python/google/appengine/api/datastore_types.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21
  22 """Higher-level, semantic data types for the datastore. These types
  23 are expected to be set as attributes of Entities.  See "Supported Data Types"
  24 in the API Guide.
  25
  26 Most of these types are based on XML elements from Atom and GData elements
  27 from the atom and gd namespaces. For more information, see:
  28
  29   http://www.atomenabled.org/developers/syndication/
  30   http://code.google.com/apis/gdata/common-elements.html
  31
  32 The namespace schemas are:
  33
  34   http://www.w3.org/2005/Atom
  35   http://schemas.google.com/g/2005
  36 """
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46 import base64
  47 import calendar
  48 import datetime
  49 import os
  50 import re
  51 import string
  52 import time
  53 import urlparse
  54 from xml.sax import saxutils
  55
  56 from google.appengine.datastore import entity_pb
  57
  58 from google.appengine.api import datastore_errors
  59 from google.appengine.api import namespace_manager
  60 from google.appengine.api import users
  61 from google.appengine.datastore import datastore_pb
  62 from google.appengine.datastore import datastore_pbs
  63 from google.appengine.datastore import entity_v4_pb
  64 from google.appengine.datastore import sortable_pb_encoder
  65
  66
  67
  68
  69
  70
  71 _MAX_STRING_LENGTH = 500
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82 _MAX_LINK_PROPERTY_LENGTH = 2083
  83
  84
  85
  86
  87
  88 _MAX_RAW_PROPERTY_BYTES = 1000 * 1000
  89
  90
  91
  92
  93
  94
  95
  96 RESERVED_PROPERTY_NAME = re.compile('^__.*__$')
  97
  98
  99
 100
 101
 102
 103
 104 KEY_SPECIAL_PROPERTY = '__key__'
 105 _KEY_SPECIAL_PROPERTY = KEY_SPECIAL_PROPERTY
 106 _UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY = '__unapplied_log_timestamp_us__'
 107 SCATTER_SPECIAL_PROPERTY = '__scatter__'
 108 _SPECIAL_PROPERTIES = frozenset(
 109     [KEY_SPECIAL_PROPERTY,
 110      _UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY,
 111      SCATTER_SPECIAL_PROPERTY])
 112
 113
 114
 115
 116
 117
 118
 119 _NAMESPACE_SEPARATOR = '!'
 120
 121
 122
 123
 124
 125 _EMPTY_NAMESPACE_ID = 1
 126
 127
 128 _EPOCH = datetime.datetime.utcfromtimestamp(0)
 129
 130
 131
 132
 133
 134 class UtcTzinfo(datetime.tzinfo):
 135   def utcoffset(self, dt): return datetime.timedelta(0)
 136   def dst(self, dt): return datetime.timedelta(0)
 137   def tzname(self, dt): return 'UTC'
 138   def __repr__(self): return 'datastore_types.UTC'
 139
 140 UTC = UtcTzinfo()
 141
 142
 143 def typename(obj):
 144   """Returns the type of obj as a string. More descriptive and specific than
 145   type(obj), and safe for any object, unlike __class__."""
 146   if hasattr(obj, '__class__'):
 147     return getattr(obj, '__class__').__name__
 148   else:
 149     return type(obj).__name__
 150
 151
 152 def ValidateString(value,
 153                    name='unused',
 154                    exception=datastore_errors.BadValueError,
 155                    max_len=_MAX_STRING_LENGTH,
 156                    empty_ok=False):
 157   """Raises an exception if value is not a valid string or a subclass thereof.
 158
 159   A string is valid if it's not empty, no more than _MAX_STRING_LENGTH bytes,
 160   and not a Blob. The exception type can be specified with the exception
 161   argument; it defaults to BadValueError.
 162
 163   Args:
 164     value: the value to validate.
 165     name: the name of this value; used in the exception message.
 166     exception: the type of exception to raise.
 167     max_len: the maximum allowed length, in bytes.
 168     empty_ok: allow empty value.
 169   """
 170   if value is None and empty_ok:
 171     return
 172   if not isinstance(value, basestring) or isinstance(value, Blob):
 173     raise exception('%s should be a string; received %s (a %s):' %
 174                     (name, value, typename(value)))
 175   if not value and not empty_ok:
 176     raise exception('%s must not be empty.' % name)
 177
 178   if len(value.encode('utf-8')) > max_len:
 179     raise exception('%s must be under %d bytes.' % (name, max_len))
 180
 181
 182 def ValidateInteger(value,
 183                    name='unused',
 184                    exception=datastore_errors.BadValueError,
 185                    empty_ok=False,
 186                    zero_ok=False,
 187                    negative_ok=False):
 188   """Raises an exception if value is not a valid integer.
 189
 190   An integer is valid if it's not negative or empty and is an integer
 191   (either int or long).  The exception type raised can be specified
 192   with the exception argument; it defaults to BadValueError.
 193
 194   Args:
 195     value: the value to validate.
 196     name: the name of this value; used in the exception message.
 197     exception: the type of exception to raise.
 198     empty_ok: allow None value.
 199     zero_ok: allow zero value.
 200     negative_ok: allow negative value.
 201   """
 202   if value is None and empty_ok:
 203     return
 204   if not isinstance(value, (int, long)):
 205     raise exception('%s should be an integer; received %s (a %s).' %
 206                     (name, value, typename(value)))
 207   if not value and not zero_ok:
 208     raise exception('%s must not be 0 (zero)' % name)
 209   if value < 0 and not negative_ok:
 210     raise exception('%s must not be negative.' % name)
 211
 212 def ResolveAppId(app):
 213   """Validate app id, providing a default.
 214
 215   If the argument is None, $APPLICATION_ID is substituted.
 216
 217   Args:
 218     app: The app id argument value to be validated.
 219
 220   Returns:
 221     The value of app, or the substituted default.  Always a non-empty string.
 222
 223   Raises:
 224     BadArgumentError if the value is empty or not a string.
 225   """
 226   if app is None:
 227     app = os.environ.get('APPLICATION_ID', '')
 228   ValidateString(app, 'app', datastore_errors.BadArgumentError)
 229   return app
 230
 231
 232 def ResolveNamespace(namespace):
 233   """Validate app namespace, providing a default.
 234
 235   If the argument is None, namespace_manager.get_namespace() is substituted.
 236
 237   Args:
 238     namespace: The namespace argument value to be validated.
 239
 240   Returns:
 241     The value of namespace, or the substituted default. The empty string is used
 242     to denote the empty namespace.
 243
 244   Raises:
 245     BadArgumentError if the value is not a string.
 246   """
 247   if namespace is None:
 248     namespace = namespace_manager.get_namespace()
 249   else:
 250     namespace_manager.validate_namespace(
 251         namespace, datastore_errors.BadArgumentError)
 252   return namespace
 253
 254
 255 def EncodeAppIdNamespace(app_id, namespace):
 256   """Concatenates app id and namespace into a single string.
 257
 258   This method is needed for xml and datastore_file_stub.
 259
 260   Args:
 261     app_id: The application id to encode
 262     namespace: The namespace to encode
 263
 264   Returns:
 265     The string encoding for the app_id, namespace pair.
 266   """
 267   if not namespace:
 268     return app_id
 269   else:
 270     return app_id + _NAMESPACE_SEPARATOR + namespace
 271
 272
 273 def DecodeAppIdNamespace(app_namespace_str):
 274   """Decodes app_namespace_str into an (app_id, namespace) pair.
 275
 276   This method is the reverse of EncodeAppIdNamespace and is needed for
 277   datastore_file_stub.
 278
 279   Args:
 280     app_namespace_str: An encoded app_id, namespace pair created by
 281       EncodeAppIdNamespace
 282
 283   Returns:
 284     (app_id, namespace) pair encoded in app_namespace_str
 285   """
 286   sep = app_namespace_str.find(_NAMESPACE_SEPARATOR)
 287   if sep < 0:
 288     return (app_namespace_str, '')
 289   else:
 290     return (app_namespace_str[0:sep], app_namespace_str[sep + 1:])
 291
 292
 293 def SetNamespace(proto, namespace):
 294   """Sets the namespace for a protocol buffer or clears the field.
 295
 296   Args:
 297     proto: the protocol buffer to update
 298     namespace: the new namespace (None or an empty string will clear out the
 299         field).
 300   """
 301   if not namespace:
 302     proto.clear_name_space()
 303   else:
 304     proto.set_name_space(namespace)
 305
 306
 307 def PartitionString(value, separator):
 308   """Equivalent to python2.5 str.partition()
 309      TODO use str.partition() when python 2.5 is adopted.
 310
 311   Args:
 312     value: String to be partitioned
 313     separator: Separator string
 314   """
 315   index = value.find(separator)
 316   if index == -1:
 317     return (value, '', value[0:0])
 318   else:
 319     return (value[0:index], separator, value[index+len(separator):len(value)])
 320
 321
 322
 323 class Key(object):
 324   """The primary key for a datastore entity.
 325
 326   A datastore GUID. A Key instance uniquely identifies an entity across all
 327   apps, and includes all information necessary to fetch the entity from the
 328   datastore with Get().
 329
 330   Key implements __hash__, and key instances are immutable, so Keys may be
 331   used in sets and as dictionary keys.
 332   """
 333   __reference = None
 334
 335   def __init__(self, encoded=None):
 336     """Constructor. Creates a Key from a string.
 337
 338     Args:
 339       # a base64-encoded primary key, generated by Key.__str__
 340       encoded: str
 341     """
 342     self._str = None
 343     if encoded is not None:
 344       if not isinstance(encoded, basestring):
 345         try:
 346           repr_encoded = repr(encoded)
 347         except:
 348           repr_encoded = "<couldn't encode>"
 349         raise datastore_errors.BadArgumentError(
 350           'Key() expects a string; received %s (a %s).' %
 351           (repr_encoded, typename(encoded)))
 352       try:
 353
 354         modulo = len(encoded) % 4
 355         if modulo != 0:
 356           encoded += ('=' * (4 - modulo))
 357
 358
 359
 360
 361
 362
 363
 364         self._str = str(encoded)
 365         encoded_pb = base64.urlsafe_b64decode(self._str)
 366         self.__reference = entity_pb.Reference(encoded_pb)
 367         assert self.__reference.IsInitialized()
 368
 369
 370         self._str = self._str.rstrip('=')
 371
 372       except (AssertionError, TypeError), e:
 373         raise datastore_errors.BadKeyError(
 374           'Invalid string key %s. Details: %s' % (encoded, e))
 375       except Exception, e:
 376
 377
 378
 379
 380
 381         if e.__class__.__name__ == 'ProtocolBufferDecodeError':
 382           raise datastore_errors.BadKeyError('Invalid string key %s.' % encoded)
 383         else:
 384           raise
 385     else:
 386
 387       self.__reference = entity_pb.Reference()
 388
 389   def to_path(self, _default_id=None, _decode=True, _fail=True):
 390     """Construct the "path" of this key as a list.
 391
 392     Returns:
 393       A list [kind_1, id_or_name_1, ..., kind_n, id_or_name_n] of the key path.
 394
 395     Raises:
 396       datastore_errors.BadKeyError if this key does not have a valid path.
 397     """
 398
 399
 400
 401
 402
 403
 404     def Decode(s):
 405       if _decode:
 406         try:
 407           return s.decode('utf-8')
 408         except UnicodeDecodeError:
 409           if _fail:
 410             raise
 411       return s
 412
 413     path = []
 414     for path_element in self.__reference.path().element_list():
 415       path.append(Decode(path_element.type()))
 416       if path_element.has_name():
 417         path.append(Decode(path_element.name()))
 418       elif path_element.has_id():
 419         path.append(path_element.id())
 420       elif _default_id is not None:
 421         path.append(_default_id)
 422       else:
 423         raise datastore_errors.BadKeyError('Incomplete key found in to_path')
 424     return path
 425
 426   @staticmethod
 427   def from_path(*args, **kwds):
 428     """Static method to construct a Key out of a "path" (kind, id or name, ...).
 429
 430     This is useful when an application wants to use just the id or name portion
 431     of a key in e.g. a URL, where the rest of the URL provides enough context to
 432     fill in the rest, i.e. the app id (always implicit), the entity kind, and
 433     possibly an ancestor key. Since ids and names are usually small, they're
 434     more attractive for use in end-user-visible URLs than the full string
 435     representation of a key.
 436
 437     Args:
 438       kind: the entity kind (a str or unicode instance)
 439       id_or_name: the id (an int or long) or name (a str or unicode instance)
 440       parent: optional parent Key; default None.
 441       namespace: optional namespace to use otherwise namespace_manager's
 442         default namespace is used.
 443
 444     Returns:
 445       A new Key instance whose .kind() and .id() or .name() methods return
 446       the *last* kind and id or name positional arguments passed.
 447
 448     Raises:
 449       BadArgumentError for invalid arguments.
 450       BadKeyError if the parent key is incomplete.
 451     """
 452
 453     parent = kwds.pop('parent', None)
 454
 455     app_id = ResolveAppId(kwds.pop('_app', None))
 456
 457
 458
 459     namespace = kwds.pop('namespace', None)
 460
 461
 462     if kwds:
 463       raise datastore_errors.BadArgumentError(
 464           'Excess keyword arguments ' + repr(kwds))
 465
 466
 467     if not args or len(args) % 2:
 468       raise datastore_errors.BadArgumentError(
 469           'A non-zero even number of positional arguments is required '
 470           '(kind, id or name, kind, id or name, ...); received %s' % repr(args))
 471
 472
 473     if parent is not None:
 474       if not isinstance(parent, Key):
 475         raise datastore_errors.BadArgumentError(
 476             'Expected None or a Key as parent; received %r (a %s).' %
 477             (parent, typename(parent)))
 478       if namespace is None:
 479         namespace = parent.namespace()
 480       if not parent.has_id_or_name():
 481         raise datastore_errors.BadKeyError(
 482             'The parent Key is incomplete.')
 483       if app_id != parent.app() or namespace != parent.namespace():
 484         raise datastore_errors.BadArgumentError(
 485             'The app/namespace arguments (%s/%s) should match '
 486             'parent.app/namespace() (%s/%s)' %
 487             (app_id, namespace, parent.app(), parent.namespace()))
 488
 489
 490     namespace = ResolveNamespace(namespace)
 491
 492
 493     key = Key()
 494     ref = key.__reference
 495     if parent is not None:
 496       ref.CopyFrom(parent.__reference)
 497     else:
 498       ref.set_app(app_id)
 499       SetNamespace(ref, namespace)
 500
 501
 502
 503     path = ref.mutable_path()
 504     for i in xrange(0, len(args), 2):
 505       kind, id_or_name = args[i:i+2]
 506       if isinstance(kind, basestring):
 507         kind = kind.encode('utf-8')
 508       else:
 509         raise datastore_errors.BadArgumentError(
 510             'Expected a string kind as argument %d; received %r (a %s).' %
 511             (i + 1, kind, typename(kind)))
 512       elem = path.add_element()
 513       elem.set_type(kind)
 514       if isinstance(id_or_name, (int, long)):
 515         elem.set_id(id_or_name)
 516       elif isinstance(id_or_name, basestring):
 517         ValidateString(id_or_name, 'name')
 518         elem.set_name(id_or_name.encode('utf-8'))
 519       else:
 520         raise datastore_errors.BadArgumentError(
 521             'Expected an integer id or string name as argument %d; '
 522             'received %r (a %s).' % (i + 2, id_or_name, typename(id_or_name)))
 523
 524
 525     assert ref.IsInitialized()
 526     return key
 527
 528   def app(self):
 529     """Returns this entity's app id, a string."""
 530     if self.__reference.app():
 531       return self.__reference.app().decode('utf-8')
 532     else:
 533       return None
 534
 535   def namespace(self):
 536     """Returns this entity's namespace, a string."""
 537     if self.__reference.has_name_space():
 538       return self.__reference.name_space().decode('utf-8')
 539     else:
 540       return ''
 541
 542
 543
 544   def kind(self):
 545     """Returns this entity's kind, as a string."""
 546     if self.__reference.path().element_size() > 0:
 547       encoded = self.__reference.path().element_list()[-1].type()
 548       return unicode(encoded.decode('utf-8'))
 549     else:
 550       return None
 551
 552   def id(self):
 553     """Returns this entity's id, or None if it doesn't have one."""
 554     elems = self.__reference.path().element_list()
 555     if elems and elems[-1].has_id() and elems[-1].id():
 556       return elems[-1].id()
 557     else:
 558       return None
 559
 560   def name(self):
 561     """Returns this entity's name, or None if it doesn't have one."""
 562     elems = self.__reference.path().element_list()
 563     if elems and elems[-1].has_name() and elems[-1].name():
 564       return elems[-1].name().decode('utf-8')
 565     else:
 566       return None
 567
 568   def id_or_name(self):
 569     """Returns this entity's id or name, whichever it has, or None."""
 570     if self.id() is not None:
 571       return self.id()
 572     else:
 573
 574       return self.name()
 575
 576   def has_id_or_name(self):
 577     """Returns True if this entity has an id or name, False otherwise.
 578     """
 579     elems = self.__reference.path().element_list()
 580     if elems:
 581       e = elems[-1]
 582       return bool(e.name() or e.id())
 583     else:
 584       return False
 585
 586   def parent(self):
 587     """Returns this entity's parent, as a Key. If this entity has no parent,
 588     returns None."""
 589     if self.__reference.path().element_size() > 1:
 590       parent = Key()
 591       parent.__reference.CopyFrom(self.__reference)
 592       del parent.__reference.path().element_list()[-1]
 593       return parent
 594     else:
 595       return None
 596
 597   def ToTagUri(self):
 598     """Returns a tag: URI for this entity for use in XML output.
 599
 600     Foreign keys for entities may be represented in XML output as tag URIs.
 601     RFC 4151 describes the tag URI scheme. From http://taguri.org/:
 602
 603       The tag algorithm lets people mint - create - identifiers that no one
 604       else using the same algorithm could ever mint. It is simple enough to do
 605       in your head, and the resulting identifiers can be easy to read, write,
 606       and remember. The identifiers conform to the URI (URL) Syntax.
 607
 608     Tag URIs for entities use the app's auth domain and the date that the URI
 609      is generated. The namespace-specific part is <kind>[<key>].
 610
 611     For example, here is the tag URI for a Kitten with the key "Fluffy" in the
 612     catsinsinks app:
 613
 614       tag:catsinsinks.googleapps.com,2006-08-29:Kitten[Fluffy]
 615
 616     Raises a BadKeyError if this entity's key is incomplete.
 617     """
 618     if not self.has_id_or_name():
 619       raise datastore_errors.BadKeyError(
 620         'ToTagUri() called for an entity with an incomplete key.')
 621
 622     return u'tag:%s.%s,%s:%s[%s]' % (
 623
 624         saxutils.escape(EncodeAppIdNamespace(self.app(), self.namespace())),
 625         os.environ['AUTH_DOMAIN'],
 626         datetime.date.today().isoformat(),
 627         saxutils.escape(self.kind()),
 628         saxutils.escape(str(self)))
 629
 630   ToXml = ToTagUri
 631
 632   def entity_group(self):
 633     """Returns this key's entity group as a Key.
 634
 635     Note that the returned Key will be incomplete if this Key is for a root
 636     entity and it is incomplete.
 637     """
 638     group = Key._FromPb(self.__reference)
 639     del group.__reference.path().element_list()[1:]
 640     return group
 641
 642   @staticmethod
 643   def _FromPb(pb):
 644     """Static factory method. Creates a Key from an entity_pb.Reference.
 645
 646     Not intended to be used by application developers. Enforced by hiding the
 647     entity_pb classes.
 648
 649     Args:
 650       pb: entity_pb.Reference
 651     """
 652     if not isinstance(pb, entity_pb.Reference):
 653       raise datastore_errors.BadArgumentError(
 654         'Key constructor takes an entity_pb.Reference; received %s (a %s).' %
 655         (pb, typename(pb)))
 656
 657     key = Key()
 658     key.__reference = entity_pb.Reference()
 659     key.__reference.CopyFrom(pb)
 660     return key
 661
 662   def _ToPb(self):
 663     """Converts this Key to its protocol buffer representation.
 664
 665     Not intended to be used by application developers. Enforced by hiding the
 666     entity_pb classes.
 667
 668     Returns:
 669       # the Reference PB representation of this Key
 670       entity_pb.Reference
 671     """
 672     pb = entity_pb.Reference()
 673     pb.CopyFrom(self.__reference)
 674     if not self.has_id_or_name():
 675       pb.mutable_path().element_list()[-1].set_id(0)
 676
 677
 678
 679     pb.app().decode('utf-8')
 680     for pathelem in pb.path().element_list():
 681       pathelem.type().decode('utf-8')
 682
 683     return pb
 684
 685   def __str__(self):
 686     """Encodes this Key as an opaque string.
 687
 688     Returns a string representation of this key, suitable for use in HTML,
 689     URLs, and other similar use cases. If the entity's key is incomplete,
 690     raises a BadKeyError.
 691
 692     Unfortunately, this string encoding isn't particularly compact, and its
 693     length varies with the length of the path. If you want a shorter identifier
 694     and you know the kind and parent (if any) ahead of time, consider using just
 695     the entity's id or name.
 696
 697     Returns:
 698       string
 699     """
 700
 701
 702
 703     try:
 704       if self._str is not None:
 705         return self._str
 706     except AttributeError:
 707       pass
 708     if (self.has_id_or_name()):
 709       encoded = base64.urlsafe_b64encode(self.__reference.Encode())
 710       self._str = encoded.replace('=', '')
 711     else:
 712       raise datastore_errors.BadKeyError(
 713         'Cannot string encode an incomplete key!\n%s' % self.__reference)
 714     return self._str
 715
 716
 717   def __repr__(self):
 718     """Returns an eval()able string representation of this key.
 719
 720     Returns a Python string of the form 'datastore_types.Key.from_path(...)'
 721     that can be used to recreate this key.
 722
 723     Returns:
 724       string
 725     """
 726     args = []
 727     for elem in self.__reference.path().element_list():
 728       args.append(repr(elem.type().decode('utf-8')))
 729       if elem.has_name():
 730         args.append(repr(elem.name().decode('utf-8')))
 731       else:
 732         args.append(repr(elem.id()))
 733
 734     args.append('_app=%r' % self.__reference.app().decode('utf-8'))
 735     if self.__reference.has_name_space():
 736       args.append('namespace=%r' %
 737           self.__reference.name_space().decode('utf-8'))
 738     return u'datastore_types.Key.from_path(%s)' % ', '.join(args)
 739
 740   def __cmp__(self, other):
 741     """Returns negative, zero, or positive when comparing two keys.
 742
 743     TODO: for API v2, we should change this to make incomplete keys, ie
 744     keys without an id or name, not equal to any other keys.
 745
 746     Args:
 747       other: Key to compare to.
 748
 749     Returns:
 750       Negative if self is less than "other"
 751       Zero if "other" is equal to self
 752       Positive if self is greater than "other"
 753     """
 754     if not isinstance(other, Key):
 755       return -2
 756
 757     self_args = [self.__reference.app(), self.__reference.name_space()]
 758     self_args += self.to_path(_default_id=0, _decode=False)
 759
 760     other_args = [other.__reference.app(), other.__reference.name_space()]
 761     other_args += other.to_path(_default_id=0, _decode=False)
 762
 763     for self_component, other_component in zip(self_args, other_args):
 764       comparison = cmp(self_component, other_component)
 765       if comparison != 0:
 766         return comparison
 767
 768     return cmp(len(self_args), len(other_args))
 769
 770   def __hash__(self):
 771     """Returns an integer hash of this key.
 772
 773     Implements Python's hash protocol so that Keys may be used in sets and as
 774     dictionary keys.
 775
 776     Returns:
 777       int
 778     """
 779     args = self.to_path(_default_id=0, _fail=False)
 780     args.append(self.__reference.app())
 781     return hash(type(args)) ^ hash(tuple(args))
 782
 783
 784 class _OverflowDateTime(long):
 785   """Container for GD_WHEN values that don't fit into a datetime.datetime.
 786
 787   This class only exists to safely round-trip GD_WHEN values that are too large
 788   to fit in a datetime.datetime instance e.g. that were created by Java
 789   applications. It should not be created directly.
 790   """
 791   pass
 792
 793
 794 def _When(val):
 795   """Coverts a GD_WHEN value to the appropriate type."""
 796   try:
 797     return _EPOCH + datetime.timedelta(microseconds=val)
 798   except OverflowError:
 799     return _OverflowDateTime(val)
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810 class Category(unicode):
 811   """A tag, ie a descriptive word or phrase. Entities may be tagged by users,
 812   and later returned by a queries for that tag. Tags can also be used for
 813   ranking results (frequency), photo captions, clustering, activity, etc.
 814
 815   Here's a more in-depth description:  http://www.zeldman.com/daily/0405d.shtml
 816
 817   This is the Atom "category" element. In XML output, the tag is provided as
 818   the term attribute. See:
 819   http://www.atomenabled.org/developers/syndication/#category
 820
 821   Raises BadValueError if tag is not a string or subtype.
 822   """
 823   TERM = 'user-tag'
 824
 825   def __init__(self, tag):
 826     super(Category, self).__init__()
 827     ValidateString(tag, 'tag')
 828
 829   def ToXml(self):
 830     return u'<category term="%s" label=%s />' % (Category.TERM,
 831                                                  saxutils.quoteattr(self))
 832
 833
 834 class Link(unicode):
 835   """A fully qualified URL. Usually http: scheme, but may also be file:, ftp:,
 836   news:, among others.
 837
 838   If you have email (mailto:) or instant messaging (aim:, xmpp:) links,
 839   consider using the Email or IM classes instead.
 840
 841   This is the Atom "link" element. In XML output, the link is provided as the
 842   href attribute. See:
 843   http://www.atomenabled.org/developers/syndication/#link
 844
 845   Raises BadValueError if link is not a fully qualified, well-formed URL.
 846   """
 847   def __init__(self, link):
 848     super(Link, self).__init__()
 849     ValidateString(link, 'link', max_len=_MAX_LINK_PROPERTY_LENGTH)
 850
 851     scheme, domain, path, params, query, fragment = urlparse.urlparse(link)
 852     if (not scheme or (scheme != 'file' and not domain) or
 853                       (scheme == 'file' and not path)):
 854       raise datastore_errors.BadValueError('Invalid URL: %s' % link)
 855
 856   def ToXml(self):
 857     return u'<link href=%s />' % saxutils.quoteattr(self)
 858
 859
 860 class Email(unicode):
 861   """An RFC2822 email address. Makes no attempt at validation; apart from
 862   checking MX records, email address validation is a rathole.
 863
 864   This is the gd:email element. In XML output, the email address is provided as
 865   the address attribute. See:
 866   http://code.google.com/apis/gdata/common-elements.html#gdEmail
 867
 868   Raises BadValueError if email is not a valid email address.
 869   """
 870   def __init__(self, email):
 871     super(Email, self).__init__()
 872     ValidateString(email, 'email')
 873
 874   def ToXml(self):
 875     return u'<gd:email address=%s />' % saxutils.quoteattr(self)
 876
 877
 878 class GeoPt(object):
 879   """A geographical point, specified by floating-point latitude and longitude
 880   coordinates. Often used to integrate with mapping sites like Google Maps.
 881   May also be used as ICBM coordinates.
 882
 883   This is the georss:point element. In XML output, the coordinates are
 884   provided as the lat and lon attributes. See: http://georss.org/
 885
 886   Serializes to '<lat>,<lon>'. Raises BadValueError if it's passed an invalid
 887   serialized string, or if lat and lon are not valid floating points in the
 888   ranges [-90, 90] and [-180, 180], respectively.
 889   """
 890   lat = None
 891   lon = None
 892
 893   def __init__(self, lat, lon=None):
 894     if lon is None:
 895
 896       try:
 897         split = lat.split(',')
 898         lat, lon = split
 899       except (AttributeError, ValueError):
 900         raise datastore_errors.BadValueError(
 901           'Expected a "lat,long" formatted string; received %s (a %s).' %
 902           (lat, typename(lat)))
 903
 904     try:
 905       lat = float(lat)
 906       lon = float(lon)
 907       if abs(lat) > 90:
 908         raise datastore_errors.BadValueError(
 909           'Latitude must be between -90 and 90; received %f' % lat)
 910       if abs(lon) > 180:
 911         raise datastore_errors.BadValueError(
 912           'Longitude must be between -180 and 180; received %f' % lon)
 913     except (TypeError, ValueError):
 914
 915       raise datastore_errors.BadValueError(
 916         'Expected floats for lat and long; received %s (a %s) and %s (a %s).' %
 917         (lat, typename(lat), lon, typename(lon)))
 918
 919     self.lat = lat
 920     self.lon = lon
 921
 922   def __cmp__(self, other):
 923     if not isinstance(other, GeoPt):
 924       try:
 925         other = GeoPt(other)
 926       except datastore_errors.BadValueError:
 927         return NotImplemented
 928
 929
 930     lat_cmp = cmp(self.lat, other.lat)
 931     if lat_cmp != 0:
 932       return lat_cmp
 933     else:
 934       return cmp(self.lon, other.lon)
 935
 936   def __hash__(self):
 937     """Returns an integer hash of this point.
 938
 939     Implements Python's hash protocol so that GeoPts may be used in sets and
 940     as dictionary keys.
 941
 942     Returns:
 943       int
 944     """
 945     return hash((self.lat, self.lon))
 946
 947   def __repr__(self):
 948     """Returns an eval()able string representation of this GeoPt.
 949
 950     The returned string is of the form 'datastore_types.GeoPt([lat], [lon])'.
 951
 952     Returns:
 953       string
 954     """
 955     return 'datastore_types.GeoPt(%r, %r)' % (self.lat, self.lon)
 956
 957   def __unicode__(self):
 958     return u'%s,%s' % (unicode(self.lat), unicode(self.lon))
 959
 960   __str__ = __unicode__
 961
 962   def ToXml(self):
 963     return u'<georss:point>%s %s</georss:point>' % (unicode(self.lat),
 964                                                     unicode(self.lon))
 965
 966
 967 class IM(object):
 968   """An instant messaging handle. Includes both an address and its protocol.
 969   The protocol value is either a standard IM scheme or a URL identifying the
 970   IM network for the protocol. Possible values include:
 971
 972     Value                           Description
 973     sip                             SIP/SIMPLE
 974     unknown                         Unknown or unspecified
 975     xmpp                            XMPP/Jabber
 976     http://aim.com/                 AIM
 977     http://icq.com/                 ICQ
 978     http://talk.google.com/         Google Talk
 979     http://messenger.msn.com/       MSN Messenger
 980     http://messenger.yahoo.com/     Yahoo Messenger
 981     http://sametime.com/            Lotus Sametime
 982     http://gadu-gadu.pl/            Gadu-Gadu
 983
 984   This is the gd:im element. In XML output, the address and protocol are
 985   provided as the address and protocol attributes, respectively. See:
 986   http://code.google.com/apis/gdata/common-elements.html#gdIm
 987
 988   Serializes to '<protocol> <address>'. Raises BadValueError if tag is not a
 989   standard IM scheme or a URL.
 990   """
 991   PROTOCOLS = [ 'sip', 'unknown', 'xmpp' ]
 992
 993   protocol = None
 994   address = None
 995
 996   def __init__(self, protocol, address=None):
 997     if address is None:
 998
 999       try:
1000         split = protocol.split(' ', 1)
1001         protocol, address = split
1002       except (AttributeError, ValueError):
1003         raise datastore_errors.BadValueError(
1004           'Expected string of format "protocol address"; received %s' %
1005           (protocol,))
1006
1007     ValidateString(address, 'address')
1008     if protocol not in self.PROTOCOLS:
1009
1010       Link(protocol)
1011
1012     self.address = address
1013     self.protocol = protocol
1014
1015   def __cmp__(self, other):
1016     if not isinstance(other, IM):
1017       try:
1018         other = IM(other)
1019       except datastore_errors.BadValueError:
1020         return NotImplemented
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032     return cmp((self.address, self.protocol),
1033                (other.address, other.protocol))
1034
1035   def __repr__(self):
1036     """Returns an eval()able string representation of this IM.
1037
1038     The returned string is of the form:
1039
1040       datastore_types.IM('address', 'protocol')
1041
1042     Returns:
1043       string
1044     """
1045     return 'datastore_types.IM(%r, %r)' % (self.protocol, self.address)
1046
1047   def __unicode__(self):
1048     return u'%s %s' % (self.protocol, self.address)
1049
1050   __str__ = __unicode__
1051
1052   def ToXml(self):
1053     return (u'<gd:im protocol=%s address=%s />' %
1054             (saxutils.quoteattr(self.protocol),
1055              saxutils.quoteattr(self.address)))
1056
1057   def __len__(self):
1058     return len(unicode(self))
1059
1060
1061 class PhoneNumber(unicode):
1062   """A human-readable phone number or address.
1063
1064   No validation is performed. Phone numbers have many different formats -
1065   local, long distance, domestic, international, internal extension, TTY,
1066   VOIP, SMS, and alternative networks like Skype, XFire and Roger Wilco. They
1067   all have their own numbering and addressing formats.
1068
1069   This is the gd:phoneNumber element. In XML output, the phone number is
1070   provided as the text of the element. See:
1071   http://code.google.com/apis/gdata/common-elements.html#gdPhoneNumber
1072
1073   Raises BadValueError if phone is not a string or subtype.
1074   """
1075   def __init__(self, phone):
1076     super(PhoneNumber, self).__init__()
1077     ValidateString(phone, 'phone')
1078
1079   def ToXml(self):
1080     return u'<gd:phoneNumber>%s</gd:phoneNumber>' % saxutils.escape(self)
1081
1082
1083 class PostalAddress(unicode):
1084   """A human-readable mailing address. Again, mailing address formats vary
1085   widely, so no validation is performed.
1086
1087   This is the gd:postalAddress element. In XML output, the address is provided
1088   as the text of the element. See:
1089   http://code.google.com/apis/gdata/common-elements.html#gdPostalAddress
1090
1091   Raises BadValueError if address is not a string or subtype.
1092   """
1093   def __init__(self, address):
1094     super(PostalAddress, self).__init__()
1095     ValidateString(address, 'address')
1096
1097   def ToXml(self):
1098     return u'<gd:postalAddress>%s</gd:postalAddress>' % saxutils.escape(self)
1099
1100
1101 class Rating(long):
1102   """A user-provided integer rating for a piece of content. Normalized to a
1103   0-100 scale.
1104
1105   This is the gd:rating element. In XML output, the address is provided
1106   as the text of the element. See:
1107   http://code.google.com/apis/gdata/common-elements.html#gdRating
1108
1109   Serializes to the decimal string representation of the rating. Raises
1110   BadValueError if the rating is not an integer in the range [0, 100].
1111   """
1112   MIN = 0
1113   MAX = 100
1114
1115   def __init__(self, rating):
1116     super(Rating, self).__init__()
1117     if isinstance(rating, float) or isinstance(rating, complex):
1118
1119       raise datastore_errors.BadValueError(
1120         'Expected int or long; received %s (a %s).' %
1121         (rating, typename(rating)))
1122
1123     try:
1124       if long(rating) < Rating.MIN or long(rating) > Rating.MAX:
1125         raise datastore_errors.BadValueError()
1126     except ValueError:
1127
1128       raise datastore_errors.BadValueError(
1129         'Expected int or long; received %s (a %s).' %
1130         (rating, typename(rating)))
1131
1132   def ToXml(self):
1133     return (u'<gd:rating value="%d" min="%d" max="%d" />' %
1134             (self, Rating.MIN, Rating.MAX))
1135
1136
1137 class Text(unicode):
1138   """A long string type.
1139
1140   Strings of any length can be stored in the datastore using this
1141   type. It behaves identically to the Python unicode type, except for
1142   the constructor, which only accepts str and unicode arguments.
1143   """
1144
1145   def __new__(cls, arg=None, encoding=None):
1146     """Constructor.
1147
1148     We only accept unicode and str instances, the latter with encoding.
1149
1150     Args:
1151       arg: optional unicode or str instance; default u''
1152       encoding: optional encoding; disallowed when isinstance(arg, unicode),
1153                 defaults to 'ascii' when isinstance(arg, str);
1154     """
1155     if arg is None:
1156       arg = u''
1157     if isinstance(arg, unicode):
1158       if encoding is not None:
1159         raise TypeError('Text() with a unicode argument '
1160                         'should not specify an encoding')
1161       return super(Text, cls).__new__(cls, arg)
1162
1163     if isinstance(arg, str):
1164       if encoding is None:
1165         encoding = 'ascii'
1166       return super(Text, cls).__new__(cls, arg, encoding)
1167
1168     raise TypeError('Text() argument should be str or unicode, not %s' %
1169                     type(arg).__name__)
1170
1171 class _BaseByteType(str):
1172   """A base class for datastore types that are encoded as bytes.
1173
1174   This behaves identically to the Python str type, except for the
1175   constructor, which only accepts str arguments.
1176   """
1177
1178   def __new__(cls, arg=None):
1179     """Constructor.
1180
1181     We only accept str instances.
1182
1183     Args:
1184       arg: optional str instance (default '')
1185     """
1186     if arg is None:
1187       arg = ''
1188     if isinstance(arg, str):
1189       return super(_BaseByteType, cls).__new__(cls, arg)
1190
1191     raise TypeError('%s() argument should be str instance, not %s' %
1192                     (cls.__name__, type(arg).__name__))
1193
1194   def ToXml(self):
1195     """Output bytes as XML.
1196
1197     Returns:
1198       Base64 encoded version of itself for safe insertion in to an XML document.
1199     """
1200     encoded = base64.urlsafe_b64encode(self)
1201     return saxutils.escape(encoded)
1202
1203
1204 class Blob(_BaseByteType):
1205   """A blob type, appropriate for storing binary data of any length.
1206
1207   This behaves identically to the Python str type, except for the
1208   constructor, which only accepts str arguments.
1209   """
1210   pass
1211
1212
1213 class EmbeddedEntity(_BaseByteType):
1214   """A proto encoded EntityProto.
1215
1216   This behaves identically to Blob, except for the
1217   constructor, which accepts a str or EntityProto argument.
1218
1219   Can be decoded using datastore.Entity.FromProto(), db.model_from_protobuf() or
1220   ndb.LocalStructuredProperty.
1221   """
1222
1223   def __new__(cls, arg=None):
1224     """Constructor.
1225
1226     Args:
1227       arg: optional str or EntityProto instance (default '')
1228     """
1229     if isinstance(arg, entity_pb.EntityProto):
1230       arg = arg.SerializePartialToString()
1231     return super(EmbeddedEntity, cls).__new__(cls, arg)
1232
1233
1234 class ByteString(_BaseByteType):
1235   """A byte-string type, appropriate for storing short amounts of indexed data.
1236
1237   This behaves identically to Blob, except it's used only for short, indexed
1238   byte strings.
1239   """
1240   pass
1241
1242
1243 class BlobKey(object):
1244   """Key used to identify a blob in Blobstore.
1245
1246   This object wraps a string that gets used internally by the Blobstore API
1247   to identify application blobs.  The BlobKey corresponds to the entity name
1248   of the underlying BlobReference entity.
1249
1250   This class is exposed in the API in both google.appengine.ext.db and
1251   google.appengine.ext.blobstore.
1252   """
1253
1254   def __init__(self, blob_key):
1255     """Constructor.
1256
1257     Used to convert a string to a BlobKey.  Normally used internally by
1258     Blobstore API.
1259
1260     Args:
1261       blob_key:  Key name of BlobReference that this key belongs to.
1262     """
1263     ValidateString(blob_key, 'blob-key')
1264     self.__blob_key = blob_key
1265
1266   def __str__(self):
1267     """Convert to string."""
1268     return self.__blob_key
1269
1270   def __repr__(self):
1271     """Returns an eval()able string representation of this key.
1272
1273     Returns a Python string of the form 'datastore_types.BlobKey(...)'
1274     that can be used to recreate this key.
1275
1276     Returns:
1277       string
1278     """
1279     return 'datastore_types.%s(%r)' % (type(self).__name__, self.__blob_key)
1280
1281   def __cmp__(self, other):
1282
1283
1284     if type(other) is type(self):
1285       return cmp(str(self), str(other))
1286     elif isinstance(other, basestring):
1287       return cmp(self.__blob_key, other)
1288     else:
1289       return NotImplemented
1290
1291   def __hash__(self):
1292     return hash(self.__blob_key)
1293
1294   def ToXml(self):
1295     return str(self)
1296
1297
1298
1299 _PROPERTY_MEANINGS = {
1300
1301
1302
1303   Blob:              entity_pb.Property.BLOB,
1304   EmbeddedEntity:    entity_pb.Property.ENTITY_PROTO,
1305   ByteString:        entity_pb.Property.BYTESTRING,
1306   Text:              entity_pb.Property.TEXT,
1307   datetime.datetime: entity_pb.Property.GD_WHEN,
1308   datetime.date:     entity_pb.Property.GD_WHEN,
1309   datetime.time:     entity_pb.Property.GD_WHEN,
1310   _OverflowDateTime: entity_pb.Property.GD_WHEN,
1311   Category:          entity_pb.Property.ATOM_CATEGORY,
1312   Link:              entity_pb.Property.ATOM_LINK,
1313   Email:             entity_pb.Property.GD_EMAIL,
1314   GeoPt:             entity_pb.Property.GEORSS_POINT,
1315   IM:                entity_pb.Property.GD_IM,
1316   PhoneNumber:       entity_pb.Property.GD_PHONENUMBER,
1317   PostalAddress:     entity_pb.Property.GD_POSTALADDRESS,
1318   Rating:            entity_pb.Property.GD_RATING,
1319   BlobKey:           entity_pb.Property.BLOBKEY,
1320 }
1321
1322
1323 _PROPERTY_TYPES = frozenset([
1324   Blob,
1325   EmbeddedEntity,
1326   ByteString,
1327   bool,
1328   Category,
1329   datetime.datetime,
1330   _OverflowDateTime,
1331   Email,
1332   float,
1333   GeoPt,
1334   IM,
1335   int,
1336   Key,
1337   Link,
1338   long,
1339   PhoneNumber,
1340   PostalAddress,
1341   Rating,
1342   str,
1343   Text,
1344   type(None),
1345   unicode,
1346   users.User,
1347   BlobKey,
1348 ])
1349
1350
1351
1352
1353 _RAW_PROPERTY_TYPES = (Blob, Text, EmbeddedEntity)
1354 _RAW_PROPERTY_MEANINGS = (entity_pb.Property.BLOB, entity_pb.Property.TEXT,
1355                           entity_pb.Property.ENTITY_PROTO)
1356
1357
1358 def ValidatePropertyInteger(name, value):
1359   """Raises an exception if the supplied integer is invalid.
1360
1361   Args:
1362     name: Name of the property this is for.
1363     value: Integer value.
1364
1365   Raises:
1366     OverflowError if the value does not fit within a signed int64.
1367   """
1368   if not (-0x8000000000000000 <= value <= 0x7fffffffffffffff):
1369     raise OverflowError('%d is out of bounds for int64' % value)
1370
1371
1372 def ValidateStringLength(name, value, max_len):
1373   """Raises an exception if the supplied string is too long.
1374
1375   Args:
1376     name: Name of the property this is for.
1377     value: String value.
1378     max_len: Maximum length the string may be.
1379
1380   Raises:
1381     OverflowError if the value is larger than the maximum length.
1382   """
1383   if len(value) > max_len:
1384     raise datastore_errors.BadValueError(
1385       'Property %s is %d bytes long; it must be %d or less. '
1386       'Consider Text instead, which can store strings of any length.' %
1387       (name, len(value), max_len))
1388
1389
1390 def ValidatePropertyString(name, value):
1391   """Validates the length of an indexed string property.
1392
1393   Args:
1394     name: Name of the property this is for.
1395     value: String value.
1396   """
1397   ValidateStringLength(name, value, max_len=_MAX_STRING_LENGTH)
1398
1399
1400 def ValidatePropertyLink(name, value):
1401   """Validates the length of an indexed Link property.
1402
1403   Args:
1404     name: Name of the property this is for.
1405     value: String value.
1406   """
1407   ValidateStringLength(name, value, max_len=_MAX_LINK_PROPERTY_LENGTH)
1408
1409
1410 def ValidatePropertyNothing(name, value):
1411   """No-op validation function.
1412
1413   Args:
1414     name: Name of the property this is for.
1415     value: Not used.
1416   """
1417   pass
1418
1419
1420 def ValidatePropertyKey(name, value):
1421   """Raises an exception if the supplied datastore.Key instance is invalid.
1422
1423   Args:
1424     name: Name of the property this is for.
1425     value: A datastore.Key instance.
1426
1427   Raises:
1428     datastore_errors.BadValueError if the value is invalid.
1429   """
1430   if not value.has_id_or_name():
1431     raise datastore_errors.BadValueError(
1432         'Incomplete key found for reference property %s.' % name)
1433
1434
1435
1436
1437
1438
1439
1440 _VALIDATE_PROPERTY_VALUES = {
1441   Blob: ValidatePropertyNothing,
1442   EmbeddedEntity: ValidatePropertyNothing,
1443   ByteString: ValidatePropertyNothing,
1444   bool: ValidatePropertyNothing,
1445   Category: ValidatePropertyNothing,
1446   datetime.datetime: ValidatePropertyNothing,
1447   _OverflowDateTime: ValidatePropertyInteger,
1448   Email: ValidatePropertyNothing,
1449   float: ValidatePropertyNothing,
1450   GeoPt: ValidatePropertyNothing,
1451   IM: ValidatePropertyNothing,
1452   int: ValidatePropertyInteger,
1453   Key: ValidatePropertyKey,
1454   Link: ValidatePropertyNothing,
1455   long: ValidatePropertyInteger,
1456   PhoneNumber: ValidatePropertyNothing,
1457   PostalAddress: ValidatePropertyNothing,
1458   Rating: ValidatePropertyInteger,
1459   str: ValidatePropertyNothing,
1460   Text: ValidatePropertyNothing,
1461   type(None): ValidatePropertyNothing,
1462   unicode: ValidatePropertyNothing,
1463   users.User: ValidatePropertyNothing,
1464   BlobKey: ValidatePropertyNothing,
1465 }
1466
1467 _PROPERTY_TYPE_TO_INDEX_VALUE_TYPE = {
1468   basestring: str,
1469   Blob: str,
1470   EmbeddedEntity: str,
1471   ByteString: str,
1472   bool: bool,
1473   Category: str,
1474   datetime.datetime: long,
1475   datetime.date: long,
1476   datetime.time: long,
1477   _OverflowDateTime: long,
1478   Email: str,
1479   float: float,
1480   GeoPt: GeoPt,
1481   IM: str,
1482   int: long,
1483   Key: Key,
1484   Link: str,
1485   long: long,
1486   PhoneNumber: str,
1487   PostalAddress: str,
1488   Rating: long,
1489   str: str,
1490   Text: str,
1491   type(None): type(None),
1492   unicode: str,
1493   users.User: users.User,
1494   BlobKey: str,
1495 }
1496
1497
1498 assert set(_VALIDATE_PROPERTY_VALUES.iterkeys()) == _PROPERTY_TYPES
1499
1500
1501 def ValidateProperty(name, values, read_only=False):
1502   """Helper function for validating property values.
1503
1504   Args:
1505     name: Name of the property this is for.
1506     value: Value for the property as a Python native type.
1507     read_only: deprecated
1508
1509   Raises:
1510     BadPropertyError if the property name is invalid. BadValueError if the
1511     property did not validate correctly or the value was an empty list. Other
1512     exception types (like OverflowError) if the property value does not meet
1513     type-specific criteria.
1514   """
1515   ValidateString(name, 'property name', datastore_errors.BadPropertyError)
1516
1517   values_type = type(values)
1518
1519
1520   if values_type is tuple:
1521     raise datastore_errors.BadValueError(
1522         'May not use tuple property value; property %s is %s.' %
1523         (name, repr(values)))
1524
1525
1526   if values_type is not list:
1527     values = [values]
1528
1529
1530   if not values:
1531     raise datastore_errors.BadValueError(
1532         'May not use the empty list as a property value; property %s is %s.' %
1533         (name, repr(values)))
1534
1535
1536
1537   try:
1538     for v in values:
1539       prop_validator = _VALIDATE_PROPERTY_VALUES.get(v.__class__)
1540       if prop_validator is None:
1541         raise datastore_errors.BadValueError(
1542           'Unsupported type for property %s: %s' % (name, v.__class__))
1543       prop_validator(name, v)
1544
1545   except (KeyError, ValueError, TypeError, IndexError, AttributeError), msg:
1546     raise datastore_errors.BadValueError(
1547       'Error type checking values for property %s: %s' % (name, msg))
1548
1549
1550
1551
1552
1553 ValidateReadProperty = ValidateProperty
1554
1555
1556
1557 def PackBlob(name, value, pbvalue):
1558   """Packs a Blob property into a entity_pb.PropertyValue.
1559
1560   Args:
1561     name: The name of the property as a string.
1562     value: A Blob instance.
1563     pbvalue: The entity_pb.PropertyValue to pack this value into.
1564   """
1565   pbvalue.set_stringvalue(value)
1566
1567
1568 def PackString(name, value, pbvalue):
1569   """Packs a string-typed property into a entity_pb.PropertyValue.
1570
1571   Args:
1572     name: The name of the property as a string.
1573     value: A string, unicode, or string-like value instance.
1574     pbvalue: The entity_pb.PropertyValue to pack this value into.
1575   """
1576   pbvalue.set_stringvalue(unicode(value).encode('utf-8'))
1577
1578
1579 def PackDatetime(name, value, pbvalue):
1580   """Packs a datetime-typed property into a entity_pb.PropertyValue.
1581
1582   Args:
1583     name: The name of the property as a string.
1584     value: A datetime.datetime instance.
1585     pbvalue: The entity_pb.PropertyValue to pack this value into.
1586   """
1587   pbvalue.set_int64value(DatetimeToTimestamp(value))
1588
1589
1590 def DatetimeToTimestamp(value):
1591   """Converts a datetime.datetime to microseconds since the epoch, as a float.
1592   Args:
1593     value: datetime.datetime
1594
1595   Returns: value as a long
1596   """
1597   if value.tzinfo:
1598
1599     value = value.astimezone(UTC)
1600   return long(calendar.timegm(value.timetuple()) * 1000000L) + value.microsecond
1601
1602
1603 def PackGeoPt(name, value, pbvalue):
1604   """Packs a GeoPt property into a entity_pb.PropertyValue.
1605
1606   Args:
1607     name: The name of the property as a string.
1608     value: A GeoPt instance.
1609     pbvalue: The entity_pb.PropertyValue to pack this value into.
1610   """
1611   pbvalue.mutable_pointvalue().set_x(value.lat)
1612   pbvalue.mutable_pointvalue().set_y(value.lon)
1613
1614
1615 def PackUser(name, value, pbvalue):
1616   """Packs a User property into a entity_pb.PropertyValue.
1617
1618   Args:
1619     name: The name of the property as a string.
1620     value: A users.User instance.
1621     pbvalue: The entity_pb.PropertyValue to pack this value into.
1622   """
1623   pbvalue.mutable_uservalue().set_email(value.email().encode('utf-8'))
1624   pbvalue.mutable_uservalue().set_auth_domain(
1625       value.auth_domain().encode('utf-8'))
1626   pbvalue.mutable_uservalue().set_gaiaid(0)
1627
1628
1629
1630
1631   if value.user_id() is not None:
1632     pbvalue.mutable_uservalue().set_obfuscated_gaiaid(
1633         value.user_id().encode('utf-8'))
1634
1635   if value.federated_identity() is not None:
1636     pbvalue.mutable_uservalue().set_federated_identity(
1637         value.federated_identity().encode('utf-8'))
1638
1639   if value.federated_provider() is not None:
1640     pbvalue.mutable_uservalue().set_federated_provider(
1641         value.federated_provider().encode('utf-8'))
1642
1643
1644 def PackKey(name, value, pbvalue):
1645   """Packs a reference property into a entity_pb.PropertyValue.
1646
1647   Args:
1648     name: The name of the property as a string.
1649     value: A Key instance.
1650     pbvalue: The entity_pb.PropertyValue to pack this value into.
1651   """
1652   ref = value._Key__reference
1653   pbvalue.mutable_referencevalue().set_app(ref.app())
1654   SetNamespace(pbvalue.mutable_referencevalue(), ref.name_space())
1655   for elem in ref.path().element_list():
1656     pbvalue.mutable_referencevalue().add_pathelement().CopyFrom(elem)
1657
1658
1659 def PackBool(name, value, pbvalue):
1660   """Packs a boolean property into a entity_pb.PropertyValue.
1661
1662   Args:
1663     name: The name of the property as a string.
1664     value: A boolean instance.
1665     pbvalue: The entity_pb.PropertyValue to pack this value into.
1666   """
1667   pbvalue.set_booleanvalue(value)
1668
1669
1670 def PackInteger(name, value, pbvalue):
1671   """Packs an integer property into a entity_pb.PropertyValue.
1672
1673   Args:
1674     name: The name of the property as a string.
1675     value: An int or long instance.
1676     pbvalue: The entity_pb.PropertyValue to pack this value into.
1677   """
1678   pbvalue.set_int64value(value)
1679
1680
1681 def PackFloat(name, value, pbvalue):
1682   """Packs a float property into a entity_pb.PropertyValue.
1683
1684   Args:
1685     name: The name of the property as a string.
1686     value: A float instance.
1687     pbvalue: The entity_pb.PropertyValue to pack this value into.
1688   """
1689   pbvalue.set_doublevalue(value)
1690
1691
1692
1693
1694
1695
1696
1697 _PACK_PROPERTY_VALUES = {
1698   Blob: PackBlob,
1699   EmbeddedEntity: PackBlob,
1700   ByteString: PackBlob,
1701   bool: PackBool,
1702   Category: PackString,
1703   datetime.datetime: PackDatetime,
1704   _OverflowDateTime: PackInteger,
1705   Email: PackString,
1706   float: PackFloat,
1707   GeoPt: PackGeoPt,
1708   IM: PackString,
1709   int: PackInteger,
1710   Key: PackKey,
1711   Link: PackString,
1712   long: PackInteger,
1713   PhoneNumber: PackString,
1714   PostalAddress: PackString,
1715   Rating: PackInteger,
1716   str: PackString,
1717   Text: PackString,
1718   type(None): lambda name, value, pbvalue: None,
1719   unicode: PackString,
1720   users.User: PackUser,
1721   BlobKey: PackString,
1722 }
1723
1724
1725 assert set(_PACK_PROPERTY_VALUES.iterkeys()) == _PROPERTY_TYPES
1726
1727
1728 def ToPropertyPb(name, values):
1729   """Creates type-specific entity_pb.PropertyValues.
1730
1731   Determines the type and meaning of the PropertyValue based on the Python
1732   type of the input value(s).
1733
1734   NOTE: This function does not validate anything!
1735
1736   Args:
1737     name: string or unicode; the property name
1738     values: The values for this property, either a single one or a list of them.
1739       All values must be a supported type. Lists of values must all be of the
1740       same type.
1741
1742   Returns:
1743     A list of entity_pb.Property instances.
1744   """
1745   encoded_name = name.encode('utf-8')
1746
1747   values_type = type(values)
1748   if values_type is list:
1749     multiple = True
1750   else:
1751     multiple = False
1752     values = [values]
1753
1754   pbs = []
1755   for v in values:
1756     pb = entity_pb.Property()
1757     pb.set_name(encoded_name)
1758     pb.set_multiple(multiple)
1759
1760     meaning = _PROPERTY_MEANINGS.get(v.__class__)
1761     if meaning is not None:
1762       pb.set_meaning(meaning)
1763
1764     pack_prop = _PACK_PROPERTY_VALUES[v.__class__]
1765     pbvalue = pack_prop(name, v, pb.mutable_value())
1766     pbs.append(pb)
1767
1768   if multiple:
1769     return pbs
1770   else:
1771     return pbs[0]
1772
1773
1774 def FromReferenceProperty(value):
1775   """Converts a reference PropertyValue to a Key.
1776
1777   Args:
1778     value: entity_pb.PropertyValue
1779
1780   Returns:
1781     Key
1782
1783   Raises:
1784     BadValueError if the value is not a PropertyValue.
1785   """
1786   assert isinstance(value, entity_pb.PropertyValue)
1787   assert value.has_referencevalue()
1788   ref = value.referencevalue()
1789
1790   key = Key()
1791   key_ref = key._Key__reference
1792   key_ref.set_app(ref.app())
1793   SetNamespace(key_ref, ref.name_space())
1794
1795   for pathelem in ref.pathelement_list():
1796     key_ref.mutable_path().add_element().CopyFrom(pathelem)
1797
1798   return key
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808 _PROPERTY_CONVERSIONS = {
1809   entity_pb.Property.GD_WHEN:           _When,
1810   entity_pb.Property.ATOM_CATEGORY:     Category,
1811   entity_pb.Property.ATOM_LINK:         Link,
1812   entity_pb.Property.GD_EMAIL:          Email,
1813   entity_pb.Property.GD_IM:             IM,
1814   entity_pb.Property.GD_PHONENUMBER:    PhoneNumber,
1815   entity_pb.Property.GD_POSTALADDRESS:  PostalAddress,
1816   entity_pb.Property.GD_RATING:         Rating,
1817   entity_pb.Property.BLOB:              Blob,
1818   entity_pb.Property.ENTITY_PROTO:      EmbeddedEntity,
1819   entity_pb.Property.BYTESTRING:        ByteString,
1820   entity_pb.Property.TEXT:              Text,
1821   entity_pb.Property.BLOBKEY:           BlobKey,
1822 }
1823
1824
1825 _NON_UTF8_MEANINGS = frozenset((entity_pb.Property.BLOB,
1826                                 entity_pb.Property.ENTITY_PROTO,
1827                                 entity_pb.Property.BYTESTRING,
1828                                 entity_pb.Property.INDEX_VALUE))
1829
1830
1831 def FromPropertyPb(pb):
1832   """Converts a property PB to a python value.
1833
1834   Args:
1835     pb: entity_pb.Property
1836
1837   Returns:
1838     # return type is determined by the type of the argument
1839     string, int, bool, double, users.User, or one of the atom or gd types
1840   """
1841
1842
1843
1844   pbval = pb.value()
1845   meaning = pb.meaning()
1846
1847   if pbval.has_stringvalue():
1848     value = pbval.stringvalue()
1849     if not pb.has_meaning() or meaning not in _NON_UTF8_MEANINGS:
1850       value = unicode(value, 'utf-8')
1851   elif pbval.has_int64value():
1852
1853
1854     value = long(pbval.int64value())
1855   elif pbval.has_booleanvalue():
1856
1857
1858     value = bool(pbval.booleanvalue())
1859   elif pbval.has_doublevalue():
1860     value = pbval.doublevalue()
1861   elif pbval.has_referencevalue():
1862     value = FromReferenceProperty(pbval)
1863   elif pbval.has_pointvalue():
1864     value = GeoPt(pbval.pointvalue().x(), pbval.pointvalue().y())
1865   elif pbval.has_uservalue():
1866     email = unicode(pbval.uservalue().email(), 'utf-8')
1867     auth_domain = unicode(pbval.uservalue().auth_domain(), 'utf-8')
1868     obfuscated_gaiaid = pbval.uservalue().obfuscated_gaiaid().decode('utf-8')
1869     obfuscated_gaiaid = unicode(pbval.uservalue().obfuscated_gaiaid(), 'utf-8')
1870
1871     federated_identity = None
1872     if pbval.uservalue().has_federated_identity():
1873       federated_identity = unicode(pbval.uservalue().federated_identity(),
1874                                    'utf-8')
1875
1876
1877
1878     value = users.User(email=email,
1879                        _auth_domain=auth_domain,
1880                        _user_id=obfuscated_gaiaid,
1881                        federated_identity=federated_identity,
1882                        _strict_mode=False)
1883   else:
1884     value = None
1885
1886   try:
1887     if pb.has_meaning() and meaning in _PROPERTY_CONVERSIONS:
1888       conversion = _PROPERTY_CONVERSIONS[meaning]
1889       value = conversion(value)
1890   except (KeyError, ValueError, IndexError, TypeError, AttributeError), msg:
1891     raise datastore_errors.BadValueError(
1892       'Error converting pb: %s\nException was: %s' % (pb, msg))
1893
1894   return value
1895
1896
1897 def RestoreFromIndexValue(index_value, data_type):
1898   """Restores a index value to the correct datastore type.
1899
1900   Projection queries return property values direclty from a datastore index.
1901   These values are the native datastore values, one of str, bool, long, float,
1902   GeoPt, Key or User. This function restores the original value when the the
1903   original type is known.
1904
1905   This function returns the value type returned when decoding a normal entity,
1906   not necessarily of type data_type. For example, data_type=int returns a
1907   long instance.
1908
1909   Args:
1910     index_value: The value returned by FromPropertyPb for the projected
1911       property.
1912     data_type: The type of the value originally given to ToPropertyPb
1913
1914   Returns:
1915     The restored property value.
1916
1917   Raises:
1918     datastore_errors.BadValueError if the value cannot be restored.
1919   """
1920   raw_type = _PROPERTY_TYPE_TO_INDEX_VALUE_TYPE.get(data_type)
1921   if raw_type is None:
1922     raise datastore_errors.BadValueError(
1923         'Unsupported data type (%r)' % data_type)
1924
1925   if index_value is None:
1926     return index_value
1927
1928
1929
1930   if not isinstance(index_value, raw_type):
1931     raise datastore_errors.BadValueError(
1932         'Unsupported converstion. Expected %r got %r' %
1933         (type(index_value), raw_type))
1934
1935   meaning = _PROPERTY_MEANINGS.get(data_type)
1936
1937
1938   if isinstance(index_value, str) and meaning not in _NON_UTF8_MEANINGS:
1939     index_value = unicode(index_value, 'utf-8')
1940
1941
1942   conv = _PROPERTY_CONVERSIONS.get(meaning)
1943   if not conv:
1944     return index_value
1945
1946   try:
1947     value = conv(index_value)
1948   except (KeyError, ValueError, IndexError, TypeError, AttributeError), msg:
1949     raise datastore_errors.BadValueError(
1950       'Error converting value: %r\nException was: %s' % (index_value, msg))
1951   return value
1952
1953
1954 def PropertyTypeName(value):
1955   """Returns the name of the type of the given property value, as a string.
1956
1957   Raises BadValueError if the value is not a valid property type.
1958
1959   Args:
1960     value: any valid property value
1961
1962   Returns:
1963     string
1964   """
1965   if value.__class__ in _PROPERTY_MEANINGS:
1966     meaning = _PROPERTY_MEANINGS[value.__class__]
1967     name = entity_pb.Property._Meaning_NAMES[meaning]
1968     return name.lower().replace('_', ':')
1969   elif isinstance(value, basestring):
1970     return 'string'
1971   elif isinstance(value, users.User):
1972     return 'user'
1973   elif isinstance(value, long):
1974     return 'int'
1975   elif value is None:
1976     return 'null'
1977   else:
1978     return typename(value).lower()
1979
1980
1981 _PROPERTY_TYPE_STRINGS = {
1982     'string':           unicode,
1983     'bool':             bool,
1984     'int':              long,
1985     'null':             type(None),
1986     'float':            float,
1987     'key':              Key,
1988     'blob':             Blob,
1989     'entity:proto':     EmbeddedEntity,
1990     'bytestring':       ByteString,
1991     'text':             Text,
1992     'user':             users.User,
1993     'atom:category':    Category,
1994     'atom:link':        Link,
1995     'gd:email':         Email,
1996     'gd:when':          datetime.datetime,
1997     'georss:point':     GeoPt,
1998     'gd:im':            IM,
1999     'gd:phonenumber':   PhoneNumber,
2000     'gd:postaladdress': PostalAddress,
2001     'gd:rating':        Rating,
2002     'blobkey':          BlobKey,
2003     }
2004
2005
2006 def FromPropertyTypeName(type_name):
2007   """Returns the python type given a type name.
2008
2009   Args:
2010     type_name: A string representation of a datastore type name.
2011
2012   Returns:
2013     A python type.
2014   """
2015   return _PROPERTY_TYPE_STRINGS[type_name]
2016
2017
2018 def PropertyValueFromString(type_,
2019                             value_string,
2020                             _auth_domain=None):
2021   """Returns an instance of a property value given a type and string value.
2022
2023   The reverse of this method is just str() and type() of the python value.
2024
2025   Note that this does *not* support non-UTC offsets in ISO 8601-formatted
2026   datetime strings, e.g. the -08:00 suffix in '2002-12-25 00:00:00-08:00'.
2027   It only supports -00:00 and +00:00 suffixes, which are UTC.
2028
2029   Args:
2030     type_: A python class.
2031     value_string: A string representation of the value of the property.
2032
2033   Returns:
2034     An instance of 'type'.
2035
2036   Raises:
2037     ValueError if type_ is datetime and value_string has a timezone offset.
2038   """
2039   if type_ == datetime.datetime:
2040     value_string = value_string.strip()
2041
2042     if value_string[-6] in ('+', '-'):
2043       if value_string[-5:] == '00:00':
2044         value_string = value_string[:-6]
2045       else:
2046
2047         raise ValueError('Non-UTC offsets in datetimes are not supported.')
2048
2049
2050     split = value_string.split('.')
2051     iso_date = split[0]
2052     microseconds = 0
2053     if len(split) > 1:
2054       microseconds = int(split[1])
2055
2056
2057
2058     time_struct = time.strptime(iso_date, '%Y-%m-%d %H:%M:%S')[0:6]
2059     value = datetime.datetime(*(time_struct + (microseconds,)))
2060     return value
2061   elif type_ == Rating:
2062
2063     return Rating(int(value_string))
2064   elif type_ == bool:
2065     return value_string == 'True'
2066   elif type_ == users.User:
2067     return users.User(value_string, _auth_domain)
2068   elif type_ == type(None):
2069     return None
2070   return type_(value_string)
2071
2072
2073 def ReferenceToKeyValue(key):
2074   """Converts a key into a comparable hashable "key" value.
2075
2076   Args:
2077     key: The entity_pb.Reference or entity_v4_pb.Key from which to construct
2078         the key value.
2079
2080   Returns:
2081     A comparable and hashable representation of the given key that is
2082     compatible with one derived from a key property value.
2083   """
2084   if isinstance(key, entity_v4_pb.Key):
2085     v4_key = key
2086     key = entity_pb.Reference()
2087     datastore_pbs.get_entity_converter().v4_to_v3_reference(v4_key, key)
2088
2089   if isinstance(key, entity_pb.Reference):
2090     element_list = key.path().element_list()
2091   elif isinstance(key, entity_pb.PropertyValue_ReferenceValue):
2092     element_list = key.pathelement_list()
2093   else:
2094     raise datastore_errors.BadArgumentError(
2095         "key arg expected to be entity_pb.Reference or entity_v4.Key (%r)"
2096         % (key,))
2097
2098   result = [entity_pb.PropertyValue.kReferenceValueGroup,
2099             key.app(), key.name_space()]
2100   for element in element_list:
2101     result.append(element.type())
2102     if element.has_name():
2103       result.append(element.name())
2104     else:
2105       result.append(element.id())
2106   return tuple(result)
2107
2108
2109 def PropertyValueToKeyValue(prop_value):
2110   """Converts a entity_pb.PropertyValue into a comparable hashable "key" value.
2111
2112   The values produces by this function mimic the native ording of the datastore
2113   and uniquely identify the given PropertyValue.
2114
2115   Args:
2116     prop_value: The entity_pb.PropertyValue from which to construct the
2117       key value.
2118
2119   Returns:
2120     A comparable and hashable representation of the given property value.
2121   """
2122   if not isinstance(prop_value, entity_pb.PropertyValue):
2123     raise datastore_errors.BadArgumentError(
2124         'prop_value arg expected to be entity_pb.PropertyValue (%r)' %
2125         (prop_value,))
2126
2127
2128
2129   if prop_value.has_stringvalue():
2130     return (entity_pb.PropertyValue.kstringValue, prop_value.stringvalue())
2131   if prop_value.has_int64value():
2132     return (entity_pb.PropertyValue.kint64Value, prop_value.int64value())
2133   if prop_value.has_booleanvalue():
2134     return (entity_pb.PropertyValue.kbooleanValue, prop_value.booleanvalue())
2135   if prop_value.has_doublevalue():
2136
2137     encoder = sortable_pb_encoder.Encoder()
2138     encoder.putDouble(prop_value.doublevalue())
2139     return (entity_pb.PropertyValue.kdoubleValue, tuple(encoder.buf))
2140   if prop_value.has_pointvalue():
2141     return (entity_pb.PropertyValue.kPointValueGroup,
2142             prop_value.pointvalue().x(), prop_value.pointvalue().y())
2143   if prop_value.has_referencevalue():
2144     return ReferenceToKeyValue(prop_value.referencevalue())
2145   if prop_value.has_uservalue():
2146     result = []
2147     uservalue = prop_value.uservalue()
2148     if uservalue.has_email():
2149       result.append((entity_pb.PropertyValue.kUserValueemail,
2150                      uservalue.email()))
2151     if uservalue.has_auth_domain():
2152       result.append((entity_pb.PropertyValue.kUserValueauth_domain,
2153                      uservalue.auth_domain()))
2154     if uservalue.has_nickname():
2155       result.append((entity_pb.PropertyValue.kUserValuenickname,
2156                      uservalue.nickname()))
2157     if uservalue.has_gaiaid():
2158       result.append((entity_pb.PropertyValue.kUserValuegaiaid,
2159                      uservalue.gaiaid()))
2160     if uservalue.has_obfuscated_gaiaid():
2161       result.append((entity_pb.PropertyValue.kUserValueobfuscated_gaiaid,
2162                      uservalue.obfuscated_gaiaid()))
2163     if uservalue.has_federated_identity():
2164       result.append((entity_pb.PropertyValue.kUserValuefederated_identity,
2165                      uservalue.federated_identity()))
2166     if uservalue.has_federated_provider():
2167       result.append((entity_pb.PropertyValue.kUserValuefederated_provider,
2168                      uservalue.federated_provider()))
2169     result.sort()
2170     return (entity_pb.PropertyValue.kUserValueGroup, tuple(result))
2171   return ()
2172
2173
2174 def GetPropertyValueTag(value_pb):
2175   """Returns the tag constant associated with the given entity_pb.PropertyValue.
2176   """
2177   if value_pb.has_booleanvalue():
2178     return entity_pb.PropertyValue.kbooleanValue
2179   elif value_pb.has_doublevalue():
2180     return entity_pb.PropertyValue.kdoubleValue
2181   elif value_pb.has_int64value():
2182     return entity_pb.PropertyValue.kint64Value
2183   elif value_pb.has_pointvalue():
2184     return entity_pb.PropertyValue.kPointValueGroup
2185   elif value_pb.has_referencevalue():
2186     return entity_pb.PropertyValue.kReferenceValueGroup
2187   elif value_pb.has_stringvalue():
2188     return entity_pb.PropertyValue.kstringValue
2189   elif value_pb.has_uservalue():
2190     return entity_pb.PropertyValue.kUserValueGroup
2191   else:
2192     return 0