python/google/appengine/api/datastore_types.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21
  22 """Higher-level, semantic data types for the datastore. These types
  23 are expected to be set as attributes of Entities.  See "Supported Data Types"
  24 in the API Guide.
  25
  26 Most of these types are based on XML elements from Atom and GData elements
  27 from the atom and gd namespaces. For more information, see:
  28
  29   http://www.atomenabled.org/developers/syndication/
  30   http://code.google.com/apis/gdata/common-elements.html
  31
  32 The namespace schemas are:
  33
  34   http://www.w3.org/2005/Atom
  35   http://schemas.google.com/g/2005
  36 """
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46 import base64
  47 import calendar
  48 import datetime
  49 import os
  50 import re
  51 import string
  52 import time
  53 import urlparse
  54 from xml.sax import saxutils
  55
  56 from google.appengine.datastore import entity_pb
  57
  58 from google.appengine.api import datastore_errors
  59 from google.appengine.api import namespace_manager
  60 from google.appengine.api import users
  61 from google.appengine.datastore import datastore_pb
  62 from google.appengine.datastore import datastore_pbs
  63 from google.appengine.datastore import entity_v4_pb
  64 from google.appengine.datastore import sortable_pb_encoder
  65
  66
  67
  68
  69
  70
  71 _MAX_STRING_LENGTH = 500
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82 _MAX_LINK_PROPERTY_LENGTH = 2083
  83
  84
  85
  86
  87
  88 _MAX_RAW_PROPERTY_BYTES = 1000 * 1000
  89
  90
  91
  92
  93
  94
  95
  96 RESERVED_PROPERTY_NAME = re.compile('^__.*__$')
  97
  98
  99
 100
 101
 102
 103
 104 KEY_SPECIAL_PROPERTY = '__key__'
 105 _KEY_SPECIAL_PROPERTY = KEY_SPECIAL_PROPERTY
 106 _UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY = '__unapplied_log_timestamp_us__'
 107 SCATTER_SPECIAL_PROPERTY = '__scatter__'
 108 _SPECIAL_PROPERTIES = frozenset(
 109     [KEY_SPECIAL_PROPERTY,
 110      _UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY,
 111      SCATTER_SPECIAL_PROPERTY])
 112
 113
 114
 115
 116
 117
 118
 119 _NAMESPACE_SEPARATOR = '!'
 120
 121
 122
 123
 124
 125 _EMPTY_NAMESPACE_ID = 1
 126
 127
 128 _EPOCH = datetime.datetime.utcfromtimestamp(0)
 129
 130
 131
 132
 133
 134 class UtcTzinfo(datetime.tzinfo):
 135   def utcoffset(self, dt): return datetime.timedelta(0)
 136   def dst(self, dt): return datetime.timedelta(0)
 137   def tzname(self, dt): return 'UTC'
 138   def __repr__(self): return 'datastore_types.UTC'
 139
 140 UTC = UtcTzinfo()
 141
 142
 143 def typename(obj):
 144   """Returns the type of obj as a string. More descriptive and specific than
 145   type(obj), and safe for any object, unlike __class__."""
 146   if hasattr(obj, '__class__'):
 147     return getattr(obj, '__class__').__name__
 148   else:
 149     return type(obj).__name__
 150
 151
 152 def ValidateString(value,
 153                    name='unused',
 154                    exception=datastore_errors.BadValueError,
 155                    max_len=_MAX_STRING_LENGTH,
 156                    empty_ok=False):
 157   """Raises an exception if value is not a valid string or a subclass thereof.
 158
 159   A string is valid if it's not empty, no more than _MAX_STRING_LENGTH bytes,
 160   and not a Blob. The exception type can be specified with the exception
 161   argument; it defaults to BadValueError.
 162
 163   Args:
 164     value: the value to validate.
 165     name: the name of this value; used in the exception message.
 166     exception: the type of exception to raise.
 167     max_len: the maximum allowed length, in bytes.
 168     empty_ok: allow empty value.
 169   """
 170   if value is None and empty_ok:
 171     return
 172   if not isinstance(value, basestring) or isinstance(value, Blob):
 173     raise exception('%s should be a string; received %s (a %s):' %
 174                     (name, value, typename(value)))
 175   if not value and not empty_ok:
 176     raise exception('%s must not be empty.' % name)
 177
 178   if len(value.encode('utf-8')) > max_len:
 179     raise exception('%s must be under %d bytes.' % (name, max_len))
 180
 181
 182 def ValidateInteger(value,
 183                    name='unused',
 184                    exception=datastore_errors.BadValueError,
 185                    empty_ok=False,
 186                    zero_ok=False,
 187                    negative_ok=False):
 188   """Raises an exception if value is not a valid integer.
 189
 190   An integer is valid if it's not negative or empty and is an integer
 191   (either int or long).  The exception type raised can be specified
 192   with the exception argument; it defaults to BadValueError.
 193
 194   Args:
 195     value: the value to validate.
 196     name: the name of this value; used in the exception message.
 197     exception: the type of exception to raise.
 198     empty_ok: allow None value.
 199     zero_ok: allow zero value.
 200     negative_ok: allow negative value.
 201   """
 202   if value is None and empty_ok:
 203     return
 204   if not isinstance(value, (int, long)):
 205     raise exception('%s should be an integer; received %s (a %s).' %
 206                     (name, value, typename(value)))
 207   if not value and not zero_ok:
 208     raise exception('%s must not be 0 (zero)' % name)
 209   if value < 0 and not negative_ok:
 210     raise exception('%s must not be negative.' % name)
 211
 212 def ResolveAppId(app):
 213   """Validate app id, providing a default.
 214
 215   If the argument is None, $APPLICATION_ID is substituted.
 216
 217   Args:
 218     app: The app id argument value to be validated.
 219
 220   Returns:
 221     The value of app, or the substituted default.  Always a non-empty string.
 222
 223   Raises:
 224     BadArgumentError if the value is empty or not a string.
 225   """
 226   if app is None:
 227     app = os.environ.get('APPLICATION_ID', '')
 228   ValidateString(app, 'app', datastore_errors.BadArgumentError)
 229   return app
 230
 231
 232 def ResolveNamespace(namespace):
 233   """Validate app namespace, providing a default.
 234
 235   If the argument is None, namespace_manager.get_namespace() is substituted.
 236
 237   Args:
 238     namespace: The namespace argument value to be validated.
 239
 240   Returns:
 241     The value of namespace, or the substituted default. The empty string is used
 242     to denote the empty namespace.
 243
 244   Raises:
 245     BadArgumentError if the value is not a string.
 246   """
 247   if namespace is None:
 248     namespace = namespace_manager.get_namespace()
 249   else:
 250     namespace_manager.validate_namespace(
 251         namespace, datastore_errors.BadArgumentError)
 252   return namespace
 253
 254
 255 def EncodeAppIdNamespace(app_id, namespace):
 256   """Concatenates app id and namespace into a single string.
 257
 258   This method is needed for xml and datastore_file_stub.
 259
 260   Args:
 261     app_id: The application id to encode
 262     namespace: The namespace to encode
 263
 264   Returns:
 265     The string encoding for the app_id, namespace pair.
 266   """
 267   if not namespace:
 268     return app_id
 269   else:
 270     return app_id + _NAMESPACE_SEPARATOR + namespace
 271
 272
 273 def DecodeAppIdNamespace(app_namespace_str):
 274   """Decodes app_namespace_str into an (app_id, namespace) pair.
 275
 276   This method is the reverse of EncodeAppIdNamespace and is needed for
 277   datastore_file_stub.
 278
 279   Args:
 280     app_namespace_str: An encoded app_id, namespace pair created by
 281       EncodeAppIdNamespace
 282
 283   Returns:
 284     (app_id, namespace) pair encoded in app_namespace_str
 285   """
 286   sep = app_namespace_str.find(_NAMESPACE_SEPARATOR)
 287   if sep < 0:
 288     return (app_namespace_str, '')
 289   else:
 290     return (app_namespace_str[0:sep], app_namespace_str[sep + 1:])
 291
 292
 293 def SetNamespace(proto, namespace):
 294   """Sets the namespace for a protocol buffer or clears the field.
 295
 296   Args:
 297     proto: the protocol buffer to update
 298     namespace: the new namespace (None or an empty string will clear out the
 299         field).
 300   """
 301   if not namespace:
 302     proto.clear_name_space()
 303   else:
 304     proto.set_name_space(namespace)
 305
 306
 307 def PartitionString(value, separator):
 308   """Equivalent to python2.5 str.partition()
 309      TODO use str.partition() when python 2.5 is adopted.
 310
 311   Args:
 312     value: String to be partitioned
 313     separator: Separator string
 314   """
 315   index = value.find(separator)
 316   if index == -1:
 317     return (value, '', value[0:0])
 318   else:
 319     return (value[0:index], separator, value[index+len(separator):len(value)])
 320
 321
 322
 323 class Key(object):
 324   """The primary key for a datastore entity.
 325
 326   A datastore GUID. A Key instance uniquely identifies an entity across all
 327   apps, and includes all information necessary to fetch the entity from the
 328   datastore with Get().
 329
 330   Key implements __hash__, and key instances are immutable, so Keys may be
 331   used in sets and as dictionary keys.
 332   """
 333   __reference = None
 334
 335   def __init__(self, encoded=None):
 336     """Constructor. Creates a Key from a string.
 337
 338     Args:
 339       # a base64-encoded primary key, generated by Key.__str__
 340       encoded: str
 341     """
 342     self._str = None
 343     if encoded is not None:
 344       if not isinstance(encoded, basestring):
 345         try:
 346           repr_encoded = repr(encoded)
 347         except:
 348           repr_encoded = "<couldn't encode>"
 349         raise datastore_errors.BadArgumentError(
 350           'Key() expects a string; received %s (a %s).' %
 351           (repr_encoded, typename(encoded)))
 352       try:
 353
 354         modulo = len(encoded) % 4
 355         if modulo != 0:
 356           encoded += ('=' * (4 - modulo))
 357
 358
 359
 360
 361
 362
 363
 364         self._str = str(encoded)
 365         encoded_pb = base64.urlsafe_b64decode(self._str)
 366         self.__reference = entity_pb.Reference(encoded_pb)
 367         assert self.__reference.IsInitialized()
 368
 369
 370         self._str = self._str.rstrip('=')
 371
 372       except (AssertionError, TypeError), e:
 373         raise datastore_errors.BadKeyError(
 374           'Invalid string key %s. Details: %s' % (encoded, e))
 375       except Exception, e:
 376
 377
 378
 379
 380
 381         if e.__class__.__name__ == 'ProtocolBufferDecodeError':
 382           raise datastore_errors.BadKeyError('Invalid string key %s.' % encoded)
 383         else:
 384           raise
 385     else:
 386
 387       self.__reference = entity_pb.Reference()
 388
 389   def to_path(self, _default_id=None, _decode=True, _fail=True):
 390     """Construct the "path" of this key as a list.
 391
 392     Returns:
 393       A list [kind_1, id_or_name_1, ..., kind_n, id_or_name_n] of the key path.
 394
 395     Raises:
 396       datastore_errors.BadKeyError if this key does not have a valid path.
 397     """
 398
 399
 400
 401
 402
 403
 404     def Decode(s):
 405       if _decode:
 406         try:
 407           return s.decode('utf-8')
 408         except UnicodeDecodeError:
 409           if _fail:
 410             raise
 411       return s
 412
 413     path = []
 414     for path_element in self.__reference.path().element_list():
 415       path.append(Decode(path_element.type()))
 416       if path_element.has_name():
 417         path.append(Decode(path_element.name()))
 418       elif path_element.has_id():
 419         path.append(path_element.id())
 420       elif _default_id is not None:
 421         path.append(_default_id)
 422       else:
 423         raise datastore_errors.BadKeyError('Incomplete key found in to_path')
 424     return path
 425
 426   @staticmethod
 427   def from_path(*args, **kwds):
 428     """Static method to construct a Key out of a "path" (kind, id or name, ...).
 429
 430     This is useful when an application wants to use just the id or name portion
 431     of a key in e.g. a URL, where the rest of the URL provides enough context to
 432     fill in the rest, i.e. the app id (always implicit), the entity kind, and
 433     possibly an ancestor key. Since ids and names are usually small, they're
 434     more attractive for use in end-user-visible URLs than the full string
 435     representation of a key.
 436
 437     Args:
 438       kind: the entity kind (a str or unicode instance)
 439       id_or_name: the id (an int or long) or name (a str or unicode instance)
 440       parent: optional parent Key; default None.
 441       namespace: optional namespace to use otherwise namespace_manager's
 442         default namespace is used.
 443
 444     Returns:
 445       A new Key instance whose .kind() and .id() or .name() methods return
 446       the *last* kind and id or name positional arguments passed.
 447
 448     Raises:
 449       BadArgumentError for invalid arguments.
 450       BadKeyError if the parent key is incomplete.
 451     """
 452
 453     parent = kwds.pop('parent', None)
 454
 455     app_id = ResolveAppId(kwds.pop('_app', None))
 456
 457
 458
 459     namespace = kwds.pop('namespace', None)
 460
 461
 462     if kwds:
 463       raise datastore_errors.BadArgumentError(
 464           'Excess keyword arguments ' + repr(kwds))
 465
 466
 467     if not args or len(args) % 2:
 468       raise datastore_errors.BadArgumentError(
 469           'A non-zero even number of positional arguments is required '
 470           '(kind, id or name, kind, id or name, ...); received %s' % repr(args))
 471
 472
 473     if parent is not None:
 474       if not isinstance(parent, Key):
 475         raise datastore_errors.BadArgumentError(
 476             'Expected None or a Key as parent; received %r (a %s).' %
 477             (parent, typename(parent)))
 478       if namespace is None:
 479         namespace = parent.namespace()
 480       if not parent.has_id_or_name():
 481         raise datastore_errors.BadKeyError(
 482             'The parent Key is incomplete.')
 483       if app_id != parent.app() or namespace != parent.namespace():
 484         raise datastore_errors.BadArgumentError(
 485             'The app/namespace arguments (%s/%s) should match '
 486             'parent.app/namespace() (%s/%s)' %
 487             (app_id, namespace, parent.app(), parent.namespace()))
 488
 489
 490     namespace = ResolveNamespace(namespace)
 491
 492
 493     key = Key()
 494     ref = key.__reference
 495     if parent is not None:
 496       ref.CopyFrom(parent.__reference)
 497     else:
 498       ref.set_app(app_id)
 499       SetNamespace(ref, namespace)
 500
 501
 502
 503     path = ref.mutable_path()
 504     for i in xrange(0, len(args), 2):
 505       kind, id_or_name = args[i:i+2]
 506       if isinstance(kind, basestring):
 507         kind = kind.encode('utf-8')
 508       else:
 509         raise datastore_errors.BadArgumentError(
 510             'Expected a string kind as argument %d; received %r (a %s).' %
 511             (i + 1, kind, typename(kind)))
 512       elem = path.add_element()
 513       elem.set_type(kind)
 514       if isinstance(id_or_name, (int, long)):
 515         elem.set_id(id_or_name)
 516       elif isinstance(id_or_name, basestring):
 517         ValidateString(id_or_name, 'name')
 518         elem.set_name(id_or_name.encode('utf-8'))
 519       else:
 520         raise datastore_errors.BadArgumentError(
 521             'Expected an integer id or string name as argument %d; '
 522             'received %r (a %s).' % (i + 2, id_or_name, typename(id_or_name)))
 523
 524
 525     assert ref.IsInitialized()
 526     return key
 527
 528   def app(self):
 529     """Returns this entity's app id, a string."""
 530     if self.__reference.app():
 531       return self.__reference.app().decode('utf-8')
 532     else:
 533       return None
 534
 535   def namespace(self):
 536     """Returns this entity's namespace, a string."""
 537     if self.__reference.has_name_space():
 538       return self.__reference.name_space().decode('utf-8')
 539     else:
 540       return ''
 541
 542
 543
 544   def kind(self):
 545     """Returns this entity's kind, as a string."""
 546     if self.__reference.path().element_size() > 0:
 547       encoded = self.__reference.path().element_list()[-1].type()
 548       return unicode(encoded.decode('utf-8'))
 549     else:
 550       return None
 551
 552   def id(self):
 553     """Returns this entity's id, or None if it doesn't have one."""
 554     elems = self.__reference.path().element_list()
 555     if elems and elems[-1].has_id() and elems[-1].id():
 556       return elems[-1].id()
 557     else:
 558       return None
 559
 560   def name(self):
 561     """Returns this entity's name, or None if it doesn't have one."""
 562     elems = self.__reference.path().element_list()
 563     if elems and elems[-1].has_name() and elems[-1].name():
 564       return elems[-1].name().decode('utf-8')
 565     else:
 566       return None
 567
 568   def id_or_name(self):
 569     """Returns this entity's id or name, whichever it has, or None."""
 570     if self.id() is not None:
 571       return self.id()
 572     else:
 573
 574       return self.name()
 575
 576   def has_id_or_name(self):
 577     """Returns True if this entity has an id or name, False otherwise.
 578     """
 579     elems = self.__reference.path().element_list()
 580     if elems:
 581       e = elems[-1]
 582       return bool(e.name() or e.id())
 583     else:
 584       return False
 585
 586   def parent(self):
 587     """Returns this entity's parent, as a Key. If this entity has no parent,
 588     returns None."""
 589     if self.__reference.path().element_size() > 1:
 590       parent = Key()
 591       parent.__reference.CopyFrom(self.__reference)
 592       del parent.__reference.path().element_list()[-1]
 593       return parent
 594     else:
 595       return None
 596
 597   def ToTagUri(self):
 598     """Returns a tag: URI for this entity for use in XML output.
 599
 600     Foreign keys for entities may be represented in XML output as tag URIs.
 601     RFC 4151 describes the tag URI scheme. From http://taguri.org/:
 602
 603       The tag algorithm lets people mint - create - identifiers that no one
 604       else using the same algorithm could ever mint. It is simple enough to do
 605       in your head, and the resulting identifiers can be easy to read, write,
 606       and remember. The identifiers conform to the URI (URL) Syntax.
 607
 608     Tag URIs for entities use the app's auth domain and the date that the URI
 609      is generated. The namespace-specific part is <kind>[<key>].
 610
 611     For example, here is the tag URI for a Kitten with the key "Fluffy" in the
 612     catsinsinks app:
 613
 614       tag:catsinsinks.googleapps.com,2006-08-29:Kitten[Fluffy]
 615
 616     Raises a BadKeyError if this entity's key is incomplete.
 617     """
 618     if not self.has_id_or_name():
 619       raise datastore_errors.BadKeyError(
 620         'ToTagUri() called for an entity with an incomplete key.')
 621
 622     return u'tag:%s.%s,%s:%s[%s]' % (
 623
 624         saxutils.escape(EncodeAppIdNamespace(self.app(), self.namespace())),
 625         os.environ['AUTH_DOMAIN'],
 626         datetime.date.today().isoformat(),
 627         saxutils.escape(self.kind()),
 628         saxutils.escape(str(self)))
 629
 630   ToXml = ToTagUri
 631
 632   def entity_group(self):
 633     """Returns this key's entity group as a Key.
 634
 635     Note that the returned Key will be incomplete if this Key is for a root
 636     entity and it is incomplete.
 637     """
 638     group = Key._FromPb(self.__reference)
 639     del group.__reference.path().element_list()[1:]
 640     return group
 641
 642   @staticmethod
 643   def _FromPb(pb):
 644     """Static factory method. Creates a Key from an entity_pb.Reference.
 645
 646     Not intended to be used by application developers. Enforced by hiding the
 647     entity_pb classes.
 648
 649     Args:
 650       pb: entity_pb.Reference
 651     """
 652     if not isinstance(pb, entity_pb.Reference):
 653       raise datastore_errors.BadArgumentError(
 654         'Key constructor takes an entity_pb.Reference; received %s (a %s).' %
 655         (pb, typename(pb)))
 656
 657     key = Key()
 658     key.__reference = entity_pb.Reference()
 659     key.__reference.CopyFrom(pb)
 660     return key
 661
 662   def _ToPb(self):
 663     """Converts this Key to its protocol buffer representation.
 664
 665     Not intended to be used by application developers. Enforced by hiding the
 666     entity_pb classes.
 667
 668     Returns:
 669       # the Reference PB representation of this Key
 670       entity_pb.Reference
 671     """
 672     pb = entity_pb.Reference()
 673     pb.CopyFrom(self.__reference)
 674
 675
 676
 677     pb.app().decode('utf-8')
 678     for pathelem in pb.path().element_list():
 679       pathelem.type().decode('utf-8')
 680
 681     return pb
 682
 683   def __str__(self):
 684     """Encodes this Key as an opaque string.
 685
 686     Returns a string representation of this key, suitable for use in HTML,
 687     URLs, and other similar use cases. If the entity's key is incomplete,
 688     raises a BadKeyError.
 689
 690     Unfortunately, this string encoding isn't particularly compact, and its
 691     length varies with the length of the path. If you want a shorter identifier
 692     and you know the kind and parent (if any) ahead of time, consider using just
 693     the entity's id or name.
 694
 695     Returns:
 696       string
 697     """
 698
 699
 700
 701     try:
 702       if self._str is not None:
 703         return self._str
 704     except AttributeError:
 705       pass
 706     if (self.has_id_or_name()):
 707       encoded = base64.urlsafe_b64encode(self.__reference.Encode())
 708       self._str = encoded.replace('=', '')
 709     else:
 710       raise datastore_errors.BadKeyError(
 711         'Cannot string encode an incomplete key!\n%s' % self.__reference)
 712     return self._str
 713
 714
 715   def __repr__(self):
 716     """Returns an eval()able string representation of this key.
 717
 718     Returns a Python string of the form 'datastore_types.Key.from_path(...)'
 719     that can be used to recreate this key.
 720
 721     Returns:
 722       string
 723     """
 724     args = []
 725     for elem in self.__reference.path().element_list():
 726       args.append(repr(elem.type().decode('utf-8')))
 727       if elem.has_name():
 728         args.append(repr(elem.name().decode('utf-8')))
 729       else:
 730         args.append(repr(elem.id()))
 731
 732     args.append('_app=%r' % self.__reference.app().decode('utf-8'))
 733     if self.__reference.has_name_space():
 734       args.append('namespace=%r' %
 735           self.__reference.name_space().decode('utf-8'))
 736     return u'datastore_types.Key.from_path(%s)' % ', '.join(args)
 737
 738   def __cmp__(self, other):
 739     """Returns negative, zero, or positive when comparing two keys.
 740
 741     TODO: for API v2, we should change this to make incomplete keys, ie
 742     keys without an id or name, not equal to any other keys.
 743
 744     Args:
 745       other: Key to compare to.
 746
 747     Returns:
 748       Negative if self is less than "other"
 749       Zero if "other" is equal to self
 750       Positive if self is greater than "other"
 751     """
 752     if not isinstance(other, Key):
 753       return -2
 754
 755     self_args = [self.__reference.app(), self.__reference.name_space()]
 756     self_args += self.to_path(_default_id=0, _decode=False)
 757
 758     other_args = [other.__reference.app(), other.__reference.name_space()]
 759     other_args += other.to_path(_default_id=0, _decode=False)
 760
 761     for self_component, other_component in zip(self_args, other_args):
 762       comparison = cmp(self_component, other_component)
 763       if comparison != 0:
 764         return comparison
 765
 766     return cmp(len(self_args), len(other_args))
 767
 768   def __hash__(self):
 769     """Returns an integer hash of this key.
 770
 771     Implements Python's hash protocol so that Keys may be used in sets and as
 772     dictionary keys.
 773
 774     Returns:
 775       int
 776     """
 777     args = self.to_path(_default_id=0, _fail=False)
 778     args.append(self.__reference.app())
 779     return hash(type(args)) ^ hash(tuple(args))
 780
 781
 782 class _OverflowDateTime(long):
 783   """Container for GD_WHEN values that don't fit into a datetime.datetime.
 784
 785   This class only exists to safely round-trip GD_WHEN values that are too large
 786   to fit in a datetime.datetime instance e.g. that were created by Java
 787   applications. It should not be created directly.
 788   """
 789   pass
 790
 791
 792 def _When(val):
 793   """Coverts a GD_WHEN value to the appropriate type."""
 794   try:
 795     return _EPOCH + datetime.timedelta(microseconds=val)
 796   except OverflowError:
 797     return _OverflowDateTime(val)
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808 class Category(unicode):
 809   """A tag, ie a descriptive word or phrase. Entities may be tagged by users,
 810   and later returned by a queries for that tag. Tags can also be used for
 811   ranking results (frequency), photo captions, clustering, activity, etc.
 812
 813   Here's a more in-depth description:  http://www.zeldman.com/daily/0405d.shtml
 814
 815   This is the Atom "category" element. In XML output, the tag is provided as
 816   the term attribute. See:
 817   http://www.atomenabled.org/developers/syndication/#category
 818
 819   Raises BadValueError if tag is not a string or subtype.
 820   """
 821   TERM = 'user-tag'
 822
 823   def __init__(self, tag):
 824     super(Category, self).__init__()
 825     ValidateString(tag, 'tag')
 826
 827   def ToXml(self):
 828     return u'<category term="%s" label=%s />' % (Category.TERM,
 829                                                  saxutils.quoteattr(self))
 830
 831
 832 class Link(unicode):
 833   """A fully qualified URL. Usually http: scheme, but may also be file:, ftp:,
 834   news:, among others.
 835
 836   If you have email (mailto:) or instant messaging (aim:, xmpp:) links,
 837   consider using the Email or IM classes instead.
 838
 839   This is the Atom "link" element. In XML output, the link is provided as the
 840   href attribute. See:
 841   http://www.atomenabled.org/developers/syndication/#link
 842
 843   Raises BadValueError if link is not a fully qualified, well-formed URL.
 844   """
 845   def __init__(self, link):
 846     super(Link, self).__init__()
 847     ValidateString(link, 'link', max_len=_MAX_LINK_PROPERTY_LENGTH)
 848
 849     scheme, domain, path, params, query, fragment = urlparse.urlparse(link)
 850     if (not scheme or (scheme != 'file' and not domain) or
 851                       (scheme == 'file' and not path)):
 852       raise datastore_errors.BadValueError('Invalid URL: %s' % link)
 853
 854   def ToXml(self):
 855     return u'<link href=%s />' % saxutils.quoteattr(self)
 856
 857
 858 class Email(unicode):
 859   """An RFC2822 email address. Makes no attempt at validation; apart from
 860   checking MX records, email address validation is a rathole.
 861
 862   This is the gd:email element. In XML output, the email address is provided as
 863   the address attribute. See:
 864   http://code.google.com/apis/gdata/common-elements.html#gdEmail
 865
 866   Raises BadValueError if email is not a valid email address.
 867   """
 868   def __init__(self, email):
 869     super(Email, self).__init__()
 870     ValidateString(email, 'email')
 871
 872   def ToXml(self):
 873     return u'<gd:email address=%s />' % saxutils.quoteattr(self)
 874
 875
 876 class GeoPt(object):
 877   """A geographical point, specified by floating-point latitude and longitude
 878   coordinates. Often used to integrate with mapping sites like Google Maps.
 879   May also be used as ICBM coordinates.
 880
 881   This is the georss:point element. In XML output, the coordinates are
 882   provided as the lat and lon attributes. See: http://georss.org/
 883
 884   Serializes to '<lat>,<lon>'. Raises BadValueError if it's passed an invalid
 885   serialized string, or if lat and lon are not valid floating points in the
 886   ranges [-90, 90] and [-180, 180], respectively.
 887   """
 888   lat = None
 889   lon = None
 890
 891   def __init__(self, lat, lon=None):
 892     if lon is None:
 893
 894       try:
 895         split = lat.split(',')
 896         lat, lon = split
 897       except (AttributeError, ValueError):
 898         raise datastore_errors.BadValueError(
 899           'Expected a "lat,long" formatted string; received %s (a %s).' %
 900           (lat, typename(lat)))
 901
 902     try:
 903       lat = float(lat)
 904       lon = float(lon)
 905       if abs(lat) > 90:
 906         raise datastore_errors.BadValueError(
 907           'Latitude must be between -90 and 90; received %f' % lat)
 908       if abs(lon) > 180:
 909         raise datastore_errors.BadValueError(
 910           'Longitude must be between -180 and 180; received %f' % lon)
 911     except (TypeError, ValueError):
 912
 913       raise datastore_errors.BadValueError(
 914         'Expected floats for lat and long; received %s (a %s) and %s (a %s).' %
 915         (lat, typename(lat), lon, typename(lon)))
 916
 917     self.lat = lat
 918     self.lon = lon
 919
 920   def __cmp__(self, other):
 921     if not isinstance(other, GeoPt):
 922       try:
 923         other = GeoPt(other)
 924       except datastore_errors.BadValueError:
 925         return NotImplemented
 926
 927
 928     lat_cmp = cmp(self.lat, other.lat)
 929     if lat_cmp != 0:
 930       return lat_cmp
 931     else:
 932       return cmp(self.lon, other.lon)
 933
 934   def __hash__(self):
 935     """Returns an integer hash of this point.
 936
 937     Implements Python's hash protocol so that GeoPts may be used in sets and
 938     as dictionary keys.
 939
 940     Returns:
 941       int
 942     """
 943     return hash((self.lat, self.lon))
 944
 945   def __repr__(self):
 946     """Returns an eval()able string representation of this GeoPt.
 947
 948     The returned string is of the form 'datastore_types.GeoPt([lat], [lon])'.
 949
 950     Returns:
 951       string
 952     """
 953     return 'datastore_types.GeoPt(%r, %r)' % (self.lat, self.lon)
 954
 955   def __unicode__(self):
 956     return u'%s,%s' % (unicode(self.lat), unicode(self.lon))
 957
 958   __str__ = __unicode__
 959
 960   def ToXml(self):
 961     return u'<georss:point>%s %s</georss:point>' % (unicode(self.lat),
 962                                                     unicode(self.lon))
 963
 964
 965 class IM(object):
 966   """An instant messaging handle. Includes both an address and its protocol.
 967   The protocol value is either a standard IM scheme or a URL identifying the
 968   IM network for the protocol. Possible values include:
 969
 970     Value                           Description
 971     sip                             SIP/SIMPLE
 972     unknown                         Unknown or unspecified
 973     xmpp                            XMPP/Jabber
 974     http://aim.com/                 AIM
 975     http://icq.com/                 ICQ
 976     http://talk.google.com/         Google Talk
 977     http://messenger.msn.com/       MSN Messenger
 978     http://messenger.yahoo.com/     Yahoo Messenger
 979     http://sametime.com/            Lotus Sametime
 980     http://gadu-gadu.pl/            Gadu-Gadu
 981
 982   This is the gd:im element. In XML output, the address and protocol are
 983   provided as the address and protocol attributes, respectively. See:
 984   http://code.google.com/apis/gdata/common-elements.html#gdIm
 985
 986   Serializes to '<protocol> <address>'. Raises BadValueError if tag is not a
 987   standard IM scheme or a URL.
 988   """
 989   PROTOCOLS = [ 'sip', 'unknown', 'xmpp' ]
 990
 991   protocol = None
 992   address = None
 993
 994   def __init__(self, protocol, address=None):
 995     if address is None:
 996
 997       try:
 998         split = protocol.split(' ', 1)
 999         protocol, address = split
1000       except (AttributeError, ValueError):
1001         raise datastore_errors.BadValueError(
1002           'Expected string of format "protocol address"; received %s' %
1003           (protocol,))
1004
1005     ValidateString(address, 'address')
1006     if protocol not in self.PROTOCOLS:
1007
1008       Link(protocol)
1009
1010     self.address = address
1011     self.protocol = protocol
1012
1013   def __cmp__(self, other):
1014     if not isinstance(other, IM):
1015       try:
1016         other = IM(other)
1017       except datastore_errors.BadValueError:
1018         return NotImplemented
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030     return cmp((self.address, self.protocol),
1031                (other.address, other.protocol))
1032
1033   def __repr__(self):
1034     """Returns an eval()able string representation of this IM.
1035
1036     The returned string is of the form:
1037
1038       datastore_types.IM('address', 'protocol')
1039
1040     Returns:
1041       string
1042     """
1043     return 'datastore_types.IM(%r, %r)' % (self.protocol, self.address)
1044
1045   def __unicode__(self):
1046     return u'%s %s' % (self.protocol, self.address)
1047
1048   __str__ = __unicode__
1049
1050   def ToXml(self):
1051     return (u'<gd:im protocol=%s address=%s />' %
1052             (saxutils.quoteattr(self.protocol),
1053              saxutils.quoteattr(self.address)))
1054
1055   def __len__(self):
1056     return len(unicode(self))
1057
1058
1059 class PhoneNumber(unicode):
1060   """A human-readable phone number or address.
1061
1062   No validation is performed. Phone numbers have many different formats -
1063   local, long distance, domestic, international, internal extension, TTY,
1064   VOIP, SMS, and alternative networks like Skype, XFire and Roger Wilco. They
1065   all have their own numbering and addressing formats.
1066
1067   This is the gd:phoneNumber element. In XML output, the phone number is
1068   provided as the text of the element. See:
1069   http://code.google.com/apis/gdata/common-elements.html#gdPhoneNumber
1070
1071   Raises BadValueError if phone is not a string or subtype.
1072   """
1073   def __init__(self, phone):
1074     super(PhoneNumber, self).__init__()
1075     ValidateString(phone, 'phone')
1076
1077   def ToXml(self):
1078     return u'<gd:phoneNumber>%s</gd:phoneNumber>' % saxutils.escape(self)
1079
1080
1081 class PostalAddress(unicode):
1082   """A human-readable mailing address. Again, mailing address formats vary
1083   widely, so no validation is performed.
1084
1085   This is the gd:postalAddress element. In XML output, the address is provided
1086   as the text of the element. See:
1087   http://code.google.com/apis/gdata/common-elements.html#gdPostalAddress
1088
1089   Raises BadValueError if address is not a string or subtype.
1090   """
1091   def __init__(self, address):
1092     super(PostalAddress, self).__init__()
1093     ValidateString(address, 'address')
1094
1095   def ToXml(self):
1096     return u'<gd:postalAddress>%s</gd:postalAddress>' % saxutils.escape(self)
1097
1098
1099 class Rating(long):
1100   """A user-provided integer rating for a piece of content. Normalized to a
1101   0-100 scale.
1102
1103   This is the gd:rating element. In XML output, the address is provided
1104   as the text of the element. See:
1105   http://code.google.com/apis/gdata/common-elements.html#gdRating
1106
1107   Serializes to the decimal string representation of the rating. Raises
1108   BadValueError if the rating is not an integer in the range [0, 100].
1109   """
1110   MIN = 0
1111   MAX = 100
1112
1113   def __init__(self, rating):
1114     super(Rating, self).__init__()
1115     if isinstance(rating, float) or isinstance(rating, complex):
1116
1117       raise datastore_errors.BadValueError(
1118         'Expected int or long; received %s (a %s).' %
1119         (rating, typename(rating)))
1120
1121     try:
1122       if long(rating) < Rating.MIN or long(rating) > Rating.MAX:
1123         raise datastore_errors.BadValueError()
1124     except ValueError:
1125
1126       raise datastore_errors.BadValueError(
1127         'Expected int or long; received %s (a %s).' %
1128         (rating, typename(rating)))
1129
1130   def ToXml(self):
1131     return (u'<gd:rating value="%d" min="%d" max="%d" />' %
1132             (self, Rating.MIN, Rating.MAX))
1133
1134
1135 class Text(unicode):
1136   """A long string type.
1137
1138   Strings of any length can be stored in the datastore using this
1139   type. It behaves identically to the Python unicode type, except for
1140   the constructor, which only accepts str and unicode arguments.
1141   """
1142
1143   def __new__(cls, arg=None, encoding=None):
1144     """Constructor.
1145
1146     We only accept unicode and str instances, the latter with encoding.
1147
1148     Args:
1149       arg: optional unicode or str instance; default u''
1150       encoding: optional encoding; disallowed when isinstance(arg, unicode),
1151                 defaults to 'ascii' when isinstance(arg, str);
1152     """
1153     if arg is None:
1154       arg = u''
1155     if isinstance(arg, unicode):
1156       if encoding is not None:
1157         raise TypeError('Text() with a unicode argument '
1158                         'should not specify an encoding')
1159       return super(Text, cls).__new__(cls, arg)
1160
1161     if isinstance(arg, str):
1162       if encoding is None:
1163         encoding = 'ascii'
1164       return super(Text, cls).__new__(cls, arg, encoding)
1165
1166     raise TypeError('Text() argument should be str or unicode, not %s' %
1167                     type(arg).__name__)
1168
1169 class _BaseByteType(str):
1170   """A base class for datastore types that are encoded as bytes.
1171
1172   This behaves identically to the Python str type, except for the
1173   constructor, which only accepts str arguments.
1174   """
1175
1176   def __new__(cls, arg=None):
1177     """Constructor.
1178
1179     We only accept str instances.
1180
1181     Args:
1182       arg: optional str instance (default '')
1183     """
1184     if arg is None:
1185       arg = ''
1186     if isinstance(arg, str):
1187       return super(_BaseByteType, cls).__new__(cls, arg)
1188
1189     raise TypeError('%s() argument should be str instance, not %s' %
1190                     (cls.__name__, type(arg).__name__))
1191
1192   def ToXml(self):
1193     """Output bytes as XML.
1194
1195     Returns:
1196       Base64 encoded version of itself for safe insertion in to an XML document.
1197     """
1198     encoded = base64.urlsafe_b64encode(self)
1199     return saxutils.escape(encoded)
1200
1201
1202 class Blob(_BaseByteType):
1203   """A blob type, appropriate for storing binary data of any length.
1204
1205   This behaves identically to the Python str type, except for the
1206   constructor, which only accepts str arguments.
1207   """
1208   pass
1209
1210
1211 class EmbeddedEntity(_BaseByteType):
1212   """A proto encoded EntityProto.
1213
1214   This behaves identically to Blob, except for the
1215   constructor, which accepts a str or EntityProto argument.
1216
1217   Can be decoded using datastore.Entity.FromProto(), db.model_from_protobuf() or
1218   ndb.LocalStructuredProperty.
1219   """
1220
1221   def __new__(cls, arg=None):
1222     """Constructor.
1223
1224     Args:
1225       arg: optional str or EntityProto instance (default '')
1226     """
1227     if isinstance(arg, entity_pb.EntityProto):
1228       arg = arg.SerializePartialToString()
1229     return super(EmbeddedEntity, cls).__new__(cls, arg)
1230
1231
1232 class ByteString(_BaseByteType):
1233   """A byte-string type, appropriate for storing short amounts of indexed data.
1234
1235   This behaves identically to Blob, except it's used only for short, indexed
1236   byte strings.
1237   """
1238   pass
1239
1240
1241 class BlobKey(object):
1242   """Key used to identify a blob in Blobstore.
1243
1244   This object wraps a string that gets used internally by the Blobstore API
1245   to identify application blobs.  The BlobKey corresponds to the entity name
1246   of the underlying BlobReference entity.
1247
1248   This class is exposed in the API in both google.appengine.ext.db and
1249   google.appengine.ext.blobstore.
1250   """
1251
1252   def __init__(self, blob_key):
1253     """Constructor.
1254
1255     Used to convert a string to a BlobKey.  Normally used internally by
1256     Blobstore API.
1257
1258     Args:
1259       blob_key:  Key name of BlobReference that this key belongs to.
1260     """
1261     ValidateString(blob_key, 'blob-key')
1262     self.__blob_key = blob_key
1263
1264   def __str__(self):
1265     """Convert to string."""
1266     return self.__blob_key
1267
1268   def __repr__(self):
1269     """Returns an eval()able string representation of this key.
1270
1271     Returns a Python string of the form 'datastore_types.BlobKey(...)'
1272     that can be used to recreate this key.
1273
1274     Returns:
1275       string
1276     """
1277     return 'datastore_types.%s(%r)' % (type(self).__name__, self.__blob_key)
1278
1279   def __cmp__(self, other):
1280
1281
1282     if type(other) is type(self):
1283       return cmp(str(self), str(other))
1284     elif isinstance(other, basestring):
1285       return cmp(self.__blob_key, other)
1286     else:
1287       return NotImplemented
1288
1289   def __hash__(self):
1290     return hash(self.__blob_key)
1291
1292   def ToXml(self):
1293     return str(self)
1294
1295
1296
1297 _PROPERTY_MEANINGS = {
1298
1299
1300
1301   Blob:              entity_pb.Property.BLOB,
1302   EmbeddedEntity:    entity_pb.Property.ENTITY_PROTO,
1303   ByteString:        entity_pb.Property.BYTESTRING,
1304   Text:              entity_pb.Property.TEXT,
1305   datetime.datetime: entity_pb.Property.GD_WHEN,
1306   datetime.date:     entity_pb.Property.GD_WHEN,
1307   datetime.time:     entity_pb.Property.GD_WHEN,
1308   _OverflowDateTime: entity_pb.Property.GD_WHEN,
1309   Category:          entity_pb.Property.ATOM_CATEGORY,
1310   Link:              entity_pb.Property.ATOM_LINK,
1311   Email:             entity_pb.Property.GD_EMAIL,
1312   GeoPt:             entity_pb.Property.GEORSS_POINT,
1313   IM:                entity_pb.Property.GD_IM,
1314   PhoneNumber:       entity_pb.Property.GD_PHONENUMBER,
1315   PostalAddress:     entity_pb.Property.GD_POSTALADDRESS,
1316   Rating:            entity_pb.Property.GD_RATING,
1317   BlobKey:           entity_pb.Property.BLOBKEY,
1318 }
1319
1320
1321 _PROPERTY_TYPES = frozenset([
1322   Blob,
1323   EmbeddedEntity,
1324   ByteString,
1325   bool,
1326   Category,
1327   datetime.datetime,
1328   _OverflowDateTime,
1329   Email,
1330   float,
1331   GeoPt,
1332   IM,
1333   int,
1334   Key,
1335   Link,
1336   long,
1337   PhoneNumber,
1338   PostalAddress,
1339   Rating,
1340   str,
1341   Text,
1342   type(None),
1343   unicode,
1344   users.User,
1345   BlobKey,
1346 ])
1347
1348
1349
1350
1351 _RAW_PROPERTY_TYPES = (Blob, Text, EmbeddedEntity)
1352 _RAW_PROPERTY_MEANINGS = (entity_pb.Property.BLOB, entity_pb.Property.TEXT,
1353                           entity_pb.Property.ENTITY_PROTO)
1354
1355
1356 def ValidatePropertyInteger(name, value):
1357   """Raises an exception if the supplied integer is invalid.
1358
1359   Args:
1360     name: Name of the property this is for.
1361     value: Integer value.
1362
1363   Raises:
1364     OverflowError if the value does not fit within a signed int64.
1365   """
1366   if not (-0x8000000000000000 <= value <= 0x7fffffffffffffff):
1367     raise OverflowError('%d is out of bounds for int64' % value)
1368
1369
1370 def ValidateStringLength(name, value, max_len):
1371   """Raises an exception if the supplied string is too long.
1372
1373   Args:
1374     name: Name of the property this is for.
1375     value: String value.
1376     max_len: Maximum length the string may be.
1377
1378   Raises:
1379     OverflowError if the value is larger than the maximum length.
1380   """
1381   if len(value) > max_len:
1382     raise datastore_errors.BadValueError(
1383       'Property %s is %d bytes long; it must be %d or less. '
1384       'Consider Text instead, which can store strings of any length.' %
1385       (name, len(value), max_len))
1386
1387
1388 def ValidatePropertyString(name, value):
1389   """Validates the length of an indexed string property.
1390
1391   Args:
1392     name: Name of the property this is for.
1393     value: String value.
1394   """
1395   ValidateStringLength(name, value, max_len=_MAX_STRING_LENGTH)
1396
1397
1398 def ValidatePropertyLink(name, value):
1399   """Validates the length of an indexed Link property.
1400
1401   Args:
1402     name: Name of the property this is for.
1403     value: String value.
1404   """
1405   ValidateStringLength(name, value, max_len=_MAX_LINK_PROPERTY_LENGTH)
1406
1407
1408 def ValidatePropertyNothing(name, value):
1409   """No-op validation function.
1410
1411   Args:
1412     name: Name of the property this is for.
1413     value: Not used.
1414   """
1415   pass
1416
1417
1418 def ValidatePropertyKey(name, value):
1419   """Raises an exception if the supplied datastore.Key instance is invalid.
1420
1421   Args:
1422     name: Name of the property this is for.
1423     value: A datastore.Key instance.
1424
1425   Raises:
1426     datastore_errors.BadValueError if the value is invalid.
1427   """
1428   if not value.has_id_or_name():
1429     raise datastore_errors.BadValueError(
1430         'Incomplete key found for reference property %s.' % name)
1431
1432
1433
1434
1435
1436
1437
1438 _VALIDATE_PROPERTY_VALUES = {
1439   Blob: ValidatePropertyNothing,
1440   EmbeddedEntity: ValidatePropertyNothing,
1441   ByteString: ValidatePropertyNothing,
1442   bool: ValidatePropertyNothing,
1443   Category: ValidatePropertyNothing,
1444   datetime.datetime: ValidatePropertyNothing,
1445   _OverflowDateTime: ValidatePropertyInteger,
1446   Email: ValidatePropertyNothing,
1447   float: ValidatePropertyNothing,
1448   GeoPt: ValidatePropertyNothing,
1449   IM: ValidatePropertyNothing,
1450   int: ValidatePropertyInteger,
1451   Key: ValidatePropertyKey,
1452   Link: ValidatePropertyNothing,
1453   long: ValidatePropertyInteger,
1454   PhoneNumber: ValidatePropertyNothing,
1455   PostalAddress: ValidatePropertyNothing,
1456   Rating: ValidatePropertyInteger,
1457   str: ValidatePropertyNothing,
1458   Text: ValidatePropertyNothing,
1459   type(None): ValidatePropertyNothing,
1460   unicode: ValidatePropertyNothing,
1461   users.User: ValidatePropertyNothing,
1462   BlobKey: ValidatePropertyNothing,
1463 }
1464
1465 _PROPERTY_TYPE_TO_INDEX_VALUE_TYPE = {
1466   basestring: str,
1467   Blob: str,
1468   EmbeddedEntity: str,
1469   ByteString: str,
1470   bool: bool,
1471   Category: str,
1472   datetime.datetime: long,
1473   datetime.date: long,
1474   datetime.time: long,
1475   _OverflowDateTime: long,
1476   Email: str,
1477   float: float,
1478   GeoPt: GeoPt,
1479   IM: str,
1480   int: long,
1481   Key: Key,
1482   Link: str,
1483   long: long,
1484   PhoneNumber: str,
1485   PostalAddress: str,
1486   Rating: long,
1487   str: str,
1488   Text: str,
1489   type(None): type(None),
1490   unicode: str,
1491   users.User: users.User,
1492   BlobKey: str,
1493 }
1494
1495
1496 assert set(_VALIDATE_PROPERTY_VALUES.iterkeys()) == _PROPERTY_TYPES
1497
1498
1499 def ValidateProperty(name, values, read_only=False):
1500   """Helper function for validating property values.
1501
1502   Args:
1503     name: Name of the property this is for.
1504     value: Value for the property as a Python native type.
1505     read_only: deprecated
1506
1507   Raises:
1508     BadPropertyError if the property name is invalid. BadValueError if the
1509     property did not validate correctly or the value was an empty list. Other
1510     exception types (like OverflowError) if the property value does not meet
1511     type-specific criteria.
1512   """
1513   ValidateString(name, 'property name', datastore_errors.BadPropertyError)
1514
1515   values_type = type(values)
1516
1517
1518   if values_type is tuple:
1519     raise datastore_errors.BadValueError(
1520         'May not use tuple property value; property %s is %s.' %
1521         (name, repr(values)))
1522
1523
1524   if values_type is not list:
1525     values = [values]
1526
1527
1528   if not values:
1529     raise datastore_errors.BadValueError(
1530         'May not use the empty list as a property value; property %s is %s.' %
1531         (name, repr(values)))
1532
1533
1534
1535   try:
1536     for v in values:
1537       prop_validator = _VALIDATE_PROPERTY_VALUES.get(v.__class__)
1538       if prop_validator is None:
1539         raise datastore_errors.BadValueError(
1540           'Unsupported type for property %s: %s' % (name, v.__class__))
1541       prop_validator(name, v)
1542
1543   except (KeyError, ValueError, TypeError, IndexError, AttributeError), msg:
1544     raise datastore_errors.BadValueError(
1545       'Error type checking values for property %s: %s' % (name, msg))
1546
1547
1548
1549
1550
1551 ValidateReadProperty = ValidateProperty
1552
1553
1554
1555 def PackBlob(name, value, pbvalue):
1556   """Packs a Blob property into a entity_pb.PropertyValue.
1557
1558   Args:
1559     name: The name of the property as a string.
1560     value: A Blob instance.
1561     pbvalue: The entity_pb.PropertyValue to pack this value into.
1562   """
1563   pbvalue.set_stringvalue(value)
1564
1565
1566 def PackString(name, value, pbvalue):
1567   """Packs a string-typed property into a entity_pb.PropertyValue.
1568
1569   Args:
1570     name: The name of the property as a string.
1571     value: A string, unicode, or string-like value instance.
1572     pbvalue: The entity_pb.PropertyValue to pack this value into.
1573   """
1574   pbvalue.set_stringvalue(unicode(value).encode('utf-8'))
1575
1576
1577 def PackDatetime(name, value, pbvalue):
1578   """Packs a datetime-typed property into a entity_pb.PropertyValue.
1579
1580   Args:
1581     name: The name of the property as a string.
1582     value: A datetime.datetime instance.
1583     pbvalue: The entity_pb.PropertyValue to pack this value into.
1584   """
1585   pbvalue.set_int64value(DatetimeToTimestamp(value))
1586
1587
1588 def DatetimeToTimestamp(value):
1589   """Converts a datetime.datetime to microseconds since the epoch, as a float.
1590   Args:
1591     value: datetime.datetime
1592
1593   Returns: value as a long
1594   """
1595   if value.tzinfo:
1596
1597     value = value.astimezone(UTC)
1598   return long(calendar.timegm(value.timetuple()) * 1000000L) + value.microsecond
1599
1600
1601 def PackGeoPt(name, value, pbvalue):
1602   """Packs a GeoPt property into a entity_pb.PropertyValue.
1603
1604   Args:
1605     name: The name of the property as a string.
1606     value: A GeoPt instance.
1607     pbvalue: The entity_pb.PropertyValue to pack this value into.
1608   """
1609   pbvalue.mutable_pointvalue().set_x(value.lat)
1610   pbvalue.mutable_pointvalue().set_y(value.lon)
1611
1612
1613 def PackUser(name, value, pbvalue):
1614   """Packs a User property into a entity_pb.PropertyValue.
1615
1616   Args:
1617     name: The name of the property as a string.
1618     value: A users.User instance.
1619     pbvalue: The entity_pb.PropertyValue to pack this value into.
1620   """
1621   pbvalue.mutable_uservalue().set_email(value.email().encode('utf-8'))
1622   pbvalue.mutable_uservalue().set_auth_domain(
1623       value.auth_domain().encode('utf-8'))
1624   pbvalue.mutable_uservalue().set_gaiaid(0)
1625
1626
1627
1628
1629   if value.user_id() is not None:
1630     pbvalue.mutable_uservalue().set_obfuscated_gaiaid(
1631         value.user_id().encode('utf-8'))
1632
1633   if value.federated_identity() is not None:
1634     pbvalue.mutable_uservalue().set_federated_identity(
1635         value.federated_identity().encode('utf-8'))
1636
1637   if value.federated_provider() is not None:
1638     pbvalue.mutable_uservalue().set_federated_provider(
1639         value.federated_provider().encode('utf-8'))
1640
1641
1642 def PackKey(name, value, pbvalue):
1643   """Packs a reference property into a entity_pb.PropertyValue.
1644
1645   Args:
1646     name: The name of the property as a string.
1647     value: A Key instance.
1648     pbvalue: The entity_pb.PropertyValue to pack this value into.
1649   """
1650   ref = value._Key__reference
1651   pbvalue.mutable_referencevalue().set_app(ref.app())
1652   SetNamespace(pbvalue.mutable_referencevalue(), ref.name_space())
1653   for elem in ref.path().element_list():
1654     pbvalue.mutable_referencevalue().add_pathelement().CopyFrom(elem)
1655
1656
1657 def PackBool(name, value, pbvalue):
1658   """Packs a boolean property into a entity_pb.PropertyValue.
1659
1660   Args:
1661     name: The name of the property as a string.
1662     value: A boolean instance.
1663     pbvalue: The entity_pb.PropertyValue to pack this value into.
1664   """
1665   pbvalue.set_booleanvalue(value)
1666
1667
1668 def PackInteger(name, value, pbvalue):
1669   """Packs an integer property into a entity_pb.PropertyValue.
1670
1671   Args:
1672     name: The name of the property as a string.
1673     value: An int or long instance.
1674     pbvalue: The entity_pb.PropertyValue to pack this value into.
1675   """
1676   pbvalue.set_int64value(value)
1677
1678
1679 def PackFloat(name, value, pbvalue):
1680   """Packs a float property into a entity_pb.PropertyValue.
1681
1682   Args:
1683     name: The name of the property as a string.
1684     value: A float instance.
1685     pbvalue: The entity_pb.PropertyValue to pack this value into.
1686   """
1687   pbvalue.set_doublevalue(value)
1688
1689
1690
1691
1692
1693
1694
1695 _PACK_PROPERTY_VALUES = {
1696   Blob: PackBlob,
1697   EmbeddedEntity: PackBlob,
1698   ByteString: PackBlob,
1699   bool: PackBool,
1700   Category: PackString,
1701   datetime.datetime: PackDatetime,
1702   _OverflowDateTime: PackInteger,
1703   Email: PackString,
1704   float: PackFloat,
1705   GeoPt: PackGeoPt,
1706   IM: PackString,
1707   int: PackInteger,
1708   Key: PackKey,
1709   Link: PackString,
1710   long: PackInteger,
1711   PhoneNumber: PackString,
1712   PostalAddress: PackString,
1713   Rating: PackInteger,
1714   str: PackString,
1715   Text: PackString,
1716   type(None): lambda name, value, pbvalue: None,
1717   unicode: PackString,
1718   users.User: PackUser,
1719   BlobKey: PackString,
1720 }
1721
1722
1723 assert set(_PACK_PROPERTY_VALUES.iterkeys()) == _PROPERTY_TYPES
1724
1725
1726 def ToPropertyPb(name, values):
1727   """Creates type-specific entity_pb.PropertyValues.
1728
1729   Determines the type and meaning of the PropertyValue based on the Python
1730   type of the input value(s).
1731
1732   NOTE: This function does not validate anything!
1733
1734   Args:
1735     name: string or unicode; the property name
1736     values: The values for this property, either a single one or a list of them.
1737       All values must be a supported type. Lists of values must all be of the
1738       same type.
1739
1740   Returns:
1741     A list of entity_pb.Property instances.
1742   """
1743   encoded_name = name.encode('utf-8')
1744
1745   values_type = type(values)
1746   if values_type is list:
1747     multiple = True
1748   else:
1749     multiple = False
1750     values = [values]
1751
1752   pbs = []
1753   for v in values:
1754     pb = entity_pb.Property()
1755     pb.set_name(encoded_name)
1756     pb.set_multiple(multiple)
1757
1758     meaning = _PROPERTY_MEANINGS.get(v.__class__)
1759     if meaning is not None:
1760       pb.set_meaning(meaning)
1761
1762     pack_prop = _PACK_PROPERTY_VALUES[v.__class__]
1763     pbvalue = pack_prop(name, v, pb.mutable_value())
1764     pbs.append(pb)
1765
1766   if multiple:
1767     return pbs
1768   else:
1769     return pbs[0]
1770
1771
1772 def FromReferenceProperty(value):
1773   """Converts a reference PropertyValue to a Key.
1774
1775   Args:
1776     value: entity_pb.PropertyValue
1777
1778   Returns:
1779     Key
1780
1781   Raises:
1782     BadValueError if the value is not a PropertyValue.
1783   """
1784   assert isinstance(value, entity_pb.PropertyValue)
1785   assert value.has_referencevalue()
1786   ref = value.referencevalue()
1787
1788   key = Key()
1789   key_ref = key._Key__reference
1790   key_ref.set_app(ref.app())
1791   SetNamespace(key_ref, ref.name_space())
1792
1793   for pathelem in ref.pathelement_list():
1794     key_ref.mutable_path().add_element().CopyFrom(pathelem)
1795
1796   return key
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806 _PROPERTY_CONVERSIONS = {
1807   entity_pb.Property.GD_WHEN:           _When,
1808   entity_pb.Property.ATOM_CATEGORY:     Category,
1809   entity_pb.Property.ATOM_LINK:         Link,
1810   entity_pb.Property.GD_EMAIL:          Email,
1811   entity_pb.Property.GD_IM:             IM,
1812   entity_pb.Property.GD_PHONENUMBER:    PhoneNumber,
1813   entity_pb.Property.GD_POSTALADDRESS:  PostalAddress,
1814   entity_pb.Property.GD_RATING:         Rating,
1815   entity_pb.Property.BLOB:              Blob,
1816   entity_pb.Property.ENTITY_PROTO:      EmbeddedEntity,
1817   entity_pb.Property.BYTESTRING:        ByteString,
1818   entity_pb.Property.TEXT:              Text,
1819   entity_pb.Property.BLOBKEY:           BlobKey,
1820 }
1821
1822
1823 _NON_UTF8_MEANINGS = frozenset((entity_pb.Property.BLOB,
1824                                 entity_pb.Property.ENTITY_PROTO,
1825                                 entity_pb.Property.BYTESTRING,
1826                                 entity_pb.Property.INDEX_VALUE))
1827
1828
1829 def FromPropertyPb(pb):
1830   """Converts a property PB to a python value.
1831
1832   Args:
1833     pb: entity_pb.Property
1834
1835   Returns:
1836     # return type is determined by the type of the argument
1837     string, int, bool, double, users.User, or one of the atom or gd types
1838   """
1839
1840
1841
1842   pbval = pb.value()
1843   meaning = pb.meaning()
1844
1845   if pbval.has_stringvalue():
1846     value = pbval.stringvalue()
1847     if not pb.has_meaning() or meaning not in _NON_UTF8_MEANINGS:
1848       value = unicode(value, 'utf-8')
1849   elif pbval.has_int64value():
1850
1851
1852     value = long(pbval.int64value())
1853   elif pbval.has_booleanvalue():
1854
1855
1856     value = bool(pbval.booleanvalue())
1857   elif pbval.has_doublevalue():
1858     value = pbval.doublevalue()
1859   elif pbval.has_referencevalue():
1860     value = FromReferenceProperty(pbval)
1861   elif pbval.has_pointvalue():
1862     value = GeoPt(pbval.pointvalue().x(), pbval.pointvalue().y())
1863   elif pbval.has_uservalue():
1864     email = unicode(pbval.uservalue().email(), 'utf-8')
1865     auth_domain = unicode(pbval.uservalue().auth_domain(), 'utf-8')
1866     obfuscated_gaiaid = pbval.uservalue().obfuscated_gaiaid().decode('utf-8')
1867     obfuscated_gaiaid = unicode(pbval.uservalue().obfuscated_gaiaid(), 'utf-8')
1868
1869     federated_identity = None
1870     if pbval.uservalue().has_federated_identity():
1871       federated_identity = unicode(pbval.uservalue().federated_identity(),
1872                                    'utf-8')
1873
1874
1875
1876     value = users.User(email=email,
1877                        _auth_domain=auth_domain,
1878                        _user_id=obfuscated_gaiaid,
1879                        federated_identity=federated_identity,
1880                        _strict_mode=False)
1881   else:
1882     value = None
1883
1884   try:
1885     if pb.has_meaning() and meaning in _PROPERTY_CONVERSIONS:
1886       conversion = _PROPERTY_CONVERSIONS[meaning]
1887       value = conversion(value)
1888   except (KeyError, ValueError, IndexError, TypeError, AttributeError), msg:
1889     raise datastore_errors.BadValueError(
1890       'Error converting pb: %s\nException was: %s' % (pb, msg))
1891
1892   return value
1893
1894
1895 def RestoreFromIndexValue(index_value, data_type):
1896   """Restores a index value to the correct datastore type.
1897
1898   Projection queries return property values direclty from a datastore index.
1899   These values are the native datastore values, one of str, bool, long, float,
1900   GeoPt, Key or User. This function restores the original value when the the
1901   original type is known.
1902
1903   This function returns the value type returned when decoding a normal entity,
1904   not necessarily of type data_type. For example, data_type=int returns a
1905   long instance.
1906
1907   Args:
1908     index_value: The value returned by FromPropertyPb for the projected
1909       property.
1910     data_type: The type of the value originally given to ToPropertyPb
1911
1912   Returns:
1913     The restored property value.
1914
1915   Raises:
1916     datastore_errors.BadValueError if the value cannot be restored.
1917   """
1918   raw_type = _PROPERTY_TYPE_TO_INDEX_VALUE_TYPE.get(data_type)
1919   if raw_type is None:
1920     raise datastore_errors.BadValueError(
1921         'Unsupported data type (%r)' % data_type)
1922
1923   if index_value is None:
1924     return index_value
1925
1926
1927
1928   if not isinstance(index_value, raw_type):
1929     raise datastore_errors.BadValueError(
1930         'Unsupported converstion. Expected %r got %r' %
1931         (type(index_value), raw_type))
1932
1933   meaning = _PROPERTY_MEANINGS.get(data_type)
1934
1935
1936   if isinstance(index_value, str) and meaning not in _NON_UTF8_MEANINGS:
1937     index_value = unicode(index_value, 'utf-8')
1938
1939
1940   conv = _PROPERTY_CONVERSIONS.get(meaning)
1941   if not conv:
1942     return index_value
1943
1944   try:
1945     value = conv(index_value)
1946   except (KeyError, ValueError, IndexError, TypeError, AttributeError), msg:
1947     raise datastore_errors.BadValueError(
1948       'Error converting value: %r\nException was: %s' % (index_value, msg))
1949   return value
1950
1951
1952 def PropertyTypeName(value):
1953   """Returns the name of the type of the given property value, as a string.
1954
1955   Raises BadValueError if the value is not a valid property type.
1956
1957   Args:
1958     value: any valid property value
1959
1960   Returns:
1961     string
1962   """
1963   if value.__class__ in _PROPERTY_MEANINGS:
1964     meaning = _PROPERTY_MEANINGS[value.__class__]
1965     name = entity_pb.Property._Meaning_NAMES[meaning]
1966     return name.lower().replace('_', ':')
1967   elif isinstance(value, basestring):
1968     return 'string'
1969   elif isinstance(value, users.User):
1970     return 'user'
1971   elif isinstance(value, long):
1972     return 'int'
1973   elif value is None:
1974     return 'null'
1975   else:
1976     return typename(value).lower()
1977
1978
1979 _PROPERTY_TYPE_STRINGS = {
1980     'string':           unicode,
1981     'bool':             bool,
1982     'int':              long,
1983     'null':             type(None),
1984     'float':            float,
1985     'key':              Key,
1986     'blob':             Blob,
1987     'entity:proto':     EmbeddedEntity,
1988     'bytestring':       ByteString,
1989     'text':             Text,
1990     'user':             users.User,
1991     'atom:category':    Category,
1992     'atom:link':        Link,
1993     'gd:email':         Email,
1994     'gd:when':          datetime.datetime,
1995     'georss:point':     GeoPt,
1996     'gd:im':            IM,
1997     'gd:phonenumber':   PhoneNumber,
1998     'gd:postaladdress': PostalAddress,
1999     'gd:rating':        Rating,
2000     'blobkey':          BlobKey,
2001     }
2002
2003
2004 def FromPropertyTypeName(type_name):
2005   """Returns the python type given a type name.
2006
2007   Args:
2008     type_name: A string representation of a datastore type name.
2009
2010   Returns:
2011     A python type.
2012   """
2013   return _PROPERTY_TYPE_STRINGS[type_name]
2014
2015
2016 def PropertyValueFromString(type_,
2017                             value_string,
2018                             _auth_domain=None):
2019   """Returns an instance of a property value given a type and string value.
2020
2021   The reverse of this method is just str() and type() of the python value.
2022
2023   Note that this does *not* support non-UTC offsets in ISO 8601-formatted
2024   datetime strings, e.g. the -08:00 suffix in '2002-12-25 00:00:00-08:00'.
2025   It only supports -00:00 and +00:00 suffixes, which are UTC.
2026
2027   Args:
2028     type_: A python class.
2029     value_string: A string representation of the value of the property.
2030
2031   Returns:
2032     An instance of 'type'.
2033
2034   Raises:
2035     ValueError if type_ is datetime and value_string has a timezone offset.
2036   """
2037   if type_ == datetime.datetime:
2038     value_string = value_string.strip()
2039
2040     if value_string[-6] in ('+', '-'):
2041       if value_string[-5:] == '00:00':
2042         value_string = value_string[:-6]
2043       else:
2044
2045         raise ValueError('Non-UTC offsets in datetimes are not supported.')
2046
2047
2048     split = value_string.split('.')
2049     iso_date = split[0]
2050     microseconds = 0
2051     if len(split) > 1:
2052       microseconds = int(split[1])
2053
2054
2055
2056     time_struct = time.strptime(iso_date, '%Y-%m-%d %H:%M:%S')[0:6]
2057     value = datetime.datetime(*(time_struct + (microseconds,)))
2058     return value
2059   elif type_ == Rating:
2060
2061     return Rating(int(value_string))
2062   elif type_ == bool:
2063     return value_string == 'True'
2064   elif type_ == users.User:
2065     return users.User(value_string, _auth_domain)
2066   elif type_ == type(None):
2067     return None
2068   return type_(value_string)
2069
2070
2071 def ReferenceToKeyValue(key):
2072   """Converts a key into a comparable hashable "key" value.
2073
2074   Args:
2075     key: The entity_pb.Reference or entity_v4_pb.Key from which to construct
2076         the key value.
2077
2078   Returns:
2079     A comparable and hashable representation of the given key that is
2080     compatible with one derived from a key property value.
2081   """
2082   if isinstance(key, entity_v4_pb.Key):
2083     v4_key = key
2084     key = entity_pb.Reference()
2085     datastore_pbs.get_entity_converter().v4_to_v3_reference(v4_key, key)
2086
2087   if isinstance(key, entity_pb.Reference):
2088     element_list = key.path().element_list()
2089   elif isinstance(key, entity_pb.PropertyValue_ReferenceValue):
2090     element_list = key.pathelement_list()
2091   else:
2092     raise datastore_errors.BadArgumentError(
2093         "key arg expected to be entity_pb.Reference or entity_v4.Key (%r)"
2094         % (key,))
2095
2096   result = [entity_pb.PropertyValue.kReferenceValueGroup,
2097             key.app(), key.name_space()]
2098   for element in element_list:
2099     result.append(element.type())
2100     if element.has_name():
2101       result.append(element.name())
2102     else:
2103       result.append(element.id())
2104   return tuple(result)
2105
2106
2107 def PropertyValueToKeyValue(prop_value):
2108   """Converts a entity_pb.PropertyValue into a comparable hashable "key" value.
2109
2110   The values produces by this function mimic the native ording of the datastore
2111   and uniquely identify the given PropertyValue.
2112
2113   Args:
2114     prop_value: The entity_pb.PropertyValue from which to construct the
2115       key value.
2116
2117   Returns:
2118     A comparable and hashable representation of the given property value.
2119   """
2120   if not isinstance(prop_value, entity_pb.PropertyValue):
2121     raise datastore_errors.BadArgumentError(
2122         'prop_value arg expected to be entity_pb.PropertyValue (%r)' %
2123         (prop_value,))
2124
2125
2126
2127   if prop_value.has_stringvalue():
2128     return (entity_pb.PropertyValue.kstringValue, prop_value.stringvalue())
2129   if prop_value.has_int64value():
2130     return (entity_pb.PropertyValue.kint64Value, prop_value.int64value())
2131   if prop_value.has_booleanvalue():
2132     return (entity_pb.PropertyValue.kbooleanValue, prop_value.booleanvalue())
2133   if prop_value.has_doublevalue():
2134
2135     encoder = sortable_pb_encoder.Encoder()
2136     encoder.putDouble(prop_value.doublevalue())
2137     return (entity_pb.PropertyValue.kdoubleValue, tuple(encoder.buf))
2138   if prop_value.has_pointvalue():
2139     return (entity_pb.PropertyValue.kPointValueGroup,
2140             prop_value.pointvalue().x(), prop_value.pointvalue().y())
2141   if prop_value.has_referencevalue():
2142     return ReferenceToKeyValue(prop_value.referencevalue())
2143   if prop_value.has_uservalue():
2144     result = []
2145     uservalue = prop_value.uservalue()
2146     if uservalue.has_email():
2147       result.append((entity_pb.PropertyValue.kUserValueemail,
2148                      uservalue.email()))
2149     if uservalue.has_auth_domain():
2150       result.append((entity_pb.PropertyValue.kUserValueauth_domain,
2151                      uservalue.auth_domain()))
2152     if uservalue.has_nickname():
2153       result.append((entity_pb.PropertyValue.kUserValuenickname,
2154                      uservalue.nickname()))
2155     if uservalue.has_gaiaid():
2156       result.append((entity_pb.PropertyValue.kUserValuegaiaid,
2157                      uservalue.gaiaid()))
2158     if uservalue.has_obfuscated_gaiaid():
2159       result.append((entity_pb.PropertyValue.kUserValueobfuscated_gaiaid,
2160                      uservalue.obfuscated_gaiaid()))
2161     if uservalue.has_federated_identity():
2162       result.append((entity_pb.PropertyValue.kUserValuefederated_identity,
2163                      uservalue.federated_identity()))
2164     if uservalue.has_federated_provider():
2165       result.append((entity_pb.PropertyValue.kUserValuefederated_provider,
2166                      uservalue.federated_provider()))
2167     result.sort()
2168     return (entity_pb.PropertyValue.kUserValueGroup, tuple(result))
2169   return ()
2170
2171
2172 def GetPropertyValueTag(value_pb):
2173   """Returns the tag constant associated with the given entity_pb.PropertyValue.
2174   """
2175   if value_pb.has_booleanvalue():
2176     return entity_pb.PropertyValue.kbooleanValue
2177   elif value_pb.has_doublevalue():
2178     return entity_pb.PropertyValue.kdoubleValue
2179   elif value_pb.has_int64value():
2180     return entity_pb.PropertyValue.kint64Value
2181   elif value_pb.has_pointvalue():
2182     return entity_pb.PropertyValue.kPointValueGroup
2183   elif value_pb.has_referencevalue():
2184     return entity_pb.PropertyValue.kReferenceValueGroup
2185   elif value_pb.has_stringvalue():
2186     return entity_pb.PropertyValue.kstringValue
2187   elif value_pb.has_uservalue():
2188     return entity_pb.PropertyValue.kUserValueGroup
2189   else:
2190     return 0