3 # Copyright 2007 Google Inc.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
18 """Higher-level, semantic data types for the datastore. These types
19 are expected to be set as attributes of Entities. See "Supported Data Types"
22 Most of these types are based on XML elements from Atom and GData elements
23 from the atom and gd namespaces. For more information, see:
25 http://www.atomenabled.org/developers/syndication/
26 http://code.google.com/apis/gdata/common-elements.html
28 The namespace schemas are:
30 http://www.w3.org/2005/Atom
31 http://schemas.google.com/g/2005
46 from xml
.sax
import saxutils
47 from google
.appengine
.datastore
import datastore_pb
48 from google
.appengine
.api
import datastore_errors
49 from google
.appengine
.api
import users
50 from google
.net
.proto
import ProtocolBuffer
51 from google
.appengine
.datastore
import entity_pb
53 _MAX_STRING_LENGTH
= 500
55 _MAX_LINK_PROPERTY_LENGTH
= 2083
57 RESERVED_PROPERTY_NAME
= re
.compile('^__.*__$')
59 _KEY_SPECIAL_PROPERTY
= '__key__'
60 _SPECIAL_PROPERTIES
= frozenset([_KEY_SPECIAL_PROPERTY
])
62 class UtcTzinfo(datetime
.tzinfo
):
63 def utcoffset(self
, dt
): return datetime
.timedelta(0)
64 def dst(self
, dt
): return datetime
.timedelta(0)
65 def tzname(self
, dt
): return 'UTC'
66 def __repr__(self
): return 'datastore_types.UTC'
72 """Returns the type of obj as a string. More descriptive and specific than
73 type(obj), and safe for any object, unlike __class__."""
74 if hasattr(obj
, '__class__'):
75 return getattr(obj
, '__class__').__name
__
77 return type(obj
).__name
__
80 def ValidateString(value
,
82 exception
=datastore_errors
.BadValueError
,
83 max_len
=_MAX_STRING_LENGTH
):
84 """Raises an exception if value is not a valid string or a subclass thereof.
86 A string is valid if it's not empty, no more than _MAX_STRING_LENGTH bytes,
87 and not a Blob. The exception type can be specified with the exception
88 argument; it defaults to BadValueError.
91 value: the value to validate.
92 name: the name of this value; used in the exception message.
93 exception: the type of exception to raise.
94 max_len: the maximum allowed length, in bytes
96 if not isinstance(value
, basestring
) or isinstance(value
, Blob
):
97 raise exception('%s should be a string; received %s (a %s):' %
98 (name
, value
, typename(value
)))
100 raise exception('%s must not be empty.' % name
)
102 if len(value
.encode('utf-8')) > max_len
:
103 raise exception('%s must be under %d bytes.' % (name
, max_len
))
106 def ResolveAppId(app
, name
='_app'):
107 """Validate app id, providing a default.
109 If the argument is None, $APPLICATION_ID is substituted.
112 app: The app id argument value to be validated.
113 name: The argument name, for error messages.
116 The value of app, or the substituted default. Always a non-empty string.
119 BadArgumentError if the value is empty or not a string.
122 app
= os
.environ
.get('APPLICATION_ID', '')
123 ValidateString(app
, '_app', datastore_errors
.BadArgumentError
)
128 """The primary key for a datastore entity.
130 A datastore GUID. A Key instance uniquely identifies an entity across all
131 apps, and includes all information necessary to fetch the entity from the
132 datastore with Get().
134 Key implements __hash__, and key instances are immutable, so Keys may be
135 used in sets and as dictionary keys.
139 def __init__(self
, encoded
=None):
140 """Constructor. Creates a Key from a string.
143 # a base64-encoded primary key, generated by Key.__str__
146 if encoded
is not None:
147 if not isinstance(encoded
, basestring
):
149 repr_encoded
= repr(encoded
)
151 repr_encoded
= "<couldn't encode>"
152 raise datastore_errors
.BadArgumentError(
153 'Key() expects a string; received %s (a %s).' %
154 (repr_encoded
, typename(encoded
)))
156 modulo
= len(encoded
) % 4
158 encoded
+= ('=' * (4 - modulo
))
160 encoded_pb
= base64
.urlsafe_b64decode(str(encoded
))
161 self
.__reference
= entity_pb
.Reference(encoded_pb
)
162 assert self
.__reference
.IsInitialized()
164 except (AssertionError, TypeError), e
:
165 raise datastore_errors
.BadKeyError(
166 'Invalid string key %s. Details: %s' % (encoded
, e
))
168 if e
.__class
__.__name
__ == 'ProtocolBufferDecodeError':
169 raise datastore_errors
.BadKeyError('Invalid string key %s.' % encoded
)
173 self
.__reference
= entity_pb
.Reference()
176 def from_path(*args
, **kwds
):
177 """Static method to construct a Key out of a "path" (kind, id or name, ...).
179 This is useful when an application wants to use just the id or name portion
180 of a key in e.g. a URL, where the rest of the URL provides enough context to
181 fill in the rest, i.e. the app id (always implicit), the entity kind, and
182 possibly an ancestor key. Since ids and names are usually small, they're
183 more attractive for use in end-user-visible URLs than the full string
184 representation of a key.
187 kind: the entity kind (a str or unicode instance)
188 id_or_name: the id (an int or long) or name (a str or unicode instance)
190 Additional positional arguments are allowed and should be
191 alternating kind and id/name.
194 parent: optional parent Key; default None.
197 A new Key instance whose .kind() and .id() or .name() methods return
198 the *last* kind and id or name positional arguments passed.
201 BadArgumentError for invalid arguments.
202 BadKeyError if the parent key is incomplete.
204 parent
= kwds
.pop('parent', None)
205 _app
= ResolveAppId(kwds
.pop('_app', None))
208 raise datastore_errors
.BadArgumentError(
209 'Excess keyword arguments ' + repr(kwds
))
211 if not args
or len(args
) % 2:
212 raise datastore_errors
.BadArgumentError(
213 'A non-zero even number of positional arguments is required '
214 '(kind, id or name, kind, id or name, ...); received %s' % repr(args
))
216 if parent
is not None:
217 if not isinstance(parent
, Key
):
218 raise datastore_errors
.BadArgumentError(
219 'Expected None or a Key as parent; received %r (a %s).' %
220 (parent
, typename(parent
)))
221 if not parent
.has_id_or_name():
222 raise datastore_errors
.BadKeyError(
223 'The parent Key is incomplete.')
224 if _app
!= parent
.app():
225 raise datastore_errors
.BadArgumentError(
226 'The _app argument (%r) should match parent.app() (%s)' %
227 (_app
, parent
.app()))
230 ref
= key
.__reference
231 if parent
is not None:
232 ref
.CopyFrom(parent
.__reference
)
236 path
= ref
.mutable_path()
237 for i
in xrange(0, len(args
), 2):
238 kind
, id_or_name
= args
[i
:i
+2]
239 if isinstance(kind
, basestring
):
240 kind
= kind
.encode('utf-8')
242 raise datastore_errors
.BadArgumentError(
243 'Expected a string kind as argument %d; received %r (a %s).' %
244 (i
+ 1, kind
, typename(kind
)))
245 elem
= path
.add_element()
247 if isinstance(id_or_name
, (int, long)):
248 elem
.set_id(id_or_name
)
249 elif isinstance(id_or_name
, basestring
):
250 ValidateString(id_or_name
, 'name')
251 if id_or_name
and id_or_name
[0] in string
.digits
:
252 raise datastore_errors
.BadArgumentError(
253 'Names may not begin with a digit; received %s.' % id_or_name
)
254 elem
.set_name(id_or_name
.encode('utf-8'))
256 raise datastore_errors
.BadArgumentError(
257 'Expected an integer id or string name as argument %d; '
258 'received %r (a %s).' % (i
+ 2, id_or_name
, typename(id_or_name
)))
260 assert ref
.IsInitialized()
264 """Returns this entity's app id, a string."""
265 if self
.__reference
.app():
266 return self
.__reference
.app().decode('utf-8')
271 """Returns this entity's kind, as a string."""
272 if self
.__reference
.path().element_size() > 0:
273 encoded
= self
.__reference
.path().element_list()[-1].type()
274 return unicode(encoded
.decode('utf-8'))
279 """Returns this entity's id, or None if it doesn't have one."""
280 elems
= self
.__reference
.path().element_list()
281 if elems
and elems
[-1].has_id() and elems
[-1].id():
282 return elems
[-1].id()
287 """Returns this entity's name, or None if it doesn't have one."""
288 elems
= self
.__reference
.path().element_list()
289 if elems
and elems
[-1].has_name() and elems
[-1].name():
290 return elems
[-1].name().decode('utf-8')
294 def id_or_name(self
):
295 """Returns this entity's id or name, whichever it has, or None."""
296 if self
.id() is not None:
301 def has_id_or_name(self
):
302 """Returns True if this entity has an id or name, False otherwise.
304 return self
.id_or_name() is not None
307 """Returns this entity's parent, as a Key. If this entity has no parent,
309 if self
.__reference
.path().element_size() > 1:
311 parent
.__reference
.CopyFrom(self
.__reference
)
312 parent
.__reference
.path().element_list().pop()
318 """Returns a tag: URI for this entity for use in XML output.
320 Foreign keys for entities may be represented in XML output as tag URIs.
321 RFC 4151 describes the tag URI scheme. From http://taguri.org/:
323 The tag algorithm lets people mint - create - identifiers that no one
324 else using the same algorithm could ever mint. It is simple enough to do
325 in your head, and the resulting identifiers can be easy to read, write,
326 and remember. The identifiers conform to the URI (URL) Syntax.
328 Tag URIs for entities use the app's auth domain and the date that the URI
329 is generated. The namespace-specific part is <kind>[<key>].
331 For example, here is the tag URI for a Kitten with the key "Fluffy" in the
334 tag:catsinsinks.googleapps.com,2006-08-29:Kitten[Fluffy]
336 Raises a BadKeyError if this entity's key is incomplete.
338 if not self
.has_id_or_name():
339 raise datastore_errors
.BadKeyError(
340 'ToTagUri() called for an entity with an incomplete key.')
342 return u
'tag:%s.%s,%s:%s[%s]' % (saxutils
.escape(self
.app()),
343 os
.environ
['AUTH_DOMAIN'],
344 datetime
.date
.today().isoformat(),
345 saxutils
.escape(self
.kind()),
346 saxutils
.escape(str(self
)))
349 def entity_group(self
):
350 """Returns this key's entity group as a Key.
352 Note that the returned Key will be incomplete if this Key is for a root
353 entity and it is incomplete.
355 group
= Key
._FromPb
(self
.__reference
)
356 del group
.__reference
.path().element_list()[1:]
361 """Static factory method. Creates a Key from an entity_pb.Reference.
363 Not intended to be used by application developers. Enforced by hiding the
367 pb: entity_pb.Reference
369 if not isinstance(pb
, entity_pb
.Reference
):
370 raise datastore_errors
.BadArgumentError(
371 'Key constructor takes an entity_pb.Reference; received %s (a %s).' %
375 key
.__reference
= entity_pb
.Reference()
376 key
.__reference
.CopyFrom(pb
)
380 """Converts this Key to its protocol buffer representation.
382 Not intended to be used by application developers. Enforced by hiding the
386 # the Reference PB representation of this Key
389 pb
= entity_pb
.Reference()
390 pb
.CopyFrom(self
.__reference
)
391 if not self
.has_id_or_name():
392 pb
.mutable_path().element_list()[-1].set_id(0)
394 pb
.app().decode('utf-8')
395 for pathelem
in pb
.path().element_list():
396 pathelem
.type().decode('utf-8')
401 """Encodes this Key as an opaque string.
403 Returns a string representation of this key, suitable for use in HTML,
404 URLs, and other similar use cases. If the entity's key is incomplete,
405 raises a BadKeyError.
407 Unfortunately, this string encoding isn't particularly compact, and its
408 length varies with the length of the path. If you want a shorter identifier
409 and you know the kind and parent (if any) ahead of time, consider using just
410 the entity's id or name.
415 if (self
.has_id_or_name()):
416 encoded
= base64
.urlsafe_b64encode(self
.__reference
.Encode())
417 return encoded
.replace('=', '')
419 raise datastore_errors
.BadKeyError(
420 'Cannot string encode an incomplete key!\n%s' % self
.__reference
)
423 """Returns an eval()able string representation of this key.
425 Returns a Python string of the form 'datastore_types.Key.from_path(...)'
426 that can be used to recreate this key.
432 for elem
in self
.__reference
.path().element_list():
433 args
.append(repr(elem
.type()))
435 args
.append(repr(elem
.name().decode('utf-8')))
437 args
.append(repr(elem
.id()))
439 args
.append('_app=%r' % self
.__reference
.app().decode('utf-8'))
440 return u
'datastore_types.Key.from_path(%s)' % ', '.join(args
)
442 def __cmp__(self
, other
):
443 """Returns negative, zero, or positive when comparing two keys.
445 TODO(ryanb): for API v2, we should change this to make incomplete keys, ie
446 keys without an id or name, not equal to any other keys.
449 other: Key to compare to.
452 Negative if self is less than "other"
453 Zero if "other" is equal to self
454 Positive if self is greater than "other"
456 if not isinstance(other
, Key
):
462 self_args
.append(self
.__reference
.app().decode('utf-8'))
463 other_args
.append(other
.__reference
.app().decode('utf-8'))
465 for elem
in self
.__reference
.path().element_list():
466 self_args
.append(repr(elem
.type()))
468 self_args
.append(repr(elem
.name().decode('utf-8')))
470 self_args
.append(elem
.id())
472 for elem
in other
.__reference
.path().element_list():
473 other_args
.append(repr(elem
.type()))
475 other_args
.append(repr(elem
.name().decode('utf-8')))
477 other_args
.append(elem
.id())
479 result
= cmp(self_args
, other_args
)
483 """Returns a 32-bit integer hash of this key.
485 Implements Python's hash protocol so that Keys may be used in sets and as
491 return hash(self
.__str
__())
494 class Category(unicode):
495 """A tag, ie a descriptive word or phrase. Entities may be tagged by users,
496 and later returned by a queries for that tag. Tags can also be used for
497 ranking results (frequency), photo captions, clustering, activity, etc.
499 Here's a more in-depth description: http://www.zeldman.com/daily/0405d.shtml
501 This is the Atom "category" element. In XML output, the tag is provided as
502 the term attribute. See:
503 http://www.atomenabled.org/developers/syndication/#category
505 Raises BadValueError if tag is not a string or subtype.
509 def __init__(self
, tag
):
510 super(Category
, self
).__init
__(self
, tag
)
511 ValidateString(tag
, 'tag')
514 return u
'<category term="%s" label=%s />' % (Category
.TERM
,
515 saxutils
.quoteattr(self
))
519 """A fully qualified URL. Usually http: scheme, but may also be file:, ftp:,
522 If you have email (mailto:) or instant messaging (aim:, xmpp:) links,
523 consider using the Email or IM classes instead.
525 This is the Atom "link" element. In XML output, the link is provided as the
527 http://www.atomenabled.org/developers/syndication/#link
529 Raises BadValueError if link is not a fully qualified, well-formed URL.
531 def __init__(self
, link
):
532 super(Link
, self
).__init
__(self
, link
)
533 ValidateString(link
, 'link', max_len
=_MAX_LINK_PROPERTY_LENGTH
)
535 scheme
, domain
, path
, params
, query
, fragment
= urlparse
.urlparse(link
)
536 if (not scheme
or (scheme
!= 'file' and not domain
) or
537 (scheme
== 'file' and not path
)):
538 raise datastore_errors
.BadValueError('Invalid URL: %s' % link
)
541 return u
'<link href=%s />' % saxutils
.quoteattr(self
)
544 class Email(unicode):
545 """An RFC2822 email address. Makes no attempt at validation; apart from
546 checking MX records, email address validation is a rathole.
548 This is the gd:email element. In XML output, the email address is provided as
549 the address attribute. See:
550 http://code.google.com/apis/gdata/common-elements.html#gdEmail
552 Raises BadValueError if email is not a valid email address.
554 def __init__(self
, email
):
555 super(Email
, self
).__init
__(self
, email
)
556 ValidateString(email
, 'email')
559 return u
'<gd:email address=%s />' % saxutils
.quoteattr(self
)
563 """A geographical point, specified by floating-point latitude and longitude
564 coordinates. Often used to integrate with mapping sites like Google Maps.
565 May also be used as ICBM coordinates.
567 This is the georss:point element. In XML output, the coordinates are
568 provided as the lat and lon attributes. See: http://georss.org/
570 Serializes to '<lat>,<lon>'. Raises BadValueError if it's passed an invalid
571 serialized string, or if lat and lon are not valid floating points in the
572 ranges [-90, 90] and [-180, 180], respectively.
577 def __init__(self
, lat
, lon
=None):
580 split
= lat
.split(',')
582 except (AttributeError, ValueError):
583 raise datastore_errors
.BadValueError(
584 'Expected a "lat,long" formatted string; received %s (a %s).' %
585 (lat
, typename(lat
)))
591 raise datastore_errors
.BadValueError(
592 'Latitude must be between -90 and 90; received %f' % lat
)
594 raise datastore_errors
.BadValueError(
595 'Longitude must be between -180 and 180; received %f' % lon
)
596 except (TypeError, ValueError):
597 raise datastore_errors
.BadValueError(
598 'Expected floats for lat and long; received %s (a %s) and %s (a %s).' %
599 (lat
, typename(lat
), lon
, typename(lon
)))
604 def __cmp__(self
, other
):
605 if not isinstance(other
, GeoPt
):
608 except datastore_errors
.BadValueError
:
609 return NotImplemented
611 lat_cmp
= cmp(self
.lat
, other
.lat
)
615 return cmp(self
.lon
, other
.lon
)
618 """Returns a 32-bit integer hash of this point.
620 Implements Python's hash protocol so that GeoPts may be used in sets and
626 return hash((self
.lat
, self
.lon
))
629 """Returns an eval()able string representation of this GeoPt.
631 The returned string is of the form 'datastore_types.GeoPt([lat], [lon])'.
636 return 'datastore_types.GeoPt(%r, %r)' % (self
.lat
, self
.lon
)
638 def __unicode__(self
):
639 return u
'%s,%s' % (unicode(self
.lat
), unicode(self
.lon
))
641 __str__
= __unicode__
644 return u
'<georss:point>%s %s</georss:point>' % (unicode(self
.lat
),
649 """An instant messaging handle. Includes both an address and its protocol.
650 The protocol value is either a standard IM scheme or a URL identifying the
651 IM network for the protocol. Possible values include:
655 unknown Unknown or unspecified
659 http://talk.google.com/ Google Talk
660 http://messenger.msn.com/ MSN Messenger
661 http://messenger.yahoo.com/ Yahoo Messenger
662 http://sametime.com/ Lotus Sametime
663 http://gadu-gadu.pl/ Gadu-Gadu
665 This is the gd:im element. In XML output, the address and protocol are
666 provided as the address and protocol attributes, respectively. See:
667 http://code.google.com/apis/gdata/common-elements.html#gdIm
669 Serializes to '<protocol> <address>'. Raises BadValueError if tag is not a
670 standard IM scheme or a URL.
672 PROTOCOLS
= [ 'sip', 'unknown', 'xmpp' ]
677 def __init__(self
, protocol
, address
=None):
680 split
= protocol
.split(' ')
681 protocol
, address
= split
682 except (AttributeError, ValueError):
683 raise datastore_errors
.BadValueError(
684 'Expected string of format "protocol address"; received %s' %
687 ValidateString(address
, 'address')
688 if protocol
not in self
.PROTOCOLS
:
691 self
.address
= address
692 self
.protocol
= protocol
694 def __cmp__(self
, other
):
695 if not isinstance(other
, IM
):
698 except datastore_errors
.BadValueError
:
699 return NotImplemented
701 return cmp((self
.address
, self
.protocol
),
702 (other
.address
, other
.protocol
))
705 """Returns an eval()able string representation of this IM.
707 The returned string is of the form:
709 datastore_types.IM('address', 'protocol')
714 return 'datastore_types.IM(%r, %r)' % (self
.protocol
, self
.address
)
716 def __unicode__(self
):
717 return u
'%s %s' % (self
.protocol
, self
.address
)
719 __str__
= __unicode__
722 return (u
'<gd:im protocol=%s address=%s />' %
723 (saxutils
.quoteattr(self
.protocol
),
724 saxutils
.quoteattr(self
.address
)))
727 return len(unicode(self
))
730 class PhoneNumber(unicode):
731 """A human-readable phone number or address.
733 No validation is performed. Phone numbers have many different formats -
734 local, long distance, domestic, international, internal extension, TTY,
735 VOIP, SMS, and alternative networks like Skype, XFire and Roger Wilco. They
736 all have their own numbering and addressing formats.
738 This is the gd:phoneNumber element. In XML output, the phone number is
739 provided as the text of the element. See:
740 http://code.google.com/apis/gdata/common-elements.html#gdPhoneNumber
742 Raises BadValueError if phone is not a string or subtype.
744 def __init__(self
, phone
):
745 super(PhoneNumber
, self
).__init
__(self
, phone
)
746 ValidateString(phone
, 'phone')
749 return u
'<gd:phoneNumber>%s</gd:phoneNumber>' % saxutils
.escape(self
)
752 class PostalAddress(unicode):
753 """A human-readable mailing address. Again, mailing address formats vary
754 widely, so no validation is performed.
756 This is the gd:postalAddress element. In XML output, the address is provided
757 as the text of the element. See:
758 http://code.google.com/apis/gdata/common-elements.html#gdPostalAddress
760 Raises BadValueError if address is not a string or subtype.
762 def __init__(self
, address
):
763 super(PostalAddress
, self
).__init
__(self
, address
)
764 ValidateString(address
, 'address')
767 return u
'<gd:postalAddress>%s</gd:postalAddress>' % saxutils
.escape(self
)
771 """A user-provided integer rating for a piece of content. Normalized to a
774 This is the gd:rating element. In XML output, the address is provided
775 as the text of the element. See:
776 http://code.google.com/apis/gdata/common-elements.html#gdRating
778 Serializes to the decimal string representation of the rating. Raises
779 BadValueError if the rating is not an integer in the range [0, 100].
784 def __init__(self
, rating
):
785 super(Rating
, self
).__init
__(self
, rating
)
786 if isinstance(rating
, float) or isinstance(rating
, complex):
787 raise datastore_errors
.BadValueError(
788 'Expected int or long; received %s (a %s).' %
789 (rating
, typename(rating
)))
792 if long(rating
) < Rating
.MIN
or long(rating
) > Rating
.MAX
:
793 raise datastore_errors
.BadValueError()
795 raise datastore_errors
.BadValueError(
796 'Expected int or long; received %s (a %s).' %
797 (rating
, typename(rating
)))
800 return (u
'<gd:rating value="%d" min="%d" max="%d" />' %
801 (self
, Rating
.MIN
, Rating
.MAX
))
805 """A long string type.
807 Strings of any length can be stored in the datastore using this
808 type. It behaves identically to the Python unicode type, except for
809 the constructor, which only accepts str and unicode arguments.
812 def __new__(cls
, arg
=None, encoding
=None):
815 We only accept unicode and str instances, the latter with encoding.
818 arg: optional unicode or str instance; default u''
819 encoding: optional encoding; disallowed when isinstance(arg, unicode),
820 defaults to 'ascii' when isinstance(arg, str);
824 if isinstance(arg
, unicode):
825 if encoding
is not None:
826 raise TypeError('Text() with a unicode argument '
827 'should not specify an encoding')
828 return super(Text
, cls
).__new
__(cls
, arg
)
830 if isinstance(arg
, str):
833 return super(Text
, cls
).__new
__(cls
, arg
, encoding
)
835 raise TypeError('Text() argument should be str or unicode, not %s' %
839 """A blob type, appropriate for storing binary data of any length.
841 This behaves identically to the Python str type, except for the
842 constructor, which only accepts str arguments.
845 def __new__(cls
, arg
=None):
848 We only accept str instances.
851 arg: optional str instance (default '')
855 if isinstance(arg
, str):
856 return super(Blob
, cls
).__new
__(cls
, arg
)
858 raise TypeError('Blob() argument should be str instance, not %s' %
862 """Output a blob as XML.
865 Base64 encoded version of itself for safe insertion in to an XML document.
867 encoded
= base64
.urlsafe_b64encode(self
)
868 return saxutils
.escape(encoded
)
870 class ByteString(str):
871 """A byte-string type, appropriate for storing short amounts of indexed data.
873 This behaves identically to Blob, except it's used only for short, indexed
877 def __new__(cls
, arg
=None):
880 We only accept str instances.
883 arg: optional str instance (default '')
887 if isinstance(arg
, str):
888 return super(ByteString
, cls
).__new
__(cls
, arg
)
890 raise TypeError('ByteString() argument should be str instance, not %s' %
894 """Output a ByteString as XML.
897 Base64 encoded version of itself for safe insertion in to an XML document.
899 encoded
= base64
.urlsafe_b64encode(self
)
900 return saxutils
.escape(encoded
)
903 _PROPERTY_MEANINGS
= {
907 Blob
: entity_pb
.Property
.BLOB
,
908 ByteString
: entity_pb
.Property
.BYTESTRING
,
909 Text
: entity_pb
.Property
.TEXT
,
910 datetime
.datetime
: entity_pb
.Property
.GD_WHEN
,
911 Category
: entity_pb
.Property
.ATOM_CATEGORY
,
912 Link
: entity_pb
.Property
.ATOM_LINK
,
913 Email
: entity_pb
.Property
.GD_EMAIL
,
914 GeoPt
: entity_pb
.Property
.GEORSS_POINT
,
915 IM
: entity_pb
.Property
.GD_IM
,
916 PhoneNumber
: entity_pb
.Property
.GD_PHONENUMBER
,
917 PostalAddress
: entity_pb
.Property
.GD_POSTALADDRESS
,
918 Rating
: entity_pb
.Property
.GD_RATING
,
921 _PROPERTY_TYPES
= frozenset([
945 _RAW_PROPERTY_TYPES
= (Blob
, Text
)
947 def ValidatePropertyInteger(name
, value
):
948 """Raises an exception if the supplied integer is invalid.
951 name: Name of the property this is for.
952 value: Integer value.
955 OverflowError if the value does not fit within a signed int64.
957 if not (-0x8000000000000000 <= value
<= 0x7fffffffffffffff):
958 raise OverflowError('%d is out of bounds for int64' % value
)
961 def ValidateStringLength(name
, value
, max_len
):
962 """Raises an exception if the supplied string is too long.
965 name: Name of the property this is for.
967 max_len: Maximum length the string may be.
970 OverflowError if the value is larger than the maximum length.
972 if len(value
) > max_len
:
973 raise datastore_errors
.BadValueError(
974 'Property %s is %d bytes long; it must be %d or less. '
975 'Consider Text instead, which can store strings of any length.' %
976 (name
, len(value
), max_len
))
979 def ValidatePropertyString(name
, value
):
980 """Validates the length of an indexed string property.
983 name: Name of the property this is for.
986 ValidateStringLength(name
, value
, max_len
=_MAX_STRING_LENGTH
)
989 def ValidatePropertyLink(name
, value
):
990 """Validates the length of an indexed Link property.
993 name: Name of the property this is for.
996 ValidateStringLength(name
, value
, max_len
=_MAX_LINK_PROPERTY_LENGTH
)
999 def ValidatePropertyNothing(name
, value
):
1000 """No-op validation function.
1003 name: Name of the property this is for.
1009 def ValidatePropertyKey(name
, value
):
1010 """Raises an exception if the supplied datastore.Key instance is invalid.
1013 name: Name of the property this is for.
1014 value: A datastore.Key instance.
1017 datastore_errors.BadValueError if the value is invalid.
1019 if not value
.has_id_or_name():
1020 raise datastore_errors
.BadValueError(
1021 'Incomplete key found for reference property %s.' % name
)
1024 _VALIDATE_PROPERTY_VALUES
= {
1025 Blob
: ValidatePropertyNothing
,
1026 ByteString
: ValidatePropertyString
,
1027 bool: ValidatePropertyNothing
,
1028 Category
: ValidatePropertyString
,
1029 datetime
.datetime
: ValidatePropertyNothing
,
1030 Email
: ValidatePropertyString
,
1031 float: ValidatePropertyNothing
,
1032 GeoPt
: ValidatePropertyNothing
,
1033 IM
: ValidatePropertyString
,
1034 int: ValidatePropertyInteger
,
1035 Key
: ValidatePropertyKey
,
1036 Link
: ValidatePropertyLink
,
1037 long: ValidatePropertyInteger
,
1038 PhoneNumber
: ValidatePropertyString
,
1039 PostalAddress
: ValidatePropertyString
,
1040 Rating
: ValidatePropertyInteger
,
1041 str: ValidatePropertyString
,
1042 Text
: ValidatePropertyNothing
,
1043 type(None): ValidatePropertyNothing
,
1044 unicode: ValidatePropertyString
,
1045 users
.User
: ValidatePropertyNothing
,
1048 assert set(_VALIDATE_PROPERTY_VALUES
.iterkeys()) == _PROPERTY_TYPES
1051 def ValidateProperty(name
, values
, read_only
=False):
1052 """Helper function for validating property values.
1055 name: Name of the property this is for.
1056 value: Value for the property as a Python native type.
1059 BadPropertyError if the property name is invalid. BadValueError if the
1060 property did not validate correctly or the value was an empty list. Other
1061 exception types (like OverflowError) if the property value does not meet
1062 type-specific criteria.
1064 ValidateString(name
, 'property name', datastore_errors
.BadPropertyError
)
1066 if not read_only
and RESERVED_PROPERTY_NAME
.match(name
):
1067 raise datastore_errors
.BadPropertyError(
1068 '%s is a reserved property name.' % name
)
1070 values_type
= type(values
)
1072 if values_type
is tuple:
1073 raise datastore_errors
.BadValueError(
1074 'May not use tuple property value; property %s is %s.' %
1075 (name
, repr(values
)))
1077 if values_type
is list:
1084 raise datastore_errors
.BadValueError(
1085 'May not use the empty list as a property value; property %s is %s.' %
1086 (name
, repr(values
)))
1090 prop_validator
= _VALIDATE_PROPERTY_VALUES
.get(v
.__class
__)
1091 if prop_validator
is None:
1092 raise datastore_errors
.BadValueError(
1093 'Unsupported type for property %s: %s' % (name
, v
.__class
__))
1094 prop_validator(name
, v
)
1096 except (KeyError, ValueError, TypeError, IndexError, AttributeError), msg
:
1097 raise datastore_errors
.BadValueError(
1098 'Error type checking values for property %s: %s' % (name
, msg
))
1101 ValidateReadProperty
= ValidateProperty
1104 def PackBlob(name
, value
, pbvalue
):
1105 """Packs a Blob property into a entity_pb.PropertyValue.
1108 name: The name of the property as a string.
1109 value: A Blob instance.
1110 pbvalue: The entity_pb.PropertyValue to pack this value into.
1112 pbvalue
.set_stringvalue(value
)
1115 def PackString(name
, value
, pbvalue
):
1116 """Packs a string-typed property into a entity_pb.PropertyValue.
1119 name: The name of the property as a string.
1120 value: A string, unicode, or string-like value instance.
1121 pbvalue: The entity_pb.PropertyValue to pack this value into.
1123 pbvalue
.set_stringvalue(unicode(value
).encode('utf-8'))
1126 def PackDatetime(name
, value
, pbvalue
):
1127 """Packs a datetime-typed property into a entity_pb.PropertyValue.
1130 name: The name of the property as a string.
1131 value: A datetime.datetime instance.
1132 pbvalue: The entity_pb.PropertyValue to pack this value into.
1134 pbvalue
.set_int64value(DatetimeToTimestamp(value
))
1137 def DatetimeToTimestamp(value
):
1138 """Converts a datetime.datetime to microseconds since the epoch, as a float.
1140 value: datetime.datetime
1142 Returns: value as a long
1145 value
= value
.astimezone(UTC
)
1146 return long(calendar
.timegm(value
.timetuple()) * 1000000L) + value
.microsecond
1149 def PackGeoPt(name
, value
, pbvalue
):
1150 """Packs a GeoPt property into a entity_pb.PropertyValue.
1153 name: The name of the property as a string.
1154 value: A GeoPt instance.
1155 pbvalue: The entity_pb.PropertyValue to pack this value into.
1157 pbvalue
.mutable_pointvalue().set_x(value
.lat
)
1158 pbvalue
.mutable_pointvalue().set_y(value
.lon
)
1161 def PackUser(name
, value
, pbvalue
):
1162 """Packs a User property into a entity_pb.PropertyValue.
1165 name: The name of the property as a string.
1166 value: A users.User instance.
1167 pbvalue: The entity_pb.PropertyValue to pack this value into.
1169 pbvalue
.mutable_uservalue().set_email(value
.email().encode('utf-8'))
1170 pbvalue
.mutable_uservalue().set_auth_domain(
1171 value
.auth_domain().encode('utf-8'))
1172 pbvalue
.mutable_uservalue().set_gaiaid(0)
1174 if value
.user_id() is not None:
1175 pbvalue
.mutable_uservalue().set_obfuscated_gaiaid(
1176 value
.user_id().encode('utf-8'))
1179 def PackKey(name
, value
, pbvalue
):
1180 """Packs a reference property into a entity_pb.PropertyValue.
1183 name: The name of the property as a string.
1184 value: A Key instance.
1185 pbvalue: The entity_pb.PropertyValue to pack this value into.
1187 ref
= value
._Key
__reference
1188 pbvalue
.mutable_referencevalue().set_app(ref
.app())
1189 for elem
in ref
.path().element_list():
1190 pbvalue
.mutable_referencevalue().add_pathelement().CopyFrom(elem
)
1193 def PackBool(name
, value
, pbvalue
):
1194 """Packs a boolean property into a entity_pb.PropertyValue.
1197 name: The name of the property as a string.
1198 value: A boolean instance.
1199 pbvalue: The entity_pb.PropertyValue to pack this value into.
1201 pbvalue
.set_booleanvalue(value
)
1204 def PackInteger(name
, value
, pbvalue
):
1205 """Packs an integer property into a entity_pb.PropertyValue.
1208 name: The name of the property as a string.
1209 value: An int or long instance.
1210 pbvalue: The entity_pb.PropertyValue to pack this value into.
1212 pbvalue
.set_int64value(value
)
1215 def PackFloat(name
, value
, pbvalue
):
1216 """Packs a float property into a entity_pb.PropertyValue.
1219 name: The name of the property as a string.
1220 value: A float instance.
1221 pbvalue: The entity_pb.PropertyValue to pack this value into.
1223 pbvalue
.set_doublevalue(value
)
1225 _PACK_PROPERTY_VALUES
= {
1227 ByteString
: PackBlob
,
1229 Category
: PackString
,
1230 datetime
.datetime
: PackDatetime
,
1239 PhoneNumber
: PackString
,
1240 PostalAddress
: PackString
,
1241 Rating
: PackInteger
,
1244 type(None): lambda name
, value
, pbvalue
: None,
1245 unicode: PackString
,
1246 users
.User
: PackUser
,
1249 assert set(_PACK_PROPERTY_VALUES
.iterkeys()) == _PROPERTY_TYPES
1252 def ToPropertyPb(name
, values
):
1253 """Creates type-specific entity_pb.PropertyValues.
1255 Determines the type and meaning of the PropertyValue based on the Python
1256 type of the input value(s).
1258 NOTE: This function does not validate anything!
1261 name: string or unicode; the property name
1262 values: The values for this property, either a single one or a list of them.
1263 All values must be a supported type. Lists of values must all be of the
1267 A list of entity_pb.PropertyValue instances.
1269 encoded_name
= name
.encode('utf-8')
1271 values_type
= type(values
)
1272 if values_type
is list:
1280 pb
= entity_pb
.Property()
1281 pb
.set_name(encoded_name
)
1282 pb
.set_multiple(multiple
)
1284 meaning
= _PROPERTY_MEANINGS
.get(v
.__class
__)
1285 if meaning
is not None:
1286 pb
.set_meaning(meaning
)
1288 pack_prop
= _PACK_PROPERTY_VALUES
[v
.__class
__]
1289 pbvalue
= pack_prop(name
, v
, pb
.mutable_value())
1298 def FromReferenceProperty(value
):
1299 """Converts a reference PropertyValue to a Key.
1302 value: entity_pb.PropertyValue
1308 BadValueError if the value is not a PropertyValue.
1310 assert isinstance(value
, entity_pb
.PropertyValue
)
1311 assert value
.has_referencevalue()
1312 ref
= value
.referencevalue()
1315 key_ref
= key
._Key
__reference
1316 key_ref
.set_app(ref
.app())
1318 for pathelem
in ref
.pathelement_list():
1319 key_ref
.mutable_path().add_element().CopyFrom(pathelem
)
1324 _EPOCH
= datetime
.datetime
.utcfromtimestamp(0)
1326 _PROPERTY_CONVERSIONS
= {
1327 entity_pb
.Property
.GD_WHEN
:
1330 lambda val
: _EPOCH
+ datetime
.timedelta(microseconds
=val
),
1331 entity_pb
.Property
.ATOM_CATEGORY
: Category
,
1332 entity_pb
.Property
.ATOM_LINK
: Link
,
1333 entity_pb
.Property
.GD_EMAIL
: Email
,
1334 entity_pb
.Property
.GEORSS_POINT
: lambda coords
: GeoPt(*coords
),
1335 entity_pb
.Property
.GD_IM
: IM
,
1336 entity_pb
.Property
.GD_PHONENUMBER
: PhoneNumber
,
1337 entity_pb
.Property
.GD_POSTALADDRESS
: PostalAddress
,
1338 entity_pb
.Property
.GD_RATING
: Rating
,
1339 entity_pb
.Property
.BLOB
: Blob
,
1340 entity_pb
.Property
.BYTESTRING
: ByteString
,
1341 entity_pb
.Property
.TEXT
: Text
,
1345 def FromPropertyPb(pb
):
1346 """Converts a property PB to a python value.
1349 pb: entity_pb.Property
1352 # return type is determined by the type of the argument
1353 string, int, bool, double, users.User, or one of the atom or gd types
1356 meaning
= pb
.meaning()
1358 if pbval
.has_stringvalue():
1359 value
= pbval
.stringvalue()
1360 if meaning
not in (entity_pb
.Property
.BLOB
, entity_pb
.Property
.BYTESTRING
):
1361 value
= unicode(value
.decode('utf-8'))
1362 elif pbval
.has_int64value():
1363 value
= long(pbval
.int64value())
1364 elif pbval
.has_booleanvalue():
1365 value
= bool(pbval
.booleanvalue())
1366 elif pbval
.has_doublevalue():
1367 value
= pbval
.doublevalue()
1368 elif pbval
.has_referencevalue():
1369 value
= FromReferenceProperty(pbval
)
1370 elif pbval
.has_pointvalue():
1371 value
= (pbval
.pointvalue().x(), pbval
.pointvalue().y())
1372 elif pbval
.has_uservalue():
1373 email
= unicode(pbval
.uservalue().email().decode('utf-8'))
1374 auth_domain
= unicode(pbval
.uservalue().auth_domain().decode('utf-8'))
1375 obfuscated_gaiaid
= pbval
.uservalue().obfuscated_gaiaid().decode('utf-8')
1376 obfuscated_gaiaid
= unicode(obfuscated_gaiaid
)
1377 value
= users
.User(email
=email
,
1378 _auth_domain
=auth_domain
,
1379 _user_id
=obfuscated_gaiaid
)
1384 if pb
.has_meaning():
1385 conversion
= _PROPERTY_CONVERSIONS
[meaning
]
1386 value
= conversion(value
)
1387 except (KeyError, ValueError, IndexError, TypeError, AttributeError), msg
:
1388 raise datastore_errors
.BadValueError(
1389 'Error converting pb: %s\nException was: %s' % (pb
, msg
))
1394 def PropertyTypeName(value
):
1395 """Returns the name of the type of the given property value, as a string.
1397 Raises BadValueError if the value is not a valid property type.
1400 value: any valid property value
1405 if value
.__class
__ in _PROPERTY_MEANINGS
:
1406 meaning
= _PROPERTY_MEANINGS
[value
.__class
__]
1407 name
= entity_pb
.Property
._Meaning
_NAMES
[meaning
]
1408 return name
.lower().replace('_', ':')
1409 elif isinstance(value
, basestring
):
1411 elif isinstance(value
, users
.User
):
1413 elif isinstance(value
, long):
1418 return typename(value
).lower()
1420 _PROPERTY_TYPE_STRINGS
= {
1428 'bytestring': ByteString
,
1431 'atom:category': Category
,
1434 'gd:when': datetime
.datetime
,
1435 'georss:point': GeoPt
,
1437 'gd:phonenumber': PhoneNumber
,
1438 'gd:postaladdress': PostalAddress
,
1439 'gd:rating': Rating
,
1443 def FromPropertyTypeName(type_name
):
1444 """Returns the python type given a type name.
1447 type_name: A string representation of a datastore type name.
1452 return _PROPERTY_TYPE_STRINGS
[type_name
]
1455 def PropertyValueFromString(type_
,
1458 """Returns an instance of a property value given a type and string value.
1460 The reverse of this method is just str() and type() of the python value.
1462 Note that this does *not* support non-UTC offsets in ISO 8601-formatted
1463 datetime strings, e.g. the -08:00 suffix in '2002-12-25 00:00:00-08:00'.
1464 It only supports -00:00 and +00:00 suffixes, which are UTC.
1467 type_: A python class.
1468 value_string: A string representation of the value of the property.
1471 An instance of 'type'.
1474 ValueError if type_ is datetime and value_string has a timezone offset.
1476 if type_
== datetime
.datetime
:
1477 value_string
= value_string
.strip()
1478 if value_string
[-6] in ('+', '-'):
1479 if value_string
[-5:] == '00:00':
1480 value_string
= value_string
[:-6]
1482 raise ValueError('Non-UTC offsets in datetimes are not supported.')
1484 split
= value_string
.split('.')
1488 microseconds
= int(split
[1])
1490 time_struct
= time
.strptime(iso_date
, '%Y-%m-%d %H:%M:%S')[0:6]
1491 value
= datetime
.datetime(*(time_struct
+ (microseconds
,)))
1493 elif type_
== Rating
:
1494 return Rating(int(value_string
))
1496 return value_string
== 'True'
1497 elif type_
== users
.User
:
1498 return users
.User(value_string
, _auth_domain
)
1499 elif type_
== type(None):
1501 return type_(value_string
)