3 # Copyright 2010 Google Inc.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
18 """Stand-alone implementation of in memory protocol messages.
21 Enum: Represents an enumerated type.
22 Variant: Hint for wire format to determine how to serialize.
23 Message: Base class for user defined messages.
24 IntegerField: Field for integer values.
25 FloatField: Field for float values.
26 BooleanField: Field for boolean values.
27 BytesField: Field for binary string values.
28 StringField: Field for UTF-8 string values.
29 MessageField: Field for other message type values.
30 EnumField: Field for enumerated type values.
32 Public Exceptions (indentation indications class hierarchy):
33 EnumDefinitionError: Raised when enumeration is incorrectly defined.
34 FieldDefinitionError: Raised when field is incorrectly defined.
35 InvalidVariantError: Raised when variant is not compatible with field type.
36 InvalidDefaultError: Raised when default is not compatiable with field.
37 InvalidNumberError: Raised when field number is out of range or reserved.
38 MessageDefinitionError: Raised when message is incorrectly defined.
39 DuplicateNumberError: Raised when field has duplicate number with another.
40 ValidationError: Raised when a message or field is not valid.
41 DefinitionNotFoundError: Raised when definition not found.
44 __author__
= 'rafek@google.com (Rafe Kaplan)'
56 __all__
= ['MAX_ENUM_VALUE',
58 'FIRST_RESERVED_FIELD_NUMBER',
59 'LAST_RESERVED_FIELD_NUMBER',
78 'EnumDefinitionError',
79 'FieldDefinitionError',
80 'InvalidVariantError',
81 'InvalidDefaultError',
83 'MessageDefinitionError',
84 'DuplicateNumberError',
86 'DefinitionNotFoundError',
90 # TODO(rafek): Add extended module test to ensure all exceptions
91 # in services extends Error.
95 class EnumDefinitionError(Error
):
96 """Enumeration definition error."""
99 class FieldDefinitionError(Error
):
100 """Field definition error."""
103 class InvalidVariantError(FieldDefinitionError
):
104 """Invalid variant provided to field."""
107 class InvalidDefaultError(FieldDefinitionError
):
108 """Invalid default provided to field."""
111 class InvalidNumberError(FieldDefinitionError
):
112 """Invalid number provided to field."""
115 class MessageDefinitionError(Error
):
116 """Message definition error."""
119 class DuplicateNumberError(Error
):
120 """Duplicate number assigned to field."""
123 class DefinitionNotFoundError(Error
):
124 """Raised when definition is not found."""
127 class DecodeError(Error
):
128 """Error found decoding message from encoded form."""
131 class EncodeError(Error
):
132 """Error found when encoding message."""
135 class ValidationError(Error
):
136 """Invalid value for message error."""
139 """Prints string with field name if present on exception."""
140 message
= Error
.__str
__(self
)
142 field_name
= self
.field_name
143 except AttributeError:
149 # Attributes that are reserved by a class definition that
150 # may not be used by either Enum or Message class definitions.
151 _RESERVED_ATTRIBUTE_NAMES
= frozenset(
152 ['__module__', '__doc__'])
154 _POST_INIT_FIELD_ATTRIBUTE_NAMES
= frozenset(
156 '_message_definition',
157 '_MessageField__type',
159 '_EnumField__resolved_default'])
161 _POST_INIT_ATTRIBUTE_NAMES
= frozenset(
162 ['_message_definition'])
164 # Maximum enumeration value as defined by the protocol buffers standard.
165 # All enum values must be less than or equal to this value.
166 MAX_ENUM_VALUE
= (2 ** 29) - 1
168 # Maximum field number as defined by the protocol buffers standard.
169 # All field numbers must be less than or equal to this value.
170 MAX_FIELD_NUMBER
= (2 ** 29) - 1
172 # Field numbers between 19000 and 19999 inclusive are reserved by the
173 # protobuf protocol and may not be used by fields.
174 FIRST_RESERVED_FIELD_NUMBER
= 19000
175 LAST_RESERVED_FIELD_NUMBER
= 19999
178 class _DefinitionClass(type):
179 """Base meta-class used for definition meta-classes.
181 The Enum and Message definition classes share some basic functionality.
182 Both of these classes may be contained by a Message definition. After
183 initialization, neither class may have attributes changed
184 except for the protected _message_definition attribute, and that attribute
185 may change only once.
188 __initialized
= False
190 def __init__(cls
, name
, bases
, dct
):
192 type.__init
__(cls
, name
, bases
, dct
)
193 # Base classes may never be initialized.
194 if cls
.__bases
__ != (object,):
195 cls
.__initialized
= True
197 def message_definition(cls
):
198 """Get outer Message definition that contains this definition.
201 Containing Message definition if definition is contained within one,
205 return cls
._message
_definition
()
206 except AttributeError:
209 def __setattr__(cls
, name
, value
):
210 """Overridden so that cannot set variables on definition classes after init.
212 Setting attributes on a class must work during the period of initialization
213 to set the enumation value class variables and build the name/number maps.
214 Once __init__ has set the __initialized flag to True prohibits setting any
215 more values on the class. The class is in effect frozen.
218 name: Name of value to set.
221 if cls
.__initialized
and name
not in _POST_INIT_ATTRIBUTE_NAMES
:
222 raise AttributeError('May not change values: %s' % name
)
224 type.__setattr
__(cls
, name
, value
)
226 def __delattr__(cls
, name
):
227 """Overridden so that cannot delete varaibles on definition classes."""
228 raise TypeError('May not delete attributes on definition class')
230 def definition_name(cls
):
231 """Helper method for creating definition name.
233 Names will be generated to include the classes package name, scope (if the
234 class is nested in another definition) and class name.
236 By default, the package name for a definition is derived from its module
237 name. However, this value can be overriden by placing a 'package' attribute
238 in the module that contains the definition class. For example:
240 package = 'some.alternate.package'
242 class MyMessage(Message):
245 >>> MyMessage.definition_name()
246 some.alternate.package.MyMessage
249 Dot-separated fully qualified name of definition.
251 outer_definition_name
= cls
.outer_definition_name()
252 if outer_definition_name
is None:
253 return unicode(cls
.__name
__)
255 return u
'%s.%s' % (outer_definition_name
, cls
.__name
__)
257 def outer_definition_name(cls
):
258 """Helper method for creating outer definition name.
261 If definition is nested, will return the outer definitions name, else the
264 outer_definition
= cls
.message_definition()
265 if not outer_definition
:
266 return util
.get_package_for_module(cls
.__module
__)
268 return outer_definition
.definition_name()
270 def definition_package(cls
):
271 """Helper method for creating creating the package of a definition.
274 Name of package that definition belongs to.
276 outer_definition
= cls
.message_definition()
277 if not outer_definition
:
278 return util
.get_package_for_module(cls
.__module
__)
280 return outer_definition
.definition_package()
283 class _EnumClass(_DefinitionClass
):
284 """Meta-class used for defining the Enum base class.
286 Meta-class enables very specific behavior for any defined Enum
287 class. All attributes defined on an Enum sub-class must be integers.
288 Each attribute defined on an Enum sub-class is translated
289 into an instance of that sub-class, with the name of the attribute
290 as its name, and the number provided as its value. It also ensures
291 that only one level of Enum class hierarchy is possible. In other
292 words it is not possible to delcare sub-classes of sub-classes of
295 This class also defines some functions in order to restrict the
296 behavior of the Enum class and its sub-classes. It is not possible
297 to change the behavior of the Enum class in later classes since
298 any new classes may be defined with only integer values, and no methods.
301 def __init__(cls
, name
, bases
, dct
):
302 # Can only define one level of sub-classes below Enum.
303 if not (bases
== (object,) or bases
== (Enum
,)):
304 raise EnumDefinitionError('Enum type %s may only inherit from Enum' %
310 # Enum base class does not need to be initialized or locked.
311 if bases
!= (object,):
312 # Replace integer with number.
313 for attribute
, value
in dct
.iteritems():
315 # Module will be in every enum class.
316 if attribute
in _RESERVED_ATTRIBUTE_NAMES
:
319 # Reject anything that is not an int.
320 if not isinstance(value
, (int, long)):
321 raise EnumDefinitionError(
322 'May only use integers in Enum definitions. Found: %s = %s' %
325 # Protocol buffer standard recommends non-negative values.
326 # Reject negative values.
328 raise EnumDefinitionError(
329 'Must use non-negative enum values. Found: %s = %d' %
332 if value
> MAX_ENUM_VALUE
:
333 raise EnumDefinitionError(
334 'Must use enum values less than or equal %d. Found: %s = %d' %
335 (MAX_ENUM_VALUE
, attribute
, value
))
337 if value
in cls
.__by
_number
:
338 raise EnumDefinitionError(
339 'Value for %s = %d is already defined: %s' %
340 (attribute
, value
, cls
.__by
_number
[value
].name
))
342 # Create enum instance and list in new Enum type.
343 instance
= object.__new
__(cls
)
344 cls
.__init
__(instance
, attribute
, value
)
345 cls
.__by
_name
[instance
.name
] = instance
346 cls
.__by
_number
[instance
.number
] = instance
347 setattr(cls
, attribute
, instance
)
349 _DefinitionClass
.__init
__(cls
, name
, bases
, dct
)
352 """Iterate over all values of enum.
355 Enumeration instances of the Enum class in arbitrary order.
357 return cls
.__by
_number
.itervalues()
360 """Get all names for Enum.
363 An iterator for names of the enumeration in arbitrary order.
365 return cls
.__by
_name
.iterkeys()
368 """Get all numbers for Enum.
371 An iterator for all numbers of the enumeration in arbitrary order.
373 return cls
.__by
_number
.iterkeys()
375 def lookup_by_name(cls
, name
):
376 """Look up Enum by name.
379 name: Name of enum to find.
382 Enum sub-class instance of that value.
384 return cls
.__by
_name
[name
]
386 def lookup_by_number(cls
, number
):
387 """Look up Enum by number.
390 number: Number of enum to find.
393 Enum sub-class instance of that value.
395 return cls
.__by
_number
[number
]
398 return len(cls
.__by
_name
)
402 """Base class for all enumerated types."""
404 __metaclass__
= _EnumClass
406 __slots__
= set(('name', 'number'))
408 def __new__(cls
, index
):
409 """Acts as look-up routine after class is initialized.
411 The purpose of overriding __new__ is to provide a way to treat
412 Enum subclasses as casting types, similar to how the int type
413 functions. A program can pass a string or an integer and this
414 method with "convert" that value in to an appropriate Enum instance.
417 index: Name or number to look up. During initialization
418 this is always the name of the new enum value.
421 TypeError: When an inappropriate index value is passed provided.
423 # If is enum type of this class, return it.
424 if isinstance(index
, cls
):
427 # If number, look up by number.
428 if isinstance(index
, (int, long)):
430 return cls
.lookup_by_number(index
)
434 # If name, look up by name.
435 if isinstance(index
, basestring
):
437 return cls
.lookup_by_name(index
)
441 raise TypeError('No such value for %s in Enum %s' %
442 (index
, cls
.__name
__))
444 def __init__(self
, name
, number
=None):
445 """Initialize new Enum instance.
447 Since this should only be called during class initialization any
448 calls that happen after the class is frozen raises an exception.
450 # Immediately return if __init__ was called after _Enum.__init__().
451 # It means that casting operator version of the class constructor
453 if getattr(type(self
), '_DefinitionClass__initialized'):
455 object.__setattr
__(self
, 'name', name
)
456 object.__setattr
__(self
, 'number', number
)
458 def __setattr__(self
, name
, value
):
459 raise TypeError('May not change enum values')
468 return '%s(%s, %d)' % (type(self
).__name
__, self
.name
, self
.number
)
470 def __cmp__(self
, other
):
471 """Order is by number."""
472 if isinstance(other
, type(self
)):
473 return cmp(self
.number
, other
.number
)
474 return NotImplemented
478 """Make dictionary version of enumerated class.
480 Dictionary created this way can be used with def_num.
483 A dict (name) -> number
485 return dict((item
.name
, item
.number
) for item
in iter(cls
))
488 def def_enum(dct
, name
):
489 """Define enum class from dictionary.
492 dct: Dictionary of enumerated values for type.
495 return type(name
, (Enum
,), dct
)
498 # TODO(rafek): Determine to what degree this enumeration should be compatible
499 # with FieldDescriptor.Type in:
501 # http://code.google.com/p/protobuf/source/browse/trunk/src/google/protobuf/descriptor.proto
503 """Wire format variant.
505 Used by the 'protobuf' wire format to determine how to transmit
506 a single piece of data. May be used by other formats.
508 See: http://code.google.com/apis/protocolbuffers/docs/encoding.html
511 DOUBLE: 64-bit floating point number.
512 FLOAT: 32-bit floating point number.
513 INT64: 64-bit signed integer.
514 UINT64: 64-bit unsigned integer.
515 INT32: 32-bit signed integer.
516 BOOL: Boolean value (True or False).
517 STRING: String of UTF-8 encoded text.
518 MESSAGE: Embedded message as byte string.
519 BYTES: String of 8-bit bytes.
520 UINT32: 32-bit unsigned integer.
521 ENUM: Enum value as integer.
522 SINT32: 32-bit signed integer. Uses "zig-zag" encoding.
523 SINT64: 64-bit signed integer. Uses "zig-zag" encoding.
540 class _MessageClass(_DefinitionClass
):
541 """Meta-class used for defining the Message base class.
543 For more details about Message classes, see the Message class docstring.
544 Information contained there may help understanding this class.
546 Meta-class enables very specific behavior for any defined Message
547 class. All attributes defined on an Message sub-class must be field
548 instances, Enum class definitions or other Message class definitions. Each
549 field attribute defined on an Message sub-class is added to the set of
550 field definitions and the attribute is translated in to a slot. It also
551 ensures that only one level of Message class hierarchy is possible. In other
552 words it is not possible to declare sub-classes of sub-classes of
555 This class also defines some functions in order to restrict the
556 behavior of the Message class and its sub-classes. It is not possible
557 to change the behavior of the Message class in later classes since
558 any new classes may be defined with only field, Enums and Messages, and
562 def __new__(cls
, name
, bases
, dct
):
563 """Create new Message class instance.
565 The __new__ method of the _MessageClass type is overridden so as to
566 allow the translation of Field instances to slots.
573 if bases
!= (object,):
574 # Can only define one level of sub-classes below Message.
575 if bases
!= (Message
,):
576 raise MessageDefinitionError(
577 'Message types may only inherit from Message')
581 # Must not use iteritems because this loop will change the state of dct.
582 for key
, field
in dct
.items():
584 if key
in _RESERVED_ATTRIBUTE_NAMES
:
587 if isinstance(field
, type) and issubclass(field
, Enum
):
591 if (isinstance(field
, type) and
592 issubclass(field
, Message
) and
593 field
is not Message
):
597 # Reject anything that is not a field.
598 if type(field
) is Field
or not isinstance(field
, Field
):
599 raise MessageDefinitionError(
600 'May only use fields in message definitions. Found: %s = %s' %
603 if field
.number
in by_number
:
604 raise DuplicateNumberError(
605 'Field with number %d declared more than once in %s' %
606 (field
.number
, name
))
610 # Place in name and number maps.
612 by_number
[field
.number
] = field
614 # Add enums if any exist.
616 dct
['__enums__'] = sorted(enums
)
618 # Add messages if any exist.
620 dct
['__messages__'] = sorted(messages
)
622 dct
['_Message__by_number'] = by_number
623 dct
['_Message__by_name'] = by_name
625 return _DefinitionClass
.__new
__(cls
, name
, bases
, dct
)
627 def __init__(cls
, name
, bases
, dct
):
628 """Initializer required to assign references to new class."""
629 if bases
!= (object,):
630 for value
in dct
.itervalues():
631 if isinstance(value
, _DefinitionClass
) and not value
is Message
:
632 value
._message
_definition
= weakref
.ref(cls
)
634 for field
in cls
.all_fields():
635 field
._message
_definition
= weakref
.ref(cls
)
637 _DefinitionClass
.__init
__(cls
, name
, bases
, dct
)
640 class Message(object):
641 """Base class for user defined message objects.
643 Used to define messages for efficient transmission across network or
644 process space. Messages are defined using the field classes (IntegerField,
645 FloatField, EnumField, etc.).
647 Messages are more restricted than normal classes in that they may only
648 contain field attributes and other Message and Enum definitions. These
649 restrictions are in place because the structure of the Message class is
650 intentended to itself be transmitted across network or process space and
651 used directly by clients or even other servers. As such methods and
652 non-field attributes could not be transmitted with the structural information
653 causing discrepancies between different languages and implementations.
655 Initialization and validation:
657 A Message object is considered to be initialized if it has all required
658 fields and any nested messages are also initialized.
660 Calling 'check_initialized' will raise a ValidationException if it is not
661 initialized; 'is_initialized' returns a boolean value indicating if it is
664 Validation automatically occurs when Message objects are created
665 and populated. Validation that a given value will be compatible with
666 a field that it is assigned to can be done through the Field instances
667 validate() method. The validate method used on a message will check that
668 all values of a message and its sub-messages are valid. Assingning an
669 invalid value to a field will raise a ValidationException.
674 class TradeType(Enum):
681 price = IntegerField(1, required=True)
682 quantity = IntegerField(2, required=True)
684 class Order(Message):
685 symbol = StringField(1, required=True)
686 total_quantity = IntegerField(2, required=True)
687 trade_type = EnumField(TradeType, 3, required=True)
688 lots = MessageField(Lot, 4, repeated=True)
689 limit = IntegerField(5)
691 order = Order(symbol='GOOG',
693 trade_type=TradeType.BUY)
695 lot1 = Lot(price=304,
698 lot2 = Lot(price = 305,
701 order.lots = [lot1, lot2]
703 # Now object is initialized!
704 order.check_initialized()
707 __metaclass__
= _MessageClass
709 def __init__(self
, **kwargs
):
710 """Initialize internal messages state.
713 A message can be initialized via the constructor by passing in keyword
714 arguments corresponding to fields. For example:
717 day = IntegerField(1)
718 month = IntegerField(2)
719 year = IntegerField(3)
723 date = Date(day=6, month=6, year=1911)
725 is the same as doing:
732 # Tag being an essential implementation detail must be private.
734 self
.__unrecognized
_fields
= {}
737 for name
, value
in kwargs
.iteritems():
738 setattr(self
, name
, value
)
741 # initialize repeated fields.
742 for field
in self
.all_fields():
743 if field
.repeated
and field
.name
not in assigned
:
744 setattr(self
, field
.name
, [])
747 def check_initialized(self
):
748 """Check class for initialization status.
750 Check that all required fields are initialized
753 ValidationError: If message is not initialized.
755 for name
, field
in self
.__by
_name
.iteritems():
756 value
= getattr(self
, name
)
759 raise ValidationError("Message %s is missing required field %s" %
760 (type(self
).__name
__, name
))
763 if (isinstance(field
, MessageField
) and
764 issubclass(field
.message_type
, Message
)):
767 item_message_value
= field
.value_to_message(item
)
768 item_message_value
.check_initialized()
770 message_value
= field
.value_to_message(value
)
771 message_value
.check_initialized()
772 except ValidationError
, err
:
773 if not hasattr(err
, 'message_name'):
774 err
.message_name
= type(self
).__name
__
777 def is_initialized(self
):
778 """Get initialization status.
781 True if message is valid, else False.
784 self
.check_initialized()
785 except ValidationError
:
792 """Get all field definition objects.
794 Ordering is arbitrary.
797 Iterator over all values in arbitrary order.
799 return cls
.__by
_name
.itervalues()
802 def field_by_name(cls
, name
):
803 """Get field by name.
806 Field object associated with name.
809 KeyError if no field found by that name.
811 return cls
.__by
_name
[name
]
814 def field_by_number(cls
, number
):
815 """Get field by number.
818 Field object associated with number.
821 KeyError if no field found by that number.
823 return cls
.__by
_number
[number
]
825 def get_assigned_value(self
, name
):
826 """Get the assigned value of an attribute.
828 Get the underlying value of an attribute. If value has not been set, will
829 not return the default for the field.
832 name: Name of attribute to get.
835 Value of attribute, None if it has not been set.
837 message_type
= type(self
)
839 field
= message_type
.field_by_name(name
)
841 raise AttributeError('Message %s has no field %s' % (
842 message_type
.__name
__, name
))
843 return self
.__tags
.get(field
.number
)
845 def reset(self
, name
):
846 """Reset assigned value for field.
848 Resetting a field will return it to its default value or None.
851 name: Name of field to reset.
853 message_type
= type(self
)
855 field
= message_type
.field_by_name(name
)
857 if name
not in message_type
.__by
_name
:
858 raise AttributeError('Message %s has no field %s' % (
859 message_type
.__name
__, name
))
860 self
.__tags
.pop(field
.number
, None)
862 def all_unrecognized_fields(self
):
863 """Get the names of all unrecognized fields in this message."""
864 return self
.__unrecognized
_fields
.keys()
866 def get_unrecognized_field_info(self
, key
, value_default
=None,
867 variant_default
=None):
868 """Get the value and variant of an unknown field in this message.
871 key: The name or number of the field to retrieve.
872 value_default: Value to be returned if the key isn't found.
873 variant_default: Value to be returned as variant if the key isn't
877 (value, variant), where value and variant are whatever was passed
878 to set_unrecognized_field.
880 value
, variant
= self
.__unrecognized
_fields
.get(key
, (value_default
,
882 return value
, variant
884 def set_unrecognized_field(self
, key
, value
, variant
):
885 """Set an unrecognized field, used when decoding a message.
888 key: The name or number used to refer to this unknown value.
889 value: The value of the field.
890 variant: Type information needed to interpret the value or re-encode it.
893 TypeError: If the variant is not an instance of messages.Variant.
895 if not isinstance(variant
, Variant
):
896 raise TypeError('Variant type %s is not valid.' % variant
)
897 self
.__unrecognized
_fields
[key
] = value
, variant
899 def __setattr__(self
, name
, value
):
900 """Change set behavior for messages.
902 Messages may only be assigned values that are fields.
904 Does not try to validate field when set.
907 name: Name of field to assign to.
908 vlaue: Value to assign to field.
911 AttributeError when trying to assign value that is not a field.
913 if name
in self
.__by
_name
or name
.startswith('_Message__'):
914 object.__setattr
__(self
, name
, value
)
916 raise AttributeError("May not assign arbitrary value %s "
917 "to message %s" % (name
, type(self
).__name
__))
920 """Make string representation of message.
924 class MyMessage(messages.Message):
925 integer_value = messages.IntegerField(1)
926 string_value = messages.StringField(2)
928 my_message = MyMessage()
929 my_message.integer_value = 42
930 my_message.string_value = u'A string'
934 ... integer_value: 42
935 ... string_value: u'A string'>
938 String representation of message, including the values
939 of all fields and repr of all sub-messages.
941 body
= ['<', type(self
).__name
__]
942 for field
in sorted(self
.all_fields(),
943 key
=lambda f
: f
.number
):
944 attribute
= field
.name
945 value
= self
.get_assigned_value(field
.name
)
946 if value
is not None:
947 body
.append('\n %s: %s' % (attribute
, repr(value
)))
951 def __eq__(self
, other
):
952 """Equality operator.
954 Does field by field comparison with other message. For
955 equality, must be same type and values of all fields must be
958 Messages not required to be initialized for comparison.
960 Does not attempt to determine equality for values that have
961 default values that are not set. In other words:
963 class HasDefault(Message):
965 attr1 = StringField(1, default='default value')
967 message1 = HasDefault()
968 message2 = HasDefault()
969 message2.attr1 = 'default value'
973 Does not compare unknown values.
976 other: Other message to compare with.
978 # TODO(rafek): Implement "equivalent" which does comparisons
979 # taking default values in to consideration.
983 if type(self
) is not type(other
):
986 return self
.__tags
== other
.__tags
988 def __ne__(self
, other
):
989 """Not equals operator.
991 Does field by field comparison with other message. For
992 non-equality, must be different type or any value of a field must be
993 non-equal to the same field in the other instance.
995 Messages not required to be initialized for comparison.
998 other: Other message to compare with.
1000 return not self
.__eq
__(other
)
1003 class FieldList(list):
1004 """List implementation that validates field values.
1006 This list implementation overrides all methods that add values in to a list
1007 in order to validate those new elements. Attempting to add or set list
1008 values that are not of the correct type will raise ValidationError.
1011 def __init__(self
, field_instance
, sequence
):
1015 field_instance: Instance of field that validates the list.
1016 sequence: List or tuple to construct list from.
1018 if not field_instance
.repeated
:
1019 raise FieldDefinitionError('FieldList may only accept repeated fields')
1020 self
.__field
= field_instance
1021 self
.__field
.validate(sequence
)
1022 list.__init
__(self
, sequence
)
1026 """Field that validates list."""
1029 def __setslice__(self
, i
, j
, sequence
):
1030 """Validate slice assignment to list."""
1031 self
.__field
.validate(sequence
)
1032 list.__setslice
__(self
, i
, j
, sequence
)
1034 def __setitem__(self
, index
, value
):
1035 """Validate item assignment to list."""
1036 self
.__field
.validate_element(value
)
1037 list.__setitem
__(self
, index
, value
)
1039 def append(self
, value
):
1040 """Validate item appending to list."""
1041 self
.__field
.validate_element(value
)
1042 return list.append(self
, value
)
1044 def extend(self
, sequence
):
1045 """Validate extension of list."""
1046 self
.__field
.validate(sequence
)
1047 return list.extend(self
, sequence
)
1049 def insert(self
, index
, value
):
1050 """Validate item insertion to list."""
1051 self
.__field
.validate_element(value
)
1052 return list.insert(self
, index
, value
)
1055 # TODO(rafek): Prevent additional field subclasses.
1056 class Field(object):
1058 __variant_to_type
= {}
1060 class __metaclass__(type):
1062 def __init__(cls
, name
, bases
, dct
):
1063 getattr(cls
, '_Field__variant_to_type').update(
1064 (variant
, cls
) for variant
in dct
.get('VARIANTS', []))
1065 type.__init
__(cls
, name
, bases
, dct
)
1067 __initialized
= False
1078 The required and repeated parameters are mutually exclusive. Setting both
1079 to True will raise a FieldDefinitionError.
1081 Sub-class Attributes:
1082 Each sub-class of Field must define the following:
1083 VARIANTS: Set of variant types accepted by that field.
1084 DEFAULT_VARIANT: Default variant type if not specified in constructor.
1087 number: Number of field. Must be unique per message class.
1088 required: Whether or not field is required. Mutually exclusive with
1090 repeated: Whether or not field is repeated. Mutually exclusive with
1092 variant: Wire-format variant hint.
1093 default: Default value for field if not found in stream.
1096 InvalidVariantError when invalid variant for field is provided.
1097 InvalidDefaultError when invalid default for field is provided.
1098 FieldDefinitionError when invalid number provided or mutually exclusive
1100 InvalidNumberError when the field number is out of range or reserved.
1102 if not isinstance(number
, int) or not 1 <= number
<= MAX_FIELD_NUMBER
:
1103 raise InvalidNumberError('Invalid number for field: %s\n'
1104 'Number must be 1 or greater and %d or less' %
1105 (number
, MAX_FIELD_NUMBER
))
1107 if FIRST_RESERVED_FIELD_NUMBER
<= number
<= LAST_RESERVED_FIELD_NUMBER
:
1108 raise InvalidNumberError('Tag number %d is a reserved number.\n'
1109 'Numbers %d to %d are reserved' %
1110 (number
, FIRST_RESERVED_FIELD_NUMBER
,
1111 LAST_RESERVED_FIELD_NUMBER
))
1113 if repeated
and required
:
1114 raise FieldDefinitionError('Cannot set both repeated and required')
1117 variant
= self
.DEFAULT_VARIANT
1119 if repeated
and default
is not None:
1120 raise FieldDefinitionError('Repeated fields may not have defaults')
1122 if variant
not in self
.VARIANTS
:
1123 raise InvalidVariantError(
1124 'Invalid variant: %s\nValid variants for %s are %r' %
1125 (variant
, type(self
).__name
__, sorted(self
.VARIANTS
)))
1127 self
.number
= number
1128 self
.required
= required
1129 self
.repeated
= repeated
1130 self
.variant
= variant
1132 if default
is not None:
1134 self
.validate_default(default
)
1135 except ValidationError
, err
:
1138 except AttributeError:
1139 # For when raising error before name initialization.
1140 raise InvalidDefaultError('Invalid default value for %s: %s: %s' %
1141 (self
.__class
__.__name
__, default
, err
))
1143 raise InvalidDefaultError('Invalid default value for field %s: '
1144 '%s: %s' % (name
, default
, err
))
1146 self
.__default
= default
1147 self
.__initialized
= True
1149 def __setattr__(self
, name
, value
):
1150 """Setter overidden to prevent assignment to fields after creation.
1153 name: Name of attribute to set.
1154 value: Value to assign.
1156 # Special case post-init names. They need to be set after constructor.
1157 if name
in _POST_INIT_FIELD_ATTRIBUTE_NAMES
:
1158 object.__setattr
__(self
, name
, value
)
1161 # All other attributes must be set before __initialized.
1162 if not self
.__initialized
:
1163 # Not initialized yet, allow assignment.
1164 object.__setattr
__(self
, name
, value
)
1166 raise AttributeError('Field objects are read-only')
1168 def __set__(self
, message_instance
, value
):
1169 """Set value on message.
1172 message_instance: Message instance to set value on.
1173 value: Value to set on message.
1175 # Reaches in to message instance directly to assign to private tags.
1178 raise ValidationError(
1179 'May not assign None to repeated field %s' % self
.name
)
1181 message_instance
._Message
__tags
.pop(self
.number
, None)
1184 value
= FieldList(self
, value
)
1186 self
.validate(value
)
1187 message_instance
._Message
__tags
[self
.number
] = value
1189 def __get__(self
, message_instance
, message_class
):
1190 if message_instance
is None:
1193 result
= message_instance
._Message
__tags
.get(self
.number
)
1199 def validate_element(self
, value
):
1200 """Validate single element of field.
1202 This is different from validate in that it is used on individual
1203 values of repeated fields.
1206 value: Value to validate.
1209 ValidationError if value is not expected type.
1211 if not isinstance(value
, self
.type):
1214 raise ValidationError('Required field is missing')
1218 except AttributeError:
1219 raise ValidationError('Expected type %s for %s, '
1220 'found %s (type %s)' %
1221 (self
.type, self
.__class
__.__name
__,
1222 value
, type(value
)))
1224 raise ValidationError('Expected type %s for field %s, '
1225 'found %s (type %s)' %
1226 (self
.type, name
, value
, type(value
)))
1228 def __validate(self
, value
, validate_element
):
1229 """Internal validation function.
1231 Validate an internal value using a function to validate individual elements.
1234 value: Value to validate.
1235 validate_element: Function to use to validate individual elements.
1238 ValidationError if value is not expected type.
1240 if not self
.repeated
:
1241 validate_element(value
)
1243 # Must be a list or tuple, may not be a string.
1244 if isinstance(value
, (list, tuple)):
1245 for element
in value
:
1249 except AttributeError:
1250 raise ValidationError('Repeated values for %s '
1251 'may not be None' % self
.__class
__.__name
__)
1253 raise ValidationError('Repeated values for field %s '
1254 'may not be None' % name
)
1255 validate_element(element
)
1256 elif value
is not None:
1259 except AttributeError:
1260 raise ValidationError('%s is repeated. Found: %s' % (
1261 self
.__class
__.__name
__, value
))
1263 raise ValidationError('Field %s is repeated. Found: %s' % (name
,
1266 def validate(self
, value
):
1267 """Validate value assigned to field.
1270 value: Value to validate.
1273 ValidationError if value is not expected type.
1275 self
.__validate
(value
, self
.validate_element
)
1277 def validate_default_element(self
, value
):
1278 """Validate value as assigned to field default field.
1280 Some fields may allow for delayed resolution of default types necessary
1281 in the case of circular definition references. In this case, the default
1282 value might be a place holder that is resolved when needed after all the
1283 message classes are defined.
1286 value: Default value to validate.
1289 ValidationError if value is not expected type.
1291 self
.validate_element(value
)
1293 def validate_default(self
, value
):
1294 """Validate default value assigned to field.
1297 value: Value to validate.
1300 ValidationError if value is not expected type.
1302 self
.__validate
(value
, self
.validate_default_element
)
1304 def message_definition(self
):
1305 """Get Message definition that contains this Field definition.
1308 Containing Message definition for Field. Will return None if for
1309 some reason Field is defined outside of a Message class.
1312 return self
._message
_definition
()
1313 except AttributeError:
1318 """Get default value for field."""
1319 return self
.__default
1322 def lookup_field_type_by_variant(cls
, variant
):
1323 return cls
.__variant
_to
_type
[variant
]
1326 class IntegerField(Field
):
1327 """Field definition for integer values."""
1329 VARIANTS
= frozenset([Variant
.INT32
,
1337 DEFAULT_VARIANT
= Variant
.INT64
1342 class FloatField(Field
):
1343 """Field definition for float values."""
1345 VARIANTS
= frozenset([Variant
.FLOAT
,
1349 DEFAULT_VARIANT
= Variant
.DOUBLE
1354 class BooleanField(Field
):
1355 """Field definition for boolean values."""
1357 VARIANTS
= frozenset([Variant
.BOOL
])
1359 DEFAULT_VARIANT
= Variant
.BOOL
1364 class BytesField(Field
):
1365 """Field definition for byte string values."""
1367 VARIANTS
= frozenset([Variant
.BYTES
])
1369 DEFAULT_VARIANT
= Variant
.BYTES
1374 class StringField(Field
):
1375 """Field definition for unicode string values."""
1377 VARIANTS
= frozenset([Variant
.STRING
])
1379 DEFAULT_VARIANT
= Variant
.STRING
1383 def validate_element(self
, value
):
1384 """Validate StringField allowing for str and unicode.
1387 ValidationError if a str value is not 7-bit ascii.
1389 # If value is str is it considered valid. Satisfies "required=True".
1390 if isinstance(value
, str):
1393 except UnicodeDecodeError, err
:
1396 except AttributeError:
1397 validation_error
= ValidationError(
1398 'Field encountered non-ASCII string %s: %s' % (value
,
1401 validation_error
= ValidationError(
1402 'Field %s encountered non-ASCII string %s: %s' % (self
.name
,
1405 validation_error
.field_name
= self
.name
1406 raise validation_error
1408 super(StringField
, self
).validate_element(value
)
1411 class MessageField(Field
):
1412 """Field definition for sub-message values.
1414 Message fields contain instance of other messages. Instances stored
1415 on messages stored on message fields are considered to be owned by
1416 the containing message instance and should not be shared between
1419 Message fields must be defined to reference a single type of message.
1420 Normally message field are defined by passing the referenced message
1421 class in to the constructor.
1423 It is possible to define a message field for a type that does not yet
1424 exist by passing the name of the message in to the constructor instead
1425 of a message class. Resolution of the actual type of the message is
1426 deferred until it is needed, for example, during message verification.
1427 Names provided to the constructor must refer to a class within the same
1428 python module as the class that is using it. Names refer to messages
1429 relative to the containing messages scope. For example, the two fields
1430 of OuterMessage refer to the same message type:
1432 class Outer(Message):
1434 inner_relative = MessageField('Inner', 1)
1435 inner_absolute = MessageField('Outer.Inner', 2)
1437 class Inner(Message):
1440 When resolving an actual type, MessageField will traverse the entire
1441 scope of nested messages to match a message name. This makes it easy
1442 for siblings to reference siblings:
1444 class Outer(Message):
1446 class Inner(Message):
1448 sibling = MessageField('Sibling', 1)
1450 class Sibling(Message):
1454 VARIANTS
= frozenset([Variant
.MESSAGE
])
1456 DEFAULT_VARIANT
= Variant
.MESSAGE
1468 message_type: Message type for field. Must be subclass of Message.
1469 number: Number of field. Must be unique per message class.
1470 required: Whether or not field is required. Mutually exclusive to
1472 repeated: Whether or not field is repeated. Mutually exclusive to
1474 variant: Wire-format variant hint.
1477 FieldDefinitionError when invalid message_type is provided.
1479 valid_type
= (isinstance(message_type
, basestring
) or
1480 (message_type
is not Message
and
1481 isinstance(message_type
, type) and
1482 issubclass(message_type
, Message
)))
1485 raise FieldDefinitionError('Invalid message class: %s' % message_type
)
1487 if isinstance(message_type
, basestring
):
1488 self
.__type
_name
= message_type
1491 self
.__type
= message_type
1493 super(MessageField
, self
).__init
__(number
,
1498 def __set__(self
, message_instance
, value
):
1499 """Set value on message.
1502 message_instance: Message instance to set value on.
1503 value: Value to set on message.
1505 message_type
= self
.type
1506 if isinstance(message_type
, type) and issubclass(message_type
, Message
):
1508 if value
and isinstance(value
, (list, tuple)):
1509 value
= [(message_type(**v
) if isinstance(v
, dict) else v
)
1511 elif isinstance(value
, dict):
1512 value
= message_type(**value
)
1513 super(MessageField
, self
).__set
__(message_instance
, value
)
1517 """Message type used for field."""
1518 if self
.__type
is None:
1519 message_type
= find_definition(self
.__type
_name
, self
.message_definition())
1520 if not (message_type
is not Message
and
1521 isinstance(message_type
, type) and
1522 issubclass(message_type
, Message
)):
1523 raise FieldDefinitionError('Invalid message class: %s' % message_type
)
1524 self
.__type
= message_type
1528 def message_type(self
):
1529 """Underlying message type used for serialization.
1531 Will always be a sub-class of Message. This is different from type
1532 which represents the python value that message_type is mapped to for
1537 def value_from_message(self
, message
):
1538 """Convert a message to a value instance.
1540 Used by deserializers to convert from underlying messages to
1541 value of expected user type.
1544 message: A message instance of type self.message_type.
1547 Value of self.message_type.
1549 if not isinstance(message
, self
.message_type
):
1550 raise DecodeError('Expected type %s, got %s: %r' %
1551 (self
.message_type
.__name
__,
1552 type(message
).__name
__,
1556 def value_to_message(self
, value
):
1557 """Convert a value instance to a message.
1559 Used by serializers to convert Python user types to underlying
1560 messages for transmission.
1563 value: A value of type self.type.
1566 An instance of type self.message_type.
1568 if not isinstance(value
, self
.type):
1569 raise EncodeError('Expected type %s, got %s: %r' %
1570 (self
.type.__name
__,
1571 type(value
).__name
__,
1576 class EnumField(Field
):
1577 """Field definition for enum values.
1579 Enum fields may have default values that are delayed until the associated enum
1580 type is resolved. This is necessary to support certain circular references.
1584 class Message1(Message):
1592 # This field default value will be validated when default is accessed.
1593 animal = EnumField('Message2.Animal', 1, default='HORSE')
1595 class Message2(Message):
1603 # This fields default value will be validated right away since Color is
1604 # already fully resolved.
1605 color = EnumField(Message1.Color, 1, default='RED')
1608 VARIANTS
= frozenset([Variant
.ENUM
])
1610 DEFAULT_VARIANT
= Variant
.ENUM
1612 def __init__(self
, enum_type
, number
, **kwargs
):
1616 enum_type: Enum type for field. Must be subclass of Enum.
1617 number: Number of field. Must be unique per message class.
1618 required: Whether or not field is required. Mutually exclusive to
1620 repeated: Whether or not field is repeated. Mutually exclusive to
1622 variant: Wire-format variant hint.
1623 default: Default value for field if not found in stream.
1626 FieldDefinitionError when invalid enum_type is provided.
1628 valid_type
= (isinstance(enum_type
, basestring
) or
1629 (enum_type
is not Enum
and
1630 isinstance(enum_type
, type) and
1631 issubclass(enum_type
, Enum
)))
1634 raise FieldDefinitionError('Invalid enum type: %s' % enum_type
)
1636 if isinstance(enum_type
, basestring
):
1637 self
.__type
_name
= enum_type
1640 self
.__type
= enum_type
1642 super(EnumField
, self
).__init
__(number
, **kwargs
)
1644 def validate_default_element(self
, value
):
1645 """Validate default element of Enum field.
1647 Enum fields allow for delayed resolution of default values when the type
1648 of the field has not been resolved. The default value of a field may be
1649 a string or an integer. If the Enum type of the field has been resolved,
1650 the default value is validated against that type.
1653 value: Value to validate.
1656 ValidationError if value is not expected message type.
1658 if isinstance(value
, (basestring
, int, long)):
1659 # Validation of the value does not happen for delayed resolution
1660 # enumerated types. Ignore if type is not yet resolved.
1665 super(EnumField
, self
).validate_default_element(value
)
1669 """Enum type used for field."""
1670 if self
.__type
is None:
1671 found_type
= find_definition(self
.__type
_name
, self
.message_definition())
1672 if not (found_type
is not Enum
and
1673 isinstance(found_type
, type) and
1674 issubclass(found_type
, Enum
)):
1675 raise FieldDefinitionError('Invalid enum type: %s' % found_type
)
1677 self
.__type
= found_type
1682 """Default for enum field.
1684 Will cause resolution of Enum type and unresolved default value.
1687 return self
.__resolved
_default
1688 except AttributeError:
1689 resolved_default
= super(EnumField
, self
).default
1690 if isinstance(resolved_default
, (basestring
, int, long)):
1691 resolved_default
= self
.type(resolved_default
)
1692 self
.__resolved
_default
= resolved_default
1693 return self
.__resolved
_default
1697 def find_definition(name
, relative_to
=None, importer
=__import__):
1698 """Find definition by name in module-space.
1700 The find algorthm will look for definitions by name relative to a message
1701 definition or by fully qualfied name. If no definition is found relative
1702 to the relative_to parameter it will do the same search against the container
1703 of relative_to. If relative_to is a nested Message, it will search its
1704 message_definition(). If that message has no message_definition() it will
1705 search its module. If relative_to is a module, it will attempt to look for
1706 the containing module and search relative to it. If the module is a top-level
1707 module, it will look for the a message using a fully qualified name. If
1708 no message is found then, the search fails and DefinitionNotFoundError is
1711 For example, when looking for any definition 'foo.bar.ADefinition' relative to
1712 an actual message definition abc.xyz.SomeMessage:
1714 find_definition('foo.bar.ADefinition', SomeMessage)
1716 It is like looking for the following fully qualified names:
1718 abc.xyz.SomeMessage. foo.bar.ADefinition
1719 abc.xyz. foo.bar.ADefinition
1720 abc. foo.bar.ADefinition
1723 When resolving the name relative to Message definitions and modules, the
1724 algorithm searches any Messages or sub-modules found in its path.
1725 Non-Message values are not searched.
1727 A name that begins with '.' is considered to be a fully qualified name. The
1728 name is always searched for from the topmost package. For example, assume
1734 Searching for '.xyz.SomeMessage' relative to 'abc' will resolve to
1735 'xyz.SomeMessage' and not 'abc.xyz.SomeMessage'. For this kind of name,
1736 the relative_to parameter is effectively ignored and always set to None.
1738 For more information about package name resolution, please see:
1740 http://code.google.com/apis/protocolbuffers/docs/proto.html#packages
1743 name: Name of definition to find. May be fully qualified or relative name.
1744 relative_to: Search for definition relative to message definition or module.
1745 None will cause a fully qualified name search.
1746 importer: Import function to use for resolving modules.
1749 Enum or Message class definition associated with name.
1752 DefinitionNotFoundError if no definition is found in any search path.
1755 if not (relative_to
is None or
1756 isinstance(relative_to
, types
.ModuleType
) or
1757 isinstance(relative_to
, type) and issubclass(relative_to
, Message
)):
1758 raise TypeError('relative_to must be None, Message definition or module. '
1759 'Found: %s' % relative_to
)
1761 name_path
= name
.split('.')
1763 # Handle absolute path reference.
1764 if not name_path
[0]:
1766 name_path
= name_path
[1:]
1769 """Performs a single iteration searching the path from relative_to.
1771 This is the function that searches up the path from a relative object.
1773 fully.qualified.object . relative.or.nested.Definition
1774 ---------------------------->
1777 this part of search --+
1780 Message or Enum at the end of name_path, else None.
1783 for node
in name_path
:
1784 # Look for attribute first.
1785 attribute
= getattr(next
, node
, None)
1787 if attribute
is not None:
1790 # If module, look for sub-module.
1791 if next
is None or isinstance(next
, types
.ModuleType
):
1795 module_name
= '%s.%s' % (next
.__name
__, node
)
1798 fromitem
= module_name
.split('.')[-1]
1799 next
= importer(module_name
, '', '', [str(fromitem
)])
1805 if (not isinstance(next
, types
.ModuleType
) and
1806 not (isinstance(next
, type) and
1807 issubclass(next
, (Message
, Enum
)))):
1813 found
= search_path()
1814 if isinstance(found
, type) and issubclass(found
, (Enum
, Message
)):
1817 # Find next relative_to to search against.
1819 # fully.qualified.object . relative.or.nested.Definition
1820 # <---------------------
1823 # does this part of search
1824 if relative_to
is None:
1825 # Fully qualified search was done. Nothing found. Fail.
1826 raise DefinitionNotFoundError('Could not find definition for %s'
1829 if isinstance(relative_to
, types
.ModuleType
):
1830 # Find parent module.
1831 module_path
= relative_to
.__name
__.split('.')[:-1]
1835 # Should not raise ImportError. If it does... weird and
1836 # unexepected. Propagate.
1837 relative_to
= importer(
1838 '.'.join(module_path
), '', '', [module_path
[-1]])
1839 elif (isinstance(relative_to
, type) and
1840 issubclass(relative_to
, Message
)):
1841 parent
= relative_to
.message_definition()
1843 last_module_name
= relative_to
.__module
__.split('.')[-1]
1844 relative_to
= importer(
1845 relative_to
.__module
__, '', '', [last_module_name
])
1847 relative_to
= parent