python/lib/protorpc-1.0/protorpc/descriptor.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2010 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18 """Services descriptor definitions.
  19
  20 Contains message definitions and functions for converting
  21 service classes into transmittable message format.
  22
  23 Describing an Enum instance, Enum class, Field class or Message class will
  24 generate an appropriate descriptor object that describes that class.
  25 This message can itself be used to transmit information to clients wishing
  26 to know the description of an enum value, enum, field or message without
  27 needing to download the source code.  This format is also compatible with
  28 other, non-Python languages.
  29
  30 The descriptors are modeled to be binary compatible with:
  31
  32   http://code.google.com/p/protobuf/source/browse/trunk/src/google/protobuf/descriptor.proto
  33
  34 NOTE: The names of types and fields are not always the same between these
  35 descriptors and the ones defined in descriptor.proto.  This was done in order
  36 to make source code files that use these descriptors easier to read.  For
  37 example, it is not necessary to prefix TYPE to all the values in
  38 FieldDescriptor.Variant as is done in descriptor.proto FieldDescriptorProto.Type.
  39
  40 Example:
  41
  42   class Pixel(messages.Message):
  43
  44     x = messages.IntegerField(1, required=True)
  45     y = messages.IntegerField(2, required=True)
  46
  47     color = messages.BytesField(3)
  48
  49   # Describe Pixel class using message descriptor.
  50   fields = []
  51
  52   field = FieldDescriptor()
  53   field.name = 'x'
  54   field.number = 1
  55   field.label = FieldDescriptor.Label.REQUIRED
  56   field.variant = FieldDescriptor.Variant.INT64
  57   fields.append(field)
  58
  59   field = FieldDescriptor()
  60   field.name = 'y'
  61   field.number = 2
  62   field.label = FieldDescriptor.Label.REQUIRED
  63   field.variant = FieldDescriptor.Variant.INT64
  64   fields.append(field)
  65
  66   field = FieldDescriptor()
  67   field.name = 'color'
  68   field.number = 3
  69   field.label = FieldDescriptor.Label.OPTIONAL
  70   field.variant = FieldDescriptor.Variant.BYTES
  71   fields.append(field)
  72
  73   message = MessageDescriptor()
  74   message.name = 'Pixel'
  75   message.fields = fields
  76
  77   # Describing is the equivalent of building the above message.
  78   message == describe_message(Pixel)
  79
  80 Public Classes:
  81   EnumValueDescriptor: Describes Enum values.
  82   EnumDescriptor: Describes Enum classes.
  83   FieldDescriptor: Describes field instances.
  84   FileDescriptor: Describes a single 'file' unit.
  85   FileSet: Describes a collection of file descriptors.
  86   MessageDescriptor: Describes Message classes.
  87   MethodDescriptor: Describes a method of a service.
  88   ServiceDescriptor: Describes a services.
  89
  90 Public Functions:
  91   describe_enum_value: Describe an individual enum-value.
  92   describe_enum: Describe an Enum class.
  93   describe_field: Describe a Field definition.
  94   describe_file: Describe a 'file' unit from a Python module or object.
  95   describe_file_set: Describe a file set from a list of modules or objects.
  96   describe_message: Describe a Message definition.
  97   describe_method: Describe a Method definition.
  98   describe_service: Describe a Service definition.
  99 """
 100
 101 __author__ = 'rafek@google.com (Rafe Kaplan)'
 102
 103 import codecs
 104 import types
 105
 106 from . import messages
 107 from . import util
 108
 109
 110 __all__ = ['EnumDescriptor',
 111            'EnumValueDescriptor',
 112            'FieldDescriptor',
 113            'MessageDescriptor',
 114            'MethodDescriptor',
 115            'FileDescriptor',
 116            'FileSet',
 117            'ServiceDescriptor',
 118            'DescriptorLibrary',
 119
 120            'describe_enum',
 121            'describe_enum_value',
 122            'describe_field',
 123            'describe_message',
 124            'describe_method',
 125            'describe_file',
 126            'describe_file_set',
 127            'describe_service',
 128            'describe',
 129            'import_descriptor_loader',
 130           ]
 131
 132
 133 # NOTE: MessageField is missing because message fields cannot have
 134 # a default value at this time.
 135 # TODO(rafek): Support default message values.
 136 #
 137 # Map to functions that convert default values of fields of a given type
 138 # to a string.  The function must return a value that is compatible with
 139 # FieldDescriptor.default_value and therefore a unicode string.
 140 _DEFAULT_TO_STRING_MAP = {
 141     messages.IntegerField: unicode,
 142     messages.FloatField: unicode,
 143     messages.BooleanField: lambda value: value and u'true' or u'false',
 144     messages.BytesField: lambda value: codecs.escape_encode(value)[0],
 145     messages.StringField: lambda value: value,
 146     messages.EnumField: lambda value: unicode(value.number),
 147 }
 148
 149 _DEFAULT_FROM_STRING_MAP = {
 150     messages.IntegerField: int,
 151     messages.FloatField: float,
 152     messages.BooleanField: lambda value: value == u'true',
 153     messages.BytesField: lambda value: codecs.escape_decode(value)[0],
 154     messages.StringField: lambda value: value,
 155     messages.EnumField: int,
 156 }
 157
 158
 159 class EnumValueDescriptor(messages.Message):
 160   """Enum value descriptor.
 161
 162   Fields:
 163     name: Name of enumeration value.
 164     number: Number of enumeration value.
 165   """
 166
 167   # TODO(rafek): Why are these listed as optional in descriptor.proto.
 168   # Harmonize?
 169   name = messages.StringField(1, required=True)
 170   number = messages.IntegerField(2,
 171                                  required=True,
 172                                  variant=messages.Variant.INT32)
 173
 174
 175 class EnumDescriptor(messages.Message):
 176   """Enum class descriptor.
 177
 178   Fields:
 179     name: Name of Enum without any qualification.
 180     values: Values defined by Enum class.
 181   """
 182
 183   name = messages.StringField(1)
 184   values = messages.MessageField(EnumValueDescriptor, 2, repeated=True)
 185
 186
 187 class FieldDescriptor(messages.Message):
 188   """Field definition descriptor.
 189
 190   Enums:
 191     Variant: Wire format hint sub-types for field.
 192     Label: Values for optional, required and repeated fields.
 193
 194   Fields:
 195     name: Name of field.
 196     number: Number of field.
 197     variant: Variant of field.
 198     type_name: Type name for message and enum fields.
 199     default_value: String representation of default value.
 200   """
 201
 202   Variant = messages.Variant
 203
 204   class Label(messages.Enum):
 205     """Field label."""
 206
 207     OPTIONAL = 1
 208     REQUIRED = 2
 209     REPEATED = 3
 210
 211   name = messages.StringField(1, required=True)
 212   number = messages.IntegerField(3,
 213                                  required=True,
 214                                  variant=messages.Variant.INT32)
 215   label = messages.EnumField(Label, 4, default=Label.OPTIONAL)
 216   variant = messages.EnumField(Variant, 5)
 217   type_name = messages.StringField(6)
 218
 219   # For numeric types, contains the original text representation of the value.
 220   # For booleans, "true" or "false".
 221   # For strings, contains the default text contents (not escaped in any way).
 222   # For bytes, contains the C escaped value.  All bytes < 128 are that are
 223   #   traditionally considered unprintable are also escaped.
 224   default_value = messages.StringField(7)
 225
 226
 227 class MessageDescriptor(messages.Message):
 228   """Message definition descriptor.
 229
 230   Fields:
 231     name: Name of Message without any qualification.
 232     fields: Fields defined for message.
 233     message_types: Nested Message classes defined on message.
 234     enum_types: Nested Enum classes defined on message.
 235   """
 236
 237   name = messages.StringField(1)
 238   fields = messages.MessageField(FieldDescriptor, 2, repeated=True)
 239
 240   message_types = messages.MessageField(
 241     'protorpc.descriptor.MessageDescriptor', 3, repeated=True)
 242   enum_types = messages.MessageField(EnumDescriptor, 4, repeated=True)
 243
 244
 245 class MethodDescriptor(messages.Message):
 246   """Service method definition descriptor.
 247
 248   Fields:
 249     name: Name of service method.
 250     request_type: Fully qualified or relative name of request message type.
 251     response_type: Fully qualified or relative name of response message type.
 252   """
 253
 254   name = messages.StringField(1)
 255
 256   request_type = messages.StringField(2)
 257   response_type = messages.StringField(3)
 258
 259
 260 class ServiceDescriptor(messages.Message):
 261   """Service definition descriptor.
 262
 263   Fields:
 264     name: Name of Service without any qualification.
 265     methods: Remote methods of Service.
 266   """
 267
 268   name = messages.StringField(1)
 269
 270   methods = messages.MessageField(MethodDescriptor, 2, repeated=True)
 271
 272
 273 class FileDescriptor(messages.Message):
 274   """Description of file containing protobuf definitions.
 275
 276   Fields:
 277     package: Fully qualified name of package that definitions belong to.
 278     message_types: Message definitions contained in file.
 279     enum_types: Enum definitions contained in file.
 280     service_types: Service definitions contained in file.
 281   """
 282
 283   package = messages.StringField(2)
 284
 285   # TODO(rafek): Add dependency field
 286
 287   message_types = messages.MessageField(MessageDescriptor, 4, repeated=True)
 288   enum_types = messages.MessageField(EnumDescriptor, 5, repeated=True)
 289   service_types = messages.MessageField(ServiceDescriptor, 6, repeated=True)
 290
 291
 292 class FileSet(messages.Message):
 293   """A collection of FileDescriptors.
 294
 295   Fields:
 296     files: Files in file-set.
 297   """
 298
 299   files = messages.MessageField(FileDescriptor, 1, repeated=True)
 300
 301
 302 def describe_enum_value(enum_value):
 303   """Build descriptor for Enum instance.
 304
 305   Args:
 306     enum_value: Enum value to provide descriptor for.
 307
 308   Returns:
 309     Initialized EnumValueDescriptor instance describing the Enum instance.
 310   """
 311   enum_value_descriptor = EnumValueDescriptor()
 312   enum_value_descriptor.name = unicode(enum_value.name)
 313   enum_value_descriptor.number = enum_value.number
 314   return enum_value_descriptor
 315
 316
 317 def describe_enum(enum_definition):
 318   """Build descriptor for Enum class.
 319
 320   Args:
 321     enum_definition: Enum class to provide descriptor for.
 322
 323   Returns:
 324     Initialized EnumDescriptor instance describing the Enum class.
 325   """
 326   enum_descriptor = EnumDescriptor()
 327   enum_descriptor.name = enum_definition.definition_name().split('.')[-1]
 328
 329   values = []
 330   for number in enum_definition.numbers():
 331     value = enum_definition.lookup_by_number(number)
 332     values.append(describe_enum_value(value))
 333
 334   if values:
 335     enum_descriptor.values = values
 336
 337   return enum_descriptor
 338
 339
 340 def describe_field(field_definition):
 341   """Build descriptor for Field instance.
 342
 343   Args:
 344     field_definition: Field instance to provide descriptor for.
 345
 346   Returns:
 347     Initialized FieldDescriptor instance describing the Field instance.
 348   """
 349   field_descriptor = FieldDescriptor()
 350   field_descriptor.name = field_definition.name
 351   field_descriptor.number = field_definition.number
 352   field_descriptor.variant = field_definition.variant
 353
 354   if isinstance(field_definition, messages.EnumField):
 355     field_descriptor.type_name = field_definition.type.definition_name()
 356
 357   if isinstance(field_definition, messages.MessageField):
 358     field_descriptor.type_name = field_definition.message_type.definition_name()
 359
 360   if field_definition.default is not None:
 361     field_descriptor.default_value = _DEFAULT_TO_STRING_MAP[
 362         type(field_definition)](field_definition.default)
 363
 364   # Set label.
 365   if field_definition.repeated:
 366     field_descriptor.label = FieldDescriptor.Label.REPEATED
 367   elif field_definition.required:
 368     field_descriptor.label = FieldDescriptor.Label.REQUIRED
 369   else:
 370     field_descriptor.label = FieldDescriptor.Label.OPTIONAL
 371
 372   return field_descriptor
 373
 374
 375 def describe_message(message_definition):
 376   """Build descriptor for Message class.
 377
 378   Args:
 379     message_definition: Message class to provide descriptor for.
 380
 381   Returns:
 382     Initialized MessageDescriptor instance describing the Message class.
 383   """
 384   message_descriptor = MessageDescriptor()
 385   message_descriptor.name = message_definition.definition_name().split('.')[-1]
 386
 387   fields = sorted(message_definition.all_fields(),
 388                   key=lambda v: v.number)
 389   if fields:
 390     message_descriptor.fields = [describe_field(field) for field in fields]
 391
 392   try:
 393     nested_messages = message_definition.__messages__
 394   except AttributeError:
 395     pass
 396   else:
 397     message_descriptors = []
 398     for name in nested_messages:
 399       value = getattr(message_definition, name)
 400       message_descriptors.append(describe_message(value))
 401
 402     message_descriptor.message_types = message_descriptors
 403
 404   try:
 405     nested_enums = message_definition.__enums__
 406   except AttributeError:
 407     pass
 408   else:
 409     enum_descriptors = []
 410     for name in nested_enums:
 411       value = getattr(message_definition, name)
 412       enum_descriptors.append(describe_enum(value))
 413
 414     message_descriptor.enum_types = enum_descriptors
 415
 416   return message_descriptor
 417
 418
 419 def describe_method(method):
 420   """Build descriptor for service method.
 421
 422   Args:
 423     method: Remote service method to describe.
 424
 425   Returns:
 426     Initialized MethodDescriptor instance describing the service method.
 427   """
 428   method_info = method.remote
 429   descriptor = MethodDescriptor()
 430   descriptor.name = method_info.method.func_name
 431   descriptor.request_type = method_info.request_type.definition_name()
 432   descriptor.response_type = method_info.response_type.definition_name()
 433
 434   return descriptor
 435
 436
 437 def describe_service(service_class):
 438   """Build descriptor for service.
 439
 440   Args:
 441     service_class: Service class to describe.
 442
 443   Returns:
 444     Initialized ServiceDescriptor instance describing the service.
 445   """
 446   descriptor = ServiceDescriptor()
 447   descriptor.name = service_class.__name__
 448   methods = []
 449   remote_methods = service_class.all_remote_methods()
 450   for name in sorted(remote_methods.iterkeys()):
 451     if name == 'get_descriptor':
 452       continue
 453
 454     method = remote_methods[name]
 455     methods.append(describe_method(method))
 456   if methods:
 457     descriptor.methods = methods
 458
 459   return descriptor
 460
 461
 462 def describe_file(module):
 463   """Build a file from a specified Python module.
 464
 465   Args:
 466     module: Python module to describe.
 467
 468   Returns:
 469     Initialized FileDescriptor instance describing the module.
 470   """
 471   # May not import remote at top of file because remote depends on this
 472   # file
 473   # TODO(rafek): Straighten out this dependency.  Possibly move these functions
 474   # from descriptor to their own module.
 475   from . import remote
 476
 477   descriptor = FileDescriptor()
 478   descriptor.package = util.get_package_for_module(module)
 479
 480   if not descriptor.package:
 481     descriptor.package = None
 482
 483   message_descriptors = []
 484   enum_descriptors = []
 485   service_descriptors = []
 486
 487   # Need to iterate over all top level attributes of the module looking for
 488   # message, enum and service definitions.  Each definition must be itself
 489   # described.
 490   for name in sorted(dir(module)):
 491     value = getattr(module, name)
 492
 493     if isinstance(value, type):
 494       if issubclass(value, messages.Message):
 495         message_descriptors.append(describe_message(value))
 496
 497       elif issubclass(value, messages.Enum):
 498         enum_descriptors.append(describe_enum(value))
 499
 500       elif issubclass(value, remote.Service):
 501         service_descriptors.append(describe_service(value))
 502
 503   if message_descriptors:
 504     descriptor.message_types = message_descriptors
 505
 506   if enum_descriptors:
 507     descriptor.enum_types = enum_descriptors
 508
 509   if service_descriptors:
 510     descriptor.service_types = service_descriptors
 511
 512   return descriptor
 513
 514
 515 def describe_file_set(modules):
 516   """Build a file set from a specified Python modules.
 517
 518   Args:
 519     modules: Iterable of Python module to describe.
 520
 521   Returns:
 522     Initialized FileSet instance describing the modules.
 523   """
 524   descriptor = FileSet()
 525   file_descriptors = []
 526   for module in modules:
 527     file_descriptors.append(describe_file(module))
 528
 529   if file_descriptors:
 530     descriptor.files = file_descriptors
 531
 532   return descriptor
 533
 534
 535 def describe(value):
 536   """Describe any value as a descriptor.
 537
 538   Helper function for describing any object with an appropriate descriptor
 539   object.
 540
 541   Args:
 542     value: Value to describe as a descriptor.
 543
 544   Returns:
 545     Descriptor message class if object is describable as a descriptor, else
 546     None.
 547   """
 548   from . import remote
 549   if isinstance(value, types.ModuleType):
 550     return describe_file(value)
 551   elif callable(value) and hasattr(value, 'remote'):
 552     return describe_method(value)
 553   elif isinstance(value, messages.Field):
 554     return describe_field(value)
 555   elif isinstance(value, messages.Enum):
 556     return describe_enum_value(value)
 557   elif isinstance(value, type):
 558     if issubclass(value, messages.Message):
 559       return describe_message(value)
 560     elif issubclass(value, messages.Enum):
 561       return describe_enum(value)
 562     elif issubclass(value, remote.Service):
 563       return describe_service(value)
 564   return None
 565
 566
 567 @util.positional(1)
 568 def import_descriptor_loader(definition_name, importer=__import__):
 569   """Find objects by importing modules as needed.
 570
 571   A definition loader is a function that resolves a definition name to a
 572   descriptor.
 573
 574   The import finder resolves definitions to their names by importing modules
 575   when necessary.
 576
 577   Args:
 578     definition_name: Name of definition to find.
 579     importer: Import function used for importing new modules.
 580
 581   Returns:
 582     Appropriate descriptor for any describable type located by name.
 583
 584   Raises:
 585     DefinitionNotFoundError when a name does not refer to either a definition
 586     or a module.
 587   """
 588   # Attempt to import descriptor as a module.
 589   if definition_name.startswith('.'):
 590     definition_name = definition_name[1:]
 591   if not definition_name.startswith('.'):
 592     leaf = definition_name.split('.')[-1]
 593     if definition_name:
 594       try:
 595         module = importer(definition_name, '', '', [leaf])
 596       except ImportError:
 597         pass
 598       else:
 599         return describe(module)
 600
 601   try:
 602     # Attempt to use messages.find_definition to find item.
 603     return describe(messages.find_definition(definition_name,
 604                                              importer=__import__))
 605   except messages.DefinitionNotFoundError, err:
 606     # There are things that find_definition will not find, but if the parent
 607     # is loaded, its children can be searched for a match.
 608     split_name = definition_name.rsplit('.', 1)
 609     if len(split_name) > 1:
 610       parent, child = split_name
 611       try:
 612         parent_definition = import_descriptor_loader(parent, importer=importer)
 613       except messages.DefinitionNotFoundError:
 614         # Fall through to original error.
 615         pass
 616       else:
 617         # Check the parent definition for a matching descriptor.
 618         if isinstance(parent_definition, FileDescriptor):
 619           search_list = parent_definition.service_types or []
 620         elif isinstance(parent_definition, ServiceDescriptor):
 621           search_list = parent_definition.methods or []
 622         elif isinstance(parent_definition, EnumDescriptor):
 623           search_list = parent_definition.values or []
 624         elif isinstance(parent_definition, MessageDescriptor):
 625           search_list = parent_definition.fields or []
 626         else:
 627           search_list = []
 628
 629         for definition in search_list:
 630           if definition.name == child:
 631             return definition
 632
 633     # Still didn't find.  Reraise original exception.
 634     raise err
 635
 636
 637 class DescriptorLibrary(object):
 638   """A descriptor library is an object that contains known definitions.
 639
 640   A descriptor library contains a cache of descriptor objects mapped by
 641   definition name.  It contains all types of descriptors except for
 642   file sets.
 643
 644   When a definition name is requested that the library does not know about
 645   it can be provided with a descriptor loader which attempt to resolve the
 646   missing descriptor.
 647   """
 648
 649   @util.positional(1)
 650   def __init__(self,
 651                descriptors=None,
 652                descriptor_loader=import_descriptor_loader):
 653     """Constructor.
 654
 655     Args:
 656       descriptors: A dictionary or dictionary-like object that can be used
 657         to store and cache descriptors by definition name.
 658       definition_loader: A function used for resolving missing descriptors.
 659         The function takes a definition name as its parameter and returns
 660         an appropriate descriptor.  It may raise DefinitionNotFoundError.
 661     """
 662     self.__descriptor_loader = descriptor_loader
 663     self.__descriptors = descriptors or {}
 664
 665   def lookup_descriptor(self, definition_name):
 666     """Lookup descriptor by name.
 667
 668     Get descriptor from library by name.  If descriptor is not found will
 669     attempt to find via descriptor loader if provided.
 670
 671     Args:
 672       definition_name: Definition name to find.
 673
 674     Returns:
 675       Descriptor that describes definition name.
 676
 677     Raises:
 678       DefinitionNotFoundError if not descriptor exists for definition name.
 679     """
 680     try:
 681       return self.__descriptors[definition_name]
 682     except KeyError:
 683       pass
 684
 685     if self.__descriptor_loader:
 686       definition = self.__descriptor_loader(definition_name)
 687       self.__descriptors[definition_name] = definition
 688       return definition
 689     else:
 690       raise messages.DefinitionNotFoundError(
 691         'Could not find definition for %s' % definition_name)
 692
 693   def lookup_package(self, definition_name):
 694     """Determines the package name for any definition.
 695
 696     Determine the package that any definition name belongs to.  May check
 697     parent for package name and will resolve missing descriptors if provided
 698     descriptor loader.
 699
 700     Args:
 701       definition_name: Definition name to find package for.
 702     """
 703     while True:
 704       descriptor = self.lookup_descriptor(definition_name)
 705       if isinstance(descriptor, FileDescriptor):
 706         return descriptor.package
 707       else:
 708         index = definition_name.rfind('.')
 709         if index < 0:
 710           return None
 711         definition_name = definition_name[:index]