python/google/appengine/datastore/datastore_v4_validator.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21 """Validators for v4 datastore protocol buffers.
  22
  23 This module is internal and should not be used by client applications.
  24 """
  25
  26
  27
  28
  29
  30
  31
  32
  33
  34
  35
  36
  37
  38
  39
  40
  41 import re
  42
  43 from google.appengine.datastore import datastore_pbs
  44 from google.appengine.datastore import datastore_v4_pb
  45
  46
  47
  48 _DATASET_ID_PARTITION_STRING = (r'[a-z\d\-]{1,%d}'
  49                                 % datastore_pbs.MAX_DATASET_ID_SECTION_LENGTH)
  50
  51 _DATASET_ID_DOMAIN_STRING = (
  52     r'[a-z\d][a-z\d\-\.]{0,%d}'
  53     % (datastore_pbs.MAX_DATASET_ID_SECTION_LENGTH - 1))
  54
  55 _DATASET_ID_DISPLAY_STRING = (
  56     r'[a-z\d][a-z\d\-]{0,%d}'
  57     % (datastore_pbs.MAX_DATASET_ID_SECTION_LENGTH - 1))
  58
  59 _DATASET_ID_STRING = ('(?:(?:%s)~)?(?:(?:%s):)?(?:%s)'
  60                       % (_DATASET_ID_PARTITION_STRING,
  61                          _DATASET_ID_DOMAIN_STRING,
  62                          _DATASET_ID_DISPLAY_STRING))
  63
  64
  65 _DATASET_ID_RE = re.compile('^%s$' % _DATASET_ID_STRING)
  66
  67
  68 _RESERVED_NAME_RE = re.compile('^__(.*)__$')
  69
  70
  71 _PARTITION_ID_RE = re.compile(r'^[0-9A-Za-z\._\-]{0,%d}$'
  72                               % datastore_pbs.MAX_PARTITION_ID_LENGTH)
  73
  74
  75 _STRING_VALUE_MEANINGS = frozenset([
  76     datastore_pbs.MEANING_TEXT,
  77     datastore_pbs.MEANING_ATOM_CATEGORY,
  78     datastore_pbs.MEANING_URL,
  79     datastore_pbs.MEANING_ATOM_TITLE,
  80     datastore_pbs.MEANING_ATOM_CONTENT,
  81     datastore_pbs.MEANING_ATOM_SUMMARY,
  82     datastore_pbs.MEANING_ATOM_AUTHOR,
  83     datastore_pbs.MEANING_GD_EMAIL,
  84     datastore_pbs.MEANING_GD_IM,
  85     datastore_pbs.MEANING_GD_PHONENUMBER,
  86     datastore_pbs.MEANING_GD_POSTALADDRESS,
  87     ])
  88
  89
  90 _BLOB_VALUE_MEANINGS = frozenset([
  91     datastore_pbs.MEANING_BYTESTRING,
  92     datastore_pbs.MEANING_ZLIB,
  93     ])
  94
  95
  96 _ENTITY_VALUE_MEANINGS = frozenset([
  97     datastore_pbs.MEANING_GEORSS_POINT,
  98     datastore_pbs.MEANING_PREDEFINED_ENTITY_POINT,
  99     datastore_pbs.MEANING_PREDEFINED_ENTITY_USER,
 100     ])
 101
 102
 103 _USER_ENTITY_PROPERTY_MAP = {
 104     datastore_pbs.PROPERTY_NAME_EMAIL: 'string',
 105     datastore_pbs.PROPERTY_NAME_AUTH_DOMAIN: 'string',
 106     datastore_pbs.PROPERTY_NAME_USER_ID: 'string',
 107     datastore_pbs.PROPERTY_NAME_INTERNAL_ID: 'integer',
 108     datastore_pbs.PROPERTY_NAME_FEDERATED_IDENTITY: 'string',
 109     datastore_pbs.PROPERTY_NAME_FEDERATED_PROVIDER: 'string',
 110     }
 111
 112
 113 _USER_ENTITY_REQUIRED_PROPERTIES = frozenset([
 114     datastore_pbs.PROPERTY_NAME_EMAIL,
 115     datastore_pbs.PROPERTY_NAME_AUTH_DOMAIN,
 116     ])
 117
 118
 119 _POINT_ENTITY_PROPERTY_MAP = {
 120     datastore_pbs.PROPERTY_NAME_X: 'double',
 121     datastore_pbs.PROPERTY_NAME_Y: 'double',
 122     }
 123
 124
 125 _POINT_ENTITY_REQUIRED_PROPERTIES = frozenset([
 126     datastore_pbs.PROPERTY_NAME_X,
 127     datastore_pbs.PROPERTY_NAME_Y,
 128     ])
 129
 130
 131
 132 class ValidationError(Exception):
 133   """Raised when validation fails."""
 134   pass
 135
 136
 137 def _assert_condition(condition, message):
 138   """Asserts a validation condition and raises an error if it's not met.
 139
 140   Args:
 141     condition: (boolean) condition to enforce
 142     message: error message
 143
 144   Raises:
 145     ValidationError: if condition is not met
 146   """
 147   if not condition:
 148     raise ValidationError(message)
 149
 150
 151 def _assert_initialized(pb):
 152   """Asserts that a protocol buffer is initialized.
 153
 154   Args:
 155     pb: a protocol buffer
 156
 157   Raises:
 158     ValidationError: if protocol buffer is not initialized
 159   """
 160   errors = []
 161   if not pb.IsInitialized(errors):
 162     _assert_condition(False, '\n\t'.join(errors))
 163
 164
 165 def _assert_valid_utf8(string, desc):
 166   """Asserts that a string is valid UTF8.
 167
 168   Args:
 169     string: string to check
 170     desc: description of the string (used in error message)
 171
 172   Raises:
 173     ValidationError: if the string is not valid UTF8
 174   """
 175   _assert_condition(datastore_pbs.is_valid_utf8(string),
 176                     'The %s is not valid UTF-8.' % desc)
 177
 178
 179 def _assert_string_not_empty(string, desc):
 180   """Asserts that a string is not empty.
 181
 182   Args:
 183     string: string to check
 184     desc: description of the string (used in error message)
 185
 186   Raises:
 187     ValidationError: if the string is empty
 188   """
 189   _assert_condition(string, 'The %s is the empty string.' % desc)
 190
 191
 192 def _assert_string_not_reserved(string, desc):
 193   """Asserts that a string is not a reserved name.
 194
 195   Args:
 196     string: string to check
 197     desc: description of the string (used in error message)
 198
 199   Raises:
 200     ValidationError: if the string is a reserved name
 201   """
 202   _assert_condition(not _RESERVED_NAME_RE.match(string),
 203                     'The %s "%s" is reserved.'  % (desc, string))
 204
 205
 206 def _assert_string_not_too_long(string, max_length, desc):
 207   """Asserts that a string is within the maximum string size bounds.
 208
 209   Args:
 210     string: string to check
 211     max_length: max length of the string (inclusive)
 212     desc: description of the string (used in error message)
 213
 214   Raises:
 215     ValidationError: if the string is a reserved name
 216   """
 217   _assert_condition(len(string) <= max_length,
 218                     'The %s is longer than %d characters.' % (desc, max_length))
 219
 220
 221 class _ValidationConstraint(object):
 222   """Container for a set of validation constraints."""
 223
 224   def __init__(self, absent_key_allowed=False, incomplete_key_path_allowed=True,
 225                complete_key_path_allowed=False, reserved_key_allowed=False,
 226                reserved_property_name_allowed=False,
 227                meaning_index_only_allowed=False):
 228     self.__absent_key_allowed = absent_key_allowed
 229     self.__incomplete_key_path_allowed = incomplete_key_path_allowed
 230     self.__complete_key_path_allowed = complete_key_path_allowed
 231     self.__reserved_key_allowed = reserved_key_allowed
 232     self.__reserved_property_name_allowed = reserved_property_name_allowed
 233     self.__meaning_index_only_allowed = meaning_index_only_allowed
 234
 235   @property
 236   def absent_key_allowed(self):
 237     """Allow keys to be absent from entities."""
 238     return self.__absent_key_allowed
 239
 240   @property
 241   def incomplete_key_path_allowed(self):
 242     """Allow key paths to be incomplete."""
 243     return self.__incomplete_key_path_allowed
 244
 245   @property
 246   def complete_key_path_allowed(self):
 247     """Allow key paths to be complete."""
 248     return self.__complete_key_path_allowed
 249
 250   @property
 251   def reserved_key_allowed(self):
 252     """Allow reserved keys and reserved partition ids."""
 253     return self.__reserved_key_allowed
 254
 255   @property
 256   def reserved_property_name_allowed(self):
 257     """Allow reserved property names."""
 258     return self.__reserved_property_name_allowed
 259
 260   @property
 261   def meaning_index_only_allowed(self):
 262     """Allow the index only meaning."""
 263     return self.__meaning_index_only_allowed
 264
 265   def __hash__(self):
 266     return hash(id(self))
 267
 268   def __eq__(self, other):
 269     return self is other
 270
 271
 272
 273 READ = _ValidationConstraint(
 274     absent_key_allowed=False,
 275     incomplete_key_path_allowed=False,
 276     complete_key_path_allowed=True,
 277     reserved_key_allowed=True,
 278     reserved_property_name_allowed=True,
 279     meaning_index_only_allowed=True)
 280
 281
 282
 283 READ_ENTITY_IN_VALUE = _ValidationConstraint(
 284     absent_key_allowed=True,
 285     incomplete_key_path_allowed=True,
 286     complete_key_path_allowed=True,
 287     reserved_key_allowed=True,
 288     reserved_property_name_allowed=True,
 289     meaning_index_only_allowed=True)
 290
 291
 292 WRITE = _ValidationConstraint(
 293     absent_key_allowed=False,
 294     incomplete_key_path_allowed=False,
 295     complete_key_path_allowed=True,
 296     reserved_key_allowed=False,
 297     reserved_property_name_allowed=False,
 298     meaning_index_only_allowed=False)
 299
 300
 301 WRITE_ENTITY_IN_VALUE = _ValidationConstraint(
 302     absent_key_allowed=True,
 303     incomplete_key_path_allowed=True,
 304     complete_key_path_allowed=True,
 305     reserved_key_allowed=True,
 306     reserved_property_name_allowed=False,
 307     meaning_index_only_allowed=False)
 308
 309
 310 ALLOCATE_KEY_ID = _ValidationConstraint(
 311     absent_key_allowed=False,
 312     incomplete_key_path_allowed=True,
 313     complete_key_path_allowed=False,
 314     reserved_key_allowed=False,
 315     reserved_property_name_allowed=False,
 316     meaning_index_only_allowed=False)
 317
 318
 319 WRITE_AUTO_ID = _ValidationConstraint(
 320     absent_key_allowed=False,
 321     incomplete_key_path_allowed=True,
 322     complete_key_path_allowed=False,
 323     reserved_key_allowed=False,
 324     reserved_property_name_allowed=False,
 325     meaning_index_only_allowed=False)
 326
 327
 328 KEY_IN_VALUE = _ValidationConstraint(
 329     absent_key_allowed=False,
 330     incomplete_key_path_allowed=False,
 331     complete_key_path_allowed=True,
 332     reserved_key_allowed=True,
 333     reserved_property_name_allowed=True,
 334     meaning_index_only_allowed=False)
 335
 336
 337 _ENTITY_IN_VALUE_CONSTRAINTS = {
 338     READ: READ_ENTITY_IN_VALUE,
 339     READ_ENTITY_IN_VALUE: READ_ENTITY_IN_VALUE,
 340     WRITE: WRITE_ENTITY_IN_VALUE,
 341     WRITE_AUTO_ID: WRITE_ENTITY_IN_VALUE,
 342     WRITE_ENTITY_IN_VALUE: WRITE_ENTITY_IN_VALUE,
 343 }
 344
 345
 346 def _get_entity_in_value_constraint(constraint):
 347   """Returns the corresponding constraint for entities in values.
 348
 349   Args:
 350     constraint: a _ValidationConstraint
 351
 352   Returns:
 353     a _ValidationConstraint for entities in values
 354
 355   Raises:
 356     ValueError: if no corresponding constraint exists
 357   """
 358   if constraint not in _ENTITY_IN_VALUE_CONSTRAINTS:
 359     raise ValueError('No corresponding constraint for entities in values.')
 360   return _ENTITY_IN_VALUE_CONSTRAINTS[constraint]
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372 class _EntityValidator(object):
 373   """Validator for v4 entities and keys."""
 374
 375   def validate_keys(self, constraint, keys):
 376     """Validates a list of keys.
 377
 378     Args:
 379       constraint: a _ValidationConstraint to apply
 380       keys: a list of entity_v4_pb.Key objects
 381
 382     Raises:
 383       ValidationError: if any of the keys is invalid
 384     """
 385     for key in keys:
 386       self.validate_key(constraint, key)
 387
 388   def validate_key(self, constraint, key):
 389     """Validates a key.
 390
 391     Args:
 392       constraint: a _ValidationConstraint to apply
 393       key: an entity_v4_pb.Key
 394
 395     Raises:
 396       ValidationError: if the key is invalid
 397     """
 398     _assert_condition(key.has_partition_id(), 'Key is missing partition id.')
 399     self.validate_partition_id(constraint, key.partition_id())
 400     num_key_path_elements = len(key.path_element_list())
 401     _assert_condition(num_key_path_elements, 'Key path is empty.')
 402     _assert_condition((num_key_path_elements
 403                        <= datastore_pbs.MAX_KEY_PATH_LENGTH),
 404                       ('Key path has more than %d elements.'
 405                        % datastore_pbs.MAX_KEY_PATH_LENGTH))
 406     num_incomplete_elements = 0
 407     for path_element in key.path_element_list():
 408       _assert_valid_utf8(path_element.kind(), 'key path kind')
 409       kind = path_element.kind()
 410       self.validate_kind(constraint, kind)
 411       has_name = path_element.has_name()
 412       if path_element.has_id():
 413         _assert_condition(not has_name,
 414                           'Key path element has both id (%d) and name ("%s").'
 415                           % (path_element.id(), path_element.name()))
 416         _assert_condition(path_element.id(),
 417                           'Key path element has an id of 0.')
 418       else:
 419         if has_name:
 420           _assert_valid_utf8(path_element.name(), 'key path name')
 421           name = path_element.name()
 422           _assert_string_not_empty(name, 'key path name')
 423           _assert_string_not_too_long(name,
 424                                       datastore_pbs.MAX_INDEXED_STRING_CHARS,
 425                                       'key path name')
 426           if not constraint.reserved_key_allowed:
 427             _assert_string_not_reserved(name, 'key path name')
 428         else:
 429           num_incomplete_elements += 1
 430     final_element = key.path_element(num_key_path_elements - 1)
 431     final_element_complete = final_element.has_id() or final_element.has_name()
 432     if not constraint.complete_key_path_allowed:
 433       _assert_condition(not final_element_complete,
 434                         'Key path is complete: %s.'
 435                         % datastore_pbs.v4_key_to_string(key))
 436     if not constraint.incomplete_key_path_allowed:
 437       _assert_condition(final_element_complete,
 438                         'Key path is incomplete: %s.'
 439                         % datastore_pbs.v4_key_to_string(key))
 440     if final_element_complete:
 441       num_expected_incomplete = 0
 442     else:
 443       num_expected_incomplete = 1
 444     if num_incomplete_elements != num_expected_incomplete:
 445
 446       _assert_condition(False, 'Key path element is incomplete: %s.'
 447                         % datastore_pbs.v4_key_to_string(key))
 448
 449   def validate_partition_id(self, constraint, partition_id):
 450     """Validates a partition ID.
 451
 452     Args:
 453       constraint: a _ValidationConstraint to apply
 454       partition_id: a datastore_v4_pb.PartitionId
 455
 456     Raises:
 457       ValidationError: if the partition ID is invalid
 458     """
 459     _assert_condition(partition_id.has_dataset_id(),
 460                       'Partition id is missing dataset id.')
 461     if partition_id.has_dataset_id():
 462       self.validate_dataset_id(constraint, partition_id.dataset_id())
 463     if partition_id.has_namespace():
 464       self.validate_partition_id_dimension(constraint, partition_id.namespace(),
 465                                            'namespace')
 466
 467   def validate_dataset_id(self, constraint, dataset_id):
 468     """Validates a dataset ID.
 469
 470     Args:
 471       constraint: a _ValidationConstraint to apply
 472       dataset_id: dataset ID
 473
 474     Raises:
 475       ValidationError: if the partition ID dimension is invalid
 476     """
 477     _assert_valid_utf8(dataset_id, 'dataset id')
 478     _assert_string_not_empty(dataset_id, 'dataset id')
 479     _assert_string_not_too_long(dataset_id, datastore_pbs.MAX_DATASET_ID_LENGTH,
 480                                 'dataset id')
 481     _assert_condition(_DATASET_ID_RE.match(dataset_id),
 482                       'Illegal string "%s" in dataset id.' % dataset_id)
 483
 484
 485
 486     if not constraint.reserved_key_allowed:
 487       _assert_string_not_reserved(dataset_id, 'dataset id')
 488
 489   def validate_partition_id_dimension(self, constraint, partition_dimension,
 490                                       desc):
 491     """Validates a dimension (e.g. namespace) of a partition ID.
 492
 493     Should not be used for datasets (see validate_dataset).
 494
 495     Args:
 496       constraint: a _ValidationConstraint to apply
 497       partition_dimension: string representing one dimension of a partition ID
 498       desc: description of the dimension (used in error messages)
 499
 500     Raises:
 501       ValidationError: if the partition ID dimension is invalid
 502     """
 503     _assert_valid_utf8(partition_dimension, desc)
 504     _assert_string_not_empty(partition_dimension, desc)
 505     _assert_string_not_too_long(partition_dimension,
 506                                 datastore_pbs.MAX_PARTITION_ID_LENGTH, desc)
 507     if not constraint.reserved_key_allowed:
 508       _assert_string_not_reserved(partition_dimension, desc)
 509     _assert_condition(_PARTITION_ID_RE.match(partition_dimension),
 510                       'Illegal string "%s" in %s.' % (partition_dimension,
 511                                                       desc))
 512
 513   def validate_kind(self, constraint, kind):
 514     """Validates a kind.
 515
 516     Args:
 517       constraint: a _ValidationConstraint to apply
 518       kind: kind string
 519
 520     Raises:
 521       ValidationError: if the kind is invalid
 522     """
 523     _assert_string_not_empty(kind, 'kind')
 524     _assert_string_not_too_long(kind, datastore_pbs.MAX_INDEXED_STRING_CHARS,
 525                                 'kind')
 526     if not constraint.reserved_key_allowed:
 527       _assert_string_not_reserved(kind, 'kind')
 528
 529   def validate_entities(self, constraint, entities):
 530     """Validates a list of entities.
 531
 532     Args:
 533       constraint: a _ValidationConstraint to apply
 534       entities: a list of entity_v4_pb.Entity objects
 535
 536     Raises:
 537       ValidationError: if any of the entities is invalid
 538     """
 539     for entity in entities:
 540       self.validate_entity(constraint, entity)
 541
 542   def validate_entity(self, constraint, entity):
 543     """Validates an entity.
 544
 545     Args:
 546       constraint: a _ValidationConstraint to apply
 547       entity: an entity_v4_pb.Entity
 548
 549     Raises:
 550       ValidationError: if the entity is invalid
 551     """
 552     if entity.has_key():
 553       self.validate_key(constraint, entity.key())
 554     else:
 555       _assert_condition(constraint.absent_key_allowed,
 556                         'Entity is missing key.')
 557     property_names = set()
 558     for prop in entity.property_list():
 559       property_name = prop.name()
 560       _assert_condition(property_name not in property_names,
 561                         ('Entity has duplicate property name "%s".'
 562                          % property_name))
 563       property_names.add(property_name)
 564       self.validate_property(constraint, prop)
 565
 566   def validate_property(self, constraint, prop):
 567     """Validates a property.
 568
 569     Args:
 570       constraint: a _ValidationConstraint to apply
 571       prop: an entity_v4_pb.Property
 572
 573     Raises:
 574       ValidationError: if the property is invalid
 575     """
 576     _assert_valid_utf8(prop.name(), 'property name')
 577     property_name = prop.name()
 578     self.validate_property_name(constraint, property_name)
 579
 580     _assert_condition(not prop.has_deprecated_multi(),
 581                       'deprecated_multi field not supported.')
 582     _assert_condition(not prop.deprecated_value_list(),
 583                       'deprecated_value field not supported.')
 584     _assert_condition(prop.has_value(),
 585                       'Property "%s" has no value.' % property_name)
 586     self.validate_value(constraint, prop.value())
 587
 588   def validate_value(self, constraint, value):
 589     """Validates a value.
 590
 591     Args:
 592       constraint: a _ValidationConstraint to apply
 593       value: an entity_v4_pb.Value
 594
 595     Raises:
 596       ValidationError: if the value is invalid
 597     """
 598     self.__validate_value_union(value)
 599     if value.has_string_value():
 600       _assert_valid_utf8(value.string_value(), 'string value')
 601     elif value.has_blob_key_value():
 602       _assert_valid_utf8(value.blob_key_value(), 'blob key value')
 603     elif value.has_key_value():
 604       self.validate_key(KEY_IN_VALUE, value.key_value())
 605     elif value.has_entity_value():
 606       entity_in_value_constraint = _get_entity_in_value_constraint(constraint)
 607       self.validate_entity(entity_in_value_constraint, value.entity_value())
 608     elif value.list_value_list():
 609       _assert_condition(not value.has_indexed(),
 610                         ('A Value containing a list_value cannot specify '
 611                          'indexed.'))
 612       _assert_condition(not value.has_meaning(),
 613                         ('A Value containing a list_value cannot specify '
 614                          'a meaning.'))
 615       for sub_value in value.list_value_list():
 616         _assert_condition(not sub_value.list_value_list(),
 617                           ('list_value cannot contain a Value containing '
 618                            'another list_value.'))
 619         self.validate_value(constraint, sub_value)
 620     self.__validate_value_meaning_matches_union(value)
 621     self.__validate_value_meaning_constraints(constraint, value)
 622     self.__validate_value_index_constraints(value)
 623
 624   def __validate_value_union(self, value):
 625     """Validates that a value is a valid union.
 626
 627     Args:
 628       value: an entity_v4_pb.Value
 629
 630     Raises:
 631       ValidationError: if the value contains more than one type
 632     """
 633     num_sub_values = (value.has_boolean_value()
 634                       + value.has_integer_value()
 635                       + value.has_double_value()
 636                       + value.has_timestamp_microseconds_value()
 637                       + value.has_key_value()
 638                       + value.has_blob_key_value()
 639                       + value.has_string_value()
 640                       + value.has_blob_value()
 641                       + value.has_entity_value()
 642                       + bool(value.list_value_list()))
 643     _assert_condition(num_sub_values <= 1,
 644                       'Value has multiple <type>_value fields set.')
 645
 646   def __validate_value_meaning_matches_union(self, value):
 647     """Validates that a value's meaning matches its value type.
 648
 649     Args:
 650       value: an entity_v4_pb.Value
 651
 652     Raises:
 653       ValidationError: if the Value's value type does not match its meaning
 654     """
 655     if not value.has_meaning():
 656       return
 657     message = 'Value meaning %d does not match %s field.'
 658     meaning = value.meaning()
 659     if meaning in _STRING_VALUE_MEANINGS:
 660       _assert_condition(value.has_string_value(),
 661                         message % (meaning, 'string_value'))
 662     elif meaning in _BLOB_VALUE_MEANINGS:
 663       _assert_condition(value.has_blob_value(),
 664                         message % (meaning, 'blob_value'))
 665     elif meaning in _ENTITY_VALUE_MEANINGS:
 666       _assert_condition(value.has_entity_value(),
 667                         message % (meaning, 'entity_value'))
 668     elif meaning == datastore_pbs.MEANING_PERCENT:
 669       _assert_condition(value.has_integer_value(),
 670                         message % (meaning, 'integer_value'))
 671     elif meaning == datastore_pbs.MEANING_INDEX_ONLY:
 672       _assert_condition(not value.has_timestamp_microseconds_value(),
 673                         message % (meaning, 'timestamp_microseconds_value'))
 674       _assert_condition(not value.has_blob_key_value(),
 675                         message % (meaning, 'blob_key_value'))
 676       _assert_condition(not value.has_entity_value(),
 677                         message % (meaning, 'entity_value'))
 678     else:
 679       _assert_condition(False,
 680                         'Unknown value meaning %d' % meaning)
 681
 682   def __validate_value_meaning_constraints(self, constraint, value):
 683     """Checks constraints on values that result from their meaning.
 684
 685     For example, some meanings cause the length of a value to be constrained.
 686
 687     Args:
 688       constraint: a _ValidationConstraint to apply
 689       value: an entity_v4_pb.Value
 690
 691     Raises:
 692       ValidationError: if the value is invalid
 693     """
 694     if not value.has_meaning():
 695       return
 696     meaning = value.meaning()
 697     if meaning == datastore_pbs.MEANING_BYTESTRING:
 698       _assert_condition((len(value.blob_value())
 699                          <= datastore_pbs.MAX_INDEXED_BLOB_BYTES),
 700                         ('Blob value with meaning %d has more than '
 701                          'permitted %d bytes.'
 702                          % (meaning, datastore_pbs.MAX_INDEXED_BLOB_BYTES)))
 703     elif meaning in (datastore_pbs.MEANING_TEXT, datastore_pbs.MEANING_ZLIB):
 704       _assert_condition(not value.indexed(),
 705                         'Indexed value has meaning %d.' % meaning)
 706     elif meaning == datastore_pbs.MEANING_URL:
 707       _assert_condition((len(value.string_value())
 708                          <= datastore_pbs.MAX_URL_CHARS),
 709                         'URL value has more than permitted %d characters.'
 710                         % datastore_pbs.MAX_URL_CHARS)
 711     elif meaning == datastore_pbs.MEANING_PERCENT:
 712       _assert_condition((value.integer_value() >= 0
 713                          and value.integer_value() <= 100),
 714                         'Percent value outside permitted range [0, 100].')
 715     elif meaning == datastore_pbs.MEANING_GEORSS_POINT:
 716       property_map = self._validate_predefined_entity_value(
 717           value.entity_value(), 'geo point', _POINT_ENTITY_PROPERTY_MAP,
 718           _POINT_ENTITY_REQUIRED_PROPERTIES)
 719       latitude = property_map[datastore_pbs.PROPERTY_NAME_X].double_value()
 720       longitude = property_map[datastore_pbs.PROPERTY_NAME_Y].double_value()
 721       _assert_condition(abs(latitude) <= 90.0,
 722                         'Latitude outside permitted range [-90.0, 90.0].')
 723       _assert_condition(abs(longitude) <= 180.0,
 724                         'Longitude outside permitted range [-180.0, 180.0].')
 725     elif meaning == datastore_pbs.MEANING_PREDEFINED_ENTITY_POINT:
 726       self._validate_predefined_entity_value(value.entity_value(), 'point',
 727                                              _POINT_ENTITY_PROPERTY_MAP,
 728                                              _POINT_ENTITY_REQUIRED_PROPERTIES)
 729     elif meaning == datastore_pbs.MEANING_PREDEFINED_ENTITY_USER:
 730       self._validate_predefined_entity_value(value.entity_value(), 'user',
 731                                              _USER_ENTITY_PROPERTY_MAP,
 732                                              _USER_ENTITY_REQUIRED_PROPERTIES)
 733     elif meaning == datastore_pbs.MEANING_INDEX_ONLY:
 734       _assert_condition(constraint.meaning_index_only_allowed,
 735                         'Value has meaning %d.'
 736                         % datastore_pbs.MEANING_INDEX_ONLY)
 737
 738   def _validate_predefined_entity_value(self, entity, entity_name,
 739                                         allowed_property_map,
 740                                         required_properties):
 741     """Validates a predefined entity (e.g. a user or a point).
 742
 743     Args:
 744       entity: the predefined entity (an entity_v4_pb.Entity)
 745       entity_name: the name of the entity (used in error messages)
 746       allowed_property_map: a dict whose keys are property names allowed in
 747           the entity and values are the expected types of these properties
 748       required_properties: a list of required property names
 749
 750     Returns:
 751       a dict of entity_v4_pb.Value objects keyed by property name
 752
 753     Raises:
 754       ValidationError: if the entity is invalid
 755     """
 756     _assert_condition(not entity.has_key(),
 757                       'The %s entity has a key.' % entity_name)
 758     property_map = {}
 759     for prop in entity.property_list():
 760       property_name = prop.name()
 761       _assert_condition(property_name in allowed_property_map,
 762                         'The %s entity property "%s" is not allowed.'
 763                         % (entity_name, property_name))
 764       value = prop.value()
 765       hasser = 'has_%s_value' % allowed_property_map[property_name]
 766       _assert_condition(
 767           getattr(value, hasser)(),
 768           ('The %s entity property "%s" is the wrong type.'
 769            % (entity_name, property_name)))
 770       _assert_condition(not value.has_meaning(),
 771                         'The %s entity property "%s" has a meaning.'
 772                         % (entity_name, property_name))
 773       _assert_condition(not value.indexed(),
 774                         'The %s entity property "%s" is indexed.'
 775                         % (entity_name, property_name))
 776       property_map[property_name] = value
 777     for required_property_name in required_properties:
 778       _assert_condition(required_property_name in property_map,
 779                         'The %s entity is missing required property "%s".'
 780                         % (entity_name, required_property_name))
 781     return property_map
 782
 783   def __validate_value_index_constraints(self, value):
 784     """Checks constraints on values that result from their being indexed.
 785
 786     Args:
 787       value: an entity_v4_pb.Value
 788
 789     Raises:
 790       ValidationError: if the value is invalid
 791     """
 792     if not value.indexed():
 793       return
 794     if (value.has_string_value()
 795         and value.meaning() != datastore_pbs.MEANING_URL):
 796
 797       _assert_condition((len(value.string_value())
 798                          <= datastore_pbs.MAX_INDEXED_STRING_CHARS),
 799                         ('Indexed string value has more than %d permitted '
 800                          'characters.'
 801                          % datastore_pbs.MAX_INDEXED_STRING_CHARS))
 802     elif value.has_blob_value():
 803       _assert_condition((len(value.blob_value())
 804                          <= datastore_pbs.MAX_INDEXED_BLOB_BYTES),
 805                         ('Indexed blob value has more than %d permitted '
 806                          'bytes.' % datastore_pbs.MAX_INDEXED_BLOB_BYTES))
 807     elif value.has_entity_value():
 808       _assert_condition(value.has_meaning(),
 809                         'Entity value is indexed.')
 810
 811   def validate_property_name(self, constraint, property_name):
 812     """Validates a property name.
 813
 814     Args:
 815       constraint: a _ValidationConstraint to apply
 816       property_name: name of a property
 817
 818     Raises:
 819       ValidationError: if the property name is invalid
 820     """
 821     desc = 'property.name'
 822     _assert_string_not_empty(property_name, desc)
 823     _assert_string_not_too_long(property_name,
 824                                 datastore_pbs.MAX_INDEXED_STRING_CHARS, desc)
 825
 826
 827     if not constraint.reserved_property_name_allowed:
 828       _assert_string_not_reserved(property_name, desc)
 829
 830
 831
 832 __entity_validator = _EntityValidator()
 833
 834
 835 def get_entity_validator():
 836   """Validator for entities and keys."""
 837   return __entity_validator
 838
 839
 840
 841 class _QueryValidator(object):
 842   """Validator for queries."""
 843
 844   def __init__(self, entity_validator):
 845     self.__entity_validator = entity_validator
 846
 847   def validate_query(self, query, is_strong_read_consistency):
 848     """Validates a Query.
 849
 850     Args:
 851       query: a datastore_v4_pb.Query
 852       is_strong_read_consistency: whether the request containing the query
 853           requested strong read consistency
 854
 855     Raises:
 856       ValidationError: if the query is invalid
 857     """
 858     _assert_condition((not is_strong_read_consistency
 859                        or self._has_ancestor(query.filter())),
 860                       'Global queries do not support strong consistency.')
 861     if query.has_filter():
 862       self.validate_filter(query.filter())
 863     for kind_expression in query.kind_list():
 864       self.__validate_kind_expression(kind_expression)
 865     group_by_properties = set()
 866     for property_reference in query.group_by_list():
 867       self.__validate_property_reference(property_reference)
 868       group_by_properties.add(property_reference.name())
 869     for property_expression in query.projection_list():
 870       self.__validate_property_expression(property_expression,
 871                                           group_by_properties)
 872     for property_order in query.order_list():
 873       self.__validate_property_order(property_order)
 874
 875   def validate_filter(self, filt):
 876     """Validates a Filter.
 877
 878     Args:
 879       filt: a datastore_v4_pb.Filter
 880
 881     Raises:
 882       ValidationError: if the filter is invalid
 883     """
 884     _assert_condition((filt.has_composite_filter()
 885                        + filt.has_property_filter()
 886                        + filt.has_bounding_circle_filter()
 887                        + filt.has_bounding_box_filter() == 1),
 888                       'A filter must have exactly one of its fields set.')
 889     if filt.has_composite_filter():
 890       comp_filter = filt.composite_filter()
 891       _assert_condition(comp_filter.filter_list(),
 892                         'A composite filter must have at least one sub-filter.')
 893       for sub_filter in comp_filter.filter_list():
 894         self.validate_filter(sub_filter)
 895     elif filt.has_property_filter():
 896       prop_filter = filt.property_filter()
 897       self.__validate_property_reference(prop_filter.property())
 898       _assert_condition(prop_filter.value().indexed(),
 899                         'A filter value must be indexed.')
 900       self.__entity_validator.validate_value(READ,
 901                                              prop_filter.value())
 902
 903   def __validate_kind_expression(self, kind_expression):
 904     """Validates a KindExpression.
 905
 906     Args:
 907       kind_expression: a datastore_v4_pb.KindExpression
 908
 909     Raises:
 910       ValidationError: if the kind expression is invalid
 911     """
 912     _assert_valid_utf8(kind_expression.name(), 'kind')
 913     self.__entity_validator.validate_kind(READ,
 914                                           kind_expression.name())
 915
 916   def __validate_property_reference(self, property_reference):
 917     """Validates a PropertyReference.
 918
 919     Args:
 920       property_reference: a datastore_v4_pb.PropertyReference
 921
 922     Raises:
 923       ValidationError: if the property reference is invalid
 924     """
 925     _assert_valid_utf8(property_reference.name(), 'property name')
 926     self.__entity_validator.validate_property_name(READ,
 927                                                    property_reference.name())
 928
 929   def __validate_property_expression(self, property_expression,
 930                                      group_by_properties):
 931     """Validates a PropertyExpression.
 932
 933     Args:
 934       property_expression: a datastore_v4_pb.PropertyExpression
 935       group_by_properties: the set of property names specified as group by
 936           properties for the query
 937
 938     Raises:
 939       ValidationError: if the property expression is invalid
 940     """
 941     self.__validate_property_reference(property_expression.property())
 942     if not group_by_properties:
 943       _assert_condition(not property_expression.has_aggregation_function(),
 944                         'Aggregation function is not allowed without group by.')
 945     elif property_expression.property().name() in group_by_properties:
 946       _assert_condition(not property_expression.has_aggregation_function(),
 947                         ('Aggregation function is not allowed for properties '
 948                          'in group by: %s.'
 949                          % property_expression.property().name()))
 950     else:
 951       _assert_condition(property_expression.has_aggregation_function(),
 952                         ('Aggregation function is required for properties '
 953                          'not in group by: %s.'
 954                          % property_expression.property().name()))
 955
 956   def __validate_property_order(self, property_order):
 957     """Validates a PropertyOrder.
 958
 959     Args:
 960       property_order: a datastore_v4_pb.PropertyOrder
 961
 962     Raises:
 963       ValidationError: if the property expression is invalid
 964     """
 965     self.__validate_property_reference(property_order.property())
 966
 967   def _has_ancestor(self, filt):
 968     """Determines if a filter includes an ancestor filter.
 969
 970     Args:
 971       filt: a datastore_v4_pb.Filter
 972
 973     Returns:
 974       True if the filter includes an ancestor filter, False otherwise
 975     """
 976     if filt.has_property_filter():
 977       op = filt.property_filter().operator()
 978       name = filt.property_filter().property().name()
 979       return (op == datastore_v4_pb.PropertyFilter.HAS_ANCESTOR
 980               and name == datastore_pbs.PROPERTY_NAME_KEY)
 981     if filt.has_composite_filter():
 982       if (filt.composite_filter().operator()
 983           == datastore_v4_pb.CompositeFilter.AND):
 984         for sub_filter in filt.composite_filter().filter_list():
 985           if self._has_ancestor(sub_filter):
 986             return True
 987     return False
 988
 989
 990
 991 __query_validator = _QueryValidator(__entity_validator)
 992
 993
 994 def get_query_validator():
 995   """Validator for queries."""
 996   return __query_validator
 997
 998
 999 class _ServiceValidator(object):
1000   """Validator for request/response protos."""
1001
1002   def __init__(self, entity_validator, query_validator):
1003     self.__entity_validator = entity_validator
1004     self.__query_validator = query_validator
1005
1006   def validate_begin_transaction_req(self, req):
1007     """Validates a normalized BeginTransactionRequest.
1008
1009     Args:
1010       req: a datastore_v4_pb.BeginTransactionRequest
1011
1012     Raises:
1013       ValidationError: if the request is invalid
1014     """
1015     _assert_initialized(req)
1016
1017   def validate_rollback_req(self, req):
1018     """Validates a normalized RunQueryRequest.
1019
1020     Args:
1021       req: a datastore_v4_pb.RunQueryRequest
1022
1023     Raises:
1024       ValidationError: if the request is invalid
1025     """
1026     _assert_initialized(req)
1027
1028   def validate_commit_req(self, req):
1029     """Validates a normalized CommitRequest.
1030
1031     Args:
1032       req: a datastore_v4_pb.CommitRequest
1033
1034     Raises:
1035       ValidationError: if the request is invalid
1036     """
1037     _assert_initialized(req)
1038     if req.mode() == datastore_v4_pb.CommitRequest.TRANSACTIONAL:
1039       _assert_condition(req.has_transaction(),
1040                         'Transactional commit requires a transaction.')
1041     elif req.mode() == datastore_v4_pb.CommitRequest.NON_TRANSACTIONAL:
1042       _assert_condition(not req.has_transaction(),
1043                         ('Non-transactional commit cannot specify a '
1044                          'transaction.'))
1045     else:
1046       _assert_condition(False,
1047                         'Unknown commit mode: %d.' % req.mode())
1048     self.__validate_deprecated_mutation(req.deprecated_mutation())
1049
1050   def validate_run_query_req(self, req):
1051     """Validates a normalized RunQueryRequest.
1052
1053     Args:
1054       req: a normalized datastore_v4_pb.RunQueryRequest
1055
1056     Raises:
1057       ValidationError: if the request is invalid
1058     """
1059
1060
1061
1062     _assert_condition(not req.has_gql_query(), 'GQL not supported.')
1063     _assert_initialized(req)
1064     self.validate_read_options(req.read_options())
1065     self.__entity_validator.validate_partition_id(READ,
1066                                                   req.partition_id())
1067     _assert_condition(req.has_query(),
1068                       ('One of fields Query.query and Query.gql_query '
1069                        'must be set.'))
1070     self.__query_validator.validate_query(
1071         req.query(),
1072         (req.read_options().read_consistency()
1073          == datastore_v4_pb.ReadOptions.STRONG))
1074
1075   def validate_continue_query_req(self, req):
1076     _assert_initialized(req)
1077
1078   def validate_lookup_req(self, req):
1079     """Validates a LookupRequest.
1080
1081     Args:
1082       req: a datastore_v4_pb.LookupRequest
1083
1084     Raises:
1085       ValidationError: if the request is invalid
1086     """
1087     _assert_initialized(req)
1088     self.validate_read_options(req.read_options())
1089     self.__entity_validator.validate_keys(READ, req.key_list())
1090
1091   def validate_allocate_ids_req(self, req):
1092     """Validates an AllocateIdsRequest.
1093
1094     Args:
1095       req: a datastore_v4_pb.AllocateIdsRequest
1096
1097     Raises:
1098       ValidationError: if the request is invalid
1099     """
1100     _assert_initialized(req)
1101     _assert_condition(not req.allocate_list() or not req.reserve_list(),
1102                       'Cannot reserve and allocate ids in the same request.')
1103     self.__entity_validator.validate_keys(ALLOCATE_KEY_ID,
1104                                           req.allocate_list())
1105     self.__entity_validator.validate_keys(WRITE,
1106                                           req.reserve_list())
1107
1108   def validate_read_options(self, read_options):
1109     _assert_condition((not read_options.has_read_consistency()
1110                        or not read_options.has_transaction()),
1111                       ('Cannot specify both a read consistency and'
1112                        ' a transaction.'))
1113
1114   def __validate_deprecated_mutation(self, deprecated_mutation):
1115     self.__entity_validator.validate_entities(WRITE,
1116                                               deprecated_mutation.upsert_list())
1117     self.__entity_validator.validate_entities(WRITE,
1118                                               deprecated_mutation.update_list())
1119     self.__entity_validator.validate_entities(WRITE,
1120                                               deprecated_mutation.insert_list())
1121     self.__entity_validator.validate_entities(
1122         WRITE_AUTO_ID,
1123         deprecated_mutation.insert_auto_id_list())
1124     self.__entity_validator.validate_keys(WRITE,
1125                                           deprecated_mutation.delete_list())
1126
1127
1128
1129 __service_validator = _ServiceValidator(__entity_validator,
1130                                         __query_validator)
1131
1132
1133 def get_service_validator():
1134   """Returns a validator for v4 service request/response protos.
1135
1136   Returns:
1137     a _ServiceValidator
1138   """
1139   return __service_validator