java/src/main/com/google/appengine/api/datastore/DataTypeTranslator.java

   1 // Copyright 2007 Google Inc. All rights reserved.
   2
   3 package com.google.appengine.api.datastore;
   4
   5 import com.google.appengine.api.blobstore.BlobKey;
   6 import com.google.appengine.api.datastore.Entity.UnindexedValue;
   7 import com.google.appengine.api.datastore.Entity.WrappedValue;
   8 import com.google.appengine.api.datastore.Entity.WrappedValueImpl;
   9 import com.google.appengine.api.users.User;
  10 import com.google.apphosting.api.ApiProxy;
  11 import com.google.common.collect.Lists;
  12 import com.google.common.collect.Maps;
  13 import com.google.datastore.v1beta3.EntityOrBuilder;
  14 import com.google.datastore.v1beta3.Key.PathElement;
  15 import com.google.datastore.v1beta3.Key.PathElement.IdTypeCase;
  16 import com.google.datastore.v1beta3.KeyOrBuilder;
  17 import com.google.datastore.v1beta3.PartitionId;
  18 import com.google.datastore.v1beta3.PartitionIdOrBuilder;
  19 import com.google.datastore.v1beta3.Value;
  20 import com.google.datastore.v1beta3.Value.NullValue;
  21 import com.google.datastore.v1beta3.Value.ValueTypeCase;
  22 import com.google.datastore.v1beta3.ValueOrBuilder;
  23 import com.google.datastore.v1beta3.client.DatastoreHelper;
  24 import com.google.io.protocol.ProtocolSupport;
  25 import com.google.protobuf.ByteString;
  26 import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
  27 import com.google.storage.onestore.v3.OnestoreEntity.Path;
  28 import com.google.storage.onestore.v3.OnestoreEntity.Path.Element;
  29 import com.google.storage.onestore.v3.OnestoreEntity.Property;
  30 import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning;
  31 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue;
  32 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue;
  33 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValuePathElement;
  34 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.UserValue;
  35 import com.google.storage.onestore.v3.OnestoreEntity.Reference;
  36
  37 import java.util.ArrayList;
  38 import java.util.Arrays;
  39 import java.util.Collection;
  40 import java.util.Collections;
  41 import java.util.Date;
  42 import java.util.HashMap;
  43 import java.util.List;
  44 import java.util.Map;
  45
  46 /**
  47  * {@code DataTypeTranslator} is a utility class for converting
  48  * between the data store's {@code Property} protocol buffers and the
  49  * user-facing classes ({@code String}, {@code User}, etc.).
  50  *
  51  */
  52 public final class DataTypeTranslator  {
  53   /**
  54    * This key points to an (optional) map from project id to app id in the
  55    * {@link com.google.apphosting.api.ApiProxy.Environment} attribute map.
  56    */
  57   static final String ADDITIONAL_APP_IDS_MAP_ATTRIBUTE_KEY =
  58       "com.google.appengine.datastore.DataTypeTranslator.AdditionalAppIdsMap";
  59
  60   private static final RawValueType RAW_VALUE_TYPE = new RawValueType();
  61   /**
  62    * The list of supported types.
  63    *
  64    * Note: If you're going to modify this list, also update
  65    * DataTypeUtils. We're not building {@link DataTypeUtils#getSupportedTypes}
  66    * directly from this typeMap, because we want {@link DataTypeUtils} to be
  67    * translatable by GWT, so that {@link Entity Entities} can be easily sent
  68    * via GWT RPC.  Also, if you add a type here that is not immutable you'll
  69    * need to add special handling for it in {@link Entity#clone()}.
  70    */
  71   private static final Map<Class<?>, Type<?>> typeMap = Maps.newHashMap();
  72   static {
  73     typeMap.put(RawValue.class, RAW_VALUE_TYPE);
  74
  75     typeMap.put(Float.class, new DoubleType());
  76     typeMap.put(Double.class, new DoubleType());
  77
  78     typeMap.put(Byte.class, new Int64Type());
  79     typeMap.put(Short.class, new Int64Type());
  80     typeMap.put(Integer.class, new Int64Type());
  81     typeMap.put(Long.class, new Int64Type());
  82     typeMap.put(Date.class, new DateType());
  83     typeMap.put(Rating.class, new RatingType());
  84
  85     typeMap.put(String.class, new StringType());
  86     typeMap.put(Link.class, new LinkType());
  87     typeMap.put(ShortBlob.class, new ShortBlobType());
  88     typeMap.put(Category.class, new CategoryType());
  89     typeMap.put(PhoneNumber.class, new PhoneNumberType());
  90     typeMap.put(PostalAddress.class, new PostalAddressType());
  91     typeMap.put(Email.class, new EmailType());
  92     typeMap.put(IMHandle.class, new IMHandleType());
  93     typeMap.put(BlobKey.class, new BlobKeyType());
  94     typeMap.put(Blob.class, new BlobType());
  95     typeMap.put(Text.class, new TextType());
  96     typeMap.put(EmbeddedEntity.class, new EmbeddedEntityType());
  97
  98     typeMap.put(Boolean.class, new BoolType());
  99     typeMap.put(User.class, new UserType());
 100     typeMap.put(Key.class, new KeyType());
 101     typeMap.put(GeoPt.class, new GeoPtType());
 102
 103     assert typeMap.keySet().equals(DataTypeUtils.getSupportedTypes())
 104         : "Warning:  DataTypeUtils and DataTypeTranslator do not agree "
 105         + "about supported classes: " + typeMap.keySet() + " vs. "
 106         + DataTypeUtils.getSupportedTypes();
 107   }
 108
 109   /**
 110    * A map with the {@link Comparable} classes returned by all the instances of
 111    * {@link Type#asComparable(Object)} as keys and the pb code point as the value.
 112    * Used for comparing values that don't map to the same pb code point.
 113    */
 114   private static final
 115   Map<Class<? extends Comparable<?>>, Integer> comparableTypeMap =
 116       new HashMap<Class<? extends Comparable<?>>, Integer>();
 117
 118   static {
 119     comparableTypeMap.put(ComparableByteArray.class, PropertyValue.kstringValue);
 120     comparableTypeMap.put(Long.class, PropertyValue.kint64Value);
 121     comparableTypeMap.put(Double.class, PropertyValue.kdoubleValue);
 122     comparableTypeMap.put(Boolean.class, PropertyValue.kbooleanValue);
 123     comparableTypeMap.put(User.class, PropertyValue.kUserValueGroup);
 124     comparableTypeMap.put(Key.class, PropertyValue.kReferenceValueGroup);
 125     comparableTypeMap.put(GeoPt.class, PropertyValue.kPointValueGroup);
 126   }
 127
 128   /**
 129    * Add all of the properties in the specified map to an {@code EntityProto}.
 130    * This involves determining the type of each property and creating the
 131    * proper type-specific protocol buffer.
 132    *
 133    * If the property value is an {@link UnindexedValue}, or if it's a
 134    * type that is never indexed, e.g. {@code Text} and {@code Blob}, it's
 135    * added to {@code EntityProto.raw_property}. Otherwise it's added to
 136    * {@code EntityProto.property}.
 137    *
 138    * @param map A not {@code null} map of all the properties which will
 139    * be set on {@code proto}
 140    * @param proto A not {@code null} protocol buffer
 141    */
 142   public static void addPropertiesToPb(Map<String, Object> map, EntityProto proto) {
 143     for (Map.Entry<String, Object> entry : map.entrySet()) {
 144       String name = entry.getKey();
 145
 146       boolean forceIndexedEmbeddedEntity = false;
 147       boolean indexed = true;
 148       Object value = entry.getValue();
 149
 150       if (entry.getValue() instanceof WrappedValue) {
 151         WrappedValue wrappedValue = (WrappedValue) entry.getValue();
 152         forceIndexedEmbeddedEntity = wrappedValue.getForceIndexedEmbeddedEntity();
 153         indexed = wrappedValue.isIndexed();
 154         value = wrappedValue.getValue();
 155       }
 156
 157       if (value instanceof Collection<?>) {
 158         Collection<?> values = (Collection<?>) value;
 159         if (values.isEmpty()) {
 160           addPropertyToPb(name, null, indexed, forceIndexedEmbeddedEntity, false, proto);
 161         } else {
 162           for (Object listValue : values) {
 163             addPropertyToPb(name, listValue, indexed, forceIndexedEmbeddedEntity, true, proto);
 164           }
 165         }
 166       } else {
 167         addPropertyToPb(name, value, indexed, forceIndexedEmbeddedEntity, false, proto);
 168       }
 169     }
 170   }
 171
 172   /**
 173    * Adds a property to {@code entity}.
 174    *
 175    * @param name the property name
 176    * @param value the property value
 177    * @param indexed whether this property should be indexed. This may be
 178    * overriden by property types like Blob and Text that are never indexed.
 179    * @param multiple whether this property has multiple values
 180    * @param entity the entity to populate
 181    */
 182   private static void addPropertyToPb(String name, Object value, boolean indexed,
 183       boolean forceIndexedEmbeddedEntity, boolean multiple, EntityProto entity) {
 184     Property property = new Property();
 185     property.setName(name);
 186     property.setMultiple(multiple);
 187     PropertyValue newValue = property.getMutableValue();
 188     if (value != null) {
 189       Type<?> type = getType(value.getClass());
 190       Meaning meaning = type.getV3Meaning();
 191       if (meaning != property.getMeaningEnum()) {
 192         property.setMeaning(meaning);
 193       }
 194       boolean indexable = type.toV3Value(value, newValue);
 195       if (!forceIndexedEmbeddedEntity || !(value instanceof EmbeddedEntity)) {
 196         indexed &= indexable;
 197       }
 198     }
 199     if (!indexed) {
 200       entity.addRawProperty(property);
 201     } else {
 202       entity.addProperty(property);
 203     }
 204   }
 205
 206   static PropertyValue toV3Value(Object value) {
 207     PropertyValue propertyValue = new PropertyValue();
 208     if (value != null) {
 209       getType(value.getClass()).toV3Value(value, propertyValue);
 210     }
 211     return propertyValue;
 212   }
 213
 214   /**
 215    * Copy all of the indexed properties present on {@code proto} into {@code map}.
 216    */
 217   public static void extractIndexedPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 218     for (Property property : proto.propertys()) {
 219       addPropertyToMap(property, true, map);
 220     }
 221   }
 222
 223   /**
 224    * Copy all of the unindexed properties present on {@code proto} into {@code map}.
 225    */
 226   private static void extractUnindexedPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 227     for (Property property : proto.rawPropertys()) {
 228       addPropertyToMap(property, false, map);
 229     }
 230   }
 231
 232   /**
 233    * Copy all of the properties present on {@code proto} into {@code map}.
 234    */
 235   public static void extractPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 236     extractIndexedPropertiesFromPb(proto, map);
 237     extractUnindexedPropertiesFromPb(proto, map);
 238   }
 239
 240   /**
 241    * Copy all of the implicit properties present on {@code proto} into {@code map}.
 242    */
 243   public static void extractImplicitPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 244     for (Property property : getImplicitProperties(proto)) {
 245       addPropertyToMap(property, true, map);
 246     }
 247   }
 248
 249   private static Iterable<Property> getImplicitProperties(EntityProto proto) {
 250     return Collections.singleton(buildImplicitKeyProperty(proto));
 251   }
 252
 253   private static Property buildImplicitKeyProperty(EntityProto proto) {
 254     Property keyProp = new Property();
 255     keyProp.setName(Entity.KEY_RESERVED_PROPERTY);
 256     PropertyValue propVal = new PropertyValue();
 257     propVal.setReferenceValue(KeyType.toReferenceValue(proto.getKey()));
 258     keyProp.setValue(propVal);
 259     return keyProp;
 260   }
 261
 262   /**
 263    * Locates and returns all indexed properties with the given name on the
 264    * given proto. If there are a mix of matching multiple and non-multiple
 265    * properties, the collection will contain the first non-multiple property.
 266    * @return A list, potentially empty, containing matching properties.
 267    */
 268   public static Collection<Property> findIndexedPropertiesOnPb(
 269       EntityProto proto, String propertyName) {
 270     if (propertyName.equals(Entity.KEY_RESERVED_PROPERTY)) {
 271       return Collections.singleton(buildImplicitKeyProperty(proto));
 272     }
 273     List<Property> matchingMultipleProps = new ArrayList<>();
 274     for (Property prop : proto.propertys()) {
 275       if (prop.getName().equals(propertyName)) {
 276         if (!prop.isMultiple()) {
 277           return Collections.singleton(prop);
 278         } else {
 279           matchingMultipleProps.add(prop);
 280         }
 281       }
 282     }
 283     return matchingMultipleProps;
 284   }
 285
 286   private static void addPropertyToMap(Property property, boolean indexed,
 287       Map<String, Object> map) {
 288     String name = property.getName();
 289     Object value = getPropertyValue(property);
 290
 291     if (property.isMultiple()) {
 292       @SuppressWarnings({"unchecked"})
 293       List<Object> results = (List<Object>) PropertyContainer.unwrapValue(map.get(name));
 294       if (results == null) {
 295         results = new ArrayList<Object>();
 296         map.put(name, indexed ? results : new UnindexedValue(results));
 297       }
 298       if (indexed && value instanceof EmbeddedEntity) {
 299         map.put(name, new WrappedValueImpl(results, true, true));
 300       }
 301       results.add(value);
 302     } else {
 303       if (indexed && value instanceof EmbeddedEntity) {
 304         value = new WrappedValueImpl(value, true, true);
 305       } else if (!indexed) {
 306         value = new UnindexedValue(value);
 307       }
 308       map.put(name, value);
 309     }
 310   }
 311
 312   /**
 313    * Returns the value for the property as its canonical type.
 314    *
 315    * @param property a not {@code null} property
 316    * @return {@code null} if no value was set for {@code property}
 317    */
 318   public static Object getPropertyValue(Property property) {
 319     PropertyValue value = property.getValue();
 320     for (Type<?> type : typeMap.values()) {
 321       if (type.isType(property.getMeaningEnum(), value)) {
 322         return type.getValue(value);
 323       }
 324     }
 325     return null;
 326   }
 327
 328   /**
 329    * @see #addPropertiesToPb(Map, EntityProto)
 330    */
 331   static void addPropertiesToPb(Map<String, Object> map,
 332       com.google.datastore.v1beta3.Entity.Builder proto) {
 333     for (Map.Entry<String, Object> entry : map.entrySet()) {
 334       proto.getMutableProperties().put(entry.getKey(), toV1Value(entry.getValue()).build());
 335     }
 336   }
 337
 338   /**
 339    * Copy all of the properties present on {@code proto} into {@code map}.
 340    *
 341    * <p>Cloud Datastore v1 entity must know if the proto came from an index-only query as the User
 342    * type overwrites the INDEX_ONLY meaning.
 343    *
 344    * @param proto the proto from which to extract properties
 345    * @param indexOnly if the proto is from an index only query (a projection query)
 346    * @param map the map to populate
 347    */
 348   static void extractPropertiesFromPb(EntityOrBuilder proto, boolean indexOnly,
 349       Map<String, Object> map) {
 350     if (indexOnly) {
 351       for (Map.Entry<String, Value> prop : proto.getProperties().entrySet()) {
 352         map.put(prop.getKey(), new RawValue(prop.getValue()));
 353       }
 354     } else {
 355       for (Map.Entry<String, Value> prop : proto.getProperties().entrySet()) {
 356         addPropertyToMap(prop.getKey(), prop.getValue(), map);
 357       }
 358     }
 359   }
 360
 361   static Value.Builder toV1Value(
 362       Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 363     if (value == null) {
 364       Value.Builder builder = Value.newBuilder();
 365       builder.setNullValue(NullValue.NULL_VALUE);
 366       builder.setExcludeFromIndexes(!indexed);
 367       return builder;
 368     }
 369     return getType(value.getClass()).toV1Value(value, indexed, forceIndexedEmbeddedEntity);
 370   }
 371
 372   private static Value.Builder toV1Value(Object value) {
 373     boolean indexed = true;
 374     boolean forceIndexedEmbeddedEntity = false;
 375
 376     if (value instanceof WrappedValue) {
 377       WrappedValue wrappedValue = (WrappedValue) value;
 378       indexed = wrappedValue.isIndexed();
 379       forceIndexedEmbeddedEntity = wrappedValue.getForceIndexedEmbeddedEntity();
 380       value = wrappedValue.getValue();
 381     }
 382
 383     if (value instanceof Collection<?>) {
 384       Collection<?> values = (Collection<?>) value;
 385       if (values.isEmpty()) {
 386         return toV1Value(null, indexed, forceIndexedEmbeddedEntity);
 387       } else {
 388         Value.Builder valueBuilder = Value.newBuilder();
 389         for (Object listValue : values) {
 390           valueBuilder.getArrayValueBuilder().addValues(
 391               toV1Value(listValue, indexed, forceIndexedEmbeddedEntity));
 392         }
 393         return valueBuilder;
 394       }
 395     } else {
 396       return toV1Value(value, indexed, forceIndexedEmbeddedEntity);
 397     }
 398   }
 399
 400   private static void addPropertyToMap(String name, Value value, Map<String, Object> map) {
 401     boolean isOrContainsIndexedEntityValue = false;
 402     boolean indexed = false;
 403     Object result;
 404
 405     if (value.getArrayValue().getValuesCount() > 0) {
 406       ArrayList<Object> resultList =
 407           new ArrayList<Object>(value.getArrayValue().getValuesCount());
 408       for (Value subValue : value.getArrayValue().getValuesList()) {
 409         if (subValue.getValueTypeCase() == ValueTypeCase.ARRAY_VALUE) {
 410           throw new IllegalArgumentException("Invalid Entity PB: list within a list.");
 411         }
 412         result = getValue(subValue);
 413         if (!subValue.getExcludeFromIndexes()) {
 414           indexed = true;
 415           if (result instanceof EmbeddedEntity) {
 416             isOrContainsIndexedEntityValue = true;
 417           }
 418         }
 419         resultList.add(result);
 420       }
 421       result = resultList;
 422     } else {
 423       indexed = !value.getExcludeFromIndexes();
 424       result = getValue(value);
 425       if (indexed && result instanceof EmbeddedEntity) {
 426         isOrContainsIndexedEntityValue = true;
 427       }
 428     }
 429
 430     if (isOrContainsIndexedEntityValue) {
 431       result = new WrappedValueImpl(result, true, true);
 432     } else if (!indexed) {
 433       result = new UnindexedValue(result);
 434     }
 435
 436     map.put(name, result);
 437   }
 438
 439   private static Object getValue(Value value) {
 440     for (Type<?> type : typeMap.values()) {
 441       if (type.isType(value)) {
 442         return type.getValue(value);
 443       }
 444     }
 445     return null;
 446   }
 447
 448   private static Meaning getV3MeaningOf(ValueOrBuilder value) {
 449     return Meaning.valueOf(value.getMeaning());
 450   }
 451
 452   private static AppIdNamespace toAppIdNamespace(PartitionIdOrBuilder partitionId) {
 453     if (partitionId.getProjectId().equals(DatastoreApiHelper.getCurrentProjectId())) {
 454       return new AppIdNamespace(DatastoreApiHelper.getCurrentAppId(), partitionId.getNamespaceId());
 455     }
 456     Map<String, String> additionalProjectIdToAppIdMap = getAdditionalProjectIdToAppIdMap();
 457     if (additionalProjectIdToAppIdMap.containsKey(partitionId.getProjectId())) {
 458       return new AppIdNamespace(additionalProjectIdToAppIdMap.get(partitionId.getProjectId()),
 459           partitionId.getNamespaceId());
 460     } else {
 461       throw new IllegalStateException(String.format(
 462           "Could not determine app id corresponding to project id \"%s\". Please add the app id "
 463           + "to %s.",
 464           partitionId.getProjectId(), RemoteCloudDatastoreV1Proxy.ADDITIONAL_APP_IDS_VAR));
 465     }
 466   }
 467
 468   @SuppressWarnings("unchecked")
 469   private static Map<String, String> getAdditionalProjectIdToAppIdMap() {
 470     if (ApiProxy.getCurrentEnvironment() != null) {
 471       Object attribute = ApiProxy.getCurrentEnvironment().getAttributes()
 472           .get(ADDITIONAL_APP_IDS_MAP_ATTRIBUTE_KEY);
 473       if (attribute != null) {
 474         return (Map<String, String>) attribute;
 475       }
 476     }
 477     return Collections.emptyMap();
 478   }
 479
 480   private static PartitionId.Builder toV1PartitionId(AppIdNamespace appNs) {
 481     PartitionId.Builder builder = PartitionId.newBuilder();
 482     builder.setProjectId(DatastoreApiHelper.toProjectId(appNs.getAppId()));
 483     if (!appNs.getNamespace().isEmpty()) {
 484       builder.setNamespaceId(appNs.getNamespace());
 485     }
 486     return builder;
 487   }
 488
 489   static com.google.datastore.v1beta3.Key.Builder toV1Key(Key key) {
 490     com.google.datastore.v1beta3.Key.Builder builder =
 491         com.google.datastore.v1beta3.Key.newBuilder();
 492     builder.setPartitionId(toV1PartitionId(key.getAppIdNamespace()));
 493     List<PathElement> pathElementList = new ArrayList<>();
 494     do {
 495       PathElement.Builder pathElement = PathElement.newBuilder();
 496       pathElement.setKind(key.getKind());
 497       if (key.getName() != null) {
 498         pathElement.setName(key.getName());
 499       } else if (key.getId() != Key.NOT_ASSIGNED) {
 500         pathElement.setId(key.getId());
 501       }
 502       pathElementList.add(pathElement.build());
 503       key = key.getParent();
 504     } while (key != null);
 505     builder.addAllPath(Lists.reverse(pathElementList));
 506     return builder;
 507   }
 508
 509   static Key toKey(KeyOrBuilder proto) {
 510     if (proto.getPathCount() == 0) {
 511       throw new IllegalArgumentException("Invalid Key PB: no elements.");
 512     }
 513     AppIdNamespace appIdNamespace = toAppIdNamespace(proto.getPartitionId());
 514     Key key = null;
 515     for (PathElement e : proto.getPathList()) {
 516       String kind = e.getKind();
 517       key = new Key(kind, key, e.getId(),
 518           e.getIdTypeCase() == IdTypeCase.NAME ? e.getName() : null, appIdNamespace);
 519     }
 520     return key;
 521   }
 522
 523   static Entity toEntity(EntityOrBuilder entityV1) {
 524     Entity entity = new Entity(DataTypeTranslator.toKey(entityV1.getKey()));
 525     DataTypeTranslator.extractPropertiesFromPb(entityV1, false,
 526         entity.getPropertyMap());
 527     return entity;
 528   }
 529
 530   static Entity toEntity(EntityOrBuilder entityV1, Collection<Projection> projections) {
 531     Entity entity = new Entity(DataTypeTranslator.toKey(entityV1.getKey()));
 532
 533     Map<String, Object> values = Maps.newHashMap();
 534     DataTypeTranslator.extractPropertiesFromPb(entityV1, true, values);
 535     for (Projection projection : projections) {
 536       entity.setProperty(projection.getName(), projection.getValue(values));
 537     }
 538     return entity;
 539   }
 540
 541   static com.google.datastore.v1beta3.Entity.Builder toV1Entity(Entity entity) {
 542     com.google.datastore.v1beta3.Entity.Builder entityV1 =
 543         com.google.datastore.v1beta3.Entity.newBuilder();
 544     entityV1.setKey(toV1Key(entity.getKey()));
 545     addPropertiesToPb(entity.getPropertyMap(), entityV1);
 546     return entityV1;
 547   }
 548
 549   /**
 550    * Returns the value for the property as its comparable representation type.
 551    *
 552    * @param property a not {@code null} property
 553    * @return {@code null} if no value was set for {@code property}
 554    */
 555   @SuppressWarnings("unchecked")
 556   public static Comparable<Object> getComparablePropertyValue(Property property) {
 557     return (Comparable<Object>) RAW_VALUE_TYPE.asComparable(new RawValue(property.getValue()));
 558   }
 559
 560   /**
 561    * Converts the given {@link Object} into a supported value then returns it as
 562    * a comparable object so it can be compared to other data types.
 563    *
 564    * @param value any Object that can be converted into a supported DataType
 565    * @return {@code null} if value is null
 566    * @throws UnsupportedOperationException if value is not supported
 567    */
 568   @SuppressWarnings("unchecked")
 569   static Comparable<Object> getComparablePropertyValue(Object value) {
 570     return value == null ? null
 571         : (Comparable<Object>) getType(value.getClass()).asComparable(value);
 572   }
 573
 574   /**
 575    * Get the rank of the given datastore type relative to other datastore
 576    * types.  Note that datastore types do not necessarily have unique ranks.
 577    */
 578   @SuppressWarnings({"unchecked", "rawtypes"})
 579   public static int getTypeRank(Class<? extends Comparable> datastoreType) {
 580     return comparableTypeMap.get(datastoreType);
 581   }
 582
 583   /**
 584    * Gets the {@link Type} that knows how to translate objects of
 585    * type {@code clazz} into protocol buffers that the data store can
 586    * handle.
 587    * @throws UnsupportedOperationException if clazz is not supported
 588    */
 589   @SuppressWarnings("unchecked")
 590   private static <T> Type<T> getType(Class<T> clazz) {
 591     if (typeMap.containsKey(clazz)) {
 592       return (Type<T>) typeMap.get(clazz);
 593     } else {
 594       throw new UnsupportedOperationException("Unsupported data type: " + clazz.getName());
 595     }
 596   }
 597
 598   /**
 599    * {@code Type} is an abstract class that knows how to convert Java
 600    * objects of one or more types into datastore representations.
 601    *
 602    * @param <T> The canonical Java class for this type.
 603    */
 604   abstract static class Type<T> {
 605     /**
 606      * @return {@code true} if the given meaning and property value matches this {@link Type}.
 607      */
 608     public final boolean isType(Meaning meaning, PropertyValue propertyValue) {
 609       return meaning == getV3Meaning() && hasValue(propertyValue);
 610     }
 611
 612     /**
 613      * @return {@code true} if the given value matches this {@link Type}.
 614      */
 615     public boolean isType(Value propertyValue) {
 616       return getV3MeaningOf(propertyValue) == getV3Meaning() && hasValue(propertyValue);
 617     }
 618
 619     /**
 620      * Returns the {@link Comparable} for the given value, or
 621      * {@code null} if values of this type are not comparable.
 622      */
 623     public abstract Comparable<?> asComparable(Object value);
 624
 625     /**
 626      * Sets the value of {@code propertyValue} to {@code value}.
 627      * @return if the value is indexable
 628      */
 629     public abstract boolean toV3Value(Object value, PropertyValue propertyValue);
 630
 631     /**
 632      * Returns a new Cloud Datastore v1 Value for the given parameters.
 633      *
 634      * @param value the Java native value to convert
 635      * @param indexed the desired indexing, ignored for types that are not indexable
 636      * @return the Cloud Datastore v1 representation of the given value and desired indexing
 637      */
 638     public abstract Value.Builder toV1Value(
 639         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity);
 640
 641     /**
 642      * Returns the value of {@code propertyValue} as its canonical Java type.
 643      *
 644      * Use {@link #isType} first to determine if the property has a value of the given type.
 645      *
 646      * @param propertyValue a not {@code null} value representing this {@code Type}
 647      * @return the canonical Java representation of {@code propertyValue}.
 648      */
 649     public abstract T getValue(PropertyValue propertyValue);
 650
 651     /**
 652      * @see Type#getValue(PropertyValue)
 653      */
 654     public abstract T getValue(Value value);
 655
 656     /**
 657      * @return {@code true} if a value of this {@code Type} is set on the given propertyValue.
 658      */
 659     public abstract boolean hasValue(PropertyValue propertyValue);
 660
 661     /**
 662      * @return {@code true} if a value of this {@code Type} is set on the given propertyValue.
 663      */
 664     public abstract boolean hasValue(Value propertyValue);
 665
 666     /**
 667      * @return the {@link Meaning} for this {@link Type}
 668      */
 669     protected Meaning getV3Meaning() {
 670       return Meaning.NO_MEANING;
 671     }
 672   }
 673
 674   /**
 675    * A base class with common functions for types that have the same datastore representation.
 676    *
 677    * @param <S> the datastore type
 678    * @param <T> the canonical Java class for this type
 679    */
 680   private abstract static class BaseVariantType<S, T> extends Type<T> {
 681     /**
 682      * @return the datastore representation of the given value
 683      */
 684     protected abstract S toDatastoreValue(Object value);
 685     /**
 686      * @return the native representation of the given value
 687      */
 688     protected abstract T fromDatastoreValue(S datastoreValue);
 689   }
 690
 691   /**
 692    * Base class for types that store strings in the datastore.
 693    *
 694    * @param <T> the canonical Java class for this type
 695    */
 696   private abstract static class BaseStringType<T> extends BaseVariantType<String, T> {
 697     @Override
 698     public boolean toV3Value(Object value, PropertyValue propertyValue) {
 699       propertyValue.setStringValue(toDatastoreValue(value));
 700       return true;
 701     }
 702
 703     @Override
 704     public Value.Builder toV1Value(
 705         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 706       Value.Builder builder = Value.newBuilder();
 707       builder.setStringValue(toDatastoreValue(value));
 708       builder.setExcludeFromIndexes(!indexed);
 709       builder.setMeaning(getV3Meaning().getValue());
 710       return builder;
 711     }
 712
 713     @Override
 714     public final T getValue(PropertyValue propertyValue) {
 715       return fromDatastoreValue(propertyValue.getStringValue());
 716     }
 717
 718     @Override
 719     public T getValue(Value propertyValue) {
 720       return fromDatastoreValue(propertyValue.getStringValue());
 721     }
 722
 723     @Override
 724     public final boolean hasValue(PropertyValue propertyValue) {
 725       return propertyValue.hasStringValue();
 726     }
 727
 728     @Override
 729     public boolean hasValue(Value propertyValue) {
 730       return propertyValue.getValueTypeCase() == ValueTypeCase.STRING_VALUE;
 731     }
 732
 733     @Override
 734     public ComparableByteArray asComparable(Object value) {
 735       return new ComparableByteArray(ProtocolSupport.toBytesUtf8(toDatastoreValue(value)));
 736     }
 737   }
 738
 739   /**
 740    * Base class for types that store bytes in the datastore.
 741    *
 742    * @param <T> the canonical Java class for this type
 743    */
 744   private abstract static class BaseBlobType<T> extends BaseVariantType<byte[], T> {
 745     protected abstract boolean isIndexable();
 746
 747     @Override
 748     public final boolean hasValue(PropertyValue propertyValue) {
 749       return propertyValue.hasStringValue();
 750     }
 751
 752     @Override
 753     public final boolean toV3Value(Object value, PropertyValue propertyValue) {
 754       propertyValue.setStringValueAsBytes(toDatastoreValue(value));
 755       return isIndexable();
 756     }
 757
 758     @Override
 759     public Value.Builder toV1Value(
 760         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 761       Value.Builder builder = Value.newBuilder();
 762       builder.setBlobValue(ByteString.copyFrom(toDatastoreValue(value)));
 763       builder.setExcludeFromIndexes(!indexed || !isIndexable());
 764       return builder;
 765     }
 766
 767     @Override
 768     public final T getValue(PropertyValue propertyValue) {
 769       return fromDatastoreValue(propertyValue.getStringValueAsBytes());
 770     }
 771
 772     @Override
 773     public final ComparableByteArray asComparable(Object value) {
 774       return isIndexable() ? new ComparableByteArray(toDatastoreValue(value)) : null;
 775     }
 776   }
 777
 778   /**
 779    * Base class for types that store predefined entities in Cloud Datastore v1.
 780    *
 781    * @param <T> the canonical Java class for this type
 782    */
 783   private abstract static class BasePredefinedEntityType<T> extends Type<T> {
 784     /**
 785      * @return the predefined entity meaning to use in Cloud Datastore v1
 786      */
 787     protected abstract int getV1Meaning();
 788
 789     /**
 790      * @return the Cloud Datastore v1 Entity representation for the given value
 791      */
 792     protected abstract com.google.datastore.v1beta3.Entity getEntity(Object value);
 793
 794     @Override
 795     public final boolean isType(Value propertyValue) {
 796       return propertyValue.getMeaning() == getV1Meaning() && hasValue(propertyValue);
 797     }
 798
 799     @Override
 800     public final boolean hasValue(Value propertyValue) {
 801       return propertyValue.getValueTypeCase() == ValueTypeCase.ENTITY_VALUE;
 802     }
 803
 804     @Override
 805     public final Value.Builder toV1Value(
 806         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 807       Value.Builder builder = Value.newBuilder();
 808       builder.setEntityValue(getEntity(value));
 809       builder.setExcludeFromIndexes(!indexed);
 810       builder.setMeaning(getV1Meaning());
 811       return builder;
 812     }
 813   }
 814
 815   /**
 816    * Returns the Cloud Datastore v1 value representation for the given value, unindexed.
 817    */
 818   private static Value makeUnindexedValue(String value) {
 819     return Value.newBuilder().setStringValue(value).setExcludeFromIndexes(true).build();
 820   }
 821
 822   /**
 823    * Base class for types that int64 values in the datastore.
 824    *
 825    * @param <T> the canonical Java class for this type
 826    */
 827   private abstract static class BaseInt64Type<T> extends BaseVariantType<Long, T> {
 828     @Override
 829     public final boolean toV3Value(Object value, PropertyValue propertyValue) {
 830       propertyValue.setInt64Value(toDatastoreValue(value));
 831       return true;
 832     }
 833
 834     @Override
 835     public Value.Builder toV1Value(
 836         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 837       Value.Builder builder = Value.newBuilder();
 838       builder.setIntegerValue(toDatastoreValue(value));
 839       builder.setExcludeFromIndexes(!indexed);
 840       builder.setMeaning(getV3Meaning().getValue());
 841       return builder;
 842     }
 843
 844     @Override
 845     public T getValue(PropertyValue propertyValue) {
 846       return fromDatastoreValue(propertyValue.getInt64Value());
 847     }
 848
 849     @Override
 850     public T getValue(Value propertyValue) {
 851       return fromDatastoreValue(propertyValue.getIntegerValue());
 852     }
 853
 854     @Override
 855     public boolean hasValue(PropertyValue propertyValue) {
 856       return propertyValue.hasInt64Value();
 857     }
 858
 859     @Override
 860     public boolean hasValue(Value propertyValue) {
 861       return propertyValue.getValueTypeCase() == ValueTypeCase.INTEGER_VALUE;
 862     }
 863
 864     @Override
 865     public Long asComparable(Object value) {
 866       return toDatastoreValue(value);
 867     }
 868   }
 869
 870   /**
 871    * The type for projected index values.
 872    */
 873   private static final class RawValueType extends Type<RawValue> {
 874     @Override
 875     public Meaning getV3Meaning() {
 876       return Meaning.INDEX_VALUE;
 877     }
 878
 879     @Override
 880     public boolean hasValue(PropertyValue propertyValue) {
 881       return true;
 882     }
 883
 884     @Override
 885     public boolean hasValue(Value propertyValue) {
 886       return true;
 887     }
 888
 889     @Override
 890     public boolean toV3Value(Object value, PropertyValue propertyValue) {
 891       throw new UnsupportedOperationException();
 892     }
 893
 894     @Override
 895     public Value.Builder toV1Value(
 896         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 897       throw new UnsupportedOperationException();
 898     }
 899
 900     @Override
 901     public RawValue getValue(PropertyValue propertyValue) {
 902       return new RawValue(propertyValue);
 903     }
 904
 905     @Override
 906     public RawValue getValue(Value propertyValue) {
 907       return new RawValue(propertyValue);
 908     }
 909
 910     @SuppressWarnings("unchecked")
 911     @Override
 912     public Comparable<?> asComparable(Object value) {
 913       value = ((RawValue) value).getValue();
 914       if (value instanceof byte[]) {
 915         return new ComparableByteArray((byte[]) value);
 916       }
 917       return (Comparable<?>) value;
 918     }
 919   }
 920
 921   /**
 922    * The raw String type.
 923    */
 924   private static final class StringType extends BaseStringType<String> {
 925     @Override
 926     protected String toDatastoreValue(Object value) {
 927       return value.toString();
 928     }
 929
 930     @Override
 931     protected String fromDatastoreValue(String datastoreValue) {
 932       return datastoreValue;
 933     }
 934   }
 935
 936   /**
 937    * The raw int64 type.
 938    */
 939   private static final class Int64Type extends BaseInt64Type<Long> {
 940     @Override
 941     protected Long toDatastoreValue(Object value) {
 942       return ((Number) value).longValue();
 943     }
 944
 945     @Override
 946     protected Long fromDatastoreValue(Long datastoreValue) {
 947       return datastoreValue;
 948     }
 949   }
 950
 951   /**
 952    * The raw double type.
 953    */
 954   private static final class DoubleType extends Type<Double> {
 955     @Override
 956     public boolean toV3Value(Object value, PropertyValue propertyValue) {
 957       propertyValue.setDoubleValue(((Number) value).doubleValue());
 958       return true;
 959     }
 960
 961     @Override
 962     public Value.Builder toV1Value(
 963         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 964       Value.Builder builder = Value.newBuilder();
 965       builder.setDoubleValue(((Number) value).doubleValue());
 966       builder.setExcludeFromIndexes(!indexed);
 967       return builder;
 968     }
 969
 970     @Override
 971     public Double getValue(PropertyValue propertyValue) {
 972       return propertyValue.getDoubleValue();
 973     }
 974
 975     @Override
 976     public Double getValue(Value propertyValue) {
 977       return propertyValue.getDoubleValue();
 978     }
 979
 980     @Override
 981     public boolean hasValue(PropertyValue propertyValue) {
 982       return propertyValue.hasDoubleValue();
 983     }
 984
 985     @Override
 986     public boolean hasValue(Value propertyValue) {
 987       return propertyValue.getValueTypeCase() == ValueTypeCase.DOUBLE_VALUE;
 988     }
 989
 990     @Override
 991     public Double asComparable(Object value) {
 992       return ((Number) value).doubleValue();
 993     }
 994   }
 995
 996   /**
 997    * The raw boolean type.
 998    */
 999   private static final class BoolType extends Type<Boolean> {
1000     @Override
1001     public boolean toV3Value(Object value, PropertyValue propertyValue) {
1002       propertyValue.setBooleanValue((Boolean) value);
1003       return true;
1004     }
1005
1006     @Override
1007     public Value.Builder toV1Value(
1008         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1009       Value.Builder builder = Value.newBuilder();
1010       builder.setBooleanValue((Boolean) value);
1011       builder.setExcludeFromIndexes(!indexed);
1012       return builder;
1013     }
1014
1015     @Override
1016     public Boolean getValue(PropertyValue propertyValue) {
1017       return propertyValue.isBooleanValue();
1018     }
1019
1020     @Override
1021     public Boolean getValue(Value propertyValue) {
1022       return propertyValue.getBooleanValue();
1023     }
1024
1025     @Override
1026     public boolean hasValue(PropertyValue propertyValue) {
1027       return propertyValue.hasBooleanValue();
1028     }
1029
1030     @Override
1031     public boolean hasValue(Value propertyValue) {
1032       return propertyValue.getValueTypeCase() == ValueTypeCase.BOOLEAN_VALUE;
1033     }
1034
1035     @Override
1036     public Boolean asComparable(Object value) {
1037       return (Boolean) value;
1038     }
1039   }
1040
1041   /**
1042    * The user type.
1043    *
1044    * <p>Stored as an entity with a special meaning in v1.
1045    */
1046   private static final class UserType extends BasePredefinedEntityType<User> {
1047     public static final int MEANING_PREDEFINED_ENTITY_USER = 20;
1048     public static final String PROPERTY_NAME_EMAIL = "email";
1049     public static final String PROPERTY_NAME_AUTH_DOMAIN = "auth_domain";
1050     public static final String PROPERTY_NAME_USER_ID = "user_id";
1051
1052     @Override
1053     public int getV1Meaning() {
1054       return MEANING_PREDEFINED_ENTITY_USER;
1055     }
1056
1057     @Override
1058     public com.google.datastore.v1beta3.Entity getEntity(Object value) {
1059       User user = (User) value;
1060       com.google.datastore.v1beta3.Entity.Builder builder =
1061           com.google.datastore.v1beta3.Entity.newBuilder();
1062       builder.getMutableProperties().put(PROPERTY_NAME_EMAIL, makeUnindexedValue(user.getEmail()));
1063       builder.getMutableProperties().put(PROPERTY_NAME_AUTH_DOMAIN,
1064           makeUnindexedValue(user.getAuthDomain()));
1065       if (user.getUserId() != null) {
1066         builder.getMutableProperties().put(PROPERTY_NAME_USER_ID,
1067             makeUnindexedValue(user.getUserId()));
1068       }
1069       return builder.build();
1070     }
1071
1072     @Override
1073     public boolean toV3Value(Object value, PropertyValue propertyValue) {
1074       User user = (User) value;
1075       UserValue userValue = new UserValue();
1076       userValue.setEmail(user.getEmail());
1077       userValue.setAuthDomain(user.getAuthDomain());
1078       if (user.getUserId() != null) {
1079         userValue.setObfuscatedGaiaid(user.getUserId());
1080       }
1081       userValue.setGaiaid(0);
1082       propertyValue.setUserValue(userValue);
1083       return true;
1084     }
1085
1086     @Override
1087     public User getValue(PropertyValue propertyValue) {
1088       UserValue userValue = propertyValue.getUserValue();
1089       String userId = userValue.hasObfuscatedGaiaid() ? userValue.getObfuscatedGaiaid() : null;
1090       return new User(userValue.getEmail(), userValue.getAuthDomain(), userId);
1091     }
1092
1093     @Override
1094     public User getValue(Value propertyValue) {
1095       String email = "";
1096       String authDomain = "";
1097       String userId = null;
1098       for (Map.Entry<String, Value> prop
1099           : propertyValue.getEntityValueOrBuilder().getProperties().entrySet()) {
1100         if (prop.getKey().equals(PROPERTY_NAME_EMAIL)) {
1101           email = prop.getValue().getStringValue();
1102         } else if (prop.getKey().equals(PROPERTY_NAME_AUTH_DOMAIN)) {
1103           authDomain = prop.getValue().getStringValue();
1104         } else if (prop.getKey().equals(PROPERTY_NAME_USER_ID)) {
1105           userId = prop.getValue().getStringValue();
1106         }
1107       }
1108       return new User(email, authDomain, userId);
1109     }
1110
1111     @Override
1112     public boolean hasValue(PropertyValue propertyValue) {
1113       return propertyValue.hasUserValue();
1114     }
1115
1116     @Override
1117     public final Comparable<User> asComparable(Object value) {
1118       return (User) value;
1119     }
1120   }
1121
1122   /**
1123    * The GeoPt type.
1124    *
1125    * <p>Stored as a GeoPoint value with no meaning in Cloud Datastore v1.
1126    */
1127   private static class GeoPtType extends Type<GeoPt> {
1128     @Override
1129     public boolean isType(Value propertyValue) {
1130       return propertyValue.getValueTypeCase() == ValueTypeCase.GEO_POINT_VALUE
1131           && propertyValue.getMeaning() == 0;
1132     }
1133
1134     @Override
1135     public boolean toV3Value(Object value, PropertyValue propertyValue) {
1136       GeoPt geoPt = (GeoPt) value;
1137       PropertyValue.PointValue pv = new PropertyValue.PointValue()
1138           .setX(geoPt.getLatitude())
1139           .setY(geoPt.getLongitude());
1140       propertyValue.setPointValue(pv);
1141       return true;
1142     }
1143
1144     @Override
1145     public final Value.Builder toV1Value(
1146         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1147       GeoPt geoPt = (GeoPt) value;
1148       Value.Builder builder = Value.newBuilder();
1149       builder.getGeoPointValueBuilder()
1150           .setLatitude(geoPt.getLatitude())
1151           .setLongitude(geoPt.getLongitude());
1152       builder.setExcludeFromIndexes(!indexed);
1153       return builder;
1154     }
1155
1156     @Override
1157     public GeoPt getValue(PropertyValue propertyValue) {
1158       PropertyValue.PointValue pv = propertyValue.getPointValue();
1159       return new GeoPt((float) pv.getX(), (float) pv.getY());
1160     }
1161
1162     @Override
1163     public GeoPt getValue(Value propertyValue) {
1164       return new GeoPt(
1165           (float) propertyValue.getGeoPointValue().getLatitude(),
1166           (float) propertyValue.getGeoPointValue().getLongitude());
1167     }
1168
1169     @Override
1170     public boolean hasValue(PropertyValue propertyValue) {
1171       return propertyValue.hasPointValue();
1172     }
1173
1174     @Override
1175     public final boolean hasValue(Value propertyValue) {
1176       return propertyValue.getValueTypeCase() == ValueTypeCase.GEO_POINT_VALUE;
1177     }
1178
1179     @Override
1180     public Meaning getV3Meaning() {
1181       return Meaning.GEORSS_POINT;
1182     }
1183
1184     @Override
1185     public final Comparable<GeoPt> asComparable(Object value) {
1186       return (GeoPt) value;
1187     }
1188   }
1189
1190   /**
1191    * The key/reference type.
1192    */
1193   private static final class KeyType extends Type<Key> {
1194     @Override
1195     public boolean toV3Value(Object value, PropertyValue propertyValue) {
1196       Reference keyRef = KeyTranslator.convertToPb((Key) value);
1197       propertyValue.setReferenceValue(toReferenceValue(keyRef));
1198       return true;
1199     }
1200
1201     @Override
1202     public Value.Builder toV1Value(
1203         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1204       Value.Builder builder = Value.newBuilder();
1205       builder.setKeyValue(toV1Key((Key) value));
1206       builder.setExcludeFromIndexes(!indexed);
1207       return builder;
1208     }
1209
1210     @Override
1211     public Key getValue(PropertyValue propertyValue) {
1212       return KeyTranslator.createFromPb(toReference(propertyValue.getReferenceValue()));
1213     }
1214
1215     @Override
1216     public Key getValue(Value propertyValue) {
1217       return toKey(propertyValue.getKeyValue());
1218     }
1219
1220     @Override
1221     public boolean hasValue(PropertyValue propertyValue) {
1222       return propertyValue.hasReferenceValue();
1223     }
1224
1225     @Override
1226     public boolean hasValue(Value propertyValue) {
1227       return propertyValue.getValueTypeCase() == ValueTypeCase.KEY_VALUE;
1228     }
1229
1230     @Override
1231     public Key asComparable(Object value) {
1232       return (Key) value;
1233     }
1234
1235     private static ReferenceValue toReferenceValue(Reference keyRef) {
1236       ReferenceValue refValue = new ReferenceValue();
1237       refValue.setApp(keyRef.getApp());
1238       if (keyRef.hasNameSpace()) {
1239         refValue.setNameSpace(keyRef.getNameSpace());
1240       }
1241       Path path = keyRef.getPath();
1242       for (Element element : path.elements()) {
1243         ReferenceValuePathElement newElement = new ReferenceValuePathElement();
1244         newElement.setType(element.getType());
1245         if (element.hasName()) {
1246           newElement.setName(element.getName());
1247         }
1248         if (element.hasId()) {
1249           newElement.setId(element.getId());
1250         }
1251         refValue.addPathElement(newElement);
1252       }
1253
1254       return refValue;
1255     }
1256
1257     private static Reference toReference(ReferenceValue refValue) {
1258       Reference reference = new Reference();
1259       reference.setApp(refValue.getApp());
1260       if (refValue.hasNameSpace()) {
1261         reference.setNameSpace(refValue.getNameSpace());
1262       }
1263       Path path = new Path();
1264       for (ReferenceValuePathElement element : refValue.pathElements()) {
1265         Element newElement = new Element();
1266         newElement.setType(element.getType());
1267         if (element.hasName()) {
1268           newElement.setName(element.getName());
1269         }
1270         if (element.hasId()) {
1271           newElement.setId(element.getId());
1272         }
1273         path.addElement(newElement);
1274       }
1275       reference.setPath(path);
1276       return reference;
1277     }
1278   }
1279
1280   /**
1281    * The non-indexable blob type.
1282    */
1283   private static class BlobType extends BaseBlobType<Blob> {
1284     @Override
1285     public Meaning getV3Meaning() {
1286       return Meaning.BLOB;
1287     }
1288
1289     @Override
1290     public boolean isType(Value propertyValue) {
1291       return getV3MeaningOf(propertyValue) == Meaning.NO_MEANING
1292           && propertyValue.getExcludeFromIndexes()
1293           && hasValue(propertyValue);
1294     }
1295
1296     @Override
1297     public boolean hasValue(Value propertyValue) {
1298       return propertyValue.getValueTypeCase() == ValueTypeCase.BLOB_VALUE;
1299     }
1300
1301     @Override
1302     public Blob getValue(Value propertyValue) {
1303       return fromDatastoreValue(propertyValue.getBlobValue().toByteArray());
1304     }
1305
1306     @Override
1307     protected Blob fromDatastoreValue(byte[] datastoreValue) {
1308       return new Blob(datastoreValue);
1309     }
1310
1311     @Override
1312     protected byte[] toDatastoreValue(Object value) {
1313       return ((Blob) value).getBytes();
1314     }
1315
1316     @Override
1317     public boolean isIndexable() {
1318       return false;
1319     }
1320   }
1321
1322   /**
1323    * The indexable blob type.
1324    */
1325   private static class ShortBlobType extends BaseBlobType<ShortBlob> {
1326     @Override
1327     public Meaning getV3Meaning() {
1328       return Meaning.BYTESTRING;
1329     }
1330
1331     @Override
1332     public boolean isType(Value propertyValue) {
1333       if (!hasValue(propertyValue)) {
1334         return false;
1335       }
1336
1337       if (propertyValue.getExcludeFromIndexes()) {
1338         return getV3MeaningOf(propertyValue) == getV3Meaning();
1339       } else {
1340         return getV3MeaningOf(propertyValue) == Meaning.NO_MEANING;
1341       }
1342     }
1343
1344     @Override
1345     public Value.Builder toV1Value(
1346         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1347       Value.Builder builder = super.toV1Value(value, indexed, forceIndexedEmbeddedEntity);
1348       if (!indexed) {
1349         builder.setMeaning(getV3Meaning().getValue());
1350       }
1351       return builder;
1352     }
1353
1354     @Override
1355     public boolean hasValue(Value propertyValue) {
1356       return propertyValue.getValueTypeCase() == ValueTypeCase.BLOB_VALUE
1357           || (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1358               && propertyValue.getValueTypeCase() == ValueTypeCase.STRING_VALUE);
1359     }
1360
1361     @Override
1362     public ShortBlob getValue(Value propertyValue) {
1363       if (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1364           && propertyValue.getValueTypeCase() == ValueTypeCase.STRING_VALUE) {
1365         return fromDatastoreValue(propertyValue.getStringValueBytes().toByteArray());
1366       } else {
1367         return fromDatastoreValue(propertyValue.getBlobValue().toByteArray());
1368       }
1369     }
1370
1371     @Override
1372     protected byte[] toDatastoreValue(Object value) {
1373       return ((ShortBlob) value).getBytes();
1374     }
1375
1376     @Override
1377     protected ShortBlob fromDatastoreValue(byte[] datastoreValue) {
1378       return new ShortBlob(datastoreValue);
1379     }
1380
1381     @Override
1382     public boolean isIndexable() {
1383       return true;
1384     }
1385   }
1386
1387   /**
1388    * The entity type.
1389    *
1390    * Stored as a partially serialized EntityProto in V3.
1391    */
1392   private static final class EmbeddedEntityType extends Type<EmbeddedEntity> {
1393     @Override
1394     public Meaning getV3Meaning() {
1395       return Meaning.ENTITY_PROTO;
1396     }
1397
1398      @Override
1399     public boolean isType(Value propertyValue) {
1400       return getV3MeaningOf(propertyValue) == Meaning.NO_MEANING
1401           && hasValue(propertyValue);
1402     }
1403
1404     @Override
1405     public boolean hasValue(PropertyValue propertyValue) {
1406       return propertyValue.hasStringValue();
1407     }
1408
1409     @Override
1410     public boolean hasValue(Value propertyValue) {
1411       return propertyValue.getValueTypeCase() == ValueTypeCase.ENTITY_VALUE;
1412     }
1413
1414     @Override
1415     public EmbeddedEntity getValue(PropertyValue propertyValue) {
1416       EntityProto proto = new EntityProto();
1417       proto.mergeFrom(propertyValue.getStringValueAsBytes());
1418       EmbeddedEntity result = new EmbeddedEntity();
1419       if (proto.hasKey() && !proto.getKey().getApp().isEmpty()) {
1420         result.setKey(KeyTranslator.createFromPb(proto.getKey()));
1421       }
1422       extractPropertiesFromPb(proto, result.getPropertyMap());
1423       return result;
1424     }
1425
1426     @Override
1427     public EmbeddedEntity getValue(Value propertyValue) {
1428       EmbeddedEntity result = new EmbeddedEntity();
1429       com.google.datastore.v1beta3.Entity proto = propertyValue.getEntityValue();
1430       if (proto.hasKey()) {
1431         result.setKey(toKey(proto.getKey()));
1432       }
1433       extractPropertiesFromPb(proto, false, result.getPropertyMap());
1434       return result;
1435     }
1436
1437     @Override
1438     public boolean toV3Value(Object value, PropertyValue propertyValue) {
1439       EmbeddedEntity structProp = (EmbeddedEntity) value;
1440       EntityProto proto = new EntityProto();
1441       if (structProp.getKey() != null) {
1442         proto.setKey(KeyTranslator.convertToPb(structProp.getKey()));
1443       }
1444       addPropertiesToPb(structProp.getPropertyMap(), proto);
1445       propertyValue.setStringValueAsBytes(proto.toByteArray());
1446       return false;
1447     }
1448
1449     @Override
1450     public Value.Builder toV1Value(
1451         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1452       EmbeddedEntity structProp = (EmbeddedEntity) value;
1453       Value.Builder builder = Value.newBuilder();
1454       com.google.datastore.v1beta3.Entity.Builder proto = builder.getEntityValueBuilder();
1455       if (structProp.getKey() != null) {
1456         proto.setKey(toV1Key(structProp.getKey()));
1457       }
1458       addPropertiesToPb(structProp.getPropertyMap(), proto);
1459       builder.setExcludeFromIndexes(!indexed || !forceIndexedEmbeddedEntity);
1460       return builder;
1461     }
1462
1463     @Override
1464     public Comparable<?> asComparable(Object value) {
1465       return null;
1466     }
1467   }
1468
1469   /**
1470    * The non-indexable {@link Text} type.
1471    */
1472   private static final class TextType extends BaseStringType<Text> {
1473     @Override
1474     public Meaning getV3Meaning() {
1475       return Meaning.TEXT;
1476     }
1477
1478     @Override
1479     public boolean toV3Value(Object value, PropertyValue propertyValue) {
1480       super.toV3Value(value, propertyValue);
1481       return false;
1482     }
1483
1484     @Override
1485     public Value.Builder toV1Value(
1486         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1487       return super.toV1Value(value, false, false);
1488     }
1489
1490     @Override
1491     protected Text fromDatastoreValue(String datastoreString) {
1492       return new Text(datastoreString);
1493     }
1494
1495     @Override
1496     protected String toDatastoreValue(Object value) {
1497       return ((Text) value).getValue();
1498     }
1499
1500     @Override
1501     public ComparableByteArray asComparable(Object value) {
1502       return null;
1503     }
1504   }
1505
1506   /**
1507    * The {@link BlobKey} type. Blob keys are just strings with a special meaning.
1508    */
1509   private static final class BlobKeyType extends BaseStringType<BlobKey> {
1510     @Override
1511     public Meaning getV3Meaning() {
1512       return Meaning.BLOBKEY;
1513     }
1514
1515     @Override
1516     public Value.Builder toV1Value(
1517         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1518       Value.Builder builder = Value.newBuilder();
1519       builder.setStringValue(toDatastoreValue(value));
1520       builder.setMeaning(Meaning.BLOBKEY.getValue());
1521       builder.setExcludeFromIndexes(!indexed);
1522       return builder;
1523     }
1524
1525     @Override
1526     public BlobKey getValue(Value propertyValue) {
1527       return fromDatastoreValue(propertyValue.getStringValue());
1528     }
1529
1530     @Override
1531     protected String toDatastoreValue(Object value) {
1532       return ((BlobKey) value).getKeyString();
1533     }
1534
1535     @Override
1536     protected BlobKey fromDatastoreValue(String datastoreString) {
1537       return new BlobKey(datastoreString);
1538     }
1539   }
1540
1541   /**
1542    * The date type.
1543    *
1544    * In V3 dates are just int64s with a special meaning.
1545    */
1546   private static final class DateType extends BaseInt64Type<Date> {
1547     @Override
1548     public Meaning getV3Meaning() {
1549       return Meaning.GD_WHEN;
1550     }
1551
1552     @Override
1553     public boolean isType(Value propertyValue) {
1554       return propertyValue.getMeaning() == 0 && hasValue(propertyValue);
1555     }
1556
1557     @Override
1558     public boolean hasValue(Value propertyValue) {
1559       return propertyValue.getValueTypeCase() == ValueTypeCase.TIMESTAMP_VALUE
1560           || (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1561               && propertyValue.getValueTypeCase() == ValueTypeCase.INTEGER_VALUE);
1562     }
1563
1564     @Override
1565     public Value.Builder toV1Value(
1566         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1567       Value.Builder builder = DatastoreHelper.makeValue((Date) value);
1568       builder.setExcludeFromIndexes(!indexed);
1569       return builder;
1570     }
1571
1572     @Override
1573     public Date getValue(Value propertyValue) {
1574       if (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1575           && propertyValue.getValueTypeCase() == ValueTypeCase.INTEGER_VALUE) {
1576         return fromDatastoreValue(propertyValue.getIntegerValue());
1577       } else {
1578         long datastoreValue = DatastoreHelper.getTimestamp(propertyValue);
1579         return fromDatastoreValue(datastoreValue);
1580       }
1581     }
1582
1583     @Override
1584     protected Long toDatastoreValue(Object value) {
1585       return ((Date) value).getTime() * 1000L;
1586     }
1587
1588     @Override
1589     protected Date fromDatastoreValue(Long datastoreValue) {
1590       return new Date(datastoreValue / 1000L);
1591     }
1592   }
1593
1594   /**
1595    * Internally a link is just a string with a special meaning.
1596    */
1597   private static final class LinkType extends BaseStringType<Link> {
1598     @Override
1599     public Meaning getV3Meaning() {
1600       return Meaning.ATOM_LINK;
1601     }
1602
1603     @Override
1604     protected String toDatastoreValue(Object value) {
1605       return ((Link) value).getValue();
1606     }
1607
1608     @Override
1609     protected Link fromDatastoreValue(String datastoreValue) {
1610       return new Link(datastoreValue);
1611     }
1612   }
1613
1614   /**
1615    * Internally a category is just a string with a special meaning.
1616    */
1617   private static final class CategoryType extends BaseStringType<Category> {
1618     @Override
1619     public Meaning getV3Meaning() {
1620       return Meaning.ATOM_CATEGORY;
1621     }
1622
1623     @Override
1624     protected String toDatastoreValue(Object value) {
1625       return ((Category) value).getCategory();
1626     }
1627
1628     @Override
1629     protected Category fromDatastoreValue(String datastoreString) {
1630       return new Category(datastoreString);
1631     }
1632   }
1633
1634   /**
1635    * Internally a rating is just an int64 with a special meaning.
1636    */
1637   private static final class RatingType extends BaseInt64Type<Rating> {
1638     @Override
1639     public Meaning getV3Meaning() {
1640       return Meaning.GD_RATING;
1641     }
1642
1643     @Override
1644     protected Long toDatastoreValue(Object value) {
1645       return (long) ((Rating) value).getRating();
1646     }
1647
1648     @Override
1649     protected Rating fromDatastoreValue(Long datastoreLong) {
1650       return new Rating(datastoreLong.intValue());
1651     }
1652   }
1653
1654   /**
1655    * Internally an email is just a string with a special meaning.
1656    */
1657   private static final class EmailType extends BaseStringType<Email> {
1658     @Override
1659     public Meaning getV3Meaning() {
1660       return Meaning.GD_EMAIL;
1661     }
1662
1663     @Override
1664     protected String toDatastoreValue(Object value) {
1665       return ((Email) value).getEmail();
1666     }
1667
1668     @Override
1669     protected Email fromDatastoreValue(String datastoreString) {
1670       return new Email(datastoreString);
1671     }
1672   }
1673
1674   /**
1675    * Internally a postal address is just a string with a special meaning.
1676    */
1677   private static final class PostalAddressType extends BaseStringType<PostalAddress> {
1678     @Override
1679     public Meaning getV3Meaning() {
1680       return Meaning.GD_POSTALADDRESS;
1681     }
1682
1683     @Override
1684     protected String toDatastoreValue(Object value) {
1685       return ((PostalAddress) value).getAddress();
1686     }
1687
1688     @Override
1689     protected PostalAddress fromDatastoreValue(String datastoreString) {
1690       return new PostalAddress(datastoreString);
1691     }
1692   }
1693
1694   /**
1695    * Internally a phone number is just a string with a special meaning.
1696    */
1697   private static final class PhoneNumberType extends BaseStringType<PhoneNumber> {
1698     @Override
1699     public Meaning getV3Meaning() {
1700       return Meaning.GD_PHONENUMBER;
1701     }
1702
1703     @Override
1704     protected String toDatastoreValue(Object value) {
1705       return ((PhoneNumber) value).getNumber();
1706     }
1707
1708     @Override
1709     protected PhoneNumber fromDatastoreValue(String datastoreString) {
1710       return new PhoneNumber(datastoreString);
1711     }
1712   }
1713
1714   /**
1715    * Internally an IM handle is just a string with a special meaning and a
1716    * well known format.
1717    */
1718   private static final class IMHandleType extends BaseStringType<IMHandle> {
1719     @Override
1720     public Meaning getV3Meaning() {
1721       return Meaning.GD_IM;
1722     }
1723
1724     @Override
1725     protected String toDatastoreValue(Object value) {
1726       return ((IMHandle) value).toDatastoreString();
1727     }
1728
1729     @Override
1730     protected IMHandle fromDatastoreValue(String datastoreString) {
1731       return IMHandle.fromDatastoreString(datastoreString);
1732     }
1733   }
1734
1735   static Map<Class<?>, Type<?>> getTypeMap() {
1736     return typeMap;
1737   }
1738
1739   /**
1740    * A wrapper for a {@code byte[]} that implements {@link Comparable}.
1741    * Comparison algorithm is the same as the prod datastore.
1742    */
1743   public static final class ComparableByteArray implements Comparable<ComparableByteArray> {
1744     private final byte[] bytes;
1745
1746     public ComparableByteArray(byte[] bytes) {
1747       this.bytes = bytes;
1748     }
1749
1750     @Override
1751     public int compareTo(ComparableByteArray other) {
1752       byte[] otherBytes = other.bytes;
1753       for (int i = 0; i < Math.min(bytes.length, otherBytes.length); i++) {
1754         int v1 = bytes[i] & 0xFF;
1755         int v2 = otherBytes[i] & 0xFF;
1756         if (v1 != v2) {
1757           return v1 - v2;
1758         }
1759       }
1760       return bytes.length - otherBytes.length;
1761     }
1762
1763     @Override
1764     public boolean equals(Object obj) {
1765       if (obj == null) {
1766         return false;
1767       }
1768       return Arrays.equals(bytes, ((ComparableByteArray) obj).bytes);
1769     }
1770
1771     @Override
1772     public int hashCode() {
1773       int result = 1;
1774       for (byte b : bytes) {
1775         result = 31 * result + b;
1776       }
1777       return result;
1778     }
1779   }
1780
1781   private DataTypeTranslator() {
1782   }
1783 }