java/src/main/com/google/appengine/api/datastore/DataTypeTranslator.java

   1 // Copyright 2007 Google Inc. All rights reserved.
   2
   3 package com.google.appengine.api.datastore;
   4
   5 import com.google.appengine.api.blobstore.BlobKey;
   6 import com.google.appengine.api.datastore.Entity.UnindexedValue;
   7 import com.google.appengine.api.datastore.Entity.WrappedValue;
   8 import com.google.appengine.api.datastore.Entity.WrappedValueImpl;
   9 import com.google.appengine.api.users.User;
  10 import com.google.apphosting.api.ApiProxy;
  11 import com.google.common.collect.Lists;
  12 import com.google.common.collect.Maps;
  13 import com.google.datastore.v1beta3.ArrayValue;
  14 import com.google.datastore.v1beta3.EntityOrBuilder;
  15 import com.google.datastore.v1beta3.Key.PathElement;
  16 import com.google.datastore.v1beta3.Key.PathElement.IdTypeCase;
  17 import com.google.datastore.v1beta3.KeyOrBuilder;
  18 import com.google.datastore.v1beta3.PartitionId;
  19 import com.google.datastore.v1beta3.PartitionIdOrBuilder;
  20 import com.google.datastore.v1beta3.Value;
  21 import com.google.datastore.v1beta3.Value.NullValue;
  22 import com.google.datastore.v1beta3.Value.ValueTypeCase;
  23 import com.google.datastore.v1beta3.ValueOrBuilder;
  24 import com.google.datastore.v1beta3.client.DatastoreHelper;
  25 import com.google.io.protocol.ProtocolSupport;
  26 import com.google.protobuf.ByteString;
  27 import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
  28 import com.google.storage.onestore.v3.OnestoreEntity.Path;
  29 import com.google.storage.onestore.v3.OnestoreEntity.Path.Element;
  30 import com.google.storage.onestore.v3.OnestoreEntity.Property;
  31 import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning;
  32 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue;
  33 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue;
  34 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValuePathElement;
  35 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.UserValue;
  36 import com.google.storage.onestore.v3.OnestoreEntity.Reference;
  37
  38 import java.util.ArrayList;
  39 import java.util.Arrays;
  40 import java.util.Collection;
  41 import java.util.Collections;
  42 import java.util.Date;
  43 import java.util.HashMap;
  44 import java.util.List;
  45 import java.util.Map;
  46
  47 /**
  48  * {@code DataTypeTranslator} is a utility class for converting
  49  * between the data store's {@code Property} protocol buffers and the
  50  * user-facing classes ({@code String}, {@code User}, etc.).
  51  *
  52  */
  53 public final class DataTypeTranslator  {
  54   /**
  55    * This key points to an (optional) map from project id to app id in the
  56    * {@link com.google.apphosting.api.ApiProxy.Environment} attribute map.
  57    */
  58   static final String ADDITIONAL_APP_IDS_MAP_ATTRIBUTE_KEY =
  59       "com.google.appengine.datastore.DataTypeTranslator.AdditionalAppIdsMap";
  60
  61   private static final RawValueType RAW_VALUE_TYPE = new RawValueType();
  62   /**
  63    * The list of supported types.
  64    *
  65    * Note: If you're going to modify this list, also update
  66    * DataTypeUtils. We're not building {@link DataTypeUtils#getSupportedTypes}
  67    * directly from this typeMap, because we want {@link DataTypeUtils} to be
  68    * translatable by GWT, so that {@link Entity Entities} can be easily sent
  69    * via GWT RPC.  Also, if you add a type here that is not immutable you'll
  70    * need to add special handling for it in {@link Entity#clone()}.
  71    */
  72   private static final Map<Class<?>, Type<?>> typeMap = Maps.newHashMap();
  73   static {
  74     typeMap.put(RawValue.class, RAW_VALUE_TYPE);
  75
  76     typeMap.put(Float.class, new DoubleType());
  77     typeMap.put(Double.class, new DoubleType());
  78
  79     typeMap.put(Byte.class, new Int64Type());
  80     typeMap.put(Short.class, new Int64Type());
  81     typeMap.put(Integer.class, new Int64Type());
  82     typeMap.put(Long.class, new Int64Type());
  83     typeMap.put(Date.class, new DateType());
  84     typeMap.put(Rating.class, new RatingType());
  85
  86     typeMap.put(String.class, new StringType());
  87     typeMap.put(Link.class, new LinkType());
  88     typeMap.put(ShortBlob.class, new ShortBlobType());
  89     typeMap.put(Category.class, new CategoryType());
  90     typeMap.put(PhoneNumber.class, new PhoneNumberType());
  91     typeMap.put(PostalAddress.class, new PostalAddressType());
  92     typeMap.put(Email.class, new EmailType());
  93     typeMap.put(IMHandle.class, new IMHandleType());
  94     typeMap.put(BlobKey.class, new BlobKeyType());
  95     typeMap.put(Blob.class, new BlobType());
  96     typeMap.put(Text.class, new TextType());
  97     typeMap.put(EmbeddedEntity.class, new EmbeddedEntityType());
  98
  99     typeMap.put(Boolean.class, new BoolType());
 100     typeMap.put(User.class, new UserType());
 101     typeMap.put(Key.class, new KeyType());
 102     typeMap.put(GeoPt.class, new GeoPtType());
 103
 104     assert typeMap.keySet().equals(DataTypeUtils.getSupportedTypes())
 105         : "Warning:  DataTypeUtils and DataTypeTranslator do not agree "
 106         + "about supported classes: " + typeMap.keySet() + " vs. "
 107         + DataTypeUtils.getSupportedTypes();
 108   }
 109
 110   /**
 111    * A map with the {@link Comparable} classes returned by all the instances of
 112    * {@link Type#asComparable(Object)} as keys and the pb code point as the value.
 113    * Used for comparing values that don't map to the same pb code point.
 114    */
 115   private static final
 116   Map<Class<? extends Comparable<?>>, Integer> comparableTypeMap =
 117       new HashMap<Class<? extends Comparable<?>>, Integer>();
 118
 119   static {
 120     comparableTypeMap.put(ComparableByteArray.class, PropertyValue.kstringValue);
 121     comparableTypeMap.put(Long.class, PropertyValue.kint64Value);
 122     comparableTypeMap.put(Double.class, PropertyValue.kdoubleValue);
 123     comparableTypeMap.put(Boolean.class, PropertyValue.kbooleanValue);
 124     comparableTypeMap.put(User.class, PropertyValue.kUserValueGroup);
 125     comparableTypeMap.put(Key.class, PropertyValue.kReferenceValueGroup);
 126     comparableTypeMap.put(GeoPt.class, PropertyValue.kPointValueGroup);
 127   }
 128
 129   /**
 130    * Add all of the properties in the specified map to an {@code EntityProto}.
 131    * This involves determining the type of each property and creating the
 132    * proper type-specific protocol buffer.
 133    *
 134    * If the property value is an {@link UnindexedValue}, or if it's a
 135    * type that is never indexed, e.g. {@code Text} and {@code Blob}, it's
 136    * added to {@code EntityProto.raw_property}. Otherwise it's added to
 137    * {@code EntityProto.property}.
 138    *
 139    * @param map A not {@code null} map of all the properties which will
 140    * be set on {@code proto}
 141    * @param proto A not {@code null} protocol buffer
 142    */
 143   public static void addPropertiesToPb(Map<String, Object> map, EntityProto proto) {
 144     for (Map.Entry<String, Object> entry : map.entrySet()) {
 145       String name = entry.getKey();
 146
 147       boolean forceIndexedEmbeddedEntity = false;
 148       boolean indexed = true;
 149       Object value = entry.getValue();
 150
 151       if (entry.getValue() instanceof WrappedValue) {
 152         WrappedValue wrappedValue = (WrappedValue) entry.getValue();
 153         forceIndexedEmbeddedEntity = wrappedValue.getForceIndexedEmbeddedEntity();
 154         indexed = wrappedValue.isIndexed();
 155         value = wrappedValue.getValue();
 156       }
 157
 158       if (value instanceof Collection<?>) {
 159         Collection<?> values = (Collection<?>) value;
 160         addListPropertyToPb(proto, name, indexed, values, forceIndexedEmbeddedEntity);
 161       } else {
 162         addPropertyToPb(name, value, indexed, forceIndexedEmbeddedEntity, false, proto);
 163       }
 164     }
 165   }
 166
 167   private static void addListPropertyToPb(EntityProto proto, String name, boolean indexed,
 168       Collection<?> values, boolean forceIndexedEmbeddedEntity) {
 169     if (values.isEmpty()) {
 170       Property property = new Property();
 171       property.setName(name);
 172       property.setMultiple(false);
 173       if (DatastoreServiceConfig.getEmptyListSupport()) {
 174         property.setMeaning(Meaning.EMPTY_LIST);
 175       } else {
 176       }
 177       property.getMutableValue();
 178       if (indexed) {
 179         proto.addProperty(property);
 180       } else {
 181         proto.addRawProperty(property);
 182       }
 183     } else {
 184       for (Object listValue : values) {
 185         addPropertyToPb(name, listValue, indexed, forceIndexedEmbeddedEntity,
 186             true, proto);
 187       }
 188     }
 189   }
 190
 191   /**
 192    * Adds a property to {@code entity}.
 193    *
 194    * @param name the property name
 195    * @param value the property value
 196    * @param indexed whether this property should be indexed. This may be
 197    * overridden by property types like Blob and Text that are never indexed.
 198    * @param multiple whether this property has multiple values
 199    * @param entity the entity to populate
 200    */
 201   private static void addPropertyToPb(String name, Object value, boolean indexed,
 202       boolean forceIndexedEmbeddedEntity, boolean multiple, EntityProto entity) {
 203     Property property = new Property();
 204     property.setName(name);
 205     property.setMultiple(multiple);
 206     PropertyValue newValue = property.getMutableValue();
 207     if (value != null) {
 208       Type<?> type = getType(value.getClass());
 209       Meaning meaning = type.getV3Meaning();
 210       if (meaning != property.getMeaningEnum()) {
 211         property.setMeaning(meaning);
 212       }
 213       type.toV3Value(value, newValue);
 214       if (!forceIndexedEmbeddedEntity || !(value instanceof EmbeddedEntity)) {
 215         indexed &= type.canBeIndexed();
 216       }
 217     }
 218     if (indexed) {
 219       entity.addProperty(property);
 220     } else {
 221       entity.addRawProperty(property);
 222     }
 223   }
 224
 225   static PropertyValue toV3Value(Object value) {
 226     PropertyValue propertyValue = new PropertyValue();
 227     if (value != null) {
 228       getType(value.getClass()).toV3Value(value, propertyValue);
 229     }
 230     return propertyValue;
 231   }
 232
 233   /**
 234    * Copy all of the indexed properties present on {@code proto} into {@code map}.
 235    */
 236   public static void extractIndexedPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 237     for (Property property : proto.propertys()) {
 238       addPropertyToMap(property, true, map);
 239     }
 240   }
 241
 242   /**
 243    * Copy all of the unindexed properties present on {@code proto} into {@code map}.
 244    */
 245   private static void extractUnindexedPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 246     for (Property property : proto.rawPropertys()) {
 247       addPropertyToMap(property, false, map);
 248     }
 249   }
 250
 251   /**
 252    * Copy all of the properties present on {@code proto} into {@code map}.
 253    */
 254   public static void extractPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 255     extractIndexedPropertiesFromPb(proto, map);
 256     extractUnindexedPropertiesFromPb(proto, map);
 257   }
 258
 259   /**
 260    * Copy all of the implicit properties present on {@code proto} into {@code map}.
 261    */
 262   public static void extractImplicitPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 263     for (Property property : getImplicitProperties(proto)) {
 264       addPropertyToMap(property, true, map);
 265     }
 266   }
 267
 268   private static Iterable<Property> getImplicitProperties(EntityProto proto) {
 269     return Collections.singleton(buildImplicitKeyProperty(proto));
 270   }
 271
 272   private static Property buildImplicitKeyProperty(EntityProto proto) {
 273     Property keyProp = new Property();
 274     keyProp.setName(Entity.KEY_RESERVED_PROPERTY);
 275     PropertyValue propVal = new PropertyValue();
 276     propVal.setReferenceValue(KeyType.toReferenceValue(proto.getKey()));
 277     keyProp.setValue(propVal);
 278     return keyProp;
 279   }
 280
 281   /**
 282    * Locates and returns all indexed properties with the given name on the
 283    * given proto. If there are a mix of matching multiple and non-multiple
 284    * properties, the collection will contain the first non-multiple property.
 285    * @return A list, potentially empty, containing matching properties.
 286    */
 287   public static Collection<Property> findIndexedPropertiesOnPb(
 288       EntityProto proto, String propertyName) {
 289     if (propertyName.equals(Entity.KEY_RESERVED_PROPERTY)) {
 290       return Collections.singleton(buildImplicitKeyProperty(proto));
 291     }
 292     List<Property> matchingMultipleProps = new ArrayList<>();
 293     for (Property prop : proto.propertys()) {
 294       if (prop.getName().equals(propertyName)) {
 295         if (!prop.isMultiple()) {
 296           return Collections.singleton(prop);
 297         } else {
 298           matchingMultipleProps.add(prop);
 299         }
 300       }
 301     }
 302     return matchingMultipleProps;
 303   }
 304
 305   private static Object wrapIfUnindexed(boolean indexed, Object value) {
 306     return indexed ? value : new UnindexedValue(value);
 307   }
 308
 309   private static void addPropertyToMap(Property property, boolean indexed,
 310       Map<String, Object> map) {
 311     String name = property.getName();
 312
 313     if (property.getMeaningEnum() == Meaning.EMPTY_LIST) {
 314       Object emptyListValue =
 315           DatastoreServiceConfig.getEmptyListSupport() ? new ArrayList<Object>() : null;
 316       map.put(name, wrapIfUnindexed(indexed, emptyListValue));
 317     } else {
 318       Object value = getPropertyValue(property);
 319       if (property.isMultiple()) {
 320         @SuppressWarnings({"unchecked"})
 321         List<Object> resultList = (List<Object>) PropertyContainer.unwrapValue(map.get(name));
 322         if (resultList == null) {
 323           resultList = new ArrayList<Object>();
 324           map.put(name, indexed ? resultList : new UnindexedValue(resultList));
 325         }
 326         if (indexed && value instanceof EmbeddedEntity) {
 327           map.put(name, new WrappedValueImpl(resultList, true, true));
 328         }
 329         resultList.add(value);
 330       } else {
 331         if (indexed && value instanceof EmbeddedEntity) {
 332           value = new WrappedValueImpl(value, true, true);
 333         } else if (!indexed) {
 334           value = new UnindexedValue(value);
 335         }
 336         map.put(name, value);
 337       }
 338     }
 339   }
 340
 341   /**
 342    * Returns the value for the property as its canonical type.
 343    *
 344    * @param property a not {@code null} property
 345    * @return {@code null} if no value was set for {@code property}
 346    */
 347   public static Object getPropertyValue(Property property) {
 348     PropertyValue value = property.getValue();
 349     for (Type<?> type : typeMap.values()) {
 350       if (type.isType(property.getMeaningEnum(), value)) {
 351         return type.getValue(value);
 352       }
 353     }
 354     return null;
 355   }
 356
 357   /**
 358    * @see #addPropertiesToPb(Map, EntityProto)
 359    */
 360   static void addPropertiesToPb(Map<String, Object> map,
 361       com.google.datastore.v1beta3.Entity.Builder proto) {
 362     for (Map.Entry<String, Object> entry : map.entrySet()) {
 363       proto.getMutableProperties().put(entry.getKey(), toV1Value(entry.getValue()).build());
 364     }
 365   }
 366
 367   /**
 368    * Copy all of the properties present on {@code proto} into {@code map}.
 369    *
 370    * <p>Cloud Datastore v1 entity must know if the proto came from an index-only query as the User
 371    * type overwrites the INDEX_ONLY meaning.
 372    *
 373    * @param proto the proto from which to extract properties
 374    * @param indexOnly if the proto is from an index only query (a projection query)
 375    * @param map the map to populate
 376    */
 377   static void extractPropertiesFromPb(EntityOrBuilder proto, boolean indexOnly,
 378       Map<String, Object> map) {
 379     if (indexOnly) {
 380       for (Map.Entry<String, Value> prop : proto.getProperties().entrySet()) {
 381         map.put(prop.getKey(), new RawValue(prop.getValue()));
 382       }
 383     } else {
 384       for (Map.Entry<String, Value> prop : proto.getProperties().entrySet()) {
 385         addPropertyToMap(prop.getKey(), prop.getValue(), map);
 386       }
 387     }
 388   }
 389
 390   static Value.Builder toV1Value(
 391       Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 392     if (value == null) {
 393       Value.Builder builder = Value.newBuilder();
 394       builder.setNullValue(NullValue.NULL_VALUE);
 395       builder.setExcludeFromIndexes(!indexed);
 396       return builder;
 397     }
 398     return getType(value.getClass()).toV1Value(value, indexed, forceIndexedEmbeddedEntity);
 399   }
 400
 401   private static Value.Builder toV1Value(Object value) {
 402     boolean indexed = true;
 403     boolean forceIndexedEmbeddedEntity = false;
 404
 405     if (value instanceof WrappedValue) {
 406       WrappedValue wrappedValue = (WrappedValue) value;
 407       indexed = wrappedValue.isIndexed();
 408       forceIndexedEmbeddedEntity = wrappedValue.getForceIndexedEmbeddedEntity();
 409       value = wrappedValue.getValue();
 410     }
 411
 412     if (value instanceof Collection<?>) {
 413       Collection<?> values = (Collection<?>) value;
 414       if (values.isEmpty()) {
 415         if (DatastoreServiceConfig.getEmptyListSupport()) {
 416           return Value.newBuilder().setExcludeFromIndexes(!indexed).setArrayValue(ArrayValue.newBuilder());
 417         } else {
 418          return toV1Value(null, indexed, forceIndexedEmbeddedEntity);
 419         }
 420       } else {
 421         Value.Builder valueBuilder = Value.newBuilder();
 422         for (Object listValue : values) {
 423           valueBuilder.getArrayValueBuilder().addValues(
 424               toV1Value(listValue, indexed, forceIndexedEmbeddedEntity));
 425         }
 426         return valueBuilder;
 427       }
 428     } else {
 429       return toV1Value(value, indexed, forceIndexedEmbeddedEntity);
 430     }
 431   }
 432
 433   private static void addPropertyToMap(String name, Value value, Map<String, Object> map) {
 434     boolean isOrContainsIndexedEntityValue = false;
 435     boolean indexed;
 436     Object result;
 437
 438     if (DatastoreServiceConfig.getEmptyListSupport()
 439         && value.getValueTypeCase() == Value.ValueTypeCase.ARRAY_VALUE
 440         && value.getArrayValue().getValuesCount() == 0) {
 441       result = new ArrayList<Object>();
 442       indexed = !value.getExcludeFromIndexes();
 443     } else if (value.getArrayValue().getValuesCount() > 0) {
 444       indexed = false;
 445       ArrayList<Object> resultList =
 446           new ArrayList<Object>(value.getArrayValue().getValuesCount());
 447       for (Value subValue : value.getArrayValue().getValuesList()) {
 448         if (subValue.getValueTypeCase() == ValueTypeCase.ARRAY_VALUE) {
 449           throw new IllegalArgumentException("Invalid Entity PB: list within a list.");
 450         }
 451         result = getValue(subValue);
 452         if (!subValue.getExcludeFromIndexes()) {
 453           indexed = true;
 454           if (result instanceof EmbeddedEntity) {
 455             isOrContainsIndexedEntityValue = true;
 456           }
 457         }
 458         resultList.add(result);
 459       }
 460       result = resultList;
 461     } else {
 462       indexed = !value.getExcludeFromIndexes();
 463       result = getValue(value);
 464       if (indexed && result instanceof EmbeddedEntity) {
 465         isOrContainsIndexedEntityValue = true;
 466       }
 467     }
 468
 469     if (isOrContainsIndexedEntityValue) {
 470       result = new WrappedValueImpl(result, true, true);
 471     } else if (!indexed) {
 472       result = new UnindexedValue(result);
 473     }
 474
 475     map.put(name, result);
 476   }
 477
 478   private static Object getValue(Value value) {
 479     for (Type<?> type : typeMap.values()) {
 480       if (type.isType(value)) {
 481         return type.getValue(value);
 482       }
 483     }
 484     return null;
 485   }
 486
 487   private static Meaning getV3MeaningOf(ValueOrBuilder value) {
 488     return Meaning.valueOf(value.getMeaning());
 489   }
 490
 491   private static AppIdNamespace toAppIdNamespace(PartitionIdOrBuilder partitionId) {
 492     if (partitionId.getProjectId().equals(DatastoreApiHelper.getCurrentProjectId())) {
 493       return new AppIdNamespace(DatastoreApiHelper.getCurrentAppId(), partitionId.getNamespaceId());
 494     }
 495     Map<String, String> additionalProjectIdToAppIdMap = getAdditionalProjectIdToAppIdMap();
 496     if (additionalProjectIdToAppIdMap.containsKey(partitionId.getProjectId())) {
 497       return new AppIdNamespace(additionalProjectIdToAppIdMap.get(partitionId.getProjectId()),
 498           partitionId.getNamespaceId());
 499     } else {
 500       throw new IllegalStateException(String.format(
 501           "Could not determine app id corresponding to project id \"%s\". Please add the app id "
 502           + "to %s.",
 503           partitionId.getProjectId(), RemoteCloudDatastoreV1Proxy.ADDITIONAL_APP_IDS_VAR));
 504     }
 505   }
 506
 507   @SuppressWarnings("unchecked")
 508   private static Map<String, String> getAdditionalProjectIdToAppIdMap() {
 509     if (ApiProxy.getCurrentEnvironment() != null) {
 510       Object attribute = ApiProxy.getCurrentEnvironment().getAttributes()
 511           .get(ADDITIONAL_APP_IDS_MAP_ATTRIBUTE_KEY);
 512       if (attribute != null) {
 513         return (Map<String, String>) attribute;
 514       }
 515     }
 516     return Collections.emptyMap();
 517   }
 518
 519   private static PartitionId.Builder toV1PartitionId(AppIdNamespace appNs) {
 520     PartitionId.Builder builder = PartitionId.newBuilder();
 521     builder.setProjectId(DatastoreApiHelper.toProjectId(appNs.getAppId()));
 522     if (!appNs.getNamespace().isEmpty()) {
 523       builder.setNamespaceId(appNs.getNamespace());
 524     }
 525     return builder;
 526   }
 527
 528   static com.google.datastore.v1beta3.Key.Builder toV1Key(Key key) {
 529     com.google.datastore.v1beta3.Key.Builder builder =
 530         com.google.datastore.v1beta3.Key.newBuilder();
 531     builder.setPartitionId(toV1PartitionId(key.getAppIdNamespace()));
 532     List<PathElement> pathElementList = new ArrayList<>();
 533     do {
 534       PathElement.Builder pathElement = PathElement.newBuilder();
 535       pathElement.setKind(key.getKind());
 536       if (key.getName() != null) {
 537         pathElement.setName(key.getName());
 538       } else if (key.getId() != Key.NOT_ASSIGNED) {
 539         pathElement.setId(key.getId());
 540       }
 541       pathElementList.add(pathElement.build());
 542       key = key.getParent();
 543     } while (key != null);
 544     builder.addAllPath(Lists.reverse(pathElementList));
 545     return builder;
 546   }
 547
 548   static Key toKey(KeyOrBuilder proto) {
 549     if (proto.getPathCount() == 0) {
 550       throw new IllegalArgumentException("Invalid Key PB: no elements.");
 551     }
 552     AppIdNamespace appIdNamespace = toAppIdNamespace(proto.getPartitionId());
 553     Key key = null;
 554     for (PathElement e : proto.getPathList()) {
 555       String kind = e.getKind();
 556       key = new Key(kind, key, e.getId(),
 557           e.getIdTypeCase() == IdTypeCase.NAME ? e.getName() : null, appIdNamespace);
 558     }
 559     return key;
 560   }
 561
 562   static Entity toEntity(EntityOrBuilder entityV1) {
 563     Entity entity = new Entity(DataTypeTranslator.toKey(entityV1.getKey()));
 564     DataTypeTranslator.extractPropertiesFromPb(entityV1, false,
 565         entity.getPropertyMap());
 566     return entity;
 567   }
 568
 569   static Entity toEntity(EntityOrBuilder entityV1, Collection<Projection> projections) {
 570     Entity entity = new Entity(DataTypeTranslator.toKey(entityV1.getKey()));
 571
 572     Map<String, Object> values = Maps.newHashMap();
 573     DataTypeTranslator.extractPropertiesFromPb(entityV1, true, values);
 574     for (Projection projection : projections) {
 575       entity.setProperty(projection.getName(), projection.getValue(values));
 576     }
 577     return entity;
 578   }
 579
 580   static com.google.datastore.v1beta3.Entity.Builder toV1Entity(Entity entity) {
 581     com.google.datastore.v1beta3.Entity.Builder entityV1 =
 582         com.google.datastore.v1beta3.Entity.newBuilder();
 583     entityV1.setKey(toV1Key(entity.getKey()));
 584     addPropertiesToPb(entity.getPropertyMap(), entityV1);
 585     return entityV1;
 586   }
 587
 588   /**
 589    * Returns the value for the property as its comparable representation type.
 590    *
 591    * @param property a not {@code null} property
 592    * @return {@code null} if no value was set for {@code property}
 593    */
 594   @SuppressWarnings("unchecked")
 595   public static Comparable<Object> getComparablePropertyValue(Property property) {
 596     return (Comparable<Object>) RAW_VALUE_TYPE.asComparable(new RawValue(property.getValue()));
 597   }
 598
 599   /**
 600    * Converts the given {@link Object} into a supported value then returns it as
 601    * a comparable object so it can be compared to other data types.
 602    *
 603    * @param value any Object that can be converted into a supported DataType
 604    * @return {@code null} if value is null
 605    * @throws UnsupportedOperationException if value is not supported
 606    */
 607   @SuppressWarnings("unchecked")
 608   static Comparable<Object> getComparablePropertyValue(Object value) {
 609     return value == null ? null
 610         : (Comparable<Object>) getType(value.getClass()).asComparable(value);
 611   }
 612
 613   /**
 614    * Get the rank of the given datastore type relative to other datastore
 615    * types.  Note that datastore types do not necessarily have unique ranks.
 616    */
 617   @SuppressWarnings({"unchecked", "rawtypes"})
 618   public static int getTypeRank(Class<? extends Comparable> datastoreType) {
 619     return comparableTypeMap.get(datastoreType);
 620   }
 621
 622   /**
 623    * Gets the {@link Type} that knows how to translate objects of
 624    * type {@code clazz} into protocol buffers that the data store can
 625    * handle.
 626    * @throws UnsupportedOperationException if clazz is not supported
 627    */
 628   @SuppressWarnings("unchecked")
 629   private static <T> Type<T> getType(Class<T> clazz) {
 630     if (typeMap.containsKey(clazz)) {
 631       return (Type<T>) typeMap.get(clazz);
 632     } else {
 633       throw new UnsupportedOperationException("Unsupported data type: " + clazz.getName());
 634     }
 635   }
 636
 637   /**
 638    * {@code Type} is an abstract class that knows how to convert Java
 639    * objects of one or more types into datastore representations.
 640    *
 641    * @param <T> The canonical Java class for this type.
 642    */
 643   abstract static class Type<T> {
 644     /**
 645      * @return {@code true} if the given meaning and property value matches this {@link Type}.
 646      */
 647     public final boolean isType(Meaning meaning, PropertyValue propertyValue) {
 648       return meaning == getV3Meaning() && hasValue(propertyValue);
 649     }
 650
 651     /**
 652      * @return {@code true} if the given value matches this {@link Type}.
 653      */
 654     public boolean isType(Value propertyValue) {
 655       return getV3MeaningOf(propertyValue) == getV3Meaning() && hasValue(propertyValue);
 656     }
 657
 658     /**
 659      * Returns the {@link Comparable} for the given value, or
 660      * {@code null} if values of this type are not comparable.
 661      */
 662     public abstract Comparable<?> asComparable(Object value);
 663
 664     /**
 665      * Sets the value of {@code propertyValue} to {@code value}.
 666      */
 667     public abstract void toV3Value(Object value, PropertyValue propertyValue);
 668
 669     /**
 670      * @returns Whether the value is indexable
 671      */
 672     public abstract boolean canBeIndexed();
 673
 674     /**
 675      * Returns a new Cloud Datastore v1 Value for the given parameters.
 676      *
 677      * @param value the Java native value to convert
 678      * @param indexed the desired indexing, ignored for types that are not indexable
 679      * @return the Cloud Datastore v1 representation of the given value and desired indexing
 680      */
 681     public abstract Value.Builder toV1Value(
 682         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity);
 683
 684     /**
 685      * Returns the value of {@code propertyValue} as its canonical Java type.
 686      *
 687      * Use {@link #isType} first to determine if the property has a value of the given type.
 688      *
 689      * @param propertyValue a not {@code null} value representing this {@code Type}
 690      * @return the canonical Java representation of {@code propertyValue}.
 691      */
 692     public abstract T getValue(PropertyValue propertyValue);
 693
 694     /**
 695      * @see Type#getValue(PropertyValue)
 696      */
 697     public abstract T getValue(Value value);
 698
 699     /**
 700      * @return {@code true} if a value of this {@code Type} is set on the given propertyValue.
 701      */
 702     public abstract boolean hasValue(PropertyValue propertyValue);
 703
 704     /**
 705      * @return {@code true} if a value of this {@code Type} is set on the given propertyValue.
 706      */
 707     public abstract boolean hasValue(Value propertyValue);
 708
 709     /**
 710      * @return the {@link Meaning} for this {@link Type}
 711      */
 712     protected Meaning getV3Meaning() {
 713       return Meaning.NO_MEANING;
 714     }
 715   }
 716
 717   /**
 718    * A base class with common functions for types that have the same datastore representation.
 719    *
 720    * @param <S> the datastore type
 721    * @param <T> the canonical Java class for this type
 722    */
 723   private abstract static class BaseVariantType<S, T> extends Type<T> {
 724     /**
 725      * @return the datastore representation of the given value
 726      */
 727     protected abstract S toDatastoreValue(Object value);
 728     /**
 729      * @return the native representation of the given value
 730      */
 731     protected abstract T fromDatastoreValue(S datastoreValue);
 732   }
 733
 734   /**
 735    * Base class for types that store strings in the datastore.
 736    *
 737    * @param <T> the canonical Java class for this type
 738    */
 739   private abstract static class BaseStringType<T> extends BaseVariantType<String, T> {
 740     @Override
 741     public void toV3Value(Object value, PropertyValue propertyValue) {
 742       propertyValue.setStringValue(toDatastoreValue(value));
 743     }
 744
 745     @Override
 746     public boolean canBeIndexed() {
 747       return true;
 748     }
 749
 750     @Override
 751     public Value.Builder toV1Value(
 752         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 753       Value.Builder builder = Value.newBuilder();
 754       builder.setStringValue(toDatastoreValue(value));
 755       builder.setExcludeFromIndexes(!indexed);
 756       builder.setMeaning(getV3Meaning().getValue());
 757       return builder;
 758     }
 759
 760     @Override
 761     public final T getValue(PropertyValue propertyValue) {
 762       return fromDatastoreValue(propertyValue.getStringValue());
 763     }
 764
 765     @Override
 766     public T getValue(Value propertyValue) {
 767       return fromDatastoreValue(propertyValue.getStringValue());
 768     }
 769
 770     @Override
 771     public final boolean hasValue(PropertyValue propertyValue) {
 772       return propertyValue.hasStringValue();
 773     }
 774
 775     @Override
 776     public boolean hasValue(Value propertyValue) {
 777       return propertyValue.getValueTypeCase() == ValueTypeCase.STRING_VALUE;
 778     }
 779
 780     @Override
 781     public ComparableByteArray asComparable(Object value) {
 782       return new ComparableByteArray(ProtocolSupport.toBytesUtf8(toDatastoreValue(value)));
 783     }
 784   }
 785
 786   /**
 787    * Base class for types that store bytes in the datastore.
 788    *
 789    * @param <T> the canonical Java class for this type
 790    */
 791   private abstract static class BaseBlobType<T> extends BaseVariantType<byte[], T> {
 792     protected abstract boolean isIndexable();
 793
 794     @Override
 795     public final boolean hasValue(PropertyValue propertyValue) {
 796       return propertyValue.hasStringValue();
 797     }
 798
 799     @Override
 800     public final void toV3Value(Object value, PropertyValue propertyValue) {
 801       propertyValue.setStringValueAsBytes(toDatastoreValue(value));
 802     }
 803
 804     @Override
 805     public boolean canBeIndexed() {
 806       return isIndexable();
 807     }
 808
 809     @Override
 810     public Value.Builder toV1Value(
 811         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 812       Value.Builder builder = Value.newBuilder();
 813       builder.setBlobValue(ByteString.copyFrom(toDatastoreValue(value)));
 814       builder.setExcludeFromIndexes(!indexed || !isIndexable());
 815       return builder;
 816     }
 817
 818     @Override
 819     public final T getValue(PropertyValue propertyValue) {
 820       return fromDatastoreValue(propertyValue.getStringValueAsBytes());
 821     }
 822
 823     @Override
 824     public final ComparableByteArray asComparable(Object value) {
 825       return isIndexable() ? new ComparableByteArray(toDatastoreValue(value)) : null;
 826     }
 827   }
 828
 829   /**
 830    * Base class for types that store predefined entities in Cloud Datastore v1.
 831    *
 832    * @param <T> the canonical Java class for this type
 833    */
 834   private abstract static class BasePredefinedEntityType<T> extends Type<T> {
 835     /**
 836      * @return the predefined entity meaning to use in Cloud Datastore v1
 837      */
 838     protected abstract int getV1Meaning();
 839
 840     /**
 841      * @return the Cloud Datastore v1 Entity representation for the given value
 842      */
 843     protected abstract com.google.datastore.v1beta3.Entity getEntity(Object value);
 844
 845     @Override
 846     public final boolean isType(Value propertyValue) {
 847       return propertyValue.getMeaning() == getV1Meaning() && hasValue(propertyValue);
 848     }
 849
 850     @Override
 851     public final boolean hasValue(Value propertyValue) {
 852       return propertyValue.getValueTypeCase() == ValueTypeCase.ENTITY_VALUE;
 853     }
 854
 855     @Override
 856     public final Value.Builder toV1Value(
 857         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 858       Value.Builder builder = Value.newBuilder();
 859       builder.setEntityValue(getEntity(value));
 860       builder.setExcludeFromIndexes(!indexed);
 861       builder.setMeaning(getV1Meaning());
 862       return builder;
 863     }
 864   }
 865
 866   /**
 867    * Returns the Cloud Datastore v1 value representation for the given value, unindexed.
 868    */
 869   private static Value makeUnindexedValue(String value) {
 870     return Value.newBuilder().setStringValue(value).setExcludeFromIndexes(true).build();
 871   }
 872
 873   /**
 874    * Base class for types that int64 values in the datastore.
 875    *
 876    * @param <T> the canonical Java class for this type
 877    */
 878   private abstract static class BaseInt64Type<T> extends BaseVariantType<Long, T> {
 879     @Override
 880     public final void toV3Value(Object value, PropertyValue propertyValue) {
 881       propertyValue.setInt64Value(toDatastoreValue(value));
 882     }
 883
 884     @Override
 885     public boolean canBeIndexed() {
 886       return true;
 887     }
 888
 889     @Override
 890     public Value.Builder toV1Value(
 891         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 892       Value.Builder builder = Value.newBuilder();
 893       builder.setIntegerValue(toDatastoreValue(value));
 894       builder.setExcludeFromIndexes(!indexed);
 895       builder.setMeaning(getV3Meaning().getValue());
 896       return builder;
 897     }
 898
 899     @Override
 900     public T getValue(PropertyValue propertyValue) {
 901       return fromDatastoreValue(propertyValue.getInt64Value());
 902     }
 903
 904     @Override
 905     public T getValue(Value propertyValue) {
 906       return fromDatastoreValue(propertyValue.getIntegerValue());
 907     }
 908
 909     @Override
 910     public boolean hasValue(PropertyValue propertyValue) {
 911       return propertyValue.hasInt64Value();
 912     }
 913
 914     @Override
 915     public boolean hasValue(Value propertyValue) {
 916       return propertyValue.getValueTypeCase() == ValueTypeCase.INTEGER_VALUE;
 917     }
 918
 919     @Override
 920     public Long asComparable(Object value) {
 921       return toDatastoreValue(value);
 922     }
 923   }
 924
 925   /**
 926    * The type for projected index values.
 927    */
 928   private static final class RawValueType extends Type<RawValue> {
 929     @Override
 930     public Meaning getV3Meaning() {
 931       return Meaning.INDEX_VALUE;
 932     }
 933
 934     @Override
 935     public boolean hasValue(PropertyValue propertyValue) {
 936       return true;
 937     }
 938
 939     @Override
 940     public boolean hasValue(Value propertyValue) {
 941       return true;
 942     }
 943
 944     @Override
 945     public void toV3Value(Object value, PropertyValue propertyValue) {
 946       throw new UnsupportedOperationException();
 947     }
 948
 949     @Override
 950     public boolean canBeIndexed() {
 951       return false;
 952     }
 953
 954     @Override
 955     public Value.Builder toV1Value(
 956         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 957       throw new UnsupportedOperationException();
 958     }
 959
 960     @Override
 961     public RawValue getValue(PropertyValue propertyValue) {
 962       return new RawValue(propertyValue);
 963     }
 964
 965     @Override
 966     public RawValue getValue(Value propertyValue) {
 967       return new RawValue(propertyValue);
 968     }
 969
 970     @SuppressWarnings("unchecked")
 971     @Override
 972     public Comparable<?> asComparable(Object value) {
 973       value = ((RawValue) value).getValue();
 974       if (value instanceof byte[]) {
 975         return new ComparableByteArray((byte[]) value);
 976       }
 977       return (Comparable<?>) value;
 978     }
 979   }
 980
 981   /**
 982    * The raw String type.
 983    */
 984   private static final class StringType extends BaseStringType<String> {
 985     @Override
 986     protected String toDatastoreValue(Object value) {
 987       return value.toString();
 988     }
 989
 990     @Override
 991     protected String fromDatastoreValue(String datastoreValue) {
 992       return datastoreValue;
 993     }
 994   }
 995
 996   /**
 997    * The raw int64 type.
 998    */
 999   private static final class Int64Type extends BaseInt64Type<Long> {
1000     @Override
1001     protected Long toDatastoreValue(Object value) {
1002       return ((Number) value).longValue();
1003     }
1004
1005     @Override
1006     protected Long fromDatastoreValue(Long datastoreValue) {
1007       return datastoreValue;
1008     }
1009   }
1010
1011   /**
1012    * The raw double type.
1013    */
1014   private static final class DoubleType extends Type<Double> {
1015     @Override
1016     public void toV3Value(Object value, PropertyValue propertyValue) {
1017       propertyValue.setDoubleValue(((Number) value).doubleValue());
1018     }
1019
1020     @Override
1021     public boolean canBeIndexed() {
1022       return true;
1023     }
1024
1025     @Override
1026     public Value.Builder toV1Value(
1027         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1028       Value.Builder builder = Value.newBuilder();
1029       builder.setDoubleValue(((Number) value).doubleValue());
1030       builder.setExcludeFromIndexes(!indexed);
1031       return builder;
1032     }
1033
1034     @Override
1035     public Double getValue(PropertyValue propertyValue) {
1036       return propertyValue.getDoubleValue();
1037     }
1038
1039     @Override
1040     public Double getValue(Value propertyValue) {
1041       return propertyValue.getDoubleValue();
1042     }
1043
1044     @Override
1045     public boolean hasValue(PropertyValue propertyValue) {
1046       return propertyValue.hasDoubleValue();
1047     }
1048
1049     @Override
1050     public boolean hasValue(Value propertyValue) {
1051       return propertyValue.getValueTypeCase() == ValueTypeCase.DOUBLE_VALUE;
1052     }
1053
1054     @Override
1055     public Double asComparable(Object value) {
1056       return ((Number) value).doubleValue();
1057     }
1058   }
1059
1060   /**
1061    * The raw boolean type.
1062    */
1063   private static final class BoolType extends Type<Boolean> {
1064     @Override
1065     public void toV3Value(Object value, PropertyValue propertyValue) {
1066       propertyValue.setBooleanValue((Boolean) value);
1067     }
1068
1069     @Override
1070     public boolean canBeIndexed() {
1071       return true;
1072     }
1073
1074     @Override
1075     public Value.Builder toV1Value(
1076         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1077       Value.Builder builder = Value.newBuilder();
1078       builder.setBooleanValue((Boolean) value);
1079       builder.setExcludeFromIndexes(!indexed);
1080       return builder;
1081     }
1082
1083     @Override
1084     public Boolean getValue(PropertyValue propertyValue) {
1085       return propertyValue.isBooleanValue();
1086     }
1087
1088     @Override
1089     public Boolean getValue(Value propertyValue) {
1090       return propertyValue.getBooleanValue();
1091     }
1092
1093     @Override
1094     public boolean hasValue(PropertyValue propertyValue) {
1095       return propertyValue.hasBooleanValue();
1096     }
1097
1098     @Override
1099     public boolean hasValue(Value propertyValue) {
1100       return propertyValue.getValueTypeCase() == ValueTypeCase.BOOLEAN_VALUE;
1101     }
1102
1103     @Override
1104     public Boolean asComparable(Object value) {
1105       return (Boolean) value;
1106     }
1107
1108   }
1109
1110   /**
1111    * The user type.
1112    *
1113    * <p>Stored as an entity with a special meaning in v1.
1114    */
1115   private static final class UserType extends BasePredefinedEntityType<User> {
1116     public static final int MEANING_PREDEFINED_ENTITY_USER = 20;
1117     public static final String PROPERTY_NAME_EMAIL = "email";
1118     public static final String PROPERTY_NAME_AUTH_DOMAIN = "auth_domain";
1119     public static final String PROPERTY_NAME_USER_ID = "user_id";
1120
1121     @Override
1122     public int getV1Meaning() {
1123       return MEANING_PREDEFINED_ENTITY_USER;
1124     }
1125
1126     @Override
1127     public com.google.datastore.v1beta3.Entity getEntity(Object value) {
1128       User user = (User) value;
1129       com.google.datastore.v1beta3.Entity.Builder builder =
1130           com.google.datastore.v1beta3.Entity.newBuilder();
1131       builder.getMutableProperties().put(PROPERTY_NAME_EMAIL, makeUnindexedValue(user.getEmail()));
1132       builder.getMutableProperties().put(PROPERTY_NAME_AUTH_DOMAIN,
1133           makeUnindexedValue(user.getAuthDomain()));
1134       if (user.getUserId() != null) {
1135         builder.getMutableProperties().put(PROPERTY_NAME_USER_ID,
1136             makeUnindexedValue(user.getUserId()));
1137       }
1138       return builder.build();
1139     }
1140
1141     @Override
1142     public void toV3Value(Object value, PropertyValue propertyValue) {
1143       User user = (User) value;
1144       UserValue userValue = new UserValue();
1145       userValue.setEmail(user.getEmail());
1146       userValue.setAuthDomain(user.getAuthDomain());
1147       if (user.getUserId() != null) {
1148         userValue.setObfuscatedGaiaid(user.getUserId());
1149       }
1150       userValue.setGaiaid(0);
1151       propertyValue.setUserValue(userValue);
1152     }
1153
1154     @Override
1155     public boolean canBeIndexed() {
1156       return true;
1157     }
1158
1159     @Override
1160     public User getValue(PropertyValue propertyValue) {
1161       UserValue userValue = propertyValue.getUserValue();
1162       String userId = userValue.hasObfuscatedGaiaid() ? userValue.getObfuscatedGaiaid() : null;
1163       return new User(userValue.getEmail(), userValue.getAuthDomain(), userId);
1164     }
1165
1166     @Override
1167     public User getValue(Value propertyValue) {
1168       String email = "";
1169       String authDomain = "";
1170       String userId = null;
1171       for (Map.Entry<String, Value> prop
1172           : propertyValue.getEntityValueOrBuilder().getProperties().entrySet()) {
1173         if (prop.getKey().equals(PROPERTY_NAME_EMAIL)) {
1174           email = prop.getValue().getStringValue();
1175         } else if (prop.getKey().equals(PROPERTY_NAME_AUTH_DOMAIN)) {
1176           authDomain = prop.getValue().getStringValue();
1177         } else if (prop.getKey().equals(PROPERTY_NAME_USER_ID)) {
1178           userId = prop.getValue().getStringValue();
1179         }
1180       }
1181       return new User(email, authDomain, userId);
1182     }
1183
1184     @Override
1185     public boolean hasValue(PropertyValue propertyValue) {
1186       return propertyValue.hasUserValue();
1187     }
1188
1189     @Override
1190     public final Comparable<User> asComparable(Object value) {
1191       return (User) value;
1192     }
1193   }
1194
1195   /**
1196    * The GeoPt type.
1197    *
1198    * <p>Stored as a GeoPoint value with no meaning in Cloud Datastore v1.
1199    */
1200   private static class GeoPtType extends Type<GeoPt> {
1201     @Override
1202     public boolean isType(Value propertyValue) {
1203       return propertyValue.getValueTypeCase() == ValueTypeCase.GEO_POINT_VALUE
1204           && propertyValue.getMeaning() == 0;
1205     }
1206
1207     @Override
1208     public void toV3Value(Object value, PropertyValue propertyValue) {
1209       GeoPt geoPt = (GeoPt) value;
1210       PropertyValue.PointValue pv = new PropertyValue.PointValue()
1211           .setX(geoPt.getLatitude())
1212           .setY(geoPt.getLongitude());
1213       propertyValue.setPointValue(pv);
1214     }
1215
1216     @Override
1217     public boolean canBeIndexed() {
1218       return true;
1219     }
1220
1221     @Override
1222     public final Value.Builder toV1Value(
1223         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1224       GeoPt geoPt = (GeoPt) value;
1225       Value.Builder builder = Value.newBuilder();
1226       builder.getGeoPointValueBuilder()
1227           .setLatitude(geoPt.getLatitude())
1228           .setLongitude(geoPt.getLongitude());
1229       builder.setExcludeFromIndexes(!indexed);
1230       return builder;
1231     }
1232
1233     @Override
1234     public GeoPt getValue(PropertyValue propertyValue) {
1235       PropertyValue.PointValue pv = propertyValue.getPointValue();
1236       return new GeoPt((float) pv.getX(), (float) pv.getY());
1237     }
1238
1239     @Override
1240     public GeoPt getValue(Value propertyValue) {
1241       return new GeoPt(
1242           (float) propertyValue.getGeoPointValue().getLatitude(),
1243           (float) propertyValue.getGeoPointValue().getLongitude());
1244     }
1245
1246     @Override
1247     public boolean hasValue(PropertyValue propertyValue) {
1248       return propertyValue.hasPointValue();
1249     }
1250
1251     @Override
1252     public final boolean hasValue(Value propertyValue) {
1253       return propertyValue.getValueTypeCase() == ValueTypeCase.GEO_POINT_VALUE;
1254     }
1255
1256     @Override
1257     public Meaning getV3Meaning() {
1258       return Meaning.GEORSS_POINT;
1259     }
1260
1261     @Override
1262     public final Comparable<GeoPt> asComparable(Object value) {
1263       return (GeoPt) value;
1264     }
1265   }
1266
1267   /**
1268    * The key/reference type.
1269    */
1270   private static final class KeyType extends Type<Key> {
1271     @Override
1272     public void toV3Value(Object value, PropertyValue propertyValue) {
1273       Reference keyRef = KeyTranslator.convertToPb((Key) value);
1274       propertyValue.setReferenceValue(toReferenceValue(keyRef));
1275     }
1276
1277     @Override
1278     public boolean canBeIndexed() {
1279       return true;
1280     }
1281
1282     @Override
1283     public Value.Builder toV1Value(
1284         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1285       Value.Builder builder = Value.newBuilder();
1286       builder.setKeyValue(toV1Key((Key) value));
1287       builder.setExcludeFromIndexes(!indexed);
1288       return builder;
1289     }
1290
1291     @Override
1292     public Key getValue(PropertyValue propertyValue) {
1293       return KeyTranslator.createFromPb(toReference(propertyValue.getReferenceValue()));
1294     }
1295
1296     @Override
1297     public Key getValue(Value propertyValue) {
1298       return toKey(propertyValue.getKeyValue());
1299     }
1300
1301     @Override
1302     public boolean hasValue(PropertyValue propertyValue) {
1303       return propertyValue.hasReferenceValue();
1304     }
1305
1306     @Override
1307     public boolean hasValue(Value propertyValue) {
1308       return propertyValue.getValueTypeCase() == ValueTypeCase.KEY_VALUE;
1309     }
1310
1311     @Override
1312     public Key asComparable(Object value) {
1313       return (Key) value;
1314     }
1315
1316     private static ReferenceValue toReferenceValue(Reference keyRef) {
1317       ReferenceValue refValue = new ReferenceValue();
1318       refValue.setApp(keyRef.getApp());
1319       if (keyRef.hasNameSpace()) {
1320         refValue.setNameSpace(keyRef.getNameSpace());
1321       }
1322       Path path = keyRef.getPath();
1323       for (Element element : path.elements()) {
1324         ReferenceValuePathElement newElement = new ReferenceValuePathElement();
1325         newElement.setType(element.getType());
1326         if (element.hasName()) {
1327           newElement.setName(element.getName());
1328         }
1329         if (element.hasId()) {
1330           newElement.setId(element.getId());
1331         }
1332         refValue.addPathElement(newElement);
1333       }
1334
1335       return refValue;
1336     }
1337
1338     private static Reference toReference(ReferenceValue refValue) {
1339       Reference reference = new Reference();
1340       reference.setApp(refValue.getApp());
1341       if (refValue.hasNameSpace()) {
1342         reference.setNameSpace(refValue.getNameSpace());
1343       }
1344       Path path = new Path();
1345       for (ReferenceValuePathElement element : refValue.pathElements()) {
1346         Element newElement = new Element();
1347         newElement.setType(element.getType());
1348         if (element.hasName()) {
1349           newElement.setName(element.getName());
1350         }
1351         if (element.hasId()) {
1352           newElement.setId(element.getId());
1353         }
1354         path.addElement(newElement);
1355       }
1356       reference.setPath(path);
1357       return reference;
1358     }
1359   }
1360
1361   /**
1362    * The non-indexable blob type.
1363    */
1364   private static class BlobType extends BaseBlobType<Blob> {
1365     @Override
1366     public Meaning getV3Meaning() {
1367       return Meaning.BLOB;
1368     }
1369
1370     @Override
1371     public boolean isType(Value propertyValue) {
1372       return getV3MeaningOf(propertyValue) == Meaning.NO_MEANING
1373           && propertyValue.getExcludeFromIndexes()
1374           && hasValue(propertyValue);
1375     }
1376
1377     @Override
1378     public boolean hasValue(Value propertyValue) {
1379       return propertyValue.getValueTypeCase() == ValueTypeCase.BLOB_VALUE;
1380     }
1381
1382     @Override
1383     public Blob getValue(Value propertyValue) {
1384       return fromDatastoreValue(propertyValue.getBlobValue().toByteArray());
1385     }
1386
1387     @Override
1388     protected Blob fromDatastoreValue(byte[] datastoreValue) {
1389       return new Blob(datastoreValue);
1390     }
1391
1392     @Override
1393     protected byte[] toDatastoreValue(Object value) {
1394       return ((Blob) value).getBytes();
1395     }
1396
1397     @Override
1398     public boolean isIndexable() {
1399       return false;
1400     }
1401   }
1402
1403   /**
1404    * The indexable blob type.
1405    */
1406   private static class ShortBlobType extends BaseBlobType<ShortBlob> {
1407     @Override
1408     public Meaning getV3Meaning() {
1409       return Meaning.BYTESTRING;
1410     }
1411
1412     @Override
1413     public boolean isType(Value propertyValue) {
1414       if (!hasValue(propertyValue)) {
1415         return false;
1416       }
1417
1418       if (propertyValue.getExcludeFromIndexes()) {
1419         return getV3MeaningOf(propertyValue) == getV3Meaning();
1420       } else {
1421         return getV3MeaningOf(propertyValue) == Meaning.NO_MEANING;
1422       }
1423     }
1424
1425     @Override
1426     public Value.Builder toV1Value(
1427         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1428       Value.Builder builder = super.toV1Value(value, indexed, forceIndexedEmbeddedEntity);
1429       if (!indexed) {
1430         builder.setMeaning(getV3Meaning().getValue());
1431       }
1432       return builder;
1433     }
1434
1435     @Override
1436     public boolean hasValue(Value propertyValue) {
1437       return propertyValue.getValueTypeCase() == ValueTypeCase.BLOB_VALUE
1438           || (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1439               && propertyValue.getValueTypeCase() == ValueTypeCase.STRING_VALUE);
1440     }
1441
1442     @Override
1443     public ShortBlob getValue(Value propertyValue) {
1444       if (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1445           && propertyValue.getValueTypeCase() == ValueTypeCase.STRING_VALUE) {
1446         return fromDatastoreValue(propertyValue.getStringValueBytes().toByteArray());
1447       } else {
1448         return fromDatastoreValue(propertyValue.getBlobValue().toByteArray());
1449       }
1450     }
1451
1452     @Override
1453     protected byte[] toDatastoreValue(Object value) {
1454       return ((ShortBlob) value).getBytes();
1455     }
1456
1457     @Override
1458     protected ShortBlob fromDatastoreValue(byte[] datastoreValue) {
1459       return new ShortBlob(datastoreValue);
1460     }
1461
1462     @Override
1463     public boolean isIndexable() {
1464       return true;
1465     }
1466   }
1467
1468   /**
1469    * The entity type.
1470    *
1471    * Stored as a partially serialized EntityProto in V3.
1472    */
1473   private static final class EmbeddedEntityType extends Type<EmbeddedEntity> {
1474     @Override
1475     public Meaning getV3Meaning() {
1476       return Meaning.ENTITY_PROTO;
1477     }
1478
1479      @Override
1480     public boolean isType(Value propertyValue) {
1481       return getV3MeaningOf(propertyValue) == Meaning.NO_MEANING
1482           && hasValue(propertyValue);
1483     }
1484
1485     @Override
1486     public boolean hasValue(PropertyValue propertyValue) {
1487       return propertyValue.hasStringValue();
1488     }
1489
1490     @Override
1491     public boolean hasValue(Value propertyValue) {
1492       return propertyValue.getValueTypeCase() == ValueTypeCase.ENTITY_VALUE;
1493     }
1494
1495     @Override
1496     public EmbeddedEntity getValue(PropertyValue propertyValue) {
1497       EntityProto proto = new EntityProto();
1498       proto.mergeFrom(propertyValue.getStringValueAsBytes());
1499       EmbeddedEntity result = new EmbeddedEntity();
1500       if (proto.hasKey() && !proto.getKey().getApp().isEmpty()) {
1501         result.setKey(KeyTranslator.createFromPb(proto.getKey()));
1502       }
1503       extractPropertiesFromPb(proto, result.getPropertyMap());
1504       return result;
1505     }
1506
1507     @Override
1508     public EmbeddedEntity getValue(Value propertyValue) {
1509       EmbeddedEntity result = new EmbeddedEntity();
1510       com.google.datastore.v1beta3.Entity proto = propertyValue.getEntityValue();
1511       if (proto.hasKey()) {
1512         result.setKey(toKey(proto.getKey()));
1513       }
1514       extractPropertiesFromPb(proto, false, result.getPropertyMap());
1515       return result;
1516     }
1517
1518     @Override
1519     public void toV3Value(Object value, PropertyValue propertyValue) {
1520       EmbeddedEntity structProp = (EmbeddedEntity) value;
1521       EntityProto proto = new EntityProto();
1522       if (structProp.getKey() != null) {
1523         proto.setKey(KeyTranslator.convertToPb(structProp.getKey()));
1524       }
1525       addPropertiesToPb(structProp.getPropertyMap(), proto);
1526       propertyValue.setStringValueAsBytes(proto.toByteArray());
1527     }
1528
1529     @Override
1530     public boolean canBeIndexed() {
1531       return false;
1532     }
1533
1534     @Override
1535     public Value.Builder toV1Value(
1536         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1537       EmbeddedEntity structProp = (EmbeddedEntity) value;
1538       Value.Builder builder = Value.newBuilder();
1539       com.google.datastore.v1beta3.Entity.Builder proto = builder.getEntityValueBuilder();
1540       if (structProp.getKey() != null) {
1541         proto.setKey(toV1Key(structProp.getKey()));
1542       }
1543       addPropertiesToPb(structProp.getPropertyMap(), proto);
1544       builder.setExcludeFromIndexes(!indexed || !forceIndexedEmbeddedEntity);
1545       return builder;
1546     }
1547
1548     @Override
1549     public Comparable<?> asComparable(Object value) {
1550       return null;
1551     }
1552   }
1553
1554   /**
1555    * The non-indexable {@link Text} type.
1556    */
1557   private static final class TextType extends BaseStringType<Text> {
1558     @Override
1559     public Meaning getV3Meaning() {
1560       return Meaning.TEXT;
1561     }
1562
1563     @Override
1564     public void toV3Value(Object value, PropertyValue propertyValue) {
1565       super.toV3Value(value, propertyValue);
1566     }
1567
1568     @Override
1569     public boolean canBeIndexed() {
1570       return false;
1571     }
1572
1573     @Override
1574     public Value.Builder toV1Value(
1575         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1576       return super.toV1Value(value, false, false);
1577     }
1578
1579     @Override
1580     protected Text fromDatastoreValue(String datastoreString) {
1581       return new Text(datastoreString);
1582     }
1583
1584     @Override
1585     protected String toDatastoreValue(Object value) {
1586       return ((Text) value).getValue();
1587     }
1588
1589     @Override
1590     public ComparableByteArray asComparable(Object value) {
1591       return null;
1592     }
1593   }
1594
1595   /**
1596    * The {@link BlobKey} type. Blob keys are just strings with a special meaning.
1597    */
1598   private static final class BlobKeyType extends BaseStringType<BlobKey> {
1599     @Override
1600     public Meaning getV3Meaning() {
1601       return Meaning.BLOBKEY;
1602     }
1603
1604     @Override
1605     public Value.Builder toV1Value(
1606         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1607       Value.Builder builder = Value.newBuilder();
1608       builder.setStringValue(toDatastoreValue(value));
1609       builder.setMeaning(Meaning.BLOBKEY.getValue());
1610       builder.setExcludeFromIndexes(!indexed);
1611       return builder;
1612     }
1613
1614     @Override
1615     public BlobKey getValue(Value propertyValue) {
1616       return fromDatastoreValue(propertyValue.getStringValue());
1617     }
1618
1619     @Override
1620     protected String toDatastoreValue(Object value) {
1621       return ((BlobKey) value).getKeyString();
1622     }
1623
1624     @Override
1625     protected BlobKey fromDatastoreValue(String datastoreString) {
1626       return new BlobKey(datastoreString);
1627     }
1628   }
1629
1630   /**
1631    * The date type.
1632    *
1633    * In V3 dates are just int64s with a special meaning.
1634    */
1635   private static final class DateType extends BaseInt64Type<Date> {
1636     @Override
1637     public Meaning getV3Meaning() {
1638       return Meaning.GD_WHEN;
1639     }
1640
1641     @Override
1642     public boolean isType(Value propertyValue) {
1643       return propertyValue.getMeaning() == 0 && hasValue(propertyValue);
1644     }
1645
1646     @Override
1647     public boolean hasValue(Value propertyValue) {
1648       return propertyValue.getValueTypeCase() == ValueTypeCase.TIMESTAMP_VALUE
1649           || (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1650               && propertyValue.getValueTypeCase() == ValueTypeCase.INTEGER_VALUE);
1651     }
1652
1653     @Override
1654     public Value.Builder toV1Value(
1655         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1656       Value.Builder builder = DatastoreHelper.makeValue((Date) value);
1657       builder.setExcludeFromIndexes(!indexed);
1658       return builder;
1659     }
1660
1661     @Override
1662     public Date getValue(Value propertyValue) {
1663       if (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1664           && propertyValue.getValueTypeCase() == ValueTypeCase.INTEGER_VALUE) {
1665         return fromDatastoreValue(propertyValue.getIntegerValue());
1666       } else {
1667         long datastoreValue = DatastoreHelper.getTimestamp(propertyValue);
1668         return fromDatastoreValue(datastoreValue);
1669       }
1670     }
1671
1672     @Override
1673     protected Long toDatastoreValue(Object value) {
1674       return ((Date) value).getTime() * 1000L;
1675     }
1676
1677     @Override
1678     protected Date fromDatastoreValue(Long datastoreValue) {
1679       return new Date(datastoreValue / 1000L);
1680     }
1681   }
1682
1683   /**
1684    * Internally a link is just a string with a special meaning.
1685    */
1686   private static final class LinkType extends BaseStringType<Link> {
1687     @Override
1688     public Meaning getV3Meaning() {
1689       return Meaning.ATOM_LINK;
1690     }
1691
1692     @Override
1693     protected String toDatastoreValue(Object value) {
1694       return ((Link) value).getValue();
1695     }
1696
1697     @Override
1698     protected Link fromDatastoreValue(String datastoreValue) {
1699       return new Link(datastoreValue);
1700     }
1701   }
1702
1703   /**
1704    * Internally a category is just a string with a special meaning.
1705    */
1706   private static final class CategoryType extends BaseStringType<Category> {
1707     @Override
1708     public Meaning getV3Meaning() {
1709       return Meaning.ATOM_CATEGORY;
1710     }
1711
1712     @Override
1713     protected String toDatastoreValue(Object value) {
1714       return ((Category) value).getCategory();
1715     }
1716
1717     @Override
1718     protected Category fromDatastoreValue(String datastoreString) {
1719       return new Category(datastoreString);
1720     }
1721   }
1722
1723   /**
1724    * Internally a rating is just an int64 with a special meaning.
1725    */
1726   private static final class RatingType extends BaseInt64Type<Rating> {
1727     @Override
1728     public Meaning getV3Meaning() {
1729       return Meaning.GD_RATING;
1730     }
1731
1732     @Override
1733     protected Long toDatastoreValue(Object value) {
1734       return (long) ((Rating) value).getRating();
1735     }
1736
1737     @Override
1738     protected Rating fromDatastoreValue(Long datastoreLong) {
1739       return new Rating(datastoreLong.intValue());
1740     }
1741   }
1742
1743   /**
1744    * Internally an email is just a string with a special meaning.
1745    */
1746   private static final class EmailType extends BaseStringType<Email> {
1747     @Override
1748     public Meaning getV3Meaning() {
1749       return Meaning.GD_EMAIL;
1750     }
1751
1752     @Override
1753     protected String toDatastoreValue(Object value) {
1754       return ((Email) value).getEmail();
1755     }
1756
1757     @Override
1758     protected Email fromDatastoreValue(String datastoreString) {
1759       return new Email(datastoreString);
1760     }
1761   }
1762
1763   /**
1764    * Internally a postal address is just a string with a special meaning.
1765    */
1766   private static final class PostalAddressType extends BaseStringType<PostalAddress> {
1767     @Override
1768     public Meaning getV3Meaning() {
1769       return Meaning.GD_POSTALADDRESS;
1770     }
1771
1772     @Override
1773     protected String toDatastoreValue(Object value) {
1774       return ((PostalAddress) value).getAddress();
1775     }
1776
1777     @Override
1778     protected PostalAddress fromDatastoreValue(String datastoreString) {
1779       return new PostalAddress(datastoreString);
1780     }
1781   }
1782
1783   /**
1784    * Internally a phone number is just a string with a special meaning.
1785    */
1786   private static final class PhoneNumberType extends BaseStringType<PhoneNumber> {
1787     @Override
1788     public Meaning getV3Meaning() {
1789       return Meaning.GD_PHONENUMBER;
1790     }
1791
1792     @Override
1793     protected String toDatastoreValue(Object value) {
1794       return ((PhoneNumber) value).getNumber();
1795     }
1796
1797     @Override
1798     protected PhoneNumber fromDatastoreValue(String datastoreString) {
1799       return new PhoneNumber(datastoreString);
1800     }
1801   }
1802
1803   /**
1804    * Internally an IM handle is just a string with a special meaning and a
1805    * well known format.
1806    */
1807   private static final class IMHandleType extends BaseStringType<IMHandle> {
1808     @Override
1809     public Meaning getV3Meaning() {
1810       return Meaning.GD_IM;
1811     }
1812
1813     @Override
1814     protected String toDatastoreValue(Object value) {
1815       return ((IMHandle) value).toDatastoreString();
1816     }
1817
1818     @Override
1819     protected IMHandle fromDatastoreValue(String datastoreString) {
1820       return IMHandle.fromDatastoreString(datastoreString);
1821     }
1822   }
1823
1824   static Map<Class<?>, Type<?>> getTypeMap() {
1825     return typeMap;
1826   }
1827
1828   /**
1829    * A wrapper for a {@code byte[]} that implements {@link Comparable}.
1830    * Comparison algorithm is the same as the prod datastore.
1831    */
1832   public static final class ComparableByteArray implements Comparable<ComparableByteArray> {
1833     private final byte[] bytes;
1834
1835     public ComparableByteArray(byte[] bytes) {
1836       this.bytes = bytes;
1837     }
1838
1839     @Override
1840     public int compareTo(ComparableByteArray other) {
1841       byte[] otherBytes = other.bytes;
1842       for (int i = 0; i < Math.min(bytes.length, otherBytes.length); i++) {
1843         int v1 = bytes[i] & 0xFF;
1844         int v2 = otherBytes[i] & 0xFF;
1845         if (v1 != v2) {
1846           return v1 - v2;
1847         }
1848       }
1849       return bytes.length - otherBytes.length;
1850     }
1851
1852     @Override
1853     public boolean equals(Object obj) {
1854       if (obj == null) {
1855         return false;
1856       }
1857       return Arrays.equals(bytes, ((ComparableByteArray) obj).bytes);
1858     }
1859
1860     @Override
1861     public int hashCode() {
1862       int result = 1;
1863       for (byte b : bytes) {
1864         result = 31 * result + b;
1865       }
1866       return result;
1867     }
1868   }
1869
1870   private DataTypeTranslator() {
1871   }
1872 }