java/src/main/com/google/appengine/api/datastore/DataTypeTranslator.java

   1 // Copyright 2007 Google Inc. All rights reserved.
   2
   3 package com.google.appengine.api.datastore;
   4
   5 import com.google.appengine.api.blobstore.BlobKey;
   6 import com.google.appengine.api.datastore.Entity.UnindexedValue;
   7 import com.google.appengine.api.datastore.Entity.WrappedValue;
   8 import com.google.appengine.api.datastore.Entity.WrappedValueImpl;
   9 import com.google.appengine.api.users.User;
  10 import com.google.apphosting.api.ApiProxy;
  11 import com.google.common.collect.Lists;
  12 import com.google.common.collect.Maps;
  13 import com.google.datastore.v1beta3.ArrayValue;
  14 import com.google.datastore.v1beta3.EntityOrBuilder;
  15 import com.google.datastore.v1beta3.Key.PathElement;
  16 import com.google.datastore.v1beta3.Key.PathElement.IdTypeCase;
  17 import com.google.datastore.v1beta3.KeyOrBuilder;
  18 import com.google.datastore.v1beta3.PartitionId;
  19 import com.google.datastore.v1beta3.PartitionIdOrBuilder;
  20 import com.google.datastore.v1beta3.Value;
  21 import com.google.datastore.v1beta3.Value.ValueTypeCase;
  22 import com.google.datastore.v1beta3.ValueOrBuilder;
  23 import com.google.datastore.v1beta3.client.DatastoreHelper;
  24 import com.google.io.protocol.ProtocolSupport;
  25 import com.google.protobuf.ByteString;
  26 import com.google.protobuf.NullValue;
  27 import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
  28 import com.google.storage.onestore.v3.OnestoreEntity.Path;
  29 import com.google.storage.onestore.v3.OnestoreEntity.Path.Element;
  30 import com.google.storage.onestore.v3.OnestoreEntity.Property;
  31 import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning;
  32 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue;
  33 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue;
  34 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValuePathElement;
  35 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.UserValue;
  36 import com.google.storage.onestore.v3.OnestoreEntity.Reference;
  37
  38 import java.util.ArrayList;
  39 import java.util.Arrays;
  40 import java.util.Collection;
  41 import java.util.Collections;
  42 import java.util.Date;
  43 import java.util.HashMap;
  44 import java.util.List;
  45 import java.util.Map;
  46
  47 /**
  48  * {@code DataTypeTranslator} is a utility class for converting
  49  * between the data store's {@code Property} protocol buffers and the
  50  * user-facing classes ({@code String}, {@code User}, etc.).
  51  *
  52  */
  53 public final class DataTypeTranslator  {
  54   /**
  55    * This key points to an (optional) map from project id to app id in the
  56    * {@link com.google.apphosting.api.ApiProxy.Environment} attribute map.
  57    */
  58   static final String ADDITIONAL_APP_IDS_MAP_ATTRIBUTE_KEY =
  59       "com.google.appengine.datastore.DataTypeTranslator.AdditionalAppIdsMap";
  60
  61   private static final RawValueType RAW_VALUE_TYPE = new RawValueType();
  62   /**
  63    * The list of supported types.
  64    *
  65    * Note: If you're going to modify this list, also update
  66    * DataTypeUtils. We're not building {@link DataTypeUtils#getSupportedTypes}
  67    * directly from this typeMap, because we want {@link DataTypeUtils} to be
  68    * translatable by GWT, so that {@link Entity Entities} can be easily sent
  69    * via GWT RPC.  Also, if you add a type here that is not immutable you'll
  70    * need to add special handling for it in {@link Entity#clone()}.
  71    */
  72   private static final Map<Class<?>, Type<?>> typeMap = Maps.newHashMap();
  73   static {
  74     typeMap.put(RawValue.class, RAW_VALUE_TYPE);
  75
  76     typeMap.put(Float.class, new DoubleType());
  77     typeMap.put(Double.class, new DoubleType());
  78
  79     typeMap.put(Byte.class, new Int64Type());
  80     typeMap.put(Short.class, new Int64Type());
  81     typeMap.put(Integer.class, new Int64Type());
  82     typeMap.put(Long.class, new Int64Type());
  83     typeMap.put(Date.class, new DateType());
  84     typeMap.put(Rating.class, new RatingType());
  85
  86     typeMap.put(String.class, new StringType());
  87     typeMap.put(Link.class, new LinkType());
  88     typeMap.put(ShortBlob.class, new ShortBlobType());
  89     typeMap.put(Category.class, new CategoryType());
  90     typeMap.put(PhoneNumber.class, new PhoneNumberType());
  91     typeMap.put(PostalAddress.class, new PostalAddressType());
  92     typeMap.put(Email.class, new EmailType());
  93     typeMap.put(IMHandle.class, new IMHandleType());
  94     typeMap.put(BlobKey.class, new BlobKeyType());
  95     typeMap.put(Blob.class, new BlobType());
  96     typeMap.put(Text.class, new TextType());
  97     typeMap.put(EmbeddedEntity.class, new EmbeddedEntityType());
  98
  99     typeMap.put(Boolean.class, new BoolType());
 100     typeMap.put(User.class, new UserType());
 101     typeMap.put(Key.class, new KeyType());
 102     typeMap.put(GeoPt.class, new GeoPtType());
 103
 104     assert typeMap.keySet().equals(DataTypeUtils.getSupportedTypes())
 105         : "Warning:  DataTypeUtils and DataTypeTranslator do not agree "
 106         + "about supported classes: " + typeMap.keySet() + " vs. "
 107         + DataTypeUtils.getSupportedTypes();
 108   }
 109
 110   /**
 111    * A map with the {@link Comparable} classes returned by all the instances of
 112    * {@link Type#asComparable(Object)} as keys and the pb code point as the value.
 113    * Used for comparing values that don't map to the same pb code point.
 114    */
 115   private static final
 116   Map<Class<? extends Comparable<?>>, Integer> comparableTypeMap =
 117       new HashMap<Class<? extends Comparable<?>>, Integer>();
 118
 119   static {
 120     comparableTypeMap.put(ComparableByteArray.class, PropertyValue.kstringValue);
 121     comparableTypeMap.put(Long.class, PropertyValue.kint64Value);
 122     comparableTypeMap.put(Double.class, PropertyValue.kdoubleValue);
 123     comparableTypeMap.put(Boolean.class, PropertyValue.kbooleanValue);
 124     comparableTypeMap.put(User.class, PropertyValue.kUserValueGroup);
 125     comparableTypeMap.put(Key.class, PropertyValue.kReferenceValueGroup);
 126     comparableTypeMap.put(GeoPt.class, PropertyValue.kPointValueGroup);
 127   }
 128
 129   /**
 130    * Add all of the properties in the specified map to an {@code EntityProto}.
 131    * This involves determining the type of each property and creating the
 132    * proper type-specific protocol buffer.
 133    *
 134    * If the property value is an {@link UnindexedValue}, or if it's a
 135    * type that is never indexed, e.g. {@code Text} and {@code Blob}, it's
 136    * added to {@code EntityProto.raw_property}. Otherwise it's added to
 137    * {@code EntityProto.property}.
 138    *
 139    * @param map A not {@code null} map of all the properties which will
 140    * be set on {@code proto}
 141    * @param proto A not {@code null} protocol buffer
 142    */
 143   public static void addPropertiesToPb(Map<String, Object> map, EntityProto proto) {
 144     for (Map.Entry<String, Object> entry : map.entrySet()) {
 145       String name = entry.getKey();
 146
 147       boolean forceIndexedEmbeddedEntity = false;
 148       boolean indexed = true;
 149       Object value = entry.getValue();
 150
 151       if (entry.getValue() instanceof WrappedValue) {
 152         WrappedValue wrappedValue = (WrappedValue) entry.getValue();
 153         forceIndexedEmbeddedEntity = wrappedValue.getForceIndexedEmbeddedEntity();
 154         indexed = wrappedValue.isIndexed();
 155         value = wrappedValue.getValue();
 156       }
 157
 158       if (value instanceof Collection<?>) {
 159         Collection<?> values = (Collection<?>) value;
 160         addListPropertyToPb(proto, name, indexed, values, forceIndexedEmbeddedEntity);
 161       } else {
 162         addPropertyToPb(name, value, indexed, forceIndexedEmbeddedEntity, false, proto);
 163       }
 164     }
 165   }
 166
 167   private static void addListPropertyToPb(EntityProto proto, String name, boolean indexed,
 168       Collection<?> values, boolean forceIndexedEmbeddedEntity) {
 169     if (values.isEmpty()) {
 170       Property property = new Property();
 171       property.setName(name);
 172       property.setMultiple(false);
 173       if (DatastoreServiceConfig.getEmptyListSupport()) {
 174         property.setMeaning(Meaning.EMPTY_LIST);
 175       } else {
 176       }
 177       property.getMutableValue();
 178       if (indexed) {
 179         proto.addProperty(property);
 180       } else {
 181         proto.addRawProperty(property);
 182       }
 183     } else {
 184       for (Object listValue : values) {
 185         addPropertyToPb(name, listValue, indexed, forceIndexedEmbeddedEntity,
 186             true, proto);
 187       }
 188     }
 189   }
 190
 191   /**
 192    * Adds a property to {@code entity}.
 193    *
 194    * @param name the property name
 195    * @param value the property value
 196    * @param indexed whether this property should be indexed. This may be
 197    *        overridden by property types like Blob and Text that are never indexed.
 198    * @param forceIndexedEmbeddedEntity whether indexed embedded entities should actually be indexed,
 199    *        as opposed to silently moved to unindexed properties (legacy behavior)
 200    * @param multiple whether this property has multiple values
 201    * @param entity the entity to populate
 202    */
 203   private static void addPropertyToPb(String name, Object value, boolean indexed,
 204       boolean forceIndexedEmbeddedEntity, boolean multiple, EntityProto entity) {
 205     Property property = new Property();
 206     property.setName(name);
 207     property.setMultiple(multiple);
 208     PropertyValue newValue = property.getMutableValue();
 209     if (value != null) {
 210       Type<?> type = getType(value.getClass());
 211       Meaning meaning = type.getV3Meaning();
 212       if (meaning != property.getMeaningEnum()) {
 213         property.setMeaning(meaning);
 214       }
 215       type.toV3Value(value, newValue);
 216       if (indexed && forceIndexedEmbeddedEntity
 217           && DataTypeUtils.isUnindexableType(value.getClass())) {
 218         throw new UnsupportedOperationException("Value must be indexable.");
 219       }
 220       if (!forceIndexedEmbeddedEntity || !(value instanceof EmbeddedEntity)) {
 221         indexed &= type.canBeIndexed();
 222       }
 223     }
 224     if (indexed) {
 225       entity.addProperty(property);
 226     } else {
 227       entity.addRawProperty(property);
 228     }
 229   }
 230
 231   static PropertyValue toV3Value(Object value) {
 232     PropertyValue propertyValue = new PropertyValue();
 233     if (value != null) {
 234       getType(value.getClass()).toV3Value(value, propertyValue);
 235     }
 236     return propertyValue;
 237   }
 238
 239   /**
 240    * Copy all of the indexed properties present on {@code proto} into {@code map}.
 241    */
 242   public static void extractIndexedPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 243     for (Property property : proto.propertys()) {
 244       addPropertyToMap(property, true, map);
 245     }
 246   }
 247
 248   /**
 249    * Copy all of the unindexed properties present on {@code proto} into {@code map}.
 250    */
 251   private static void extractUnindexedPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 252     for (Property property : proto.rawPropertys()) {
 253       addPropertyToMap(property, false, map);
 254     }
 255   }
 256
 257   /**
 258    * Copy all of the properties present on {@code proto} into {@code map}.
 259    */
 260   public static void extractPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 261     extractIndexedPropertiesFromPb(proto, map);
 262     extractUnindexedPropertiesFromPb(proto, map);
 263   }
 264
 265   /**
 266    * Copy all of the implicit properties present on {@code proto} into {@code map}.
 267    */
 268   public static void extractImplicitPropertiesFromPb(EntityProto proto, Map<String, Object> map) {
 269     for (Property property : getImplicitProperties(proto)) {
 270       addPropertyToMap(property, true, map);
 271     }
 272   }
 273
 274   private static Iterable<Property> getImplicitProperties(EntityProto proto) {
 275     return Collections.singleton(buildImplicitKeyProperty(proto));
 276   }
 277
 278   private static Property buildImplicitKeyProperty(EntityProto proto) {
 279     Property keyProp = new Property();
 280     keyProp.setName(Entity.KEY_RESERVED_PROPERTY);
 281     PropertyValue propVal = new PropertyValue();
 282     propVal.setReferenceValue(KeyType.toReferenceValue(proto.getKey()));
 283     keyProp.setValue(propVal);
 284     return keyProp;
 285   }
 286
 287   /**
 288    * Locates and returns all indexed properties with the given name on the
 289    * given proto. If there are a mix of matching multiple and non-multiple
 290    * properties, the collection will contain the first non-multiple property.
 291    * @return A list, potentially empty, containing matching properties.
 292    */
 293   public static Collection<Property> findIndexedPropertiesOnPb(
 294       EntityProto proto, String propertyName) {
 295     if (propertyName.equals(Entity.KEY_RESERVED_PROPERTY)) {
 296       return Collections.singleton(buildImplicitKeyProperty(proto));
 297     }
 298     List<Property> matchingMultipleProps = new ArrayList<>();
 299     for (Property prop : proto.propertys()) {
 300       if (prop.getName().equals(propertyName)) {
 301         if (!prop.isMultiple()) {
 302           return Collections.singleton(prop);
 303         } else {
 304           matchingMultipleProps.add(prop);
 305         }
 306       }
 307     }
 308     return matchingMultipleProps;
 309   }
 310
 311   private static Object wrapIfUnindexed(boolean indexed, Object value) {
 312     return indexed ? value : new UnindexedValue(value);
 313   }
 314
 315   private static void addPropertyToMap(Property property, boolean indexed,
 316       Map<String, Object> map) {
 317     String name = property.getName();
 318
 319     if (property.getMeaningEnum() == Meaning.EMPTY_LIST) {
 320       Object emptyListValue =
 321           DatastoreServiceConfig.getEmptyListSupport() ? new ArrayList<Object>() : null;
 322       map.put(name, wrapIfUnindexed(indexed, emptyListValue));
 323     } else {
 324       Object value = getPropertyValue(property);
 325       if (property.isMultiple()) {
 326         @SuppressWarnings({"unchecked"})
 327         List<Object> resultList = (List<Object>) PropertyContainer.unwrapValue(map.get(name));
 328         if (resultList == null) {
 329           resultList = new ArrayList<Object>();
 330           map.put(name, indexed ? resultList : new UnindexedValue(resultList));
 331         }
 332         if (indexed && value instanceof EmbeddedEntity) {
 333           map.put(name, new WrappedValueImpl(resultList, true, true));
 334         }
 335         resultList.add(value);
 336       } else {
 337         if (indexed && value instanceof EmbeddedEntity) {
 338           value = new WrappedValueImpl(value, true, true);
 339         } else if (!indexed) {
 340           value = new UnindexedValue(value);
 341         }
 342         map.put(name, value);
 343       }
 344     }
 345   }
 346
 347   /**
 348    * Returns the value for the property as its canonical type.
 349    *
 350    * @param property a not {@code null} property
 351    * @return {@code null} if no value was set for {@code property}
 352    */
 353   public static Object getPropertyValue(Property property) {
 354     PropertyValue value = property.getValue();
 355     for (Type<?> type : typeMap.values()) {
 356       if (type.isType(property.getMeaningEnum(), value)) {
 357         return type.getValue(value);
 358       }
 359     }
 360     return null;
 361   }
 362
 363   /**
 364    * @see #addPropertiesToPb(Map, EntityProto)
 365    */
 366   static void addPropertiesToPb(Map<String, Object> map,
 367       com.google.datastore.v1beta3.Entity.Builder proto) {
 368     for (Map.Entry<String, Object> entry : map.entrySet()) {
 369       proto.getMutableProperties().put(entry.getKey(), toV1Value(entry.getValue()).build());
 370     }
 371   }
 372
 373   /**
 374    * Copy all of the properties present on {@code proto} into {@code map}.
 375    *
 376    * <p>Cloud Datastore v1 entity must know if the proto came from an index-only query as the User
 377    * type overwrites the INDEX_ONLY meaning.
 378    *
 379    * @param proto the proto from which to extract properties
 380    * @param indexOnly if the proto is from an index only query (a projection query)
 381    * @param map the map to populate
 382    */
 383   static void extractPropertiesFromPb(EntityOrBuilder proto, boolean indexOnly,
 384       Map<String, Object> map) {
 385     if (indexOnly) {
 386       for (Map.Entry<String, Value> prop : proto.getProperties().entrySet()) {
 387         map.put(prop.getKey(), new RawValue(prop.getValue()));
 388       }
 389     } else {
 390       for (Map.Entry<String, Value> prop : proto.getProperties().entrySet()) {
 391         addPropertyToMap(prop.getKey(), prop.getValue(), map);
 392       }
 393     }
 394   }
 395
 396   static Value.Builder toV1Value(
 397       Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 398     if (value == null) {
 399       Value.Builder builder = Value.newBuilder();
 400       builder.setNullValue(NullValue.NULL_VALUE);
 401       builder.setExcludeFromIndexes(!indexed);
 402       return builder;
 403     } else if (indexed && forceIndexedEmbeddedEntity
 404         && DataTypeUtils.isUnindexableType(value.getClass())) {
 405       throw new UnsupportedOperationException("Value must be indexable.");
 406     }
 407     return getType(value.getClass()).toV1Value(value, indexed, forceIndexedEmbeddedEntity);
 408   }
 409
 410   private static Value.Builder toV1Value(Object value) {
 411     boolean indexed = true;
 412     boolean forceIndexedEmbeddedEntity = false;
 413
 414     if (value instanceof WrappedValue) {
 415       WrappedValue wrappedValue = (WrappedValue) value;
 416       indexed = wrappedValue.isIndexed();
 417       forceIndexedEmbeddedEntity = wrappedValue.getForceIndexedEmbeddedEntity();
 418       value = wrappedValue.getValue();
 419     }
 420
 421     if (value instanceof Collection<?>) {
 422       Collection<?> values = (Collection<?>) value;
 423       if (values.isEmpty()) {
 424         if (DatastoreServiceConfig.getEmptyListSupport()) {
 425           return Value.newBuilder().setExcludeFromIndexes(!indexed).setArrayValue(ArrayValue.newBuilder());
 426         } else {
 427          return toV1Value(null, indexed, forceIndexedEmbeddedEntity);
 428         }
 429       } else {
 430         Value.Builder valueBuilder = Value.newBuilder();
 431         for (Object listValue : values) {
 432           valueBuilder.getArrayValueBuilder().addValues(
 433               toV1Value(listValue, indexed, forceIndexedEmbeddedEntity));
 434         }
 435         return valueBuilder;
 436       }
 437     } else {
 438       return toV1Value(value, indexed, forceIndexedEmbeddedEntity);
 439     }
 440   }
 441
 442   private static void addPropertyToMap(String name, Value value, Map<String, Object> map) {
 443     boolean isOrContainsIndexedEntityValue = false;
 444     boolean indexed;
 445     Object result;
 446
 447     if (DatastoreServiceConfig.getEmptyListSupport()
 448         && value.getValueTypeCase() == Value.ValueTypeCase.ARRAY_VALUE
 449         && value.getArrayValue().getValuesCount() == 0) {
 450       result = new ArrayList<Object>();
 451       indexed = !value.getExcludeFromIndexes();
 452     } else if (value.getArrayValue().getValuesCount() > 0) {
 453       indexed = false;
 454       ArrayList<Object> resultList =
 455           new ArrayList<Object>(value.getArrayValue().getValuesCount());
 456       for (Value subValue : value.getArrayValue().getValuesList()) {
 457         if (subValue.getValueTypeCase() == ValueTypeCase.ARRAY_VALUE) {
 458           throw new IllegalArgumentException("Invalid Entity PB: list within a list.");
 459         }
 460         result = getValue(subValue);
 461         if (!subValue.getExcludeFromIndexes()) {
 462           indexed = true;
 463           if (result instanceof EmbeddedEntity) {
 464             isOrContainsIndexedEntityValue = true;
 465           }
 466         }
 467         resultList.add(result);
 468       }
 469       result = resultList;
 470     } else {
 471       indexed = !value.getExcludeFromIndexes();
 472       result = getValue(value);
 473       if (indexed && result instanceof EmbeddedEntity) {
 474         isOrContainsIndexedEntityValue = true;
 475       }
 476     }
 477
 478     if (isOrContainsIndexedEntityValue) {
 479       result = new WrappedValueImpl(result, true, true);
 480     } else if (!indexed) {
 481       result = new UnindexedValue(result);
 482     }
 483
 484     map.put(name, result);
 485   }
 486
 487   private static Object getValue(Value value) {
 488     for (Type<?> type : typeMap.values()) {
 489       if (type.isType(value)) {
 490         return type.getValue(value);
 491       }
 492     }
 493     return null;
 494   }
 495
 496   private static Meaning getV3MeaningOf(ValueOrBuilder value) {
 497     return Meaning.valueOf(value.getMeaning());
 498   }
 499
 500   private static AppIdNamespace toAppIdNamespace(PartitionIdOrBuilder partitionId) {
 501     if (partitionId.getProjectId().equals(DatastoreApiHelper.getCurrentProjectId())) {
 502       return new AppIdNamespace(DatastoreApiHelper.getCurrentAppId(), partitionId.getNamespaceId());
 503     }
 504     Map<String, String> additionalProjectIdToAppIdMap = getAdditionalProjectIdToAppIdMap();
 505     if (additionalProjectIdToAppIdMap.containsKey(partitionId.getProjectId())) {
 506       return new AppIdNamespace(additionalProjectIdToAppIdMap.get(partitionId.getProjectId()),
 507           partitionId.getNamespaceId());
 508     } else {
 509       throw new IllegalStateException(String.format(
 510           "Could not determine app id corresponding to project id \"%s\". Please add the app id "
 511           + "to %s.",
 512           partitionId.getProjectId(), RemoteCloudDatastoreV1Proxy.ADDITIONAL_APP_IDS_VAR));
 513     }
 514   }
 515
 516   @SuppressWarnings("unchecked")
 517   private static Map<String, String> getAdditionalProjectIdToAppIdMap() {
 518     if (ApiProxy.getCurrentEnvironment() != null) {
 519       Object attribute = ApiProxy.getCurrentEnvironment().getAttributes()
 520           .get(ADDITIONAL_APP_IDS_MAP_ATTRIBUTE_KEY);
 521       if (attribute != null) {
 522         return (Map<String, String>) attribute;
 523       }
 524     }
 525     return Collections.emptyMap();
 526   }
 527
 528   private static PartitionId.Builder toV1PartitionId(AppIdNamespace appNs) {
 529     PartitionId.Builder builder = PartitionId.newBuilder();
 530     builder.setProjectId(DatastoreApiHelper.toProjectId(appNs.getAppId()));
 531     if (!appNs.getNamespace().isEmpty()) {
 532       builder.setNamespaceId(appNs.getNamespace());
 533     }
 534     return builder;
 535   }
 536
 537   static com.google.datastore.v1beta3.Key.Builder toV1Key(Key key) {
 538     com.google.datastore.v1beta3.Key.Builder builder =
 539         com.google.datastore.v1beta3.Key.newBuilder();
 540     builder.setPartitionId(toV1PartitionId(key.getAppIdNamespace()));
 541     List<PathElement> pathElementList = new ArrayList<>();
 542     do {
 543       PathElement.Builder pathElement = PathElement.newBuilder();
 544       pathElement.setKind(key.getKind());
 545       if (key.getName() != null) {
 546         pathElement.setName(key.getName());
 547       } else if (key.getId() != Key.NOT_ASSIGNED) {
 548         pathElement.setId(key.getId());
 549       }
 550       pathElementList.add(pathElement.build());
 551       key = key.getParent();
 552     } while (key != null);
 553     builder.addAllPath(Lists.reverse(pathElementList));
 554     return builder;
 555   }
 556
 557   static Key toKey(KeyOrBuilder proto) {
 558     if (proto.getPathCount() == 0) {
 559       throw new IllegalArgumentException("Invalid Key PB: no elements.");
 560     }
 561     AppIdNamespace appIdNamespace = toAppIdNamespace(proto.getPartitionId());
 562     Key key = null;
 563     for (PathElement e : proto.getPathList()) {
 564       String kind = e.getKind();
 565       key = new Key(kind, key, e.getId(),
 566           e.getIdTypeCase() == IdTypeCase.NAME ? e.getName() : null, appIdNamespace);
 567     }
 568     return key;
 569   }
 570
 571   static Entity toEntity(EntityOrBuilder entityV1) {
 572     Entity entity = new Entity(DataTypeTranslator.toKey(entityV1.getKey()));
 573     DataTypeTranslator.extractPropertiesFromPb(entityV1, false,
 574         entity.getPropertyMap());
 575     return entity;
 576   }
 577
 578   static Entity toEntity(EntityOrBuilder entityV1, Collection<Projection> projections) {
 579     Entity entity = new Entity(DataTypeTranslator.toKey(entityV1.getKey()));
 580
 581     Map<String, Object> values = Maps.newHashMap();
 582     DataTypeTranslator.extractPropertiesFromPb(entityV1, true, values);
 583     for (Projection projection : projections) {
 584       entity.setProperty(projection.getName(), projection.getValue(values));
 585     }
 586     return entity;
 587   }
 588
 589   static com.google.datastore.v1beta3.Entity.Builder toV1Entity(Entity entity) {
 590     com.google.datastore.v1beta3.Entity.Builder entityV1 =
 591         com.google.datastore.v1beta3.Entity.newBuilder();
 592     entityV1.setKey(toV1Key(entity.getKey()));
 593     addPropertiesToPb(entity.getPropertyMap(), entityV1);
 594     return entityV1;
 595   }
 596
 597   /**
 598    * Returns the value for the property as its comparable representation type.
 599    *
 600    * @param property a not {@code null} property
 601    * @return {@code null} if no value was set for {@code property}
 602    */
 603   @SuppressWarnings("unchecked")
 604   public static Comparable<Object> getComparablePropertyValue(Property property) {
 605     return (Comparable<Object>) RAW_VALUE_TYPE.asComparable(new RawValue(property.getValue()));
 606   }
 607
 608   /**
 609    * Converts the given {@link Object} into a supported value then returns it as
 610    * a comparable object so it can be compared to other data types.
 611    *
 612    * @param value any Object that can be converted into a supported DataType
 613    * @return {@code null} if value is null
 614    * @throws UnsupportedOperationException if value is not supported
 615    */
 616   @SuppressWarnings("unchecked")
 617   static Comparable<Object> getComparablePropertyValue(Object value) {
 618     return value == null ? null
 619         : (Comparable<Object>) getType(value.getClass()).asComparable(value);
 620   }
 621
 622   /**
 623    * Get the rank of the given datastore type relative to other datastore
 624    * types.  Note that datastore types do not necessarily have unique ranks.
 625    */
 626   @SuppressWarnings({"unchecked", "rawtypes"})
 627   public static int getTypeRank(Class<? extends Comparable> datastoreType) {
 628     return comparableTypeMap.get(datastoreType);
 629   }
 630
 631   /**
 632    * Gets the {@link Type} that knows how to translate objects of
 633    * type {@code clazz} into protocol buffers that the data store can
 634    * handle.
 635    * @throws UnsupportedOperationException if clazz is not supported
 636    */
 637   @SuppressWarnings("unchecked")
 638   private static <T> Type<T> getType(Class<T> clazz) {
 639     if (typeMap.containsKey(clazz)) {
 640       return (Type<T>) typeMap.get(clazz);
 641     } else {
 642       throw new UnsupportedOperationException("Unsupported data type: " + clazz.getName());
 643     }
 644   }
 645
 646   /**
 647    * {@code Type} is an abstract class that knows how to convert Java
 648    * objects of one or more types into datastore representations.
 649    *
 650    * @param <T> The canonical Java class for this type.
 651    */
 652   abstract static class Type<T> {
 653     /**
 654      * @return {@code true} if the given meaning and property value matches this {@link Type}.
 655      */
 656     public final boolean isType(Meaning meaning, PropertyValue propertyValue) {
 657       return meaning == getV3Meaning() && hasValue(propertyValue);
 658     }
 659
 660     /**
 661      * @return {@code true} if the given value matches this {@link Type}.
 662      */
 663     public boolean isType(Value propertyValue) {
 664       return getV3MeaningOf(propertyValue) == getV3Meaning() && hasValue(propertyValue);
 665     }
 666
 667     /**
 668      * Returns the {@link Comparable} for the given value, or
 669      * {@code null} if values of this type are not comparable.
 670      */
 671     public abstract Comparable<?> asComparable(Object value);
 672
 673     /**
 674      * Sets the value of {@code propertyValue} to {@code value}.
 675      */
 676     public abstract void toV3Value(Object value, PropertyValue propertyValue);
 677
 678     /**
 679      * @returns Whether the value is indexable
 680      */
 681     public abstract boolean canBeIndexed();
 682
 683     /**
 684      * Returns a new Cloud Datastore v1 Value for the given parameters.
 685      *
 686      * @param value the Java native value to convert
 687      * @param indexed the desired indexing, ignored for types that are not indexable
 688      * @return the Cloud Datastore v1 representation of the given value and desired indexing
 689      */
 690     public abstract Value.Builder toV1Value(
 691         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity);
 692
 693     /**
 694      * Returns the value of {@code propertyValue} as its canonical Java type.
 695      *
 696      * Use {@link #isType} first to determine if the property has a value of the given type.
 697      *
 698      * @param propertyValue a not {@code null} value representing this {@code Type}
 699      * @return the canonical Java representation of {@code propertyValue}.
 700      */
 701     public abstract T getValue(PropertyValue propertyValue);
 702
 703     /**
 704      * @see Type#getValue(PropertyValue)
 705      */
 706     public abstract T getValue(Value value);
 707
 708     /**
 709      * @return {@code true} if a value of this {@code Type} is set on the given propertyValue.
 710      */
 711     public abstract boolean hasValue(PropertyValue propertyValue);
 712
 713     /**
 714      * @return {@code true} if a value of this {@code Type} is set on the given propertyValue.
 715      */
 716     public abstract boolean hasValue(Value propertyValue);
 717
 718     /**
 719      * @return the {@link Meaning} for this {@link Type}
 720      */
 721     protected Meaning getV3Meaning() {
 722       return Meaning.NO_MEANING;
 723     }
 724   }
 725
 726   /**
 727    * A base class with common functions for types that have the same datastore representation.
 728    *
 729    * @param <S> the datastore type
 730    * @param <T> the canonical Java class for this type
 731    */
 732   private abstract static class BaseVariantType<S, T> extends Type<T> {
 733     /**
 734      * @return the datastore representation of the given value
 735      */
 736     protected abstract S toDatastoreValue(Object value);
 737     /**
 738      * @return the native representation of the given value
 739      */
 740     protected abstract T fromDatastoreValue(S datastoreValue);
 741   }
 742
 743   /**
 744    * Base class for types that store strings in the datastore.
 745    *
 746    * @param <T> the canonical Java class for this type
 747    */
 748   private abstract static class BaseStringType<T> extends BaseVariantType<String, T> {
 749     @Override
 750     public void toV3Value(Object value, PropertyValue propertyValue) {
 751       propertyValue.setStringValue(toDatastoreValue(value));
 752     }
 753
 754     @Override
 755     public boolean canBeIndexed() {
 756       return true;
 757     }
 758
 759     @Override
 760     public Value.Builder toV1Value(
 761         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 762       Value.Builder builder = Value.newBuilder();
 763       builder.setStringValue(toDatastoreValue(value));
 764       builder.setExcludeFromIndexes(!indexed);
 765       builder.setMeaning(getV3Meaning().getValue());
 766       return builder;
 767     }
 768
 769     @Override
 770     public final T getValue(PropertyValue propertyValue) {
 771       return fromDatastoreValue(propertyValue.getStringValue());
 772     }
 773
 774     @Override
 775     public T getValue(Value propertyValue) {
 776       return fromDatastoreValue(propertyValue.getStringValue());
 777     }
 778
 779     @Override
 780     public final boolean hasValue(PropertyValue propertyValue) {
 781       return propertyValue.hasStringValue();
 782     }
 783
 784     @Override
 785     public boolean hasValue(Value propertyValue) {
 786       return propertyValue.getValueTypeCase() == ValueTypeCase.STRING_VALUE;
 787     }
 788
 789     @Override
 790     public ComparableByteArray asComparable(Object value) {
 791       return new ComparableByteArray(ProtocolSupport.toBytesUtf8(toDatastoreValue(value)));
 792     }
 793   }
 794
 795   /**
 796    * Base class for types that store bytes in the datastore.
 797    *
 798    * @param <T> the canonical Java class for this type
 799    */
 800   private abstract static class BaseBlobType<T> extends BaseVariantType<byte[], T> {
 801     protected abstract boolean isIndexable();
 802
 803     @Override
 804     public final boolean hasValue(PropertyValue propertyValue) {
 805       return propertyValue.hasStringValue();
 806     }
 807
 808     @Override
 809     public final void toV3Value(Object value, PropertyValue propertyValue) {
 810       propertyValue.setStringValueAsBytes(toDatastoreValue(value));
 811     }
 812
 813     @Override
 814     public boolean canBeIndexed() {
 815       return isIndexable();
 816     }
 817
 818     @Override
 819     public Value.Builder toV1Value(
 820         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 821       Value.Builder builder = Value.newBuilder();
 822       builder.setBlobValue(ByteString.copyFrom(toDatastoreValue(value)));
 823       builder.setExcludeFromIndexes(!indexed || !isIndexable());
 824       return builder;
 825     }
 826
 827     @Override
 828     public final T getValue(PropertyValue propertyValue) {
 829       return fromDatastoreValue(propertyValue.getStringValueAsBytes());
 830     }
 831
 832     @Override
 833     public final ComparableByteArray asComparable(Object value) {
 834       return isIndexable() ? new ComparableByteArray(toDatastoreValue(value)) : null;
 835     }
 836   }
 837
 838   /**
 839    * Base class for types that store predefined entities in Cloud Datastore v1.
 840    *
 841    * @param <T> the canonical Java class for this type
 842    */
 843   private abstract static class BasePredefinedEntityType<T> extends Type<T> {
 844     /**
 845      * @return the predefined entity meaning to use in Cloud Datastore v1
 846      */
 847     protected abstract int getV1Meaning();
 848
 849     /**
 850      * @return the Cloud Datastore v1 Entity representation for the given value
 851      */
 852     protected abstract com.google.datastore.v1beta3.Entity getEntity(Object value);
 853
 854     @Override
 855     public final boolean isType(Value propertyValue) {
 856       return propertyValue.getMeaning() == getV1Meaning() && hasValue(propertyValue);
 857     }
 858
 859     @Override
 860     public final boolean hasValue(Value propertyValue) {
 861       return propertyValue.getValueTypeCase() == ValueTypeCase.ENTITY_VALUE;
 862     }
 863
 864     @Override
 865     public final Value.Builder toV1Value(
 866         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 867       Value.Builder builder = Value.newBuilder();
 868       builder.setEntityValue(getEntity(value));
 869       builder.setExcludeFromIndexes(!indexed);
 870       builder.setMeaning(getV1Meaning());
 871       return builder;
 872     }
 873   }
 874
 875   /**
 876    * Returns the Cloud Datastore v1 value representation for the given value, unindexed.
 877    */
 878   private static Value makeUnindexedValue(String value) {
 879     return Value.newBuilder().setStringValue(value).setExcludeFromIndexes(true).build();
 880   }
 881
 882   /**
 883    * Base class for types that int64 values in the datastore.
 884    *
 885    * @param <T> the canonical Java class for this type
 886    */
 887   private abstract static class BaseInt64Type<T> extends BaseVariantType<Long, T> {
 888     @Override
 889     public final void toV3Value(Object value, PropertyValue propertyValue) {
 890       propertyValue.setInt64Value(toDatastoreValue(value));
 891     }
 892
 893     @Override
 894     public boolean canBeIndexed() {
 895       return true;
 896     }
 897
 898     @Override
 899     public Value.Builder toV1Value(
 900         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 901       Value.Builder builder = Value.newBuilder();
 902       builder.setIntegerValue(toDatastoreValue(value));
 903       builder.setExcludeFromIndexes(!indexed);
 904       builder.setMeaning(getV3Meaning().getValue());
 905       return builder;
 906     }
 907
 908     @Override
 909     public T getValue(PropertyValue propertyValue) {
 910       return fromDatastoreValue(propertyValue.getInt64Value());
 911     }
 912
 913     @Override
 914     public T getValue(Value propertyValue) {
 915       return fromDatastoreValue(propertyValue.getIntegerValue());
 916     }
 917
 918     @Override
 919     public boolean hasValue(PropertyValue propertyValue) {
 920       return propertyValue.hasInt64Value();
 921     }
 922
 923     @Override
 924     public boolean hasValue(Value propertyValue) {
 925       return propertyValue.getValueTypeCase() == ValueTypeCase.INTEGER_VALUE;
 926     }
 927
 928     @Override
 929     public Long asComparable(Object value) {
 930       return toDatastoreValue(value);
 931     }
 932   }
 933
 934   /**
 935    * The type for projected index values.
 936    */
 937   private static final class RawValueType extends Type<RawValue> {
 938     @Override
 939     public Meaning getV3Meaning() {
 940       return Meaning.INDEX_VALUE;
 941     }
 942
 943     @Override
 944     public boolean hasValue(PropertyValue propertyValue) {
 945       return true;
 946     }
 947
 948     @Override
 949     public boolean hasValue(Value propertyValue) {
 950       return true;
 951     }
 952
 953     @Override
 954     public void toV3Value(Object value, PropertyValue propertyValue) {
 955       throw new UnsupportedOperationException();
 956     }
 957
 958     @Override
 959     public boolean canBeIndexed() {
 960       return false;
 961     }
 962
 963     @Override
 964     public Value.Builder toV1Value(
 965         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
 966       throw new UnsupportedOperationException();
 967     }
 968
 969     @Override
 970     public RawValue getValue(PropertyValue propertyValue) {
 971       return new RawValue(propertyValue);
 972     }
 973
 974     @Override
 975     public RawValue getValue(Value propertyValue) {
 976       return new RawValue(propertyValue);
 977     }
 978
 979     @SuppressWarnings("unchecked")
 980     @Override
 981     public Comparable<?> asComparable(Object value) {
 982       value = ((RawValue) value).getValue();
 983       if (value instanceof byte[]) {
 984         return new ComparableByteArray((byte[]) value);
 985       }
 986       return (Comparable<?>) value;
 987     }
 988   }
 989
 990   /**
 991    * The raw String type.
 992    */
 993   private static final class StringType extends BaseStringType<String> {
 994     @Override
 995     protected String toDatastoreValue(Object value) {
 996       return value.toString();
 997     }
 998
 999     @Override
1000     protected String fromDatastoreValue(String datastoreValue) {
1001       return datastoreValue;
1002     }
1003   }
1004
1005   /**
1006    * The raw int64 type.
1007    */
1008   private static final class Int64Type extends BaseInt64Type<Long> {
1009     @Override
1010     protected Long toDatastoreValue(Object value) {
1011       return ((Number) value).longValue();
1012     }
1013
1014     @Override
1015     protected Long fromDatastoreValue(Long datastoreValue) {
1016       return datastoreValue;
1017     }
1018   }
1019
1020   /**
1021    * The raw double type.
1022    */
1023   private static final class DoubleType extends Type<Double> {
1024     @Override
1025     public void toV3Value(Object value, PropertyValue propertyValue) {
1026       propertyValue.setDoubleValue(((Number) value).doubleValue());
1027     }
1028
1029     @Override
1030     public boolean canBeIndexed() {
1031       return true;
1032     }
1033
1034     @Override
1035     public Value.Builder toV1Value(
1036         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1037       Value.Builder builder = Value.newBuilder();
1038       builder.setDoubleValue(((Number) value).doubleValue());
1039       builder.setExcludeFromIndexes(!indexed);
1040       return builder;
1041     }
1042
1043     @Override
1044     public Double getValue(PropertyValue propertyValue) {
1045       return propertyValue.getDoubleValue();
1046     }
1047
1048     @Override
1049     public Double getValue(Value propertyValue) {
1050       return propertyValue.getDoubleValue();
1051     }
1052
1053     @Override
1054     public boolean hasValue(PropertyValue propertyValue) {
1055       return propertyValue.hasDoubleValue();
1056     }
1057
1058     @Override
1059     public boolean hasValue(Value propertyValue) {
1060       return propertyValue.getValueTypeCase() == ValueTypeCase.DOUBLE_VALUE;
1061     }
1062
1063     @Override
1064     public Double asComparable(Object value) {
1065       return ((Number) value).doubleValue();
1066     }
1067   }
1068
1069   /**
1070    * The raw boolean type.
1071    */
1072   private static final class BoolType extends Type<Boolean> {
1073     @Override
1074     public void toV3Value(Object value, PropertyValue propertyValue) {
1075       propertyValue.setBooleanValue((Boolean) value);
1076     }
1077
1078     @Override
1079     public boolean canBeIndexed() {
1080       return true;
1081     }
1082
1083     @Override
1084     public Value.Builder toV1Value(
1085         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1086       Value.Builder builder = Value.newBuilder();
1087       builder.setBooleanValue((Boolean) value);
1088       builder.setExcludeFromIndexes(!indexed);
1089       return builder;
1090     }
1091
1092     @Override
1093     public Boolean getValue(PropertyValue propertyValue) {
1094       return propertyValue.isBooleanValue();
1095     }
1096
1097     @Override
1098     public Boolean getValue(Value propertyValue) {
1099       return propertyValue.getBooleanValue();
1100     }
1101
1102     @Override
1103     public boolean hasValue(PropertyValue propertyValue) {
1104       return propertyValue.hasBooleanValue();
1105     }
1106
1107     @Override
1108     public boolean hasValue(Value propertyValue) {
1109       return propertyValue.getValueTypeCase() == ValueTypeCase.BOOLEAN_VALUE;
1110     }
1111
1112     @Override
1113     public Boolean asComparable(Object value) {
1114       return (Boolean) value;
1115     }
1116
1117   }
1118
1119   /**
1120    * The user type.
1121    *
1122    * <p>Stored as an entity with a special meaning in v1.
1123    */
1124   private static final class UserType extends BasePredefinedEntityType<User> {
1125     public static final int MEANING_PREDEFINED_ENTITY_USER = 20;
1126     public static final String PROPERTY_NAME_EMAIL = "email";
1127     public static final String PROPERTY_NAME_AUTH_DOMAIN = "auth_domain";
1128     public static final String PROPERTY_NAME_USER_ID = "user_id";
1129
1130     @Override
1131     public int getV1Meaning() {
1132       return MEANING_PREDEFINED_ENTITY_USER;
1133     }
1134
1135     @Override
1136     public com.google.datastore.v1beta3.Entity getEntity(Object value) {
1137       User user = (User) value;
1138       com.google.datastore.v1beta3.Entity.Builder builder =
1139           com.google.datastore.v1beta3.Entity.newBuilder();
1140       builder.getMutableProperties().put(PROPERTY_NAME_EMAIL, makeUnindexedValue(user.getEmail()));
1141       builder.getMutableProperties().put(PROPERTY_NAME_AUTH_DOMAIN,
1142           makeUnindexedValue(user.getAuthDomain()));
1143       if (user.getUserId() != null) {
1144         builder.getMutableProperties().put(PROPERTY_NAME_USER_ID,
1145             makeUnindexedValue(user.getUserId()));
1146       }
1147       return builder.build();
1148     }
1149
1150     @Override
1151     public void toV3Value(Object value, PropertyValue propertyValue) {
1152       User user = (User) value;
1153       UserValue userValue = new UserValue();
1154       userValue.setEmail(user.getEmail());
1155       userValue.setAuthDomain(user.getAuthDomain());
1156       if (user.getUserId() != null) {
1157         userValue.setObfuscatedGaiaid(user.getUserId());
1158       }
1159       userValue.setGaiaid(0);
1160       propertyValue.setUserValue(userValue);
1161     }
1162
1163     @Override
1164     public boolean canBeIndexed() {
1165       return true;
1166     }
1167
1168     @Override
1169     public User getValue(PropertyValue propertyValue) {
1170       UserValue userValue = propertyValue.getUserValue();
1171       String userId = userValue.hasObfuscatedGaiaid() ? userValue.getObfuscatedGaiaid() : null;
1172       return new User(userValue.getEmail(), userValue.getAuthDomain(), userId);
1173     }
1174
1175     @Override
1176     public User getValue(Value propertyValue) {
1177       String email = "";
1178       String authDomain = "";
1179       String userId = null;
1180       for (Map.Entry<String, Value> prop
1181           : propertyValue.getEntityValueOrBuilder().getProperties().entrySet()) {
1182         if (prop.getKey().equals(PROPERTY_NAME_EMAIL)) {
1183           email = prop.getValue().getStringValue();
1184         } else if (prop.getKey().equals(PROPERTY_NAME_AUTH_DOMAIN)) {
1185           authDomain = prop.getValue().getStringValue();
1186         } else if (prop.getKey().equals(PROPERTY_NAME_USER_ID)) {
1187           userId = prop.getValue().getStringValue();
1188         }
1189       }
1190       return new User(email, authDomain, userId);
1191     }
1192
1193     @Override
1194     public boolean hasValue(PropertyValue propertyValue) {
1195       return propertyValue.hasUserValue();
1196     }
1197
1198     @Override
1199     public final Comparable<User> asComparable(Object value) {
1200       return (User) value;
1201     }
1202   }
1203
1204   /**
1205    * The GeoPt type.
1206    *
1207    * <p>Stored as a GeoPoint value with no meaning in Cloud Datastore v1.
1208    */
1209   private static class GeoPtType extends Type<GeoPt> {
1210     @Override
1211     public boolean isType(Value propertyValue) {
1212       return propertyValue.getValueTypeCase() == ValueTypeCase.GEO_POINT_VALUE
1213           && propertyValue.getMeaning() == 0;
1214     }
1215
1216     @Override
1217     public void toV3Value(Object value, PropertyValue propertyValue) {
1218       GeoPt geoPt = (GeoPt) value;
1219       PropertyValue.PointValue pv = new PropertyValue.PointValue()
1220           .setX(geoPt.getLatitude())
1221           .setY(geoPt.getLongitude());
1222       propertyValue.setPointValue(pv);
1223     }
1224
1225     @Override
1226     public boolean canBeIndexed() {
1227       return true;
1228     }
1229
1230     @Override
1231     public final Value.Builder toV1Value(
1232         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1233       GeoPt geoPt = (GeoPt) value;
1234       Value.Builder builder = Value.newBuilder();
1235       builder.getGeoPointValueBuilder()
1236           .setLatitude(geoPt.getLatitude())
1237           .setLongitude(geoPt.getLongitude());
1238       builder.setExcludeFromIndexes(!indexed);
1239       return builder;
1240     }
1241
1242     @Override
1243     public GeoPt getValue(PropertyValue propertyValue) {
1244       PropertyValue.PointValue pv = propertyValue.getPointValue();
1245       return new GeoPt((float) pv.getX(), (float) pv.getY());
1246     }
1247
1248     @Override
1249     public GeoPt getValue(Value propertyValue) {
1250       return new GeoPt(
1251           (float) propertyValue.getGeoPointValue().getLatitude(),
1252           (float) propertyValue.getGeoPointValue().getLongitude());
1253     }
1254
1255     @Override
1256     public boolean hasValue(PropertyValue propertyValue) {
1257       return propertyValue.hasPointValue();
1258     }
1259
1260     @Override
1261     public final boolean hasValue(Value propertyValue) {
1262       return propertyValue.getValueTypeCase() == ValueTypeCase.GEO_POINT_VALUE;
1263     }
1264
1265     @Override
1266     public Meaning getV3Meaning() {
1267       return Meaning.GEORSS_POINT;
1268     }
1269
1270     @Override
1271     public final Comparable<GeoPt> asComparable(Object value) {
1272       return (GeoPt) value;
1273     }
1274   }
1275
1276   /**
1277    * The key/reference type.
1278    */
1279   private static final class KeyType extends Type<Key> {
1280     @Override
1281     public void toV3Value(Object value, PropertyValue propertyValue) {
1282       Reference keyRef = KeyTranslator.convertToPb((Key) value);
1283       propertyValue.setReferenceValue(toReferenceValue(keyRef));
1284     }
1285
1286     @Override
1287     public boolean canBeIndexed() {
1288       return true;
1289     }
1290
1291     @Override
1292     public Value.Builder toV1Value(
1293         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1294       Value.Builder builder = Value.newBuilder();
1295       builder.setKeyValue(toV1Key((Key) value));
1296       builder.setExcludeFromIndexes(!indexed);
1297       return builder;
1298     }
1299
1300     @Override
1301     public Key getValue(PropertyValue propertyValue) {
1302       return KeyTranslator.createFromPb(toReference(propertyValue.getReferenceValue()));
1303     }
1304
1305     @Override
1306     public Key getValue(Value propertyValue) {
1307       return toKey(propertyValue.getKeyValue());
1308     }
1309
1310     @Override
1311     public boolean hasValue(PropertyValue propertyValue) {
1312       return propertyValue.hasReferenceValue();
1313     }
1314
1315     @Override
1316     public boolean hasValue(Value propertyValue) {
1317       return propertyValue.getValueTypeCase() == ValueTypeCase.KEY_VALUE;
1318     }
1319
1320     @Override
1321     public Key asComparable(Object value) {
1322       return (Key) value;
1323     }
1324
1325     private static ReferenceValue toReferenceValue(Reference keyRef) {
1326       ReferenceValue refValue = new ReferenceValue();
1327       refValue.setApp(keyRef.getApp());
1328       if (keyRef.hasNameSpace()) {
1329         refValue.setNameSpace(keyRef.getNameSpace());
1330       }
1331       Path path = keyRef.getPath();
1332       for (Element element : path.elements()) {
1333         ReferenceValuePathElement newElement = new ReferenceValuePathElement();
1334         newElement.setType(element.getType());
1335         if (element.hasName()) {
1336           newElement.setName(element.getName());
1337         }
1338         if (element.hasId()) {
1339           newElement.setId(element.getId());
1340         }
1341         refValue.addPathElement(newElement);
1342       }
1343
1344       return refValue;
1345     }
1346
1347     private static Reference toReference(ReferenceValue refValue) {
1348       Reference reference = new Reference();
1349       reference.setApp(refValue.getApp());
1350       if (refValue.hasNameSpace()) {
1351         reference.setNameSpace(refValue.getNameSpace());
1352       }
1353       Path path = new Path();
1354       for (ReferenceValuePathElement element : refValue.pathElements()) {
1355         Element newElement = new Element();
1356         newElement.setType(element.getType());
1357         if (element.hasName()) {
1358           newElement.setName(element.getName());
1359         }
1360         if (element.hasId()) {
1361           newElement.setId(element.getId());
1362         }
1363         path.addElement(newElement);
1364       }
1365       reference.setPath(path);
1366       return reference;
1367     }
1368   }
1369
1370   /**
1371    * The non-indexable blob type.
1372    */
1373   private static class BlobType extends BaseBlobType<Blob> {
1374     @Override
1375     public Meaning getV3Meaning() {
1376       return Meaning.BLOB;
1377     }
1378
1379     @Override
1380     public boolean isType(Value propertyValue) {
1381       return getV3MeaningOf(propertyValue) == Meaning.NO_MEANING
1382           && propertyValue.getExcludeFromIndexes()
1383           && hasValue(propertyValue);
1384     }
1385
1386     @Override
1387     public boolean hasValue(Value propertyValue) {
1388       return propertyValue.getValueTypeCase() == ValueTypeCase.BLOB_VALUE;
1389     }
1390
1391     @Override
1392     public Blob getValue(Value propertyValue) {
1393       return fromDatastoreValue(propertyValue.getBlobValue().toByteArray());
1394     }
1395
1396     @Override
1397     protected Blob fromDatastoreValue(byte[] datastoreValue) {
1398       return new Blob(datastoreValue);
1399     }
1400
1401     @Override
1402     protected byte[] toDatastoreValue(Object value) {
1403       return ((Blob) value).getBytes();
1404     }
1405
1406     @Override
1407     public boolean isIndexable() {
1408       return false;
1409     }
1410   }
1411
1412   /**
1413    * The indexable blob type.
1414    */
1415   private static class ShortBlobType extends BaseBlobType<ShortBlob> {
1416     @Override
1417     public Meaning getV3Meaning() {
1418       return Meaning.BYTESTRING;
1419     }
1420
1421     @Override
1422     public boolean isType(Value propertyValue) {
1423       if (!hasValue(propertyValue)) {
1424         return false;
1425       }
1426
1427       if (propertyValue.getExcludeFromIndexes()) {
1428         return getV3MeaningOf(propertyValue) == getV3Meaning();
1429       } else {
1430         return getV3MeaningOf(propertyValue) == Meaning.NO_MEANING;
1431       }
1432     }
1433
1434     @Override
1435     public Value.Builder toV1Value(
1436         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1437       Value.Builder builder = super.toV1Value(value, indexed, forceIndexedEmbeddedEntity);
1438       if (!indexed) {
1439         builder.setMeaning(getV3Meaning().getValue());
1440       }
1441       return builder;
1442     }
1443
1444     @Override
1445     public boolean hasValue(Value propertyValue) {
1446       return propertyValue.getValueTypeCase() == ValueTypeCase.BLOB_VALUE
1447           || (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1448               && propertyValue.getValueTypeCase() == ValueTypeCase.STRING_VALUE);
1449     }
1450
1451     @Override
1452     public ShortBlob getValue(Value propertyValue) {
1453       if (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1454           && propertyValue.getValueTypeCase() == ValueTypeCase.STRING_VALUE) {
1455         return fromDatastoreValue(propertyValue.getStringValueBytes().toByteArray());
1456       } else {
1457         return fromDatastoreValue(propertyValue.getBlobValue().toByteArray());
1458       }
1459     }
1460
1461     @Override
1462     protected byte[] toDatastoreValue(Object value) {
1463       return ((ShortBlob) value).getBytes();
1464     }
1465
1466     @Override
1467     protected ShortBlob fromDatastoreValue(byte[] datastoreValue) {
1468       return new ShortBlob(datastoreValue);
1469     }
1470
1471     @Override
1472     public boolean isIndexable() {
1473       return true;
1474     }
1475   }
1476
1477   /**
1478    * The entity type.
1479    *
1480    * Stored as a partially serialized EntityProto in V3.
1481    */
1482   private static final class EmbeddedEntityType extends Type<EmbeddedEntity> {
1483     @Override
1484     public Meaning getV3Meaning() {
1485       return Meaning.ENTITY_PROTO;
1486     }
1487
1488      @Override
1489     public boolean isType(Value propertyValue) {
1490       return getV3MeaningOf(propertyValue) == Meaning.NO_MEANING
1491           && hasValue(propertyValue);
1492     }
1493
1494     @Override
1495     public boolean hasValue(PropertyValue propertyValue) {
1496       return propertyValue.hasStringValue();
1497     }
1498
1499     @Override
1500     public boolean hasValue(Value propertyValue) {
1501       return propertyValue.getValueTypeCase() == ValueTypeCase.ENTITY_VALUE;
1502     }
1503
1504     @Override
1505     public EmbeddedEntity getValue(PropertyValue propertyValue) {
1506       EntityProto proto = new EntityProto();
1507       boolean parsed = proto.mergeFrom(propertyValue.getStringValueAsBytes());
1508       if (!parsed) {
1509         throw new IllegalArgumentException("Could not parse EntityProto value");
1510       }
1511       EmbeddedEntity result = new EmbeddedEntity();
1512       if (proto.hasKey() && !proto.getKey().getApp().isEmpty()) {
1513         result.setKey(KeyTranslator.createFromPb(proto.getKey()));
1514       }
1515       extractPropertiesFromPb(proto, result.getPropertyMap());
1516       return result;
1517     }
1518
1519     @Override
1520     public EmbeddedEntity getValue(Value propertyValue) {
1521       EmbeddedEntity result = new EmbeddedEntity();
1522       com.google.datastore.v1beta3.Entity proto = propertyValue.getEntityValue();
1523       if (proto.hasKey()) {
1524         result.setKey(toKey(proto.getKey()));
1525       }
1526       extractPropertiesFromPb(proto, false, result.getPropertyMap());
1527       return result;
1528     }
1529
1530     @Override
1531     public void toV3Value(Object value, PropertyValue propertyValue) {
1532       EmbeddedEntity structProp = (EmbeddedEntity) value;
1533       EntityProto proto = new EntityProto();
1534       if (structProp.getKey() != null) {
1535         proto.setKey(KeyTranslator.convertToPb(structProp.getKey()));
1536       }
1537       addPropertiesToPb(structProp.getPropertyMap(), proto);
1538       propertyValue.setStringValueAsBytes(proto.toByteArray());
1539     }
1540
1541     @Override
1542     public boolean canBeIndexed() {
1543       return false;
1544     }
1545
1546     @Override
1547     public Value.Builder toV1Value(
1548         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1549       EmbeddedEntity structProp = (EmbeddedEntity) value;
1550       Value.Builder builder = Value.newBuilder();
1551       com.google.datastore.v1beta3.Entity.Builder proto = builder.getEntityValueBuilder();
1552       if (structProp.getKey() != null) {
1553         proto.setKey(toV1Key(structProp.getKey()));
1554       }
1555       addPropertiesToPb(structProp.getPropertyMap(), proto);
1556       builder.setExcludeFromIndexes(!indexed || !forceIndexedEmbeddedEntity);
1557       return builder;
1558     }
1559
1560     @Override
1561     public Comparable<?> asComparable(Object value) {
1562       return null;
1563     }
1564   }
1565
1566   /**
1567    * The non-indexable {@link Text} type.
1568    */
1569   private static final class TextType extends BaseStringType<Text> {
1570     @Override
1571     public Meaning getV3Meaning() {
1572       return Meaning.TEXT;
1573     }
1574
1575     @Override
1576     public void toV3Value(Object value, PropertyValue propertyValue) {
1577       super.toV3Value(value, propertyValue);
1578     }
1579
1580     @Override
1581     public boolean canBeIndexed() {
1582       return false;
1583     }
1584
1585     @Override
1586     public Value.Builder toV1Value(
1587         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1588       return super.toV1Value(value, false, false);
1589     }
1590
1591     @Override
1592     protected Text fromDatastoreValue(String datastoreString) {
1593       return new Text(datastoreString);
1594     }
1595
1596     @Override
1597     protected String toDatastoreValue(Object value) {
1598       return ((Text) value).getValue();
1599     }
1600
1601     @Override
1602     public ComparableByteArray asComparable(Object value) {
1603       return null;
1604     }
1605   }
1606
1607   /**
1608    * The {@link BlobKey} type. Blob keys are just strings with a special meaning.
1609    */
1610   private static final class BlobKeyType extends BaseStringType<BlobKey> {
1611     @Override
1612     public Meaning getV3Meaning() {
1613       return Meaning.BLOBKEY;
1614     }
1615
1616     @Override
1617     public Value.Builder toV1Value(
1618         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1619       Value.Builder builder = Value.newBuilder();
1620       builder.setStringValue(toDatastoreValue(value));
1621       builder.setMeaning(Meaning.BLOBKEY.getValue());
1622       builder.setExcludeFromIndexes(!indexed);
1623       return builder;
1624     }
1625
1626     @Override
1627     public BlobKey getValue(Value propertyValue) {
1628       return fromDatastoreValue(propertyValue.getStringValue());
1629     }
1630
1631     @Override
1632     protected String toDatastoreValue(Object value) {
1633       return ((BlobKey) value).getKeyString();
1634     }
1635
1636     @Override
1637     protected BlobKey fromDatastoreValue(String datastoreString) {
1638       return new BlobKey(datastoreString);
1639     }
1640   }
1641
1642   /**
1643    * The date type.
1644    *
1645    * In V3 dates are just int64s with a special meaning.
1646    */
1647   private static final class DateType extends BaseInt64Type<Date> {
1648     @Override
1649     public Meaning getV3Meaning() {
1650       return Meaning.GD_WHEN;
1651     }
1652
1653     @Override
1654     public boolean isType(Value propertyValue) {
1655       return propertyValue.getMeaning() == 0 && hasValue(propertyValue);
1656     }
1657
1658     @Override
1659     public boolean hasValue(Value propertyValue) {
1660       return propertyValue.getValueTypeCase() == ValueTypeCase.TIMESTAMP_VALUE
1661           || (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1662               && propertyValue.getValueTypeCase() == ValueTypeCase.INTEGER_VALUE);
1663     }
1664
1665     @Override
1666     public Value.Builder toV1Value(
1667         Object value, boolean indexed, boolean forceIndexedEmbeddedEntity) {
1668       Value.Builder builder = DatastoreHelper.makeValue((Date) value);
1669       builder.setExcludeFromIndexes(!indexed);
1670       return builder;
1671     }
1672
1673     @Override
1674     public Date getValue(Value propertyValue) {
1675       if (getV3MeaningOf(propertyValue) == Meaning.INDEX_VALUE
1676           && propertyValue.getValueTypeCase() == ValueTypeCase.INTEGER_VALUE) {
1677         return fromDatastoreValue(propertyValue.getIntegerValue());
1678       } else {
1679         long datastoreValue = DatastoreHelper.getTimestamp(propertyValue);
1680         return fromDatastoreValue(datastoreValue);
1681       }
1682     }
1683
1684     @Override
1685     protected Long toDatastoreValue(Object value) {
1686       return ((Date) value).getTime() * 1000L;
1687     }
1688
1689     @Override
1690     protected Date fromDatastoreValue(Long datastoreValue) {
1691       return new Date(datastoreValue / 1000L);
1692     }
1693   }
1694
1695   /**
1696    * Internally a link is just a string with a special meaning.
1697    */
1698   private static final class LinkType extends BaseStringType<Link> {
1699     @Override
1700     public Meaning getV3Meaning() {
1701       return Meaning.ATOM_LINK;
1702     }
1703
1704     @Override
1705     protected String toDatastoreValue(Object value) {
1706       return ((Link) value).getValue();
1707     }
1708
1709     @Override
1710     protected Link fromDatastoreValue(String datastoreValue) {
1711       return new Link(datastoreValue);
1712     }
1713   }
1714
1715   /**
1716    * Internally a category is just a string with a special meaning.
1717    */
1718   private static final class CategoryType extends BaseStringType<Category> {
1719     @Override
1720     public Meaning getV3Meaning() {
1721       return Meaning.ATOM_CATEGORY;
1722     }
1723
1724     @Override
1725     protected String toDatastoreValue(Object value) {
1726       return ((Category) value).getCategory();
1727     }
1728
1729     @Override
1730     protected Category fromDatastoreValue(String datastoreString) {
1731       return new Category(datastoreString);
1732     }
1733   }
1734
1735   /**
1736    * Internally a rating is just an int64 with a special meaning.
1737    */
1738   private static final class RatingType extends BaseInt64Type<Rating> {
1739     @Override
1740     public Meaning getV3Meaning() {
1741       return Meaning.GD_RATING;
1742     }
1743
1744     @Override
1745     protected Long toDatastoreValue(Object value) {
1746       return (long) ((Rating) value).getRating();
1747     }
1748
1749     @Override
1750     protected Rating fromDatastoreValue(Long datastoreLong) {
1751       return new Rating(datastoreLong.intValue());
1752     }
1753   }
1754
1755   /**
1756    * Internally an email is just a string with a special meaning.
1757    */
1758   private static final class EmailType extends BaseStringType<Email> {
1759     @Override
1760     public Meaning getV3Meaning() {
1761       return Meaning.GD_EMAIL;
1762     }
1763
1764     @Override
1765     protected String toDatastoreValue(Object value) {
1766       return ((Email) value).getEmail();
1767     }
1768
1769     @Override
1770     protected Email fromDatastoreValue(String datastoreString) {
1771       return new Email(datastoreString);
1772     }
1773   }
1774
1775   /**
1776    * Internally a postal address is just a string with a special meaning.
1777    */
1778   private static final class PostalAddressType extends BaseStringType<PostalAddress> {
1779     @Override
1780     public Meaning getV3Meaning() {
1781       return Meaning.GD_POSTALADDRESS;
1782     }
1783
1784     @Override
1785     protected String toDatastoreValue(Object value) {
1786       return ((PostalAddress) value).getAddress();
1787     }
1788
1789     @Override
1790     protected PostalAddress fromDatastoreValue(String datastoreString) {
1791       return new PostalAddress(datastoreString);
1792     }
1793   }
1794
1795   /**
1796    * Internally a phone number is just a string with a special meaning.
1797    */
1798   private static final class PhoneNumberType extends BaseStringType<PhoneNumber> {
1799     @Override
1800     public Meaning getV3Meaning() {
1801       return Meaning.GD_PHONENUMBER;
1802     }
1803
1804     @Override
1805     protected String toDatastoreValue(Object value) {
1806       return ((PhoneNumber) value).getNumber();
1807     }
1808
1809     @Override
1810     protected PhoneNumber fromDatastoreValue(String datastoreString) {
1811       return new PhoneNumber(datastoreString);
1812     }
1813   }
1814
1815   /**
1816    * Internally an IM handle is just a string with a special meaning and a
1817    * well known format.
1818    */
1819   private static final class IMHandleType extends BaseStringType<IMHandle> {
1820     @Override
1821     public Meaning getV3Meaning() {
1822       return Meaning.GD_IM;
1823     }
1824
1825     @Override
1826     protected String toDatastoreValue(Object value) {
1827       return ((IMHandle) value).toDatastoreString();
1828     }
1829
1830     @Override
1831     protected IMHandle fromDatastoreValue(String datastoreString) {
1832       return IMHandle.fromDatastoreString(datastoreString);
1833     }
1834   }
1835
1836   static Map<Class<?>, Type<?>> getTypeMap() {
1837     return typeMap;
1838   }
1839
1840   /**
1841    * A wrapper for a {@code byte[]} that implements {@link Comparable}.
1842    * Comparison algorithm is the same as the prod datastore.
1843    */
1844   public static final class ComparableByteArray implements Comparable<ComparableByteArray> {
1845     private final byte[] bytes;
1846
1847     public ComparableByteArray(byte[] bytes) {
1848       this.bytes = bytes;
1849     }
1850
1851     @Override
1852     public int compareTo(ComparableByteArray other) {
1853       byte[] otherBytes = other.bytes;
1854       for (int i = 0; i < Math.min(bytes.length, otherBytes.length); i++) {
1855         int v1 = bytes[i] & 0xFF;
1856         int v2 = otherBytes[i] & 0xFF;
1857         if (v1 != v2) {
1858           return v1 - v2;
1859         }
1860       }
1861       return bytes.length - otherBytes.length;
1862     }
1863
1864     @Override
1865     public boolean equals(Object obj) {
1866       if (obj == null) {
1867         return false;
1868       }
1869       return Arrays.equals(bytes, ((ComparableByteArray) obj).bytes);
1870     }
1871
1872     @Override
1873     public int hashCode() {
1874       int result = 1;
1875       for (byte b : bytes) {
1876         result = 31 * result + b;
1877       }
1878       return result;
1879     }
1880   }
1881
1882   private DataTypeTranslator() {
1883   }
1884 }