java/src/main/com/google/appengine/api/datastore/DatastoreServiceConfig.java

   1 // Copyright 2010 Google Inc. All Rights Reserved.
   2 package com.google.appengine.api.datastore;
   3
   4 import static com.google.appengine.api.datastore.RemoteCloudDatastoreV1Proxy.APP_ID_VAR;
   5 import static com.google.common.base.Preconditions.checkNotNull;
   6 import static com.google.datastore.v1beta3.client.DatastoreHelper.PROJECT_ID_ENV_VAR;
   7
   8 import com.google.appengine.api.datastore.ReadPolicy.Consistency;
   9 import com.google.apphosting.api.ApiProxy;
  10 import com.google.common.base.Preconditions;
  11
  12 import java.io.ByteArrayInputStream;
  13 import java.io.InputStream;
  14 import java.util.Collection;
  15
  16 /**
  17  * User-configurable properties of the datastore.
  18  *
  19  * Notes on usage:<br>
  20  * The recommended way to instantiate a {@code DatastoreServiceConfig} object
  21  * is to statically import {@link Builder}.* and invoke a static creation
  22  * method followed by an instance mutator (if needed):
  23  *
  24  * <pre>
  25  * import static com.google.appengine.api.datastore.DatastoreServiceConfig.Builder.*;
  26  * import com.google.appengine.api.datastore.ReadPolicy.Consistency;
  27  *
  28  * ...
  29  *
  30  * // eventually consistent reads
  31  * DatastoreServiceConfig config = withReadPolicy(new ReadPolicy(Consistency.EVENTUAL));
  32  *
  33  * // eventually consistent reads with a 5 second deadline
  34  * DatastoreServiceConfig config =
  35  *   withReadPolicy(new ReadPolicy(Consistency.EVENTUAL)).deadline(5.0);
  36  * </pre>
  37  *
  38  */
  39 public final class DatastoreServiceConfig {
  40
  41   /**
  42    * This is the name of a system property that determines how the Java SDK writes/reads empty lists
  43    * to/from Datastore.
  44    * <p>
  45    * Historically Datastore has not had the ability to represent an empty list in its persistent
  46    * store. Different SDKs have made different decisions on what to write to Datastore when a client
  47    * attempts to insert an empty list to the persistent store. The Java SDK has historically written
  48    * both empty lists and null values as null values to the persistent store.
  49    * <p>
  50    * With the release of this flag, Datastore can now represent empty lists within its persistent
  51    * store. This means that Datastore SDKs can distinguish between empty lists and null values. This
  52    * property controls whether this SDK writes an empty list as empty list or null to the persistent
  53    * store.
  54    * <p>
  55    * A note on queries: Null values can be indexed by Datastore which means they can interact with
  56    * queries. For example queries that find all entities with null values or order by a property
  57    * will include null values. Empty lists are not indexable by Datastore and so cannot appear in
  58    * similar queries.
  59    * <p>
  60    * Thus, if this flag was unset (the old behavior) and an empty list was stored into the
  61    * persistent store, it could appear in query results because it was really stored as a null
  62    * value.
  63    * <p>
  64    * If this flag is set (the new behavior) and an empty list is stored into the persistent store,
  65    * it will not appear it query results because it is stored as an empty list.
  66    *
  67    * <p>
  68    * When this variable is set to anything other than "true" the system provides legacy behavior.
  69    *
  70    * <ul>
  71    * <li>Null properties are written as null to Datastore
  72    * <li>Empty collections are written as null to Datastore
  73    * <li>A null is read as null from Datastore
  74    * <li>An empty collection is read as null. <b>Note that a read modify write of an entity with an
  75    * empty list will cause that list to be turned into a null value.</b>
  76    * </ul>
  77    *
  78    * <p>
  79    * When this variable is set to "true":
  80    * <ul>
  81    * <li>Null properties are written as null to Datastore
  82    * <li>Empty collections (#{@link Collection#isEmpty()}) are written as empty list to Datastore
  83    * <li>A null is read as null from Datastore
  84    * <li>An empty collection is read as null
  85    * <li>When reading from Datastore an empty list is returned as an empty {@link Collection}.
  86    * </ul>
  87    *
  88    * <p>
  89    * It is strongly recommended that this property be set to true in order to provide compatibility
  90    * with other Datastore SDK's, as well as future versions of Datastore.
  91    *
  92    * To set the flag: <code>
  93    * System.setProperty(DatastoreServiceConfig.DATASTORE_EMPTY_LIST_SUPPORT,
  94    *                    Boolean.TRUE.toString());
  95    * </code>
  96    */
  97   public static final String DATASTORE_EMPTY_LIST_SUPPORT = "DATASTORE_EMPTY_LIST_SUPPORT";
  98
  99   /**
 100    * Returns whether or not empty list support is enabled; see
 101    * {@link #DATASTORE_EMPTY_LIST_SUPPORT}.
 102    */
 103   public static boolean getEmptyListSupport() {
 104     String value = System.getProperty(DATASTORE_EMPTY_LIST_SUPPORT);
 105     return value != null && value.equals(Boolean.TRUE.toString());
 106   }
 107
 108   /**
 109    * Datastore service API version.
 110    */
 111   enum ApiVersion {
 112     V3(false),
 113     CLOUD_DATASTORE_V1_VIA_API_PROXY(false),
 114     CLOUD_DATASTORE_V1_REMOTE(true);
 115
 116     private final boolean isCloudService;
 117
 118     private ApiVersion(boolean isCloudService) {
 119       this.isCloudService = isCloudService;
 120     }
 121
 122     boolean isCloudService() {
 123       return isCloudService;
 124     }
 125   }
 126
 127   /**
 128    * The default maximum size a request RPC can be.
 129    */
 130   static final int DEFAULT_RPC_SIZE_LIMIT_BYTES = 1024 * 1024;
 131
 132   /**
 133    * The default maximum number of keys allowed in a Get request.
 134    */
 135   static final int DEFAULT_MAX_BATCH_GET_KEYS = 1000;
 136
 137   /**
 138    * The default maximum number of entities allowed in a Put or Delete request.
 139    */
 140   static final int DEFAULT_MAX_BATCH_WRITE_ENTITIES = 500;
 141
 142   /**
 143    * Name of a system property that users can specify to control the default
 144    * max entity groups per rpc.
 145    */
 146   static final String DEFAULT_MAX_ENTITY_GROUPS_PER_RPC_SYS_PROP =
 147       "appengine.datastore.defaultMaxEntityGroupsPerRpc";
 148
 149   /**
 150    * The default number of max entity groups per rpc.
 151    */
 152   static final int DEFAULT_MAX_ENTITY_GROUPS_PER_RPC = getDefaultMaxEntityGroupsPerRpc();
 153
 154   static final String CALLBACKS_CONFIG_SYS_PROP = "appengine.datastore.callbacksConfig";
 155
 156   static volatile DatastoreCallbacks CALLBACKS = null;
 157
 158   /**
 159    * Keep in sync with
 160    * com.google.appengine.tools.compilation.DatastoreCallbacksProcessor.CALLBACKS_CONFIG_FILE
 161    */
 162   private static final String CALLBACKS_CONFIG_FILE = "/META-INF/datastorecallbacks.xml";
 163
 164   static int getDefaultMaxEntityGroupsPerRpc() {
 165     return getDefaultMaxEntityGroupsPerRpc(DEFAULT_MAX_ENTITY_GROUPS_PER_RPC_SYS_PROP, 10);
 166   }
 167
 168   static final String DATASTORE_SERVICE_VERSION_SYS_PROP = "appengine.datastore.v1service";
 169   static final String FORCE_LOCAL_CLOUD_DATASTORE_V1_SYS_PROP_VAL = "force";
 170
 171   /**
 172    * Returns the value of the given system property as an int, or return the
 173    * default value if there is no property with that name set.
 174    */
 175   static int getDefaultMaxEntityGroupsPerRpc(String sysPropName, int defaultVal) {
 176     String sysPropVal = System.getProperty(sysPropName);
 177     return sysPropVal == null ? defaultVal : Integer.parseInt(sysPropVal);
 178   }
 179
 180   private static InputStream getCallbacksConfigInputStream() {
 181     InputStream is;
 182     String callbacksConfig = System.getProperty(CALLBACKS_CONFIG_SYS_PROP);
 183     if (callbacksConfig != null) {
 184       is = new ByteArrayInputStream(callbacksConfig.getBytes());
 185     } else {
 186       is = DatastoreServiceConfig.class.getResourceAsStream(CALLBACKS_CONFIG_FILE);
 187     }
 188     return is;
 189   }
 190
 191   private final DatastoreCallbacks instanceDatastoreCallbacks;
 192
 193   private ImplicitTransactionManagementPolicy implicitTransactionManagementPolicy =
 194       ImplicitTransactionManagementPolicy.NONE;
 195
 196   private ReadPolicy readPolicy = new ReadPolicy(Consistency.STRONG);
 197
 198   private Double deadline;
 199
 200   private AppIdNamespace appIdNamespace;
 201
 202   int maxRpcSizeBytes = DEFAULT_RPC_SIZE_LIMIT_BYTES;
 203   int maxBatchWriteEntities = DEFAULT_MAX_BATCH_WRITE_ENTITIES;
 204   int maxBatchReadEntities = DEFAULT_MAX_BATCH_GET_KEYS;
 205   int maxBatchAllocateIdKeys = DEFAULT_MAX_BATCH_WRITE_ENTITIES;
 206   int maxEntityGroupsPerRpc = DEFAULT_MAX_ENTITY_GROUPS_PER_RPC;
 207
 208   private EntityCacheConfig entityCacheConfig = null;
 209
 210   /**
 211    * If set, overrides the service API version.
 212    */
 213   private ApiVersion apiVersion = null;
 214
 215   private CloudDatastoreV1Proxy cloudDatastoreV1Proxy = null;
 216
 217   /**
 218    * Cannot be directly instantiated, use {@link Builder} instead.
 219    *
 220    * @param datastoreCallbacks the callback methods to invoke on specific datastore operations.
 221    * If null the datastore callbacks are derived from {@link #getCallbacksConfigInputStream}.
 222    */
 223   private DatastoreServiceConfig( DatastoreCallbacks datastoreCallbacks) {
 224     instanceDatastoreCallbacks = datastoreCallbacks;
 225   }
 226
 227   /**
 228    * Copy constructor
 229    */
 230   DatastoreServiceConfig(DatastoreServiceConfig config) {
 231     implicitTransactionManagementPolicy = config.implicitTransactionManagementPolicy;
 232     readPolicy = config.readPolicy;
 233     deadline = config.deadline;
 234     maxRpcSizeBytes = config.maxRpcSizeBytes;
 235     maxBatchWriteEntities = config.maxBatchWriteEntities;
 236     maxBatchReadEntities = config.maxBatchReadEntities;
 237     maxEntityGroupsPerRpc = config.maxEntityGroupsPerRpc;
 238     instanceDatastoreCallbacks = config.instanceDatastoreCallbacks;
 239     appIdNamespace = config.appIdNamespace;
 240     apiVersion = config.apiVersion;
 241     cloudDatastoreV1Proxy = config.cloudDatastoreV1Proxy;
 242   }
 243
 244   /**
 245    * Sets the implicit transaction management policy.
 246    * @param p the implicit transaction management policy to set.
 247    * @return {@code this} (for chaining)
 248    */
 249   public DatastoreServiceConfig implicitTransactionManagementPolicy(
 250       ImplicitTransactionManagementPolicy p) {
 251     if (p == null) {
 252       throw new NullPointerException("implicit transaction management policy must not be null");
 253     }
 254     implicitTransactionManagementPolicy = p;
 255     return this;
 256   }
 257
 258   /**
 259    * Sets the read policy.
 260    * @param readPolicy the read policy to set.
 261    * @return {@code this} (for chaining)
 262    */
 263   public DatastoreServiceConfig readPolicy(ReadPolicy readPolicy) {
 264     if (readPolicy == null) {
 265       throw new NullPointerException("read policy must not be null");
 266     }
 267     this.readPolicy = readPolicy;
 268     return this;
 269   }
 270
 271   /**
 272    * Sets the deadline, in seconds, for all rpcs initiated by the
 273    * {@link DatastoreService} with which this config is associated.
 274    * @param deadline the deadline to set.
 275    * @throws IllegalArgumentException if deadline is not positive
 276    * @return {@code this} (for chaining)
 277    */
 278   public DatastoreServiceConfig deadline(double deadline) {
 279     if (deadline <= 0.0) {
 280       throw new IllegalArgumentException("deadline must be > 0, got " + deadline);
 281     }
 282     this.deadline = deadline;
 283     return this;
 284   }
 285
 286   DatastoreServiceConfig appIdNamespace(AppIdNamespace appIdNamespace) {
 287     this.appIdNamespace = appIdNamespace;
 288     return this;
 289   }
 290
 291   /**
 292    * Sets the maximum number of entities that can be modified in a single RPC.
 293    * @param maxBatchWriteEntities the limit to set
 294    * @throws IllegalArgumentException if maxBatchWriteEntities is not greater
 295    * than zero
 296    * @return {@code this} (for chaining)
 297    */
 298   DatastoreServiceConfig maxBatchWriteEntities(int maxBatchWriteEntities) {
 299     if (maxBatchWriteEntities <= 0) {
 300       throw new IllegalArgumentException("maxBatchWriteEntities must be > 0, got "
 301           + maxBatchWriteEntities);
 302     }
 303     this.maxBatchWriteEntities = maxBatchWriteEntities;
 304     return this;
 305   }
 306
 307   /**
 308    * Sets the maximum number of entities that can be read in a single RPC.
 309    * @param maxBatchReadEntities the limit to set
 310    * @throws IllegalArgumentException if maxBatchReadEntities is not greater
 311    * than zero
 312    * @return {@code this} (for chaining)
 313    */
 314   DatastoreServiceConfig maxBatchReadEntities(int maxBatchReadEntities) {
 315     if (maxBatchReadEntities <= 0) {
 316       throw new IllegalArgumentException("maxBatchReadEntities must be > 0, got "
 317           + maxBatchReadEntities);
 318     }
 319     this.maxBatchReadEntities = maxBatchReadEntities;
 320     return this;
 321   }
 322
 323   /**
 324    * Sets the maximum size in bytes an RPC can be.
 325    *
 326    * The size of the request can be exceeded if the RPC cannot be split enough
 327    * to respect this limit. However there may be a hard limit on the RPC which,
 328    * if exceeded, will cause an exception to be thrown.
 329    *
 330    * @param maxRpcSizeBytes the limit to set
 331    * @throws IllegalArgumentException if maxRpcSizeBytes is not positive
 332    * @return {@code this} (for chaining)
 333    */
 334   DatastoreServiceConfig maxRpcSizeBytes(int maxRpcSizeBytes) {
 335     if (maxRpcSizeBytes < 0) {
 336       throw new IllegalArgumentException("maxRpcSizeBytes must be >= 0, got "
 337           + maxRpcSizeBytes);
 338     }
 339     this.maxRpcSizeBytes = maxRpcSizeBytes;
 340     return this;
 341   }
 342
 343   /**
 344    * Sets the maximum number of entity groups that can be represented in a
 345    * single rpc.
 346    *
 347    * For a non-transactional operation that involves more entity groups than the
 348    * maximum, the operation will be performed by executing multiple, asynchronous
 349    * rpcs to the datastore, each of which has no more entity groups represented
 350    * than the maximum.  So, if a put() operation has 8 entity groups and the
 351    * maximum is 3, we will send 3 rpcs, 2 with 3 entity groups and 1 with 2
 352    * entity groups.  This is a performance optimization - in many cases
 353    * multiple, small, asynchronous rpcs will finish faster than a single large
 354    * asynchronous rpc.  The optimal value for this property will be
 355    * application-specific, so experimentation is encouraged.
 356    *
 357    * @param maxEntityGroupsPerRpc the maximum number of entity groups per rpc
 358    * @throws IllegalArgumentException if maxEntityGroupsPerRpc is not greater
 359    * than zero
 360    * @return {@code this} (for chaining)
 361    */
 362   public DatastoreServiceConfig maxEntityGroupsPerRpc(int maxEntityGroupsPerRpc) {
 363     if (maxEntityGroupsPerRpc <= 0) {
 364       throw new IllegalArgumentException("maxEntityGroupsPerRpc must be > 0, got "
 365           + maxEntityGroupsPerRpc);
 366     }
 367     this.maxEntityGroupsPerRpc = maxEntityGroupsPerRpc;
 368     return this;
 369   }
 370
 371   /**
 372    * Sets the entity cache configuration to use for entity caching.
 373    *
 374    * To successfully use entity caching it is important that all application code that updates
 375    * or reads the same set of entities uses a {@code DatastoreServiceConfig} with the same entity
 376    * cache configuration.
 377    *
 378    * By default the datastore service does not do any entity caching.
 379    *
 380    * @param entityCacheConfig the entity cache configuration to use for entity caching.
 381    * @return {@code this} (for chaining)
 382    */
 383   DatastoreServiceConfig entityCacheConfig(EntityCacheConfig entityCacheConfig) {
 384     checkNotNull(entityCacheConfig, "The entityCacheConfig argument can not be null");
 385     this.entityCacheConfig = entityCacheConfig;
 386     return this;
 387   }
 388
 389   /**
 390    * @return The {@code ImplicitTransactionManagementPolicy} to use.
 391    */
 392   public ImplicitTransactionManagementPolicy getImplicitTransactionManagementPolicy() {
 393     return implicitTransactionManagementPolicy;
 394   }
 395
 396   /**
 397    * @return The {@code ReadPolicy} to use.
 398    */
 399   public ReadPolicy getReadPolicy() {
 400     return readPolicy;
 401   }
 402
 403   /**
 404    * @return The maximum number of entity groups per rpc.
 405    */
 406   public Integer getMaxEntityGroupsPerRpc() {
 407     return getMaxEntityGroupsPerRpcInternal();
 408   }
 409
 410   int getMaxEntityGroupsPerRpcInternal() {
 411     return maxEntityGroupsPerRpc;
 412   }
 413
 414   /**
 415    * @return The deadline to use.  Can be {@code null}.
 416    */
 417   public Double getDeadline() {
 418     return deadline;
 419   }
 420
 421   AppIdNamespace getAppIdNamespace() {
 422     return appIdNamespace == null ? DatastoreApiHelper.getCurrentAppIdNamespace() : appIdNamespace;
 423   }
 424
 425   DatastoreCallbacks getDatastoreCallbacks() {
 426     if (instanceDatastoreCallbacks != null) {
 427       return instanceDatastoreCallbacks;
 428     }
 429
 430     if (CALLBACKS == null) {
 431       InputStream is = getCallbacksConfigInputStream();
 432       if (is == null) {
 433         CALLBACKS = DatastoreCallbacks.NoOpDatastoreCallbacks.INSTANCE;
 434       } else {
 435         CALLBACKS = new DatastoreCallbacksImpl(is, false);
 436       }
 437     }
 438     return CALLBACKS;
 439   }
 440
 441   /**
 442    * @return the {@link EntityCacheConfig} to use or {@code null} if no entity cache configuration
 443    *     was specified.
 444    */
 445   EntityCacheConfig getEntityCacheConfig() {
 446     return entityCacheConfig;
 447   }
 448
 449   /**
 450    * Datastore service API version to use.
 451    */
 452   ApiVersion getApiVersion() {
 453     if (apiVersion != null) {
 454       return apiVersion;
 455     }
 456
 457     boolean useLocalCloudDatastoreV1 = FORCE_LOCAL_CLOUD_DATASTORE_V1_SYS_PROP_VAL.equals(
 458         System.getProperty(DATASTORE_SERVICE_VERSION_SYS_PROP));
 459     boolean useCloudDatastore = Boolean.valueOf(EnvProxy.getenv("DATASTORE_USE_CLOUD_DATASTORE"))
 460         || EnvProxy.getenv(PROJECT_ID_ENV_VAR) != null
 461         || EnvProxy.getenv(APP_ID_VAR) != null;
 462
 463     ApiVersion localApiVersion = ApiVersion.V3;
 464     if (useLocalCloudDatastoreV1 && useCloudDatastore) {
 465       throw new IllegalArgumentException("Cannot use both local and remote Cloud Datastore APIs.");
 466     } else if (useLocalCloudDatastoreV1) {
 467       localApiVersion = ApiVersion.CLOUD_DATASTORE_V1_VIA_API_PROXY;
 468     } else if (useCloudDatastore) {
 469       localApiVersion = ApiVersion.CLOUD_DATASTORE_V1_REMOTE;
 470     }
 471
 472     apiVersion = localApiVersion;
 473     return apiVersion;
 474   }
 475
 476   DatastoreServiceConfig setApiVersion( ApiVersion apiVersion) {
 477     this.apiVersion = apiVersion;
 478     return this;
 479   }
 480
 481   ApiProxy.ApiConfig constructApiConfig() {
 482     ApiProxy.ApiConfig apiConfig = new ApiProxy.ApiConfig();
 483     apiConfig.setDeadlineInSeconds(deadline);
 484     return apiConfig;
 485   }
 486  CloudDatastoreV1Proxy getCloudDatastoreV1Proxy() {
 487     return cloudDatastoreV1Proxy;
 488   }
 489
 490   /**
 491    * Contains static creation methods for {@link DatastoreServiceConfig}.
 492    */
 493   public static final class Builder {
 494
 495     /**
 496      * Create a {@link DatastoreServiceConfig} with the given implicit
 497      * transaction management policy.
 498      * @param p the implicit transaction management policy to set.
 499      * @return The newly created DatastoreServiceConfig instance.
 500      */
 501     public static DatastoreServiceConfig withImplicitTransactionManagementPolicy(
 502         ImplicitTransactionManagementPolicy p) {
 503       return withDefaults().implicitTransactionManagementPolicy(p);
 504     }
 505
 506     /**
 507      * Create a {@link DatastoreServiceConfig} with the given read
 508      * policy.
 509      * @param readPolicy the read policy to set.
 510      * @return The newly created DatastoreServiceConfig instance.
 511      */
 512     public static DatastoreServiceConfig withReadPolicy(ReadPolicy readPolicy) {
 513       return withDefaults().readPolicy(readPolicy);
 514     }
 515
 516     /**
 517      * Create a {@link DatastoreServiceConfig} with the given deadline, in
 518      * seconds.
 519      * @param deadline the deadline to set.
 520      * @throws IllegalArgumentException if deadline is not positive
 521      * @return The newly created DatastoreServiceConfig instance.
 522      */
 523     public static DatastoreServiceConfig withDeadline(double deadline) {
 524       return withDefaults().deadline(deadline);
 525     }
 526
 527     /**
 528      * Create a {@link DatastoreServiceConfig} with the given maximum entity
 529      * groups per rpc.
 530      * @param maxEntityGroupsPerRpc the maximum entity groups per rpc to set.
 531      * @return The newly created DatastoreServiceConfig instance.
 532      *
 533      * @see DatastoreServiceConfig#maxEntityGroupsPerRpc(int)
 534      */
 535     public static DatastoreServiceConfig withMaxEntityGroupsPerRpc(int maxEntityGroupsPerRpc) {
 536       return withDefaults().maxEntityGroupsPerRpc(maxEntityGroupsPerRpc);
 537     }
 538
 539     /**
 540      * Create a {@link DatastoreServiceConfig} with the given entity cache configuration.
 541      *
 542      * @param entityCacheConfig the entity cache configuration to use for entity caching.
 543      *
 544      * @see DatastoreServiceConfig#entityCacheConfig
 545      */
 546     static DatastoreServiceConfig withEntityCacheConfig(
 547         EntityCacheConfig entityCacheConfig) {
 548       return withDefaults().entityCacheConfig(entityCacheConfig);
 549     }
 550
 551     /**
 552      * Helper method for creating a {@link DatastoreServiceConfig} instance with the
 553      * specified {@code datastoreCallbacks}. The callbacks defined for the application are
 554      * bypassed and the specified callbacks are used instead.
 555      *
 556      * @return The newly created DatastoreServiceConfig instance.
 557      */
 558     static DatastoreServiceConfig withDatastoreCallbacks(DatastoreCallbacks datastoreCallbacks) {
 559       Preconditions.checkNotNull(datastoreCallbacks);
 560       return new DatastoreServiceConfig(datastoreCallbacks);
 561     }
 562
 563     /**
 564      * Helper method for creating a {@link DatastoreServiceConfig}
 565      * instance with default values: Implicit transactions are disabled, reads
 566      * execute with {@link Consistency#STRONG}, and no deadline is
 567      * provided.  When no deadline is provided, datastore rpcs execute with the
 568      * system-defined deadline.
 569      *
 570      * @return The newly created DatastoreServiceConfig instance.
 571      */
 572     public static DatastoreServiceConfig withDefaults() {
 573       return new DatastoreServiceConfig((DatastoreCallbacks) null);
 574     }
 575
 576     private Builder() {}
 577   }
 578 }