java/src/main/com/google/appengine/api/datastore/RemoteCloudDatastoreV1Proxy.java

   1 package com.google.appengine.api.datastore;
   2
   3 import static com.google.common.base.Preconditions.checkArgument;
   4 import static com.google.common.base.Preconditions.checkNotNull;
   5 import static com.google.datastore.v1beta3.client.DatastoreHelper.LOCAL_HOST_ENV_VAR;
   6 import static com.google.datastore.v1beta3.client.DatastoreHelper.PRIVATE_KEY_FILE_ENV_VAR;
   7 import static com.google.datastore.v1beta3.client.DatastoreHelper.PROJECT_ID_ENV_VAR;
   8 import static com.google.datastore.v1beta3.client.DatastoreHelper.SERVICE_ACCOUNT_ENV_VAR;
   9
  10 import com.google.api.client.auth.oauth2.Credential;
  11 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
  12 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
  13 import com.google.api.client.json.jackson.JacksonFactory;
  14 import com.google.appengine.api.datastore.DatastoreServiceConfig.ApiVersion;
  15 import com.google.appengine.api.taskqueue.TaskQueuePb.TaskQueueAddRequest;
  16 import com.google.appengine.api.taskqueue.TaskQueuePb.TaskQueueBulkAddRequest;
  17 import com.google.apphosting.api.ApiProxy;
  18 import com.google.apphosting.api.ApiProxy.ApiConfig;
  19 import com.google.apphosting.api.ApiProxy.ApiProxyException;
  20 import com.google.apphosting.api.ApiProxy.Delegate;
  21 import com.google.apphosting.api.ApiProxy.Environment;
  22 import com.google.apphosting.api.ApiProxy.EnvironmentFactory;
  23 import com.google.apphosting.api.ApiProxy.LogRecord;
  24 import com.google.datastore.v1beta3.AllocateIdsRequest;
  25 import com.google.datastore.v1beta3.AllocateIdsResponse;
  26 import com.google.datastore.v1beta3.BeginTransactionRequest;
  27 import com.google.datastore.v1beta3.BeginTransactionResponse;
  28 import com.google.datastore.v1beta3.CommitRequest;
  29 import com.google.datastore.v1beta3.CommitResponse;
  30 import com.google.datastore.v1beta3.Datastore.Method;
  31 import com.google.datastore.v1beta3.LookupRequest;
  32 import com.google.datastore.v1beta3.LookupResponse;
  33 import com.google.datastore.v1beta3.RollbackRequest;
  34 import com.google.datastore.v1beta3.RollbackResponse;
  35 import com.google.datastore.v1beta3.RunQueryRequest;
  36 import com.google.datastore.v1beta3.RunQueryResponse;
  37 import com.google.datastore.v1beta3.client.Datastore;
  38 import com.google.datastore.v1beta3.client.DatastoreException;
  39 import com.google.datastore.v1beta3.client.DatastoreFactory;
  40 import com.google.datastore.v1beta3.client.DatastoreHelper;
  41 import com.google.datastore.v1beta3.client.DatastoreOptions;
  42 import com.google.protobuf.InvalidProtocolBufferException;
  43 import com.google.protobuf.Message;
  44
  45 import java.io.File;
  46 import java.io.IOException;
  47 import java.security.GeneralSecurityException;
  48 import java.util.HashMap;
  49 import java.util.List;
  50 import java.util.Map;
  51 import java.util.concurrent.Callable;
  52 import java.util.concurrent.ExecutorService;
  53 import java.util.concurrent.Executors;
  54 import java.util.concurrent.Future;
  55 import java.util.logging.Level;
  56 import java.util.logging.Logger;
  57
  58 /**
  59  * {@link CloudDatastoreV1Proxy} that makes remote calls (currently over HTTP).
  60  *
  61  * <p>Methods in this class do not populate the project id field in outgoing requests since it
  62  * is not required when using the API over HTTP.
  63  *
  64  * <p>This class is used if (and only if) the {@code DATASTORE_USE_CLOUD_DATASTORE},
  65  * {@code DATASTORE_PROJECT_ID}, or {@code DATASTORE_APP_ID} environment variable is set.
  66  *
  67  * <p>This class is designed to run outside of App Engine in an environment where the app id is
  68  * potentially unknown. The user has several ways to specify it:
  69  * <ul>
  70  * <li>Install the Remote API. The Remote API can retrieve the app id by making a call to the
  71  * server.
  72  * <li>Specify the {@code DATASTORE_APP_ID} environment variable. If the project id has also been
  73  * specified, then the value of {@code DATASTORE_APP_ID} is required to match.
  74  * <li>Explicitly opt to use the project id instead by setting
  75  * {@code DATASTORE_USE_PROJECT_ID_AS_APP_ID=true}. This changes the serialized form of entities,
  76  * so integration with services such as memcache will not work correctly.
  77  * </ul>
  78  *
  79  * <p>{@code DATASTORE_APP_ID} and {@code DATASTORE_USE_PROJECT_ID_AS_APP_ID} cannot both be
  80  * specified.
  81  *
  82  * <p>Separately, {@code DATASTORE_ADDITIONAL_APP_IDS} can be set to a comma-separated list of
  83  * app ids in order to support foreign keys.
  84  */
  85 final class RemoteCloudDatastoreV1Proxy implements CloudDatastoreV1Proxy {
  86
  87   private static final Logger logger =
  88       Logger.getLogger(RemoteCloudDatastoreV1Proxy.class.getName());
  89
  90   private static final String URL_OVERRIDE_ENV_VAR = "__DATASTORE_URL_OVERRIDE";
  91
  92   static final String ADDITIONAL_APP_IDS_VAR = "DATASTORE_ADDITIONAL_APP_IDS";
  93   private static final String USE_PROJECT_ID_AS_APP_ID_VAR =
  94       "DATASTORE_USE_PROJECT_ID_AS_APP_ID";
  95   static final String APP_ID_VAR = "DATASTORE_APP_ID";
  96   private static final String ALLOW_TRANSACTIONAL_TASKS = "__DATASTORE_ALLOW_TRANSACTIONAL_TASKS";
  97
  98   private static final ExecutorService executor = Executors.newCachedThreadPool();
  99
 100   private final Datastore datastore;
 101
 102   RemoteCloudDatastoreV1Proxy(Datastore datastore) {
 103     this.datastore = checkNotNull(datastore);
 104   }
 105
 106   /**
 107    * Creates a {@link RemoteCloudDatastoreV1Proxy}. This method has the side effect
 108    * of installing minimal stubs ({@link EnvironmentFactory} and
 109    * {@link Delegate}) in the API proxy if they have not already been installed.
 110    */
 111   static RemoteCloudDatastoreV1Proxy create(DatastoreServiceConfig config) {
 112     checkArgument(config.getApiVersion() == ApiVersion.CLOUD_DATASTORE_V1_REMOTE);
 113     String projectId = getProjectId();
 114     DatastoreOptions options;
 115     try {
 116       options = getDatastoreOptions(projectId);
 117     } catch (GeneralSecurityException | IOException e) {
 118       throw new RuntimeException(
 119           "Could not get Cloud Datastore options from environment.", e);
 120     }
 121     ensureApiProxyIsConfigured(projectId);
 122     populateAdditionalAppIdsMap();
 123     return new RemoteCloudDatastoreV1Proxy(DatastoreFactory.get().create(options));
 124   }
 125
 126   private static void populateAdditionalAppIdsMap() {
 127     String appIdsVar = EnvProxy.getenv(ADDITIONAL_APP_IDS_VAR);
 128     if (appIdsVar == null) {
 129       return;
 130     }
 131     String[] appIds = appIdsVar.split(",");
 132     Map<String, String> projectIdToAppId = new HashMap<>();
 133     for (String appId : appIds) {
 134       appId = appId.trim();
 135       if (!appId.isEmpty()) {
 136         projectIdToAppId.put(DatastoreApiHelper.toProjectId(appId), appId);
 137       }
 138     }
 139     ApiProxy.getCurrentEnvironment().getAttributes()
 140         .put(DataTypeTranslator.ADDITIONAL_APP_IDS_MAP_ATTRIBUTE_KEY, projectIdToAppId);
 141   }
 142
 143   @Override
 144   public Future<BeginTransactionResponse> beginTransaction(final BeginTransactionRequest req) {
 145     return makeCall(new Callable<BeginTransactionResponse>() {
 146       @Override
 147       public BeginTransactionResponse call() throws DatastoreException {
 148         return datastore.beginTransaction(req);
 149       }
 150     }, Method.BeginTransaction);
 151   }
 152
 153   @Override
 154   public Future<RollbackResponse> rollback(final RollbackRequest req) {
 155     return makeCall(new Callable<RollbackResponse>() {
 156       @Override
 157       public RollbackResponse call() throws DatastoreException {
 158         return datastore.rollback(req);
 159       }
 160     }, Method.Rollback);
 161   }
 162
 163   @Override
 164   public Future<RunQueryResponse> runQuery(final RunQueryRequest req) {
 165     return makeCall(new Callable<RunQueryResponse>() {
 166       @Override
 167       public RunQueryResponse call() throws DatastoreException {
 168         return datastore.runQuery(req);
 169       }
 170     }, Method.RunQuery);
 171   }
 172
 173   @Override
 174   public Future<LookupResponse> lookup(final LookupRequest req) {
 175     return makeCall(new Callable<LookupResponse>() {
 176       @Override
 177       public LookupResponse call() throws DatastoreException {
 178         return datastore.lookup(req);
 179       }
 180     }, Method.Lookup);
 181   }
 182
 183   @Override
 184   public Future<AllocateIdsResponse> allocateIds(final AllocateIdsRequest req) {
 185     return makeCall(new Callable<AllocateIdsResponse>() {
 186       @Override
 187       public AllocateIdsResponse call() throws DatastoreException {
 188         return datastore.allocateIds(req);
 189       }
 190     }, Method.AllocateIds);
 191   }
 192
 193   @Override
 194   public Future<CommitResponse> commit(final CommitRequest req) {
 195     return makeCall(new Callable<CommitResponse>() {
 196       @Override
 197       public CommitResponse call() throws DatastoreException {
 198         return datastore.commit(req);
 199       }
 200     }, Method.Commit);
 201   }
 202
 203   @Override
 204   public Future<CommitResponse> rawCommit(byte[] bytes) {
 205     try {
 206       return commit(CommitRequest.parseFrom(bytes));
 207     } catch (InvalidProtocolBufferException e) {
 208       throw new IllegalStateException(e);
 209     }
 210   }
 211
 212   private static <T extends Message> Future<T> makeCall(final Callable<T> request,
 213       final Method method) {
 214     return executor.submit(new Callable<T>() {
 215       @Override
 216       public T call() throws Exception {
 217         try {
 218           return request.call();
 219         } catch (DatastoreException e) {
 220           throw DatastoreApiHelper.createV1Exception(e.getCode(), e.getMessage(), method);
 221         }
 222       }
 223     });
 224   }
 225
 226   private static DatastoreOptions getDatastoreOptions(String projectId)
 227       throws GeneralSecurityException, IOException {
 228     DatastoreOptions.Builder options = new DatastoreOptions.Builder();
 229     setProjectEndpoint(projectId, options);
 230     options.credential(getCredential());
 231     return options.build();
 232   }
 233
 234   private static Credential getCredential() throws GeneralSecurityException, IOException {
 235     if (Boolean.valueOf(EnvProxy.getenv("__DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST"))) {
 236       return null;
 237     } else if (EnvProxy.getenv(LOCAL_HOST_ENV_VAR) != null) {
 238       logger.log(Level.INFO, "{0} environment variable was set. Not using credentials.",
 239           new Object[] {LOCAL_HOST_ENV_VAR});
 240       return null;
 241     }
 242     String serviceAccount = EnvProxy.getenv(SERVICE_ACCOUNT_ENV_VAR);
 243     String privateKeyFile = EnvProxy.getenv(PRIVATE_KEY_FILE_ENV_VAR);
 244     if (serviceAccount != null && privateKeyFile != null) {
 245       logger.log(Level.INFO, "{0} and {1} environment variables were set. "
 246           + "Using service account credential.",
 247           new Object[] {SERVICE_ACCOUNT_ENV_VAR, PRIVATE_KEY_FILE_ENV_VAR});
 248       return getServiceAccountCredential(serviceAccount, privateKeyFile);
 249     }
 250     return GoogleCredential.getApplicationDefault()
 251         .createScoped(DatastoreOptions.SCOPES);
 252   }
 253
 254   private static void setProjectEndpoint(String projectId, DatastoreOptions.Builder options) {
 255     if (EnvProxy.getenv("DATASTORE_HOST") != null) {
 256       logger.warning(String.format(
 257           "Ignoring value of environment variable DATASTORE_HOST. "
 258           + "To point datastore to a host running locally, use "
 259           + "the environment variable %s.",
 260           LOCAL_HOST_ENV_VAR));
 261     }
 262     if (EnvProxy.getenv(URL_OVERRIDE_ENV_VAR) != null) {
 263       options.projectEndpoint(String.format("%s/projects/%s",
 264           EnvProxy.getenv(URL_OVERRIDE_ENV_VAR), projectId));
 265       return;
 266     }
 267     if (EnvProxy.getenv(LOCAL_HOST_ENV_VAR) != null) {
 268       options.projectId(projectId);
 269       options.localHost(EnvProxy.getenv(LOCAL_HOST_ENV_VAR));
 270       return;
 271     }
 272     options.projectId(projectId);
 273     return;
 274   }
 275
 276   private static String getProjectId() {
 277     String projectIdFromEnv = EnvProxy.getenv(PROJECT_ID_ENV_VAR);
 278     if (projectIdFromEnv != null) {
 279       return projectIdFromEnv;
 280     }
 281     String appIdFromEnv = EnvProxy.getenv(APP_ID_VAR);
 282     if (appIdFromEnv != null) {
 283       return DatastoreApiHelper.toProjectId(appIdFromEnv);
 284     }
 285     String projectIdFromComputeEngine = DatastoreHelper.getProjectIdFromComputeEngine();
 286     if (projectIdFromComputeEngine != null) {
 287       return projectIdFromComputeEngine;
 288     }
 289     throw new IllegalStateException(String.format("Could not determine project ID."
 290         + " If you are not running on Compute Engine, set the "
 291         + " %s environment variable.", PROJECT_ID_ENV_VAR));
 292   }
 293
 294   private static Credential getServiceAccountCredential(String account, String privateKeyFile)
 295       throws GeneralSecurityException, IOException {
 296     return new GoogleCredential.Builder()
 297         .setTransport(GoogleNetHttpTransport.newTrustedTransport())
 298         .setJsonFactory(new JacksonFactory())
 299         .setServiceAccountId(account)
 300         .setServiceAccountScopes(DatastoreOptions.SCOPES)
 301         .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile))
 302         .build();
 303   }
 304
 305   /**
 306    * Make sure that the API proxy has been configured. If it's already
 307    * configured (e.g. because the Remote API has been installed or the factory
 308    * has already been used), do nothing. Otherwise, install a stub environment
 309    * and delegate.
 310    */
 311   private static synchronized void ensureApiProxyIsConfigured(String projectId) {
 312     boolean hasEnvironmentOrFactory = (ApiProxy.getCurrentEnvironment() != null);
 313     boolean hasDelegate = (ApiProxy.getDelegate() != null);
 314
 315     if (hasEnvironmentOrFactory && hasDelegate) {
 316       boolean allowTransactionalTasks =
 317           Boolean.valueOf(EnvProxy.getenv(ALLOW_TRANSACTIONAL_TASKS));
 318       if (!allowTransactionalTasks
 319           && !(ApiProxy.getDelegate() instanceof TransactionalTaskDisallowingApiProxyDelegate)) {
 320         @SuppressWarnings("unchecked")
 321         Delegate<Environment> originalDelegate = ApiProxy.getDelegate();
 322         ApiProxy.setDelegate(new TransactionalTaskDisallowingApiProxyDelegate(originalDelegate));
 323       }
 324       return;
 325     }
 326
 327     if (hasEnvironmentOrFactory) {
 328       throw new IllegalStateException(
 329           "An ApiProxy.Environment or ApiProxy.EnvironmentFactory was already installed. "
 330           + "Cannot use Cloud Datastore.");
 331     } else if (hasDelegate) {
 332       throw new IllegalStateException(
 333           "An ApiProxy.Delegate was already installed. Cannot use Cloud Datastore.");
 334     }
 335
 336     String appId = EnvProxy.getenv(APP_ID_VAR);
 337     boolean useProjectIdAsAppId =
 338         Boolean.valueOf(EnvProxy.getenv(USE_PROJECT_ID_AS_APP_ID_VAR));
 339
 340     if (appId == null && !useProjectIdAsAppId) {
 341       throw new IllegalStateException(String.format(
 342           "Could not not determine app id. To use project id (%s) instead, set "
 343           + "%s=true. This will affect the serialized form "
 344           + "of entities and should not be used if serialized entities will be shared between "
 345           + "code running on App Engine and code running off App Engine. Alternatively, set "
 346           + "%s=<app id>.",
 347           projectId, USE_PROJECT_ID_AS_APP_ID_VAR, APP_ID_VAR));
 348     } else if (appId != null) {
 349       if (useProjectIdAsAppId) {
 350         throw new IllegalStateException(String.format(
 351             "App id was provided (%s) but %s was set to true. "
 352             + "Please unset either %s or %s.",
 353             appId, USE_PROJECT_ID_AS_APP_ID_VAR, APP_ID_VAR, USE_PROJECT_ID_AS_APP_ID_VAR));
 354       } else if (!DatastoreApiHelper.toProjectId(appId).equals(projectId)) {
 355         throw new IllegalStateException(String.format(
 356             "App id \"%s\" does not match project id \"%s\".",
 357             appId, projectId));
 358       }
 359     }
 360
 361     ApiProxy.setEnvironmentFactory(new StubApiProxyEnvironmentFactory(
 362         useProjectIdAsAppId ? projectId : appId));
 363     ApiProxy.setDelegate(new StubApiProxyDelegate());
 364   }
 365
 366   /**
 367    * A {@link Delegate} that disallows transactional tasks. Requests not
 368    * containing a transactional task are delegated to another {@link Delegate}.
 369    */
 370   static class TransactionalTaskDisallowingApiProxyDelegate implements Delegate<Environment> {
 371     static final String NOT_SUPPORTED_MESSAGE =
 372         "Transactional tasks are not supported under this configuration.";
 373     private final Delegate<Environment> delegate;
 374
 375     public TransactionalTaskDisallowingApiProxyDelegate(Delegate<Environment> delegate) {
 376       this.delegate = delegate;
 377     }
 378
 379     @Override
 380     public byte[] makeSyncCall(Environment environment, String packageName,
 381         String methodName, byte[] request) throws ApiProxyException {
 382       checkAllowed(packageName, methodName, request);
 383       return delegate.makeSyncCall(environment, packageName, methodName, request);
 384     }
 385
 386     @Override
 387     public Future<byte[]> makeAsyncCall(Environment environment, String packageName,
 388         String methodName, byte[] request, ApiConfig apiConfig) {
 389       checkAllowed(packageName, methodName, request);
 390       return delegate.makeAsyncCall(environment, packageName, methodName, request, apiConfig);
 391     }
 392
 393     @Override
 394     public void log(Environment environment, LogRecord record) {
 395       delegate.log(environment, record);
 396     }
 397
 398     @Override
 399     public void flushLogs(Environment environment) {
 400       delegate.flushLogs(environment);
 401     }
 402
 403     @Override
 404     public List<Thread> getRequestThreads(Environment environment) {
 405       return delegate.getRequestThreads(environment);
 406     }
 407
 408     private static void checkAllowed(String packageName, String methodName,
 409         byte[] requestBytes) {
 410       if (!"taskqueue".equals(packageName)) {
 411         return;
 412       }
 413
 414       try {
 415         if ("Add".equals(methodName)) {
 416           checkNonTransactional(TaskQueueAddRequest.parser().parseFrom(requestBytes));
 417         } else if ("BulkAdd".equals(methodName)) {
 418           TaskQueueBulkAddRequest req = TaskQueueBulkAddRequest.parser().parseFrom(requestBytes);
 419           for (TaskQueueAddRequest subReq : req.addRequests()) {
 420             checkNonTransactional(subReq);
 421           }
 422         }
 423       } catch (InvalidProtocolBufferException e) {
 424         throw new IllegalArgumentException(e);
 425       }
 426     }
 427
 428     private static void checkNonTransactional(TaskQueueAddRequest req) {
 429       if (req.hasTransaction() || req.hasDatastoreTransaction()) {
 430         throw new UnsupportedOperationException(NOT_SUPPORTED_MESSAGE);
 431       }
 432     }
 433   }
 434
 435   /**
 436    * A {@link Delegate} that throws {@link UnsupportedOperationException} for
 437    * all methods.
 438    */
 439   static class StubApiProxyDelegate implements Delegate<Environment> {
 440     private static final String UNSUPPORTED_API_PATTERN =
 441         "Calls to %s.%s are not supported under this configuration, only "
 442         + "calls to Cloud Datastore. To use other APIs, first install the "
 443         + "Remote API.";
 444
 445     @Override
 446     public byte[] makeSyncCall(Environment environment, String packageName,
 447         String methodName, byte[] request) throws ApiProxyException {
 448       throw new UnsupportedOperationException(
 449           String.format(UNSUPPORTED_API_PATTERN, packageName, methodName));
 450     }
 451
 452     @Override
 453     public Future<byte[]> makeAsyncCall(Environment environment, String packageName,
 454         String methodName, byte[] request, ApiConfig apiConfig) {
 455       throw new UnsupportedOperationException(
 456           String.format(UNSUPPORTED_API_PATTERN, packageName, methodName));
 457     }
 458
 459     @Override
 460     public void log(Environment environment, LogRecord record) {
 461       throw new UnsupportedOperationException();
 462     }
 463
 464     @Override
 465     public void flushLogs(Environment environment) {
 466       throw new UnsupportedOperationException();
 467     }
 468
 469     @Override
 470     public List<Thread> getRequestThreads(Environment environment) {
 471       throw new UnsupportedOperationException();
 472     }
 473   }
 474
 475   /**
 476    * An {@link EnvironmentFactory} that builds {@link StubApiProxyEnvironment}s.
 477    */
 478   static class StubApiProxyEnvironmentFactory implements EnvironmentFactory {
 479     private final String appId;
 480
 481     public StubApiProxyEnvironmentFactory(String appId) {
 482       this.appId = appId;
 483     }
 484
 485     @Override
 486     public Environment newEnvironment() {
 487       return new StubApiProxyEnvironment(appId);
 488     }
 489   }
 490
 491   /**
 492    * An {@link Environment} that supports the minimal subset of features needed
 493    * to run code from the datastore package outside of App Engine. All other
 494    * methods throw {@link UnsupportedOperationException}.
 495    */
 496   static class StubApiProxyEnvironment implements Environment {
 497     private final Map<String, Object> attributes;
 498     private final String appId;
 499
 500     public StubApiProxyEnvironment(String appId) {
 501       this.attributes = new HashMap<>();
 502       this.appId = appId;
 503     }
 504
 505     @Override
 506     public boolean isLoggedIn() {
 507       throw new UnsupportedOperationException();
 508     }
 509
 510     @Override
 511     public boolean isAdmin() {
 512       throw new UnsupportedOperationException();
 513     }
 514
 515     @Override
 516     public String getVersionId() {
 517       throw new UnsupportedOperationException();
 518     }
 519
 520     @Deprecated
 521     @Override
 522     public String getRequestNamespace() {
 523       throw new UnsupportedOperationException();
 524     }
 525
 526     @Override
 527     public long getRemainingMillis() {
 528       throw new UnsupportedOperationException();
 529     }
 530
 531     @Override
 532     public String getModuleId() {
 533       throw new UnsupportedOperationException();
 534     }
 535
 536     @Override
 537     public String getEmail() {
 538       throw new UnsupportedOperationException();
 539     }
 540
 541     @Override
 542     public String getAuthDomain() {
 543       throw new UnsupportedOperationException();
 544     }
 545
 546     @Override
 547     public Map<String, Object> getAttributes() {
 548       return attributes;
 549     }
 550
 551     @Override
 552     public String getAppId() {
 553       return appId;
 554     }
 555   }
 556 }