java/src/main/com/google/appengine/api/datastore/RemoteCloudDatastoreV1Proxy.java

   1 package com.google.appengine.api.datastore;
   2
   3 import static com.google.common.base.Preconditions.checkArgument;
   4 import static com.google.common.base.Preconditions.checkNotNull;
   5 import static com.google.datastore.v1beta3.client.DatastoreHelper.LOCAL_HOST_ENV_VAR;
   6 import static com.google.datastore.v1beta3.client.DatastoreHelper.PRIVATE_KEY_FILE_ENV_VAR;
   7 import static com.google.datastore.v1beta3.client.DatastoreHelper.PROJECT_ID_ENV_VAR;
   8 import static com.google.datastore.v1beta3.client.DatastoreHelper.SERVICE_ACCOUNT_ENV_VAR;
   9
  10 import com.google.api.client.auth.oauth2.Credential;
  11 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
  12 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
  13 import com.google.api.client.json.jackson.JacksonFactory;
  14 import com.google.appengine.api.datastore.DatastoreServiceConfig.ApiVersion;
  15 import com.google.appengine.api.taskqueue.TaskQueuePb.TaskQueueAddRequest;
  16 import com.google.appengine.api.taskqueue.TaskQueuePb.TaskQueueBulkAddRequest;
  17 import com.google.apphosting.api.ApiProxy;
  18 import com.google.apphosting.api.ApiProxy.ApiConfig;
  19 import com.google.apphosting.api.ApiProxy.ApiProxyException;
  20 import com.google.apphosting.api.ApiProxy.Delegate;
  21 import com.google.apphosting.api.ApiProxy.Environment;
  22 import com.google.apphosting.api.ApiProxy.EnvironmentFactory;
  23 import com.google.apphosting.api.ApiProxy.LogRecord;
  24 import com.google.datastore.v1beta3.AllocateIdsRequest;
  25 import com.google.datastore.v1beta3.AllocateIdsResponse;
  26 import com.google.datastore.v1beta3.BeginTransactionRequest;
  27 import com.google.datastore.v1beta3.BeginTransactionResponse;
  28 import com.google.datastore.v1beta3.CommitRequest;
  29 import com.google.datastore.v1beta3.CommitResponse;
  30 import com.google.datastore.v1beta3.LookupRequest;
  31 import com.google.datastore.v1beta3.LookupResponse;
  32 import com.google.datastore.v1beta3.RollbackRequest;
  33 import com.google.datastore.v1beta3.RollbackResponse;
  34 import com.google.datastore.v1beta3.RunQueryRequest;
  35 import com.google.datastore.v1beta3.RunQueryResponse;
  36 import com.google.datastore.v1beta3.client.Datastore;
  37 import com.google.datastore.v1beta3.client.DatastoreException;
  38 import com.google.datastore.v1beta3.client.DatastoreFactory;
  39 import com.google.datastore.v1beta3.client.DatastoreHelper;
  40 import com.google.datastore.v1beta3.client.DatastoreOptions;
  41 import com.google.protobuf.InvalidProtocolBufferException;
  42 import com.google.protobuf.Message;
  43
  44 import java.io.File;
  45 import java.io.IOException;
  46 import java.security.GeneralSecurityException;
  47 import java.util.HashMap;
  48 import java.util.List;
  49 import java.util.Map;
  50 import java.util.concurrent.Callable;
  51 import java.util.concurrent.ExecutorService;
  52 import java.util.concurrent.Executors;
  53 import java.util.concurrent.Future;
  54 import java.util.logging.Level;
  55 import java.util.logging.Logger;
  56
  57 /**
  58  * {@link CloudDatastoreV1Proxy} that makes remote calls (currently over HTTP).
  59  *
  60  * <p>Methods in this class do not populate the project id field in outgoing requests since it
  61  * is not required when using the API over HTTP.
  62  *
  63  * <p>This class is used if (and only if) the {@code DATASTORE_USE_CLOUD_DATASTORE} environment
  64  * variable is set.
  65  *
  66  * <p>This class is designed to run outside of App Engine in an environment where the app id is
  67  * potentially unknown. The user has several ways to specify it:
  68  * <ul>
  69  * <li>Install the Remote API. The Remote API can retrieve the app id by making a call to the
  70  * server.
  71  * <li>Specify the {@code DATASTORE_APP_ID} environment variable. If the project id has also been
  72  * specified, then the value of {@code DATASTORE_APP_ID} is required to match.
  73  * <li>Explicitly opt to use the project id instead by setting
  74  * {@code DATASTORE_USE_PROJECT_ID_AS_APP_ID=true}. This changes the serialized form of entities,
  75  * so integration with services such as memcache will not work correctly.
  76  * </ul>
  77  *
  78  * <p>{@code DATASTORE_APP_ID} and {@code DATASTORE_USE_PROJECT_ID_AS_APP_ID} cannot both be
  79  * specified.
  80  *
  81  * <p>Separately, {@code DATASTORE_ADDITIONAL_APP_IDS} can be set to a comma-separated list of
  82  * app ids in order to support foreign keys.
  83  */
  84 final class RemoteCloudDatastoreV1Proxy implements CloudDatastoreV1Proxy {
  85
  86   private static final Logger logger =
  87       Logger.getLogger(RemoteCloudDatastoreV1Proxy.class.getName());
  88
  89   private static final String URL_OVERRIDE_ENV_VAR = "__DATASTORE_URL_OVERRIDE";
  90
  91   static final String ADDITIONAL_APP_IDS_VAR = "DATASTORE_ADDITIONAL_APP_IDS";
  92   private static final String USE_PROJECT_ID_AS_APP_ID_VAR =
  93       "DATASTORE_USE_PROJECT_ID_AS_APP_ID";
  94   private static final String APP_ID_VAR = "DATASTORE_APP_ID";
  95
  96   private static final ExecutorService executor = Executors.newCachedThreadPool();
  97
  98   private final Datastore datastore;
  99
 100   RemoteCloudDatastoreV1Proxy(Datastore datastore) {
 101     this.datastore = checkNotNull(datastore);
 102   }
 103
 104   /**
 105    * Creates a {@link RemoteCloudDatastoreV1Proxy}. This method has the side effect
 106    * of installing minimal stubs ({@link EnvironmentFactory} and
 107    * {@link Delegate}) in the API proxy if they have not already been installed.
 108    */
 109   static RemoteCloudDatastoreV1Proxy create(DatastoreServiceConfig config) {
 110     checkArgument(config.getApiVersion() == ApiVersion.CLOUD_DATASTORE_V1_REMOTE);
 111     String projectId = getProjectId();
 112     DatastoreOptions options;
 113     try {
 114       options = getDatastoreOptions(projectId);
 115     } catch (GeneralSecurityException | IOException e) {
 116       throw new RuntimeException(
 117           "Could not get Cloud Datastore options from environment.", e);
 118     }
 119     ensureApiProxyIsConfigured(projectId);
 120     populateAdditionalAppIdsMap();
 121     return new RemoteCloudDatastoreV1Proxy(DatastoreFactory.get().create(options));
 122   }
 123
 124   private static void populateAdditionalAppIdsMap() {
 125     String appIdsVar = EnvProxy.getenv(ADDITIONAL_APP_IDS_VAR);
 126     if (appIdsVar == null) {
 127       return;
 128     }
 129     String[] appIds = appIdsVar.split(",");
 130     Map<String, String> projectIdToAppId = new HashMap<>();
 131     for (String appId : appIds) {
 132       appId = appId.trim();
 133       if (!appId.isEmpty()) {
 134         projectIdToAppId.put(DatastoreApiHelper.toProjectId(appId), appId);
 135       }
 136     }
 137     ApiProxy.getCurrentEnvironment().getAttributes()
 138         .put(DataTypeTranslator.ADDITIONAL_APP_IDS_MAP_ATTRIBUTE_KEY, projectIdToAppId);
 139   }
 140
 141   @Override
 142   public Future<BeginTransactionResponse> beginTransaction(final BeginTransactionRequest req) {
 143     return makeCall(new Callable<BeginTransactionResponse>() {
 144       @Override
 145       public BeginTransactionResponse call() throws DatastoreException {
 146         return datastore.beginTransaction(req);
 147       }
 148     });
 149   }
 150
 151   @Override
 152   public Future<RollbackResponse> rollback(final RollbackRequest req) {
 153     return makeCall(new Callable<RollbackResponse>() {
 154       @Override
 155       public RollbackResponse call() throws DatastoreException {
 156         return datastore.rollback(req);
 157       }
 158     });
 159   }
 160
 161   @Override
 162   public Future<RunQueryResponse> runQuery(final RunQueryRequest req) {
 163     return makeCall(new Callable<RunQueryResponse>() {
 164       @Override
 165       public RunQueryResponse call() throws DatastoreException {
 166         return datastore.runQuery(req);
 167       }
 168     });
 169   }
 170
 171   @Override
 172   public Future<LookupResponse> lookup(final LookupRequest req) {
 173     return makeCall(new Callable<LookupResponse>() {
 174       @Override
 175       public LookupResponse call() throws DatastoreException {
 176         return datastore.lookup(req);
 177       }
 178     });
 179   }
 180
 181   @Override
 182   public Future<AllocateIdsResponse> allocateIds(final AllocateIdsRequest req) {
 183     return makeCall(new Callable<AllocateIdsResponse>() {
 184       @Override
 185       public AllocateIdsResponse call() throws DatastoreException {
 186         return datastore.allocateIds(req);
 187       }
 188     });
 189   }
 190
 191   @Override
 192   public Future<CommitResponse> commit(final CommitRequest req) {
 193     return makeCall(new Callable<CommitResponse>() {
 194       @Override
 195       public CommitResponse call() throws DatastoreException {
 196         return datastore.commit(req);
 197       }
 198     });
 199   }
 200
 201   @Override
 202   public Future<CommitResponse> rawCommit(byte[] bytes) {
 203     try {
 204       return commit(CommitRequest.parseFrom(bytes));
 205     } catch (InvalidProtocolBufferException e) {
 206       throw new IllegalStateException(e);
 207     }
 208   }
 209
 210   private static <T extends Message> Future<T> makeCall(final Callable<T> request) {
 211     return executor.submit(new Callable<T>() {
 212       @Override
 213       public T call() throws Exception {
 214         try {
 215           return request.call();
 216         } catch (DatastoreException e) {
 217           throw DatastoreApiHelper.createException(e.getCode(), e.getMessage());
 218         }
 219       }
 220     });
 221   }
 222
 223   private static DatastoreOptions getDatastoreOptions(String projectId)
 224       throws GeneralSecurityException, IOException {
 225     DatastoreOptions.Builder options = new DatastoreOptions.Builder();
 226     setProjectEndpoint(projectId, options);
 227     options.credential(getCredential());
 228     return options.build();
 229   }
 230
 231   private static Credential getCredential() throws GeneralSecurityException, IOException {
 232     if (Boolean.valueOf(EnvProxy.getenv("__DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST"))) {
 233       return null;
 234     } else if (EnvProxy.getenv(LOCAL_HOST_ENV_VAR) != null) {
 235       logger.log(Level.INFO, "{0} environment variable was set. Not using credentials.",
 236           new Object[] {LOCAL_HOST_ENV_VAR});
 237       return null;
 238     }
 239     String serviceAccount = EnvProxy.getenv(SERVICE_ACCOUNT_ENV_VAR);
 240     String privateKeyFile = EnvProxy.getenv(PRIVATE_KEY_FILE_ENV_VAR);
 241     if (serviceAccount != null && privateKeyFile != null) {
 242       logger.log(Level.INFO, "{0} and {1} environment variables were set. "
 243           + "Using service account credential.",
 244           new Object[] {SERVICE_ACCOUNT_ENV_VAR, PRIVATE_KEY_FILE_ENV_VAR});
 245       return getServiceAccountCredential(serviceAccount, privateKeyFile);
 246     }
 247     return GoogleCredential.getApplicationDefault();
 248   }
 249
 250   private static void setProjectEndpoint(String projectId, DatastoreOptions.Builder options) {
 251     if (EnvProxy.getenv("DATASTORE_HOST") != null) {
 252       throw new IllegalArgumentException(String.format(
 253           "The environment variable DATASTORE_HOST is not supported. "
 254           + "To point datastore to a host running locally, use "
 255           + "the environment variable %s.",
 256           LOCAL_HOST_ENV_VAR));
 257     }
 258     if (EnvProxy.getenv(URL_OVERRIDE_ENV_VAR) != null) {
 259       options.projectEndpoint(String.format("%s/projects/%s",
 260           EnvProxy.getenv(URL_OVERRIDE_ENV_VAR), projectId));
 261       return;
 262     }
 263     if (EnvProxy.getenv(LOCAL_HOST_ENV_VAR) != null) {
 264       options.projectId(projectId);
 265       options.localHost(EnvProxy.getenv(LOCAL_HOST_ENV_VAR));
 266       return;
 267     }
 268     options.projectId(projectId);
 269     return;
 270   }
 271
 272   private static String getProjectId() {
 273     String projectIdFromEnv = EnvProxy.getenv(PROJECT_ID_ENV_VAR);
 274     if (projectIdFromEnv != null) {
 275       return projectIdFromEnv;
 276     }
 277     String appIdFromEnv = EnvProxy.getenv(APP_ID_VAR);
 278     if (appIdFromEnv != null) {
 279       return DatastoreApiHelper.toProjectId(appIdFromEnv);
 280     }
 281     String projectIdFromComputeEngine = DatastoreHelper.getProjectIdFromComputeEngine();
 282     if (projectIdFromComputeEngine != null) {
 283       return projectIdFromComputeEngine;
 284     }
 285     throw new IllegalStateException(String.format("Could not determine project ID."
 286         + " If you are not running on Compute Engine, set the "
 287         + " %s environment variable.", PROJECT_ID_ENV_VAR));
 288   }
 289
 290   private static Credential getServiceAccountCredential(String account, String privateKeyFile)
 291       throws GeneralSecurityException, IOException {
 292     return new GoogleCredential.Builder()
 293         .setTransport(GoogleNetHttpTransport.newTrustedTransport())
 294         .setJsonFactory(new JacksonFactory())
 295         .setServiceAccountId(account)
 296         .setServiceAccountScopes(DatastoreOptions.SCOPES)
 297         .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile))
 298         .build();
 299   }
 300
 301   /**
 302    * Make sure that the API proxy has been configured. If it's already
 303    * configured (e.g. because the Remote API has been installed or the factory
 304    * has already been used), do nothing. Otherwise, install a stub environment
 305    * and delegate.
 306    */
 307   private static synchronized void ensureApiProxyIsConfigured(String projectId) {
 308     boolean hasEnvironmentOrFactory = (ApiProxy.getCurrentEnvironment() != null);
 309     boolean hasDelegate = (ApiProxy.getDelegate() != null);
 310
 311     if (hasEnvironmentOrFactory && hasDelegate) {
 312       if (!(ApiProxy.getDelegate() instanceof TransactionalTaskDisallowingApiProxyDelegate)) {
 313         @SuppressWarnings("unchecked")
 314         Delegate<Environment> originalDelegate = ApiProxy.getDelegate();
 315         ApiProxy.setDelegate(new TransactionalTaskDisallowingApiProxyDelegate(originalDelegate));
 316       }
 317       return;
 318     }
 319
 320     if (hasEnvironmentOrFactory) {
 321       throw new IllegalStateException(
 322           "An ApiProxy.Environment or ApiProxy.EnvironmentFactory was already installed. "
 323           + "Cannot use Cloud Datastore.");
 324     } else if (hasDelegate) {
 325       throw new IllegalStateException(
 326           "An ApiProxy.Delegate was already installed. Cannot use Cloud Datastore.");
 327     }
 328
 329     String appId = EnvProxy.getenv(APP_ID_VAR);
 330     boolean useProjectIdAsAppId =
 331         Boolean.valueOf(EnvProxy.getenv(USE_PROJECT_ID_AS_APP_ID_VAR));
 332
 333     if (appId == null && !useProjectIdAsAppId) {
 334       throw new IllegalStateException(String.format(
 335           "Could not not determine app id. To use project id (%s) instead, set "
 336           + "%s=true. This will affect the serialized form "
 337           + "of entities and should not be used if serialized entities will be shared between "
 338           + "code running on App Engine and code running off App Engine. Alternatively, set "
 339           + "%s=<app id>.",
 340           projectId, USE_PROJECT_ID_AS_APP_ID_VAR, APP_ID_VAR));
 341     } else if (appId != null) {
 342       if (useProjectIdAsAppId) {
 343         throw new IllegalStateException(String.format(
 344             "App id was provided (%s) but %s was set to true. "
 345             + "Please unset either %s or %s.",
 346             appId, USE_PROJECT_ID_AS_APP_ID_VAR, APP_ID_VAR, USE_PROJECT_ID_AS_APP_ID_VAR));
 347       } else if (!DatastoreApiHelper.toProjectId(appId).equals(projectId)) {
 348         throw new IllegalStateException(String.format(
 349             "App id \"%s\" does not match project id \"%s\".",
 350             appId, projectId));
 351       }
 352     }
 353
 354     ApiProxy.setEnvironmentFactory(new StubApiProxyEnvironmentFactory(
 355         useProjectIdAsAppId ? projectId : appId));
 356     ApiProxy.setDelegate(new StubApiProxyDelegate());
 357   }
 358
 359   /**
 360    * A {@link Delegate} that disallows transactional tasks. Requests not
 361    * containing a transactional task are delegated to another {@link Delegate}.
 362    */
 363   static class TransactionalTaskDisallowingApiProxyDelegate implements Delegate<Environment> {
 364     static final String NOT_SUPPORTED_MESSAGE =
 365         "Transactional tasks are not supported under this configuration.";
 366     private final Delegate<Environment> delegate;
 367
 368     public TransactionalTaskDisallowingApiProxyDelegate(Delegate<Environment> delegate) {
 369       this.delegate = delegate;
 370     }
 371
 372     @Override
 373     public byte[] makeSyncCall(Environment environment, String packageName,
 374         String methodName, byte[] request) throws ApiProxyException {
 375       checkAllowed(packageName, methodName, request);
 376       return delegate.makeSyncCall(environment, packageName, methodName, request);
 377     }
 378
 379     @Override
 380     public Future<byte[]> makeAsyncCall(Environment environment, String packageName,
 381         String methodName, byte[] request, ApiConfig apiConfig) {
 382       checkAllowed(packageName, methodName, request);
 383       return delegate.makeAsyncCall(environment, packageName, methodName, request, apiConfig);
 384     }
 385
 386     @Override
 387     public void log(Environment environment, LogRecord record) {
 388       delegate.log(environment, record);
 389     }
 390
 391     @Override
 392     public void flushLogs(Environment environment) {
 393       delegate.flushLogs(environment);
 394     }
 395
 396     @Override
 397     public List<Thread> getRequestThreads(Environment environment) {
 398       return delegate.getRequestThreads(environment);
 399     }
 400
 401     private static void checkAllowed(String packageName, String methodName,
 402         byte[] requestBytes) {
 403       if (!"taskqueue".equals(packageName)) {
 404         return;
 405       }
 406
 407       try {
 408         if ("Add".equals(methodName)) {
 409           checkNonTransactional(TaskQueueAddRequest.parser().parseFrom(requestBytes));
 410         } else if ("BulkAdd".equals(methodName)) {
 411           TaskQueueBulkAddRequest req = TaskQueueBulkAddRequest.parser().parseFrom(requestBytes);
 412           for (TaskQueueAddRequest subReq : req.addRequests()) {
 413             checkNonTransactional(subReq);
 414           }
 415         }
 416       } catch (InvalidProtocolBufferException e) {
 417         throw new IllegalArgumentException(e);
 418       }
 419     }
 420
 421     private static void checkNonTransactional(TaskQueueAddRequest req) {
 422       if (req.hasTransaction()) {
 423         throw new UnsupportedOperationException(NOT_SUPPORTED_MESSAGE);
 424       }
 425     }
 426   }
 427
 428   /**
 429    * A {@link Delegate} that throws {@link UnsupportedOperationException} for
 430    * all methods.
 431    */
 432   static class StubApiProxyDelegate implements Delegate<Environment> {
 433     private static final String UNSUPPORTED_API_PATTERN =
 434         "Calls to %s.%s are not supported under this configuration, only "
 435         + "calls to Cloud Datastore. To use other APIs, first install the "
 436         + "Remote API.";
 437
 438     @Override
 439     public byte[] makeSyncCall(Environment environment, String packageName,
 440         String methodName, byte[] request) throws ApiProxyException {
 441       throw new UnsupportedOperationException(
 442           String.format(UNSUPPORTED_API_PATTERN, packageName, methodName));
 443     }
 444
 445     @Override
 446     public Future<byte[]> makeAsyncCall(Environment environment, String packageName,
 447         String methodName, byte[] request, ApiConfig apiConfig) {
 448       throw new UnsupportedOperationException(
 449           String.format(UNSUPPORTED_API_PATTERN, packageName, methodName));
 450     }
 451
 452     @Override
 453     public void log(Environment environment, LogRecord record) {
 454       throw new UnsupportedOperationException();
 455     }
 456
 457     @Override
 458     public void flushLogs(Environment environment) {
 459       throw new UnsupportedOperationException();
 460     }
 461
 462     @Override
 463     public List<Thread> getRequestThreads(Environment environment) {
 464       throw new UnsupportedOperationException();
 465     }
 466   }
 467
 468   /**
 469    * An {@link EnvironmentFactory} that builds {@link StubApiProxyEnvironment}s.
 470    */
 471   static class StubApiProxyEnvironmentFactory implements EnvironmentFactory {
 472     private final String appId;
 473
 474     public StubApiProxyEnvironmentFactory(String appId) {
 475       this.appId = appId;
 476     }
 477
 478     @Override
 479     public Environment newEnvironment() {
 480       return new StubApiProxyEnvironment(appId);
 481     }
 482   }
 483
 484   /**
 485    * An {@link Environment} that supports the minimal subset of features needed
 486    * to run code from the datastore package outside of App Engine. All other
 487    * methods throw {@link UnsupportedOperationException}.
 488    */
 489   static class StubApiProxyEnvironment implements Environment {
 490     private final Map<String, Object> attributes;
 491     private final String appId;
 492
 493     public StubApiProxyEnvironment(String appId) {
 494       this.attributes = new HashMap<>();
 495       this.appId = appId;
 496     }
 497
 498     @Override
 499     public boolean isLoggedIn() {
 500       throw new UnsupportedOperationException();
 501     }
 502
 503     @Override
 504     public boolean isAdmin() {
 505       throw new UnsupportedOperationException();
 506     }
 507
 508     @Override
 509     public String getVersionId() {
 510       throw new UnsupportedOperationException();
 511     }
 512
 513     @Deprecated
 514     @Override
 515     public String getRequestNamespace() {
 516       throw new UnsupportedOperationException();
 517     }
 518
 519     @Override
 520     public long getRemainingMillis() {
 521       throw new UnsupportedOperationException();
 522     }
 523
 524     @Override
 525     public String getModuleId() {
 526       throw new UnsupportedOperationException();
 527     }
 528
 529     @Override
 530     public String getEmail() {
 531       throw new UnsupportedOperationException();
 532     }
 533
 534     @Override
 535     public String getAuthDomain() {
 536       throw new UnsupportedOperationException();
 537     }
 538
 539     @Override
 540     public Map<String, Object> getAttributes() {
 541       return attributes;
 542     }
 543
 544     @Override
 545     public String getAppId() {
 546       return appId;
 547     }
 548   }
 549 }