java/src/main/com/google/appengine/api/datastore/RemoteCloudDatastoreV1Proxy.java

   1 package com.google.appengine.api.datastore;
   2
   3 import static com.google.common.base.Preconditions.checkArgument;
   4 import static com.google.common.base.Preconditions.checkNotNull;
   5
   6 import com.google.api.client.auth.oauth2.Credential;
   7 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
   8 import com.google.api.client.googleapis.compute.ComputeCredential;
   9 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
  10 import com.google.api.client.http.GenericUrl;
  11 import com.google.api.client.http.HttpRequest;
  12 import com.google.api.client.http.HttpTransport;
  13 import com.google.api.client.http.javanet.NetHttpTransport;
  14 import com.google.api.client.json.jackson.JacksonFactory;
  15 import com.google.appengine.api.datastore.DatastoreServiceConfig.ApiVersion;
  16 import com.google.appengine.api.taskqueue.TaskQueuePb.TaskQueueAddRequest;
  17 import com.google.appengine.api.taskqueue.TaskQueuePb.TaskQueueBulkAddRequest;
  18 import com.google.apphosting.api.ApiProxy;
  19 import com.google.apphosting.api.ApiProxy.ApiConfig;
  20 import com.google.apphosting.api.ApiProxy.ApiProxyException;
  21 import com.google.apphosting.api.ApiProxy.Delegate;
  22 import com.google.apphosting.api.ApiProxy.Environment;
  23 import com.google.apphosting.api.ApiProxy.EnvironmentFactory;
  24 import com.google.apphosting.api.ApiProxy.LogRecord;
  25 import com.google.datastore.v1beta3.AllocateIdsRequest;
  26 import com.google.datastore.v1beta3.AllocateIdsResponse;
  27 import com.google.datastore.v1beta3.BeginTransactionRequest;
  28 import com.google.datastore.v1beta3.BeginTransactionResponse;
  29 import com.google.datastore.v1beta3.CommitRequest;
  30 import com.google.datastore.v1beta3.CommitResponse;
  31 import com.google.datastore.v1beta3.LookupRequest;
  32 import com.google.datastore.v1beta3.LookupResponse;
  33 import com.google.datastore.v1beta3.RollbackRequest;
  34 import com.google.datastore.v1beta3.RollbackResponse;
  35 import com.google.datastore.v1beta3.RunQueryRequest;
  36 import com.google.datastore.v1beta3.RunQueryResponse;
  37 import com.google.datastore.v1beta3.client.Datastore;
  38 import com.google.datastore.v1beta3.client.DatastoreException;
  39 import com.google.datastore.v1beta3.client.DatastoreFactory;
  40 import com.google.datastore.v1beta3.client.DatastoreOptions;
  41 import com.google.protobuf.InvalidProtocolBufferException;
  42 import com.google.protobuf.Message;
  43
  44 import java.io.File;
  45 import java.io.IOException;
  46 import java.security.GeneralSecurityException;
  47 import java.util.HashMap;
  48 import java.util.List;
  49 import java.util.Map;
  50 import java.util.concurrent.Callable;
  51 import java.util.concurrent.ExecutorService;
  52 import java.util.concurrent.Executors;
  53 import java.util.concurrent.Future;
  54 import java.util.logging.Level;
  55 import java.util.logging.Logger;
  56
  57 /**
  58  * {@link CloudDatastoreV1Proxy} that makes remote calls (currently over HTTP).
  59  *
  60  * <p>Methods in this class do not populate the project id field in outgoing requests since it
  61  * is not required when using the API over HTTP.
  62  *
  63  * <p>This class is used if (and only if) the {@code DATASTORE_USE_CLOUD_DATASTORE} environment
  64  * variable is set.
  65  *
  66  * <p>This class is designed to run outside of App Engine in an environment where the app id is
  67  * potentially unknown. The user has several ways to specify it:
  68  * <ul>
  69  * <li>Install the Remote API. The Remote API can retrieve the app id by making a call to the
  70  * server.
  71  * <li>Specify the {@code DATASTORE_APP_ID} environment variable. If the project id has also been
  72  * specified, then the value of {@code DATASTORE_APP_ID} is required to match.
  73  * <li>Explicitly opt to use the project id instead by setting
  74  * {@code DATASTORE_USE_PROJECT_ID_AS_APP_ID=true}. This changes the serialized form of entities,
  75  * so integration with services such as memcache will not work correctly.
  76  * </ul>
  77  *
  78  * <p>{@code DATASTORE_APP_ID} and {@code DATASTORE_USE_PROJECT_ID_AS_APP_ID} cannot both be
  79  * specified.
  80  *
  81  * <p>Separately, {@code DATASTORE_ADDITIONAL_APP_IDS} can be set to a comma-separated list of
  82  * app ids in order to support foreign keys.
  83  */
  84 final class RemoteCloudDatastoreV1Proxy implements CloudDatastoreV1Proxy {
  85
  86   private static final Logger logger =
  87       Logger.getLogger(RemoteCloudDatastoreV1Proxy.class.getName());
  88
  89   static final String ADDITIONAL_APP_IDS_VAR = "DATASTORE_ADDITIONAL_APP_IDS";
  90   private static final String USE_PROJECT_ID_AS_APP_ID_VAR =
  91       "DATASTORE_USE_PROJECT_ID_AS_APP_ID";
  92   private static final String APP_ID_VAR = "DATASTORE_APP_ID";
  93
  94   private static final ExecutorService executor = Executors.newCachedThreadPool();
  95
  96   private final Datastore datastore;
  97
  98   RemoteCloudDatastoreV1Proxy(Datastore datastore) {
  99     this.datastore = checkNotNull(datastore);
 100   }
 101
 102   /**
 103    * Creates a {@link RemoteCloudDatastoreV1Proxy}. This method has the side effect
 104    * of installing minimal stubs ({@link EnvironmentFactory} and
 105    * {@link Delegate}) in the API proxy if they have not already been installed.
 106    */
 107   static RemoteCloudDatastoreV1Proxy create(DatastoreServiceConfig config) {
 108     checkArgument(config.getApiVersion() == ApiVersion.CLOUD_DATASTORE_V1_REMOTE);
 109     DatastoreOptions options;
 110     try {
 111       options = getDatastoreOptions();
 112     } catch (GeneralSecurityException | IOException e) {
 113       throw new RuntimeException(
 114           "Could not get Cloud Datastore options from environment.", e);
 115     }
 116     ensureApiProxyIsConfigured(options);
 117     populateAdditionalAppIdsMap();
 118     return new RemoteCloudDatastoreV1Proxy(DatastoreFactory.get().create(options));
 119   }
 120
 121   private static void populateAdditionalAppIdsMap() {
 122     String appIdsVar = EnvProxy.getenv(ADDITIONAL_APP_IDS_VAR);
 123     if (appIdsVar == null) {
 124       return;
 125     }
 126     String[] appIds = appIdsVar.split(",");
 127     Map<String, String> projectIdToAppId = new HashMap<>();
 128     for (String appId : appIds) {
 129       appId = appId.trim();
 130       if (!appId.isEmpty()) {
 131         projectIdToAppId.put(DatastoreApiHelper.toProjectId(appId), appId);
 132       }
 133     }
 134     ApiProxy.getCurrentEnvironment().getAttributes()
 135         .put(DataTypeTranslator.ADDITIONAL_APP_IDS_MAP_ATTRIBUTE_KEY, projectIdToAppId);
 136   }
 137
 138   @Override
 139   public Future<BeginTransactionResponse> beginTransaction(final BeginTransactionRequest req) {
 140     return makeCall(new Callable<BeginTransactionResponse>() {
 141       @Override
 142       public BeginTransactionResponse call() throws DatastoreException {
 143         return datastore.beginTransaction(req);
 144       }
 145     });
 146   }
 147
 148   @Override
 149   public Future<RollbackResponse> rollback(final RollbackRequest req) {
 150     return makeCall(new Callable<RollbackResponse>() {
 151       @Override
 152       public RollbackResponse call() throws DatastoreException {
 153         return datastore.rollback(req);
 154       }
 155     });
 156   }
 157
 158   @Override
 159   public Future<RunQueryResponse> runQuery(final RunQueryRequest req) {
 160     return makeCall(new Callable<RunQueryResponse>() {
 161       @Override
 162       public RunQueryResponse call() throws DatastoreException {
 163         return datastore.runQuery(req);
 164       }
 165     });
 166   }
 167
 168   @Override
 169   public Future<LookupResponse> lookup(final LookupRequest req) {
 170     return makeCall(new Callable<LookupResponse>() {
 171       @Override
 172       public LookupResponse call() throws DatastoreException {
 173         return datastore.lookup(req);
 174       }
 175     });
 176   }
 177
 178   @Override
 179   public Future<AllocateIdsResponse> allocateIds(final AllocateIdsRequest req) {
 180     return makeCall(new Callable<AllocateIdsResponse>() {
 181       @Override
 182       public AllocateIdsResponse call() throws DatastoreException {
 183         return datastore.allocateIds(req);
 184       }
 185     });
 186   }
 187
 188   @Override
 189   public Future<CommitResponse> commit(final CommitRequest req) {
 190     return makeCall(new Callable<CommitResponse>() {
 191       @Override
 192       public CommitResponse call() throws DatastoreException {
 193         return datastore.commit(req);
 194       }
 195     });
 196   }
 197
 198   @Override
 199   public Future<CommitResponse> rawCommit(byte[] bytes) {
 200     try {
 201       return commit(CommitRequest.parseFrom(bytes));
 202     } catch (InvalidProtocolBufferException e) {
 203       throw new IllegalStateException(e);
 204     }
 205   }
 206
 207   private static <T extends Message> Future<T> makeCall(final Callable<T> request) {
 208     return executor.submit(new Callable<T>() {
 209       @Override
 210       public T call() throws Exception {
 211         try {
 212           return request.call();
 213         } catch (DatastoreException e) {
 214           throw DatastoreApiHelper.createException(e.getCode(), e.getMessage());
 215         }
 216       }
 217     });
 218   }
 219
 220   private static DatastoreOptions getDatastoreOptions()
 221       throws GeneralSecurityException, IOException {
 222     DatastoreOptions.Builder options = new DatastoreOptions.Builder();
 223     options.projectId(getProjectId());
 224     options.host(EnvProxy.getenv("DATASTORE_HOST"));
 225
 226     String serviceAccount = EnvProxy.getenv("DATASTORE_SERVICE_ACCOUNT");
 227     String privateKeyFile = EnvProxy.getenv("DATASTORE_PRIVATE_KEY_FILE");
 228     Credential credential;
 229     if (Boolean.valueOf(EnvProxy.getenv("__DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST"))) {
 230       credential = null;
 231     } else if (serviceAccount != null && privateKeyFile != null) {
 232       credential = getServiceAccountCredential(serviceAccount, privateKeyFile);
 233     } else {
 234       credential = getComputeEngineCredential();
 235     }
 236     options.credential(credential);
 237     return options.build();
 238   }
 239
 240   private static String getProjectId() {
 241     String projectIdFromEnv = EnvProxy.getenv("DATASTORE_PROJECT_ID");
 242     if (projectIdFromEnv != null) {
 243       return projectIdFromEnv;
 244     }
 245     String appIdFromEnv = EnvProxy.getenv(APP_ID_VAR);
 246     if (appIdFromEnv != null) {
 247       return DatastoreApiHelper.toProjectId(appIdFromEnv);
 248     }
 249     try {
 250       HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
 251       GenericUrl projectIdUrl =
 252           new GenericUrl("http://metadata/computeMetadata/v1/project/project-id");
 253       HttpRequest request = transport.createRequestFactory().buildGetRequest(projectIdUrl);
 254       request.getHeaders().set("Metadata-Flavor", "Google");
 255       return request.execute().parseAsString();
 256     } catch (GeneralSecurityException | IOException e) {
 257       logger.log(Level.INFO, "Could not retrieve project id from GCE metadata server", e);
 258       return null;
 259     }
 260   }
 261
 262   private static Credential getServiceAccountCredential(String account, String privateKeyFile)
 263       throws GeneralSecurityException, IOException {
 264     return new GoogleCredential.Builder()
 265         .setTransport(GoogleNetHttpTransport.newTrustedTransport())
 266         .setJsonFactory(new JacksonFactory())
 267         .setServiceAccountId(account)
 268         .setServiceAccountScopes(DatastoreOptions.SCOPES)
 269         .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile))
 270         .build();
 271   }
 272
 273   private static Credential getComputeEngineCredential()
 274       throws GeneralSecurityException, IOException {
 275     NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
 276     try {
 277       ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory());
 278       credential.refreshToken();
 279       return credential;
 280     } catch (IOException e) {
 281       return null;
 282     }
 283   }
 284
 285   /**
 286    * Make sure that the API proxy has been configured. If it's already
 287    * configured (e.g. because the Remote API has been installed or the factory
 288    * has already been used), do nothing. Otherwise, install a stub environment
 289    * and delegate.
 290    */
 291   private static synchronized void ensureApiProxyIsConfigured(DatastoreOptions options) {
 292     boolean hasEnvironmentOrFactory = (ApiProxy.getCurrentEnvironment() != null);
 293     boolean hasDelegate = (ApiProxy.getDelegate() != null);
 294
 295     if (hasEnvironmentOrFactory && hasDelegate) {
 296       if (!(ApiProxy.getDelegate() instanceof TransactionalTaskDisallowingApiProxyDelegate)) {
 297         @SuppressWarnings("unchecked")
 298         Delegate<Environment> originalDelegate = ApiProxy.getDelegate();
 299         ApiProxy.setDelegate(new TransactionalTaskDisallowingApiProxyDelegate(originalDelegate));
 300       }
 301       return;
 302     }
 303
 304     if (hasEnvironmentOrFactory) {
 305       throw new IllegalStateException(
 306           "An ApiProxy.Environment or ApiProxy.EnvironmentFactory was already installed. "
 307           + "Cannot use Cloud Datastore.");
 308     } else if (hasDelegate) {
 309       throw new IllegalStateException(
 310           "An ApiProxy.Delegate was already installed. Cannot use Cloud Datastore.");
 311     }
 312
 313     String appId = EnvProxy.getenv("DATASTORE_APP_ID");
 314     boolean useProjectIdAsAppId =
 315         Boolean.valueOf(EnvProxy.getenv(USE_PROJECT_ID_AS_APP_ID_VAR));
 316
 317     if (appId == null && !useProjectIdAsAppId) {
 318       throw new IllegalStateException(String.format(
 319           "Could not not determine app id. To use project id (%s) instead, set "
 320           + "%s=true. This will affect the serialized form "
 321           + "of entities and should not be used if serialized entities will be shared between "
 322           + "code running on App Engine and code running off App Engine. Alternatively, set "
 323           + "%s=<app id>.",
 324           options.getProjectId(), USE_PROJECT_ID_AS_APP_ID_VAR, APP_ID_VAR));
 325     } else if (appId != null) {
 326       if (useProjectIdAsAppId) {
 327         throw new IllegalStateException(String.format(
 328             "App id was provided (%s) but %s was set to true. "
 329             + "Please unset either %s or %s.",
 330             appId, USE_PROJECT_ID_AS_APP_ID_VAR, APP_ID_VAR, USE_PROJECT_ID_AS_APP_ID_VAR));
 331       } else if (!DatastoreApiHelper.toProjectId(appId).equals(options.getProjectId())) {
 332         throw new IllegalStateException(String.format(
 333             "App id \"%s\" does not match project id \"%s\".",
 334             appId, options.getProjectId()));
 335       }
 336     }
 337
 338     ApiProxy.setEnvironmentFactory(new StubApiProxyEnvironmentFactory(
 339         useProjectIdAsAppId ? options.getProjectId() : appId));
 340     ApiProxy.setDelegate(new StubApiProxyDelegate());
 341   }
 342
 343   /**
 344    * A {@link Delegate} that disallows transactional tasks. Requests not
 345    * containing a transactional task are delegated to another {@link Delegate}.
 346    */
 347   static class TransactionalTaskDisallowingApiProxyDelegate implements Delegate<Environment> {
 348     static final String NOT_SUPPORTED_MESSAGE =
 349         "Transactional tasks are not supported under this configuration.";
 350     private final Delegate<Environment> delegate;
 351
 352     public TransactionalTaskDisallowingApiProxyDelegate(Delegate<Environment> delegate) {
 353       this.delegate = delegate;
 354     }
 355
 356     @Override
 357     public byte[] makeSyncCall(Environment environment, String packageName,
 358         String methodName, byte[] request) throws ApiProxyException {
 359       checkAllowed(packageName, methodName, request);
 360       return delegate.makeSyncCall(environment, packageName, methodName, request);
 361     }
 362
 363     @Override
 364     public Future<byte[]> makeAsyncCall(Environment environment, String packageName,
 365         String methodName, byte[] request, ApiConfig apiConfig) {
 366       checkAllowed(packageName, methodName, request);
 367       return delegate.makeAsyncCall(environment, packageName, methodName, request, apiConfig);
 368     }
 369
 370     @Override
 371     public void log(Environment environment, LogRecord record) {
 372       delegate.log(environment, record);
 373     }
 374
 375     @Override
 376     public void flushLogs(Environment environment) {
 377       delegate.flushLogs(environment);
 378     }
 379
 380     @Override
 381     public List<Thread> getRequestThreads(Environment environment) {
 382       return delegate.getRequestThreads(environment);
 383     }
 384
 385     private static void checkAllowed(String packageName, String methodName,
 386         byte[] requestBytes) {
 387       if (!"taskqueue".equals(packageName)) {
 388         return;
 389       }
 390
 391       try {
 392         if ("Add".equals(methodName)) {
 393           checkNonTransactional(TaskQueueAddRequest.PARSER.parseFrom(requestBytes));
 394         } else if ("BulkAdd".equals(methodName)) {
 395           TaskQueueBulkAddRequest req = TaskQueueBulkAddRequest.PARSER.parseFrom(requestBytes);
 396           for (TaskQueueAddRequest subReq : req.addRequests()) {
 397             checkNonTransactional(subReq);
 398           }
 399         }
 400       } catch (InvalidProtocolBufferException e) {
 401         throw new IllegalArgumentException(e);
 402       }
 403     }
 404
 405     private static void checkNonTransactional(TaskQueueAddRequest req) {
 406       if (req.hasTransaction()) {
 407         throw new UnsupportedOperationException(NOT_SUPPORTED_MESSAGE);
 408       }
 409     }
 410   }
 411
 412   /**
 413    * A {@link Delegate} that throws {@link UnsupportedOperationException} for
 414    * all methods.
 415    */
 416   static class StubApiProxyDelegate implements Delegate<Environment> {
 417     private static final String UNSUPPORTED_API_PATTERN =
 418         "Calls to %s.%s are not supported under this configuration, only "
 419         + "calls to Cloud Datastore. To use other APIs, first install the "
 420         + "Remote API.";
 421
 422     @Override
 423     public byte[] makeSyncCall(Environment environment, String packageName,
 424         String methodName, byte[] request) throws ApiProxyException {
 425       throw new UnsupportedOperationException(
 426           String.format(UNSUPPORTED_API_PATTERN, packageName, methodName));
 427     }
 428
 429     @Override
 430     public Future<byte[]> makeAsyncCall(Environment environment, String packageName,
 431         String methodName, byte[] request, ApiConfig apiConfig) {
 432       throw new UnsupportedOperationException(
 433           String.format(UNSUPPORTED_API_PATTERN, packageName, methodName));
 434     }
 435
 436     @Override
 437     public void log(Environment environment, LogRecord record) {
 438       throw new UnsupportedOperationException();
 439     }
 440
 441     @Override
 442     public void flushLogs(Environment environment) {
 443       throw new UnsupportedOperationException();
 444     }
 445
 446     @Override
 447     public List<Thread> getRequestThreads(Environment environment) {
 448       throw new UnsupportedOperationException();
 449     }
 450   }
 451
 452   /**
 453    * An {@link EnvironmentFactory} that builds {@link StubApiProxyEnvironment}s.
 454    */
 455   static class StubApiProxyEnvironmentFactory implements EnvironmentFactory {
 456     private final String appId;
 457
 458     public StubApiProxyEnvironmentFactory(String appId) {
 459       this.appId = appId;
 460     }
 461
 462     @Override
 463     public Environment newEnvironment() {
 464       return new StubApiProxyEnvironment(appId);
 465     }
 466   }
 467
 468   /**
 469    * An {@link Environment} that supports the minimal subset of features needed
 470    * to run code from the datastore package outside of App Engine. All other
 471    * methods throw {@link UnsupportedOperationException}.
 472    */
 473   static class StubApiProxyEnvironment implements Environment {
 474     private final Map<String, Object> attributes;
 475     private final String appId;
 476
 477     public StubApiProxyEnvironment(String appId) {
 478       this.attributes = new HashMap<>();
 479       this.appId = appId;
 480     }
 481
 482     @Override
 483     public boolean isLoggedIn() {
 484       throw new UnsupportedOperationException();
 485     }
 486
 487     @Override
 488     public boolean isAdmin() {
 489       throw new UnsupportedOperationException();
 490     }
 491
 492     @Override
 493     public String getVersionId() {
 494       throw new UnsupportedOperationException();
 495     }
 496
 497     @Deprecated
 498     @Override
 499     public String getRequestNamespace() {
 500       throw new UnsupportedOperationException();
 501     }
 502
 503     @Override
 504     public long getRemainingMillis() {
 505       throw new UnsupportedOperationException();
 506     }
 507
 508     @Override
 509     public String getModuleId() {
 510       throw new UnsupportedOperationException();
 511     }
 512
 513     @Override
 514     public String getEmail() {
 515       throw new UnsupportedOperationException();
 516     }
 517
 518     @Override
 519     public String getAuthDomain() {
 520       throw new UnsupportedOperationException();
 521     }
 522
 523     @Override
 524     public Map<String, Object> getAttributes() {
 525       return attributes;
 526     }
 527
 528     @Override
 529     public String getAppId() {
 530       return appId;
 531     }
 532   }
 533 }