java/src/main/com/google/appengine/api/datastore/CloudDatastoreProxy.java

   1 package com.google.appengine.api.datastore;
   2
   3 import static com.google.common.base.Preconditions.checkArgument;
   4 import static com.google.common.base.Preconditions.checkNotNull;
   5
   6 import com.google.api.client.auth.oauth2.Credential;
   7 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
   8 import com.google.api.client.googleapis.compute.ComputeCredential;
   9 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
  10 import com.google.api.client.http.HttpRequest;
  11 import com.google.api.client.http.HttpRequestInitializer;
  12 import com.google.api.client.http.javanet.NetHttpTransport;
  13 import com.google.api.client.json.jackson.JacksonFactory;
  14 import com.google.api.services.datastore.DatastoreV1;
  15 import com.google.api.services.datastore.client.Datastore;
  16 import com.google.api.services.datastore.client.DatastoreException;
  17 import com.google.api.services.datastore.client.DatastoreFactory;
  18 import com.google.api.services.datastore.client.DatastoreOptions;
  19 import com.google.appengine.api.datastore.DatastoreServiceConfig.ApiVersion;
  20 import com.google.apphosting.api.ApiProxy;
  21 import com.google.apphosting.api.ApiProxy.ApiConfig;
  22 import com.google.apphosting.api.ApiProxy.ApiProxyException;
  23 import com.google.apphosting.api.ApiProxy.Delegate;
  24 import com.google.apphosting.api.ApiProxy.Environment;
  25 import com.google.apphosting.api.ApiProxy.EnvironmentFactory;
  26 import com.google.apphosting.api.ApiProxy.LogRecord;
  27 import com.google.apphosting.datastore.DatastoreV4.AllocateIdsRequest;
  28 import com.google.apphosting.datastore.DatastoreV4.AllocateIdsResponse;
  29 import com.google.apphosting.datastore.DatastoreV4.BeginTransactionRequest;
  30 import com.google.apphosting.datastore.DatastoreV4.BeginTransactionResponse;
  31 import com.google.apphosting.datastore.DatastoreV4.CommitRequest;
  32 import com.google.apphosting.datastore.DatastoreV4.CommitResponse;
  33 import com.google.apphosting.datastore.DatastoreV4.ContinueQueryRequest;
  34 import com.google.apphosting.datastore.DatastoreV4.ContinueQueryResponse;
  35 import com.google.apphosting.datastore.DatastoreV4.LookupRequest;
  36 import com.google.apphosting.datastore.DatastoreV4.LookupResponse;
  37 import com.google.apphosting.datastore.DatastoreV4.RollbackRequest;
  38 import com.google.apphosting.datastore.DatastoreV4.RollbackResponse;
  39 import com.google.apphosting.datastore.DatastoreV4.RunQueryRequest;
  40 import com.google.apphosting.datastore.DatastoreV4.RunQueryResponse;
  41 import com.google.protobuf.Message;
  42
  43 import java.io.File;
  44 import java.io.IOException;
  45 import java.security.GeneralSecurityException;
  46 import java.util.ConcurrentModificationException;
  47 import java.util.HashMap;
  48 import java.util.List;
  49 import java.util.Map;
  50 import java.util.concurrent.Callable;
  51 import java.util.concurrent.ExecutorService;
  52 import java.util.concurrent.Executors;
  53 import java.util.concurrent.Future;
  54 import java.util.regex.Matcher;
  55 import java.util.regex.Pattern;
  56
  57 import javax.servlet.http.HttpServletResponse;
  58
  59 /**
  60  * Redirects API calls to Google Cloud Datastore.
  61  */
  62 final class CloudDatastoreProxy implements DatastoreV4Proxy {
  63
  64   private static final ExecutorService executor = Executors.newCachedThreadPool();
  65
  66   private static final CloudDatastoreProtoConverter CONVERTER =
  67       CloudDatastoreProtoConverter.getInstance();
  68
  69   private final Datastore datastore;
  70
  71   CloudDatastoreProxy(Datastore datastore) {
  72     this.datastore = checkNotNull(datastore);
  73   }
  74
  75   /**
  76    * Creates a {@link CloudDatastoreProxy}. This method has the side effect
  77    * of installing minimal stubs ({@link EnvironmentFactory} and
  78    * {@link Delegate}) in the API proxy if they have not already been installed.
  79    */
  80   static CloudDatastoreProxy create(DatastoreServiceConfig config) {
  81     checkArgument(config.getApiVersion() == ApiVersion.CLOUD_DATASTORE);
  82     DatastoreOptions options;
  83     try {
  84       options = getDatastoreOptions();
  85     } catch (GeneralSecurityException | IOException e) {
  86       throw new RuntimeException(
  87           "Could not get Cloud Datastore options from environment.", e);
  88     }
  89     ensureApiProxyIsConfigured(options);
  90     return new CloudDatastoreProxy(DatastoreFactory.get().create(options));
  91   }
  92
  93   @Override
  94   public Future<BeginTransactionResponse> beginTransaction(BeginTransactionRequest v4Request) {
  95     return makeCall(new Callable<BeginTransactionResponse>() {
  96       @Override
  97       public BeginTransactionResponse call() throws DatastoreException {
  98         return CONVERTER.toV4BeginTransactionResponse(datastore.beginTransaction(
  99             DatastoreV1.BeginTransactionRequest.getDefaultInstance())).build();
 100       }
 101     });
 102   }
 103
 104   @Override
 105   public Future<RollbackResponse> rollback(final RollbackRequest v4Request) {
 106     return makeCall(new Callable<RollbackResponse>() {
 107       @Override
 108       public RollbackResponse call() throws DatastoreException {
 109         datastore.rollback(CONVERTER.toV1RollbackRequest(v4Request).build());
 110         return RollbackResponse.getDefaultInstance();
 111       }
 112     });
 113   }
 114
 115   @Override
 116   public Future<RunQueryResponse> runQuery(final RunQueryRequest v4Request) {
 117     return makeCall(new Callable<RunQueryResponse>() {
 118       @Override
 119       public RunQueryResponse call() throws DatastoreException {
 120         return CONVERTER.toV4RunQueryResponse(datastore.runQuery(
 121             CONVERTER.toV1RunQueryRequest(v4Request).build())).build();
 122       }
 123     });
 124   }
 125
 126   @Override
 127   public Future<ContinueQueryResponse> continueQuery(ContinueQueryRequest v4Request) {
 128     throw new UnsupportedOperationException();
 129   }
 130
 131   @Override
 132   public Future<LookupResponse> lookup(final LookupRequest v4Request) {
 133     return makeCall(new Callable<LookupResponse>() {
 134       @Override
 135       public LookupResponse call() throws DatastoreException {
 136         return CONVERTER.toV4LookupResponse(datastore.lookup(
 137             CONVERTER.toV1LookupRequest(v4Request).build())).build();
 138       }
 139     });
 140   }
 141
 142   @Override
 143   public Future<AllocateIdsResponse> allocateIds(final AllocateIdsRequest v4Request) {
 144     return makeCall(new Callable<AllocateIdsResponse>() {
 145       @Override
 146       public AllocateIdsResponse call() throws DatastoreException {
 147         return CONVERTER.toV4AllocateIdsResponse(datastore.allocateIds(
 148             CONVERTER.toV1AllocateIdsRequest(v4Request).build())).build();
 149       }
 150     });
 151   }
 152
 153   @Override
 154   public Future<CommitResponse> commit(final CommitRequest v4Request) {
 155     return makeCall(new Callable<CommitResponse>() {
 156       @Override
 157       public CommitResponse call() throws DatastoreException {
 158         return CONVERTER.toV4CommitResponse(datastore.commit(
 159             CONVERTER.toV1CommitRequest(v4Request).build()), v4Request).build();
 160       }
 161     });
 162   }
 163
 164   @Override
 165   public Future<CommitResponse> rawCommit(byte[] v4Request) {
 166     throw new UnsupportedOperationException();
 167   }
 168
 169   private static <T extends Message> Future<T> makeCall(final Callable<T> request) {
 170     return executor.submit(new Callable<T>() {
 171       @Override
 172       public T call() throws Exception {
 173         try {
 174           return request.call();
 175         } catch (DatastoreException e) {
 176           throw extractException(e.getMessage(), e.getCode());
 177         }
 178       }
 179     });
 180   }
 181
 182   private static final Pattern reasonPattern = Pattern.compile("\"reason\": \"(.*)\",?\\n");
 183   private static final Pattern messagePattern = Pattern.compile("\"message\": \"(.*)\",?\\n");
 184
 185   /**
 186    * Convert the exception thrown by Cloud Datastore to version comparable
 187    * to the Exceptions thrown by ApiProxy.
 188    */
 189   protected static Exception extractException(String rawMessage, int httpCode) {
 190     Matcher msgMatcher = messagePattern.matcher(rawMessage);
 191     String message = msgMatcher.find() ? msgMatcher.group(1) : "[" + rawMessage + "]\n";
 192     switch (httpCode) {
 193       case HttpServletResponse.SC_BAD_REQUEST:
 194         return new IllegalArgumentException(message);
 195       case HttpServletResponse.SC_FORBIDDEN:
 196         Matcher reasonMatch = reasonPattern.matcher(rawMessage);
 197         if (reasonMatch.find() && reasonMatch.group(1).equals("DEADLINE_EXCEEDED")) {
 198           return new DatastoreTimeoutException(message);
 199         } else {
 200           return new IllegalStateException(message);
 201         }
 202       case HttpServletResponse.SC_PRECONDITION_FAILED:
 203         return new DatastoreNeedIndexException(message);
 204       case HttpServletResponse.SC_CONFLICT:
 205         return new ConcurrentModificationException(message);
 206       case HttpServletResponse.SC_SERVICE_UNAVAILABLE:
 207         return new IllegalStateException(message);
 208       case HttpServletResponse.SC_INTERNAL_SERVER_ERROR:
 209         return new DatastoreFailureException(message);
 210       case HttpServletResponse.SC_PAYMENT_REQUIRED:
 211       default:
 212         return new RuntimeException(message);
 213     }
 214   }
 215
 216   private static DatastoreOptions getDatastoreOptions()
 217       throws GeneralSecurityException, IOException {
 218     DatastoreOptions.Builder options = new DatastoreOptions.Builder();
 219     options.dataset(EnvProxy.getenv("DATASTORE_DATASET"));
 220     options.host(EnvProxy.getenv("DATASTORE_HOST"));
 221
 222     String serviceAccount = EnvProxy.getenv("DATASTORE_SERVICE_ACCOUNT");
 223     String privateKeyFile = EnvProxy.getenv("DATASTORE_PRIVATE_KEY_FILE");
 224     Credential credential;
 225     if (Boolean.valueOf(EnvProxy.getenv("__DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST"))) {
 226       credential = null;
 227     } else if (serviceAccount != null && privateKeyFile != null) {
 228       credential = getServiceAccountCredential(serviceAccount, privateKeyFile);
 229     } else {
 230       credential = getComputeEngineCredential();
 231     }
 232     options.credential(credential);
 233
 234     final String versionOverrideForTest = EnvProxy.getenv("__DATASTORE_VERSION_OVERRIDE_FOR_TEST");
 235     if (versionOverrideForTest != null) {
 236       options.initializer(new HttpRequestInitializer() {
 237         @Override
 238         public void initialize(HttpRequest request) throws IOException {
 239           request.getUrl().setRawPath(
 240               request.getUrl().getRawPath().replaceFirst(
 241                   DatastoreFactory.VERSION, versionOverrideForTest));
 242         }
 243       });
 244     }
 245
 246     return options.build();
 247   }
 248
 249   private static Credential getServiceAccountCredential(String account, String privateKeyFile)
 250       throws GeneralSecurityException, IOException {
 251     return new GoogleCredential.Builder()
 252         .setTransport(GoogleNetHttpTransport.newTrustedTransport())
 253         .setJsonFactory(new JacksonFactory())
 254         .setServiceAccountId(account)
 255         .setServiceAccountScopes(DatastoreOptions.SCOPES)
 256         .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile))
 257         .build();
 258   }
 259
 260   private static Credential getComputeEngineCredential()
 261       throws GeneralSecurityException, IOException {
 262     NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
 263     try {
 264       ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory());
 265       credential.refreshToken();
 266       return credential;
 267     } catch (IOException e) {
 268       return null;
 269     }
 270   }
 271
 272   /**
 273    * Make sure that the API proxy has been configured. If it's already
 274    * configured (e.g. because the Remote API has been installed or the factory
 275    * has already been used), do nothing. Otherwise, install a stub environment
 276    * and delegate.
 277    */
 278   private static synchronized void ensureApiProxyIsConfigured(DatastoreOptions options) {
 279     boolean hasEnvironmentOrFactory = (ApiProxy.getCurrentEnvironment() != null);
 280     boolean hasDelegate = (ApiProxy.getDelegate() != null);
 281
 282     if (hasEnvironmentOrFactory && hasDelegate) {
 283       return;
 284     }
 285
 286     if (hasEnvironmentOrFactory) {
 287       throw new IllegalStateException(
 288           "An ApiProxy.Environment or ApiProxy.EnvironmentFactory was already installed. "
 289           + "Cannot use Cloud Datastore.");
 290     } else if (hasDelegate) {
 291       throw new IllegalStateException(
 292           "An ApiProxy.Delegate was already installed. Cannot use Cloud Datastore.");
 293     }
 294
 295     ApiProxy.setEnvironmentFactory(
 296         new StubApiProxyEnvironmentFactory(getFullAppId(options)));
 297     ApiProxy.setDelegate(new StubApiProxyDelegate());
 298   }
 299
 300   /**
 301    * Attempt to determine the full app id. This is only necessary if the client
 302    * did not install the Remote API (which will determine it automatically).
 303    * <p>
 304    * By default, take the dataset from the provided {@link DatastoreOptions} and
 305    * prepend {@code s~}. Apps for which this is incorrect (e.g. apps running in
 306    * Europe) can specify the full app id via the {@code _DATASTORE_FULL_DATASET}
 307    * environment variable.
 308    */
 309   private static String getFullAppId(DatastoreOptions options) {
 310     String fullDataset = EnvProxy.getenv("_DATASTORE_FULL_DATASET");
 311     if (fullDataset != null) {
 312       return fullDataset;
 313     } else if (options.getHost().startsWith("http://localhost")) {
 314       return options.getDataset();
 315     }
 316     return "s~" + options.getDataset();
 317   }
 318
 319   /**
 320    * A {@link Delegate} that throws {@link UnsupportedOperationException} for
 321    * all methods.
 322    */
 323   static class StubApiProxyDelegate implements Delegate<Environment> {
 324     private static final String UNSUPPORTED_API_PATTERN =
 325         "Calls to %s.%s are not supported under this configuration, only "
 326         + "calls to Cloud Datastore. To use other APIs, first install the "
 327         + "Remote API.";
 328
 329     @Override
 330     public byte[] makeSyncCall(Environment environment, String packageName,
 331         String methodName, byte[] request) throws ApiProxyException {
 332       throw new UnsupportedOperationException(
 333           String.format(UNSUPPORTED_API_PATTERN, packageName, methodName));
 334     }
 335
 336     @Override
 337     public Future<byte[]> makeAsyncCall(Environment environment, String packageName,
 338         String methodName, byte[] request, ApiConfig apiConfig) {
 339       throw new UnsupportedOperationException(
 340           String.format(UNSUPPORTED_API_PATTERN, packageName, methodName));
 341     }
 342
 343     @Override
 344     public void log(Environment environment, LogRecord record) {
 345       throw new UnsupportedOperationException();
 346     }
 347
 348     @Override
 349     public void flushLogs(Environment environment) {
 350       throw new UnsupportedOperationException();
 351     }
 352
 353     @Override
 354     public List<Thread> getRequestThreads(Environment environment) {
 355       throw new UnsupportedOperationException();
 356     }
 357   }
 358
 359   /**
 360    * An {@link EnvironmentFactory} that builds {@link StubApiProxyEnvironment}s.
 361    */
 362   static class StubApiProxyEnvironmentFactory implements EnvironmentFactory {
 363     private final String appId;
 364
 365     public StubApiProxyEnvironmentFactory(String appId) {
 366       this.appId = appId;
 367     }
 368
 369     @Override
 370     public Environment newEnvironment() {
 371       return new StubApiProxyEnvironment(appId);
 372     }
 373   }
 374
 375   /**
 376    * An {@link Environment} that supports the minimal subset of features needed
 377    * to run code from the datastore package outside of App Engine. All other
 378    * methods throw {@link UnsupportedOperationException}.
 379    */
 380   static class StubApiProxyEnvironment implements Environment {
 381     private final Map<String, Object> attributes;
 382     private final String appId;
 383
 384     public StubApiProxyEnvironment(String appId) {
 385       this.attributes = new HashMap<>();
 386       this.appId = appId;
 387     }
 388
 389     @Override
 390     public boolean isLoggedIn() {
 391       throw new UnsupportedOperationException();
 392     }
 393
 394     @Override
 395     public boolean isAdmin() {
 396       throw new UnsupportedOperationException();
 397     }
 398
 399     @Override
 400     public String getVersionId() {
 401       throw new UnsupportedOperationException();
 402     }
 403
 404     @Deprecated
 405     @Override
 406     public String getRequestNamespace() {
 407       throw new UnsupportedOperationException();
 408     }
 409
 410     @Override
 411     public long getRemainingMillis() {
 412       throw new UnsupportedOperationException();
 413     }
 414
 415     @Override
 416     public String getModuleId() {
 417       throw new UnsupportedOperationException();
 418     }
 419
 420     @Override
 421     public String getEmail() {
 422       throw new UnsupportedOperationException();
 423     }
 424
 425     @Override
 426     public String getAuthDomain() {
 427       throw new UnsupportedOperationException();
 428     }
 429
 430     @Override
 431     public Map<String, Object> getAttributes() {
 432       return attributes;
 433     }
 434
 435     @Override
 436     public String getAppId() {
 437       return appId;
 438     }
 439   }
 440 }