java/src/main/com/google/appengine/api/datastore/RemoteCloudDatastoreV1Proxy.java

   1 package com.google.appengine.api.datastore;
   2
   3 import static com.google.common.base.Preconditions.checkArgument;
   4 import static com.google.common.base.Preconditions.checkNotNull;
   5
   6 import com.google.api.client.auth.oauth2.Credential;
   7 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
   8 import com.google.api.client.googleapis.compute.ComputeCredential;
   9 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
  10 import com.google.api.client.http.HttpRequest;
  11 import com.google.api.client.http.HttpRequestInitializer;
  12 import com.google.api.client.http.javanet.NetHttpTransport;
  13 import com.google.api.client.json.jackson.JacksonFactory;
  14 import com.google.appengine.api.datastore.DatastoreServiceConfig.ApiVersion;
  15 import com.google.apphosting.api.ApiProxy;
  16 import com.google.apphosting.api.ApiProxy.ApiConfig;
  17 import com.google.apphosting.api.ApiProxy.ApiProxyException;
  18 import com.google.apphosting.api.ApiProxy.Delegate;
  19 import com.google.apphosting.api.ApiProxy.Environment;
  20 import com.google.apphosting.api.ApiProxy.EnvironmentFactory;
  21 import com.google.apphosting.api.ApiProxy.LogRecord;
  22 import com.google.datastore.v1beta3.AllocateIdsRequest;
  23 import com.google.datastore.v1beta3.AllocateIdsResponse;
  24 import com.google.datastore.v1beta3.BeginTransactionRequest;
  25 import com.google.datastore.v1beta3.BeginTransactionResponse;
  26 import com.google.datastore.v1beta3.CommitRequest;
  27 import com.google.datastore.v1beta3.CommitResponse;
  28 import com.google.datastore.v1beta3.LookupRequest;
  29 import com.google.datastore.v1beta3.LookupResponse;
  30 import com.google.datastore.v1beta3.RollbackRequest;
  31 import com.google.datastore.v1beta3.RollbackResponse;
  32 import com.google.datastore.v1beta3.RunQueryRequest;
  33 import com.google.datastore.v1beta3.RunQueryResponse;
  34 import com.google.datastore.v1beta3.client.Datastore;
  35 import com.google.datastore.v1beta3.client.DatastoreException;
  36 import com.google.datastore.v1beta3.client.DatastoreFactory;
  37 import com.google.datastore.v1beta3.client.DatastoreOptions;
  38 import com.google.protobuf.InvalidProtocolBufferException;
  39 import com.google.protobuf.Message;
  40
  41 import java.io.File;
  42 import java.io.IOException;
  43 import java.security.GeneralSecurityException;
  44 import java.util.ConcurrentModificationException;
  45 import java.util.HashMap;
  46 import java.util.List;
  47 import java.util.Map;
  48 import java.util.concurrent.Callable;
  49 import java.util.concurrent.ExecutorService;
  50 import java.util.concurrent.Executors;
  51 import java.util.concurrent.Future;
  52 import java.util.regex.Matcher;
  53 import java.util.regex.Pattern;
  54
  55 import javax.servlet.http.HttpServletResponse;
  56
  57 /**
  58  * {@link CloudDatastoreV1Proxy} that makes remote calls (currently over HTTP).
  59  *
  60  * <p>Methods in this class do not populate the project id field in outgoing requests since it
  61  * is not required when using the API over HTTP.
  62  */
  63 final class RemoteCloudDatastoreV1Proxy implements CloudDatastoreV1Proxy {
  64
  65   private static final ExecutorService executor = Executors.newCachedThreadPool();
  66
  67   private final Datastore datastore;
  68
  69   RemoteCloudDatastoreV1Proxy(Datastore datastore) {
  70     this.datastore = checkNotNull(datastore);
  71   }
  72
  73   /**
  74    * Creates a {@link RemoteCloudDatastoreV1Proxy}. This method has the side effect
  75    * of installing minimal stubs ({@link EnvironmentFactory} and
  76    * {@link Delegate}) in the API proxy if they have not already been installed.
  77    */
  78   static RemoteCloudDatastoreV1Proxy create(DatastoreServiceConfig config) {
  79     checkArgument(config.getApiVersion() == ApiVersion.CLOUD_DATASTORE_V1_REMOTE);
  80     DatastoreOptions options;
  81     try {
  82       options = getDatastoreOptions();
  83     } catch (GeneralSecurityException | IOException e) {
  84       throw new RuntimeException(
  85           "Could not get Cloud Datastore options from environment.", e);
  86     }
  87     ensureApiProxyIsConfigured(options);
  88     return new RemoteCloudDatastoreV1Proxy(DatastoreFactory.get().create(options));
  89   }
  90
  91   @Override
  92   public Future<BeginTransactionResponse> beginTransaction(final BeginTransactionRequest req) {
  93     return makeCall(new Callable<BeginTransactionResponse>() {
  94       @Override
  95       public BeginTransactionResponse call() throws DatastoreException {
  96         return datastore.beginTransaction(req);
  97       }
  98     });
  99   }
 100
 101   @Override
 102   public Future<RollbackResponse> rollback(final RollbackRequest req) {
 103     return makeCall(new Callable<RollbackResponse>() {
 104       @Override
 105       public RollbackResponse call() throws DatastoreException {
 106         return datastore.rollback(req);
 107       }
 108     });
 109   }
 110
 111   @Override
 112   public Future<RunQueryResponse> runQuery(final RunQueryRequest req) {
 113     return makeCall(new Callable<RunQueryResponse>() {
 114       @Override
 115       public RunQueryResponse call() throws DatastoreException {
 116         return datastore.runQuery(req);
 117       }
 118     });
 119   }
 120
 121   @Override
 122   public Future<LookupResponse> lookup(final LookupRequest req) {
 123     return makeCall(new Callable<LookupResponse>() {
 124       @Override
 125       public LookupResponse call() throws DatastoreException {
 126         return datastore.lookup(req);
 127       }
 128     });
 129   }
 130
 131   @Override
 132   public Future<AllocateIdsResponse> allocateIds(final AllocateIdsRequest req) {
 133     return makeCall(new Callable<AllocateIdsResponse>() {
 134       @Override
 135       public AllocateIdsResponse call() throws DatastoreException {
 136         return datastore.allocateIds(req);
 137       }
 138     });
 139   }
 140
 141   @Override
 142   public Future<CommitResponse> commit(final CommitRequest req) {
 143     return makeCall(new Callable<CommitResponse>() {
 144       @Override
 145       public CommitResponse call() throws DatastoreException {
 146         return datastore.commit(req);
 147       }
 148     });
 149   }
 150
 151   @Override
 152   public Future<CommitResponse> rawCommit(byte[] bytes) {
 153     try {
 154       return commit(CommitRequest.parseFrom(bytes));
 155     } catch (InvalidProtocolBufferException e) {
 156       throw new IllegalStateException(e);
 157     }
 158   }
 159
 160   private static <T extends Message> Future<T> makeCall(final Callable<T> request) {
 161     return executor.submit(new Callable<T>() {
 162       @Override
 163       public T call() throws Exception {
 164         try {
 165           return request.call();
 166         } catch (DatastoreException e) {
 167           throw extractException(e.getMessage(), e.getCode());
 168         }
 169       }
 170     });
 171   }
 172
 173   private static final Pattern reasonPattern = Pattern.compile("\"reason\": \"(.*)\",?\\n");
 174   private static final Pattern messagePattern = Pattern.compile("\"message\": \"(.*)\",?\\n");
 175
 176   /**
 177    * Convert the exception thrown by Cloud Datastore to version comparable
 178    * to the Exceptions thrown by ApiProxy.
 179    */
 180   protected static Exception extractException(String rawMessage, int httpCode) {
 181     Matcher msgMatcher = messagePattern.matcher(rawMessage);
 182     String message = msgMatcher.find() ? msgMatcher.group(1) : "[" + rawMessage + "]\n";
 183     switch (httpCode) {
 184       case HttpServletResponse.SC_BAD_REQUEST:
 185         return new IllegalArgumentException(message);
 186       case HttpServletResponse.SC_FORBIDDEN:
 187         Matcher reasonMatch = reasonPattern.matcher(rawMessage);
 188         if (reasonMatch.find() && reasonMatch.group(1).equals("DEADLINE_EXCEEDED")) {
 189           return new DatastoreTimeoutException(message);
 190         } else {
 191           return new IllegalStateException(message);
 192         }
 193       case HttpServletResponse.SC_PRECONDITION_FAILED:
 194         return new DatastoreNeedIndexException(message);
 195       case HttpServletResponse.SC_CONFLICT:
 196         return new ConcurrentModificationException(message);
 197       case HttpServletResponse.SC_SERVICE_UNAVAILABLE:
 198         return new IllegalStateException(message);
 199       case HttpServletResponse.SC_INTERNAL_SERVER_ERROR:
 200         return new DatastoreFailureException(message);
 201       case HttpServletResponse.SC_PAYMENT_REQUIRED:
 202       default:
 203         return new RuntimeException(message);
 204     }
 205   }
 206
 207   private static DatastoreOptions getDatastoreOptions()
 208       throws GeneralSecurityException, IOException {
 209     DatastoreOptions.Builder options = new DatastoreOptions.Builder();
 210     options.projectId(EnvProxy.getenv("DATASTORE_PROJECT_ID"));
 211     options.host(EnvProxy.getenv("DATASTORE_HOST"));
 212
 213     String serviceAccount = EnvProxy.getenv("DATASTORE_SERVICE_ACCOUNT");
 214     String privateKeyFile = EnvProxy.getenv("DATASTORE_PRIVATE_KEY_FILE");
 215     Credential credential;
 216     if (Boolean.valueOf(EnvProxy.getenv("__DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST"))) {
 217       credential = null;
 218     } else if (serviceAccount != null && privateKeyFile != null) {
 219       credential = getServiceAccountCredential(serviceAccount, privateKeyFile);
 220     } else {
 221       credential = getComputeEngineCredential();
 222     }
 223     options.credential(credential);
 224
 225     final String versionOverrideForTest = EnvProxy.getenv("__DATASTORE_VERSION_OVERRIDE_FOR_TEST");
 226     if (versionOverrideForTest != null) {
 227       options.initializer(new HttpRequestInitializer() {
 228         @Override
 229         public void initialize(HttpRequest request) throws IOException {
 230           request.getUrl().setRawPath(
 231               request.getUrl().getRawPath().replaceFirst(
 232                   DatastoreFactory.VERSION, versionOverrideForTest));
 233         }
 234       });
 235     }
 236
 237     return options.build();
 238   }
 239
 240   private static Credential getServiceAccountCredential(String account, String privateKeyFile)
 241       throws GeneralSecurityException, IOException {
 242     return new GoogleCredential.Builder()
 243         .setTransport(GoogleNetHttpTransport.newTrustedTransport())
 244         .setJsonFactory(new JacksonFactory())
 245         .setServiceAccountId(account)
 246         .setServiceAccountScopes(DatastoreOptions.SCOPES)
 247         .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile))
 248         .build();
 249   }
 250
 251   private static Credential getComputeEngineCredential()
 252       throws GeneralSecurityException, IOException {
 253     NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
 254     try {
 255       ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory());
 256       credential.refreshToken();
 257       return credential;
 258     } catch (IOException e) {
 259       return null;
 260     }
 261   }
 262
 263   /**
 264    * Make sure that the API proxy has been configured. If it's already
 265    * configured (e.g. because the Remote API has been installed or the factory
 266    * has already been used), do nothing. Otherwise, install a stub environment
 267    * and delegate.
 268    */
 269   private static synchronized void ensureApiProxyIsConfigured(DatastoreOptions options) {
 270     boolean hasEnvironmentOrFactory = (ApiProxy.getCurrentEnvironment() != null);
 271     boolean hasDelegate = (ApiProxy.getDelegate() != null);
 272
 273     if (hasEnvironmentOrFactory && hasDelegate) {
 274       return;
 275     }
 276
 277     if (hasEnvironmentOrFactory) {
 278       throw new IllegalStateException(
 279           "An ApiProxy.Environment or ApiProxy.EnvironmentFactory was already installed. "
 280           + "Cannot use Cloud Datastore.");
 281     } else if (hasDelegate) {
 282       throw new IllegalStateException(
 283           "An ApiProxy.Delegate was already installed. Cannot use Cloud Datastore.");
 284     }
 285
 286     ApiProxy.setEnvironmentFactory(
 287         new StubApiProxyEnvironmentFactory(options.getProjectId()));
 288     ApiProxy.setDelegate(new StubApiProxyDelegate());
 289   }
 290
 291   /**
 292    * A {@link Delegate} that throws {@link UnsupportedOperationException} for
 293    * all methods.
 294    */
 295   static class StubApiProxyDelegate implements Delegate<Environment> {
 296     private static final String UNSUPPORTED_API_PATTERN =
 297         "Calls to %s.%s are not supported under this configuration, only "
 298         + "calls to Cloud Datastore. To use other APIs, first install the "
 299         + "Remote API.";
 300
 301     @Override
 302     public byte[] makeSyncCall(Environment environment, String packageName,
 303         String methodName, byte[] request) throws ApiProxyException {
 304       throw new UnsupportedOperationException(
 305           String.format(UNSUPPORTED_API_PATTERN, packageName, methodName));
 306     }
 307
 308     @Override
 309     public Future<byte[]> makeAsyncCall(Environment environment, String packageName,
 310         String methodName, byte[] request, ApiConfig apiConfig) {
 311       throw new UnsupportedOperationException(
 312           String.format(UNSUPPORTED_API_PATTERN, packageName, methodName));
 313     }
 314
 315     @Override
 316     public void log(Environment environment, LogRecord record) {
 317       throw new UnsupportedOperationException();
 318     }
 319
 320     @Override
 321     public void flushLogs(Environment environment) {
 322       throw new UnsupportedOperationException();
 323     }
 324
 325     @Override
 326     public List<Thread> getRequestThreads(Environment environment) {
 327       throw new UnsupportedOperationException();
 328     }
 329   }
 330
 331   /**
 332    * An {@link EnvironmentFactory} that builds {@link StubApiProxyEnvironment}s.
 333    */
 334   static class StubApiProxyEnvironmentFactory implements EnvironmentFactory {
 335     private final String appId;
 336
 337     public StubApiProxyEnvironmentFactory(String appId) {
 338       this.appId = appId;
 339     }
 340
 341     @Override
 342     public Environment newEnvironment() {
 343       return new StubApiProxyEnvironment(appId);
 344     }
 345   }
 346
 347   /**
 348    * An {@link Environment} that supports the minimal subset of features needed
 349    * to run code from the datastore package outside of App Engine. All other
 350    * methods throw {@link UnsupportedOperationException}.
 351    */
 352   static class StubApiProxyEnvironment implements Environment {
 353     private final Map<String, Object> attributes;
 354     private final String appId;
 355
 356     public StubApiProxyEnvironment(String appId) {
 357       this.attributes = new HashMap<>();
 358       this.appId = appId;
 359     }
 360
 361     @Override
 362     public boolean isLoggedIn() {
 363       throw new UnsupportedOperationException();
 364     }
 365
 366     @Override
 367     public boolean isAdmin() {
 368       throw new UnsupportedOperationException();
 369     }
 370
 371     @Override
 372     public String getVersionId() {
 373       throw new UnsupportedOperationException();
 374     }
 375
 376     @Deprecated
 377     @Override
 378     public String getRequestNamespace() {
 379       throw new UnsupportedOperationException();
 380     }
 381
 382     @Override
 383     public long getRemainingMillis() {
 384       throw new UnsupportedOperationException();
 385     }
 386
 387     @Override
 388     public String getModuleId() {
 389       throw new UnsupportedOperationException();
 390     }
 391
 392     @Override
 393     public String getEmail() {
 394       throw new UnsupportedOperationException();
 395     }
 396
 397     @Override
 398     public String getAuthDomain() {
 399       throw new UnsupportedOperationException();
 400     }
 401
 402     @Override
 403     public Map<String, Object> getAttributes() {
 404       return attributes;
 405     }
 406
 407     @Override
 408     public String getAppId() {
 409       return appId;
 410     }
 411   }
 412 }