1 package com
.google
.appengine
.api
.datastore
;
3 import static com
.google
.common
.base
.Preconditions
.checkArgument
;
4 import static com
.google
.common
.base
.Preconditions
.checkNotNull
;
6 import com
.google
.api
.client
.auth
.oauth2
.Credential
;
7 import com
.google
.api
.client
.googleapis
.auth
.oauth2
.GoogleCredential
;
8 import com
.google
.api
.client
.googleapis
.compute
.ComputeCredential
;
9 import com
.google
.api
.client
.googleapis
.javanet
.GoogleNetHttpTransport
;
10 import com
.google
.api
.client
.http
.HttpRequest
;
11 import com
.google
.api
.client
.http
.HttpRequestInitializer
;
12 import com
.google
.api
.client
.http
.javanet
.NetHttpTransport
;
13 import com
.google
.api
.client
.json
.jackson
.JacksonFactory
;
14 import com
.google
.api
.services
.datastore
.DatastoreV1
;
15 import com
.google
.api
.services
.datastore
.client
.Datastore
;
16 import com
.google
.api
.services
.datastore
.client
.DatastoreException
;
17 import com
.google
.api
.services
.datastore
.client
.DatastoreFactory
;
18 import com
.google
.api
.services
.datastore
.client
.DatastoreOptions
;
19 import com
.google
.appengine
.api
.datastore
.DatastoreServiceConfig
.ApiVersion
;
20 import com
.google
.apphosting
.api
.ApiProxy
;
21 import com
.google
.apphosting
.api
.ApiProxy
.ApiConfig
;
22 import com
.google
.apphosting
.api
.ApiProxy
.ApiProxyException
;
23 import com
.google
.apphosting
.api
.ApiProxy
.Delegate
;
24 import com
.google
.apphosting
.api
.ApiProxy
.Environment
;
25 import com
.google
.apphosting
.api
.ApiProxy
.EnvironmentFactory
;
26 import com
.google
.apphosting
.api
.ApiProxy
.LogRecord
;
27 import com
.google
.apphosting
.datastore
.DatastoreV4
.AllocateIdsRequest
;
28 import com
.google
.apphosting
.datastore
.DatastoreV4
.AllocateIdsResponse
;
29 import com
.google
.apphosting
.datastore
.DatastoreV4
.BeginTransactionRequest
;
30 import com
.google
.apphosting
.datastore
.DatastoreV4
.BeginTransactionResponse
;
31 import com
.google
.apphosting
.datastore
.DatastoreV4
.CommitRequest
;
32 import com
.google
.apphosting
.datastore
.DatastoreV4
.CommitResponse
;
33 import com
.google
.apphosting
.datastore
.DatastoreV4
.ContinueQueryRequest
;
34 import com
.google
.apphosting
.datastore
.DatastoreV4
.ContinueQueryResponse
;
35 import com
.google
.apphosting
.datastore
.DatastoreV4
.LookupRequest
;
36 import com
.google
.apphosting
.datastore
.DatastoreV4
.LookupResponse
;
37 import com
.google
.apphosting
.datastore
.DatastoreV4
.RollbackRequest
;
38 import com
.google
.apphosting
.datastore
.DatastoreV4
.RollbackResponse
;
39 import com
.google
.apphosting
.datastore
.DatastoreV4
.RunQueryRequest
;
40 import com
.google
.apphosting
.datastore
.DatastoreV4
.RunQueryResponse
;
41 import com
.google
.protobuf
.Message
;
44 import java
.io
.IOException
;
45 import java
.security
.GeneralSecurityException
;
46 import java
.util
.ConcurrentModificationException
;
47 import java
.util
.HashMap
;
48 import java
.util
.List
;
50 import java
.util
.concurrent
.Callable
;
51 import java
.util
.concurrent
.ExecutorService
;
52 import java
.util
.concurrent
.Executors
;
53 import java
.util
.concurrent
.Future
;
54 import java
.util
.regex
.Matcher
;
55 import java
.util
.regex
.Pattern
;
57 import javax
.servlet
.http
.HttpServletResponse
;
60 * Redirects API calls to Google Cloud Datastore.
62 final class CloudDatastoreProxy
implements DatastoreV4Proxy
{
64 private static final ExecutorService executor
= Executors
.newCachedThreadPool();
66 private static final CloudDatastoreProtoConverter CONVERTER
=
67 CloudDatastoreProtoConverter
.getInstance();
69 private final Datastore datastore
;
71 CloudDatastoreProxy(Datastore datastore
) {
72 this.datastore
= checkNotNull(datastore
);
76 * Creates a {@link CloudDatastoreProxy}. This method has the side effect
77 * of installing minimal stubs ({@link EnvironmentFactory} and
78 * {@link Delegate}) in the API proxy if they have not already been installed.
80 static CloudDatastoreProxy
create(DatastoreServiceConfig config
) {
81 checkArgument(config
.getApiVersion() == ApiVersion
.CLOUD_DATASTORE
);
82 DatastoreOptions options
;
84 options
= getDatastoreOptions();
85 } catch (GeneralSecurityException
| IOException e
) {
86 throw new RuntimeException(
87 "Could not get Cloud Datastore options from environment.", e
);
89 ensureApiProxyIsConfigured(options
);
90 return new CloudDatastoreProxy(DatastoreFactory
.get().create(options
));
94 public Future
<BeginTransactionResponse
> beginTransaction(BeginTransactionRequest v4Request
) {
95 return makeCall(new Callable
<BeginTransactionResponse
>() {
97 public BeginTransactionResponse
call() throws DatastoreException
{
98 return CONVERTER
.toV4BeginTransactionResponse(datastore
.beginTransaction(
99 DatastoreV1
.BeginTransactionRequest
.getDefaultInstance())).build();
105 public Future
<RollbackResponse
> rollback(final RollbackRequest v4Request
) {
106 return makeCall(new Callable
<RollbackResponse
>() {
108 public RollbackResponse
call() throws DatastoreException
{
109 datastore
.rollback(CONVERTER
.toV1RollbackRequest(v4Request
).build());
110 return RollbackResponse
.getDefaultInstance();
116 public Future
<RunQueryResponse
> runQuery(final RunQueryRequest v4Request
) {
117 return makeCall(new Callable
<RunQueryResponse
>() {
119 public RunQueryResponse
call() throws DatastoreException
{
120 return CONVERTER
.toV4RunQueryResponse(datastore
.runQuery(
121 CONVERTER
.toV1RunQueryRequest(v4Request
).build())).build();
127 public Future
<ContinueQueryResponse
> continueQuery(ContinueQueryRequest v4Request
) {
128 throw new UnsupportedOperationException();
132 public Future
<LookupResponse
> lookup(final LookupRequest v4Request
) {
133 return makeCall(new Callable
<LookupResponse
>() {
135 public LookupResponse
call() throws DatastoreException
{
136 return CONVERTER
.toV4LookupResponse(datastore
.lookup(
137 CONVERTER
.toV1LookupRequest(v4Request
).build())).build();
143 public Future
<AllocateIdsResponse
> allocateIds(final AllocateIdsRequest v4Request
) {
144 return makeCall(new Callable
<AllocateIdsResponse
>() {
146 public AllocateIdsResponse
call() throws DatastoreException
{
147 return CONVERTER
.toV4AllocateIdsResponse(datastore
.allocateIds(
148 CONVERTER
.toV1AllocateIdsRequest(v4Request
).build())).build();
154 public Future
<CommitResponse
> commit(final CommitRequest v4Request
) {
155 return makeCall(new Callable
<CommitResponse
>() {
157 public CommitResponse
call() throws DatastoreException
{
158 return CONVERTER
.toV4CommitResponse(datastore
.commit(
159 CONVERTER
.toV1CommitRequest(v4Request
).build()), v4Request
).build();
165 public Future
<CommitResponse
> rawCommit(byte[] v4Request
) {
166 throw new UnsupportedOperationException();
169 private static <T
extends Message
> Future
<T
> makeCall(final Callable
<T
> request
) {
170 return executor
.submit(new Callable
<T
>() {
172 public T
call() throws Exception
{
174 return request
.call();
175 } catch (DatastoreException e
) {
176 throw extractException(e
.getMessage(), e
.getCode());
182 private static final Pattern reasonPattern
= Pattern
.compile("\"reason\": \"(.*)\",?\\n");
183 private static final Pattern messagePattern
= Pattern
.compile("\"message\": \"(.*)\",?\\n");
186 * Convert the exception thrown by Cloud Datastore to version comparable
187 * to the Exceptions thrown by ApiProxy.
189 protected static Exception
extractException(String rawMessage
, int httpCode
) {
190 Matcher msgMatcher
= messagePattern
.matcher(rawMessage
);
191 String message
= msgMatcher
.find() ? msgMatcher
.group(1) : "[" + rawMessage
+ "]\n";
193 case HttpServletResponse
.SC_BAD_REQUEST
:
194 return new IllegalArgumentException(message
);
195 case HttpServletResponse
.SC_FORBIDDEN
:
196 Matcher reasonMatch
= reasonPattern
.matcher(rawMessage
);
197 if (reasonMatch
.find() && reasonMatch
.group(1).equals("DEADLINE_EXCEEDED")) {
198 return new DatastoreTimeoutException(message
);
200 return new IllegalStateException(message
);
202 case HttpServletResponse
.SC_PRECONDITION_FAILED
:
203 return new DatastoreNeedIndexException(message
);
204 case HttpServletResponse
.SC_CONFLICT
:
205 return new ConcurrentModificationException(message
);
206 case HttpServletResponse
.SC_SERVICE_UNAVAILABLE
:
207 return new IllegalStateException(message
);
208 case HttpServletResponse
.SC_INTERNAL_SERVER_ERROR
:
209 return new DatastoreFailureException(message
);
210 case HttpServletResponse
.SC_PAYMENT_REQUIRED
:
212 return new RuntimeException(message
);
216 private static DatastoreOptions
getDatastoreOptions()
217 throws GeneralSecurityException
, IOException
{
218 DatastoreOptions
.Builder options
= new DatastoreOptions
.Builder();
219 options
.dataset(EnvProxy
.getenv("DATASTORE_DATASET"));
220 options
.host(EnvProxy
.getenv("DATASTORE_HOST"));
222 String serviceAccount
= EnvProxy
.getenv("DATASTORE_SERVICE_ACCOUNT");
223 String privateKeyFile
= EnvProxy
.getenv("DATASTORE_PRIVATE_KEY_FILE");
224 Credential credential
;
225 if (Boolean
.valueOf(EnvProxy
.getenv("__DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST"))) {
227 } else if (serviceAccount
!= null && privateKeyFile
!= null) {
228 credential
= getServiceAccountCredential(serviceAccount
, privateKeyFile
);
230 credential
= getComputeEngineCredential();
232 options
.credential(credential
);
234 final String versionOverrideForTest
= EnvProxy
.getenv("__DATASTORE_VERSION_OVERRIDE_FOR_TEST");
235 if (versionOverrideForTest
!= null) {
236 options
.initializer(new HttpRequestInitializer() {
238 public void initialize(HttpRequest request
) throws IOException
{
239 request
.getUrl().setRawPath(
240 request
.getUrl().getRawPath().replaceFirst(
241 DatastoreFactory
.VERSION
, versionOverrideForTest
));
246 return options
.build();
249 private static Credential
getServiceAccountCredential(String account
, String privateKeyFile
)
250 throws GeneralSecurityException
, IOException
{
251 return new GoogleCredential
.Builder()
252 .setTransport(GoogleNetHttpTransport
.newTrustedTransport())
253 .setJsonFactory(new JacksonFactory())
254 .setServiceAccountId(account
)
255 .setServiceAccountScopes(DatastoreOptions
.SCOPES
)
256 .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile
))
260 private static Credential
getComputeEngineCredential()
261 throws GeneralSecurityException
, IOException
{
262 NetHttpTransport transport
= GoogleNetHttpTransport
.newTrustedTransport();
264 ComputeCredential credential
= new ComputeCredential(transport
, new JacksonFactory());
265 credential
.refreshToken();
267 } catch (IOException e
) {
273 * Make sure that the API proxy has been configured. If it's already
274 * configured (e.g. because the Remote API has been installed or the factory
275 * has already been used), do nothing. Otherwise, install a stub environment
278 private static synchronized void ensureApiProxyIsConfigured(DatastoreOptions options
) {
279 boolean hasEnvironmentOrFactory
= (ApiProxy
.getCurrentEnvironment() != null);
280 boolean hasDelegate
= (ApiProxy
.getDelegate() != null);
282 if (hasEnvironmentOrFactory
&& hasDelegate
) {
286 if (hasEnvironmentOrFactory
) {
287 throw new IllegalStateException(
288 "An ApiProxy.Environment or ApiProxy.EnvironmentFactory was already installed. "
289 + "Cannot use Cloud Datastore.");
290 } else if (hasDelegate
) {
291 throw new IllegalStateException(
292 "An ApiProxy.Delegate was already installed. Cannot use Cloud Datastore.");
295 ApiProxy
.setEnvironmentFactory(
296 new StubApiProxyEnvironmentFactory(getFullAppId(options
)));
297 ApiProxy
.setDelegate(new StubApiProxyDelegate());
301 * Attempt to determine the full app id. This is only necessary if the client
302 * did not install the Remote API (which will determine it automatically).
304 * By default, take the dataset from the provided {@link DatastoreOptions} and
305 * prepend {@code s~}. Apps for which this is incorrect (e.g. apps running in
306 * Europe) can specify the full app id via the {@code _DATASTORE_FULL_DATASET}
307 * environment variable.
309 private static String
getFullAppId(DatastoreOptions options
) {
310 String fullDataset
= EnvProxy
.getenv("_DATASTORE_FULL_DATASET");
311 if (fullDataset
!= null) {
313 } else if (options
.getHost().startsWith("http://localhost")) {
314 return options
.getDataset();
316 return "s~" + options
.getDataset();
320 * A {@link Delegate} that throws {@link UnsupportedOperationException} for
323 static class StubApiProxyDelegate
implements Delegate
<Environment
> {
324 private static final String UNSUPPORTED_API_PATTERN
=
325 "Calls to %s.%s are not supported under this configuration, only "
326 + "calls to Cloud Datastore. To use other APIs, first install the "
330 public byte[] makeSyncCall(Environment environment
, String packageName
,
331 String methodName
, byte[] request
) throws ApiProxyException
{
332 throw new UnsupportedOperationException(
333 String
.format(UNSUPPORTED_API_PATTERN
, packageName
, methodName
));
337 public Future
<byte[]> makeAsyncCall(Environment environment
, String packageName
,
338 String methodName
, byte[] request
, ApiConfig apiConfig
) {
339 throw new UnsupportedOperationException(
340 String
.format(UNSUPPORTED_API_PATTERN
, packageName
, methodName
));
344 public void log(Environment environment
, LogRecord record
) {
345 throw new UnsupportedOperationException();
349 public void flushLogs(Environment environment
) {
350 throw new UnsupportedOperationException();
354 public List
<Thread
> getRequestThreads(Environment environment
) {
355 throw new UnsupportedOperationException();
360 * An {@link EnvironmentFactory} that builds {@link StubApiProxyEnvironment}s.
362 static class StubApiProxyEnvironmentFactory
implements EnvironmentFactory
{
363 private final String appId
;
365 public StubApiProxyEnvironmentFactory(String appId
) {
370 public Environment
newEnvironment() {
371 return new StubApiProxyEnvironment(appId
);
376 * An {@link Environment} that supports the minimal subset of features needed
377 * to run code from the datastore package outside of App Engine. All other
378 * methods throw {@link UnsupportedOperationException}.
380 static class StubApiProxyEnvironment
implements Environment
{
381 private final Map
<String
, Object
> attributes
;
382 private final String appId
;
384 public StubApiProxyEnvironment(String appId
) {
385 this.attributes
= new HashMap
<>();
390 public boolean isLoggedIn() {
391 throw new UnsupportedOperationException();
395 public boolean isAdmin() {
396 throw new UnsupportedOperationException();
400 public String
getVersionId() {
401 throw new UnsupportedOperationException();
406 public String
getRequestNamespace() {
407 throw new UnsupportedOperationException();
411 public long getRemainingMillis() {
412 throw new UnsupportedOperationException();
416 public String
getModuleId() {
417 throw new UnsupportedOperationException();
421 public String
getEmail() {
422 throw new UnsupportedOperationException();
426 public String
getAuthDomain() {
427 throw new UnsupportedOperationException();
431 public Map
<String
, Object
> getAttributes() {
436 public String
getAppId() {