1 package com
.google
.appengine
.api
.datastore
;
3 import static com
.google
.common
.base
.Preconditions
.checkArgument
;
4 import static com
.google
.common
.base
.Preconditions
.checkNotNull
;
5 import static com
.google
.datastore
.v1beta3
.client
.DatastoreHelper
.LOCAL_HOST_ENV_VAR
;
6 import static com
.google
.datastore
.v1beta3
.client
.DatastoreHelper
.PRIVATE_KEY_FILE_ENV_VAR
;
7 import static com
.google
.datastore
.v1beta3
.client
.DatastoreHelper
.PROJECT_ID_ENV_VAR
;
8 import static com
.google
.datastore
.v1beta3
.client
.DatastoreHelper
.SERVICE_ACCOUNT_ENV_VAR
;
10 import com
.google
.api
.client
.auth
.oauth2
.Credential
;
11 import com
.google
.api
.client
.googleapis
.auth
.oauth2
.GoogleCredential
;
12 import com
.google
.api
.client
.googleapis
.javanet
.GoogleNetHttpTransport
;
13 import com
.google
.api
.client
.json
.jackson
.JacksonFactory
;
14 import com
.google
.appengine
.api
.datastore
.DatastoreServiceConfig
.ApiVersion
;
15 import com
.google
.appengine
.api
.taskqueue
.TaskQueuePb
.TaskQueueAddRequest
;
16 import com
.google
.appengine
.api
.taskqueue
.TaskQueuePb
.TaskQueueBulkAddRequest
;
17 import com
.google
.apphosting
.api
.ApiProxy
;
18 import com
.google
.apphosting
.api
.ApiProxy
.ApiConfig
;
19 import com
.google
.apphosting
.api
.ApiProxy
.ApiProxyException
;
20 import com
.google
.apphosting
.api
.ApiProxy
.Delegate
;
21 import com
.google
.apphosting
.api
.ApiProxy
.Environment
;
22 import com
.google
.apphosting
.api
.ApiProxy
.EnvironmentFactory
;
23 import com
.google
.apphosting
.api
.ApiProxy
.LogRecord
;
24 import com
.google
.datastore
.v1beta3
.AllocateIdsRequest
;
25 import com
.google
.datastore
.v1beta3
.AllocateIdsResponse
;
26 import com
.google
.datastore
.v1beta3
.BeginTransactionRequest
;
27 import com
.google
.datastore
.v1beta3
.BeginTransactionResponse
;
28 import com
.google
.datastore
.v1beta3
.CommitRequest
;
29 import com
.google
.datastore
.v1beta3
.CommitResponse
;
30 import com
.google
.datastore
.v1beta3
.Datastore
.Method
;
31 import com
.google
.datastore
.v1beta3
.LookupRequest
;
32 import com
.google
.datastore
.v1beta3
.LookupResponse
;
33 import com
.google
.datastore
.v1beta3
.RollbackRequest
;
34 import com
.google
.datastore
.v1beta3
.RollbackResponse
;
35 import com
.google
.datastore
.v1beta3
.RunQueryRequest
;
36 import com
.google
.datastore
.v1beta3
.RunQueryResponse
;
37 import com
.google
.datastore
.v1beta3
.client
.Datastore
;
38 import com
.google
.datastore
.v1beta3
.client
.DatastoreException
;
39 import com
.google
.datastore
.v1beta3
.client
.DatastoreFactory
;
40 import com
.google
.datastore
.v1beta3
.client
.DatastoreHelper
;
41 import com
.google
.datastore
.v1beta3
.client
.DatastoreOptions
;
42 import com
.google
.protobuf
.InvalidProtocolBufferException
;
43 import com
.google
.protobuf
.Message
;
46 import java
.io
.IOException
;
47 import java
.security
.GeneralSecurityException
;
48 import java
.util
.HashMap
;
49 import java
.util
.List
;
51 import java
.util
.concurrent
.Callable
;
52 import java
.util
.concurrent
.ExecutorService
;
53 import java
.util
.concurrent
.Executors
;
54 import java
.util
.concurrent
.Future
;
55 import java
.util
.logging
.Level
;
56 import java
.util
.logging
.Logger
;
59 * {@link CloudDatastoreV1Proxy} that makes remote calls (currently over HTTP).
61 * <p>Methods in this class do not populate the project id field in outgoing requests since it
62 * is not required when using the API over HTTP.
64 * <p>This class is used if (and only if) the {@code DATASTORE_USE_CLOUD_DATASTORE},
65 * {@code DATASTORE_PROJECT_ID}, or {@code DATASTORE_APP_ID} environment variable is set.
67 * <p>This class is designed to run outside of App Engine in an environment where the app id is
68 * potentially unknown. The user has several ways to specify it:
70 * <li>Install the Remote API. The Remote API can retrieve the app id by making a call to the
72 * <li>Specify the {@code DATASTORE_APP_ID} environment variable. If the project id has also been
73 * specified, then the value of {@code DATASTORE_APP_ID} is required to match.
74 * <li>Explicitly opt to use the project id instead by setting
75 * {@code DATASTORE_USE_PROJECT_ID_AS_APP_ID=true}. This changes the serialized form of entities,
76 * so integration with services such as memcache will not work correctly.
79 * <p>{@code DATASTORE_APP_ID} and {@code DATASTORE_USE_PROJECT_ID_AS_APP_ID} cannot both be
82 * <p>Separately, {@code DATASTORE_ADDITIONAL_APP_IDS} can be set to a comma-separated list of
83 * app ids in order to support foreign keys.
85 final class RemoteCloudDatastoreV1Proxy
implements CloudDatastoreV1Proxy
{
87 private static final Logger logger
=
88 Logger
.getLogger(RemoteCloudDatastoreV1Proxy
.class.getName());
90 private static final String URL_OVERRIDE_ENV_VAR
= "__DATASTORE_URL_OVERRIDE";
92 static final String ADDITIONAL_APP_IDS_VAR
= "DATASTORE_ADDITIONAL_APP_IDS";
93 private static final String USE_PROJECT_ID_AS_APP_ID_VAR
=
94 "DATASTORE_USE_PROJECT_ID_AS_APP_ID";
95 static final String APP_ID_VAR
= "DATASTORE_APP_ID";
96 private static final String ALLOW_TRANSACTIONAL_TASKS
= "__DATASTORE_ALLOW_TRANSACTIONAL_TASKS";
98 private static final ExecutorService executor
= Executors
.newCachedThreadPool();
100 private final Datastore datastore
;
102 RemoteCloudDatastoreV1Proxy(Datastore datastore
) {
103 this.datastore
= checkNotNull(datastore
);
107 * Creates a {@link RemoteCloudDatastoreV1Proxy}. This method has the side effect
108 * of installing minimal stubs ({@link EnvironmentFactory} and
109 * {@link Delegate}) in the API proxy if they have not already been installed.
111 static RemoteCloudDatastoreV1Proxy
create(DatastoreServiceConfig config
) {
112 checkArgument(config
.getApiVersion() == ApiVersion
.CLOUD_DATASTORE_V1_REMOTE
);
113 String projectId
= getProjectId();
114 DatastoreOptions options
;
116 options
= getDatastoreOptions(projectId
);
117 } catch (GeneralSecurityException
| IOException e
) {
118 throw new RuntimeException(
119 "Could not get Cloud Datastore options from environment.", e
);
121 ensureApiProxyIsConfigured(projectId
);
122 populateAdditionalAppIdsMap();
123 return new RemoteCloudDatastoreV1Proxy(DatastoreFactory
.get().create(options
));
126 private static void populateAdditionalAppIdsMap() {
127 String appIdsVar
= EnvProxy
.getenv(ADDITIONAL_APP_IDS_VAR
);
128 if (appIdsVar
== null) {
131 String
[] appIds
= appIdsVar
.split(",");
132 Map
<String
, String
> projectIdToAppId
= new HashMap
<>();
133 for (String appId
: appIds
) {
134 appId
= appId
.trim();
135 if (!appId
.isEmpty()) {
136 projectIdToAppId
.put(DatastoreApiHelper
.toProjectId(appId
), appId
);
139 ApiProxy
.getCurrentEnvironment().getAttributes()
140 .put(DataTypeTranslator
.ADDITIONAL_APP_IDS_MAP_ATTRIBUTE_KEY
, projectIdToAppId
);
144 public Future
<BeginTransactionResponse
> beginTransaction(final BeginTransactionRequest req
) {
145 return makeCall(new Callable
<BeginTransactionResponse
>() {
147 public BeginTransactionResponse
call() throws DatastoreException
{
148 return datastore
.beginTransaction(req
);
150 }, Method
.BeginTransaction
);
154 public Future
<RollbackResponse
> rollback(final RollbackRequest req
) {
155 return makeCall(new Callable
<RollbackResponse
>() {
157 public RollbackResponse
call() throws DatastoreException
{
158 return datastore
.rollback(req
);
164 public Future
<RunQueryResponse
> runQuery(final RunQueryRequest req
) {
165 return makeCall(new Callable
<RunQueryResponse
>() {
167 public RunQueryResponse
call() throws DatastoreException
{
168 return datastore
.runQuery(req
);
174 public Future
<LookupResponse
> lookup(final LookupRequest req
) {
175 return makeCall(new Callable
<LookupResponse
>() {
177 public LookupResponse
call() throws DatastoreException
{
178 return datastore
.lookup(req
);
184 public Future
<AllocateIdsResponse
> allocateIds(final AllocateIdsRequest req
) {
185 return makeCall(new Callable
<AllocateIdsResponse
>() {
187 public AllocateIdsResponse
call() throws DatastoreException
{
188 return datastore
.allocateIds(req
);
190 }, Method
.AllocateIds
);
194 public Future
<CommitResponse
> commit(final CommitRequest req
) {
195 return makeCall(new Callable
<CommitResponse
>() {
197 public CommitResponse
call() throws DatastoreException
{
198 return datastore
.commit(req
);
204 public Future
<CommitResponse
> rawCommit(byte[] bytes
) {
206 return commit(CommitRequest
.parseFrom(bytes
));
207 } catch (InvalidProtocolBufferException e
) {
208 throw new IllegalStateException(e
);
212 private static <T
extends Message
> Future
<T
> makeCall(final Callable
<T
> request
,
213 final Method method
) {
214 return executor
.submit(new Callable
<T
>() {
216 public T
call() throws Exception
{
218 return request
.call();
219 } catch (DatastoreException e
) {
220 throw DatastoreApiHelper
.createV1Exception(e
.getCode(), e
.getMessage(), method
);
226 private static DatastoreOptions
getDatastoreOptions(String projectId
)
227 throws GeneralSecurityException
, IOException
{
228 DatastoreOptions
.Builder options
= new DatastoreOptions
.Builder();
229 setProjectEndpoint(projectId
, options
);
230 options
.credential(getCredential());
231 return options
.build();
234 private static Credential
getCredential() throws GeneralSecurityException
, IOException
{
235 if (Boolean
.valueOf(EnvProxy
.getenv("__DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST"))) {
237 } else if (EnvProxy
.getenv(LOCAL_HOST_ENV_VAR
) != null) {
238 logger
.log(Level
.INFO
, "{0} environment variable was set. Not using credentials.",
239 new Object
[] {LOCAL_HOST_ENV_VAR
});
242 String serviceAccount
= EnvProxy
.getenv(SERVICE_ACCOUNT_ENV_VAR
);
243 String privateKeyFile
= EnvProxy
.getenv(PRIVATE_KEY_FILE_ENV_VAR
);
244 if (serviceAccount
!= null && privateKeyFile
!= null) {
245 logger
.log(Level
.INFO
, "{0} and {1} environment variables were set. "
246 + "Using service account credential.",
247 new Object
[] {SERVICE_ACCOUNT_ENV_VAR
, PRIVATE_KEY_FILE_ENV_VAR
});
248 return getServiceAccountCredential(serviceAccount
, privateKeyFile
);
250 return GoogleCredential
.getApplicationDefault()
251 .createScoped(DatastoreOptions
.SCOPES
);
254 private static void setProjectEndpoint(String projectId
, DatastoreOptions
.Builder options
) {
255 if (EnvProxy
.getenv("DATASTORE_HOST") != null) {
256 logger
.warning(String
.format(
257 "Ignoring value of environment variable DATASTORE_HOST. "
258 + "To point datastore to a host running locally, use "
259 + "the environment variable %s.",
260 LOCAL_HOST_ENV_VAR
));
262 if (EnvProxy
.getenv(URL_OVERRIDE_ENV_VAR
) != null) {
263 options
.projectEndpoint(String
.format("%s/projects/%s",
264 EnvProxy
.getenv(URL_OVERRIDE_ENV_VAR
), projectId
));
267 if (EnvProxy
.getenv(LOCAL_HOST_ENV_VAR
) != null) {
268 options
.projectId(projectId
);
269 options
.localHost(EnvProxy
.getenv(LOCAL_HOST_ENV_VAR
));
272 options
.projectId(projectId
);
276 private static String
getProjectId() {
277 String projectIdFromEnv
= EnvProxy
.getenv(PROJECT_ID_ENV_VAR
);
278 if (projectIdFromEnv
!= null) {
279 return projectIdFromEnv
;
281 String appIdFromEnv
= EnvProxy
.getenv(APP_ID_VAR
);
282 if (appIdFromEnv
!= null) {
283 return DatastoreApiHelper
.toProjectId(appIdFromEnv
);
285 String projectIdFromComputeEngine
= DatastoreHelper
.getProjectIdFromComputeEngine();
286 if (projectIdFromComputeEngine
!= null) {
287 return projectIdFromComputeEngine
;
289 throw new IllegalStateException(String
.format("Could not determine project ID."
290 + " If you are not running on Compute Engine, set the "
291 + " %s environment variable.", PROJECT_ID_ENV_VAR
));
294 private static Credential
getServiceAccountCredential(String account
, String privateKeyFile
)
295 throws GeneralSecurityException
, IOException
{
296 return new GoogleCredential
.Builder()
297 .setTransport(GoogleNetHttpTransport
.newTrustedTransport())
298 .setJsonFactory(new JacksonFactory())
299 .setServiceAccountId(account
)
300 .setServiceAccountScopes(DatastoreOptions
.SCOPES
)
301 .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile
))
306 * Make sure that the API proxy has been configured. If it's already
307 * configured (e.g. because the Remote API has been installed or the factory
308 * has already been used), do nothing. Otherwise, install a stub environment
311 private static synchronized void ensureApiProxyIsConfigured(String projectId
) {
312 boolean hasEnvironmentOrFactory
= (ApiProxy
.getCurrentEnvironment() != null);
313 boolean hasDelegate
= (ApiProxy
.getDelegate() != null);
315 if (hasEnvironmentOrFactory
&& hasDelegate
) {
316 boolean allowTransactionalTasks
=
317 Boolean
.valueOf(EnvProxy
.getenv(ALLOW_TRANSACTIONAL_TASKS
));
318 if (!allowTransactionalTasks
319 && !(ApiProxy
.getDelegate() instanceof TransactionalTaskDisallowingApiProxyDelegate
)) {
320 @SuppressWarnings("unchecked")
321 Delegate
<Environment
> originalDelegate
= ApiProxy
.getDelegate();
322 ApiProxy
.setDelegate(new TransactionalTaskDisallowingApiProxyDelegate(originalDelegate
));
327 if (hasEnvironmentOrFactory
) {
328 throw new IllegalStateException(
329 "An ApiProxy.Environment or ApiProxy.EnvironmentFactory was already installed. "
330 + "Cannot use Cloud Datastore.");
331 } else if (hasDelegate
) {
332 throw new IllegalStateException(
333 "An ApiProxy.Delegate was already installed. Cannot use Cloud Datastore.");
336 String appId
= EnvProxy
.getenv(APP_ID_VAR
);
337 boolean useProjectIdAsAppId
=
338 Boolean
.valueOf(EnvProxy
.getenv(USE_PROJECT_ID_AS_APP_ID_VAR
));
340 if (appId
== null && !useProjectIdAsAppId
) {
341 throw new IllegalStateException(String
.format(
342 "Could not not determine app id. To use project id (%s) instead, set "
343 + "%s=true. This will affect the serialized form "
344 + "of entities and should not be used if serialized entities will be shared between "
345 + "code running on App Engine and code running off App Engine. Alternatively, set "
347 projectId
, USE_PROJECT_ID_AS_APP_ID_VAR
, APP_ID_VAR
));
348 } else if (appId
!= null) {
349 if (useProjectIdAsAppId
) {
350 throw new IllegalStateException(String
.format(
351 "App id was provided (%s) but %s was set to true. "
352 + "Please unset either %s or %s.",
353 appId
, USE_PROJECT_ID_AS_APP_ID_VAR
, APP_ID_VAR
, USE_PROJECT_ID_AS_APP_ID_VAR
));
354 } else if (!DatastoreApiHelper
.toProjectId(appId
).equals(projectId
)) {
355 throw new IllegalStateException(String
.format(
356 "App id \"%s\" does not match project id \"%s\".",
361 ApiProxy
.setEnvironmentFactory(new StubApiProxyEnvironmentFactory(
362 useProjectIdAsAppId ? projectId
: appId
));
363 ApiProxy
.setDelegate(new StubApiProxyDelegate());
367 * A {@link Delegate} that disallows transactional tasks. Requests not
368 * containing a transactional task are delegated to another {@link Delegate}.
370 static class TransactionalTaskDisallowingApiProxyDelegate
implements Delegate
<Environment
> {
371 static final String NOT_SUPPORTED_MESSAGE
=
372 "Transactional tasks are not supported under this configuration.";
373 private final Delegate
<Environment
> delegate
;
375 public TransactionalTaskDisallowingApiProxyDelegate(Delegate
<Environment
> delegate
) {
376 this.delegate
= delegate
;
380 public byte[] makeSyncCall(Environment environment
, String packageName
,
381 String methodName
, byte[] request
) throws ApiProxyException
{
382 checkAllowed(packageName
, methodName
, request
);
383 return delegate
.makeSyncCall(environment
, packageName
, methodName
, request
);
387 public Future
<byte[]> makeAsyncCall(Environment environment
, String packageName
,
388 String methodName
, byte[] request
, ApiConfig apiConfig
) {
389 checkAllowed(packageName
, methodName
, request
);
390 return delegate
.makeAsyncCall(environment
, packageName
, methodName
, request
, apiConfig
);
394 public void log(Environment environment
, LogRecord record
) {
395 delegate
.log(environment
, record
);
399 public void flushLogs(Environment environment
) {
400 delegate
.flushLogs(environment
);
404 public List
<Thread
> getRequestThreads(Environment environment
) {
405 return delegate
.getRequestThreads(environment
);
408 private static void checkAllowed(String packageName
, String methodName
,
409 byte[] requestBytes
) {
410 if (!"taskqueue".equals(packageName
)) {
415 if ("Add".equals(methodName
)) {
416 checkNonTransactional(TaskQueueAddRequest
.parser().parseFrom(requestBytes
));
417 } else if ("BulkAdd".equals(methodName
)) {
418 TaskQueueBulkAddRequest req
= TaskQueueBulkAddRequest
.parser().parseFrom(requestBytes
);
419 for (TaskQueueAddRequest subReq
: req
.addRequests()) {
420 checkNonTransactional(subReq
);
423 } catch (InvalidProtocolBufferException e
) {
424 throw new IllegalArgumentException(e
);
428 private static void checkNonTransactional(TaskQueueAddRequest req
) {
429 if (req
.hasTransaction() || req
.hasDatastoreTransaction()) {
430 throw new UnsupportedOperationException(NOT_SUPPORTED_MESSAGE
);
436 * A {@link Delegate} that throws {@link UnsupportedOperationException} for
439 static class StubApiProxyDelegate
implements Delegate
<Environment
> {
440 private static final String UNSUPPORTED_API_PATTERN
=
441 "Calls to %s.%s are not supported under this configuration, only "
442 + "calls to Cloud Datastore. To use other APIs, first install the "
446 public byte[] makeSyncCall(Environment environment
, String packageName
,
447 String methodName
, byte[] request
) throws ApiProxyException
{
448 throw new UnsupportedOperationException(
449 String
.format(UNSUPPORTED_API_PATTERN
, packageName
, methodName
));
453 public Future
<byte[]> makeAsyncCall(Environment environment
, String packageName
,
454 String methodName
, byte[] request
, ApiConfig apiConfig
) {
455 throw new UnsupportedOperationException(
456 String
.format(UNSUPPORTED_API_PATTERN
, packageName
, methodName
));
460 public void log(Environment environment
, LogRecord record
) {
461 throw new UnsupportedOperationException();
465 public void flushLogs(Environment environment
) {
466 throw new UnsupportedOperationException();
470 public List
<Thread
> getRequestThreads(Environment environment
) {
471 throw new UnsupportedOperationException();
476 * An {@link EnvironmentFactory} that builds {@link StubApiProxyEnvironment}s.
478 static class StubApiProxyEnvironmentFactory
implements EnvironmentFactory
{
479 private final String appId
;
481 public StubApiProxyEnvironmentFactory(String appId
) {
486 public Environment
newEnvironment() {
487 return new StubApiProxyEnvironment(appId
);
492 * An {@link Environment} that supports the minimal subset of features needed
493 * to run code from the datastore package outside of App Engine. All other
494 * methods throw {@link UnsupportedOperationException}.
496 static class StubApiProxyEnvironment
implements Environment
{
497 private final Map
<String
, Object
> attributes
;
498 private final String appId
;
500 public StubApiProxyEnvironment(String appId
) {
501 this.attributes
= new HashMap
<>();
506 public boolean isLoggedIn() {
507 throw new UnsupportedOperationException();
511 public boolean isAdmin() {
512 throw new UnsupportedOperationException();
516 public String
getVersionId() {
517 throw new UnsupportedOperationException();
522 public String
getRequestNamespace() {
523 throw new UnsupportedOperationException();
527 public long getRemainingMillis() {
528 throw new UnsupportedOperationException();
532 public String
getModuleId() {
533 throw new UnsupportedOperationException();
537 public String
getEmail() {
538 throw new UnsupportedOperationException();
542 public String
getAuthDomain() {
543 throw new UnsupportedOperationException();
547 public Map
<String
, Object
> getAttributes() {
552 public String
getAppId() {