1 package com
.google
.appengine
.api
.datastore
;
3 import static com
.google
.common
.base
.Preconditions
.checkArgument
;
4 import static com
.google
.common
.base
.Preconditions
.checkNotNull
;
6 import com
.google
.api
.client
.auth
.oauth2
.Credential
;
7 import com
.google
.api
.client
.googleapis
.auth
.oauth2
.GoogleCredential
;
8 import com
.google
.api
.client
.googleapis
.compute
.ComputeCredential
;
9 import com
.google
.api
.client
.googleapis
.javanet
.GoogleNetHttpTransport
;
10 import com
.google
.api
.client
.http
.GenericUrl
;
11 import com
.google
.api
.client
.http
.HttpRequest
;
12 import com
.google
.api
.client
.http
.HttpRequestInitializer
;
13 import com
.google
.api
.client
.http
.HttpTransport
;
14 import com
.google
.api
.client
.http
.javanet
.NetHttpTransport
;
15 import com
.google
.api
.client
.json
.jackson
.JacksonFactory
;
16 import com
.google
.appengine
.api
.datastore
.DatastoreServiceConfig
.ApiVersion
;
17 import com
.google
.apphosting
.api
.ApiProxy
;
18 import com
.google
.apphosting
.api
.ApiProxy
.ApiConfig
;
19 import com
.google
.apphosting
.api
.ApiProxy
.ApiProxyException
;
20 import com
.google
.apphosting
.api
.ApiProxy
.Delegate
;
21 import com
.google
.apphosting
.api
.ApiProxy
.Environment
;
22 import com
.google
.apphosting
.api
.ApiProxy
.EnvironmentFactory
;
23 import com
.google
.apphosting
.api
.ApiProxy
.LogRecord
;
24 import com
.google
.datastore
.v1beta3
.AllocateIdsRequest
;
25 import com
.google
.datastore
.v1beta3
.AllocateIdsResponse
;
26 import com
.google
.datastore
.v1beta3
.BeginTransactionRequest
;
27 import com
.google
.datastore
.v1beta3
.BeginTransactionResponse
;
28 import com
.google
.datastore
.v1beta3
.CommitRequest
;
29 import com
.google
.datastore
.v1beta3
.CommitResponse
;
30 import com
.google
.datastore
.v1beta3
.LookupRequest
;
31 import com
.google
.datastore
.v1beta3
.LookupResponse
;
32 import com
.google
.datastore
.v1beta3
.RollbackRequest
;
33 import com
.google
.datastore
.v1beta3
.RollbackResponse
;
34 import com
.google
.datastore
.v1beta3
.RunQueryRequest
;
35 import com
.google
.datastore
.v1beta3
.RunQueryResponse
;
36 import com
.google
.datastore
.v1beta3
.client
.Datastore
;
37 import com
.google
.datastore
.v1beta3
.client
.DatastoreException
;
38 import com
.google
.datastore
.v1beta3
.client
.DatastoreFactory
;
39 import com
.google
.datastore
.v1beta3
.client
.DatastoreOptions
;
40 import com
.google
.protobuf
.InvalidProtocolBufferException
;
41 import com
.google
.protobuf
.Message
;
44 import java
.io
.IOException
;
45 import java
.security
.GeneralSecurityException
;
46 import java
.util
.HashMap
;
47 import java
.util
.List
;
49 import java
.util
.concurrent
.Callable
;
50 import java
.util
.concurrent
.ExecutorService
;
51 import java
.util
.concurrent
.Executors
;
52 import java
.util
.concurrent
.Future
;
53 import java
.util
.logging
.Level
;
54 import java
.util
.logging
.Logger
;
57 * {@link CloudDatastoreV1Proxy} that makes remote calls (currently over HTTP).
59 * <p>Methods in this class do not populate the project id field in outgoing requests since it
60 * is not required when using the API over HTTP.
62 * <p>This class is used if (and only if) the {@code DATASTORE_USE_CLOUD_DATASTORE} environment
65 * <p>This class is designed to run outside of App Engine in an environment where the app id is
66 * potentially unknown. The user has several ways to specify it:
68 * <li>Install the Remote API. The Remote API can retrieve the app id by making a call to the
70 * <li>Specify the {@code DATASTORE_APP_ID} environment variable. If the project id has also been
71 * specified, then the value of {@code DATASTORE_APP_ID} is required to match.
72 * <li>Explicitly opt to use the project id instead by setting
73 * {@code DATASTORE_USE_PROJECT_ID_AS_APP_ID=true}. This changes the serialized form of entities,
74 * so integration with services such as memcache will not work correctly.
77 * <p>{@code DATASTORE_APP_ID} and {@code DATASTORE_USE_PROJECT_ID_AS_APP_ID} cannot both be
80 * <p>Separately, {@code DATASTORE_ADDITIONAL_APP_IDS} can be set to a comma-separated list of
81 * app ids in order to support foreign keys.
83 final class RemoteCloudDatastoreV1Proxy
implements CloudDatastoreV1Proxy
{
85 private static final Logger logger
=
86 Logger
.getLogger(RemoteCloudDatastoreV1Proxy
.class.getName());
88 static final String ADDITIONAL_APP_IDS_VAR
= "DATASTORE_ADDITIONAL_APP_IDS";
89 private static final String USE_PROJECT_ID_AS_APP_ID_VAR
=
90 "DATASTORE_USE_PROJECT_ID_AS_APP_ID";
91 private static final String APP_ID_VAR
= "DATASTORE_APP_ID";
93 private static final ExecutorService executor
= Executors
.newCachedThreadPool();
95 private final Datastore datastore
;
97 RemoteCloudDatastoreV1Proxy(Datastore datastore
) {
98 this.datastore
= checkNotNull(datastore
);
102 * Creates a {@link RemoteCloudDatastoreV1Proxy}. This method has the side effect
103 * of installing minimal stubs ({@link EnvironmentFactory} and
104 * {@link Delegate}) in the API proxy if they have not already been installed.
106 static RemoteCloudDatastoreV1Proxy
create(DatastoreServiceConfig config
) {
107 checkArgument(config
.getApiVersion() == ApiVersion
.CLOUD_DATASTORE_V1_REMOTE
);
108 DatastoreOptions options
;
110 options
= getDatastoreOptions();
111 } catch (GeneralSecurityException
| IOException e
) {
112 throw new RuntimeException(
113 "Could not get Cloud Datastore options from environment.", e
);
115 ensureApiProxyIsConfigured(options
);
116 populateAdditionalAppIdsMap();
117 return new RemoteCloudDatastoreV1Proxy(DatastoreFactory
.get().create(options
));
120 private static void populateAdditionalAppIdsMap() {
121 String appIdsVar
= EnvProxy
.getenv(ADDITIONAL_APP_IDS_VAR
);
122 if (appIdsVar
== null) {
125 String
[] appIds
= appIdsVar
.split(",");
126 Map
<String
, String
> projectIdToAppId
= new HashMap
<>();
127 for (String appId
: appIds
) {
128 appId
= appId
.trim();
129 if (!appId
.isEmpty()) {
130 projectIdToAppId
.put(DatastoreApiHelper
.toProjectId(appId
), appId
);
133 ApiProxy
.getCurrentEnvironment().getAttributes()
134 .put(DataTypeTranslator
.ADDITIONAL_APP_IDS_MAP_ATTRIBUTE_KEY
, projectIdToAppId
);
138 public Future
<BeginTransactionResponse
> beginTransaction(final BeginTransactionRequest req
) {
139 return makeCall(new Callable
<BeginTransactionResponse
>() {
141 public BeginTransactionResponse
call() throws DatastoreException
{
142 return datastore
.beginTransaction(req
);
148 public Future
<RollbackResponse
> rollback(final RollbackRequest req
) {
149 return makeCall(new Callable
<RollbackResponse
>() {
151 public RollbackResponse
call() throws DatastoreException
{
152 return datastore
.rollback(req
);
158 public Future
<RunQueryResponse
> runQuery(final RunQueryRequest req
) {
159 return makeCall(new Callable
<RunQueryResponse
>() {
161 public RunQueryResponse
call() throws DatastoreException
{
162 return datastore
.runQuery(req
);
168 public Future
<LookupResponse
> lookup(final LookupRequest req
) {
169 return makeCall(new Callable
<LookupResponse
>() {
171 public LookupResponse
call() throws DatastoreException
{
172 return datastore
.lookup(req
);
178 public Future
<AllocateIdsResponse
> allocateIds(final AllocateIdsRequest req
) {
179 return makeCall(new Callable
<AllocateIdsResponse
>() {
181 public AllocateIdsResponse
call() throws DatastoreException
{
182 return datastore
.allocateIds(req
);
188 public Future
<CommitResponse
> commit(final CommitRequest req
) {
189 return makeCall(new Callable
<CommitResponse
>() {
191 public CommitResponse
call() throws DatastoreException
{
192 return datastore
.commit(req
);
198 public Future
<CommitResponse
> rawCommit(byte[] bytes
) {
200 return commit(CommitRequest
.parseFrom(bytes
));
201 } catch (InvalidProtocolBufferException e
) {
202 throw new IllegalStateException(e
);
206 private static <T
extends Message
> Future
<T
> makeCall(final Callable
<T
> request
) {
207 return executor
.submit(new Callable
<T
>() {
209 public T
call() throws Exception
{
211 return request
.call();
212 } catch (DatastoreException e
) {
213 throw DatastoreApiHelper
.createException(e
.getCode(), e
.getMessage());
219 private static DatastoreOptions
getDatastoreOptions()
220 throws GeneralSecurityException
, IOException
{
221 DatastoreOptions
.Builder options
= new DatastoreOptions
.Builder();
222 options
.projectId(getProjectId());
223 options
.host(EnvProxy
.getenv("DATASTORE_HOST"));
225 String serviceAccount
= EnvProxy
.getenv("DATASTORE_SERVICE_ACCOUNT");
226 String privateKeyFile
= EnvProxy
.getenv("DATASTORE_PRIVATE_KEY_FILE");
227 Credential credential
;
228 if (Boolean
.valueOf(EnvProxy
.getenv("__DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST"))) {
230 } else if (serviceAccount
!= null && privateKeyFile
!= null) {
231 credential
= getServiceAccountCredential(serviceAccount
, privateKeyFile
);
233 credential
= getComputeEngineCredential();
235 options
.credential(credential
);
237 final String versionOverrideForTest
= EnvProxy
.getenv("__DATASTORE_VERSION_OVERRIDE_FOR_TEST");
238 if (versionOverrideForTest
!= null) {
239 options
.initializer(new HttpRequestInitializer() {
241 public void initialize(HttpRequest request
) throws IOException
{
242 request
.getUrl().setRawPath(
243 request
.getUrl().getRawPath().replaceFirst(
244 DatastoreFactory
.VERSION
, versionOverrideForTest
));
249 return options
.build();
252 private static String
getProjectId() {
253 String projectIdFromEnv
= EnvProxy
.getenv("DATASTORE_PROJECT_ID");
254 if (projectIdFromEnv
!= null) {
255 return projectIdFromEnv
;
257 String appIdFromEnv
= EnvProxy
.getenv(APP_ID_VAR
);
258 if (appIdFromEnv
!= null) {
259 return DatastoreApiHelper
.toProjectId(appIdFromEnv
);
262 HttpTransport transport
= GoogleNetHttpTransport
.newTrustedTransport();
263 GenericUrl projectIdUrl
=
264 new GenericUrl("http://metadata/computeMetadata/v1/project/project-id");
265 HttpRequest request
= transport
.createRequestFactory().buildGetRequest(projectIdUrl
);
266 request
.getHeaders().set("Metadata-Flavor", "Google");
267 return request
.execute().parseAsString();
268 } catch (GeneralSecurityException
| IOException e
) {
269 logger
.log(Level
.INFO
, "Could not retrieve project id from GCE metadata server", e
);
274 private static Credential
getServiceAccountCredential(String account
, String privateKeyFile
)
275 throws GeneralSecurityException
, IOException
{
276 return new GoogleCredential
.Builder()
277 .setTransport(GoogleNetHttpTransport
.newTrustedTransport())
278 .setJsonFactory(new JacksonFactory())
279 .setServiceAccountId(account
)
280 .setServiceAccountScopes(DatastoreOptions
.SCOPES
)
281 .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile
))
285 private static Credential
getComputeEngineCredential()
286 throws GeneralSecurityException
, IOException
{
287 NetHttpTransport transport
= GoogleNetHttpTransport
.newTrustedTransport();
289 ComputeCredential credential
= new ComputeCredential(transport
, new JacksonFactory());
290 credential
.refreshToken();
292 } catch (IOException e
) {
298 * Make sure that the API proxy has been configured. If it's already
299 * configured (e.g. because the Remote API has been installed or the factory
300 * has already been used), do nothing. Otherwise, install a stub environment
303 private static synchronized void ensureApiProxyIsConfigured(DatastoreOptions options
) {
304 boolean hasEnvironmentOrFactory
= (ApiProxy
.getCurrentEnvironment() != null);
305 boolean hasDelegate
= (ApiProxy
.getDelegate() != null);
307 if (hasEnvironmentOrFactory
&& hasDelegate
) {
311 if (hasEnvironmentOrFactory
) {
312 throw new IllegalStateException(
313 "An ApiProxy.Environment or ApiProxy.EnvironmentFactory was already installed. "
314 + "Cannot use Cloud Datastore.");
315 } else if (hasDelegate
) {
316 throw new IllegalStateException(
317 "An ApiProxy.Delegate was already installed. Cannot use Cloud Datastore.");
320 String appId
= EnvProxy
.getenv("DATASTORE_APP_ID");
321 boolean useProjectIdAsAppId
=
322 Boolean
.valueOf(EnvProxy
.getenv(USE_PROJECT_ID_AS_APP_ID_VAR
));
324 if (appId
== null && !useProjectIdAsAppId
) {
325 throw new IllegalStateException(String
.format(
326 "Could not not determine app id. To use project id (%s) instead, set "
327 + "%s=true. This will affect the serialized form "
328 + "of entities and should not be used if serialized entities will be shared between "
329 + "code running on App Engine and code running off App Engine. Alternatively, set "
331 options
.getProjectId(), USE_PROJECT_ID_AS_APP_ID_VAR
, APP_ID_VAR
));
332 } else if (appId
!= null) {
333 if (useProjectIdAsAppId
) {
334 throw new IllegalStateException(String
.format(
335 "App id was provided (%s) but %s was set to true. "
336 + "Please unset either %s or %s.",
337 appId
, USE_PROJECT_ID_AS_APP_ID_VAR
, APP_ID_VAR
, USE_PROJECT_ID_AS_APP_ID_VAR
));
338 } else if (!DatastoreApiHelper
.toProjectId(appId
).equals(options
.getProjectId())) {
339 throw new IllegalStateException(String
.format(
340 "App id \"%s\" does not match project id \"%s\".",
341 appId
, options
.getProjectId()));
345 ApiProxy
.setEnvironmentFactory(new StubApiProxyEnvironmentFactory(
346 useProjectIdAsAppId ? options
.getProjectId() : appId
));
347 ApiProxy
.setDelegate(new StubApiProxyDelegate());
351 * A {@link Delegate} that throws {@link UnsupportedOperationException} for
354 static class StubApiProxyDelegate
implements Delegate
<Environment
> {
355 private static final String UNSUPPORTED_API_PATTERN
=
356 "Calls to %s.%s are not supported under this configuration, only "
357 + "calls to Cloud Datastore. To use other APIs, first install the "
361 public byte[] makeSyncCall(Environment environment
, String packageName
,
362 String methodName
, byte[] request
) throws ApiProxyException
{
363 throw new UnsupportedOperationException(
364 String
.format(UNSUPPORTED_API_PATTERN
, packageName
, methodName
));
368 public Future
<byte[]> makeAsyncCall(Environment environment
, String packageName
,
369 String methodName
, byte[] request
, ApiConfig apiConfig
) {
370 throw new UnsupportedOperationException(
371 String
.format(UNSUPPORTED_API_PATTERN
, packageName
, methodName
));
375 public void log(Environment environment
, LogRecord record
) {
376 throw new UnsupportedOperationException();
380 public void flushLogs(Environment environment
) {
381 throw new UnsupportedOperationException();
385 public List
<Thread
> getRequestThreads(Environment environment
) {
386 throw new UnsupportedOperationException();
391 * An {@link EnvironmentFactory} that builds {@link StubApiProxyEnvironment}s.
393 static class StubApiProxyEnvironmentFactory
implements EnvironmentFactory
{
394 private final String appId
;
396 public StubApiProxyEnvironmentFactory(String appId
) {
401 public Environment
newEnvironment() {
402 return new StubApiProxyEnvironment(appId
);
407 * An {@link Environment} that supports the minimal subset of features needed
408 * to run code from the datastore package outside of App Engine. All other
409 * methods throw {@link UnsupportedOperationException}.
411 static class StubApiProxyEnvironment
implements Environment
{
412 private final Map
<String
, Object
> attributes
;
413 private final String appId
;
415 public StubApiProxyEnvironment(String appId
) {
416 this.attributes
= new HashMap
<>();
421 public boolean isLoggedIn() {
422 throw new UnsupportedOperationException();
426 public boolean isAdmin() {
427 throw new UnsupportedOperationException();
431 public String
getVersionId() {
432 throw new UnsupportedOperationException();
437 public String
getRequestNamespace() {
438 throw new UnsupportedOperationException();
442 public long getRemainingMillis() {
443 throw new UnsupportedOperationException();
447 public String
getModuleId() {
448 throw new UnsupportedOperationException();
452 public String
getEmail() {
453 throw new UnsupportedOperationException();
457 public String
getAuthDomain() {
458 throw new UnsupportedOperationException();
462 public Map
<String
, Object
> getAttributes() {
467 public String
getAppId() {