1 // Copyright 2009 Google Inc. All Rights Reserved.
3 package com
.google
.apphosting
.utils
.remoteapi
;
5 import static com
.google
.apphosting
.datastore
.DatastoreV3Pb
.Error
.ErrorCode
.BAD_REQUEST
;
6 import static com
.google
.apphosting
.datastore
.DatastoreV3Pb
.Error
.ErrorCode
.CONCURRENT_TRANSACTION
;
8 import com
.google
.appengine
.api
.oauth
.OAuthRequestException
;
9 import com
.google
.appengine
.api
.oauth
.OAuthService
;
10 import com
.google
.appengine
.api
.oauth
.OAuthServiceFactory
;
11 import com
.google
.appengine
.api
.users
.UserService
;
12 import com
.google
.appengine
.api
.users
.UserServiceFactory
;
13 import com
.google
.apphosting
.api
.ApiBasePb
.VoidProto
;
14 import com
.google
.apphosting
.api
.ApiProxy
;
15 import com
.google
.apphosting
.datastore
.DatastoreV3Pb
.BeginTransactionRequest
;
16 import com
.google
.apphosting
.datastore
.DatastoreV3Pb
.DeleteRequest
;
17 import com
.google
.apphosting
.datastore
.DatastoreV3Pb
.GetRequest
;
18 import com
.google
.apphosting
.datastore
.DatastoreV3Pb
.GetResponse
;
19 import com
.google
.apphosting
.datastore
.DatastoreV3Pb
.NextRequest
;
20 import com
.google
.apphosting
.datastore
.DatastoreV3Pb
.PutRequest
;
21 import com
.google
.apphosting
.datastore
.DatastoreV3Pb
.Query
;
22 import com
.google
.apphosting
.datastore
.DatastoreV3Pb
.QueryResult
;
23 import com
.google
.apphosting
.utils
.remoteapi
.RemoteApiPb
.ApplicationError
;
24 import com
.google
.apphosting
.utils
.remoteapi
.RemoteApiPb
.Request
;
25 import com
.google
.apphosting
.utils
.remoteapi
.RemoteApiPb
.Response
;
26 import com
.google
.apphosting
.utils
.remoteapi
.RemoteApiPb
.TransactionRequest
;
27 import com
.google
.apphosting
.utils
.remoteapi
.RemoteApiPb
.TransactionRequest
.Precondition
;
28 import com
.google
.io
.protocol
.ProtocolMessage
;
29 import com
.google
.storage
.onestore
.v3
.OnestoreEntity
;
30 import com
.google
.storage
.onestore
.v3
.OnestoreEntity
.EntityProto
;
31 import com
.google
.storage
.onestore
.v3
.OnestoreEntity
.Path
.Element
;
33 import java
.io
.ByteArrayOutputStream
;
34 import java
.io
.InputStream
;
35 import java
.io
.ObjectOutput
;
36 import java
.io
.ObjectOutputStream
;
37 import java
.security
.MessageDigest
;
38 import java
.security
.NoSuchAlgorithmException
;
39 import java
.util
.Arrays
;
40 import java
.util
.HashSet
;
41 import java
.util
.List
;
42 import java
.util
.logging
.Logger
;
44 import javax
.servlet
.http
.HttpServlet
;
45 import javax
.servlet
.http
.HttpServletRequest
;
46 import javax
.servlet
.http
.HttpServletResponse
;
49 * Remote API servlet handler.
52 public class RemoteApiServlet
extends HttpServlet
{
53 private static final Logger log
= Logger
.getLogger(RemoteApiServlet
.class.getName());
55 private static final String
[] OAUTH_SCOPES
= new String
[] {
56 "https://www.googleapis.com/auth/appengine.apis",
57 "https://www.googleapis.com/auth/cloud-platform",
59 private static final String INBOUND_APP_SYSTEM_PROPERTY
= "HTTP_X_APPENGINE_INBOUND_APPID";
60 private static final String INBOUND_APP_HEADER_NAME
= "X-AppEngine-Inbound-AppId";
62 private HashSet
<String
> allowedApps
= null;
63 private final OAuthService oauthService
;
65 public RemoteApiServlet() {
66 this(OAuthServiceFactory
.getOAuthService());
69 RemoteApiServlet(OAuthService oauthService
) {
70 this.oauthService
= oauthService
;
73 private static boolean toBoolean(String initParam
) {
74 initParam
= initParam
.trim();
75 return initParam
.equalsIgnoreCase("true") || initParam
.equals("1");
79 * Exception for unknown errors from a Python remote_api handler.
81 public static class UnknownPythonServerException
extends RuntimeException
{
82 public UnknownPythonServerException(String message
) {
88 * Checks if the inbound request is valid.
90 * @param req the {@link HttpServletRequest}
91 * @param res the {@link HttpServletResponse}
92 * @return true if the application is known.
93 * @throws java.io.IOException
95 boolean checkIsValidRequest(HttpServletRequest req
, HttpServletResponse res
)
96 throws java
.io
.IOException
{
97 if (!checkIsKnownInbound(req
) && !checkIsAdmin(req
, res
)) {
100 return checkIsValidHeader(req
, res
);
104 * Checks if the request is coming from a known application.
106 * @param req the {@link HttpServletRequest}
107 * @return true if the application is known.
108 * @throws java.io.IOException
110 private synchronized boolean checkIsKnownInbound(HttpServletRequest req
)
111 throws java
.io
.IOException
{
112 if (allowedApps
== null) {
113 allowedApps
= new HashSet
<String
>();
114 String allowedAppsStr
= System
.getProperty(INBOUND_APP_SYSTEM_PROPERTY
);
115 if (allowedAppsStr
!= null) {
116 String
[] apps
= allowedAppsStr
.split(",");
117 for (String app
: apps
) {
118 allowedApps
.add(app
);
122 String inboundAppId
= req
.getHeader(INBOUND_APP_HEADER_NAME
);
123 return inboundAppId
!= null && allowedApps
.contains(inboundAppId
);
127 * Checks for the api-version header to prevent XSRF
129 * @param req the {@link HttpServletRequest}
130 * @param res the {@link HttpServletResponse}
131 * @return true if the header exists.
132 * @throws java.io.IOException
134 private boolean checkIsValidHeader(HttpServletRequest req
, HttpServletResponse res
)
135 throws java
.io
.IOException
{
136 if (req
.getHeader("X-appcfg-api-version") == null) {
138 res
.setContentType("text/plain");
139 res
.getWriter().println("This request did not contain a necessary header");
146 * Check that the current user is signed is with admin access.
148 * @return true if the current user is logged in with admin access, false
151 private boolean checkIsAdmin(HttpServletRequest req
, HttpServletResponse res
)
152 throws java
.io
.IOException
{
153 UserService userService
= UserServiceFactory
.getUserService();
155 if (userService
.getCurrentUser() != null) {
156 if (userService
.isUserAdmin()) {
159 respondNotAdmin(res
);
165 if (oauthService
.isUserAdmin(OAUTH_SCOPES
)) {
168 respondNotAdmin(res
);
171 } catch (OAuthRequestException e
) {
174 res
.sendRedirect(userService
.createLoginURL(req
.getRequestURI()));
178 private void respondNotAdmin(HttpServletResponse res
) throws java
.io
.IOException
{
180 res
.setContentType("text/plain");
181 res
.getWriter().println(
182 "You must be logged in as an administrator, or access from an approved application.");
186 * Serve GET requests with a YAML encoding of the app-id and a validation
190 public void doGet(HttpServletRequest req
, HttpServletResponse res
) throws java
.io
.IOException
{
191 if (!checkIsValidRequest(req
, res
)) {
194 res
.setContentType("text/plain");
195 String appId
= ApiProxy
.getCurrentEnvironment().getAppId();
196 StringBuilder outYaml
=
197 new StringBuilder().append("{rtok: ").append(req
.getParameter("rtok")).append(", app_id: ")
198 .append(appId
).append("}");
199 res
.getWriter().println(outYaml
);
203 * Serve POST requests by forwarding calls to ApiProxy.
206 public void doPost(HttpServletRequest req
, HttpServletResponse res
) throws java
.io
.IOException
{
207 if (!checkIsValidRequest(req
, res
)) {
210 res
.setContentType("application/octet-stream");
212 Response response
= new Response();
215 byte[] responseData
= executeRequest(req
);
216 response
.setResponseAsBytes(responseData
);
218 } catch (Exception e
) {
219 log
.warning("Caught exception while executing remote_api command:\n" + e
);
221 ByteArrayOutputStream byteStream
= new ByteArrayOutputStream();
222 ObjectOutput out
= new ObjectOutputStream(byteStream
);
225 byte[] serializedException
= byteStream
.toByteArray();
226 response
.setJavaExceptionAsBytes(serializedException
);
227 if (e
instanceof ApiProxy
.ApplicationException
) {
228 ApiProxy
.ApplicationException ae
= (ApiProxy
.ApplicationException
) e
;
229 ApplicationError appError
= response
.getMutableApplicationError();
230 appError
.setCode(ae
.getApplicationError());
231 appError
.setDetail(ae
.getErrorDetail());
234 res
.getOutputStream().write(response
.toByteArray());
237 private byte[] executeRunQuery(Request request
) {
238 Query queryRequest
= new Query();
239 parseFromBytes(queryRequest
, request
.getRequestAsBytes());
240 int batchSize
= Math
.max(1000, queryRequest
.getLimit());
241 queryRequest
.setCount(batchSize
);
243 QueryResult runQueryResponse
= new QueryResult();
244 byte[] res
= ApiProxy
.makeSyncCall("datastore_v3", "RunQuery", request
.getRequestAsBytes());
245 parseFromBytes(runQueryResponse
, res
);
247 if (queryRequest
.hasLimit()) {
248 while (runQueryResponse
.isMoreResults()) {
249 NextRequest nextRequest
= new NextRequest();
250 nextRequest
.getMutableCursor().mergeFrom(runQueryResponse
.getCursor());
251 nextRequest
.setCount(batchSize
);
252 byte[] nextRes
= ApiProxy
.makeSyncCall("datastore_v3", "Next", nextRequest
.toByteArray());
253 parseFromBytes(runQueryResponse
, nextRes
);
256 return runQueryResponse
.toByteArray();
259 private byte[] executeTxQuery(Request request
) {
260 RemoteApiPb
.TransactionQueryResult result
= new RemoteApiPb
.TransactionQueryResult();
262 Query query
= new Query();
263 parseFromBytes(query
, request
.getRequestAsBytes());
265 if (!query
.hasAncestor()) {
266 throw new ApiProxy
.ApplicationException(BAD_REQUEST
.getValue(),
267 "No ancestor in transactional query.");
269 OnestoreEntity
.Reference egKey
=
270 result
.getMutableEntityGroupKey().mergeFrom(query
.getAncestor());
271 OnestoreEntity
.Path
.Element root
= egKey
.getPath().getElement(0);
272 egKey
.getMutablePath().clearElement().addElement(root
);
273 OnestoreEntity
.Path
.Element egElement
= new OnestoreEntity
.Path
.Element();
274 egElement
.setType("__entity_group__").setId(1);
275 egKey
.getMutablePath().addElement(egElement
);
277 byte[] tx
= beginTransaction(false);
278 parseFromBytes(query
.getMutableTransaction(), tx
);
279 byte[] queryBytes
= ApiProxy
.makeSyncCall("datastore_v3", "RunQuery", query
.toByteArray());
280 parseFromBytes(result
.getMutableResult(), queryBytes
);
282 GetRequest egRequest
= new GetRequest();
283 egRequest
.addKey(egKey
);
284 GetResponse egResponse
= txGet(tx
, egRequest
);
285 if (egResponse
.getEntity(0).hasEntity()) {
286 result
.setEntityGroup(egResponse
.getEntity(0).getEntity());
290 return result
.toByteArray();
294 * Throws a CONCURRENT_TRANSACTION exception if the entity does not match the precondition.
296 private void assertEntityResultMatchesPrecondition(
297 GetResponse
.Entity entityResult
, Precondition precondition
) {
298 if (precondition
.hasHash() != entityResult
.hasEntity()) {
299 throw new ApiProxy
.ApplicationException(CONCURRENT_TRANSACTION
.getValue(),
300 "Transaction precondition failed");
303 if (entityResult
.hasEntity()) {
304 EntityProto entity
= entityResult
.getEntity();
305 if (Arrays
.equals(precondition
.getHashAsBytes(), computeSha1(entity
))) {
309 byte[] backwardsCompatibleHash
= computeSha1OmittingLastByteForBackwardsCompatibility(entity
);
310 if (!Arrays
.equals(precondition
.getHashAsBytes(), backwardsCompatibleHash
)) {
311 throw new ApiProxy
.ApplicationException(
312 CONCURRENT_TRANSACTION
.getValue(), "Transaction precondition failed");
317 private byte[] executeTx(Request request
) {
318 TransactionRequest txRequest
= new TransactionRequest();
319 parseFromBytes(txRequest
, request
.getRequestAsBytes());
321 byte[] tx
= beginTransaction(txRequest
.isAllowMultipleEg());
323 List
<Precondition
> preconditions
= txRequest
.preconditions();
325 if (!preconditions
.isEmpty()) {
326 GetRequest getRequest
= new GetRequest();
327 for (Precondition precondition
: preconditions
) {
328 OnestoreEntity
.Reference key
= precondition
.getKey();
329 OnestoreEntity
.Reference requestKey
= getRequest
.addKey();
330 requestKey
.mergeFrom(key
);
333 GetResponse getResponse
= txGet(tx
, getRequest
);
334 List
<GetResponse
.Entity
> entities
= getResponse
.entitys();
336 assert (entities
.size() == preconditions
.size());
337 for (int i
= 0; i
< entities
.size(); i
++) {
338 assertEntityResultMatchesPrecondition(entities
.get(i
), preconditions
.get(i
));
341 byte[] res
= new VoidProto().toByteArray();
342 if (txRequest
.hasPuts()) {
343 PutRequest putRequest
= txRequest
.getPuts();
344 parseFromBytes(putRequest
.getMutableTransaction(), tx
);
345 res
= ApiProxy
.makeSyncCall("datastore_v3", "Put", putRequest
.toByteArray());
347 if (txRequest
.hasDeletes()) {
348 DeleteRequest deleteRequest
= txRequest
.getDeletes();
349 parseFromBytes(deleteRequest
.getMutableTransaction(), tx
);
350 ApiProxy
.makeSyncCall("datastore_v3", "Delete", deleteRequest
.toByteArray());
352 ApiProxy
.makeSyncCall("datastore_v3", "Commit", tx
);
356 private byte[] executeGetIDs(Request request
, boolean isXG
) {
357 PutRequest putRequest
= new PutRequest();
358 parseFromBytes(putRequest
, request
.getRequestAsBytes());
359 for (EntityProto entity
: putRequest
.entitys()) {
360 assert (entity
.propertySize() == 0);
361 assert (entity
.rawPropertySize() == 0);
362 assert (entity
.getEntityGroup().elementSize() == 0);
363 List
<Element
> elementList
= entity
.getKey().getPath().elements();
364 Element lastPart
= elementList
.get(elementList
.size() - 1);
365 assert (lastPart
.getId() == 0);
366 assert (!lastPart
.hasName());
369 byte[] tx
= beginTransaction(isXG
);
370 parseFromBytes(putRequest
.getMutableTransaction(), tx
);
372 byte[] res
= ApiProxy
.makeSyncCall("datastore_v3", "Put", putRequest
.toByteArray());
378 private byte[] executeRequest(HttpServletRequest req
) throws java
.io
.IOException
{
379 Request request
= new Request();
380 parseFromInputStream(request
, req
.getInputStream());
381 String service
= request
.getServiceName();
382 String method
= request
.getMethod();
384 log
.fine("remote API call: " + service
+ ", " + method
);
386 if (service
.equals("remote_datastore")) {
387 if (method
.equals("RunQuery")) {
388 return executeRunQuery(request
);
389 } else if (method
.equals("Transaction")) {
390 return executeTx(request
);
391 } else if (method
.equals("TransactionQuery")) {
392 return executeTxQuery(request
);
393 } else if (method
.equals("GetIDs")) {
394 return executeGetIDs(request
, false);
395 } else if (method
.equals("GetIDsXG")) {
396 return executeGetIDs(request
, true);
398 throw new ApiProxy
.CallNotFoundException(service
, method
);
401 return ApiProxy
.makeSyncCall(service
, method
, request
.getRequestAsBytes());
405 private static byte[] beginTransaction(boolean allowMultipleEg
) {
406 String appId
= ApiProxy
.getCurrentEnvironment().getAppId();
407 byte[] req
= new BeginTransactionRequest().setApp(appId
)
408 .setAllowMultipleEg(allowMultipleEg
).toByteArray();
409 return ApiProxy
.makeSyncCall("datastore_v3", "BeginTransaction", req
);
412 private static void rollback(byte[] tx
) {
413 ApiProxy
.makeSyncCall("datastore_v3", "Rollback", tx
);
416 private static GetResponse
txGet(byte[] tx
, GetRequest request
) {
417 parseFromBytes(request
.getMutableTransaction(), tx
);
418 GetResponse response
= new GetResponse();
419 byte[] resultBytes
= ApiProxy
.makeSyncCall("datastore_v3", "Get", request
.toByteArray());
420 parseFromBytes(response
, resultBytes
);
424 static byte[] computeSha1(EntityProto entity
) {
425 byte[] entityBytes
= entity
.toByteArray();
426 return computeSha1(entityBytes
, entityBytes
.length
);
430 * This is a HACK. There used to be a bug in RemoteDatastore.java in that it would omit the last
431 * byte of the Entity when calculating the hash for the Precondition. If an app has not updated
432 * that library, we may still receive hashes like this. For backwards compatibility, we'll
433 * consider the transaction valid if omitting the last byte of the Entity matches the
436 static byte[] computeSha1OmittingLastByteForBackwardsCompatibility(EntityProto entity
) {
437 byte[] entityBytes
= entity
.toByteArray();
438 return computeSha1(entityBytes
, entityBytes
.length
- 1);
441 private static byte[] computeSha1(byte[] bytes
, int length
) {
444 md
= MessageDigest
.getInstance("SHA-1");
445 } catch (NoSuchAlgorithmException e
) {
446 throw new ApiProxy
.ApplicationException(
447 CONCURRENT_TRANSACTION
.getValue(), "Transaction precondition could not be computed");
450 md
.update(bytes
, 0, length
);
454 private static void parseFromBytes(ProtocolMessage
<?
> message
, byte[] bytes
) {
455 boolean parsed
= message
.parseFrom(bytes
);
456 checkParse(message
, parsed
);
459 private static void parseFromInputStream(ProtocolMessage
<?
> message
, InputStream inputStream
) {
460 boolean parsed
= message
.parseFrom(inputStream
);
461 checkParse(message
, parsed
);
464 private static void checkParse(ProtocolMessage
<?
> message
, boolean parsed
) {
466 throw new ApiProxy
.ApiProxyException("Could not parse protobuf");
468 String error
= message
.findInitializationError();
470 throw new ApiProxy
.ApiProxyException("Could not parse protobuf: " + error
);