java/src/main/com/google/apphosting/utils/remoteapi/RemoteApiServlet.java

   1 // Copyright 2009 Google Inc. All Rights Reserved.
   2
   3 package com.google.apphosting.utils.remoteapi;
   4
   5 import static com.google.apphosting.datastore.DatastoreV3Pb.Error.ErrorCode.BAD_REQUEST;
   6 import static com.google.apphosting.datastore.DatastoreV3Pb.Error.ErrorCode.CONCURRENT_TRANSACTION;
   7
   8 import com.google.appengine.api.oauth.OAuthRequestException;
   9 import com.google.appengine.api.oauth.OAuthService;
  10 import com.google.appengine.api.oauth.OAuthServiceFactory;
  11 import com.google.appengine.api.users.UserService;
  12 import com.google.appengine.api.users.UserServiceFactory;
  13 import com.google.apphosting.api.ApiBasePb.VoidProto;
  14 import com.google.apphosting.api.ApiProxy;
  15 import com.google.apphosting.datastore.DatastoreV3Pb.BeginTransactionRequest;
  16 import com.google.apphosting.datastore.DatastoreV3Pb.DeleteRequest;
  17 import com.google.apphosting.datastore.DatastoreV3Pb.GetRequest;
  18 import com.google.apphosting.datastore.DatastoreV3Pb.GetResponse;
  19 import com.google.apphosting.datastore.DatastoreV3Pb.NextRequest;
  20 import com.google.apphosting.datastore.DatastoreV3Pb.PutRequest;
  21 import com.google.apphosting.datastore.DatastoreV3Pb.Query;
  22 import com.google.apphosting.datastore.DatastoreV3Pb.QueryResult;
  23 import com.google.apphosting.utils.remoteapi.RemoteApiPb.ApplicationError;
  24 import com.google.apphosting.utils.remoteapi.RemoteApiPb.Request;
  25 import com.google.apphosting.utils.remoteapi.RemoteApiPb.Response;
  26 import com.google.apphosting.utils.remoteapi.RemoteApiPb.TransactionRequest;
  27 import com.google.apphosting.utils.remoteapi.RemoteApiPb.TransactionRequest.Precondition;
  28 import com.google.io.protocol.ProtocolMessage;
  29 import com.google.storage.onestore.v3.OnestoreEntity;
  30 import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
  31 import com.google.storage.onestore.v3.OnestoreEntity.Path.Element;
  32
  33 import java.io.ByteArrayOutputStream;
  34 import java.io.InputStream;
  35 import java.io.ObjectOutput;
  36 import java.io.ObjectOutputStream;
  37 import java.security.MessageDigest;
  38 import java.security.NoSuchAlgorithmException;
  39 import java.util.Arrays;
  40 import java.util.HashSet;
  41 import java.util.List;
  42 import java.util.logging.Logger;
  43
  44 import javax.servlet.http.HttpServlet;
  45 import javax.servlet.http.HttpServletRequest;
  46 import javax.servlet.http.HttpServletResponse;
  47
  48 /**
  49  * Remote API servlet handler.
  50  *
  51  */
  52 public class RemoteApiServlet extends HttpServlet {
  53   private static final Logger log = Logger.getLogger(RemoteApiServlet.class.getName());
  54
  55   private static final String[] OAUTH_SCOPES = new String[] {
  56       "https://www.googleapis.com/auth/appengine.apis",
  57       "https://www.googleapis.com/auth/cloud-platform",
  58   };
  59   private static final String INBOUND_APP_SYSTEM_PROPERTY = "HTTP_X_APPENGINE_INBOUND_APPID";
  60   private static final String INBOUND_APP_HEADER_NAME = "X-AppEngine-Inbound-AppId";
  61
  62   private HashSet<String> allowedApps = null;
  63   private final OAuthService oauthService;
  64
  65   public RemoteApiServlet() {
  66     this(OAuthServiceFactory.getOAuthService());
  67   }
  68
  69   RemoteApiServlet(OAuthService oauthService) {
  70     this.oauthService = oauthService;
  71   }
  72
  73   private static boolean toBoolean(String initParam) {
  74     initParam = initParam.trim();
  75     return initParam.equalsIgnoreCase("true") || initParam.equals("1");
  76   }
  77
  78   /**
  79    * Exception for unknown errors from a Python remote_api handler.
  80    */
  81   public static class UnknownPythonServerException extends RuntimeException {
  82     public UnknownPythonServerException(String message) {
  83       super(message);
  84     }
  85   }
  86
  87   /**
  88    * Checks if the inbound request is valid.
  89    *
  90    * @param req the {@link HttpServletRequest}
  91    * @param res the {@link HttpServletResponse}
  92    * @return true if the application is known.
  93    * @throws java.io.IOException
  94    */
  95   boolean checkIsValidRequest(HttpServletRequest req, HttpServletResponse res)
  96       throws java.io.IOException {
  97     if (!checkIsKnownInbound(req) && !checkIsAdmin(req, res)) {
  98       return false;
  99     }
 100     return checkIsValidHeader(req, res);
 101   }
 102
 103   /**
 104    * Checks if the request is coming from a known application.
 105    *
 106    * @param req the {@link HttpServletRequest}
 107    * @return true if the application is known.
 108    * @throws java.io.IOException
 109    */
 110   private synchronized boolean checkIsKnownInbound(HttpServletRequest req)
 111       throws java.io.IOException {
 112     if (allowedApps == null) {
 113       allowedApps = new HashSet<String>();
 114       String allowedAppsStr = System.getProperty(INBOUND_APP_SYSTEM_PROPERTY);
 115       if (allowedAppsStr != null) {
 116         String[] apps = allowedAppsStr.split(",");
 117         for (String app : apps) {
 118           allowedApps.add(app);
 119         }
 120       }
 121     }
 122     String inboundAppId = req.getHeader(INBOUND_APP_HEADER_NAME);
 123     return inboundAppId != null && allowedApps.contains(inboundAppId);
 124   }
 125
 126   /**
 127    * Checks for the api-version header to prevent XSRF
 128    *
 129    * @param req the {@link HttpServletRequest}
 130    * @param res the {@link HttpServletResponse}
 131    * @return true if the header exists.
 132    * @throws java.io.IOException
 133    */
 134   private boolean checkIsValidHeader(HttpServletRequest req, HttpServletResponse res)
 135       throws java.io.IOException {
 136     if (req.getHeader("X-appcfg-api-version") == null) {
 137       res.setStatus(403);
 138       res.setContentType("text/plain");
 139       res.getWriter().println("This request did not contain a necessary header");
 140       return false;
 141     }
 142     return true;
 143   }
 144
 145   /**
 146    * Check that the current user is signed is with admin access.
 147    *
 148    * @return true if the current user is logged in with admin access, false
 149    *         otherwise.
 150    */
 151   private boolean checkIsAdmin(HttpServletRequest req, HttpServletResponse res)
 152       throws java.io.IOException {
 153     UserService userService = UserServiceFactory.getUserService();
 154
 155     if (userService.getCurrentUser() != null) {
 156       if (userService.isUserAdmin()) {
 157         return true;
 158       } else {
 159         respondNotAdmin(res);
 160         return false;
 161       }
 162     }
 163
 164     try {
 165       if (oauthService.isUserAdmin(OAUTH_SCOPES)) {
 166         return true;
 167       } else {
 168         respondNotAdmin(res);
 169         return false;
 170       }
 171     } catch (OAuthRequestException e) {
 172     }
 173
 174     res.sendRedirect(userService.createLoginURL(req.getRequestURI()));
 175     return false;
 176   }
 177
 178   private void respondNotAdmin(HttpServletResponse res) throws java.io.IOException {
 179     res.setStatus(401);
 180     res.setContentType("text/plain");
 181     res.getWriter().println(
 182         "You must be logged in as an administrator, or access from an approved application.");
 183   }
 184
 185   /**
 186    * Serve GET requests with a YAML encoding of the app-id and a validation
 187    * token.
 188    */
 189   @Override
 190   public void doGet(HttpServletRequest req, HttpServletResponse res) throws java.io.IOException {
 191     if (!checkIsValidRequest(req, res)) {
 192       return;
 193     }
 194     res.setContentType("text/plain");
 195     String appId = ApiProxy.getCurrentEnvironment().getAppId();
 196     StringBuilder outYaml =
 197         new StringBuilder().append("{rtok: ").append(req.getParameter("rtok")).append(", app_id: ")
 198             .append(appId).append("}");
 199     res.getWriter().println(outYaml);
 200   }
 201
 202   /**
 203    * Serve POST requests by forwarding calls to ApiProxy.
 204    */
 205   @Override
 206   public void doPost(HttpServletRequest req, HttpServletResponse res) throws java.io.IOException {
 207     if (!checkIsValidRequest(req, res)) {
 208       return;
 209     }
 210     res.setContentType("application/octet-stream");
 211
 212     Response response = new Response();
 213
 214     try {
 215       byte[] responseData = executeRequest(req);
 216       response.setResponseAsBytes(responseData);
 217       res.setStatus(200);
 218     } catch (Exception e) {
 219       log.warning("Caught exception while executing remote_api command:\n" + e);
 220       res.setStatus(200);
 221       ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
 222       ObjectOutput out = new ObjectOutputStream(byteStream);
 223       out.writeObject(e);
 224       out.close();
 225       byte[] serializedException = byteStream.toByteArray();
 226       response.setJavaExceptionAsBytes(serializedException);
 227       if (e instanceof ApiProxy.ApplicationException) {
 228         ApiProxy.ApplicationException ae = (ApiProxy.ApplicationException) e;
 229         ApplicationError appError = response.getMutableApplicationError();
 230         appError.setCode(ae.getApplicationError());
 231         appError.setDetail(ae.getErrorDetail());
 232       }
 233     }
 234     res.getOutputStream().write(response.toByteArray());
 235   }
 236
 237   private byte[] executeRunQuery(Request request) {
 238     Query queryRequest = new Query();
 239     parseFromBytes(queryRequest, request.getRequestAsBytes());
 240     int batchSize = Math.max(1000, queryRequest.getLimit());
 241     queryRequest.setCount(batchSize);
 242
 243     QueryResult runQueryResponse = new QueryResult();
 244     byte[] res = ApiProxy.makeSyncCall("datastore_v3", "RunQuery", request.getRequestAsBytes());
 245     parseFromBytes(runQueryResponse, res);
 246
 247     if (queryRequest.hasLimit()) {
 248       while (runQueryResponse.isMoreResults()) {
 249         NextRequest nextRequest = new NextRequest();
 250         nextRequest.getMutableCursor().mergeFrom(runQueryResponse.getCursor());
 251         nextRequest.setCount(batchSize);
 252         byte[] nextRes = ApiProxy.makeSyncCall("datastore_v3", "Next", nextRequest.toByteArray());
 253         parseFromBytes(runQueryResponse, nextRes);
 254       }
 255     }
 256     return runQueryResponse.toByteArray();
 257   }
 258
 259   private byte[] executeTxQuery(Request request) {
 260     RemoteApiPb.TransactionQueryResult result = new RemoteApiPb.TransactionQueryResult();
 261
 262     Query query = new Query();
 263     parseFromBytes(query, request.getRequestAsBytes());
 264
 265     if (!query.hasAncestor()) {
 266       throw new ApiProxy.ApplicationException(BAD_REQUEST.getValue(),
 267                                               "No ancestor in transactional query.");
 268     }
 269     OnestoreEntity.Reference egKey =
 270         result.getMutableEntityGroupKey().mergeFrom(query.getAncestor());
 271     OnestoreEntity.Path.Element root = egKey.getPath().getElement(0);
 272     egKey.getMutablePath().clearElement().addElement(root);
 273     OnestoreEntity.Path.Element egElement = new OnestoreEntity.Path.Element();
 274     egElement.setType("__entity_group__").setId(1);
 275     egKey.getMutablePath().addElement(egElement);
 276
 277     byte[] tx = beginTransaction(false);
 278     parseFromBytes(query.getMutableTransaction(), tx);
 279     byte[] queryBytes = ApiProxy.makeSyncCall("datastore_v3", "RunQuery", query.toByteArray());
 280     parseFromBytes(result.getMutableResult(), queryBytes);
 281
 282     GetRequest egRequest = new GetRequest();
 283     egRequest.addKey(egKey);
 284     GetResponse egResponse = txGet(tx, egRequest);
 285     if (egResponse.getEntity(0).hasEntity()) {
 286       result.setEntityGroup(egResponse.getEntity(0).getEntity());
 287     }
 288     rollback(tx);
 289
 290     return result.toByteArray();
 291   }
 292
 293   /**
 294    * Throws a CONCURRENT_TRANSACTION exception if the entity does not match the precondition.
 295    */
 296   private void assertEntityResultMatchesPrecondition(
 297       GetResponse.Entity entityResult, Precondition precondition) {
 298     if (precondition.hasHash() != entityResult.hasEntity()) {
 299       throw new ApiProxy.ApplicationException(CONCURRENT_TRANSACTION.getValue(),
 300           "Transaction precondition failed");
 301     }
 302
 303     if (entityResult.hasEntity()) {
 304       EntityProto entity = entityResult.getEntity();
 305       if (Arrays.equals(precondition.getHashAsBytes(), computeSha1(entity))) {
 306         return;
 307       }
 308
 309       byte[] backwardsCompatibleHash = computeSha1OmittingLastByteForBackwardsCompatibility(entity);
 310       if (!Arrays.equals(precondition.getHashAsBytes(), backwardsCompatibleHash)) {
 311         throw new ApiProxy.ApplicationException(
 312             CONCURRENT_TRANSACTION.getValue(), "Transaction precondition failed");
 313       }
 314     }
 315   }
 316
 317   private byte[] executeTx(Request request) {
 318     TransactionRequest txRequest = new TransactionRequest();
 319     parseFromBytes(txRequest, request.getRequestAsBytes());
 320
 321     byte[] tx = beginTransaction(txRequest.isAllowMultipleEg());
 322
 323     List<Precondition> preconditions = txRequest.preconditions();
 324
 325     if (!preconditions.isEmpty()) {
 326       GetRequest getRequest = new GetRequest();
 327       for (Precondition precondition : preconditions) {
 328         OnestoreEntity.Reference key = precondition.getKey();
 329         OnestoreEntity.Reference requestKey = getRequest.addKey();
 330         requestKey.mergeFrom(key);
 331       }
 332
 333       GetResponse getResponse = txGet(tx, getRequest);
 334       List<GetResponse.Entity> entities = getResponse.entitys();
 335
 336       assert (entities.size() == preconditions.size());
 337       for (int i = 0; i < entities.size(); i++) {
 338         assertEntityResultMatchesPrecondition(entities.get(i), preconditions.get(i));
 339       }
 340     }
 341     byte[] res = new VoidProto().toByteArray();
 342     if (txRequest.hasPuts()) {
 343       PutRequest putRequest = txRequest.getPuts();
 344       parseFromBytes(putRequest.getMutableTransaction(), tx);
 345       res = ApiProxy.makeSyncCall("datastore_v3", "Put", putRequest.toByteArray());
 346     }
 347     if (txRequest.hasDeletes()) {
 348       DeleteRequest deleteRequest = txRequest.getDeletes();
 349       parseFromBytes(deleteRequest.getMutableTransaction(), tx);
 350       ApiProxy.makeSyncCall("datastore_v3", "Delete", deleteRequest.toByteArray());
 351     }
 352     ApiProxy.makeSyncCall("datastore_v3", "Commit", tx);
 353     return res;
 354   }
 355
 356   private byte[] executeGetIDs(Request request, boolean isXG) {
 357     PutRequest putRequest = new PutRequest();
 358     parseFromBytes(putRequest, request.getRequestAsBytes());
 359     for (EntityProto entity : putRequest.entitys()) {
 360       assert (entity.propertySize() == 0);
 361       assert (entity.rawPropertySize() == 0);
 362       assert (entity.getEntityGroup().elementSize() == 0);
 363       List<Element> elementList = entity.getKey().getPath().elements();
 364       Element lastPart = elementList.get(elementList.size() - 1);
 365       assert (lastPart.getId() == 0);
 366       assert (!lastPart.hasName());
 367     }
 368
 369     byte[] tx = beginTransaction(isXG);
 370     parseFromBytes(putRequest.getMutableTransaction(), tx);
 371
 372     byte[] res = ApiProxy.makeSyncCall("datastore_v3", "Put", putRequest.toByteArray());
 373
 374     rollback(tx);
 375     return res;
 376   }
 377
 378   private byte[] executeRequest(HttpServletRequest req) throws java.io.IOException {
 379     Request request = new Request();
 380     parseFromInputStream(request, req.getInputStream());
 381     String service = request.getServiceName();
 382     String method = request.getMethod();
 383
 384     log.fine("remote API call: " + service + ", " + method);
 385
 386     if (service.equals("remote_datastore")) {
 387       if (method.equals("RunQuery")) {
 388         return executeRunQuery(request);
 389       } else if (method.equals("Transaction")) {
 390         return executeTx(request);
 391       } else if (method.equals("TransactionQuery")) {
 392         return executeTxQuery(request);
 393       } else if (method.equals("GetIDs")) {
 394         return executeGetIDs(request, false);
 395       } else if (method.equals("GetIDsXG")) {
 396         return executeGetIDs(request, true);
 397       } else {
 398         throw new ApiProxy.CallNotFoundException(service, method);
 399       }
 400     } else {
 401       return ApiProxy.makeSyncCall(service, method, request.getRequestAsBytes());
 402     }
 403   }
 404
 405   private static byte[] beginTransaction(boolean allowMultipleEg) {
 406     String appId = ApiProxy.getCurrentEnvironment().getAppId();
 407     byte[] req = new BeginTransactionRequest().setApp(appId)
 408         .setAllowMultipleEg(allowMultipleEg).toByteArray();
 409     return ApiProxy.makeSyncCall("datastore_v3", "BeginTransaction", req);
 410   }
 411
 412   private static void rollback(byte[] tx) {
 413     ApiProxy.makeSyncCall("datastore_v3", "Rollback", tx);
 414   }
 415
 416   private static GetResponse txGet(byte[] tx, GetRequest request) {
 417     parseFromBytes(request.getMutableTransaction(), tx);
 418     GetResponse response = new GetResponse();
 419     byte[] resultBytes = ApiProxy.makeSyncCall("datastore_v3", "Get", request.toByteArray());
 420     parseFromBytes(response, resultBytes);
 421     return response;
 422   }
 423
 424   static byte[] computeSha1(EntityProto entity) {
 425     byte[] entityBytes = entity.toByteArray();
 426     return computeSha1(entityBytes, entityBytes.length);
 427   }
 428
 429   /**
 430    * This is a HACK.  There used to be a bug in RemoteDatastore.java in that it would omit the last
 431    * byte of the Entity when calculating the hash for the Precondition.  If an app has not updated
 432    * that library, we may still receive hashes like this.  For backwards compatibility, we'll
 433    * consider the transaction valid if omitting the last byte of the Entity matches the
 434    * Precondition.
 435    */
 436   static byte[] computeSha1OmittingLastByteForBackwardsCompatibility(EntityProto entity) {
 437     byte[] entityBytes = entity.toByteArray();
 438     return computeSha1(entityBytes, entityBytes.length - 1);
 439   }
 440
 441   private static byte[] computeSha1(byte[] bytes, int length) {
 442     MessageDigest md;
 443     try {
 444       md = MessageDigest.getInstance("SHA-1");
 445     } catch (NoSuchAlgorithmException e) {
 446       throw new ApiProxy.ApplicationException(
 447           CONCURRENT_TRANSACTION.getValue(), "Transaction precondition could not be computed");
 448     }
 449
 450     md.update(bytes, 0, length);
 451     return md.digest();
 452   }
 453
 454   private static void parseFromBytes(ProtocolMessage<?> message, byte[] bytes) {
 455     boolean parsed = message.parseFrom(bytes);
 456     checkParse(message, parsed);
 457   }
 458
 459   private static void parseFromInputStream(ProtocolMessage<?> message, InputStream inputStream) {
 460     boolean parsed = message.parseFrom(inputStream);
 461     checkParse(message, parsed);
 462   }
 463
 464   private static void checkParse(ProtocolMessage<?> message, boolean parsed) {
 465     if (!parsed) {
 466       throw new ApiProxy.ApiProxyException("Could not parse protobuf");
 467     }
 468     String error = message.findInitializationError();
 469     if (error != null) {
 470       throw new ApiProxy.ApiProxyException("Could not parse protobuf: " + error);
 471     }
 472   }
 473 }