java/src/main/com/google/appengine/api/urlfetch/URLFetchServiceImpl.java

   1 // Copyright 2007 Google Inc. All rights reserved.
   2
   3 package com.google.appengine.api.urlfetch;
   4
   5 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchRequest;
   6 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchRequest.RequestMethod;
   7 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchResponse;
   8 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchResponse.Header;
   9 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchServiceError.ErrorCode;
  10 import com.google.appengine.api.utils.FutureWrapper;
  11 import com.google.apphosting.api.ApiProxy;
  12 import com.google.protobuf.ByteString;
  13
  14 import java.io.IOException;
  15 import java.net.MalformedURLException;
  16 import java.net.SocketTimeoutException;
  17 import java.net.URL;
  18 import java.net.UnknownHostException;
  19 import java.util.concurrent.Future;
  20 import java.util.logging.Logger;
  21
  22 import javax.net.ssl.SSLHandshakeException;
  23
  24 class URLFetchServiceImpl implements URLFetchService {
  25   static final String PACKAGE = "urlfetch";
  26
  27   private static final Logger logger = Logger.getLogger(URLFetchServiceImpl.class.getName());
  28
  29   @Override
  30   public HTTPResponse fetch(URL url) throws IOException {
  31     return fetch(new HTTPRequest(url));
  32   }
  33
  34   @Override
  35   public HTTPResponse fetch(HTTPRequest request) throws IOException {
  36     byte[] responseBytes;
  37     try {
  38       responseBytes = ApiProxy.makeSyncCall(PACKAGE, "Fetch", convertToPb(request).toByteArray(),
  39           createApiConfig(request.getFetchOptions()));
  40     } catch (ApiProxy.RequestTooLargeException ex) {
  41       throw new IOException("The request exceeded the maximum permissible size");
  42     } catch (ApiProxy.ApplicationException ex) {
  43       Throwable cause = convertApplicationException(request.getURL(), ex);
  44       if (cause instanceof RuntimeException) {
  45         throw (RuntimeException) cause;
  46       } else if (cause instanceof IOException) {
  47         throw (IOException) cause;
  48       } else {
  49         throw new RuntimeException(cause);
  50       }
  51     } catch (ApiProxy.ApiDeadlineExceededException ex) {
  52       throw new SocketTimeoutException("Timeout while fetching: " + request.getURL());
  53     }
  54
  55     URLFetchResponse responseProto = URLFetchResponse.newBuilder().mergeFrom(responseBytes).build();
  56     if (!request.getFetchOptions().getAllowTruncate() && responseProto.getContentWasTruncated()) {
  57       throw new ResponseTooLargeException(request.getURL().toString());
  58     }
  59     return convertFromPb(responseProto);
  60   }
  61
  62   @Override
  63   public Future<HTTPResponse> fetchAsync(URL url) {
  64     return fetchAsync(new HTTPRequest(url));
  65   }
  66
  67   @Override
  68   public Future<HTTPResponse> fetchAsync(HTTPRequest request) {
  69     final FetchOptions fetchOptions = request.getFetchOptions();
  70     final URL url = request.getURL();
  71
  72     Future<byte[]> response = ApiProxy.makeAsyncCall(
  73         PACKAGE, "Fetch", convertToPb(request).toByteArray(), createApiConfig(fetchOptions));
  74
  75     return new FutureWrapper<byte[], HTTPResponse>(response) {
  76       @Override
  77       protected HTTPResponse wrap(byte[] responseBytes) throws IOException {
  78         URLFetchResponse responseProto =
  79             URLFetchResponse.newBuilder().mergeFrom(responseBytes).build();
  80         if (!fetchOptions.getAllowTruncate() && responseProto.getContentWasTruncated()) {
  81           throw new ResponseTooLargeException(url.toString());
  82         }
  83         return convertFromPb(responseProto);
  84       }
  85
  86       @Override
  87       protected Throwable convertException(Throwable cause) {
  88         if (cause instanceof ApiProxy.ApplicationException) {
  89           return convertApplicationException(url, (ApiProxy.ApplicationException) cause);
  90         } else if (cause instanceof ApiProxy.ApiDeadlineExceededException) {
  91           return new SocketTimeoutException("Timeout while fetching: " + url);
  92         }
  93         return cause;
  94       }
  95     };
  96   }
  97
  98   static class DeadlineParser {
  99     static final DeadlineParser INSTANCE = new DeadlineParser();
 100     volatile int deadlineMs = -1;
 101
 102     private DeadlineParser() {
 103       refresh();
 104     }
 105
 106     void refresh() {
 107       String globalDefault = System.getProperty(URLFetchService.DEFAULT_DEADLINE_PROPERTY);
 108       if (globalDefault != null) {
 109         try {
 110           deadlineMs = (int) (Double.parseDouble(globalDefault) * 1000);
 111         } catch (NumberFormatException e) {
 112           deadlineMs = -1;
 113           logger.warning("Cannot parse deadline: " + globalDefault);
 114         }
 115       } else {
 116         deadlineMs = -1;
 117       }
 118     }
 119   }
 120
 121   private ApiProxy.ApiConfig createApiConfig(FetchOptions options) {
 122     ApiProxy.ApiConfig apiConfig = new ApiProxy.ApiConfig();
 123
 124     if (options.getDeadline() != null) {
 125       apiConfig.setDeadlineInSeconds(options.getDeadline());
 126     } else if (DeadlineParser.INSTANCE.deadlineMs >= 0) {
 127       apiConfig.setDeadlineInSeconds(DeadlineParser.INSTANCE.deadlineMs / 1000.0);
 128     }
 129
 130     return apiConfig;
 131   }
 132
 133   private String getURLExceptionMessage(String formatString, String url, String errorDetail) {
 134     if (errorDetail == null || errorDetail.trim().isEmpty()) {
 135       return String.format(formatString, url);
 136     } else {
 137       return String.format(formatString + ", error: %s", url, errorDetail);
 138     }
 139   }
 140
 141   private Throwable convertApplicationException(URL requestUrl, ApiProxy.ApplicationException ex) {
 142     String urlString = requestUrl.toString();
 143     ErrorCode errorCode = ErrorCode.valueOf(ex.getApplicationError());
 144     String errorDetail = ex.getErrorDetail();
 145     switch (errorCode) {
 146       case INVALID_URL:
 147         return new MalformedURLException(
 148             getURLExceptionMessage("Invalid URL specified: %s", urlString, errorDetail));
 149       case PAYLOAD_TOO_LARGE:
 150         return new RequestPayloadTooLargeException(urlString);
 151       case CLOSED:
 152         return new IOException(getURLExceptionMessage(
 153             "Connection closed unexpectedly by server at URL: %s", urlString, null));
 154       case TOO_MANY_REDIRECTS:
 155         return new IOException(getURLExceptionMessage(
 156             "Too many redirects at URL: %s with redirect=true", urlString, null));
 157       case MALFORMED_REPLY:
 158         return new IOException(getURLExceptionMessage(
 159             "Malformed HTTP reply received from server at URL: %s", urlString, errorDetail));
 160       case RESPONSE_TOO_LARGE:
 161         return new ResponseTooLargeException(urlString);
 162       case DNS_ERROR:
 163         return new UnknownHostException(
 164             getURLExceptionMessage("DNS host lookup failed for URL: %s", urlString, null));
 165       case FETCH_ERROR:
 166         return new IOException(
 167             getURLExceptionMessage("Could not fetch URL: %s", urlString, errorDetail));
 168       case INTERNAL_TRANSIENT_ERROR:
 169         return new InternalTransientException(urlString);
 170       case DEADLINE_EXCEEDED:
 171         return new SocketTimeoutException(
 172             getURLExceptionMessage("Timeout while fetching URL: %s", urlString, null));
 173       case SSL_CERTIFICATE_ERROR:
 174         return new SSLHandshakeException(getURLExceptionMessage(
 175             "Could not verify SSL certificate for URL: %s", urlString, null));
 176       case UNSPECIFIED_ERROR:
 177       default:
 178         return new IOException(ex.getErrorDetail());
 179     }
 180   }
 181
 182   private URLFetchRequest convertToPb(HTTPRequest request) {
 183     URLFetchRequest.Builder requestProto = URLFetchRequest.newBuilder();
 184     requestProto.setUrl(request.getURL().toExternalForm());
 185
 186     byte[] payload = request.getPayload();
 187     if (payload != null) {
 188       requestProto.setPayload(ByteString.copyFrom(payload));
 189     }
 190
 191     switch (request.getMethod()) {
 192       case GET:
 193         requestProto.setMethod(RequestMethod.GET);
 194         break;
 195       case POST:
 196         requestProto.setMethod(RequestMethod.POST);
 197         break;
 198       case HEAD:
 199         requestProto.setMethod(RequestMethod.HEAD);
 200         break;
 201       case PUT:
 202         requestProto.setMethod(RequestMethod.PUT);
 203         break;
 204       case DELETE:
 205         requestProto.setMethod(RequestMethod.DELETE);
 206         break;
 207       case PATCH:
 208         requestProto.setMethod(RequestMethod.PATCH);
 209         break;
 210       default:
 211         throw new IllegalArgumentException("unknown method: " + request.getMethod());
 212     }
 213
 214     for (HTTPHeader header : request.getHeaders()) {
 215       URLFetchRequest.Header.Builder headerProto = URLFetchRequest.Header.newBuilder();
 216       headerProto.setKey(header.getName());
 217       headerProto.setValue(header.getValue());
 218       requestProto.addHeader(headerProto);
 219     }
 220
 221     requestProto.setFollowRedirects(request.getFetchOptions().getFollowRedirects());
 222
 223     switch (request.getFetchOptions().getCertificateValidationBehavior()) {
 224       case VALIDATE:
 225         requestProto.setMustValidateServerCertificate(true);
 226         break;
 227       case DO_NOT_VALIDATE:
 228         requestProto.setMustValidateServerCertificate(false);
 229         break;
 230       default:
 231     }
 232
 233     return requestProto.build();
 234   }
 235
 236   private HTTPResponse convertFromPb(URLFetchResponse responseProto) {
 237     HTTPResponse response = new HTTPResponse(responseProto.getStatusCode());
 238     if (responseProto.hasContent()) {
 239       response.setContent(responseProto.getContent().toByteArray());
 240     }
 241
 242     for (Header header : responseProto.getHeaderList()) {
 243       response.addHeader(header.getKey(), header.getValue());
 244     }
 245
 246     if (responseProto.hasFinalUrl() && responseProto.getFinalUrl().length() > 0) {
 247       try {
 248         response.setFinalUrl(new URL(responseProto.getFinalUrl()));
 249       } catch (MalformedURLException e) {
 250         logger.severe("malformed final URL: " + e);
 251       }
 252     }
 253
 254     return response;
 255   }
 256 }