java/src/main/com/google/appengine/api/urlfetch/URLFetchServiceImpl.java

   1 // Copyright 2007 Google Inc. All rights reserved.
   2
   3 package com.google.appengine.api.urlfetch;
   4
   5 import com.google.apphosting.api.ApiProxy;
   6 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchRequest;
   7 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchRequest.RequestMethod;
   8 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchResponse.Header;
   9 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchResponse;
  10 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchServiceError.ErrorCode;
  11 import com.google.appengine.api.utils.FutureWrapper;
  12 import com.google.protobuf.ByteString;
  13
  14 import java.io.IOException;
  15 import java.net.MalformedURLException;
  16 import java.net.SocketTimeoutException;
  17 import java.net.UnknownHostException;
  18 import java.net.URL;
  19 import java.util.concurrent.Future;
  20 import java.util.logging.Logger;
  21
  22 import javax.net.ssl.SSLHandshakeException;
  23
  24 class URLFetchServiceImpl implements URLFetchService {
  25   static final String PACKAGE = "urlfetch";
  26
  27   private static final Logger logger = Logger.getLogger(URLFetchServiceImpl.class.getName());
  28
  29   public HTTPResponse fetch(URL url) throws IOException {
  30     return fetch(new HTTPRequest(url));
  31   }
  32
  33   public HTTPResponse fetch(HTTPRequest request) throws IOException {
  34     URLFetchRequest requestProto = convertToPb(request);
  35
  36     byte[] responseBytes;
  37     try {
  38       responseBytes = ApiProxy.makeSyncCall(
  39           PACKAGE, "Fetch",
  40           requestProto.toByteArray(),
  41           createApiConfig(request.getFetchOptions()));
  42     } catch (ApiProxy.ApplicationException ex) {
  43       Throwable cause = convertApplicationException(requestProto, ex);
  44       if (cause instanceof RuntimeException) {
  45         throw (RuntimeException) cause;
  46       } else if (cause instanceof IOException) {
  47         throw (IOException) cause;
  48       } else {
  49         throw new RuntimeException(cause);
  50       }
  51     } catch (ApiProxy.ApiDeadlineExceededException ex) {
  52       throw new SocketTimeoutException("Timeout while fetching: " + requestProto.getUrl());
  53     }
  54
  55     URLFetchResponse responseProto = URLFetchResponse.newBuilder().mergeFrom(responseBytes).build();
  56     if (!request.getFetchOptions().getAllowTruncate() && responseProto.getContentWasTruncated()) {
  57       throw new ResponseTooLargeException(request.getURL().toString());
  58     }
  59     return convertFromPb(responseProto);
  60   }
  61
  62   public Future<HTTPResponse> fetchAsync(URL url) {
  63     return fetchAsync(new HTTPRequest(url));
  64   }
  65
  66   public Future<HTTPResponse> fetchAsync(final HTTPRequest request) {
  67     final URLFetchRequest requestProto = convertToPb(request);
  68
  69     Future<byte[]> response = ApiProxy.makeAsyncCall(
  70         PACKAGE, "Fetch",
  71         requestProto.toByteArray(),
  72         createApiConfig(request.getFetchOptions()));
  73     return new FutureWrapper<byte[], HTTPResponse>(response) {
  74       @Override
  75       protected HTTPResponse wrap(byte[] responseBytes) throws IOException {
  76         URLFetchResponse responseProto =
  77             URLFetchResponse.newBuilder()
  78             .mergeFrom(responseBytes)
  79             .build();
  80         if (!request.getFetchOptions().getAllowTruncate() &&
  81             responseProto.getContentWasTruncated()) {
  82           throw new ResponseTooLargeException(request.getURL().toString());
  83         }
  84         return convertFromPb(responseProto);
  85       }
  86
  87       @Override
  88       protected Throwable convertException(Throwable cause) {
  89         if (cause instanceof ApiProxy.ApplicationException) {
  90           return convertApplicationException(requestProto, (ApiProxy.ApplicationException) cause);
  91         } else if (cause instanceof ApiProxy.ApiDeadlineExceededException) {
  92           return new SocketTimeoutException("Timeout while fetching: " + requestProto.getUrl());
  93         }
  94         return cause;
  95       }
  96     };
  97   }
  98
  99   private ApiProxy.ApiConfig createApiConfig(FetchOptions options) {
 100     ApiProxy.ApiConfig apiConfig = new ApiProxy.ApiConfig();
 101     apiConfig.setDeadlineInSeconds(options.getDeadline());
 102     return apiConfig;
 103   }
 104
 105   private String getURLExceptionMessage(String formatString, String url, String errorDetail) {
 106     if (errorDetail == null || errorDetail.trim().equals("")) {
 107       return String.format(formatString, url);
 108     } else {
 109       return String.format(formatString + ", error: %s", url, errorDetail);
 110     }
 111   }
 112
 113   private Throwable convertApplicationException(URLFetchRequest request,
 114                                                 ApiProxy.ApplicationException ex) {
 115     ErrorCode errorCode = ErrorCode.valueOf(ex.getApplicationError());
 116     String errorDetail = ex.getErrorDetail();
 117     switch (errorCode) {
 118       case INVALID_URL:
 119         return new MalformedURLException(
 120             getURLExceptionMessage("Invalid URL specified: %s", request.getUrl(), errorDetail));
 121       case CLOSED:
 122         return new IOException(getURLExceptionMessage(
 123             "Connection closed unexpectedly by server at URL: %s", request.getUrl(), null));
 124       case TOO_MANY_REDIRECTS:
 125         return new IOException(getURLExceptionMessage(
 126             "Too many redirects at URL: %s with redirect=true", request.getUrl(), null));
 127       case MALFORMED_REPLY:
 128         return new IOException(getURLExceptionMessage(
 129             "Malformed HTTP reply received from server at URL: %s", request.getUrl(), errorDetail));
 130       case RESPONSE_TOO_LARGE:
 131         return new ResponseTooLargeException(request.getUrl());
 132       case DNS_ERROR:
 133         return new UnknownHostException(getURLExceptionMessage(
 134             "DNS host lookup failed for URL: %s", request.getUrl(), null));
 135       case FETCH_ERROR:
 136         return new IOException(getURLExceptionMessage(
 137             "Could not fetch URL: %s", request.getUrl(), null));
 138       case INTERNAL_TRANSIENT_ERROR:
 139         return new InternalTransientException(request.getUrl());
 140       case DEADLINE_EXCEEDED:
 141         return new SocketTimeoutException(getURLExceptionMessage(
 142             "Timeout while fetching URL: %s", request.getUrl(), null));
 143       case SSL_CERTIFICATE_ERROR:
 144         return new SSLHandshakeException(getURLExceptionMessage(
 145             "Could not verify SSL certificate for URL: %s", request.getUrl(), null));
 146       case UNSPECIFIED_ERROR:
 147       default:
 148         return new IOException(ex.getErrorDetail());
 149     }
 150   }
 151
 152   private URLFetchRequest convertToPb(HTTPRequest request) {
 153     URLFetchRequest.Builder requestProto = URLFetchRequest.newBuilder();
 154     requestProto.setUrl(request.getURL().toExternalForm());
 155
 156     byte[] payload = request.getPayload();
 157     if (payload != null) {
 158       requestProto.setPayload(ByteString.copyFrom(payload));
 159     }
 160
 161     switch(request.getMethod()) {
 162       case GET:
 163         requestProto.setMethod(RequestMethod.GET);
 164         break;
 165       case POST:
 166         requestProto.setMethod(RequestMethod.POST);
 167         break;
 168       case HEAD:
 169         requestProto.setMethod(RequestMethod.HEAD);
 170         break;
 171       case PUT:
 172         requestProto.setMethod(RequestMethod.PUT);
 173         break;
 174       case DELETE:
 175         requestProto.setMethod(RequestMethod.DELETE);
 176         break;
 177       default:
 178         throw new IllegalArgumentException("unknown method: " + request.getMethod());
 179     }
 180
 181     for (HTTPHeader header : request.getHeaders()) {
 182       URLFetchRequest.Header.Builder headerProto = URLFetchRequest.Header.newBuilder();
 183       headerProto.setKey(header.getName());
 184       headerProto.setValue(header.getValue());
 185       requestProto.addHeader(headerProto);
 186     }
 187
 188     requestProto.setFollowRedirects(
 189         request.getFetchOptions().getFollowRedirects());
 190
 191     switch (request.getFetchOptions().getCertificateValidationBehavior()) {
 192       case VALIDATE:
 193         requestProto.setMustValidateServerCertificate(true);
 194         break;
 195       case DO_NOT_VALIDATE:
 196         requestProto.setMustValidateServerCertificate(false);
 197         break;
 198       default:
 199     }
 200
 201     return requestProto.build();
 202   }
 203
 204   private HTTPResponse convertFromPb(URLFetchResponse responseProto) {
 205     HTTPResponse response = new HTTPResponse(responseProto.getStatusCode());
 206     if (responseProto.hasContent()) {
 207       response.setContent(responseProto.getContent().toByteArray());
 208     }
 209
 210     for (Header header : responseProto.getHeaderList()) {
 211       response.addHeader(header.getKey(), header.getValue());
 212     }
 213
 214     if (responseProto.hasFinalUrl() &&
 215         responseProto.getFinalUrl().length() > 0) {
 216       try {
 217         response.setFinalUrl(new URL(responseProto.getFinalUrl()));
 218       } catch (MalformedURLException e) {
 219         logger.severe("malformed final URL: " + e);
 220       }
 221     }
 222
 223     return response;
 224   }
 225 }