java/src/main/com/google/appengine/api/urlfetch/URLFetchServiceImpl.java

   1 // Copyright 2007 Google Inc. All rights reserved.
   2
   3 package com.google.appengine.api.urlfetch;
   4
   5 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchRequest;
   6 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchRequest.RequestMethod;
   7 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchResponse;
   8 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchResponse.Header;
   9 import com.google.appengine.api.urlfetch.URLFetchServicePb.URLFetchServiceError.ErrorCode;
  10 import com.google.appengine.api.utils.FutureWrapper;
  11 import com.google.apphosting.api.ApiProxy;
  12 import com.google.protobuf.ByteString;
  13
  14 import java.io.IOException;
  15 import java.net.MalformedURLException;
  16 import java.net.SocketTimeoutException;
  17 import java.net.URL;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.List;
  21 import java.util.concurrent.Future;
  22 import java.util.logging.Logger;
  23
  24 import javax.net.ssl.SSLHandshakeException;
  25
  26 class URLFetchServiceImpl implements URLFetchService {
  27   static final String PACKAGE = "urlfetch";
  28
  29   private static final Logger logger = Logger.getLogger(URLFetchServiceImpl.class.getName());
  30
  31   @Override
  32   public HTTPResponse fetch(URL url) throws IOException {
  33     return fetch(new HTTPRequest(url));
  34   }
  35
  36   @Override
  37   public HTTPResponse fetch(HTTPRequest request) throws IOException {
  38     byte[] responseBytes;
  39     try {
  40       responseBytes = ApiProxy.makeSyncCall(PACKAGE, "Fetch", convertToPb(request).toByteArray(),
  41           createApiConfig(request.getFetchOptions()));
  42     } catch (ApiProxy.RequestTooLargeException ex) {
  43       throw new IOException("The request exceeded the maximum permissible size");
  44     } catch (ApiProxy.ApplicationException ex) {
  45       Throwable cause = convertApplicationException(request.getURL(), ex);
  46       if (cause instanceof RuntimeException) {
  47         throw (RuntimeException) cause;
  48       } else if (cause instanceof IOException) {
  49         throw (IOException) cause;
  50       } else {
  51         throw new RuntimeException(cause);
  52       }
  53     } catch (ApiProxy.ApiDeadlineExceededException ex) {
  54       throw new SocketTimeoutException("Timeout while fetching: " + request.getURL());
  55     }
  56
  57     URLFetchResponse responseProto = URLFetchResponse.newBuilder().mergeFrom(responseBytes).build();
  58     if (!request.getFetchOptions().getAllowTruncate() && responseProto.getContentWasTruncated()) {
  59       throw new ResponseTooLargeException(request.getURL().toString());
  60     }
  61     return convertFromPb(responseProto);
  62   }
  63
  64   @Override
  65   public Future<HTTPResponse> fetchAsync(URL url) {
  66     return fetchAsync(new HTTPRequest(url));
  67   }
  68
  69   @Override
  70   public Future<HTTPResponse> fetchAsync(HTTPRequest request) {
  71     final FetchOptions fetchOptions = request.getFetchOptions();
  72     final URL url = request.getURL();
  73
  74     Future<byte[]> response = ApiProxy.makeAsyncCall(
  75         PACKAGE, "Fetch", convertToPb(request).toByteArray(), createApiConfig(fetchOptions));
  76
  77     return new FutureWrapper<byte[], HTTPResponse>(response) {
  78       @Override
  79       protected HTTPResponse wrap(byte[] responseBytes) throws IOException {
  80         URLFetchResponse responseProto =
  81             URLFetchResponse.newBuilder().mergeFrom(responseBytes).build();
  82         if (!fetchOptions.getAllowTruncate() && responseProto.getContentWasTruncated()) {
  83           throw new ResponseTooLargeException(url.toString());
  84         }
  85         return convertFromPb(responseProto);
  86       }
  87
  88       @Override
  89       protected Throwable convertException(Throwable cause) {
  90         if (cause instanceof ApiProxy.ApplicationException) {
  91           return convertApplicationException(url, (ApiProxy.ApplicationException) cause);
  92         } else if (cause instanceof ApiProxy.ApiDeadlineExceededException) {
  93           return new SocketTimeoutException("Timeout while fetching: " + url);
  94         }
  95         return cause;
  96       }
  97     };
  98   }
  99
 100   static class DeadlineParser {
 101     static final DeadlineParser INSTANCE = new DeadlineParser();
 102     volatile int deadlineMs = -1;
 103
 104     private DeadlineParser() {
 105       refresh();
 106     }
 107
 108     void refresh() {
 109       String globalDefault = System.getProperty(URLFetchService.DEFAULT_DEADLINE_PROPERTY);
 110       if (globalDefault != null) {
 111         try {
 112           deadlineMs = (int) (Double.parseDouble(globalDefault) * 1000);
 113         } catch (NumberFormatException e) {
 114           deadlineMs = -1;
 115           logger.warning("Cannot parse deadline: " + globalDefault);
 116         }
 117       } else {
 118         deadlineMs = -1;
 119       }
 120     }
 121   }
 122
 123   private ApiProxy.ApiConfig createApiConfig(FetchOptions options) {
 124     ApiProxy.ApiConfig apiConfig = new ApiProxy.ApiConfig();
 125
 126     if (options.getDeadline() != null) {
 127       apiConfig.setDeadlineInSeconds(options.getDeadline());
 128     } else if (DeadlineParser.INSTANCE.deadlineMs >= 0) {
 129       apiConfig.setDeadlineInSeconds(DeadlineParser.INSTANCE.deadlineMs / 1000.0);
 130     }
 131
 132     return apiConfig;
 133   }
 134
 135   private String getURLExceptionMessage(String formatString, String url, String errorDetail) {
 136     if (errorDetail == null || errorDetail.trim().isEmpty()) {
 137       return String.format(formatString, url);
 138     }
 139     return String.format(formatString + ", error: %s", url, errorDetail);
 140   }
 141
 142   private Throwable convertApplicationException(URL requestUrl, ApiProxy.ApplicationException ex) {
 143     String urlString = requestUrl.toString();
 144     ErrorCode errorCode = ErrorCode.valueOf(ex.getApplicationError());
 145     String errorDetail = ex.getErrorDetail();
 146     switch (errorCode) {
 147       case INVALID_URL:
 148         return new MalformedURLException(
 149             getURLExceptionMessage("Invalid URL specified: %s", urlString, errorDetail));
 150       case PAYLOAD_TOO_LARGE:
 151         return new RequestPayloadTooLargeException(urlString);
 152       case CLOSED:
 153         return new IOException(getURLExceptionMessage(
 154             "Connection closed unexpectedly by server at URL: %s", urlString, null));
 155       case TOO_MANY_REDIRECTS:
 156         return new IOException(getURLExceptionMessage(
 157             "Too many redirects at URL: %s with redirect=true", urlString, null));
 158       case MALFORMED_REPLY:
 159         return new IOException(getURLExceptionMessage(
 160             "Malformed HTTP reply received from server at URL: %s", urlString, errorDetail));
 161       case RESPONSE_TOO_LARGE:
 162         return new ResponseTooLargeException(urlString);
 163       case DNS_ERROR:
 164         return new UnknownHostException(
 165             getURLExceptionMessage("DNS host lookup failed for URL: %s", urlString, null));
 166       case FETCH_ERROR:
 167         return new IOException(
 168             getURLExceptionMessage("Could not fetch URL: %s", urlString, errorDetail));
 169       case INTERNAL_TRANSIENT_ERROR:
 170         return new InternalTransientException(urlString);
 171       case DEADLINE_EXCEEDED:
 172         return new SocketTimeoutException(
 173             getURLExceptionMessage("Timeout while fetching URL: %s", urlString, null));
 174       case SSL_CERTIFICATE_ERROR:
 175         return new SSLHandshakeException(getURLExceptionMessage(
 176             "Could not verify SSL certificate for URL: %s", urlString, null));
 177       case UNSPECIFIED_ERROR:
 178       default:
 179         return new IOException(ex.getErrorDetail());
 180     }
 181   }
 182
 183   private URLFetchRequest convertToPb(HTTPRequest request) {
 184     URLFetchRequest.Builder requestProto = URLFetchRequest.newBuilder();
 185     requestProto.setUrl(request.getURL().toExternalForm());
 186
 187     byte[] payload = request.getPayload();
 188     if (payload != null) {
 189       requestProto.setPayload(ByteString.copyFrom(payload));
 190     }
 191
 192     switch (request.getMethod()) {
 193       case GET:
 194         requestProto.setMethod(RequestMethod.GET);
 195         break;
 196       case POST:
 197         requestProto.setMethod(RequestMethod.POST);
 198         break;
 199       case HEAD:
 200         requestProto.setMethod(RequestMethod.HEAD);
 201         break;
 202       case PUT:
 203         requestProto.setMethod(RequestMethod.PUT);
 204         break;
 205       case DELETE:
 206         requestProto.setMethod(RequestMethod.DELETE);
 207         break;
 208       case PATCH:
 209         requestProto.setMethod(RequestMethod.PATCH);
 210         break;
 211       default:
 212         throw new IllegalArgumentException("unknown method: " + request.getMethod());
 213     }
 214
 215     for (HTTPHeader header : request.getHeaders()) {
 216       URLFetchRequest.Header.Builder headerProto = URLFetchRequest.Header.newBuilder();
 217       headerProto.setKey(header.getName());
 218       headerProto.setValue(header.getValue());
 219       requestProto.addHeader(headerProto);
 220     }
 221
 222     requestProto.setFollowRedirects(request.getFetchOptions().getFollowRedirects());
 223
 224     switch (request.getFetchOptions().getCertificateValidationBehavior()) {
 225       case VALIDATE:
 226         requestProto.setMustValidateServerCertificate(true);
 227         break;
 228       case DO_NOT_VALIDATE:
 229         requestProto.setMustValidateServerCertificate(false);
 230         break;
 231       default:
 232     }
 233
 234     return requestProto.build();
 235   }
 236
 237   private HTTPResponse convertFromPb(URLFetchResponse responseProto) {
 238
 239     byte[] content = responseProto.hasContent() ? responseProto.getContent().toByteArray() : null;
 240
 241     List<HTTPHeader> headers = new ArrayList<>(responseProto.getHeaderCount());
 242     for (Header header : responseProto.getHeaderList()) {
 243       headers.add(new HTTPHeader(header.getKey(), header.getValue()));
 244     }
 245
 246     URL finalURL = null;
 247     if (responseProto.hasFinalUrl() && responseProto.getFinalUrl().length() > 0) {
 248       try {
 249         finalURL = new URL(responseProto.getFinalUrl());
 250       } catch (MalformedURLException e) {
 251         logger.severe("malformed final URL: " + e);
 252       }
 253     }
 254
 255     return new HTTPResponse(responseProto.getStatusCode(), content, finalURL, headers);
 256   }
 257 }