1 // Copyright 2007 Google Inc. All rights reserved.
3 package com
.google
.appengine
.api
.urlfetch
;
5 import com
.google
.apphosting
.api
.ApiProxy
;
6 import com
.google
.appengine
.api
.urlfetch
.URLFetchServicePb
.URLFetchRequest
;
7 import com
.google
.appengine
.api
.urlfetch
.URLFetchServicePb
.URLFetchRequest
.RequestMethod
;
8 import com
.google
.appengine
.api
.urlfetch
.URLFetchServicePb
.URLFetchResponse
.Header
;
9 import com
.google
.appengine
.api
.urlfetch
.URLFetchServicePb
.URLFetchResponse
;
10 import com
.google
.appengine
.api
.urlfetch
.URLFetchServicePb
.URLFetchServiceError
.ErrorCode
;
11 import com
.google
.appengine
.api
.utils
.FutureWrapper
;
12 import com
.google
.protobuf
.ByteString
;
14 import java
.io
.IOException
;
15 import java
.net
.MalformedURLException
;
16 import java
.net
.SocketTimeoutException
;
17 import java
.net
.UnknownHostException
;
19 import java
.util
.concurrent
.Future
;
20 import java
.util
.logging
.Logger
;
22 import javax
.net
.ssl
.SSLHandshakeException
;
24 class URLFetchServiceImpl
implements URLFetchService
{
25 static final String PACKAGE
= "urlfetch";
27 private static final Logger logger
= Logger
.getLogger(URLFetchServiceImpl
.class.getName());
29 public HTTPResponse
fetch(URL url
) throws IOException
{
30 return fetch(new HTTPRequest(url
));
33 public HTTPResponse
fetch(HTTPRequest request
) throws IOException
{
34 URLFetchRequest requestProto
= convertToPb(request
);
38 responseBytes
= ApiProxy
.makeSyncCall(
40 requestProto
.toByteArray(),
41 createApiConfig(request
.getFetchOptions()));
42 } catch (ApiProxy
.ApplicationException ex
) {
43 Throwable cause
= convertApplicationException(requestProto
, ex
);
44 if (cause
instanceof RuntimeException
) {
45 throw (RuntimeException
) cause
;
46 } else if (cause
instanceof IOException
) {
47 throw (IOException
) cause
;
49 throw new RuntimeException(cause
);
51 } catch (ApiProxy
.ApiDeadlineExceededException ex
) {
52 throw new SocketTimeoutException("Timeout while fetching: " + requestProto
.getUrl());
55 URLFetchResponse responseProto
= URLFetchResponse
.newBuilder().mergeFrom(responseBytes
).build();
56 if (!request
.getFetchOptions().getAllowTruncate() && responseProto
.getContentWasTruncated()) {
57 throw new ResponseTooLargeException(request
.getURL().toString());
59 return convertFromPb(responseProto
);
62 public Future
<HTTPResponse
> fetchAsync(URL url
) {
63 return fetchAsync(new HTTPRequest(url
));
66 public Future
<HTTPResponse
> fetchAsync(final HTTPRequest request
) {
67 final URLFetchRequest requestProto
= convertToPb(request
);
69 Future
<byte[]> response
= ApiProxy
.makeAsyncCall(
71 requestProto
.toByteArray(),
72 createApiConfig(request
.getFetchOptions()));
73 return new FutureWrapper
<byte[], HTTPResponse
>(response
) {
75 protected HTTPResponse
wrap(byte[] responseBytes
) throws IOException
{
76 URLFetchResponse responseProto
=
77 URLFetchResponse
.newBuilder()
78 .mergeFrom(responseBytes
)
80 if (!request
.getFetchOptions().getAllowTruncate() &&
81 responseProto
.getContentWasTruncated()) {
82 throw new ResponseTooLargeException(request
.getURL().toString());
84 return convertFromPb(responseProto
);
88 protected Throwable
convertException(Throwable cause
) {
89 if (cause
instanceof ApiProxy
.ApplicationException
) {
90 return convertApplicationException(requestProto
, (ApiProxy
.ApplicationException
) cause
);
91 } else if (cause
instanceof ApiProxy
.ApiDeadlineExceededException
) {
92 return new SocketTimeoutException("Timeout while fetching: " + requestProto
.getUrl());
99 private ApiProxy
.ApiConfig
createApiConfig(FetchOptions options
) {
100 ApiProxy
.ApiConfig apiConfig
= new ApiProxy
.ApiConfig();
101 apiConfig
.setDeadlineInSeconds(options
.getDeadline());
105 private String
getURLExceptionMessage(String formatString
, String url
, String errorDetail
) {
106 if (errorDetail
== null || errorDetail
.trim().equals("")) {
107 return String
.format(formatString
, url
);
109 return String
.format(formatString
+ ", error: %s", url
, errorDetail
);
113 private Throwable
convertApplicationException(URLFetchRequest request
,
114 ApiProxy
.ApplicationException ex
) {
115 ErrorCode errorCode
= ErrorCode
.valueOf(ex
.getApplicationError());
116 String errorDetail
= ex
.getErrorDetail();
119 return new MalformedURLException(
120 getURLExceptionMessage("Invalid URL specified: %s", request
.getUrl(), errorDetail
));
122 return new IOException(getURLExceptionMessage(
123 "Connection closed unexpectedly by server at URL: %s", request
.getUrl(), null));
124 case TOO_MANY_REDIRECTS
:
125 return new IOException(getURLExceptionMessage(
126 "Too many redirects at URL: %s with redirect=true", request
.getUrl(), null));
127 case MALFORMED_REPLY
:
128 return new IOException(getURLExceptionMessage(
129 "Malformed HTTP reply received from server at URL: %s", request
.getUrl(), errorDetail
));
130 case RESPONSE_TOO_LARGE
:
131 return new ResponseTooLargeException(request
.getUrl());
133 return new UnknownHostException(getURLExceptionMessage(
134 "DNS host lookup failed for URL: %s", request
.getUrl(), null));
136 return new IOException(getURLExceptionMessage(
137 "Could not fetch URL: %s", request
.getUrl(), null));
138 case INTERNAL_TRANSIENT_ERROR
:
139 return new InternalTransientException(request
.getUrl());
140 case DEADLINE_EXCEEDED
:
141 return new SocketTimeoutException(getURLExceptionMessage(
142 "Timeout while fetching URL: %s", request
.getUrl(), null));
143 case SSL_CERTIFICATE_ERROR
:
144 return new SSLHandshakeException(getURLExceptionMessage(
145 "Could not verify SSL certificate for URL: %s", request
.getUrl(), null));
146 case UNSPECIFIED_ERROR
:
148 return new IOException(ex
.getErrorDetail());
152 private URLFetchRequest
convertToPb(HTTPRequest request
) {
153 URLFetchRequest
.Builder requestProto
= URLFetchRequest
.newBuilder();
154 requestProto
.setUrl(request
.getURL().toExternalForm());
156 byte[] payload
= request
.getPayload();
157 if (payload
!= null) {
158 requestProto
.setPayload(ByteString
.copyFrom(payload
));
161 switch(request
.getMethod()) {
163 requestProto
.setMethod(RequestMethod
.GET
);
166 requestProto
.setMethod(RequestMethod
.POST
);
169 requestProto
.setMethod(RequestMethod
.HEAD
);
172 requestProto
.setMethod(RequestMethod
.PUT
);
175 requestProto
.setMethod(RequestMethod
.DELETE
);
178 requestProto
.setMethod(RequestMethod
.PATCH
);
181 throw new IllegalArgumentException("unknown method: " + request
.getMethod());
184 for (HTTPHeader header
: request
.getHeaders()) {
185 URLFetchRequest
.Header
.Builder headerProto
= URLFetchRequest
.Header
.newBuilder();
186 headerProto
.setKey(header
.getName());
187 headerProto
.setValue(header
.getValue());
188 requestProto
.addHeader(headerProto
);
191 requestProto
.setFollowRedirects(
192 request
.getFetchOptions().getFollowRedirects());
194 switch (request
.getFetchOptions().getCertificateValidationBehavior()) {
196 requestProto
.setMustValidateServerCertificate(true);
198 case DO_NOT_VALIDATE
:
199 requestProto
.setMustValidateServerCertificate(false);
204 return requestProto
.build();
207 private HTTPResponse
convertFromPb(URLFetchResponse responseProto
) {
208 HTTPResponse response
= new HTTPResponse(responseProto
.getStatusCode());
209 if (responseProto
.hasContent()) {
210 response
.setContent(responseProto
.getContent().toByteArray());
213 for (Header header
: responseProto
.getHeaderList()) {
214 response
.addHeader(header
.getKey(), header
.getValue());
217 if (responseProto
.hasFinalUrl() &&
218 responseProto
.getFinalUrl().length() > 0) {
220 response
.setFinalUrl(new URL(responseProto
.getFinalUrl()));
221 } catch (MalformedURLException e
) {
222 logger
.severe("malformed final URL: " + e
);