java/src/main/com/google/appengine/tools/admin/AbstractServerConnection.java

   1 // Copyright 2009 Google Inc. All rights reserved.
   2
   3 package com.google.appengine.tools.admin;
   4
   5 import com.google.appengine.tools.admin.AppAdminFactory.ConnectOptions;
   6 import com.google.common.collect.Multimap;
   7 import com.google.common.collect.Multimaps;
   8
   9 import java.io.BufferedReader;
  10 import java.io.File;
  11 import java.io.FileInputStream;
  12 import java.io.IOException;
  13 import java.io.InputStream;
  14 import java.io.InputStreamReader;
  15 import java.io.OutputStream;
  16 import java.io.OutputStreamWriter;
  17 import java.io.PrintWriter;
  18 import java.io.StringReader;
  19 import java.io.UnsupportedEncodingException;
  20 import java.net.HttpURLConnection;
  21 import java.net.MalformedURLException;
  22 import java.net.URL;
  23 import java.net.URLEncoder;
  24 import java.util.HashMap;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.logging.Logger;
  28
  29 /**
  30  * Connection to the AppEngine hosting service, as set by {@link ConnectOptions}
  31  *
  32  */
  33 public abstract class AbstractServerConnection implements ServerConnection {
  34
  35   private static final int MAX_SEND_RETRIES = 3;
  36   private static final String USER_AGENT_HEADER = "X-appcfg-user-agent";
  37   private static final String USER_AGENT_KEY = "appengine.useragent";
  38
  39   protected interface DataPoster {
  40     void post(OutputStream s) throws IOException;
  41   }
  42
  43   private static class FilePoster implements DataPoster {
  44     private static final int BUFFER_SIZE = 4 * 1024;
  45     private File file;
  46
  47     public FilePoster(File file) {
  48       assert (file != null && file.exists());
  49       this.file = file;
  50     }
  51
  52     @Override
  53     public void post(OutputStream out) throws IOException {
  54       InputStream in = new FileInputStream(file);
  55       try {
  56         byte[] buf = new byte[BUFFER_SIZE];
  57         int len;
  58         while ((len = in.read(buf)) != -1) {
  59           out.write(buf, 0, len);
  60         }
  61       } finally {
  62         in.close();
  63       }
  64     }
  65   }
  66
  67   static class StringPoster implements DataPoster {
  68     private String str;
  69
  70     public StringPoster(String s) {
  71       str = s;
  72     }
  73
  74     @Override
  75     public void post(OutputStream s) throws IOException {
  76       s.write(str.getBytes("UTF-8"));
  77     }
  78   }
  79
  80   protected static final String POST = "POST";
  81
  82   protected static final String GET = "GET";
  83
  84   protected ConnectOptions options;
  85
  86   protected static Logger logger =
  87       Logger.getLogger(AbstractServerConnection.class.getCanonicalName());
  88
  89   protected AbstractServerConnection() {
  90   }
  91
  92   protected AbstractServerConnection(ConnectOptions options) {
  93     this.options = options;
  94     if (System.getProperty("http.proxyHost") != null) {
  95       logger.info("proxying HTTP through " + System.getProperty("http.proxyHost") + ":"
  96           + System.getProperty("http.proxyPort"));
  97     }
  98     if (System.getProperty("https.proxyHost") != null) {
  99       logger.info("proxying HTTPS through " + System.getProperty("https.proxyHost") + ":"
 100           + System.getProperty("https.proxyPort"));
 101     }
 102   }
 103
 104   protected String buildQuery(Map<String, String> params) throws UnsupportedEncodingException {
 105     return buildQuery(Multimaps.forMap(params));
 106   }
 107
 108   protected String buildQuery(Multimap<String, String> params) throws UnsupportedEncodingException {
 109     StringBuffer buf = new StringBuffer();
 110     for (String key : params.keySet()) {
 111       String encodedKey = URLEncoder.encode(key, "UTF-8");
 112       for (String value : params.get(key)) {
 113         buf.append(encodedKey);
 114         buf.append('=');
 115         buf.append(URLEncoder.encode(value, "UTF-8"));
 116         buf.append('&');
 117       }
 118     }
 119     return buf.toString();
 120   }
 121
 122   protected URL buildURL(String path) throws MalformedURLException {
 123     String protocol = options.getSecure() ? "https" : "http";
 124     return new URL(protocol + "://" + options.getServer() + path);
 125   }
 126
 127   protected IOException connect(String method, HttpURLConnection conn, DataPoster data)
 128       throws IOException {
 129     doPreConnect(method, conn, data);
 130     conn.setInstanceFollowRedirects(false);
 131     conn.setRequestMethod(method);
 132
 133     if (POST.equals(method)) {
 134       conn.setDoOutput(true);
 135       OutputStream out = conn.getOutputStream();
 136       if (data != null) {
 137         data.post(out);
 138       }
 139       out.close();
 140     }
 141
 142     try {
 143       conn.getInputStream();
 144     } catch (IOException ex) {
 145       return ex;
 146     } finally {
 147       doPostConnect(method, conn, data);
 148     }
 149
 150     return null;
 151   }
 152
 153   protected String constructHttpErrorMessage(HttpURLConnection conn, BufferedReader reader)
 154       throws IOException {
 155     StringBuilder sb = new StringBuilder("Error posting to URL: ");
 156     sb.append(conn.getURL());
 157     sb.append('\n');
 158     sb.append(conn.getResponseCode());
 159     sb.append(' ');
 160     sb.append(conn.getResponseMessage());
 161     sb.append('\n');
 162     if (reader != null) {
 163       for (String line; (line = reader.readLine()) != null;) {
 164         sb.append(line);
 165         sb.append('\n');
 166       }
 167     }
 168     return sb.toString();
 169   }
 170
 171   protected abstract void doHandleSendErrors(int status, URL url, HttpURLConnection conn,
 172       BufferedReader connReader) throws IOException;
 173
 174   protected abstract void doPostConnect(String method, HttpURLConnection conn, DataPoster data)
 175       throws IOException;
 176
 177   protected abstract void doPreConnect(String method, HttpURLConnection conn, DataPoster data)
 178       throws IOException;
 179
 180   @Override
 181   public String get(String url, Map<String, String> params) throws IOException {
 182     return send(GET, url, null, null, Multimaps.forMap(params));
 183   }
 184
 185   @Override
 186   public String get(String url, Multimap<String, String> params) throws IOException {
 187     return send(GET, url, null, null, params);
 188   }
 189
 190   private static InputStream getInputStream(HttpURLConnection conn) {
 191     InputStream is;
 192     try {
 193       return conn.getInputStream();
 194     } catch (IOException ex) {
 195       return conn.getErrorStream();
 196     }
 197   }
 198
 199   private static BufferedReader getReader(InputStream is) {
 200     if (is == null) {
 201       return null;
 202     }
 203     return new BufferedReader(new InputStreamReader(is));
 204   }
 205
 206   protected static String getString(InputStream is) throws IOException {
 207     StringBuffer response = new StringBuffer();
 208     InputStreamReader reader = new InputStreamReader(is);
 209     char[] buffer = new char[8192];
 210     int read = 0;
 211     while ((read = reader.read(buffer)) != -1) {
 212       response.append(buffer, 0, read);
 213     }
 214     return response.toString();
 215   }
 216
 217   protected static BufferedReader getReader(HttpURLConnection conn) {
 218     return getReader(getInputStream(conn));
 219   }
 220
 221   @Override
 222   public String post(String url, File payload, String contentType, Map<String, String> params)
 223       throws IOException {
 224     return send(POST, url, new FilePoster(payload), contentType, Multimaps.forMap(params));
 225   }
 226
 227   @Override
 228   public String post(String url, File payload, String contentType, Multimap<String, String> params)
 229       throws IOException {
 230     return send(POST, url, new FilePoster(payload), contentType, params);
 231   }
 232
 233   protected InputStream postAndGetInputStream(String url, DataPoster poster, String... params)
 234       throws IOException {
 235     return send1(POST, url, poster, null, Multimaps.forMap(getParamMap(params)));
 236   }
 237
 238   @Override
 239   public String post(String url, File payload, String contentType, String... params)
 240       throws IOException {
 241     return send(POST, url, new FilePoster(payload), contentType,
 242         Multimaps.forMap(getParamMap(params)));
 243   }
 244
 245   @Override
 246   public String post(String url, List<AppVersionUpload.FileInfo> payload, String... params)
 247       throws IOException {
 248     return sendBatch(POST, url, payload, getParamMap(params));
 249   }
 250
 251   @Override
 252   public String post(String url, String payload, Map<String, String> params)
 253       throws IOException {
 254     return send(POST, url, new StringPoster(payload), null, Multimaps.forMap(params));
 255   }
 256
 257   @Override
 258   public String post(String url, String payload, Multimap<String, String> params)
 259       throws IOException {
 260     return send(POST, url, new StringPoster(payload), null, params);
 261   }
 262
 263   @Override
 264   public String post(String url, String payload, String... params) throws IOException {
 265     return send(POST, url, new StringPoster(payload), null, Multimaps.forMap(getParamMap(params)));
 266   }
 267
 268   /**
 269    * Returns an input stream that reads from this open connection. If the server returned
 270    * an error (404, etc), return the error stream instead.
 271    *
 272    * @return {@code InputStream} for the server response.
 273    */
 274   @Override
 275   public InputStream postAndGetInputStream(String url, String payload, String... params)
 276     throws IOException {
 277     return send1(POST, url, new StringPoster(payload), null, Multimaps.forMap(getParamMap(params)));
 278   }
 279
 280   protected HttpURLConnection openConnection(URL url) throws IOException {
 281     return (HttpURLConnection) url.openConnection();
 282   }
 283
 284   protected String send(String method, String path, DataPoster payload, String content_type,
 285                         Multimap<String, String> params) throws IOException {
 286     return getString(send1(method, path, payload, content_type, params));
 287   }
 288
 289   private Map<String, String> getParamMap(String... params) {
 290     Map<String, String> paramMap = new HashMap<String, String>();
 291     for (int i = 0; i < params.length; i += 2) {
 292       paramMap.put(params[i], params[i + 1]);
 293     }
 294     return paramMap;
 295   }
 296
 297   private InputStream send1(String method, String path, DataPoster payload, String content_type,
 298       Multimap<String, String> params) throws IOException {
 299
 300     URL url = buildURL(path + '?' + buildQuery(params));
 301
 302     if (content_type == null) {
 303       content_type = Application.guessContentTypeFromName(path);
 304     }
 305
 306     int tries = 0;
 307     while (true) {
 308       HttpURLConnection conn = openConnection(url);
 309       conn.setRequestProperty("Content-type", content_type);
 310       conn.setRequestProperty("X-appcfg-api-version", "1");
 311
 312       String userAgentValue = System.getProperty(USER_AGENT_KEY);
 313       if (userAgentValue != null) {
 314         conn.setRequestProperty(USER_AGENT_HEADER, userAgentValue);
 315       }
 316
 317       if (options.getHost() != null) {
 318         conn.setRequestProperty("Host", options.getHost());
 319       }
 320
 321       IOException ioe = connect(method, conn, payload);
 322
 323       int status = conn.getResponseCode();
 324
 325       if (status == HttpURLConnection.HTTP_OK) {
 326         return getInputStream(conn);
 327       } else {
 328         BufferedReader reader = getReader(conn);
 329         StringBuilder sb = new StringBuilder();
 330         for (String line; (line = reader.readLine()) != null;) {
 331           sb.append(line);
 332         }
 333         String response = sb.toString();
 334         String httpErrorMessage = constructHttpErrorMessage(
 335             conn,
 336             new BufferedReader(new StringReader(response)));
 337         logger.warning(httpErrorMessage + "This is try #" + tries);
 338         if (++tries > MAX_SEND_RETRIES) {
 339           throw new HttpIoException(httpErrorMessage, status);
 340         }
 341         doHandleSendErrors(status, url, conn, new BufferedReader(new StringReader(response)));
 342       }
 343     }
 344   }
 345
 346   protected String sendBatch(String method, String path, List<AppVersionUpload.FileInfo> payload,
 347       Map<String, String> params) throws IOException {
 348     return getString(sendBatchPayload(method, path, payload, params));
 349   }
 350
 351   private InputStream sendBatchPayload(String method, String path,
 352       List<AppVersionUpload.FileInfo> payload,
 353       Map<String, String> params) throws IOException {
 354
 355     URL url = buildURL(path + '?' + buildQuery(params));
 356
 357     int tries = 0;
 358     while (true) {
 359       String boundary = "boundary" + Long
 360           .toHexString(System.currentTimeMillis());
 361       HttpURLConnection conn = openConnection(url);
 362       conn.setRequestProperty("MIME-Version", "1.0");
 363       conn.setRequestProperty("Content-Type", "message/rfc822");
 364       conn.setRequestProperty("X-appcfg-api-version", "1");
 365       doPreConnect(method, conn, null);
 366       conn.setDoOutput(true);
 367       conn.setRequestMethod(method);
 368
 369       if (options.getHost() != null) {
 370         conn.setRequestProperty("Host", options.getHost());
 371       }
 372       conn.setInstanceFollowRedirects(false);
 373
 374       populateMixedPayloadStream(conn.getOutputStream(), payload, boundary);
 375
 376       doPostConnect(method, conn, null);
 377
 378       int status = conn.getResponseCode();
 379       if (status == HttpURLConnection.HTTP_OK) {
 380         return getInputStream(conn);
 381       } else {
 382         BufferedReader reader = getReader(conn);
 383         StringBuilder sb = new StringBuilder();
 384         for (String line; (line = reader.readLine()) != null;) {
 385           sb.append(line);
 386         }
 387         String response = sb.toString();
 388         String httpErrorMessage = constructHttpErrorMessage(
 389             conn,
 390             new BufferedReader(new StringReader(response)));
 391         logger.warning(httpErrorMessage + "This is try #" + tries);
 392         if (++tries > MAX_SEND_RETRIES) {
 393           throw new IOException(httpErrorMessage);
 394         }
 395         doHandleSendErrors(status, url, conn, new BufferedReader(new StringReader(response)));
 396       }
 397
 398     }
 399   }
 400
 401   void populateMixedPayloadStream(OutputStream output,
 402       List<AppVersionUpload.FileInfo> payload, String boundary) throws IOException {
 403
 404     String charset = "UTF-8";
 405     String lf = "\n";
 406     PrintWriter writer = null;
 407     try {
 408       writer = new PrintWriter(new OutputStreamWriter(output, charset),
 409           true);
 410       writer.append("Content-Type: multipart/mixed; boundary=\"" + boundary + "\"").append(lf);
 411       writer.append("MIME-Version: 1").append(lf);
 412       writer.append(lf).append("This is a message with multiple parts in MIME format.").append(lf);
 413
 414       for (AppVersionUpload.FileInfo fileInfo : payload) {
 415
 416         File binaryFile = fileInfo.file;
 417         writer.append("--" + boundary).append(lf);
 418         writer.append("X-Appcfg-File: ").append(URLEncoder.encode(fileInfo.path, "UTF-8"))
 419             .append(lf);
 420         writer.append("X-Appcfg-Hash: ").append(fileInfo.hash).append(lf);
 421         String mimeValue = fileInfo.mimeType;
 422         if (mimeValue == null) {
 423           mimeValue = Application.guessContentTypeFromName(binaryFile.getName());
 424         }
 425         writer.append("Content-Type: " + mimeValue).append(lf);
 426         writer.append("Content-Length: ").append("" + binaryFile.length()).append(lf);
 427         writer.append("Content-Transfer-Encoding: 8bit").append(lf);
 428
 429         writer.append(lf).flush();
 430         FilePoster filePoster = new FilePoster(binaryFile);
 431         filePoster.post(output);
 432         writer.append(lf).flush();
 433       }
 434       writer.append("--" + boundary + "--").append(lf);
 435       writer.append(lf).flush();
 436
 437     } finally {
 438       if (writer != null) {
 439         writer.close();
 440       }
 441     }
 442   }
 443 }