java/src/main/com/google/appengine/tools/remoteapi/RemoteApiInstaller.java

   1 // Copyright 2010 Google Inc. All Rights Reserved.
   2
   3 package com.google.appengine.tools.remoteapi;
   4
   5 import com.google.appengine.api.users.dev.LoginCookieUtils;
   6 import com.google.apphosting.api.ApiProxy;
   7 import com.google.apphosting.api.ApiProxy.Delegate;
   8 import com.google.apphosting.api.ApiProxy.Environment;
   9 import com.google.apphosting.api.ApiProxy.EnvironmentFactory;
  10
  11 import org.apache.commons.httpclient.Cookie;
  12
  13 import java.io.IOException;
  14 import java.util.ArrayList;
  15 import java.util.Arrays;
  16 import java.util.Collections;
  17 import java.util.HashMap;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.logging.ConsoleHandler;
  21 import java.util.logging.Formatter;
  22 import java.util.logging.Level;
  23 import java.util.logging.LogRecord;
  24 import java.util.logging.Logger;
  25 import java.util.logging.StreamHandler;
  26 import java.util.regex.Matcher;
  27 import java.util.regex.Pattern;
  28
  29 /**
  30  * Installs and uninstalls the remote API. While the RemoteApi is installed,
  31  * all App Engine calls made by the same thread that performed the installation
  32  * will be sent to a remote server.
  33  *
  34  * <p>Instances of this class can only be used on a single thread.</p>
  35  *
  36  */
  37 public class RemoteApiInstaller {
  38   private static final Pattern PAIR_REGEXP =
  39       Pattern.compile("([a-z0-9_-]+): +(['\\\"]?)([:~.a-z0-9_-]+)\\2");
  40
  41   /**
  42    * A key that can be put into {@link Environment#getAttributes()} to override the app id used by
  43    * the Datastore API.  Note that this is copied from
  44    * com.google.appengine.api.datastore.DatastoreApiHelper to avoid a dependency on that class.  It
  45    * must be kept in sync.
  46    */
  47   static final String DATASTORE_APP_ID_OVERRIDE_KEY =
  48       "com.google.appengine.datastore.AppIdOverride";
  49
  50   private static ConsoleHandler remoteMethodHandler;
  51
  52   private static synchronized StreamHandler getStreamHandler() {
  53     if (remoteMethodHandler == null) {
  54       remoteMethodHandler = new ConsoleHandler();
  55       remoteMethodHandler.setFormatter(new Formatter() {
  56         @Override
  57         public String format(LogRecord record) {
  58           return record.getMessage() + "\n";
  59         }
  60       });
  61       remoteMethodHandler.setLevel(Level.FINE);
  62     }
  63     return remoteMethodHandler;
  64   }
  65
  66   private InstallerState installerState;
  67   private static boolean installedForAllThreads = false;
  68
  69   void validateOptions(RemoteApiOptions options) {
  70     if (options.getHostname() == null) {
  71       throw new IllegalArgumentException("server not set in options");
  72     }
  73     if (options.getUserEmail() == null
  74         && options.getOAuthCredential() == null) {
  75       throw new IllegalArgumentException("credentials not set in options");
  76     }
  77   }
  78
  79   private boolean installed() {
  80     return installerState != null || installedForAllThreads;
  81   }
  82
  83   /**
  84    * Installs the remote API on all threads using the provided options. Logs
  85    * into the remote application using the credentials available via these
  86    * options.
  87    *
  88    * <p>Note that if installed using this method, the remote API cannot be
  89    * uninstalled.</p>
  90    *
  91    * @throws IllegalArgumentException if the server or credentials weren't provided.
  92    * @throws IllegalStateException if already installed
  93    * @throws LoginException if unable to log in.
  94    * @throws IOException if unable to connect to the remote API.
  95    */
  96   public void installOnAllThreads(RemoteApiOptions options) throws IOException {
  97     final RemoteApiOptions finalOptions = options.copy();
  98     validateOptions(finalOptions);
  99
 100     synchronized (getClass()) {
 101       if (installed()) {
 102         throw new IllegalStateException("remote API is already installed");
 103       }
 104       installedForAllThreads = true;
 105
 106       final RemoteApiClient client = login(finalOptions);
 107       @SuppressWarnings("unchecked")
 108       Delegate<Environment> originalDelegate = ApiProxy.getDelegate();
 109
 110       RemoteApiDelegate globalRemoteApiDelegate =
 111           createDelegate(finalOptions, client, originalDelegate);
 112       ApiProxy.setDelegate(globalRemoteApiDelegate);
 113
 114       ApiProxy.setEnvironmentFactory(new EnvironmentFactory() {
 115         @Override
 116         public Environment newEnvironment() {
 117           return createEnv(finalOptions, client);
 118         }
 119       });
 120     }
 121   }
 122
 123   /**
 124    * Installs the remote API using the provided options.  Logs into the remote
 125    * application using the credentials available via these options.
 126    *
 127    * <p>Warning: This method only installs the remote API on the current
 128    * thread.  Do not share this instance across threads!</p>
 129    *
 130    * @throws IllegalArgumentException if the server or credentials weren't provided.
 131    * @throws IllegalStateException if already installed
 132    * @throws LoginException if unable to log in.
 133    * @throws IOException if unable to connect to the remote API.
 134    */
 135   public void install(RemoteApiOptions options) throws IOException {
 136     options = options.copy();
 137     validateOptions(options);
 138
 139     synchronized (getClass()) {
 140       if (installed()) {
 141         throw new IllegalStateException("remote API is already installed");
 142       }
 143       @SuppressWarnings("unchecked")
 144       Delegate<Environment> originalDelegate = ApiProxy.getDelegate();
 145       Environment originalEnv = ApiProxy.getCurrentEnvironment();
 146       RemoteApiClient installedClient = login(options);
 147       RemoteApiDelegate remoteApiDelegate;
 148       if (originalDelegate instanceof ThreadLocalDelegate) {
 149         ThreadLocalDelegate<Environment> installedDelegate =
 150             (ThreadLocalDelegate<Environment>) originalDelegate;
 151         Delegate<Environment> globalDelegate = installedDelegate.getGlobalDelegate();
 152         remoteApiDelegate = createDelegate(options, installedClient, globalDelegate);
 153         if (installedDelegate.getDelegateForThread() != null) {
 154           throw new IllegalStateException("remote API is already installed");
 155         }
 156         installedDelegate.setDelegateForThread(remoteApiDelegate);
 157       } else {
 158         remoteApiDelegate = createDelegate(options, installedClient, originalDelegate);
 159         ApiProxy.setDelegate(new ThreadLocalDelegate<Environment>(
 160             originalDelegate, remoteApiDelegate));
 161       }
 162       Environment installedEnv = null;
 163       String appIdOverrideToRestore = null;
 164       if (originalEnv == null) {
 165         installedEnv = createEnv(options, installedClient);
 166         ApiProxy.setEnvironmentForCurrentThread(installedEnv);
 167       } else {
 168         appIdOverrideToRestore =
 169             (String) originalEnv.getAttributes().get(DATASTORE_APP_ID_OVERRIDE_KEY);
 170         originalEnv.getAttributes().put(DATASTORE_APP_ID_OVERRIDE_KEY, installedClient.getAppId());
 171       }
 172
 173       installerState = new InstallerState(
 174           originalEnv,
 175           installedClient,
 176           remoteApiDelegate,
 177           installedEnv,
 178           appIdOverrideToRestore);
 179     }
 180   }
 181
 182   /**
 183    * The state related to the installation of a {@link RemoteApiInstaller}.
 184    * It's just a struct, but it makes it easy for us to ensure that we don't
 185    * end up in an inconsistent state when installation fails part-way through.
 186    */
 187   private static class InstallerState { private final Environment originalEnv;
 188     private final RemoteApiClient installedClient;
 189     private final RemoteApiDelegate remoteApiDelegate; private final Environment installedEnv; String appIdOverrideToRestore;
 190
 191     InstallerState(
 192         Environment originalEnv,
 193         RemoteApiClient installedClient,
 194         RemoteApiDelegate remoteApiDelegate,
 195         Environment installedEnv,
 196         String appIdOverrideToRestore) {
 197       this.originalEnv = originalEnv;
 198       this.installedClient = installedClient;
 199       this.remoteApiDelegate = remoteApiDelegate;
 200       this.installedEnv = installedEnv;
 201       this.appIdOverrideToRestore = appIdOverrideToRestore;
 202     }
 203   }
 204   /**
 205    * Uninstalls the remote API. If any async calls are in progress, waits for
 206    * them to finish.
 207    *
 208    * <p>If the remote API isn't installed, this method has no effect.</p>
 209    */
 210   public void uninstall() {
 211     synchronized (getClass()) {
 212       if (installedForAllThreads) {
 213         throw new IllegalArgumentException(
 214             "cannot uninstall the remote API after installing on all threads");
 215       }
 216       if (installerState == null) {
 217         throw new IllegalArgumentException("remote API is already uninstalled");
 218       }
 219       if (installerState.installedEnv != null
 220           && installerState.installedEnv != ApiProxy.getCurrentEnvironment()) {
 221         throw new IllegalStateException(
 222           "Can't uninstall because the current environment has been modified.");
 223       }
 224       ApiProxy.Delegate<?> currentDelegate = ApiProxy.getDelegate();
 225       if (!(currentDelegate instanceof ThreadLocalDelegate)) {
 226         throw new IllegalStateException(
 227             "Can't uninstall because the current delegate has been modified.");
 228       }
 229       ThreadLocalDelegate<?> tld = (ThreadLocalDelegate<?>) currentDelegate;
 230       if (tld.getDelegateForThread() == null) {
 231         throw new IllegalArgumentException("remote API is already uninstalled");
 232       }
 233       tld.clearThreadDelegate();
 234
 235       if (installerState.installedEnv != null) {
 236         ApiProxy.setEnvironmentForCurrentThread(installerState.originalEnv);
 237       } else {
 238         if (installerState.appIdOverrideToRestore != null) {
 239           ApiProxy.getCurrentEnvironment().getAttributes().put(
 240               DATASTORE_APP_ID_OVERRIDE_KEY, installerState.appIdOverrideToRestore);
 241         } else {
 242           ApiProxy.getCurrentEnvironment().getAttributes().remove(DATASTORE_APP_ID_OVERRIDE_KEY);
 243         }
 244       }
 245
 246       installerState.remoteApiDelegate.shutdown();
 247       installerState = null;
 248     }
 249   }
 250
 251   /**
 252    * Returns a string containing the cookies associated with this
 253    * connection. The string can be used to create a new connection
 254    * without logging in again by using {@link RemoteApiOptions#reuseCredentials}.
 255    * By storing credentials to a file, we can avoid repeated password
 256    * prompts in command-line tools. (Note that the cookies will expire
 257    * based on the setting under Application Settings in the admin console.)
 258    *
 259    * <p>Beware: it's important to keep this string private, as it
 260    * allows admin access to the app as the current user.</p>
 261    */
 262   public String serializeCredentials() {
 263     return installerState.installedClient.serializeCredentials();
 264   }
 265
 266   /**
 267    * Starts logging remote API method calls to the console. (Useful within tests.)
 268    */
 269   public void logMethodCalls() {
 270     Logger logger = Logger.getLogger(RemoteApiDelegate.class.getName());
 271     logger.setLevel(Level.FINE);
 272     if (!Arrays.asList(logger.getHandlers()).contains(getStreamHandler())) {
 273       logger.addHandler(getStreamHandler());
 274     }
 275   }
 276
 277   public void resetRpcCount() {
 278     installerState.remoteApiDelegate.resetRpcCount();
 279   }
 280
 281   /**
 282    * Returns the number of RPC calls made since the API was installed
 283    * or {@link #resetRpcCount} was called.
 284    */
 285   public int getRpcCount() {
 286     return installerState.remoteApiDelegate.getRpcCount();
 287   }
 288
 289   RemoteApiClient login(RemoteApiOptions options) throws IOException {
 290     return loginImpl(options);
 291   }
 292
 293   RemoteApiDelegate createDelegate(RemoteApiOptions options, RemoteApiClient client, Delegate<Environment> originalDelegate) {
 294     return RemoteApiDelegate.newInstance(new RemoteRpc(client), options, originalDelegate);
 295   }
 296
 297   Environment createEnv(RemoteApiOptions options, RemoteApiClient client) {
 298     return new ToolEnvironment(client.getAppId(), options.getUserEmail());
 299   }
 300
 301   /**
 302    * Submits credentials and gets cookies for logging in to App Engine.
 303    * (Also downloads the appId from the remote API.)
 304    * @return an AppEngineClient containing credentials (if successful)
 305    * @throws LoginException for a login failure
 306    * @throws IOException for other connection failures
 307    */
 308   private RemoteApiClient loginImpl(RemoteApiOptions options) throws IOException {
 309     List<Cookie> authCookies;
 310     if (!authenticationRequiresCookies(options)) {
 311       authCookies = Collections.emptyList();
 312     } else if (options.getCredentialsToReuse() != null) {
 313       authCookies = parseSerializedCredentials(options.getUserEmail(), options.getHostname(),
 314           options.getCredentialsToReuse());
 315     } else if (options.getHostname().equals("localhost")) {
 316       authCookies = Collections.singletonList(
 317           makeDevAppServerCookie(options.getHostname(), options.getUserEmail()));
 318     } else if (ApiProxy.getCurrentEnvironment() != null) {
 319       authCookies = new HostedClientLogin().login(
 320           options.getHostname(), options.getUserEmail(), options.getPassword());
 321     } else {
 322       authCookies = new StandaloneClientLogin().login(
 323           options.getHostname(), options.getUserEmail(), options.getPassword());
 324     }
 325
 326     String appId = getAppIdFromServer(authCookies, options);
 327     return createAppEngineClient(options, authCookies, appId);
 328   }
 329
 330   /**
 331    * @return {@code true} if the authentication to support the {@link RemoteApiOptions} requires
 332    *         cookies, {@code false} otherwise
 333    */
 334   boolean authenticationRequiresCookies(final RemoteApiOptions options) {
 335     return options.getOAuthCredential() == null;
 336   }
 337
 338   RemoteApiClient createAppEngineClient(RemoteApiOptions options,
 339       List<Cookie> authCookies, String appId) {
 340     if (options.getOAuthCredential() != null) {
 341       return new OAuthClient(options, appId);
 342     }
 343     if (ApiProxy.getCurrentEnvironment() != null) {
 344       return new HostedAppEngineClient(options, authCookies, appId);
 345     }
 346     return new StandaloneAppEngineClient(options, authCookies, appId);
 347   }
 348
 349   public static Cookie makeDevAppServerCookie(String hostname, String email) {
 350     String cookieValue = email + ":true:" + LoginCookieUtils.encodeEmailAsUserId(email);
 351     Cookie cookie = new Cookie(hostname, LoginCookieUtils.COOKIE_NAME, cookieValue);
 352     cookie.setPath("/");
 353     return cookie;
 354   }
 355
 356   String getAppIdFromServer(List<Cookie> authCookies, RemoteApiOptions options)
 357       throws IOException {
 358     RemoteApiClient tempClient = createAppEngineClient(options, authCookies, null);
 359     AppEngineClient.Response response = tempClient.get(options.getRemoteApiPath());
 360     int status = response.getStatusCode();
 361     if (status != 200) {
 362       if (response.getBodyAsBytes() == null) {
 363         throw new IOException("can't get appId from remote api; status code = " + status);
 364       } else {
 365         throw new IOException("can't get appId from remote api; status code = " + status
 366             + ", body: " + response.getBodyAsString());
 367       }
 368     }
 369     String body = response.getBodyAsString();
 370     Map<String, String> props = parseYamlMap(body);
 371     String appId = props.get("app_id");
 372     if (appId == null) {
 373       throw new IOException("unexpected response from remote api: " + body);
 374     }
 375     return appId;
 376   }
 377
 378   /**
 379    * Parses the response from the remote API as a YAML map.
 380    */
 381   static Map<String, String> parseYamlMap(String input) {
 382
 383     Map<String, String> result = new HashMap<String, String>();
 384     input = input.trim();
 385     if (!input.startsWith("{") || !input.endsWith("}")) {
 386       return Collections.emptyMap();
 387     }
 388     input = input.substring(1, input.length() - 1);
 389
 390     String[] pairs = input.split(", +");
 391     for (String pair : pairs) {
 392       Matcher matcher = PAIR_REGEXP.matcher(pair);
 393       if (matcher.matches()) {
 394         result.put(matcher.group(1), matcher.group(3));
 395       }
 396     }
 397     return result;
 398   }
 399
 400   static List<Cookie> parseSerializedCredentials(String expectedEmail, String expectedHost,
 401       String serializedCredentials) throws IOException {
 402
 403     Map<String, List<String>> props = parseProperties(serializedCredentials);
 404     checkOneProperty(props, "email");
 405     checkOneProperty(props, "host");
 406     String email = props.get("email").get(0);
 407     if (!expectedEmail.equals(email)) {
 408       throw new IOException("credentials don't match current user email");
 409     }
 410     String host = props.get("host").get(0);
 411     if (!expectedHost.equals(host)) {
 412       throw new IOException("credentials don't match current host");
 413     }
 414
 415     List<Cookie> result = new ArrayList<Cookie>();
 416     for (String line : props.get("cookie")) {
 417       result.add(parseCookie(line, host));
 418     }
 419     return result;
 420   }
 421
 422   private static Cookie parseCookie(String line, String host) throws IOException {
 423     int firstEqual = line.indexOf('=');
 424     if (firstEqual < 1) {
 425       throw new IOException("invalid cookie in credentials");
 426     }
 427     String key = line.substring(0, firstEqual);
 428     String value = line.substring(firstEqual + 1);
 429     Cookie cookie = new Cookie(host, key, value);
 430     cookie.setPath("/");
 431     return cookie;
 432   }
 433
 434   private static void checkOneProperty(Map<String, List<String>> props, String key)
 435       throws IOException {
 436     if (props.get(key).size() != 1) {
 437       String message = "invalid credential file (should have one property named '" + key + "')";
 438       throw new IOException(message);
 439     }
 440   }
 441
 442   private static Map<String, List<String>> parseProperties(String serializedCredentials) {
 443     Map<String, List<String>> props = new HashMap<String, List<String>>();
 444     for (String line : serializedCredentials.split("\n")) {
 445       line = line.trim();
 446       if (!line.startsWith("#") && line.contains("=")) {
 447         int firstEqual = line.indexOf('=');
 448         String key = line.substring(0, firstEqual);
 449         String value = line.substring(firstEqual + 1);
 450         List<String> values = props.get(key);
 451         if (values == null) {
 452           values = new ArrayList<String>();
 453           props.put(key, values);
 454         }
 455         values.add(value);
 456       }
 457     }
 458     return props;
 459   }
 460 }