java/src/main/com/google/appengine/tools/development/AbstractBackendServers.java

   1 // Copyright 2011 Google Inc. All Rights Reserved.
   2
   3 package com.google.appengine.tools.development;
   4
   5 import com.google.appengine.api.backends.dev.LocalServerController;
   6 import com.google.appengine.tools.development.AbstractContainerService.LocalInitializationEnvironment;
   7 import com.google.apphosting.api.ApiProxy;
   8 import com.google.apphosting.api.ApiProxy.Environment;
   9 import com.google.apphosting.utils.config.AppEngineWebXml;
  10 import com.google.apphosting.utils.config.BackendsXml;
  11 import com.google.common.collect.ImmutableMap;
  12 import com.google.common.collect.Maps;
  13
  14 import org.apache.commons.httpclient.HttpClient;
  15 import org.apache.commons.httpclient.methods.GetMethod;
  16
  17 import java.io.File;
  18 import java.io.IOException;
  19 import java.io.InputStream;
  20 import java.lang.reflect.Method;
  21 import java.net.MalformedURLException;
  22 import java.util.HashMap;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.Map.Entry;
  26 import java.util.TreeMap;
  27 import java.util.concurrent.Semaphore;
  28 import java.util.concurrent.TimeUnit;
  29 import java.util.logging.Logger;
  30
  31 import javax.servlet.ServletException;
  32 import javax.servlet.http.HttpServletRequest;
  33 import javax.servlet.http.HttpServletResponse;
  34
  35 /**
  36  * Controls backend servers configured in appengine-web.xml. Each server is
  37  * started on a separate port. All servers run the same code as the main app.
  38  *
  39  *
  40  */
  41 public abstract class AbstractBackendServers implements BackendContainer, LocalServerController {
  42   private static final String X_GOOGLE_DEV_APPSERVER_SKIPADMINCHECK =
  43       "X-Google-DevAppserver-SkipAdminCheck";
  44   public static final String SYSTEM_PROPERTY_STATIC_PORT_NUM_PREFIX =
  45       "com.google.appengine.devappserver.";
  46
  47   private static final int AH_REQUEST_DEFAULT_TIMEOUT = 30 * 1000;
  48   private static final int AH_REQUEST_INFINITE_TIMEOUT = 0;
  49
  50   private static final Integer DEFAULT_INSTANCES = 1;
  51   private static final String DEFAULT_INSTANCE_CLASS = "B1";
  52   private static final Integer DEFAULT_MAX_CONCURRENT_REQUESTS = 10;
  53
  54   private enum BackendServerState {
  55     INITIALIZING, SLEEPING, RUNNING_START_REQUEST, RUNNING, STOPPING, STOPPED, SHUTDOWN;
  56   }
  57
  58   private static final int MAX_PENDING_QUEUE_LENGTH = 20;
  59   private static final int MAX_PENDING_QUEUE_TIME_MS = 10 * 1000;
  60   private static final int MAX_START_QUEUE_TIME_MS = 30 * 1000;
  61
  62   private String address;
  63   private File appDir;
  64   private File externalResourceDir;
  65   private File webXmlLocation;
  66   private File appEngineWebXmlLocation;
  67   private Map<String, Object> containerConfigProperties;
  68   private Map<AbstractBackendServers.ServerInstanceEntry, ServerWrapper> backendServers =
  69       ImmutableMap.copyOf(new HashMap<ServerInstanceEntry, ServerWrapper>());
  70   private Map<String, String> portMapping =
  71       ImmutableMap.copyOf(new HashMap<String, String>());
  72   protected Logger logger = Logger.getLogger(AbstractBackendServers.class.getName());
  73
  74   private Map<String, String> serviceProperties = new HashMap<String, String>();
  75
  76   private DevAppServer devAppServer;
  77
  78   public void init(File appDir, File externalResourceDir, File webXmlLocation,
  79       File appEngineWebXmlLocation, String address, Map<String, Object> containerConfigProperties,
  80       DevAppServer devAppServer) {
  81     this.appDir = appDir;
  82     this.externalResourceDir = externalResourceDir;
  83     this.webXmlLocation = webXmlLocation;
  84     this.appEngineWebXmlLocation = appEngineWebXmlLocation;
  85     this.address = address;
  86     this.containerConfigProperties = containerConfigProperties;
  87     this.devAppServer = devAppServer;
  88   }
  89
  90   @Override
  91   public void setServiceProperties(Map<String, String> properties) {
  92     this.serviceProperties = properties;
  93   }
  94
  95   @Override
  96   public void shutdownAll() throws Exception {
  97     for (ServerWrapper server : backendServers.values()) {
  98       logger.finer("server shutdown: " + server);
  99       server.shutdown();
 100     }
 101     backendServers = ImmutableMap.copyOf(new HashMap<ServerInstanceEntry, ServerWrapper>());
 102   }
 103
 104   @Override
 105   public TreeMap<String, BackendStateInfo> getBackendState(String requestHostName) {
 106     TreeMap<String, BackendStateInfo> serverInfoMap = new TreeMap<String, BackendStateInfo>();
 107     for (ServerWrapper serverWrapper : backendServers.values()) {
 108       String name = serverWrapper.serverEntry.getName();
 109
 110       String listenAddress;
 111       if (requestHostName == null) {
 112         listenAddress = portMapping.get(serverWrapper.getDnsPrefix());
 113       } else {
 114         listenAddress = requestHostName + ":" + serverWrapper.port;
 115       }
 116
 117       BackendStateInfo ssi = serverInfoMap.get(name);
 118       if (ssi == null) {
 119         ssi = new BackendStateInfo(serverWrapper.serverEntry);
 120         serverInfoMap.put(name, ssi);
 121       }
 122       if (serverWrapper.isLoadBalanceServer()) {
 123         ssi.setState(serverWrapper.serverState.name().toLowerCase());
 124         ssi.setAddress(listenAddress);
 125       } else {
 126         ssi.add(new InstanceStateInfo(serverWrapper.serverInstance, listenAddress,
 127             serverWrapper.serverState.name().toLowerCase()));
 128       }
 129     }
 130     return serverInfoMap;
 131   }
 132
 133   @Override
 134   public synchronized void startBackend(String serverToStart) throws IllegalStateException {
 135     if (!checkServerExists(serverToStart)) {
 136       String message = String.format("Tried to start unknown server %s", serverToStart);
 137       logger.warning(message);
 138       throw new IllegalStateException(message);
 139     }
 140     for (ServerWrapper server : backendServers.values()) {
 141       if (server.getName().equals(serverToStart)) {
 142         if (server.getState() != BackendServerState.STOPPED) {
 143           continue;
 144         }
 145         if (server.isLoadBalanceServer()) {
 146           server.compareAndSetServerState(
 147               BackendServerState.RUNNING, BackendServerState.STOPPED);
 148           continue;
 149         }
 150         server.compareAndSetServerState(
 151             BackendServerState.SLEEPING, BackendServerState.STOPPED);
 152         server.sendStartRequest();
 153       }
 154     }
 155   }
 156
 157   @Override
 158   public synchronized void stopBackend(String serverToStop) throws Exception {
 159     if (!checkServerExists(serverToStop)) {
 160       String message = String.format("Tried to stop unknown server %s", serverToStop);
 161       logger.warning(message);
 162       throw new IllegalStateException(message);
 163     }
 164     for (ServerWrapper server : backendServers.values()) {
 165       if (server.getName().equals(serverToStop)) {
 166         if (server.getState() == BackendServerState.STOPPED) {
 167           continue;
 168         }
 169         if (server.isLoadBalanceServer()) {
 170           server.compareAndSetServerState(
 171               BackendServerState.STOPPED, BackendServerState.RUNNING);
 172           continue;
 173         }
 174         logger.fine("Stopping server: " + server.getDnsPrefix());
 175         server.shutdown();
 176         server.startup(true);
 177       }
 178     }
 179   }
 180
 181   @Override
 182   public void startupAll(BackendsXml backendsXml, ApiProxyLocal local) throws Exception {
 183     if (backendsXml == null) {
 184       logger.fine("Got null backendsXml config.");
 185       return;
 186     }
 187     List<BackendsXml.Entry> servers = backendsXml.getBackends();
 188     if (servers.size() == 0) {
 189       logger.fine("No backends configured.");
 190       return;
 191     }
 192
 193     if (backendServers.size() != 0) {
 194       throw new Exception("Tried to start backendservers but some are already running.");
 195     }
 196     logger.finer("Found " + servers.size() + " configured backends.");
 197
 198     Map<AbstractBackendServers.ServerInstanceEntry, ServerWrapper> serverMap = Maps.newHashMap();
 199     for (BackendsXml.Entry entry : servers) {
 200       entry = resolveDefaults(entry);
 201
 202       for (int serverInstance = -1; serverInstance < entry.getInstances(); serverInstance++) {
 203         int port = checkForStaticPort(entry.getName(), serverInstance);
 204         ServerWrapper serverWrapper =
 205             new ServerWrapper(ContainerUtils.loadContainer(), entry, serverInstance, port);
 206         serverMap.put(new ServerInstanceEntry(entry.getName(), serverInstance), serverWrapper);
 207       }
 208     }
 209     this.backendServers = ImmutableMap.copyOf(serverMap);
 210
 211     String prettyAddress = address;
 212     if ("0.0.0.0".equals(address)) {
 213       prettyAddress = "127.0.0.1";
 214     }
 215
 216     Map<String, String> portMap = Maps.newHashMap();
 217     for (ServerWrapper serverWrapper : backendServers.values()) {
 218       logger.finer(
 219           "starting server: " + serverWrapper.serverInstance + "." + serverWrapper.getName()
 220               + " on " + address + ":" + serverWrapper.port);
 221       ApiProxy.Delegate configured = ApiProxy.getDelegate();
 222       try {
 223         ApiProxy.setDelegate(local);
 224         serverWrapper.startup(false);
 225       } finally {
 226         ApiProxy.setDelegate(configured);
 227       }
 228
 229       portMap.put(serverWrapper.getDnsPrefix(), prettyAddress + ":" + serverWrapper.port);
 230     }
 231     this.portMapping = ImmutableMap.copyOf(portMap);;
 232
 233     for (ServerWrapper serverWrapper : backendServers.values()) {
 234       if (serverWrapper.isLoadBalanceServer()) {
 235         continue;
 236       }
 237       serverWrapper.sendStartRequest();
 238     }
 239   }
 240
 241   private BackendsXml.Entry resolveDefaults(BackendsXml.Entry entry) {
 242     return new BackendsXml.Entry(
 243         entry.getName(),
 244         entry.getInstances() == null ? DEFAULT_INSTANCES : entry.getInstances(),
 245         entry.getInstanceClass() == null ? DEFAULT_INSTANCE_CLASS : entry.getInstanceClass(),
 246         entry.getMaxConcurrentRequests() == null ? DEFAULT_MAX_CONCURRENT_REQUESTS :
 247                                                    entry.getMaxConcurrentRequests(),
 248         entry.getOptions(),
 249         entry.getState() == null ? BackendsXml.State.STOP : entry.getState());
 250   }
 251
 252   /**
 253    * Forward a request to a specific server and instance. This will call the
 254    * specified instance request dispatcher so the request is handled in the
 255    * right server context.
 256    */
 257   protected abstract void forwardToServer(String requestedServer, int instance,
 258       HttpServletRequest hrequest,
 259       HttpServletResponse hresponse) throws IOException, ServletException;
 260
 261   /**
 262    * This method guards access to servers to limit the number of concurrent
 263    * requests. Each request running on a server must acquire a serving permit.
 264    * If no permits are available a 500 response should be sent.
 265    *
 266    * @param serverName The server for which to acquire a permit.
 267    * @param instanceNumber The server instance for which to acquire a permit.
 268    * @param allowQueueOnBackends If set to false the method will return
 269    *        instantly, if set to true (and the specified server allows pending
 270    *        queues) this method can block for up to 10 s waiting for a serving
 271    *        permit to become available.
 272    * @return true if a permit was acquired, false otherwise
 273    */
 274   public boolean acquireServingPermit(
 275       String serverName, int instanceNumber, boolean allowQueueOnBackends) {
 276     logger.finest(
 277         String.format("trying to get serving permit for server %d.%s", instanceNumber, serverName));
 278     try {
 279       ServerWrapper server = getServerWrapper(serverName, instanceNumber);
 280       int maxQueueTime = 0;
 281
 282       synchronized (server) {
 283         if (!server.acceptsConnections()) {
 284           logger.finest(server + ": got request but server is not in a serving state");
 285           return false;
 286         }
 287         if (server.getApproximateQueueLength() > MAX_PENDING_QUEUE_LENGTH) {
 288           logger.finest(server + ": server queue is full");
 289           return false;
 290         }
 291         if (server.getState() == BackendServerState.SLEEPING) {
 292           logger.finest(server + ": waking up sleeping server");
 293           server.sendStartRequest();
 294         }
 295
 296         if (server.getState() == BackendServerState.RUNNING_START_REQUEST) {
 297           maxQueueTime = MAX_START_QUEUE_TIME_MS;
 298         } else if (allowQueueOnBackends && server.getMaxPendingQueueSize() > 0) {
 299           maxQueueTime = MAX_PENDING_QUEUE_TIME_MS;
 300         }
 301       }
 302       boolean gotPermit = server.acquireServingPermit(maxQueueTime);
 303       logger.finest(server + ": tried to get server permit, timeout=" + maxQueueTime + " success="
 304           + gotPermit);
 305       return gotPermit;
 306     } catch (InterruptedException e) {
 307       logger.finest(
 308           instanceNumber + "." + serverName + ": got interrupted while waiting for serving permit");
 309       return false;
 310     }
 311   }
 312
 313   /**
 314    * Reserves an instance for this request. For workers this method will return
 315    * -1 if no free instances are available. For backends this method will assign
 316    * this request to the instance with the shortest queue and block until that
 317    * instance is ready to serve the request.
 318    *
 319    * @param requestedServer Name of the server the request is to.
 320    * @return the instance id of an available server instance, or -1 if no
 321    *         instance is available.
 322    */
 323   public int getAndReserveFreeInstance(String requestedServer) {
 324     logger.finest("trying to get serving permit for server " + requestedServer);
 325
 326     ServerWrapper server = getServerWrapper(requestedServer, -1);
 327     if (server == null) {
 328       return -1;
 329     }
 330     if (!server.acceptsConnections()) {
 331       return -1;
 332     }
 333     int instanceNum = server.getInstances();
 334     for (int i = 0; i < instanceNum; i++) {
 335       if (acquireServingPermit(requestedServer, i, false)) {
 336         return i;
 337       }
 338     }
 339     if (server.getMaxPendingQueueSize() > 0) {
 340       return addToShortestInstanceQueue(requestedServer);
 341     } else {
 342       logger.finest("no servers free");
 343       return -1;
 344     }
 345   }
 346
 347   /**
 348    * Will add this request to the queue of the instance with the approximate
 349    * shortest queue. This method will block for up to 10 seconds until a permit
 350    * is received.
 351    *
 352    * @param requestedServer the server name
 353    * @return the instance where the serving permit was reserved, or -1 if all
 354    *         instance queues are full
 355    */
 356   int addToShortestInstanceQueue(String requestedServer) {
 357     logger.finest(requestedServer + ": no instances free, trying to find a queue");
 358     int shortestQueue = MAX_PENDING_QUEUE_LENGTH;
 359     ServerWrapper instanceWithShortestQueue = null;
 360     for (ServerWrapper server : backendServers.values()) {
 361       if (!server.acceptsConnections()) {
 362         continue;
 363       }
 364       int serverQueue = server.getApproximateQueueLength();
 365       if (shortestQueue > serverQueue) {
 366         instanceWithShortestQueue = server;
 367         shortestQueue = serverQueue;
 368       }
 369     }
 370
 371     try {
 372       if (shortestQueue < MAX_PENDING_QUEUE_LENGTH) {
 373         logger.finest("adding request to queue on instance: " + instanceWithShortestQueue);
 374         if (instanceWithShortestQueue.acquireServingPermit(MAX_PENDING_QUEUE_TIME_MS)) {
 375           logger.finest("ready to serve request on instance: " + instanceWithShortestQueue);
 376           return instanceWithShortestQueue.serverInstance;
 377         }
 378       }
 379     } catch (InterruptedException e) {
 380       logger.finer("interupted while queued at server " + instanceWithShortestQueue);
 381     }
 382     return -1;
 383   }
 384
 385   /**
 386    * Method for returning a serving permit after a request has completed.
 387    *
 388    * @param serverName The server name
 389    * @param instance The server instance
 390    */
 391   public void returnServingPermit(String serverName, int instance) {
 392     ServerWrapper server = getServerWrapper(serverName, instance);
 393     server.releaseServingPermit();
 394   }
 395
 396   /**
 397    * Verifies if a specific server/instance is configured.
 398    *
 399    * @param serverName The server name
 400    * @param instance The server instance
 401    * @return true if the server/instance is configured, false otherwise.
 402    */
 403   public boolean checkInstanceExists(String serverName, int instance) {
 404     return getServerWrapper(serverName, instance) != null;
 405   }
 406
 407   /**
 408    * Verifies if a specific server is configured.
 409    *
 410    * @param serverName The server name
 411    * @return true if the server is configured, false otherwise.
 412    */
 413   public boolean checkServerExists(String serverName) {
 414     return checkInstanceExists(serverName, -1);
 415   }
 416
 417   /**
 418    * Verifies if a specific server is stopped.
 419    *
 420    * @param serverName The server name
 421    * @return true if the server is stopped, false otherwise.
 422    */
 423   public boolean checkServerStopped(String serverName) {
 424     return checkInstanceStopped(serverName, -1);
 425   }
 426
 427   /**
 428    * Verifies if a specific server/instance is stopped.
 429    *
 430    * @param serverName The server name
 431    * @param instance The server instance
 432    * @return true if the server/instance is stopped, false otherwise.
 433    */
 434   public boolean checkInstanceStopped(String serverName, int instance) {
 435     return !getServerWrapper(serverName, instance).acceptsConnections();
 436   }
 437
 438   /**
 439    * Allows the servers API to get the current mapping from server and instance
 440    * to listening ports.
 441    *
 442    * @return The port map
 443    */
 444   public Map<String, String> getPortMapping() {
 445     return this.portMapping;
 446   }
 447
 448   /**
 449    * Returns the server instance serving on a specific local port
 450    *
 451    * @param port the local tcp port that received the request
 452    * @return the server instance, or -1 if no server instance is running on that
 453    *         port
 454    */
 455   public int getServerInstanceFromPort(int port) {
 456     ServerWrapper server = getServerWrapperFromPort(port);
 457     if (server != null) {
 458       return server.serverInstance;
 459     } else {
 460       return -1;
 461     }
 462   }
 463
 464   /**
 465    * Returns the server serving on a specific local port
 466    *
 467    * @param port the local tcp port that received the request
 468    * @return the server name, or null if no server instance is running on that
 469    *         port
 470    */
 471   public String getServerNameFromPort(int port) {
 472     ServerWrapper server = getServerWrapperFromPort(port);
 473     if (server != null) {
 474       return server.getName();
 475     } else {
 476       return null;
 477     }
 478   }
 479
 480   /**
 481    * Convenience method for getting the ServerWrapper running on a specific port
 482    */
 483   private ServerWrapper getServerWrapperFromPort(int port) {
 484     for (Entry<ServerInstanceEntry, ServerWrapper> entry : backendServers.entrySet()) {
 485       if (entry.getValue().port == port) {
 486         return entry.getValue();
 487       }
 488     }
 489     return null;
 490   }
 491
 492   /**
 493    * Convenience method for getting the ServerWrapper for a specific
 494    * server/instance
 495    */
 496   protected ServerWrapper getServerWrapper(String serverName, int instanceNumber) {
 497     return backendServers.get(new ServerInstanceEntry(serverName, instanceNumber));
 498   }
 499
 500   /**
 501    * Verifies if the specific port is statically configured, if not it will
 502    * return 0 which instructs jetty to pick the port
 503    *
 504    *  Ports can be statically configured by a system property at
 505    * com.google.appengine.server.<server-name>.port for the server and
 506    * com.google.appengine.server.<server-name>.<instance-id>.port for individual
 507    * instances
 508    *
 509    * @param server the name of the configured server
 510    * @param instance the instance number to configure
 511    * @return the statically configured port, or 0 if none is configured
 512    */
 513   private int checkForStaticPort(String server, int instance) {
 514     StringBuilder key = new StringBuilder();
 515     key.append(SYSTEM_PROPERTY_STATIC_PORT_NUM_PREFIX);
 516     key.append(server);
 517     if (instance >= 0) {
 518       key.append("." + instance);
 519     }
 520     key.append(".port");
 521     String configuredPort = serviceProperties.get(key.toString());
 522     if (configuredPort != null) {
 523       return Integer.parseInt(configuredPort);
 524     } else {
 525       return 0;
 526     }
 527   }
 528
 529   /**
 530    * Class that allows the key in the server map to be the
 531    * (servername,instanceid) tuple. Overrides equals() and hashcode() to
 532    * function as a hashtable key
 533    *
 534    */
 535   static class ServerInstanceEntry {
 536     private final int instanceNumber;
 537     private final String serverName;
 538
 539     /**
 540      * @param serverName
 541      * @param instanceNumber
 542      */
 543     public ServerInstanceEntry(String serverName, int instanceNumber) {
 544       this.serverName = serverName;
 545       this.instanceNumber = instanceNumber;
 546     }
 547
 548     @Override
 549     public boolean equals(Object o) {
 550       if (!(o instanceof ServerInstanceEntry)) {
 551         return false;
 552       }
 553
 554       ServerInstanceEntry that = (ServerInstanceEntry) o;
 555       if (this.serverName != null) {
 556         if (!this.serverName.equals(that.serverName)) {
 557           return false;
 558         }
 559       } else {
 560         if (that.serverName != null) {
 561           return false;
 562         }
 563       }
 564
 565       if (this.instanceNumber != that.instanceNumber) {
 566         return false;
 567       }
 568       return true;
 569     }
 570
 571     @Override
 572     public int hashCode() {
 573       int hash = 17;
 574       hash = 31 * hash + instanceNumber;
 575       if (serverName != null) {
 576         hash = 31 * hash + serverName.hashCode();
 577       }
 578       return hash;
 579     }
 580
 581     @Override
 582     public String toString() {
 583       return instanceNumber + "." + serverName;
 584     }
 585   }
 586
 587   /**
 588    * Wraps a container service and contains extra information such as the
 589    * instanceid of the current container as well as the port number it is
 590    * running on.
 591    */
 592   protected class ServerWrapper {
 593
 594     private final ContainerService container;
 595     private final int serverInstance;
 596     private int port;
 597     private final BackendsXml.Entry serverEntry;
 598
 599     private BackendServerState serverState = BackendServerState.SHUTDOWN;
 600
 601     private final Semaphore servingQueue = new Semaphore(0, true);
 602
 603     public ServerWrapper(ContainerService containerService,
 604         BackendsXml.Entry serverEntry,
 605         int instance, int port) {
 606       this.container = containerService;
 607       this.serverEntry = serverEntry;
 608       this.serverInstance = instance;
 609       this.port = port;
 610       containerService.setEnvironmentVariableMismatchSeverity(
 611           ContainerService.EnvironmentVariableMismatchSeverity.IGNORE);
 612     }
 613
 614     /**
 615      * Triggers an HTTP GET to /_ah/start
 616      *
 617      *  This method will keep on trying until it receives a non-error response
 618      * code from the server.
 619      *
 620      * @param timeoutInMs Timeout in milliseconds, 0 indicates no timeout.
 621      *
 622      */
 623     private void sendStartRequest(int timeoutInMs) {
 624       try {
 625         String urlString =
 626             String.format("http://%s:%d/_ah/start", address, ServerWrapper.this.port);
 627         logger.finer("sending start request to: " + urlString);
 628
 629         HttpClient httpClient = new HttpClient();
 630         httpClient.getParams().setConnectionManagerTimeout(timeoutInMs);
 631
 632         GetMethod request = new GetMethod(urlString);
 633         request.addRequestHeader(X_GOOGLE_DEV_APPSERVER_SKIPADMINCHECK, "true");
 634         try {
 635           int returnCode = httpClient.executeMethod(request);
 636
 637           byte[] buffer = new byte[1024];
 638           InputStream in = request.getResponseBodyAsStream();
 639           while (in.read(buffer) != -1) {
 640           }
 641           if ((returnCode >= 200 && returnCode < 300) || returnCode == 404) {
 642             logger.fine(
 643                 String.format("backend server %d.%s request to /_ah/start completed, code=%d",
 644                     serverInstance, serverEntry.getName(), returnCode));
 645             compareAndSetServerState(
 646                 BackendServerState.RUNNING, BackendServerState.RUNNING_START_REQUEST);
 647             servingQueue.release(serverEntry.getMaxConcurrentRequests());
 648           } else {
 649             logger.warning("Start request to /_ah/start on server " + serverInstance + "."
 650                 + serverEntry.getName() + " failed (HTTP status code=" + returnCode
 651                 + "). Retrying...");
 652             Thread.sleep(1000);
 653             sendStartRequest(timeoutInMs);
 654           }
 655         } finally {
 656           request.releaseConnection();
 657         }
 658       } catch (MalformedURLException e) {
 659         logger.severe(String.format(
 660             "Unable to send start request to server: %d.%s, " + "MalformedURLException: %s",
 661             serverInstance, serverEntry.getName(), e.getMessage()));
 662       } catch (Exception e) {
 663         logger.warning(String.format(
 664             "Got exception while performing /_ah/start " + "request on server: %d.%s, %s: %s",
 665             serverInstance, serverEntry.getName(), e.getClass().getName(), e.getMessage()));
 666       }
 667     }
 668
 669     /**
 670      * Shut down the server.
 671      *
 672      * Will trigger any shutdown hooks installed by the
 673      * {@link com.google.appengine.api.LifecycleManager}
 674      *
 675      * @throws Exception
 676      */
 677     public void shutdown() throws Exception {
 678       synchronized (ServerWrapper.this) {
 679         if (serverState == BackendServerState.RUNNING
 680             || serverState == BackendServerState.RUNNING_START_REQUEST) {
 681           triggerLifecycleShutdownHook();
 682         }
 683         getContainer().shutdown();
 684         serverState = BackendServerState.SHUTDOWN;
 685       }
 686     }
 687
 688     void startup(boolean setStateToStopped) throws Exception {
 689       compareAndSetServerState(BackendServerState.INITIALIZING, BackendServerState.SHUTDOWN);
 690
 691       getContainer().configure(ContainerUtils.getServerInfo(),
 692           appDir,
 693           externalResourceDir,
 694           webXmlLocation,
 695           appEngineWebXmlLocation,
 696           address,
 697           port,
 698           containerConfigProperties,
 699           devAppServer);
 700       getContainer().startup();
 701
 702       this.port = getContainer().getPort();
 703       if (setStateToStopped) {
 704         compareAndSetServerState(BackendServerState.STOPPED, BackendServerState.INITIALIZING);
 705       } else {
 706         logger.info(
 707             "server: " + serverInstance + "." + serverEntry.getName() + " is running on port "
 708                 + this.port);
 709         if (isLoadBalanceServer()) {
 710           compareAndSetServerState(
 711               BackendServerState.RUNNING, BackendServerState.INITIALIZING);
 712         } else {
 713           compareAndSetServerState(
 714               BackendServerState.SLEEPING, BackendServerState.INITIALIZING);
 715         }
 716       }
 717     }
 718
 719     void sendStartRequest() {
 720       compareAndSetServerState(
 721           BackendServerState.RUNNING_START_REQUEST, BackendServerState.SLEEPING);
 722       if (serverInstance >= 0) {
 723         Thread requestThread = new Thread(new Runnable() {
 724           @Override
 725           public void run() {
 726             sendStartRequest(AH_REQUEST_INFINITE_TIMEOUT);
 727           }
 728         });
 729         requestThread.setDaemon(true);
 730         requestThread.setName(
 731             "BackendServersStartRequestThread." + serverInstance + "." + serverEntry.getName());
 732         requestThread.start();
 733       }
 734     }
 735
 736     /**
 737      * This method will trigger any shutdown hooks registered with the current
 738      * server.
 739      *
 740      * Some class loader trickery is required to make sure that we get the
 741      * {@link com.google.appengine.api.LifecycleManager} responsible for this
 742      * server instance.
 743      */
 744     private void triggerLifecycleShutdownHook() {
 745       Environment prevEnvironment = ApiProxy.getCurrentEnvironment();
 746       try {
 747         ClassLoader serverClassLoader = getContainer().getAppContext().getClassLoader();
 748
 749         Class<?> lifeCycleManagerClass =
 750             Class.forName("com.google.appengine.api.LifecycleManager", true, serverClassLoader);
 751         Method lifeCycleManagerGetter = lifeCycleManagerClass.getMethod("getInstance");
 752         Object userThreadLifeCycleManager = lifeCycleManagerGetter.invoke(null, new Object[0]);
 753
 754         Method beginShutdown = lifeCycleManagerClass.getMethod("beginShutdown", long.class);
 755         AppEngineWebXml appEngineWebXml = getContainer().getAppEngineWebXmlConfig();
 756         ApiProxy.setEnvironmentForCurrentThread(new LocalInitializationEnvironment(
 757             appEngineWebXml.getAppId(), appEngineWebXml.getMajorVersionId()));
 758
 759         try {
 760           beginShutdown.invoke(userThreadLifeCycleManager, AH_REQUEST_DEFAULT_TIMEOUT);
 761         } catch (Exception e) {
 762           logger.warning(
 763               String.format("got exception when running shutdown hook on server %d.%s",
 764                   serverInstance, serverEntry.getName()));
 765           e.printStackTrace();
 766         }
 767       } catch (Exception e) {
 768         logger.severe(
 769             String.format("Exception during reflective call to "
 770                 + "LifecycleManager.beginShutdown on server %d.%s, got %s: %s", serverInstance,
 771                 serverEntry.getName(), e.getClass().getName(), e.getMessage()));
 772       } finally {
 773         ApiProxy.setEnvironmentForCurrentThread(prevEnvironment);
 774       }
 775     }
 776
 777     /**
 778      * Checks if the server is in a state where it can accept incoming requests.
 779      *
 780      * @return true if the server can accept incoming requests, false otherwise.
 781      */
 782     boolean acceptsConnections() {
 783       synchronized (ServerWrapper.this) {
 784         return (serverState == BackendServerState.RUNNING
 785             || serverState == BackendServerState.RUNNING_START_REQUEST
 786             || serverState == BackendServerState.SLEEPING);
 787       }
 788     }
 789
 790     /**
 791      * Updates the current server state and verifies that the previous state is
 792      * what is expected.
 793      *
 794      * @param newState The new state to change to
 795      * @param acceptablePreviousStates Acceptable previous states
 796      * @throws IllegalStateException If the current state is not one of the
 797      *         acceptable previous states
 798      */
 799     private void compareAndSetServerState(
 800         BackendServerState newState, BackendServerState... acceptablePreviousStates)
 801         throws IllegalStateException {
 802       synchronized (ServerWrapper.this) {
 803         for (BackendServerState acceptableStates : acceptablePreviousStates) {
 804           if (serverState == acceptableStates) {
 805             serverState = newState;
 806             return;
 807           }
 808         }
 809       }
 810       StringBuilder error = new StringBuilder();
 811       error.append("Tried to change state to " + newState);
 812       error.append(" on server " + toString());
 813       error.append(" but previous state is not ");
 814       for (int i = 0; i < acceptablePreviousStates.length; i++) {
 815         error.append(acceptablePreviousStates[i].name() + " | ");
 816       }
 817       throw new IllegalStateException(error.toString());
 818     }
 819
 820     /**
 821      * Acquires a serving permit for this server.
 822      *
 823      * @param maxWaitTimeInMs Max wait time in ms
 824      * @return true if a serving permit was acquired within the allowed time,
 825      *         false if not permit was required.
 826      * @throws InterruptedException If the thread was interrupted while waiting.
 827      */
 828     boolean acquireServingPermit(int maxWaitTimeInMs) throws InterruptedException {
 829       logger.finest(
 830           this + ": accuiring serving permit, available: " + servingQueue.availablePermits());
 831       return servingQueue.tryAcquire(maxWaitTimeInMs, TimeUnit.MILLISECONDS);
 832     }
 833
 834     /**
 835      * Returns a serving permit to the pool of available permits
 836      */
 837     void releaseServingPermit() {
 838       servingQueue.release();
 839       logger.finest(
 840           this + ": returned serving permit, available: " + servingQueue.availablePermits());
 841     }
 842
 843     /**
 844      * Returns the approximate number of threads waiting for serving permits.
 845      *
 846      * @return the number of waiting threads.
 847      */
 848     int getApproximateQueueLength() {
 849       return servingQueue.getQueueLength();
 850     }
 851
 852     /**
 853      * Returns the number of requests that can be queued on this specific
 854      * server. For servers without pending queue no queue (size=0) is allowed.
 855      */
 856     int getMaxPendingQueueSize() {
 857       return serverEntry.isFailFast() ? 0 : MAX_PENDING_QUEUE_LENGTH;
 858     }
 859
 860     /**
 861      * The dns prefix for this server, basically the first part of:
 862      * <instance>.<server_name>.<app-id>.appspot.com for a specific instance,
 863      * and <server_name>.<app-id>.appspot.com for just the server.
 864      */
 865     public String getDnsPrefix() {
 866       if (!isLoadBalanceServer()) {
 867         return serverInstance + "." + getName();
 868       } else {
 869         return getName();
 870       }
 871     }
 872
 873     String getName() {
 874       return serverEntry.getName();
 875     }
 876
 877     public int getInstances() {
 878       return serverEntry.getInstances();
 879     }
 880
 881     BackendServerState getState() {
 882       synchronized (ServerWrapper.this) {
 883         return serverState;
 884       }
 885     }
 886
 887     boolean isLoadBalanceServer() {
 888       return serverInstance == -1;
 889     }
 890
 891     @Override
 892     public String toString() {
 893       return serverInstance + "." + serverEntry.getName() + " state=" + serverState;
 894     }
 895
 896     public ContainerService getContainer() {
 897       return container;
 898     }
 899   }
 900 }