java/src/main/com/google/appengine/tools/development/AbstractBackendServers.java

   1 // Copyright 2011 Google Inc. All Rights Reserved.
   2
   3 package com.google.appengine.tools.development;
   4
   5 import com.google.appengine.api.backends.dev.LocalServerController;
   6 import com.google.appengine.tools.development.AbstractContainerService.LocalInitializationEnvironment;
   7 import com.google.appengine.tools.development.ApplicationConfigurationManager.ServerConfigurationHandle;
   8 import com.google.apphosting.api.ApiProxy;
   9 import com.google.apphosting.api.ApiProxy.Environment;
  10 import com.google.apphosting.utils.config.AppEngineWebXml;
  11 import com.google.apphosting.utils.config.BackendsXml;
  12 import com.google.apphosting.utils.config.WebModule;
  13 import com.google.common.collect.ImmutableMap;
  14 import com.google.common.collect.Maps;
  15
  16 import org.apache.commons.httpclient.HttpClient;
  17 import org.apache.commons.httpclient.methods.GetMethod;
  18
  19 import java.io.File;
  20 import java.io.IOException;
  21 import java.io.InputStream;
  22 import java.lang.reflect.Method;
  23 import java.net.MalformedURLException;
  24 import java.util.HashMap;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Map.Entry;
  28 import java.util.TreeMap;
  29 import java.util.concurrent.Semaphore;
  30 import java.util.concurrent.TimeUnit;
  31 import java.util.logging.Logger;
  32
  33 import javax.servlet.ServletException;
  34 import javax.servlet.http.HttpServletRequest;
  35 import javax.servlet.http.HttpServletResponse;
  36
  37 /**
  38  * Controls backend servers configured in appengine-web.xml. Each server is
  39  * started on a separate port. All servers run the same code as the main app.
  40  *
  41  *
  42  */
  43 public abstract class AbstractBackendServers implements BackendContainer, LocalServerController {
  44   private static final String X_GOOGLE_DEV_APPSERVER_SKIPADMINCHECK =
  45       "X-Google-DevAppserver-SkipAdminCheck";
  46   public static final String SYSTEM_PROPERTY_STATIC_PORT_NUM_PREFIX =
  47       "com.google.appengine.devappserver.";
  48
  49   private static final int AH_REQUEST_DEFAULT_TIMEOUT = 30 * 1000;
  50   private static final int AH_REQUEST_INFINITE_TIMEOUT = 0;
  51
  52   private static final Integer DEFAULT_INSTANCES = 1;
  53   private static final String DEFAULT_INSTANCE_CLASS = "B1";
  54   private static final Integer DEFAULT_MAX_CONCURRENT_REQUESTS = 10;
  55
  56   private enum BackendServerState {
  57     INITIALIZING, SLEEPING, RUNNING_START_REQUEST, RUNNING, STOPPING, STOPPED, SHUTDOWN;
  58   }
  59
  60   private static final int MAX_PENDING_QUEUE_LENGTH = 20;
  61   private static final int MAX_PENDING_QUEUE_TIME_MS = 10 * 1000;
  62   private static final int MAX_START_QUEUE_TIME_MS = 30 * 1000;
  63
  64   private String address;
  65   private ServerConfigurationHandle serverConfigurationHandle;
  66   private File externalResourceDir;
  67   private Map<String, Object> containerConfigProperties;
  68   private Map<AbstractBackendServers.ServerInstanceEntry, ServerWrapper> backendServers =
  69       ImmutableMap.copyOf(new HashMap<ServerInstanceEntry, ServerWrapper>());
  70   private Map<String, String> portMapping =
  71       ImmutableMap.copyOf(new HashMap<String, String>());
  72   protected Logger logger = Logger.getLogger(AbstractBackendServers.class.getName());
  73
  74   private Map<String, String> serviceProperties = new HashMap<String, String>();
  75
  76   private DevAppServer devAppServer;
  77
  78   @Override
  79   public void init(String address, ServerConfigurationHandle serverConfigurationHandle,
  80       File externalResourceDirectory, Map<String, Object> containerConfigProperties,
  81       DevAppServer devAppServer) {
  82     this.serverConfigurationHandle = serverConfigurationHandle;
  83     this.externalResourceDir = externalResourceDirectory;
  84     this.address = address;
  85     this.containerConfigProperties = containerConfigProperties;
  86     this.devAppServer = devAppServer;
  87   }
  88
  89   @Override
  90   public void setServiceProperties(Map<String, String> properties) {
  91     this.serviceProperties = properties;
  92   }
  93
  94   @Override
  95   public void shutdownAll() throws Exception {
  96     for (ServerWrapper server : backendServers.values()) {
  97       logger.finer("server shutdown: " + server);
  98       server.shutdown();
  99     }
 100     backendServers = ImmutableMap.copyOf(new HashMap<ServerInstanceEntry, ServerWrapper>());
 101   }
 102
 103   @Override
 104   public TreeMap<String, BackendStateInfo> getBackendState(String requestHostName) {
 105     TreeMap<String, BackendStateInfo> serverInfoMap = new TreeMap<String, BackendStateInfo>();
 106     for (ServerWrapper serverWrapper : backendServers.values()) {
 107       String name = serverWrapper.serverEntry.getName();
 108
 109       String listenAddress;
 110       if (requestHostName == null) {
 111         listenAddress = portMapping.get(serverWrapper.getDnsPrefix());
 112       } else {
 113         listenAddress = requestHostName + ":" + serverWrapper.port;
 114       }
 115
 116       BackendStateInfo ssi = serverInfoMap.get(name);
 117       if (ssi == null) {
 118         ssi = new BackendStateInfo(serverWrapper.serverEntry);
 119         serverInfoMap.put(name, ssi);
 120       }
 121       if (serverWrapper.isLoadBalanceServer()) {
 122         ssi.setState(serverWrapper.serverState.name().toLowerCase());
 123         ssi.setAddress(listenAddress);
 124       } else {
 125         ssi.add(new InstanceStateInfo(serverWrapper.serverInstance, listenAddress,
 126             serverWrapper.serverState.name().toLowerCase()));
 127       }
 128     }
 129     return serverInfoMap;
 130   }
 131
 132   @Override
 133   public synchronized void startBackend(String serverToStart) throws IllegalStateException {
 134     if (!checkServerExists(serverToStart)) {
 135       String message = String.format("Tried to start unknown server %s", serverToStart);
 136       logger.warning(message);
 137       throw new IllegalStateException(message);
 138     }
 139     for (ServerWrapper server : backendServers.values()) {
 140       if (server.getName().equals(serverToStart)) {
 141         if (server.getState() != BackendServerState.STOPPED) {
 142           continue;
 143         }
 144         if (server.isLoadBalanceServer()) {
 145           server.compareAndSetServerState(
 146               BackendServerState.RUNNING, BackendServerState.STOPPED);
 147           continue;
 148         }
 149         server.compareAndSetServerState(
 150             BackendServerState.SLEEPING, BackendServerState.STOPPED);
 151         server.sendStartRequest();
 152       }
 153     }
 154   }
 155
 156   @Override
 157   public synchronized void stopBackend(String serverToStop) throws Exception {
 158     if (!checkServerExists(serverToStop)) {
 159       String message = String.format("Tried to stop unknown server %s", serverToStop);
 160       logger.warning(message);
 161       throw new IllegalStateException(message);
 162     }
 163     for (ServerWrapper server : backendServers.values()) {
 164       if (server.getName().equals(serverToStop)) {
 165         if (server.getState() == BackendServerState.STOPPED) {
 166           continue;
 167         }
 168         if (server.isLoadBalanceServer()) {
 169           server.compareAndSetServerState(
 170               BackendServerState.STOPPED, BackendServerState.RUNNING);
 171           continue;
 172         }
 173         logger.fine("Stopping server: " + server.getDnsPrefix());
 174         server.shutdown();
 175         server.startup(true);
 176       }
 177     }
 178   }
 179
 180   @Override
 181   public void startupAll(ApiProxyLocal local) throws Exception {
 182     BackendsXml backendsXml = serverConfigurationHandle.getBackendsXml();
 183     if (backendsXml == null) {
 184       logger.fine("Got null backendsXml config.");
 185       return;
 186     }
 187     List<BackendsXml.Entry> servers = backendsXml.getBackends();
 188     if (servers.size() == 0) {
 189       logger.fine("No backends configured.");
 190       return;
 191     }
 192
 193     if (backendServers.size() != 0) {
 194       throw new Exception("Tried to start backendservers but some are already running.");
 195     }
 196     logger.finer("Found " + servers.size() + " configured backends.");
 197
 198     Map<AbstractBackendServers.ServerInstanceEntry, ServerWrapper> serverMap = Maps.newHashMap();
 199     for (BackendsXml.Entry entry : servers) {
 200       entry = resolveDefaults(entry);
 201
 202       for (int serverInstance = -1; serverInstance < entry.getInstances(); serverInstance++) {
 203         int port = checkForStaticPort(entry.getName(), serverInstance);
 204         ServerWrapper serverWrapper =
 205             new ServerWrapper(ContainerUtils.loadContainer(), entry, serverInstance, port);
 206         serverMap.put(new ServerInstanceEntry(entry.getName(), serverInstance), serverWrapper);
 207       }
 208     }
 209     this.backendServers = ImmutableMap.copyOf(serverMap);
 210
 211     String prettyAddress = address;
 212     if ("0.0.0.0".equals(address)) {
 213       prettyAddress = "127.0.0.1";
 214     }
 215
 216     Map<String, String> portMap = Maps.newHashMap();
 217     for (ServerWrapper serverWrapper : backendServers.values()) {
 218       logger.finer(
 219           "starting server: " + serverWrapper.serverInstance + "." + serverWrapper.getName()
 220               + " on " + address + ":" + serverWrapper.port);
 221       ApiProxy.Delegate configured = ApiProxy.getDelegate();
 222       try {
 223         ApiProxy.setDelegate(local);
 224         serverWrapper.startup(false);
 225       } finally {
 226         ApiProxy.setDelegate(configured);
 227       }
 228
 229       portMap.put(serverWrapper.getDnsPrefix(), prettyAddress + ":" + serverWrapper.port);
 230     }
 231     this.portMapping = ImmutableMap.copyOf(portMap);;
 232
 233     for (ServerWrapper serverWrapper : backendServers.values()) {
 234       if (serverWrapper.isLoadBalanceServer()) {
 235         continue;
 236       }
 237       serverWrapper.sendStartRequest();
 238     }
 239   }
 240
 241   private BackendsXml.Entry resolveDefaults(BackendsXml.Entry entry) {
 242     return new BackendsXml.Entry(
 243         entry.getName(),
 244         entry.getInstances() == null ? DEFAULT_INSTANCES : entry.getInstances(),
 245         entry.getInstanceClass() == null ? DEFAULT_INSTANCE_CLASS : entry.getInstanceClass(),
 246         entry.getMaxConcurrentRequests() == null ? DEFAULT_MAX_CONCURRENT_REQUESTS :
 247                                                    entry.getMaxConcurrentRequests(),
 248         entry.getOptions(),
 249         entry.getState() == null ? BackendsXml.State.STOP : entry.getState());
 250   }
 251
 252   /**
 253    * Forward a request to a specific server and instance. This will call the
 254    * specified instance request dispatcher so the request is handled in the
 255    * right server context.
 256    */
 257   protected abstract void forwardToServer(String requestedServer, int instance,
 258       HttpServletRequest hrequest,
 259       HttpServletResponse hresponse) throws IOException, ServletException;
 260
 261   /**
 262    * This method guards access to servers to limit the number of concurrent
 263    * requests. Each request running on a server must acquire a serving permit.
 264    * If no permits are available a 500 response should be sent.
 265    *
 266    * @param serverName The server for which to acquire a permit.
 267    * @param instanceNumber The server instance for which to acquire a permit.
 268    * @param allowQueueOnBackends If set to false the method will return
 269    *        instantly, if set to true (and the specified server allows pending
 270    *        queues) this method can block for up to 10 s waiting for a serving
 271    *        permit to become available.
 272    * @return true if a permit was acquired, false otherwise
 273    */
 274   public boolean acquireServingPermit(
 275       String serverName, int instanceNumber, boolean allowQueueOnBackends) {
 276     logger.finest(
 277         String.format("trying to get serving permit for server %d.%s", instanceNumber, serverName));
 278     try {
 279       ServerWrapper server = getServerWrapper(serverName, instanceNumber);
 280       int maxQueueTime = 0;
 281
 282       synchronized (server) {
 283         if (!server.acceptsConnections()) {
 284           logger.finest(server + ": got request but server is not in a serving state");
 285           return false;
 286         }
 287         if (server.getApproximateQueueLength() > MAX_PENDING_QUEUE_LENGTH) {
 288           logger.finest(server + ": server queue is full");
 289           return false;
 290         }
 291         if (server.getState() == BackendServerState.SLEEPING) {
 292           logger.finest(server + ": waking up sleeping server");
 293           server.sendStartRequest();
 294         }
 295
 296         if (server.getState() == BackendServerState.RUNNING_START_REQUEST) {
 297           maxQueueTime = MAX_START_QUEUE_TIME_MS;
 298         } else if (allowQueueOnBackends && server.getMaxPendingQueueSize() > 0) {
 299           maxQueueTime = MAX_PENDING_QUEUE_TIME_MS;
 300         }
 301       }
 302       boolean gotPermit = server.acquireServingPermit(maxQueueTime);
 303       logger.finest(server + ": tried to get server permit, timeout=" + maxQueueTime + " success="
 304           + gotPermit);
 305       return gotPermit;
 306     } catch (InterruptedException e) {
 307       logger.finest(
 308           instanceNumber + "." + serverName + ": got interrupted while waiting for serving permit");
 309       return false;
 310     }
 311   }
 312
 313   /**
 314    * Reserves an instance for this request. For workers this method will return
 315    * -1 if no free instances are available. For backends this method will assign
 316    * this request to the instance with the shortest queue and block until that
 317    * instance is ready to serve the request.
 318    *
 319    * @param requestedServer Name of the server the request is to.
 320    * @return the instance id of an available server instance, or -1 if no
 321    *         instance is available.
 322    */
 323   public int getAndReserveFreeInstance(String requestedServer) {
 324     logger.finest("trying to get serving permit for server " + requestedServer);
 325
 326     ServerWrapper server = getServerWrapper(requestedServer, -1);
 327     if (server == null) {
 328       return -1;
 329     }
 330     if (!server.acceptsConnections()) {
 331       return -1;
 332     }
 333     int instanceNum = server.getInstances();
 334     for (int i = 0; i < instanceNum; i++) {
 335       if (acquireServingPermit(requestedServer, i, false)) {
 336         return i;
 337       }
 338     }
 339     if (server.getMaxPendingQueueSize() > 0) {
 340       return addToShortestInstanceQueue(requestedServer);
 341     } else {
 342       logger.finest("no servers free");
 343       return -1;
 344     }
 345   }
 346
 347   /**
 348    * Will add this request to the queue of the instance with the approximate
 349    * shortest queue. This method will block for up to 10 seconds until a permit
 350    * is received.
 351    *
 352    * @param requestedServer the server name
 353    * @return the instance where the serving permit was reserved, or -1 if all
 354    *         instance queues are full
 355    */
 356   int addToShortestInstanceQueue(String requestedServer) {
 357     logger.finest(requestedServer + ": no instances free, trying to find a queue");
 358     int shortestQueue = MAX_PENDING_QUEUE_LENGTH;
 359     ServerWrapper instanceWithShortestQueue = null;
 360     for (ServerWrapper server : backendServers.values()) {
 361       if (!server.acceptsConnections()) {
 362         continue;
 363       }
 364       int serverQueue = server.getApproximateQueueLength();
 365       if (shortestQueue > serverQueue) {
 366         instanceWithShortestQueue = server;
 367         shortestQueue = serverQueue;
 368       }
 369     }
 370
 371     try {
 372       if (shortestQueue < MAX_PENDING_QUEUE_LENGTH) {
 373         logger.finest("adding request to queue on instance: " + instanceWithShortestQueue);
 374         if (instanceWithShortestQueue.acquireServingPermit(MAX_PENDING_QUEUE_TIME_MS)) {
 375           logger.finest("ready to serve request on instance: " + instanceWithShortestQueue);
 376           return instanceWithShortestQueue.serverInstance;
 377         }
 378       }
 379     } catch (InterruptedException e) {
 380       logger.finer("interupted while queued at server " + instanceWithShortestQueue);
 381     }
 382     return -1;
 383   }
 384
 385   /**
 386    * Method for returning a serving permit after a request has completed.
 387    *
 388    * @param serverName The server name
 389    * @param instance The server instance
 390    */
 391   public void returnServingPermit(String serverName, int instance) {
 392     ServerWrapper server = getServerWrapper(serverName, instance);
 393     server.releaseServingPermit();
 394   }
 395
 396   /**
 397    * Verifies if a specific server/instance is configured.
 398    *
 399    * @param serverName The server name
 400    * @param instance The server instance
 401    * @return true if the server/instance is configured, false otherwise.
 402    */
 403   public boolean checkInstanceExists(String serverName, int instance) {
 404     return getServerWrapper(serverName, instance) != null;
 405   }
 406
 407   /**
 408    * Verifies if a specific server is configured.
 409    *
 410    * @param serverName The server name
 411    * @return true if the server is configured, false otherwise.
 412    */
 413   public boolean checkServerExists(String serverName) {
 414     return checkInstanceExists(serverName, -1);
 415   }
 416
 417   /**
 418    * Verifies if a specific server is stopped.
 419    *
 420    * @param serverName The server name
 421    * @return true if the server is stopped, false otherwise.
 422    */
 423   public boolean checkServerStopped(String serverName) {
 424     return checkInstanceStopped(serverName, -1);
 425   }
 426
 427   /**
 428    * Verifies if a specific server/instance is stopped.
 429    *
 430    * @param serverName The server name
 431    * @param instance The server instance
 432    * @return true if the server/instance is stopped, false otherwise.
 433    */
 434   public boolean checkInstanceStopped(String serverName, int instance) {
 435     return !getServerWrapper(serverName, instance).acceptsConnections();
 436   }
 437
 438   /**
 439    * Allows the servers API to get the current mapping from server and instance
 440    * to listening ports.
 441    *
 442    * @return The port map
 443    */
 444   public Map<String, String> getPortMapping() {
 445     return this.portMapping;
 446   }
 447
 448   /**
 449    * Returns the server instance serving on a specific local port
 450    *
 451    * @param port the local tcp port that received the request
 452    * @return the server instance, or -1 if no server instance is running on that
 453    *         port
 454    */
 455   public int getServerInstanceFromPort(int port) {
 456     ServerWrapper server = getServerWrapperFromPort(port);
 457     if (server != null) {
 458       return server.serverInstance;
 459     } else {
 460       return -1;
 461     }
 462   }
 463
 464   /**
 465    * Returns the server serving on a specific local port
 466    *
 467    * @param port the local tcp port that received the request
 468    * @return the server name, or null if no server instance is running on that
 469    *         port
 470    */
 471   public String getServerNameFromPort(int port) {
 472     ServerWrapper server = getServerWrapperFromPort(port);
 473     if (server != null) {
 474       return server.getName();
 475     } else {
 476       return null;
 477     }
 478   }
 479
 480   /**
 481    * Convenience method for getting the ServerWrapper running on a specific port
 482    */
 483   private ServerWrapper getServerWrapperFromPort(int port) {
 484     for (Entry<ServerInstanceEntry, ServerWrapper> entry : backendServers.entrySet()) {
 485       if (entry.getValue().port == port) {
 486         return entry.getValue();
 487       }
 488     }
 489     return null;
 490   }
 491
 492   /**
 493    * Convenience method for getting the ServerWrapper for a specific
 494    * server/instance
 495    */
 496   protected ServerWrapper getServerWrapper(String serverName, int instanceNumber) {
 497     return backendServers.get(new ServerInstanceEntry(serverName, instanceNumber));
 498   }
 499
 500   /**
 501    * Verifies if the specific port is statically configured, if not it will
 502    * return 0 which instructs jetty to pick the port
 503    *
 504    *  Ports can be statically configured by a system property at
 505    * com.google.appengine.server.<server-name>.port for the server and
 506    * com.google.appengine.server.<server-name>.<instance-id>.port for individual
 507    * instances
 508    *
 509    * @param server the name of the configured server
 510    * @param instance the instance number to configure
 511    * @return the statically configured port, or 0 if none is configured
 512    */
 513   private int checkForStaticPort(String server, int instance) {
 514     StringBuilder key = new StringBuilder();
 515     key.append(SYSTEM_PROPERTY_STATIC_PORT_NUM_PREFIX);
 516     key.append(server);
 517     if (instance >= 0) {
 518       key.append("." + instance);
 519     }
 520     key.append(".port");
 521     String configuredPort = serviceProperties.get(key.toString());
 522     if (configuredPort != null) {
 523       return Integer.parseInt(configuredPort);
 524     } else {
 525       return 0;
 526     }
 527   }
 528
 529   /**
 530    * Class that allows the key in the server map to be the
 531    * (servername,instanceid) tuple. Overrides equals() and hashcode() to
 532    * function as a hashtable key
 533    *
 534    */
 535   static class ServerInstanceEntry {
 536     private final int instanceNumber;
 537     private final String serverName;
 538
 539     /**
 540      * @param serverName
 541      * @param instanceNumber
 542      */
 543     public ServerInstanceEntry(String serverName, int instanceNumber) {
 544       this.serverName = serverName;
 545       this.instanceNumber = instanceNumber;
 546     }
 547
 548     @Override
 549     public boolean equals(Object o) {
 550       if (!(o instanceof ServerInstanceEntry)) {
 551         return false;
 552       }
 553
 554       ServerInstanceEntry that = (ServerInstanceEntry) o;
 555       if (this.serverName != null) {
 556         if (!this.serverName.equals(that.serverName)) {
 557           return false;
 558         }
 559       } else {
 560         if (that.serverName != null) {
 561           return false;
 562         }
 563       }
 564
 565       if (this.instanceNumber != that.instanceNumber) {
 566         return false;
 567       }
 568       return true;
 569     }
 570
 571     @Override
 572     public int hashCode() {
 573       int hash = 17;
 574       hash = 31 * hash + instanceNumber;
 575       if (serverName != null) {
 576         hash = 31 * hash + serverName.hashCode();
 577       }
 578       return hash;
 579     }
 580
 581     @Override
 582     public String toString() {
 583       return instanceNumber + "." + serverName;
 584     }
 585   }
 586
 587   /**
 588    * Wraps a container service and contains extra information such as the
 589    * instanceid of the current container as well as the port number it is
 590    * running on.
 591    */
 592   protected class ServerWrapper {
 593
 594     private final ContainerService container;
 595     private final int serverInstance;
 596     private int port;
 597     private final BackendsXml.Entry serverEntry;
 598
 599     private BackendServerState serverState = BackendServerState.SHUTDOWN;
 600
 601     private final Semaphore servingQueue = new Semaphore(0, true);
 602
 603     public ServerWrapper(ContainerService containerService,
 604         BackendsXml.Entry serverEntry,
 605         int instance, int port) {
 606       this.container = containerService;
 607       this.serverEntry = serverEntry;
 608       this.serverInstance = instance;
 609       this.port = port;
 610     }
 611
 612     /**
 613      * Triggers an HTTP GET to /_ah/start
 614      *
 615      *  This method will keep on trying until it receives a non-error response
 616      * code from the server.
 617      *
 618      * @param timeoutInMs Timeout in milliseconds, 0 indicates no timeout.
 619      *
 620      */
 621     private void sendStartRequest(int timeoutInMs) {
 622       try {
 623         String urlString =
 624             String.format("http://%s:%d/_ah/start", address, ServerWrapper.this.port);
 625         logger.finer("sending start request to: " + urlString);
 626
 627         HttpClient httpClient = new HttpClient();
 628         httpClient.getParams().setConnectionManagerTimeout(timeoutInMs);
 629
 630         GetMethod request = new GetMethod(urlString);
 631         request.addRequestHeader(X_GOOGLE_DEV_APPSERVER_SKIPADMINCHECK, "true");
 632         try {
 633           int returnCode = httpClient.executeMethod(request);
 634
 635           byte[] buffer = new byte[1024];
 636           InputStream in = request.getResponseBodyAsStream();
 637           while (in.read(buffer) != -1) {
 638           }
 639           if ((returnCode >= 200 && returnCode < 300) || returnCode == 404) {
 640             logger.fine(
 641                 String.format("backend server %d.%s request to /_ah/start completed, code=%d",
 642                     serverInstance, serverEntry.getName(), returnCode));
 643             compareAndSetServerState(
 644                 BackendServerState.RUNNING, BackendServerState.RUNNING_START_REQUEST);
 645             servingQueue.release(serverEntry.getMaxConcurrentRequests());
 646           } else {
 647             logger.warning("Start request to /_ah/start on server " + serverInstance + "."
 648                 + serverEntry.getName() + " failed (HTTP status code=" + returnCode
 649                 + "). Retrying...");
 650             Thread.sleep(1000);
 651             sendStartRequest(timeoutInMs);
 652           }
 653         } finally {
 654           request.releaseConnection();
 655         }
 656       } catch (MalformedURLException e) {
 657         logger.severe(String.format(
 658             "Unable to send start request to server: %d.%s, " + "MalformedURLException: %s",
 659             serverInstance, serverEntry.getName(), e.getMessage()));
 660       } catch (Exception e) {
 661         logger.warning(String.format(
 662             "Got exception while performing /_ah/start " + "request on server: %d.%s, %s: %s",
 663             serverInstance, serverEntry.getName(), e.getClass().getName(), e.getMessage()));
 664       }
 665     }
 666
 667     /**
 668      * Shut down the server.
 669      *
 670      * Will trigger any shutdown hooks installed by the
 671      * {@link com.google.appengine.api.LifecycleManager}
 672      *
 673      * @throws Exception
 674      */
 675     public void shutdown() throws Exception {
 676       synchronized (ServerWrapper.this) {
 677         if (serverState == BackendServerState.RUNNING
 678             || serverState == BackendServerState.RUNNING_START_REQUEST) {
 679           triggerLifecycleShutdownHook();
 680         }
 681         getContainer().shutdown();
 682         serverState = BackendServerState.SHUTDOWN;
 683       }
 684     }
 685
 686     void startup(boolean setStateToStopped) throws Exception {
 687       compareAndSetServerState(BackendServerState.INITIALIZING, BackendServerState.SHUTDOWN);
 688
 689       getContainer().configure(ContainerUtils.getServerInfo(),
 690           address,
 691           port,
 692           serverConfigurationHandle,
 693           externalResourceDir,
 694           containerConfigProperties,
 695           serverInstance,
 696           devAppServer);
 697       getContainer().createConnection();
 698       getContainer().startup();
 699
 700       this.port = getContainer().getPort();
 701       if (setStateToStopped) {
 702         compareAndSetServerState(BackendServerState.STOPPED, BackendServerState.INITIALIZING);
 703       } else {
 704         logger.info(
 705             "server: " + serverInstance + "." + serverEntry.getName() + " is running on port "
 706                 + this.port);
 707         if (isLoadBalanceServer()) {
 708           compareAndSetServerState(
 709               BackendServerState.RUNNING, BackendServerState.INITIALIZING);
 710         } else {
 711           compareAndSetServerState(
 712               BackendServerState.SLEEPING, BackendServerState.INITIALIZING);
 713         }
 714       }
 715     }
 716
 717     void sendStartRequest() {
 718       compareAndSetServerState(
 719           BackendServerState.RUNNING_START_REQUEST, BackendServerState.SLEEPING);
 720       if (serverInstance >= 0) {
 721         Thread requestThread = new Thread(new Runnable() {
 722           @Override
 723           public void run() {
 724             sendStartRequest(AH_REQUEST_INFINITE_TIMEOUT);
 725           }
 726         });
 727         requestThread.setDaemon(true);
 728         requestThread.setName(
 729             "BackendServersStartRequestThread." + serverInstance + "." + serverEntry.getName());
 730         requestThread.start();
 731       }
 732     }
 733
 734     /**
 735      * This method will trigger any shutdown hooks registered with the current
 736      * server.
 737      *
 738      * Some class loader trickery is required to make sure that we get the
 739      * {@link com.google.appengine.api.LifecycleManager} responsible for this
 740      * server instance.
 741      */
 742     private void triggerLifecycleShutdownHook() {
 743       Environment prevEnvironment = ApiProxy.getCurrentEnvironment();
 744       try {
 745         ClassLoader serverClassLoader = getContainer().getAppContext().getClassLoader();
 746
 747         Class<?> lifeCycleManagerClass =
 748             Class.forName("com.google.appengine.api.LifecycleManager", true, serverClassLoader);
 749         Method lifeCycleManagerGetter = lifeCycleManagerClass.getMethod("getInstance");
 750         Object userThreadLifeCycleManager = lifeCycleManagerGetter.invoke(null, new Object[0]);
 751
 752         Method beginShutdown = lifeCycleManagerClass.getMethod("beginShutdown", long.class);
 753         AppEngineWebXml appEngineWebXml = getContainer().getAppEngineWebXmlConfig();
 754         ApiProxy.setEnvironmentForCurrentThread(new LocalInitializationEnvironment(
 755             appEngineWebXml.getAppId(), WebModule.getServerName(appEngineWebXml),
 756             appEngineWebXml.getMajorVersionId(), serverInstance));
 757
 758         try {
 759           beginShutdown.invoke(userThreadLifeCycleManager, AH_REQUEST_DEFAULT_TIMEOUT);
 760         } catch (Exception e) {
 761           logger.warning(
 762               String.format("got exception when running shutdown hook on server %d.%s",
 763                   serverInstance, serverEntry.getName()));
 764           e.printStackTrace();
 765         }
 766       } catch (Exception e) {
 767         logger.severe(
 768             String.format("Exception during reflective call to "
 769                 + "LifecycleManager.beginShutdown on server %d.%s, got %s: %s", serverInstance,
 770                 serverEntry.getName(), e.getClass().getName(), e.getMessage()));
 771       } finally {
 772         ApiProxy.setEnvironmentForCurrentThread(prevEnvironment);
 773       }
 774     }
 775
 776     /**
 777      * Checks if the server is in a state where it can accept incoming requests.
 778      *
 779      * @return true if the server can accept incoming requests, false otherwise.
 780      */
 781     boolean acceptsConnections() {
 782       synchronized (ServerWrapper.this) {
 783         return (serverState == BackendServerState.RUNNING
 784             || serverState == BackendServerState.RUNNING_START_REQUEST
 785             || serverState == BackendServerState.SLEEPING);
 786       }
 787     }
 788
 789     /**
 790      * Updates the current server state and verifies that the previous state is
 791      * what is expected.
 792      *
 793      * @param newState The new state to change to
 794      * @param acceptablePreviousStates Acceptable previous states
 795      * @throws IllegalStateException If the current state is not one of the
 796      *         acceptable previous states
 797      */
 798     private void compareAndSetServerState(
 799         BackendServerState newState, BackendServerState... acceptablePreviousStates)
 800         throws IllegalStateException {
 801       synchronized (ServerWrapper.this) {
 802         for (BackendServerState acceptableStates : acceptablePreviousStates) {
 803           if (serverState == acceptableStates) {
 804             serverState = newState;
 805             return;
 806           }
 807         }
 808       }
 809       StringBuilder error = new StringBuilder();
 810       error.append("Tried to change state to " + newState);
 811       error.append(" on server " + toString());
 812       error.append(" but previous state is not ");
 813       for (int i = 0; i < acceptablePreviousStates.length; i++) {
 814         error.append(acceptablePreviousStates[i].name() + " | ");
 815       }
 816       throw new IllegalStateException(error.toString());
 817     }
 818
 819     /**
 820      * Acquires a serving permit for this server.
 821      *
 822      * @param maxWaitTimeInMs Max wait time in ms
 823      * @return true if a serving permit was acquired within the allowed time,
 824      *         false if not permit was required.
 825      * @throws InterruptedException If the thread was interrupted while waiting.
 826      */
 827     boolean acquireServingPermit(int maxWaitTimeInMs) throws InterruptedException {
 828       logger.finest(
 829           this + ": accuiring serving permit, available: " + servingQueue.availablePermits());
 830       return servingQueue.tryAcquire(maxWaitTimeInMs, TimeUnit.MILLISECONDS);
 831     }
 832
 833     /**
 834      * Returns a serving permit to the pool of available permits
 835      */
 836     void releaseServingPermit() {
 837       servingQueue.release();
 838       logger.finest(
 839           this + ": returned serving permit, available: " + servingQueue.availablePermits());
 840     }
 841
 842     /**
 843      * Returns the approximate number of threads waiting for serving permits.
 844      *
 845      * @return the number of waiting threads.
 846      */
 847     int getApproximateQueueLength() {
 848       return servingQueue.getQueueLength();
 849     }
 850
 851     /**
 852      * Returns the number of requests that can be queued on this specific
 853      * server. For servers without pending queue no queue (size=0) is allowed.
 854      */
 855     int getMaxPendingQueueSize() {
 856       return serverEntry.isFailFast() ? 0 : MAX_PENDING_QUEUE_LENGTH;
 857     }
 858
 859     /**
 860      * The dns prefix for this server, basically the first part of:
 861      * <instance>.<server_name>.<app-id>.appspot.com for a specific instance,
 862      * and <server_name>.<app-id>.appspot.com for just the server.
 863      */
 864     public String getDnsPrefix() {
 865       if (!isLoadBalanceServer()) {
 866         return serverInstance + "." + getName();
 867       } else {
 868         return getName();
 869       }
 870     }
 871
 872     String getName() {
 873       return serverEntry.getName();
 874     }
 875
 876     public int getInstances() {
 877       return serverEntry.getInstances();
 878     }
 879
 880     BackendServerState getState() {
 881       synchronized (ServerWrapper.this) {
 882         return serverState;
 883       }
 884     }
 885
 886     boolean isLoadBalanceServer() {
 887       return serverInstance == -1;
 888     }
 889
 890     @Override
 891     public String toString() {
 892       return serverInstance + "." + serverEntry.getName() + " state=" + serverState;
 893     }
 894
 895     public ContainerService getContainer() {
 896       return container;
 897     }
 898   }
 899 }