hphp/runtime/server/http-server.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/server/http-server.h"
  18
  19 #include "hphp/runtime/base/backtrace.h"
  20 #include "hphp/runtime/base/externals.h"
  21 #include "hphp/runtime/base/file-util.h"
  22 #include "hphp/runtime/base/http-client.h"
  23 #include "hphp/runtime/base/init-fini-node.h"
  24 #include "hphp/runtime/base/memory-manager.h"
  25 #include "hphp/runtime/base/program-functions.h"
  26 #include "hphp/runtime/base/runtime-option.h"
  27 #include "hphp/runtime/debugger/debugger.h"
  28 #include "hphp/runtime/ext/apc/ext_apc.h"
  29 #include "hphp/runtime/server/admin-request-handler.h"
  30 #include "hphp/runtime/server/http-request-handler.h"
  31 #include "hphp/runtime/server/replay-transport.h"
  32 #include "hphp/runtime/server/server-stats.h"
  33 #include "hphp/runtime/server/static-content-cache.h"
  34 #include "hphp/runtime/server/warmup-request-handler.h"
  35 #include "hphp/runtime/server/xbox-server.h"
  36 #include "hphp/runtime/vm/vm-regs.h"
  37
  38 #include "hphp/util/alloc.h"
  39 #include "hphp/util/boot-stats.h"
  40 #include "hphp/util/light-process.h"
  41 #include "hphp/util/logger.h"
  42 #include "hphp/util/process.h"
  43 #include "hphp/util/ssl-init.h"
  44 #include "hphp/util/stack-trace.h"
  45 #include "hphp/util/struct-log.h"
  46 #include "hphp/util/sync-signal.h"
  47
  48 #include <folly/Conv.h>
  49 #include <folly/Format.h>
  50 #include <folly/portability/Unistd.h>
  51
  52 #include <signal.h>
  53 #include <fstream>
  54
  55 #ifdef __linux__
  56 void DisableFork() __attribute__((__weak__));
  57 void EnableForkLogging() __attribute__((__weak__));
  58 extern "C" void __gcov_flush() __attribute__((__weak__));
  59 #endif
  60
  61 namespace HPHP {
  62
  63 ///////////////////////////////////////////////////////////////////////////////
  64 // statics
  65
  66 std::shared_ptr<HttpServer> HttpServer::Server;
  67 time_t HttpServer::StartTime;
  68 std::atomic<double> HttpServer::LoadFactor{1.0};
  69 std::atomic_int HttpServer::QueueDiscount{0};
  70 std::atomic_int HttpServer::SignalReceived{0};
  71 std::atomic_int_fast64_t HttpServer::PrepareToStopTime{0};
  72 time_t HttpServer::OldServerStopTime;
  73 std::vector<ShutdownStat> HttpServer::ShutdownStats;
  74 folly::MicroSpinLock HttpServer::StatsLock;
  75
  76 // signals upon which the server shuts down gracefully
  77 static const int kTermSignals[] = { SIGHUP, SIGINT, SIGTERM, SIGUSR1 };
  78
  79 static void on_kill_server(int sig) {   // runs in signal handler thread.
  80   static std::atomic_flag flag = ATOMIC_FLAG_INIT;
  81   if (flag.test_and_set()) return;      // it was already called once.
  82
  83   if (HttpServer::Server) {
  84     // Stop handling signals, because the
  85     // server is already shutting down, and we don't want to use the default
  86     // handlers which may immediately terminate the process.
  87     ignore_sync_signals();
  88
  89     int zero = 0;
  90     HttpServer::SignalReceived.compare_exchange_strong(zero, sig);
  91     // SignalReceived may possibly be set in HttpServer::stopOnSignal().
  92     HttpServer::Server->stop();
  93   } else {
  94     reset_sync_signals();
  95     raise(sig);                         // invoke default handler
  96   }
  97   pthread_exit(nullptr);                // terminate signal handler thread
  98 }
  99
 100 static void exit_on_timeout(int sig) {
 101   // Must only call async-signal-safe functions.
 102   kill(getpid(), SIGKILL);
 103   // we really shouldn't get here, but who knows.
 104   // abort so we catch it as a crash.
 105   abort();
 106 }
 107
 108 ///////////////////////////////////////////////////////////////////////////////
 109
 110 HttpServer::HttpServer() {
 111   LoadFactor = RuntimeOption::EvalInitialLoadFactor;
 112
 113   // enabling mutex profiling, but it's not turned on
 114   LockProfiler::s_pfunc_profile = server_stats_log_mutex;
 115
 116   int startingThreadCount = RuntimeOption::ServerThreadCount;
 117   if (RuntimeOption::ServerWarmupThrottleRequestCount > 0) {
 118     startingThreadCount =
 119       std::min(startingThreadCount,
 120                RuntimeOption::ServerWarmupThrottleThreadCount);
 121   }
 122   auto serverFactory = ServerFactoryRegistry::getInstance()->getFactory
 123       (RuntimeOption::ServerType);
 124   const std::string address = RuntimeOption::ServerFileSocket.empty()
 125     ? RuntimeOption::ServerIP : RuntimeOption::ServerFileSocket;
 126   ServerOptions options(address, RuntimeOption::ServerPort,
 127     RuntimeOption::ServerThreadCount, startingThreadCount,
 128     RuntimeOption::ServerQueueCount);
 129   options.m_useFileSocket = !RuntimeOption::ServerFileSocket.empty();
 130   options.m_serverFD = RuntimeOption::ServerPortFd;
 131   options.m_sslFD = RuntimeOption::SSLPortFd;
 132   options.m_takeoverFilename = RuntimeOption::TakeoverFilename;
 133   options.m_hugeThreads = RuntimeOption::ServerHugeThreadCount;
 134   options.m_hugeStackKb = RuntimeOption::ServerHugeStackKb;
 135   options.m_loop_sample_rate = RuntimeOption::ServerLoopSampleRate;
 136   m_pageServer = serverFactory->createServer(options);
 137   m_pageServer->addTakeoverListener(this);
 138   m_pageServer->addServerEventListener(this);
 139
 140   if (startingThreadCount != RuntimeOption::ServerThreadCount) {
 141     auto handlerFactory = std::make_shared<WarmupRequestHandlerFactory>(
 142       m_pageServer.get(),
 143       RuntimeOption::ServerWarmupThrottleRequestCount,
 144       RuntimeOption::RequestTimeoutSeconds);
 145     m_pageServer->setRequestHandlerFactory([handlerFactory] {
 146       return handlerFactory->createHandler();
 147     });
 148   } else {
 149     m_pageServer->setRequestHandlerFactory<HttpRequestHandler>(
 150       RuntimeOption::RequestTimeoutSeconds);
 151   }
 152
 153   if (RuntimeOption::EnableSSL) {
 154     assertx(SSLInit::IsInited());
 155     m_pageServer->enableSSL(RuntimeOption::SSLPort);
 156   }
 157
 158   if (RuntimeOption::EnableSSLWithPlainText) {
 159     assertx(SSLInit::IsInited());
 160     m_pageServer->enableSSLWithPlainText();
 161   }
 162
 163   ServerOptions admin_options(RuntimeOption::AdminServerIP,
 164                               RuntimeOption::AdminServerPort,
 165                               RuntimeOption::AdminThreadCount);
 166   m_adminServer = serverFactory->createServer(admin_options);
 167   m_adminServer->setRequestHandlerFactory<AdminRequestHandler>(
 168     RuntimeOption::RequestTimeoutSeconds);
 169
 170   for (auto const& info : RuntimeOption::SatelliteServerInfos) {
 171     auto satellite(SatelliteServer::Create(info));
 172     if (satellite) {
 173       m_satellites.push_back(std::move(satellite));
 174     }
 175   }
 176
 177   if (RuntimeOption::XboxServerPort != 0) {
 178     auto xboxInfo = std::make_shared<XboxServerInfo>();
 179     auto satellite = SatelliteServer::Create(xboxInfo);
 180     if (satellite) {
 181       m_satellites.push_back(std::move(satellite));
 182     }
 183   }
 184
 185   StaticContentCache::TheCache.load();
 186
 187   m_counterCallback.init(
 188     [this](std::map<std::string, int64_t>& counters) {
 189       counters["ev_connections"] = m_pageServer->getLibEventConnectionCount();
 190       counters["inflight_requests"] = m_pageServer->getActiveWorker();
 191       counters["queued_requests"] = m_pageServer->getQueuedJobs();
 192
 193       auto const sat_requests = getSatelliteRequestCount();
 194       counters["satellite_inflight_requests"] = sat_requests.first;
 195       counters["satellite_queued_requests"] = sat_requests.second;
 196     }
 197   );
 198
 199   for (auto const sig : kTermSignals) {
 200     sync_signal(sig, on_kill_server);
 201   }
 202 }
 203
 204 // Synchronously stop satellites
 205 void HttpServer::onServerShutdown() {
 206   InitFiniNode::ServerFini();
 207
 208   Eval::Debugger::Stop();
 209   if (RuntimeOption::EnableDebuggerServer) {
 210     Logger::Info("debugger server stopped");
 211   }
 212
 213   // When a new instance of HPHP has taken over our page server socket,
 214   // stop our admin server and satellites so it can acquire those
 215   // ports.
 216   if (RuntimeOption::AdminServerPort) {
 217     m_adminServer->stop();
 218   }
 219   for (unsigned int i = 0; i < m_satellites.size(); i++) {
 220     std::string name = m_satellites[i]->getName();
 221     m_satellites[i]->stop();
 222     Logger::Info("satellite server %s stopped", name.c_str());
 223   }
 224   XboxServer::Stop();
 225   if (RuntimeOption::AdminServerPort) {
 226     m_adminServer->waitForEnd();
 227     Logger::Info("admin server stopped");
 228   }
 229 }
 230
 231 void HttpServer::takeoverShutdown() {
 232   // We want to synchronously shut down our satellite servers to free up ports,
 233   // then asynchronously shut down everything else.
 234   onServerShutdown();
 235   stop();
 236 }
 237
 238 void HttpServer::serverStopped(HPHP::Server* server) {
 239   Logger::Info("Page server stopped");
 240   assertx(server == m_pageServer.get());
 241   removePid();
 242
 243   auto sockFile = RuntimeOption::ServerFileSocket;
 244   if (!sockFile.empty()) {
 245     unlink(sockFile.c_str());
 246   }
 247
 248   LogShutdownStats();
 249 }
 250
 251 void HttpServer::playShutdownRequest(const std::string& fileName) {
 252   if (fileName.empty()) return;
 253   Logger::Info("playing request upon shutdown %s", fileName.c_str());
 254   try {
 255     ReplayTransport rt;
 256     rt.replayInput(fileName.c_str());
 257     HttpRequestHandler handler(0);
 258     handler.run(&rt);
 259     if (rt.getResponseCode() == 200) {
 260       Logger::Info("successfully finished request: %s", rt.getUrl());
 261     } else {
 262       Logger::Error("request unsuccessful: %s", rt.getUrl());
 263     }
 264   } catch (...) {
 265     Logger::Error("got exception when playing request: %s",
 266                   fileName.c_str());
 267   }
 268 }
 269
 270 HttpServer::~HttpServer() {
 271   m_counterCallback.deinit();
 272   stop();
 273 }
 274
 275 static StaticString s_file{"file"}, s_line{"line"};
 276
 277 void HttpServer::runOrExitProcess() {
 278   if (StaticContentCache::TheFileCache &&
 279       StructuredLog::enabled() &&
 280       StructuredLog::coinflip(RuntimeOption::EvalStaticContentsLogRate)) {
 281     CacheManager::setLogger([](bool existsCheck, const std::string& name) {
 282         auto record = StructuredLogEntry{};
 283         record.setInt("existsCheck", existsCheck);
 284         record.setStr("file", name);
 285         bool needsCppStack = true;
 286         if (!g_context.isNull()) {
 287           VMRegAnchor _;
 288           if (vmfp()) {
 289             auto const bt =
 290               createBacktrace(BacktraceArgs().withArgValues(false));
 291             std::vector<std::string> frameStrings;
 292             std::vector<folly::StringPiece> frames;
 293             for (int i = 0; i < bt.size(); i++) {
 294               auto f = tvCastToArrayLike(bt.rvalAt(i).tv());
 295               if (f.exists(s_file)) {
 296                 auto s = tvCastToString(f.rvalAt(s_file).tv()).toCppString();
 297                 if (f.exists(s_line)) {
 298                   s += folly::sformat(
 299                     ":{}",
 300                     tvCastToInt64(f.rvalAt(s_line).tv())
 301                   );
 302                 }
 303                 frameStrings.emplace_back(std::move(s));
 304                 frames.push_back(frameStrings.back());
 305               }
 306             }
 307             record.setVec("stack", frames);
 308             needsCppStack = false;
 309           }
 310         }
 311         if (needsCppStack) {
 312           record.setStackTrace("stack", StackTrace{StackTrace::Force{}});
 313         }
 314         StructuredLog::log("hhvm_file_cache", record);
 315       });
 316   }
 317   auto startupFailure = [] (const std::string& msg) {
 318     Logger::Error(msg);
 319     Logger::Error("Shutting down due to failure(s) to bind in "
 320                   "HttpServer::runAndExitProcess");
 321     // Logger flushes itself---we don't need to run any atexit handlers
 322     // (historically we've mostly just SEGV'd while trying) ...
 323     _Exit(1);
 324   };
 325
 326   if (!RuntimeOption::InstanceId.empty()) {
 327     std::string msg = "Starting instance " + RuntimeOption::InstanceId;
 328     if (!RuntimeOption::DeploymentId.empty()) {
 329       msg += " from deployment " + RuntimeOption::DeploymentId;
 330     }
 331     Logger::Info(msg);
 332   }
 333
 334   block_sync_signals_and_start_handler_thread();
 335
 336   if (RuntimeOption::ServerPort) {
 337     if (!startServer(true)) {
 338       startupFailure("Unable to start page server");
 339       not_reached();
 340     }
 341     Logger::Info("page server started");
 342   }
 343
 344   StartTime = time(nullptr);
 345
 346   if (RuntimeOption::AdminServerPort) {
 347     if (!startServer(false)) {
 348       startupFailure("Unable to start admin server");
 349       not_reached();
 350     }
 351     Logger::Info("admin server started");
 352   }
 353
 354   for (unsigned int i = 0; i < m_satellites.size(); i++) {
 355     std::string name = m_satellites[i]->getName();
 356     try {
 357       m_satellites[i]->start();
 358       Logger::Info("satellite server %s started", name.c_str());
 359     } catch (Exception& e) {
 360       startupFailure(
 361         folly::format("Unable to start satellite server {}: {}",
 362                       name, e.getMessage()).str()
 363       );
 364       not_reached();
 365     }
 366   }
 367
 368   if (!Eval::Debugger::StartServer()) {
 369     startupFailure("Unable to start debugger server");
 370     not_reached();
 371   } else if (RuntimeOption::EnableDebuggerServer) {
 372     Logger::Info("debugger server started");
 373   }
 374
 375   try {
 376     InitFiniNode::ServerInit();
 377   } catch (std::exception& e) {
 378     startupFailure(
 379       folly::sformat("Exception in InitFiniNode::ServerInit(): {}",
 380                      e.what()));
 381   }
 382
 383   {
 384     BootStats::mark("servers started");
 385     Logger::Info("all servers started");
 386     if (!RuntimeOption::ServerForkEnabled) {
 387 #ifdef __linux__
 388       if (DisableFork) {
 389         // We should not fork from the server process.  Use light process
 390         // instead.  This will intercept subsequent fork() calls and make them
 391         // fail immediately.
 392         DisableFork();
 393       } else {
 394         // Otherwise, the binary we are building is not HHVM.  It is probably
 395         // some tests, don't intercept fork().
 396         Logger::Warning("ignored runtime option Server.Forking.Enabled=false");
 397       }
 398 #else
 399       Logger::Warning("ignored Server.Forking.Enabled=false "
 400                       "as it only works on Linux");
 401 #endif
 402     } else {
 403 #if FOLLY_HAVE_PTHREAD_ATFORK
 404       pthread_atfork(nullptr, nullptr,
 405                      [] {
 406                        Process::OOMScoreAdj(1000);
 407                      });
 408 #endif
 409     }
 410     if (RuntimeOption::ServerForkLogging) {
 411 #ifdef __linux__
 412       if (EnableForkLogging) {
 413         EnableForkLogging();
 414       } else {
 415         // the binary we are building is not HHVM.  It is probably
 416         // some tests, don't intercept fork().
 417         Logger::Warning("ignored runtime option Server.Forking.Log=true");
 418       }
 419 #else
 420       Logger::Warning("ignored Server.Forking.Log=true "
 421                       "as it only works on Linux");
 422 #endif
 423     }
 424     if (RuntimeOption::EvalServerOOMAdj < 0) {
 425       // Avoid HHVM getting killed when a forked process uses too much memory.
 426       // A positive adjustment makes it more likely for the server to be killed,
 427       // and that's not what we want.
 428       Process::OOMScoreAdj(RuntimeOption::EvalServerOOMAdj);
 429     }
 430     createPid();
 431     Lock lock(this);
 432     BootStats::done();
 433     // Play extended warmup requests after server starts running.
 434     if (isJitSerializing() &&
 435         RuntimeOption::ServerExtendedWarmupThreadCount > 0 &&
 436         !RuntimeOption::ServerExtendedWarmupRequests.empty()) {
 437       auto const threadCount = RuntimeOption::ServerExtendedWarmupThreadCount;
 438       auto const delay = RuntimeOption::ServerExtendedWarmupDelaySeconds;
 439       auto const nTimes = RuntimeOption::ServerExtendedWarmupRepeat;
 440       InternalWarmupRequestPlayer{threadCount}.
 441         runAfterDelay(RuntimeOption::ServerExtendedWarmupRequests,
 442                       nTimes, delay);
 443     }
 444     // continously running until /stop is received on admin server, or
 445     // takeover is requested.
 446     while (!m_stopped) {
 447       wait();
 448     }
 449     if (m_stopReason) {
 450       Logger::Warning("Server stopping with reason: %s", m_stopReason);
 451     } else if (auto signo = SignalReceived.load(std::memory_order_acquire)) {
 452       Logger::Warning("Server stopping with reason: %s", strsignal(signo));
 453     }
 454   }
 455
 456   if (RuntimeOption::ServerPort) {
 457     Logger::Info("stopping page server");
 458     m_pageServer->stop();
 459   }
 460   onServerShutdown();
 461
 462   EvictFileCache();
 463
 464   waitForServers();
 465   playShutdownRequest(RuntimeOption::ServerCleanupRequest);
 466 }
 467
 468 void HttpServer::waitForServers() {
 469   if (RuntimeOption::ServerPort) {
 470     m_pageServer->waitForEnd();
 471   }
 472   if (RuntimeOption::AdminServerPort) {
 473     m_adminServer->waitForEnd();
 474   }
 475   // all other servers invoke waitForEnd on stop
 476 }
 477
 478 void HttpServer::ProfileFlush() {
 479   #ifdef __linux__
 480   if (__gcov_flush) __gcov_flush();
 481   #endif
 482 }
 483
 484 void HttpServer::stop(const char* stopReason) {
 485   if (m_stopping.exchange(true)) return;
 486
 487   ProfileFlush();
 488   // Let all worker threads know that the server is shutting down. If some
 489   // request installed a PHP-level signal handler through `pcntl_signal(SIGTERM,
 490   // handler_func)`, `handler_func()` will run in the corresponding request
 491   // context, to perform cleanup tasks. If no handler is registered, it is
 492   // ignored.
 493   RequestInfo::BroadcastSignal(SIGTERM);
 494
 495   // we're shutting down flush http logs
 496   Logger::FlushAll();
 497   HttpRequestHandler::GetAccessLog().flushAllWriters();
 498   Process::OOMScoreAdj(1000);
 499   MarkShutdownStat(ShutdownEvent::SHUTDOWN_INITIATED);
 500
 501   if (RuntimeOption::ServerKillOnTimeout) {
 502     int totalWait =
 503       RuntimeOption::ServerPreShutdownWait +
 504       RuntimeOption::ServerShutdownListenWait +
 505       RuntimeOption::ServerGracefulShutdownWait;
 506
 507     if (totalWait > 0) {
 508       // Use a killer thread to _Exit() after totalWait seconds.  If
 509       // the main thread returns before that, the killer thread will
 510       // exit.  So don't do join() on this thread.
 511       auto killer = std::thread([totalWait] {
 512 #ifdef __linux__
 513           sched_param param{};
 514           param.sched_priority = 5;
 515           pthread_setschedparam(pthread_self(), SCHED_RR, &param);
 516           // It is OK if we fail to increase thread priority.
 517 #endif
 518           /* sleep override */
 519           std::this_thread::sleep_for(std::chrono::seconds{totalWait});
 520           _Exit(1);
 521         });
 522       killer.detach();
 523     }
 524   }
 525
 526   Lock lock(this);
 527   m_stopped = true;
 528   m_stopReason = stopReason;
 529   notify();
 530 }
 531
 532 void HttpServer::stopOnSignal(int sig) {
 533   int zero = 0;
 534   if (SignalReceived.compare_exchange_strong(zero, sig) &&
 535       RuntimeOption::ServerKillOnTimeout) {
 536     auto const totalWait =
 537       RuntimeOption::ServerPreShutdownWait +
 538       RuntimeOption::ServerShutdownListenWait +
 539       RuntimeOption::ServerGracefulShutdownWait;
 540     // In case HttpServer::stop() isn't invoked in the synchronous signal
 541     // handler.
 542     if (totalWait > 0) {
 543         signal(SIGALRM, exit_on_timeout);
 544         sigset_t s;
 545         sigemptyset(&s);
 546         sigaddset(&s, SIGALRM);
 547         pthread_sigmask(SIG_UNBLOCK, &s, nullptr);
 548         alarm(totalWait);
 549     }
 550   }
 551   // Invoke HttpServer::stop() from the synchronous signal handler thread. This
 552   // way, stopOnSignal() is asynchronous-signal safe.
 553   raise(SIGTERM);
 554 }
 555
 556 void HttpServer::EvictFileCache() {
 557   // In theory, the kernel should do just as well even if we don't
 558   // explicitly advise files out.  But we can do it anyway when we
 559   // need more free memory, e.g., when a new instance of the server is
 560   // about to start.
 561   advise_out(RuntimeOption::RepoLocalPath);
 562   advise_out(RuntimeOption::FileCache);
 563   apc_advise_out();
 564 }
 565
 566 void HttpServer::PrepareToStop() {
 567   MarkShutdownStat(ShutdownEvent::SHUTDOWN_PREPARE);
 568   PrepareToStopTime.store(time(nullptr), std::memory_order_release);
 569   EvictFileCache();
 570 }
 571
 572 void HttpServer::createPid() {
 573   if (!RuntimeOption::PidFile.empty()) {
 574     FILE * f = fopen(RuntimeOption::PidFile.c_str(), "w");
 575     if (f) {
 576       auto const pid = getpid();
 577       char buf[64];
 578       snprintf(buf, sizeof(buf), "%" PRId64, (int64_t)pid);
 579       fwrite(buf, strlen(buf), 1, f);
 580       fclose(f);
 581     } else {
 582       Logger::Error("Unable to open pid file %s for write",
 583                     RuntimeOption::PidFile.c_str());
 584     }
 585   }
 586 }
 587
 588 void HttpServer::removePid() {
 589   if (!RuntimeOption::PidFile.empty()) {
 590     unlink(RuntimeOption::PidFile.c_str());
 591   }
 592 }
 593
 594 void HttpServer::killPid() {
 595   if (!RuntimeOption::PidFile.empty()) {
 596     std::ifstream in(RuntimeOption::PidFile);
 597     int64_t pid;
 598     if ((in >> pid) && pid > 0) {
 599       in.close();
 600       kill((pid_t)pid, SIGKILL);
 601       return;
 602     }
 603     Logger::Error("Unable to read pid file %s for any meaningful pid",
 604                   RuntimeOption::PidFile.c_str());
 605   }
 606 }
 607
 608 static bool sendAdminCommand(const char* cmd) {
 609   if (RuntimeOption::AdminServerPort <= 0) return false;
 610   std::string host = RuntimeOption::AdminServerIP;
 611   if (host.empty()) host = "localhost";
 612   auto passwords = RuntimeOption::AdminPasswords;
 613   if (passwords.empty() && !RuntimeOption::AdminPassword.empty()) {
 614     passwords.insert(RuntimeOption::AdminPassword);
 615   }
 616   auto passwordIter = passwords.begin();
 617   do {
 618     std::string url;
 619     if (passwordIter != passwords.end()) {
 620       url = folly::sformat("http://{}:{}/{}?auth={}", host,
 621                            RuntimeOption::AdminServerPort,
 622                            cmd, *passwordIter);
 623       ++passwordIter;
 624     } else {
 625       url = folly::sformat("http://{}:{}/{}", host,
 626                            RuntimeOption::AdminServerPort, cmd);
 627     }
 628     if (CURL* curl = curl_easy_init()) {
 629       curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
 630       curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
 631       curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
 632       curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3);
 633       auto code = curl_easy_perform(curl);
 634       curl_easy_cleanup(curl);
 635       if (code == CURLE_OK) {
 636         Logger::Info("sent %s via admin port", cmd);
 637         return true;
 638       }
 639     }
 640   } while (passwordIter != passwords.end());
 641   return false;
 642 }
 643
 644 bool HttpServer::ReduceOldServerLoad() {
 645   if (!RuntimeOption::StopOldServer) return false;
 646   if (OldServerStopTime > 0) return true;
 647   Logger::Info("trying to reduce load of the old HPHP server");
 648   return sendAdminCommand("prepare-to-stop");
 649 }
 650
 651 bool HttpServer::StopOldServer() {
 652   if (OldServerStopTime > 0) return true;
 653   SCOPE_EXIT { OldServerStopTime = time(nullptr); };
 654   Logger::Info("trying to shut down old HPHP server by /stop command");
 655   return sendAdminCommand("stop");
 656 }
 657
 658 // Return the estimated amount of memory that can be safely taken into
 659 // the current process RSS.
 660 static inline int64_t availableMemory(const MemInfo& mem, int64_t rss,
 661                                       int factor) {
 662   // Estimation of page cache that are readily evictable without
 663   // causing big memory pressure.  We consider it safe to take
 664   // `pageFreeFactor` percent of cached pages (excluding those used by
 665   // the current process, estimated to be equal to the current RSS)
 666   auto const otherCacheMb = std::max((int64_t)0,  mem.cachedMb - rss);
 667   auto const availableMb = mem.freeMb + otherCacheMb * factor / 100;
 668   return availableMb;
 669 }
 670
 671 bool HttpServer::CanContinue(const MemInfo& mem, int64_t rssMb,
 672                              int64_t rssNeeded, int cacheFreeFactor) {
 673   if (!mem.valid()) return false;
 674   if (mem.freeMb < RuntimeOption::ServerCriticalFreeMb) return false;
 675   auto const availableMb = availableMemory(mem, rssMb, cacheFreeFactor);
 676   auto const result = (rssMb + availableMb >= rssNeeded);
 677   if (result) assertx(CanStep(mem, rssMb, rssNeeded, cacheFreeFactor));
 678   return result;
 679 }
 680
 681 bool HttpServer::CanStep(const MemInfo& mem, int64_t rssMb,
 682                          int64_t rssNeeded, int cacheFreeFactor) {
 683   if (!mem.valid()) return false;
 684   if (mem.freeMb < RuntimeOption::ServerCriticalFreeMb) return false;
 685   auto const availableMb = availableMemory(mem, rssMb, cacheFreeFactor);
 686   // Estimation of the memory needed till the next check point.  Since
 687   // the current check point is not the last one, we try to be more
 688   // optimistic, by assuming that memory requirement won't grow
 689   // drastically between successive check points, and that it won't
 690   // grow over our estimate.
 691   auto const neededToStep = std::min(rssNeeded / 4, rssNeeded - rssMb);
 692   return (availableMb >= neededToStep);
 693 }
 694
 695 void HttpServer::CheckMemAndWait(bool final) {
 696   if (!RuntimeOption::StopOldServer) return;
 697   if (RuntimeOption::OldServerWait <= 0) return;
 698
 699   auto const rssNeeded = RuntimeOption::ServerRSSNeededMb;
 700   auto const factor = RuntimeOption::CacheFreeFactor;
 701   do {
 702     // Don't wait too long
 703     if (OldServerStopTime > 0 &&
 704         time(nullptr) - OldServerStopTime >= RuntimeOption::OldServerWait) {
 705       return;
 706     }
 707
 708     auto const rssMb = Process::GetMemUsageMb();
 709     MemInfo memInfo;
 710     if (!Process::GetMemoryInfo(memInfo)) {
 711       Logger::Error("Failed to obtain memory information");
 712       HttpServer::StopOldServer();
 713       return;
 714     }
 715     Logger::FInfo("Memory pressure check: free/cached/buffers {}/{}/{} "
 716                   "currentRss {}Mb, required {}Mb.",
 717                   memInfo.freeMb, memInfo.cachedMb, memInfo.buffersMb,
 718                   rssMb, rssNeeded);
 719
 720     if (!final) {
 721       if (CanStep(memInfo, rssMb, rssNeeded, factor)) return;
 722     } else {
 723       if (CanContinue(memInfo, rssMb, rssNeeded, factor)) return;
 724     }
 725
 726     // OK, we don't have enough memory, let's do something.
 727     HttpServer::StopOldServer();  // nop if already called before
 728     /* sleep override */ sleep(1);
 729   } while (true); // Guaranteed to return in the loop, at least upon timeout.
 730   not_reached();
 731 }
 732
 733 void HttpServer::MarkShutdownStat(ShutdownEvent event) {
 734   if (!RuntimeOption::EvalLogServerRestartStats) return;
 735   std::lock_guard<folly::MicroSpinLock> lock(StatsLock);
 736   MemInfo mem;
 737   Process::GetMemoryInfo(mem);
 738   auto const rss = Process::GetMemUsageMb();
 739   auto const requests = requestCount();
 740   if (event == ShutdownEvent::SHUTDOWN_PREPARE) {
 741     ShutdownStats.clear();
 742     ShutdownStats.reserve(ShutdownEvent::kNumEvents);
 743   }
 744 #ifndef NDEBUG
 745   if (!ShutdownStats.empty()) {
 746     assertx(ShutdownStats.back().event <= event);
 747   }
 748 #endif
 749   ShutdownStats.push_back({event, time(nullptr), mem, rss, requests});
 750 }
 751
 752 void HttpServer::LogShutdownStats() {
 753   if (!RuntimeOption::EvalLogServerRestartStats) return;
 754   StructuredLogEntry entry;
 755   std::lock_guard<folly::MicroSpinLock> lock(StatsLock);
 756   if (ShutdownStats.empty()) return;
 757   for (size_t i = 0; i < ShutdownStats.size(); ++i) {
 758     const auto& stat = ShutdownStats[i];
 759     auto const eventName = stat.eventName();
 760     entry.setInt(folly::sformat("{}_rss", eventName), stat.rss);
 761     entry.setInt(folly::sformat("{}_free", eventName),
 762                  stat.memUsage.freeMb);
 763     entry.setInt(folly::sformat("{}_cached", eventName),
 764                  stat.memUsage.cachedMb);
 765     entry.setInt(folly::sformat("{}_buffers", eventName),
 766                  stat.memUsage.buffersMb);
 767     // Log the difference since last event, if available
 768     if (i > 0) {
 769       const auto& last = ShutdownStats[i - 1];
 770       auto const lastEvent = last.eventName();
 771       entry.setInt(folly::sformat("{}_duration", lastEvent),
 772                    stat.time - last.time);
 773       entry.setInt(folly::sformat("{}_requests", lastEvent),
 774                    stat.requestsServed - last.requestsServed);
 775       entry.setInt(folly::sformat("{}_rss_delta", lastEvent),
 776                    stat.rss - last.rss);
 777     }
 778   }
 779   StructuredLog::log("webserver_shutdown_timing", entry);
 780   ShutdownStats.clear();
 781 }
 782
 783 void HttpServer::getSatelliteStats(
 784     std::vector<std::pair<std::string, int>> *stats) {
 785   for (const auto& i : m_satellites) {
 786     std::pair<std::string, int> active("satellite." + i->getName() + ".load",
 787                                        i->getActiveWorker());
 788     std::pair<std::string, int> queued("satellite." + i->getName() + ".queued",
 789                                        i->getQueuedJobs());
 790     stats->push_back(active);
 791     stats->push_back(queued);
 792   }
 793 }
 794
 795 std::pair<int, int> HttpServer::getSatelliteRequestCount() const {
 796   int inflight = 0;
 797   int queued = 0;
 798   for (const auto& i : m_satellites) {
 799     inflight += i->getActiveWorker();
 800     queued += i->getQueuedJobs();
 801   }
 802   return std::make_pair(inflight, queued);
 803 }
 804
 805 ///////////////////////////////////////////////////////////////////////////////
 806 // page server
 807
 808 bool HttpServer::startServer(bool pageServer) {
 809   int port = pageServer ?
 810     RuntimeOption::ServerPort : RuntimeOption::AdminServerPort;
 811
 812   // 1. try something nice
 813   for (unsigned int i = 0; i < 60; i++) {
 814     try {
 815       if (pageServer) {
 816         m_pageServer->start();
 817       } else {
 818         m_adminServer->start();
 819       }
 820       return true;
 821     } catch (FailedToListenException& e) {
 822       if (RuntimeOption::ServerExitOnBindFail) return false;
 823
 824       StopOldServer();
 825
 826       if (errno == EACCES) {
 827         Logger::Error("%s: Permission denied.", e.what());
 828         return false;
 829       }
 830
 831       if (pageServer && !RuntimeOption::ServerFileSocket.empty()) {
 832         if (i == 0) {
 833           Logger::Info("Unlinking unused socket at %s",
 834                        RuntimeOption::ServerFileSocket.c_str());
 835         }
 836         struct stat stat_buf;
 837         if (stat(RuntimeOption::ServerFileSocket.c_str(), &stat_buf) == 0
 838             && S_ISSOCK(stat_buf.st_mode)) {
 839           std::string cmd = "bash -c '! fuser ";
 840           cmd += RuntimeOption::ServerFileSocket;
 841           cmd += "'";
 842           if (FileUtil::ssystem(cmd.c_str()) == 0) {
 843             unlink(RuntimeOption::ServerFileSocket.c_str());
 844           }
 845         }
 846       }
 847       sleep(1);
 848     }
 849   }
 850
 851   // 2. try something harsh
 852   if (RuntimeOption::ServerHarshShutdown) {
 853     for (unsigned int i = 0; i < 5; i++) {
 854       try {
 855         if (pageServer) {
 856           m_pageServer->start();
 857         } else {
 858           m_adminServer->start();
 859         }
 860         return true;
 861       } catch (FailedToListenException& e) {
 862         if (i == 0) {
 863           Logger::Info("shutting down old HPHP server by pid file");
 864         }
 865         killPid();
 866         sleep(1);
 867       }
 868     }
 869   }
 870
 871   // 3. try something evil
 872   if (RuntimeOption::ServerEvilShutdown) {
 873     for (unsigned int i = 0; i < 60; i++) {
 874       try {
 875         if (pageServer) {
 876           m_pageServer->start();
 877         } else {
 878           m_adminServer->start();
 879         }
 880         return true;
 881       } catch (FailedToListenException& e) {
 882         if (pageServer && !RuntimeOption::ServerFileSocket.empty()) {
 883           if (i == 0) {
 884             Logger::Info("unlinking socket at %s",
 885                          RuntimeOption::ServerFileSocket.c_str());
 886           }
 887
 888           struct stat stat_buf;
 889           if (stat(RuntimeOption::ServerFileSocket.c_str(), &stat_buf) == 0
 890               && S_ISSOCK(stat_buf.st_mode)) {
 891             unlink(RuntimeOption::ServerFileSocket.c_str());
 892           }
 893         } else {
 894           if (i == 0) {
 895             Logger::Info("killing anything listening on port %d", port);
 896           }
 897
 898           std::string cmd = "lsof -t -i :";
 899           cmd += folly::to<std::string>(port);
 900           cmd += " | xargs kill -9";
 901           FileUtil::ssystem(cmd.c_str());
 902         }
 903         sleep(1);
 904       }
 905     }
 906   }
 907
 908   return false;
 909 }
 910
 911 ///////////////////////////////////////////////////////////////////////////////
 912 }