hphp/util/light-process.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #include "hphp/util/light-process.h"
  17
  18 #include <string>
  19 #include <vector>
  20
  21 #include <boost/scoped_array.hpp>
  22 #include <boost/thread/barrier.hpp>
  23
  24 #include <folly/portability/SysMman.h>
  25 #include <folly/portability/SysResource.h>
  26 #include <folly/portability/Sockets.h>
  27 #include <sys/types.h>
  28 #include <sys/wait.h>
  29
  30 #include <afdt.h>
  31 #include <grp.h>
  32 #include <stdlib.h>
  33 #include <folly/portability/Unistd.h>
  34
  35 #include <pwd.h>
  36 #include <signal.h>
  37
  38 #include <folly/Memory.h>
  39 #include <folly/String.h>
  40
  41 #include "hphp/util/afdt-util.h"
  42 #include "hphp/util/compatibility.h"
  43 #include "hphp/util/hardware-counter.h"
  44 #include "hphp/util/hugetlb.h"
  45 #include "hphp/util/logger.h"
  46 #include "hphp/util/process.h"
  47 #include "hphp/util/timer.h"
  48
  49 namespace HPHP {
  50
  51 ///////////////////////////////////////////////////////////////////////////////
  52 // helper functions
  53
  54 namespace {
  55
  56 __thread LightProcess* tl_proc;
  57 bool s_trackProcessTimes = false;
  58 Mutex s_mutex;
  59
  60 using afdt::send_fd;
  61 using afdt::recv_fd;
  62
  63 char **build_envp(const std::vector<std::string> &env) {
  64   char **envp = nullptr;
  65   int size = env.size();
  66   if (size) {
  67     envp = (char **)malloc((size + 1) * sizeof(char *));
  68     int j = 0;
  69     for (unsigned int i = 0; i < env.size(); i++, j++) {
  70       *(envp + j) = (char *)env[i].c_str();
  71     }
  72     *(envp + j) = nullptr;
  73   }
  74   return envp;
  75 }
  76
  77 void close_fds(const std::vector<int> &fds) {
  78   for (auto fd : fds) {
  79     ::close(fd);
  80   }
  81 }
  82
  83 template<class Head, class... Tail>
  84 void lwp_write(int afdt_fd, const Head& h, Tail&&... args) {
  85   if (afdt::sendRaw(afdt_fd, h, std::forward<Tail>(args)...)) {
  86     throw Exception("Failed in afdt::sendRaw: %s",
  87                     folly::errnoStr(errno).c_str());
  88   }
  89 }
  90
  91 template<class Head, class... Tail>
  92 void lwp_read(int afdt_fd, Head& h, Tail&... args) {
  93   if (afdt::recvRaw(afdt_fd, h, args...)) {
  94     throw Exception("Failed in afdt::recvRaw: %s",
  95                     folly::errnoStr(errno).c_str());
  96   }
  97 }
  98
  99 // not thread safe - needs a lock when called from the main
 100 // process.
 101 int popen_impl(const char* cmd, const char* mode, pid_t* out_pid) {
 102   int p[2];
 103   auto const read = *mode == 'r';
 104   if (!read && *mode != 'w') return -1;
 105
 106   if (pipe2(p, O_CLOEXEC) < 0) {
 107     return -1;
 108   }
 109
 110   auto pid = fork();
 111   if (pid < 0) {
 112     close(p[0]);
 113     close(p[1]);
 114     return -1;
 115   }
 116   int child_pipe = read ? 1 : 0;
 117   if (pid == 0) {
 118     // child
 119     mprotect_1g_pages(PROT_READ);
 120     // replace stdin or stdout with the appropriate end
 121     // of the pipe
 122     if (p[child_pipe] == child_pipe) {
 123       // pretty unlikely, but if it was we must clear CLOEXEC,
 124       // and the only way to do that is to dup it to a new fd,
 125       // and then dup2 it back
 126       p[child_pipe] = fcntl(child_pipe, F_DUPFD_CLOEXEC, 3);
 127     }
 128     dup2(p[child_pipe], child_pipe);
 129     // no need to close p[child_pipe] because of O_CLOEXEC
 130
 131     signal(SIGINT, SIG_DFL);
 132     sigset_t eset;
 133     sigemptyset(&eset);
 134     sigprocmask(SIG_SETMASK, &eset, nullptr);
 135     execl("/bin/sh", "sh", "-c", cmd, nullptr);
 136     Logger::Warning("Failed to exec: `%s'", cmd);
 137     _Exit(1);
 138   }
 139   // parent
 140
 141   // close the pipe we're not using
 142   close(p[child_pipe]);
 143   *out_pid = pid;
 144   return p[1-child_pipe];
 145 }
 146
 147 int64_t ru2microseconds(const rusage& ru) {
 148   int64_t time_us = ru.ru_utime.tv_usec;
 149   time_us += ru.ru_stime.tv_usec;
 150   int64_t time_s = ru.ru_utime.tv_sec;
 151   time_s += ru.ru_stime.tv_sec;
 152   return time_us + time_s * 1000000;
 153 }
 154
 155 /*
 156  * Hardware counters can be configured to measure sub-process times,
 157  * but note that any given LightProcess will be running jobs for
 158  * multiple request threads (each request thread binds to a single
 159  * LightProcess, but its a many to one mapping).
 160  *
 161  * This means that between the fork/exec and the waitpid, the LightProcess
 162  * could fork/exec more processes for different requests. The solution
 163  * is to fork/exec in a thread, and start the hardware counters there.
 164  *
 165  * The hardware counter will then measure time for that thread, and all
 166  * its children - which is exactly what we want.
 167  *
 168  * HardwareCounterWrapper & co take care of that.
 169  */
 170 struct HardwareCounterWrapperArg {
 171   boost::barrier barrier{2};
 172   pthread_t thr;
 173   int afdt_fd;
 174   pid_t (*func)(int);
 175   pid_t pid;
 176   int64_t *events;
 177 };
 178
 179 std::map<pid_t, std::unique_ptr<HardwareCounterWrapperArg>> s_pidToHCWMap;
 180
 181 void* hardwareCounterWrapper(void* varg) {
 182   auto arg = (HardwareCounterWrapperArg*)varg;
 183
 184   HardwareCounter::s_counter.getCheck();
 185   HardwareCounter::Reset();
 186   arg->pid = arg->func(arg->afdt_fd);
 187   // tell the main thread that pid has been set.
 188   arg->barrier.wait();
 189   if (arg->pid < 0) return nullptr;
 190
 191   // Wait until the main thread is ready to join us.
 192   // Note that even though we have multiple threads running in the LightProcess
 193   // now, at the time any of the do_* functions are called, any live threads are
 194   // waiting here on this barrier; so there is no problem with fork, malloc,
 195   // exec.
 196   arg->barrier.wait();
 197   HardwareCounter::GetPerfEvents(
 198     [](const std::string& event, int64_t value, void* data) {
 199       auto events = reinterpret_cast<int64_t*>(data);
 200       if (event == "instructions") {
 201         events[0] = value;
 202       } else if (event == "loads") {
 203         events[1] = value;
 204       } else if (event == "stores") {
 205         events[2] = value;
 206       }
 207     },
 208     arg->events);
 209
 210   return nullptr;
 211 }
 212
 213 void hardwareCounterWrapperHelper(pid_t (*func)(int), int afdt_fd) {
 214   if (!s_trackProcessTimes) {
 215     func(afdt_fd);
 216     return;
 217   }
 218
 219   auto arg = std::make_unique<HardwareCounterWrapperArg>();
 220   arg->afdt_fd = afdt_fd;
 221   arg->func = func;
 222   if (pthread_create(&arg->thr, nullptr, hardwareCounterWrapper, arg.get())) {
 223     throw Exception("Failed to pthread_create: %s",
 224                     folly::errnoStr(errno).c_str());
 225   }
 226   // Wait for the pid to be set. Note that we must not add any code here that
 227   // could cause issues for the fork (eg malloc); we should wait on the barrier
 228   // immediately after the thread is created.
 229   arg->barrier.wait();
 230   if (arg->pid > 0) {
 231     // successfully forked, so don't join until waitpid.
 232     auto& map_entry = s_pidToHCWMap[arg->pid];
 233     map_entry = std::move(arg);
 234   } else {
 235     // fork failed, join now.
 236     pthread_join(arg->thr, nullptr);
 237   }
 238 }
 239
 240 ///////////////////////////////////////////////////////////////////////////////
 241 // shadow process tasks
 242
 243 pid_t do_popen_helper(int afdt_fd) {
 244   std::string mode, buf, cwd;
 245
 246   lwp_read(afdt_fd, mode, buf, cwd);
 247
 248   std::string old_cwd;
 249
 250   if (!cwd.empty()) {
 251     old_cwd = Process::GetCurrentDirectory();
 252     if (old_cwd != cwd) {
 253       if (chdir(cwd.c_str())) {
 254         // Ignore chdir failures, because the compiled version might not
 255         // have the directory any more.
 256         Logger::Warning("Light Process failed chdir to %s.", cwd.c_str());
 257       }
 258     }
 259   }
 260
 261   pid_t pid;
 262
 263   auto fd = buf.empty() ? -1 :
 264     popen_impl(buf.c_str(), mode.c_str(), &pid);
 265
 266   if (!old_cwd.empty() && chdir(old_cwd.c_str())) {
 267     // only here if we can't change the cwd back
 268   }
 269
 270   if (fd < 0) {
 271     Logger::Error("Light process failed popen: %d (%s).", errno,
 272                   folly::errnoStr(errno).c_str());
 273     lwp_write(afdt_fd, "error");
 274   } else {
 275     lwp_write(afdt_fd, "success", pid);
 276     send_fd(afdt_fd, fd);
 277     // the fd is now owned by the main process, close our copy
 278     close(fd);
 279   }
 280   return pid;
 281 }
 282
 283 void do_popen(int afdt_fd) {
 284   hardwareCounterWrapperHelper(do_popen_helper, afdt_fd);
 285 }
 286
 287 pid_t do_proc_open_helper(int afdt_fd) {
 288   std::string cmd, cwd;
 289   std::vector<std::string> env;
 290   std::vector<int> pvals;
 291   lwp_read(afdt_fd, cmd, cwd, env, pvals);
 292
 293   std::vector<int> pkeys;
 294   for (int i = 0; i < pvals.size(); i++) {
 295     int fd = recv_fd(afdt_fd);
 296     if (fd < 0) {
 297       lwp_write(afdt_fd, "error", (int32_t)EPROTO);
 298       close_fds(pkeys);
 299       return -1;
 300     }
 301     pkeys.push_back(fd);
 302   }
 303
 304   // indicate error if an empty command was received
 305   if (cmd.length() == 0) {
 306     lwp_write(afdt_fd, "error", (int32_t)ENOENT);
 307     return -1;
 308   }
 309
 310   // now ready to start the child process
 311   pid_t child = fork();
 312   if (child == 0) {
 313     mprotect_1g_pages(PROT_READ);
 314     for (int i = 0; i < pvals.size(); i++) {
 315       dup2(pkeys[i], pvals[i]);
 316     }
 317     if (!cwd.empty() && chdir(cwd.c_str())) {
 318       // non-zero for error
 319       // chdir failed, the working directory remains unchanged
 320     }
 321     if (!env.empty()) {
 322       char **envp = build_envp(env);
 323       execle("/bin/sh", "sh", "-c", cmd.c_str(), nullptr, envp);
 324       free(envp);
 325     } else {
 326       execl("/bin/sh", "sh", "-c", cmd.c_str(), nullptr);
 327     }
 328     _Exit(HPHP_EXIT_FAILURE);
 329   }
 330
 331   if (child > 0) {
 332     // successfully created the child process
 333     lwp_write(afdt_fd, "success", child);
 334   } else {
 335     // failed creating the child process
 336     lwp_write(afdt_fd, "error", errno);
 337   }
 338
 339   close_fds(pkeys);
 340   return child;
 341 }
 342
 343 void do_proc_open(int afdt_fd) {
 344   hardwareCounterWrapperHelper(do_proc_open_helper, afdt_fd);
 345 }
 346
 347 pid_t waited = 0;
 348
 349 void kill_handler(int sig) {
 350   if (sig == SIGALRM && waited) {
 351     kill(waited, SIGKILL);
 352   }
 353 }
 354
 355 void do_waitpid(int afdt_fd) {
 356   pid_t pid = -1;
 357   int options = 0;
 358   int timeout = 0;
 359   lwp_read(afdt_fd, pid, options, timeout);
 360
 361   int stat;
 362   if (timeout > 0) {
 363     waited = pid;
 364     signal(SIGALRM, kill_handler);
 365     alarm(timeout);
 366   }
 367
 368   rusage ru;
 369   const auto ret = ::wait4(pid, &stat, options, &ru);
 370   alarm(0); // cancel the previous alarm if not triggered yet
 371   waited = 0;
 372   const auto time_us = ru2microseconds(ru);
 373   int64_t events[] = { 0, 0, 0 };
 374   if (ret > 0 && s_trackProcessTimes) {
 375     auto it = s_pidToHCWMap.find(ret);
 376     if (it == s_pidToHCWMap.end()) {
 377       throw Exception("pid not in map: %s",
 378                       folly::errnoStr(errno).c_str());
 379     }
 380
 381     auto hcw = std::move(it->second);
 382     s_pidToHCWMap.erase(it);
 383     hcw->events = events;
 384     hcw->barrier.wait();
 385     pthread_join(hcw->thr, nullptr);
 386   }
 387
 388   lwp_write(afdt_fd, ret, errno, stat,
 389             time_us, events[0], events[1], events[2]);
 390 }
 391
 392 void do_change_user(int afdt_fd) {
 393   std::string uname;
 394   lwp_read(afdt_fd, uname);
 395   if (uname.length() > 0) {
 396     struct passwd *pw = getpwnam(uname.c_str());
 397     if (pw) {
 398       if (pw->pw_gid) {
 399         initgroups(pw->pw_name, pw->pw_gid);
 400         setgid(pw->pw_gid);
 401       }
 402       if (pw->pw_uid) {
 403         setuid(pw->pw_uid);
 404       }
 405     }
 406   }
 407 }
 408
 409 ///////////////////////////////////////////////////////////////////////////////
 410 // light-weight process
 411
 412 boost::scoped_array<LightProcess> g_procs;
 413 int g_procsCount = 0;
 414 bool s_handlerInited = false;
 415 LightProcess::LostChildHandler s_lostChildHandler;
 416 std::map<FILE*, pid_t> s_popenMap;
 417
 418 } // namespace
 419
 420 LightProcess::LightProcess() : m_shadowProcess(0), m_afdt_fd(-1) { }
 421
 422 LightProcess::~LightProcess() {
 423 }
 424
 425 void LightProcess::SigChldHandler(int /*sig*/, siginfo_t* info, void* /*ctx*/) {
 426   if (info->si_code != CLD_EXITED &&
 427       info->si_code != CLD_KILLED &&
 428       info->si_code != CLD_DUMPED) {
 429     return;
 430   }
 431   pid_t pid = info->si_pid;
 432   for (int i = 0; i < g_procsCount; ++i) {
 433     if (g_procs && g_procs[i].m_shadowProcess == pid) {
 434       // The exited process was a light process. Notify the callback, if any.
 435       if (s_lostChildHandler) {
 436         s_lostChildHandler(pid);
 437       }
 438       break;
 439     }
 440   }
 441 }
 442
 443 void LightProcess::Initialize(const std::string &prefix, int count,
 444                               bool trackProcessTimes,
 445                               const std::vector<int> &inherited_fds) {
 446   s_trackProcessTimes = trackProcessTimes;
 447
 448   if (prefix.empty() || count <= 0) {
 449     return;
 450   }
 451
 452   if (Available()) {
 453     // already initialized
 454     return;
 455   }
 456
 457   g_procs.reset(new LightProcess[count]);
 458   g_procsCount = count;
 459
 460   auto afdt_filename = folly::sformat("{}.{}", prefix, getpid());
 461
 462   // remove the possible leftover
 463   remove(afdt_filename.c_str());
 464
 465   afdt_error_t err = AFDT_ERROR_T_INIT;
 466   auto afdt_lid = afdt_listen(afdt_filename.c_str(), &err);
 467   if (afdt_lid < 0) {
 468     Logger::Error("Unable to afdt_listen to %s: %d %s",
 469                   afdt_filename.c_str(),
 470                   errno, folly::errnoStr(errno).c_str());
 471     abort();
 472   }
 473
 474   SCOPE_EXIT {
 475     ::close(afdt_lid);
 476     remove(afdt_filename.c_str());
 477   };
 478
 479   for (int i = 0; i < count; i++) {
 480     if (!g_procs[i].initShadow(afdt_lid, afdt_filename, i, inherited_fds)) {
 481       for (int j = 0; j < i; j++) {
 482         g_procs[j].closeShadow();
 483       }
 484       g_procs.reset();
 485       g_procsCount = 0;
 486       break;
 487     }
 488   }
 489
 490   if (!s_handlerInited) {
 491     struct sigaction sa = {};
 492     sa.sa_sigaction = &LightProcess::SigChldHandler;
 493     sa.sa_flags = SA_SIGINFO | SA_NOCLDSTOP;
 494     if (sigaction(SIGCHLD, &sa, nullptr) != 0) {
 495       Logger::Error("Couldn't install SIGCHLD handler");
 496       abort();
 497     }
 498     s_handlerInited = true;
 499   }
 500 }
 501
 502 bool LightProcess::initShadow(int afdt_lid,
 503                               const std::string &afdt_filename, int id,
 504                               const std::vector<int> &inherited_fds) {
 505   Lock lock(m_procMutex);
 506
 507   pid_t child = fork();
 508   if (child == 0) {
 509     // child
 510     mprotect_1g_pages(PROT_READ);
 511     if (s_trackProcessTimes) {
 512       HardwareCounter::RecordSubprocessTimes();
 513     }
 514     Logger::ResetPid();
 515     pid_t sid = setsid();
 516     if (sid < 0) {
 517       Logger::Warning("Unable to setsid");
 518       _Exit(HPHP_EXIT_FAILURE);
 519     }
 520     afdt_error_t err = AFDT_ERROR_T_INIT;
 521     auto afdt_fd = afdt_connect(afdt_filename.c_str(), &err);
 522     if (afdt_fd < 0) {
 523       Logger::Warning("Unable to afdt_connect, filename %s: %d %s",
 524                       afdt_filename.c_str(),
 525                       errno, folly::errnoStr(errno).c_str());
 526       _Exit(HPHP_EXIT_FAILURE);
 527     }
 528
 529     // shadow process doesn't use g_procs
 530     for (int i = 0; i < id; i++) {
 531       g_procs[i].closeFiles();
 532     }
 533     g_procs.reset();
 534     g_procsCount = 0;
 535     close_fds(inherited_fds);
 536     ::close(afdt_lid);
 537
 538     runShadow(afdt_fd);
 539   } else if (child < 0) {
 540     // failed
 541     Logger::Warning("Unable to fork lightly: %d %s", errno,
 542                     folly::errnoStr(errno).c_str());
 543     return false;
 544   } else {
 545     // parent
 546     m_shadowProcess = child;
 547
 548     sockaddr addr;
 549     socklen_t addrlen = sizeof(addr);
 550     m_afdt_fd = accept(afdt_lid, &addr, &addrlen);
 551     if (m_afdt_fd < 0) {
 552       Logger::Warning("Unable to establish afdt connection: %d %s",
 553                       errno, folly::errnoStr(errno).c_str());
 554       closeShadow();
 555       return false;
 556     }
 557   }
 558   return true;
 559 }
 560
 561 void LightProcess::runShadow(int afdt_fd) {
 562   std::string buf;
 563
 564   pollfd pfd[1];
 565   pfd[0].fd = afdt_fd;
 566   pfd[0].events = POLLIN;
 567   try {
 568     while (true) {
 569       int ret = poll(pfd, 1, -1);
 570       if (ret < 0 && errno == EINTR) {
 571         continue;
 572       }
 573       if (pfd[0].revents & POLLHUP) {
 574         // no more command can come in
 575         Logger::Error("Lost parent, LightProcess exiting");
 576         break;
 577       }
 578       if (pfd[0].revents & POLLIN) {
 579         lwp_read(afdt_fd, buf);
 580         if (buf == "exit") {
 581           Logger::Verbose("LightProcess exiting upon request");
 582           break;
 583         } else if (buf == "popen") {
 584           do_popen(afdt_fd);
 585         } else if (buf == "proc_open") {
 586           do_proc_open(afdt_fd);
 587         } else if (buf == "waitpid") {
 588           do_waitpid(afdt_fd);
 589         } else if (buf == "change_user") {
 590           do_change_user(afdt_fd);
 591         } else if (buf[0]) {
 592           Logger::Info("LightProcess got invalid command: %.20s", buf.c_str());
 593         }
 594       }
 595     }
 596   } catch (const std::exception& e) {
 597     Logger::Error("LightProcess exiting due to exception: %s", e.what());
 598   } catch (...) {
 599     Logger::Error("LightProcess exiting due to unknown exception");
 600   }
 601
 602   ::close(afdt_fd);
 603   _Exit(0);
 604 }
 605
 606 namespace {
 607
 608 int GetId() {
 609   return (long)pthread_self() % g_procsCount;
 610 }
 611
 612 NEVER_INLINE
 613 void handleException(const char* call) {
 614   try {
 615     throw;
 616   } catch (const std::exception& e) {
 617     Logger::Error("LightProcess::%s failed due to exception: %s",
 618                   call, e.what());
 619   } catch (...) {
 620     Logger::Error("LightProcess::%s failed due to unknown exception",
 621                   call);
 622   }
 623 }
 624
 625 template <class R, class F1>
 626 R runLight(const char* call, F1 body, R failureResult) {
 627   try {
 628     auto proc = tl_proc ? tl_proc : &g_procs[GetId()];
 629     Lock lock(proc->mutex());
 630
 631     return body(proc);
 632   } catch (...) {
 633     handleException(call);
 634   }
 635   return failureResult;
 636 }
 637
 638 }
 639
 640 void LightProcess::Close() {
 641   boost::scoped_array<LightProcess> procs;
 642   procs.swap(g_procs);
 643   int count = g_procsCount;
 644   g_procs.reset();
 645   g_procsCount = 0;
 646
 647   for (int i = 0; i < count; i++) {
 648     procs[i].closeShadow();
 649   }
 650 }
 651
 652 void LightProcess::closeShadow() {
 653   Lock lock(m_procMutex);
 654   if (m_shadowProcess) {
 655     try {
 656       lwp_write(m_afdt_fd, "exit");
 657     } catch (...) {
 658       handleException("closeShadow");
 659     }
 660     // removes the "zombie" process, so not to interfere with later waits
 661     while (true) {
 662       auto r = ::waitpid(m_shadowProcess, nullptr, 0);
 663       // retry on EINTR
 664       if (r != -1 || errno != EINTR) {
 665         break;
 666       }
 667     }
 668
 669     m_shadowProcess = 0;
 670   }
 671   closeFiles();
 672 }
 673
 674 void LightProcess::closeFiles() {
 675   if (m_afdt_fd >= 0) {
 676     ::close(m_afdt_fd);
 677     m_afdt_fd = -1;
 678   }
 679 }
 680
 681 bool LightProcess::Available() {
 682   if (tl_proc) return true;
 683   return g_procsCount > 0;
 684 }
 685
 686 FILE *LightProcess::popen(const char *cmd, const char *type,
 687                           const char *cwd /* = NULL */) {
 688   if (!Available()) {
 689     // fallback to normal popen
 690     Logger::Verbose("Light-weight fork not available; "
 691                     "use the heavy one instead.");
 692   } else {
 693     FILE *f = LightPopenImpl(cmd, type, cwd);
 694     if (f) {
 695       return f;
 696     }
 697     if (tl_proc) {
 698       Logger::Warning("Light-weight fork failed in remote CLI mode.");
 699       return nullptr;
 700     }
 701     Logger::Verbose("Light-weight fork failed; use the heavy one instead.");
 702   }
 703   return HeavyPopenImpl(cmd, type, cwd);
 704 }
 705
 706 FILE *LightProcess::HeavyPopenImpl(const char *cmd, const char *type,
 707                                    const char *cwd) {
 708   int fd = -1;
 709   pid_t pid;
 710   Lock lock(s_mutex);
 711   if (cwd && *cwd) {
 712     auto old_cwd = Process::GetCurrentDirectory();
 713     if (old_cwd != cwd) {
 714       if (chdir(cwd)) {
 715         Logger::Warning("Failed to chdir to %s.", cwd);
 716       }
 717       fd = popen_impl(cmd, type, &pid);
 718       if (chdir(old_cwd.c_str())) {
 719         // error occurred changing cwd back
 720       }
 721       if (fd < 0) return nullptr;
 722     }
 723   }
 724   if (fd < 0) {
 725     fd = popen_impl(cmd, type, &pid);
 726   }
 727   if (fd < 0) return nullptr;
 728   auto f = fdopen(fd, type);
 729   if (f) {
 730     s_popenMap[f] = pid;
 731   }
 732   return f;
 733 }
 734
 735 FILE *LightProcess::LightPopenImpl(const char *cmd, const char *type,
 736                                    const char *cwd) {
 737   return runLight("popen", [&] (LightProcess* proc) -> FILE* {
 738       auto afdt_fd = proc->m_afdt_fd;
 739       lwp_write(afdt_fd, "popen", type, cmd, cwd ? cwd : "");
 740
 741       std::string buf;
 742       lwp_read(afdt_fd, buf);
 743       if (buf == "error") {
 744         return nullptr;
 745       }
 746
 747       pid_t pid;
 748       lwp_read(afdt_fd, pid);
 749       assert(pid);
 750       int fd = recv_fd(afdt_fd);
 751       if (fd < 0) {
 752         Logger::Error("Light process failed to send the file descriptor.");
 753         return nullptr;
 754       }
 755       FILE *f = fdopen(fd, type);
 756       if (f) {
 757         proc->m_popenMap[f] = pid;
 758       }
 759       return f;
 760     }, static_cast<FILE*>(nullptr));
 761 }
 762
 763 int LightProcess::pclose(FILE *f) {
 764   pid_t pid;
 765   if (!Available()) {
 766     Lock lock(s_mutex);
 767     auto it = s_popenMap.find(f);
 768     if (it == s_popenMap.end()) return -1;
 769     pid = it->second;
 770     s_popenMap.erase(it);
 771   } else {
 772     auto proc = [] {
 773       if (tl_proc) return tl_proc;
 774       return &g_procs[GetId()];
 775     }();
 776     Lock lock(proc->m_procMutex);
 777
 778     auto it = proc->m_popenMap.find(f);
 779     if (it == proc->m_popenMap.end()) return -1;
 780     pid = it->second;
 781     proc->m_popenMap.erase(it);
 782   }
 783
 784   fclose(f);
 785   int status;
 786   if (LightProcess::waitpid(pid, &status, 0, 0) < 0) return -1;
 787   return status;
 788 }
 789
 790 pid_t LightProcess::proc_open(const char *cmd, const std::vector<int> &created,
 791                               const std::vector<int> &desired,
 792                               const char *cwd,
 793                               const std::vector<std::string> &env) {
 794   always_assert(Available());
 795   always_assert(created.size() == desired.size());
 796
 797   return runLight("proc_open", [&] (LightProcess* proc) -> pid_t {
 798       auto fout = proc->m_afdt_fd;
 799       lwp_write(fout, "proc_open", cmd, cwd ? cwd : "",
 800                 env, desired);
 801
 802       bool error_send = false;
 803       int save_errno = 0;
 804       for (auto cfd : created) {
 805         if (!send_fd(proc->m_afdt_fd, cfd)) {
 806           error_send = true;
 807           save_errno = errno;
 808           break;
 809         }
 810       }
 811
 812       std::string buf;
 813       auto fin = proc->m_afdt_fd;
 814       lwp_read(fin, buf);
 815       if (buf == "error") {
 816         lwp_read(fin, errno);
 817         if (error_send) {
 818           // On this error, the receiver side returns dummy errno,
 819           // use the sender side errno here.
 820           errno = save_errno;
 821         }
 822         return -1;
 823       }
 824       always_assert_flog(buf == "success",
 825                          "Unexpected message from light process: `{}'", buf);
 826       pid_t pid = -1;
 827       lwp_read(fin, pid);
 828       always_assert(pid);
 829       return pid;
 830     }, static_cast<pid_t>(-1));
 831 }
 832
 833 pid_t LightProcess::waitpid(pid_t pid, int *stat_loc, int options,
 834                             int timeout) {
 835   if (!Available()) {
 836     // light process is not really there
 837     rusage ru;
 838     const auto ret = wait4(pid, stat_loc, options, &ru);
 839     if (s_trackProcessTimes) {
 840       s_extra_request_nanoseconds += ru2microseconds(ru) * 1000;
 841     }
 842     return ret;
 843   }
 844
 845   return runLight("waitpid", [&] (LightProcess* proc) -> pid_t {
 846       lwp_write(proc->m_afdt_fd, "waitpid", pid, options, timeout);
 847
 848       pid_t ret;
 849       int stat;
 850       int err;
 851       int64_t time_us, events[3];
 852       lwp_read(proc->m_afdt_fd, ret, err, stat,
 853                time_us, events[0], events[1], events[2]);
 854
 855       *stat_loc = stat;
 856       if (ret < 0) {
 857         errno = err;
 858       } else if (s_trackProcessTimes) {
 859         s_extra_request_nanoseconds += time_us * 1000;
 860         HardwareCounter::IncInstructionCount(events[0]);
 861         HardwareCounter::IncLoadCount(events[1]);
 862         HardwareCounter::IncStoreCount(events[2]);
 863       }
 864
 865       return ret;
 866     }, static_cast<pid_t>(-1));
 867 }
 868
 869 pid_t LightProcess::pcntl_waitpid(pid_t pid, int *stat_loc, int options) {
 870   rusage ru;
 871   const auto ret = wait4(pid, stat_loc, options, &ru);
 872   if (s_trackProcessTimes) {
 873     s_extra_request_nanoseconds += ru2microseconds(ru) * 1000;
 874   }
 875   return ret;
 876 }
 877
 878 void LightProcess::ChangeUser(int afdt, const std::string& username) {
 879   if (!username.empty()) lwp_write(afdt, "change_user", username);
 880 }
 881
 882 void LightProcess::ChangeUser(const std::string& username) {
 883   if (username.empty()) return;
 884   for (int i = 0; i < g_procsCount; i++) {
 885     Lock lock(g_procs[i].m_procMutex);
 886     lwp_write(g_procs[i].m_afdt_fd, "change_user", username);
 887   }
 888 }
 889
 890 void LightProcess::SetLostChildHandler(const LostChildHandler& handler) {
 891   s_lostChildHandler = handler;
 892 }
 893
 894 std::unique_ptr<LightProcess> LightProcess::setThreadLocalAfdtOverride(
 895   std::unique_ptr<LightProcess> p
 896 ) {
 897   auto ret = std::unique_ptr<LightProcess>(tl_proc);
 898   tl_proc = p.release();
 899   return ret;
 900 }
 901
 902 std::unique_ptr<LightProcess> LightProcess::setThreadLocalAfdtOverride(int fd) {
 903   auto ret = std::unique_ptr<LightProcess>(tl_proc);
 904   tl_proc = new LightProcess;
 905   tl_proc->m_afdt_fd = fd;
 906   return ret;
 907 }
 908
 909 int LightProcess::createDelegate() {
 910   int pair[2];
 911   if (socketpair(AF_UNIX, SOCK_STREAM, 0, pair)) {
 912     Logger::Warning("Unable to create a unix socket pair: %s",
 913                     folly::errnoStr(errno).c_str());
 914     return -1;
 915   }
 916
 917   pid_t child = fork();
 918
 919   if (child < 0) {
 920     Logger::Warning("Unable to fork delegate process: %s",
 921                     folly::errnoStr(errno).c_str());
 922     close(pair[0]);
 923     close(pair[1]);
 924     return -1;
 925   }
 926
 927   if (child == 0) {
 928     // child
 929     fclose(stdin);
 930     fclose(stdout);
 931     fclose(stderr);
 932
 933     mprotect_1g_pages(PROT_READ);
 934     if (s_trackProcessTimes) {
 935       HardwareCounter::RecordSubprocessTimes();
 936     }
 937     Logger::ResetPid();
 938     pid_t sid = setsid();
 939     if (sid < 0) {
 940       Logger::Warning("Unable to setsid");
 941       _Exit(HPHP_EXIT_FAILURE);
 942     }
 943
 944     close(pair[0]);
 945     runShadow(pair[1]);
 946   }
 947
 948   always_assert(child > 0);
 949   close(pair[1]);
 950   return pair[0];
 951 }
 952
 953 ///////////////////////////////////////////////////////////////////////////////
 954 }