hphp/util/light-process.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #include "hphp/util/light-process.h"
  17
  18 #include <string>
  19 #include <memory>
  20 #include <vector>
  21
  22 #include <boost/thread/barrier.hpp>
  23
  24 #include <folly/portability/SysMman.h>
  25 #include <folly/portability/SysResource.h>
  26 #include <folly/portability/Sockets.h>
  27 #include <sys/types.h>
  28 #include <sys/wait.h>
  29
  30 #include <afdt.h>
  31 #include <grp.h>
  32 #include <stdlib.h>
  33 #include <folly/portability/Unistd.h>
  34
  35 #include <pwd.h>
  36 #include <signal.h>
  37
  38 #include <folly/Memory.h>
  39 #include <folly/String.h>
  40
  41 #include "hphp/util/afdt-util.h"
  42 #include "hphp/util/compatibility.h"
  43 #include "hphp/util/hardware-counter.h"
  44 #include "hphp/util/hugetlb.h"
  45 #include "hphp/util/logger.h"
  46 #include "hphp/util/process.h"
  47 #include "hphp/util/timer.h"
  48
  49 namespace HPHP {
  50
  51 ///////////////////////////////////////////////////////////////////////////////
  52 // helper functions
  53
  54 namespace {
  55
  56 __thread LightProcess* tl_proc;
  57 bool s_trackProcessTimes = false;
  58 Mutex s_mutex;
  59
  60 using afdt::send_fd;
  61 using afdt::recv_fd;
  62
  63 char **build_envp(const std::vector<std::string> &env) {
  64   char **envp = nullptr;
  65   int size = env.size();
  66   if (size) {
  67     envp = (char **)malloc((size + 1) * sizeof(char *));
  68     int j = 0;
  69     for (unsigned int i = 0; i < env.size(); i++, j++) {
  70       *(envp + j) = (char *)env[i].c_str();
  71     }
  72     *(envp + j) = nullptr;
  73   }
  74   return envp;
  75 }
  76
  77 void close_fds(const std::vector<int> &fds) {
  78   for (auto fd : fds) {
  79     ::close(fd);
  80   }
  81 }
  82
  83 template<class Head, class... Tail>
  84 void lwp_write(int afdt_fd, const Head& h, Tail&&... args) {
  85   if (afdt::sendRaw(afdt_fd, h, std::forward<Tail>(args)...)) {
  86     throw Exception("Failed in afdt::sendRaw: %s",
  87                     folly::errnoStr(errno).c_str());
  88   }
  89 }
  90
  91 template<class Head, class... Tail>
  92 void lwp_read(int afdt_fd, Head& h, Tail&... args) {
  93   if (afdt::recvRaw(afdt_fd, h, args...)) {
  94     throw Exception("Failed in afdt::recvRaw: %s",
  95                     folly::errnoStr(errno).c_str());
  96   }
  97 }
  98
  99 // not thread safe - needs a lock when called from the main
 100 // process.
 101 int popen_impl(const char* cmd, const char* mode, pid_t* out_pid) {
 102   int p[2];
 103   auto const read = *mode == 'r';
 104   if (!read && *mode != 'w') return -1;
 105
 106   if (pipe2(p, O_CLOEXEC) < 0) {
 107     return -1;
 108   }
 109
 110   auto pid = fork();
 111   if (pid < 0) {
 112     close(p[0]);
 113     close(p[1]);
 114     return -1;
 115   }
 116   int child_pipe = read ? 1 : 0;
 117   if (pid == 0) {
 118     // child
 119     mprotect_1g_pages(PROT_READ);
 120     // If anything goes wrong, let the OOM killer kill this child process.
 121     Process::OOMScoreAdj(1000);
 122     // replace stdin or stdout with the appropriate end
 123     // of the pipe
 124     if (p[child_pipe] == child_pipe) {
 125       // pretty unlikely, but if it was we must clear CLOEXEC,
 126       // and the only way to do that is to dup it to a new fd,
 127       // and then dup2 it back
 128       p[child_pipe] = fcntl(child_pipe, F_DUPFD_CLOEXEC, 3);
 129     }
 130     dup2(p[child_pipe], child_pipe);
 131     // no need to close p[child_pipe] because of O_CLOEXEC
 132
 133     signal(SIGINT, SIG_DFL);
 134     sigset_t eset;
 135     sigemptyset(&eset);
 136     sigprocmask(SIG_SETMASK, &eset, nullptr);
 137     execl("/bin/sh", "sh", "-c", cmd, nullptr);
 138     Logger::Warning("Failed to exec: `%s'", cmd);
 139     _Exit(1);
 140   }
 141   // parent
 142
 143   // close the pipe we're not using
 144   close(p[child_pipe]);
 145   *out_pid = pid;
 146   return p[1-child_pipe];
 147 }
 148
 149 int64_t ru2microseconds(const rusage& ru) {
 150   int64_t time_us = ru.ru_utime.tv_usec;
 151   time_us += ru.ru_stime.tv_usec;
 152   int64_t time_s = ru.ru_utime.tv_sec;
 153   time_s += ru.ru_stime.tv_sec;
 154   return time_us + time_s * 1000000;
 155 }
 156
 157 /*
 158  * Hardware counters can be configured to measure sub-process times,
 159  * but note that any given LightProcess will be running jobs for
 160  * multiple request threads (each request thread binds to a single
 161  * LightProcess, but its a many to one mapping).
 162  *
 163  * This means that between the fork/exec and the waitpid, the LightProcess
 164  * could fork/exec more processes for different requests. The solution
 165  * is to fork/exec in a thread, and start the hardware counters there.
 166  *
 167  * The hardware counter will then measure time for that thread, and all
 168  * its children - which is exactly what we want.
 169  *
 170  * HardwareCounterWrapper & co take care of that.
 171  */
 172 struct HardwareCounterWrapperArg {
 173   boost::barrier barrier{2};
 174   pthread_t thr;
 175   int afdt_fd;
 176   pid_t (*func)(int);
 177   pid_t pid;
 178   int64_t *events;
 179 };
 180
 181 std::map<pid_t, std::unique_ptr<HardwareCounterWrapperArg>> s_pidToHCWMap;
 182
 183 void* hardwareCounterWrapper(void* varg) {
 184   auto arg = (HardwareCounterWrapperArg*)varg;
 185
 186   HardwareCounter::s_counter.getCheck();
 187   HardwareCounter::Reset();
 188   arg->pid = arg->func(arg->afdt_fd);
 189   // tell the main thread that pid has been set.
 190   arg->barrier.wait();
 191   if (arg->pid < 0) return nullptr;
 192
 193   // Wait until the main thread is ready to join us.
 194   // Note that even though we have multiple threads running in the LightProcess
 195   // now, at the time any of the do_* functions are called, any live threads are
 196   // waiting here on this barrier; so there is no problem with fork, malloc,
 197   // exec.
 198   arg->barrier.wait();
 199   HardwareCounter::GetPerfEvents(
 200     [](const std::string& event, int64_t value, void* data) {
 201       auto events = reinterpret_cast<int64_t*>(data);
 202       if (event == "instructions") {
 203         events[0] = value;
 204       } else if (event == "loads") {
 205         events[1] = value;
 206       } else if (event == "stores") {
 207         events[2] = value;
 208       }
 209     },
 210     arg->events);
 211
 212   return nullptr;
 213 }
 214
 215 void hardwareCounterWrapperHelper(pid_t (*func)(int), int afdt_fd) {
 216   if (!s_trackProcessTimes) {
 217     func(afdt_fd);
 218     return;
 219   }
 220
 221   auto arg = std::make_unique<HardwareCounterWrapperArg>();
 222   arg->afdt_fd = afdt_fd;
 223   arg->func = func;
 224   if (pthread_create(&arg->thr, nullptr, hardwareCounterWrapper, arg.get())) {
 225     throw Exception("Failed to pthread_create: %s",
 226                     folly::errnoStr(errno).c_str());
 227   }
 228   // Wait for the pid to be set. Note that we must not add any code here that
 229   // could cause issues for the fork (eg malloc); we should wait on the barrier
 230   // immediately after the thread is created.
 231   arg->barrier.wait();
 232   if (arg->pid > 0) {
 233     // successfully forked, so don't join until waitpid.
 234     auto& map_entry = s_pidToHCWMap[arg->pid];
 235     map_entry = std::move(arg);
 236   } else {
 237     // fork failed, join now.
 238     pthread_join(arg->thr, nullptr);
 239   }
 240 }
 241
 242 ///////////////////////////////////////////////////////////////////////////////
 243 // shadow process tasks
 244
 245 pid_t do_popen_helper(int afdt_fd) {
 246   std::string mode, buf, cwd;
 247
 248   lwp_read(afdt_fd, mode, buf, cwd);
 249
 250   std::string old_cwd;
 251
 252   if (!cwd.empty()) {
 253     old_cwd = Process::GetCurrentDirectory();
 254     if (old_cwd != cwd) {
 255       if (chdir(cwd.c_str())) {
 256         // Ignore chdir failures, because the compiled version might not
 257         // have the directory any more.
 258         Logger::Warning("Light Process failed chdir to %s.", cwd.c_str());
 259       }
 260     }
 261   }
 262
 263   pid_t pid;
 264
 265   auto fd = buf.empty() ? -1 :
 266     popen_impl(buf.c_str(), mode.c_str(), &pid);
 267
 268   if (!old_cwd.empty() && chdir(old_cwd.c_str())) {
 269     // only here if we can't change the cwd back
 270   }
 271
 272   if (fd < 0) {
 273     Logger::Error("Light process failed popen: %d (%s).", errno,
 274                   folly::errnoStr(errno).c_str());
 275     lwp_write(afdt_fd, "error");
 276   } else {
 277     lwp_write(afdt_fd, "success", pid);
 278     send_fd(afdt_fd, fd);
 279     // the fd is now owned by the main process, close our copy
 280     close(fd);
 281   }
 282   return pid;
 283 }
 284
 285 void do_popen(int afdt_fd) {
 286   hardwareCounterWrapperHelper(do_popen_helper, afdt_fd);
 287 }
 288
 289 pid_t do_proc_open_helper(int afdt_fd) {
 290   std::string cmd, cwd;
 291   std::vector<std::string> env;
 292   std::vector<int> pvals;
 293   lwp_read(afdt_fd, cmd, cwd, env, pvals);
 294
 295   std::vector<int> pkeys;
 296   for (int i = 0; i < pvals.size(); i++) {
 297     int fd = recv_fd(afdt_fd);
 298     if (fd < 0) {
 299       lwp_write(afdt_fd, "error", (int32_t)EPROTO);
 300       close_fds(pkeys);
 301       return -1;
 302     }
 303     pkeys.push_back(fd);
 304   }
 305
 306   // indicate error if an empty command was received
 307   if (cmd.length() == 0) {
 308     lwp_write(afdt_fd, "error", (int32_t)ENOENT);
 309     return -1;
 310   }
 311
 312   // now ready to start the child process
 313   pid_t child = fork();
 314   if (child == 0) {
 315     mprotect_1g_pages(PROT_READ);
 316     Process::OOMScoreAdj(1000);
 317     for (int i = 0; i < pvals.size(); i++) {
 318       dup2(pkeys[i], pvals[i]);
 319     }
 320     if (!cwd.empty() && chdir(cwd.c_str())) {
 321       // non-zero for error
 322       // chdir failed, the working directory remains unchanged
 323     }
 324     if (!env.empty()) {
 325       char **envp = build_envp(env);
 326       execle("/bin/sh", "sh", "-c", cmd.c_str(), nullptr, envp);
 327       free(envp);
 328     } else {
 329       execl("/bin/sh", "sh", "-c", cmd.c_str(), nullptr);
 330     }
 331     _Exit(HPHP_EXIT_FAILURE);
 332   }
 333
 334   if (child > 0) {
 335     // successfully created the child process
 336     lwp_write(afdt_fd, "success", child);
 337   } else {
 338     // failed creating the child process
 339     lwp_write(afdt_fd, "error", errno);
 340   }
 341
 342   close_fds(pkeys);
 343   return child;
 344 }
 345
 346 void do_proc_open(int afdt_fd) {
 347   hardwareCounterWrapperHelper(do_proc_open_helper, afdt_fd);
 348 }
 349
 350 pid_t waited = 0;
 351
 352 void kill_handler(int sig) {
 353   if (sig == SIGALRM && waited) {
 354     kill(waited, SIGKILL);
 355   }
 356 }
 357
 358 void do_waitpid(int afdt_fd) {
 359   pid_t pid = -1;
 360   int options = 0;
 361   int timeout = 0;
 362   lwp_read(afdt_fd, pid, options, timeout);
 363
 364   int stat;
 365   if (timeout > 0) {
 366     waited = pid;
 367     signal(SIGALRM, kill_handler);
 368     alarm(timeout);
 369   }
 370
 371   rusage ru;
 372   int64_t time_us = 0;
 373   const auto ret = ::wait4(pid, &stat, options, &ru);
 374   alarm(0); // cancel the previous alarm if not triggered yet
 375   waited = 0;
 376   int64_t events[] = { 0, 0, 0 };
 377   if (ret > 0 && s_trackProcessTimes) {
 378     time_us = ru2microseconds(ru);
 379     auto it = s_pidToHCWMap.find(ret);
 380     if (it == s_pidToHCWMap.end()) {
 381       throw Exception("pid not in map: %s",
 382                       folly::errnoStr(errno).c_str());
 383     }
 384
 385     auto hcw = std::move(it->second);
 386     s_pidToHCWMap.erase(it);
 387     hcw->events = events;
 388     hcw->barrier.wait();
 389     pthread_join(hcw->thr, nullptr);
 390   }
 391
 392   lwp_write(afdt_fd, ret, errno, stat,
 393             time_us, events[0], events[1], events[2]);
 394 }
 395
 396 void do_change_user(int afdt_fd) {
 397   std::string uname;
 398   lwp_read(afdt_fd, uname);
 399   if (uname.length() > 0) {
 400     struct passwd *pw = getpwnam(uname.c_str());
 401     if (pw) {
 402       if (pw->pw_gid) {
 403         initgroups(pw->pw_name, pw->pw_gid);
 404         setgid(pw->pw_gid);
 405       }
 406       if (pw->pw_uid) {
 407         setuid(pw->pw_uid);
 408       }
 409     }
 410   }
 411 }
 412
 413 ///////////////////////////////////////////////////////////////////////////////
 414 // light-weight process
 415
 416 std::unique_ptr<LightProcess[]> g_procs;
 417 int g_procsCount = 0;
 418 bool s_handlerInited = false;
 419 LightProcess::LostChildHandler s_lostChildHandler;
 420 std::map<FILE*, pid_t> s_popenMap;
 421
 422 } // namespace
 423
 424 LightProcess::LightProcess() : m_shadowProcess(0), m_afdt_fd(-1) { }
 425
 426 LightProcess::~LightProcess() {
 427 }
 428
 429 void LightProcess::SigChldHandler(int /*sig*/, siginfo_t* info, void* /*ctx*/) {
 430   if (info->si_code != CLD_EXITED &&
 431       info->si_code != CLD_KILLED &&
 432       info->si_code != CLD_DUMPED) {
 433     return;
 434   }
 435   pid_t pid = info->si_pid;
 436   for (int i = 0; i < g_procsCount; ++i) {
 437     if (g_procs && g_procs[i].m_shadowProcess == pid) {
 438       // The exited process was a light process. Notify the callback, if any.
 439       if (s_lostChildHandler) {
 440         s_lostChildHandler(pid);
 441       }
 442       break;
 443     }
 444   }
 445 }
 446
 447 void LightProcess::Initialize(const std::string &prefix, int count,
 448                               bool trackProcessTimes,
 449                               const std::vector<int> &inherited_fds) {
 450   s_trackProcessTimes = trackProcessTimes;
 451
 452   if (prefix.empty() || count <= 0) {
 453     return;
 454   }
 455
 456   if (Available()) {
 457     // already initialized
 458     return;
 459   }
 460
 461   g_procs.reset(new LightProcess[count]);
 462   g_procsCount = count;
 463
 464   auto afdt_filename = folly::sformat("{}.{}", prefix, getpid());
 465
 466   // remove the possible leftover
 467   remove(afdt_filename.c_str());
 468
 469   afdt_error_t err = AFDT_ERROR_T_INIT;
 470   auto afdt_lid = afdt_listen(afdt_filename.c_str(), &err);
 471   if (afdt_lid < 0) {
 472     Logger::Error("Unable to afdt_listen to %s: %d %s",
 473                   afdt_filename.c_str(),
 474                   errno, folly::errnoStr(errno).c_str());
 475     abort();
 476   }
 477
 478   SCOPE_EXIT {
 479     ::close(afdt_lid);
 480     remove(afdt_filename.c_str());
 481   };
 482
 483   for (int i = 0; i < count; i++) {
 484     if (!g_procs[i].initShadow(afdt_lid, afdt_filename, i, inherited_fds)) {
 485       for (int j = 0; j < i; j++) {
 486         g_procs[j].closeShadow();
 487       }
 488       g_procs.reset();
 489       g_procsCount = 0;
 490       break;
 491     }
 492   }
 493
 494   if (!s_handlerInited) {
 495     struct sigaction sa = {};
 496     sa.sa_sigaction = &LightProcess::SigChldHandler;
 497     sa.sa_flags = SA_SIGINFO | SA_NOCLDSTOP;
 498     if (sigaction(SIGCHLD, &sa, nullptr) != 0) {
 499       Logger::Error("Couldn't install SIGCHLD handler");
 500       abort();
 501     }
 502     s_handlerInited = true;
 503   }
 504 }
 505
 506 bool LightProcess::initShadow(int afdt_lid,
 507                               const std::string &afdt_filename, int id,
 508                               const std::vector<int> &inherited_fds) {
 509   Lock lock(m_procMutex);
 510
 511   pid_t child = fork();
 512   if (child == 0) {
 513     // child
 514     mprotect_1g_pages(PROT_READ);
 515     if (s_trackProcessTimes) {
 516       HardwareCounter::RecordSubprocessTimes();
 517     }
 518     Logger::ResetPid();
 519     pid_t sid = setsid();
 520     if (sid < 0) {
 521       Logger::Warning("Unable to setsid");
 522       _Exit(HPHP_EXIT_FAILURE);
 523     }
 524     afdt_error_t err = AFDT_ERROR_T_INIT;
 525     auto afdt_fd = afdt_connect(afdt_filename.c_str(), &err);
 526     if (afdt_fd < 0) {
 527       Logger::Warning("Unable to afdt_connect, filename %s: %d %s",
 528                       afdt_filename.c_str(),
 529                       errno, folly::errnoStr(errno).c_str());
 530       _Exit(HPHP_EXIT_FAILURE);
 531     }
 532
 533     // shadow process doesn't use g_procs
 534     for (int i = 0; i < id; i++) {
 535       g_procs[i].closeFiles();
 536     }
 537     g_procs.reset();
 538     g_procsCount = 0;
 539     close_fds(inherited_fds);
 540     ::close(afdt_lid);
 541     // Tell the OOM killer never to kill a light process.  Killing it will cause
 542     // the entire server to exit, and won't free much memory anyway.
 543     Process::OOMScoreAdj(-1000);
 544     runShadow(afdt_fd);
 545   } else if (child < 0) {
 546     // failed
 547     Logger::Warning("Unable to fork lightly: %d %s", errno,
 548                     folly::errnoStr(errno).c_str());
 549     return false;
 550   } else {
 551     // parent
 552     m_shadowProcess = child;
 553
 554     sockaddr addr;
 555     socklen_t addrlen = sizeof(addr);
 556     m_afdt_fd = accept(afdt_lid, &addr, &addrlen);
 557     if (m_afdt_fd < 0) {
 558       Logger::Warning("Unable to establish afdt connection: %d %s",
 559                       errno, folly::errnoStr(errno).c_str());
 560       closeShadow();
 561       return false;
 562     }
 563   }
 564   return true;
 565 }
 566
 567 void LightProcess::runShadow(int afdt_fd) {
 568   std::string buf;
 569
 570   pollfd pfd[1];
 571   pfd[0].fd = afdt_fd;
 572   pfd[0].events = POLLIN;
 573   try {
 574     while (true) {
 575       int ret = poll(pfd, 1, -1);
 576       if (ret < 0 && errno == EINTR) {
 577         continue;
 578       }
 579       if (pfd[0].revents & POLLHUP) {
 580         // no more command can come in
 581         Logger::Error("Lost parent, LightProcess exiting");
 582         break;
 583       }
 584       if (pfd[0].revents & POLLIN) {
 585         lwp_read(afdt_fd, buf);
 586         if (buf == "exit") {
 587           Logger::Verbose("LightProcess exiting upon request");
 588           break;
 589         } else if (buf == "popen") {
 590           do_popen(afdt_fd);
 591         } else if (buf == "proc_open") {
 592           do_proc_open(afdt_fd);
 593         } else if (buf == "waitpid") {
 594           do_waitpid(afdt_fd);
 595         } else if (buf == "change_user") {
 596           do_change_user(afdt_fd);
 597         } else if (buf[0]) {
 598           Logger::Info("LightProcess got invalid command: %.20s", buf.c_str());
 599         }
 600       }
 601     }
 602   } catch (const std::exception& e) {
 603     Logger::Error("LightProcess exiting due to exception: %s", e.what());
 604   } catch (...) {
 605     Logger::Error("LightProcess exiting due to unknown exception");
 606   }
 607
 608   ::close(afdt_fd);
 609   _Exit(0);
 610 }
 611
 612 namespace {
 613
 614 int GetId() {
 615   return (long)pthread_self() % g_procsCount;
 616 }
 617
 618 NEVER_INLINE
 619 void handleException(const char* call) {
 620   try {
 621     throw;
 622   } catch (const std::exception& e) {
 623     Logger::Error("LightProcess::%s failed due to exception: %s",
 624                   call, e.what());
 625   } catch (...) {
 626     Logger::Error("LightProcess::%s failed due to unknown exception",
 627                   call);
 628   }
 629 }
 630
 631 template <class R, class F1>
 632 R runLight(const char* call, F1 body, R failureResult) {
 633   try {
 634     auto proc = tl_proc ? tl_proc : &g_procs[GetId()];
 635     Lock lock(proc->mutex());
 636
 637     return body(proc);
 638   } catch (...) {
 639     handleException(call);
 640   }
 641   return failureResult;
 642 }
 643
 644 }
 645
 646 void LightProcess::Close() {
 647   std::unique_ptr<LightProcess[]> procs;
 648   procs.swap(g_procs);
 649   int count = g_procsCount;
 650   g_procs.reset();
 651   g_procsCount = 0;
 652
 653   for (int i = 0; i < count; i++) {
 654     procs[i].closeShadow();
 655   }
 656 }
 657
 658 void LightProcess::closeShadow() {
 659   Lock lock(m_procMutex);
 660   if (m_shadowProcess) {
 661     try {
 662       lwp_write(m_afdt_fd, "exit");
 663     } catch (...) {
 664       handleException("closeShadow");
 665     }
 666     // removes the "zombie" process, so not to interfere with later waits
 667     while (true) {
 668       auto r = ::waitpid(m_shadowProcess, nullptr, 0);
 669       // retry on EINTR
 670       if (r != -1 || errno != EINTR) {
 671         break;
 672       }
 673     }
 674
 675     m_shadowProcess = 0;
 676   }
 677   closeFiles();
 678 }
 679
 680 void LightProcess::closeFiles() {
 681   if (m_afdt_fd >= 0) {
 682     ::close(m_afdt_fd);
 683     m_afdt_fd = -1;
 684   }
 685 }
 686
 687 bool LightProcess::Available() {
 688   if (tl_proc) return true;
 689   return g_procsCount > 0;
 690 }
 691
 692 FILE *LightProcess::popen(const char *cmd, const char *type,
 693                           const char *cwd /* = NULL */) {
 694   if (!Available()) {
 695     // fallback to normal popen
 696     Logger::Verbose("Light-weight fork not available; "
 697                     "use the heavy one instead.");
 698   } else {
 699     FILE *f = LightPopenImpl(cmd, type, cwd);
 700     if (f) {
 701       return f;
 702     }
 703     if (tl_proc) {
 704       Logger::Verbose("Light-weight fork failed in remote CLI mode.");
 705       return nullptr;
 706     }
 707     Logger::Verbose("Light-weight fork failed; use the heavy one instead.");
 708   }
 709   return HeavyPopenImpl(cmd, type, cwd);
 710 }
 711
 712 FILE *LightProcess::HeavyPopenImpl(const char *cmd, const char *type,
 713                                    const char *cwd) {
 714   int fd = -1;
 715   pid_t pid;
 716   Lock lock(s_mutex);
 717   if (cwd && *cwd) {
 718     auto old_cwd = Process::GetCurrentDirectory();
 719     if (old_cwd != cwd) {
 720       if (chdir(cwd)) {
 721         Logger::Warning("Failed to chdir to %s.", cwd);
 722       }
 723       fd = popen_impl(cmd, type, &pid);
 724       if (chdir(old_cwd.c_str())) {
 725         // error occurred changing cwd back
 726       }
 727       if (fd < 0) return nullptr;
 728     }
 729   }
 730   if (fd < 0) {
 731     fd = popen_impl(cmd, type, &pid);
 732   }
 733   if (fd < 0) return nullptr;
 734   auto f = fdopen(fd, type);
 735   if (f) {
 736     s_popenMap[f] = pid;
 737   }
 738   return f;
 739 }
 740
 741 FILE *LightProcess::LightPopenImpl(const char *cmd, const char *type,
 742                                    const char *cwd) {
 743   return runLight("popen", [&] (LightProcess* proc) -> FILE* {
 744       auto afdt_fd = proc->m_afdt_fd;
 745       lwp_write(afdt_fd, "popen", type, cmd, cwd ? cwd : "");
 746
 747       std::string buf;
 748       lwp_read(afdt_fd, buf);
 749       if (buf == "error") {
 750         return nullptr;
 751       }
 752
 753       pid_t pid;
 754       lwp_read(afdt_fd, pid);
 755       assert(pid);
 756       int fd = recv_fd(afdt_fd);
 757       if (fd < 0) {
 758         Logger::Error("Light process failed to send the file descriptor.");
 759         return nullptr;
 760       }
 761       FILE *f = fdopen(fd, type);
 762       if (f) {
 763         proc->m_popenMap[f] = pid;
 764       }
 765       return f;
 766     }, static_cast<FILE*>(nullptr));
 767 }
 768
 769 int LightProcess::pclose(FILE *f) {
 770   pid_t pid;
 771   if (!Available()) {
 772     Lock lock(s_mutex);
 773     auto it = s_popenMap.find(f);
 774     if (it == s_popenMap.end()) return -1;
 775     pid = it->second;
 776     s_popenMap.erase(it);
 777   } else {
 778     auto proc = [] {
 779       if (tl_proc) return tl_proc;
 780       return &g_procs[GetId()];
 781     }();
 782     Lock lock(proc->m_procMutex);
 783
 784     auto it = proc->m_popenMap.find(f);
 785     if (it == proc->m_popenMap.end()) return -1;
 786     pid = it->second;
 787     proc->m_popenMap.erase(it);
 788   }
 789
 790   fclose(f);
 791   int status;
 792   if (LightProcess::waitpid(pid, &status, 0, 0) < 0) return -1;
 793   return status;
 794 }
 795
 796 pid_t LightProcess::proc_open(const char *cmd, const std::vector<int> &created,
 797                               const std::vector<int> &desired,
 798                               const char *cwd,
 799                               const std::vector<std::string> &env) {
 800   always_assert(Available());
 801   always_assert(created.size() == desired.size());
 802
 803   return runLight("proc_open", [&] (LightProcess* proc) -> pid_t {
 804       auto fout = proc->m_afdt_fd;
 805       lwp_write(fout, "proc_open", cmd, cwd ? cwd : "",
 806                 env, desired);
 807
 808       bool error_send = false;
 809       int save_errno = 0;
 810       for (auto cfd : created) {
 811         if (!send_fd(proc->m_afdt_fd, cfd)) {
 812           error_send = true;
 813           save_errno = errno;
 814           break;
 815         }
 816       }
 817
 818       std::string buf;
 819       auto fin = proc->m_afdt_fd;
 820       lwp_read(fin, buf);
 821       if (buf == "error") {
 822         lwp_read(fin, errno);
 823         if (error_send) {
 824           // On this error, the receiver side returns dummy errno,
 825           // use the sender side errno here.
 826           errno = save_errno;
 827         }
 828         return -1;
 829       }
 830       always_assert_flog(buf == "success",
 831                          "Unexpected message from light process: `{}'", buf);
 832       pid_t pid = -1;
 833       lwp_read(fin, pid);
 834       always_assert(pid);
 835       return pid;
 836     }, static_cast<pid_t>(-1));
 837 }
 838
 839 pid_t LightProcess::waitpid(pid_t pid, int *stat_loc, int options,
 840                             int timeout) {
 841   if (!Available()) {
 842     // light process is not really there
 843     rusage ru;
 844     const auto ret = wait4(pid, stat_loc, options, &ru);
 845     if (ret > 0 && s_trackProcessTimes) {
 846       s_extra_request_nanoseconds += ru2microseconds(ru) * 1000;
 847     }
 848     return ret;
 849   }
 850
 851   return runLight("waitpid", [&] (LightProcess* proc) -> pid_t {
 852       lwp_write(proc->m_afdt_fd, "waitpid", pid, options, timeout);
 853
 854       pid_t ret;
 855       int stat;
 856       int err;
 857       int64_t time_us, events[3];
 858       lwp_read(proc->m_afdt_fd, ret, err, stat,
 859                time_us, events[0], events[1], events[2]);
 860
 861       *stat_loc = stat;
 862       if (ret < 0) {
 863         errno = err;
 864       } else if (s_trackProcessTimes) {
 865         s_extra_request_nanoseconds += time_us * 1000;
 866         HardwareCounter::IncInstructionCount(events[0]);
 867         HardwareCounter::IncLoadCount(events[1]);
 868         HardwareCounter::IncStoreCount(events[2]);
 869       }
 870
 871       return ret;
 872     }, static_cast<pid_t>(-1));
 873 }
 874
 875 pid_t LightProcess::pcntl_waitpid(pid_t pid, int *stat_loc, int options) {
 876   rusage ru;
 877   const auto ret = wait4(pid, stat_loc, options, &ru);
 878   if (ret > 0 && s_trackProcessTimes) {
 879     s_extra_request_nanoseconds += ru2microseconds(ru) * 1000;
 880   }
 881   return ret;
 882 }
 883
 884 void LightProcess::ChangeUser(int afdt, const std::string& username) {
 885   if (!username.empty()) lwp_write(afdt, "change_user", username);
 886 }
 887
 888 void LightProcess::ChangeUser(const std::string& username) {
 889   if (username.empty()) return;
 890   for (int i = 0; i < g_procsCount; i++) {
 891     Lock lock(g_procs[i].m_procMutex);
 892     lwp_write(g_procs[i].m_afdt_fd, "change_user", username);
 893   }
 894 }
 895
 896 void LightProcess::SetLostChildHandler(const LostChildHandler& handler) {
 897   s_lostChildHandler = handler;
 898 }
 899
 900 std::unique_ptr<LightProcess> LightProcess::setThreadLocalAfdtOverride(
 901   std::unique_ptr<LightProcess> p
 902 ) {
 903   auto ret = std::unique_ptr<LightProcess>(tl_proc);
 904   tl_proc = p.release();
 905   return ret;
 906 }
 907
 908 std::unique_ptr<LightProcess> LightProcess::setThreadLocalAfdtOverride(int fd) {
 909   auto ret = std::unique_ptr<LightProcess>(tl_proc);
 910   tl_proc = new LightProcess;
 911   tl_proc->m_afdt_fd = fd;
 912   return ret;
 913 }
 914
 915 int LightProcess::createDelegate() {
 916   int pair[2];
 917   if (socketpair(AF_UNIX, SOCK_STREAM, 0, pair)) {
 918     Logger::Warning("Unable to create a unix socket pair: %s",
 919                     folly::errnoStr(errno).c_str());
 920     return -1;
 921   }
 922
 923   pid_t child = fork();
 924
 925   if (child < 0) {
 926     Logger::Warning("Unable to fork delegate process: %s",
 927                     folly::errnoStr(errno).c_str());
 928     close(pair[0]);
 929     close(pair[1]);
 930     return -1;
 931   }
 932
 933   if (child == 0) {
 934     // child
 935     fclose(stdin);
 936     fclose(stdout);
 937     fclose(stderr);
 938
 939     mprotect_1g_pages(PROT_READ);
 940     if (s_trackProcessTimes) {
 941       HardwareCounter::RecordSubprocessTimes();
 942     }
 943     Logger::ResetPid();
 944     pid_t sid = setsid();
 945     if (sid < 0) {
 946       Logger::Warning("Unable to setsid");
 947       _Exit(HPHP_EXIT_FAILURE);
 948     }
 949
 950     close(pair[0]);
 951 #ifdef __APPLE__
 952     {
 953       int newfd = dup2(pair[1], 0);
 954       always_assert(newfd == 0);
 955     }
 956     close(pair[1]);
 957     pair[1] = 0;
 958 #endif
 959     runShadow(pair[1]);
 960   }
 961
 962   always_assert(child > 0);
 963   close(pair[1]);
 964   return pair[0];
 965 }
 966
 967 ///////////////////////////////////////////////////////////////////////////////
 968 }