hphp/runtime/base/unit-cache.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #include "hphp/runtime/base/unit-cache.h"
  17
  18 #include <sys/types.h>
  19 #include <sys/stat.h>
  20 #include <memory>
  21 #include <string>
  22 #include <cstdlib>
  23 #include <thread>
  24
  25 #include <folly/ScopeGuard.h>
  26 #include <folly/portability/Fcntl.h>
  27
  28 #include "hphp/util/assertions.h"
  29 #include "hphp/util/rank.h"
  30 #include "hphp/util/mutex.h"
  31 #include "hphp/util/process.h"
  32 #include "hphp/util/struct-log.h"
  33 #include "hphp/util/timer.h"
  34 #include "hphp/runtime/base/system-profiler.h"
  35 #include "hphp/runtime/base/runtime-option.h"
  36 #include "hphp/runtime/base/zend-string.h"
  37 #include "hphp/runtime/base/string-util.h"
  38 #include "hphp/runtime/base/file-util.h"
  39 #include "hphp/runtime/base/plain-file.h"
  40 #include "hphp/runtime/base/stat-cache.h"
  41 #include "hphp/runtime/base/stream-wrapper-registry.h"
  42 #include "hphp/runtime/base/file-stream-wrapper.h"
  43 #include "hphp/runtime/base/program-functions.h"
  44 #include "hphp/runtime/base/rds.h"
  45 #include "hphp/runtime/server/cli-server.h"
  46 #include "hphp/runtime/server/source-root-info.h"
  47 #include "hphp/runtime/vm/debugger-hook.h"
  48 #include "hphp/runtime/vm/extern-compiler.h"
  49 #include "hphp/runtime/vm/repo.h"
  50 #include "hphp/runtime/vm/runtime.h"
  51 #include "hphp/runtime/vm/treadmill.h"
  52
  53 #ifdef __APPLE__
  54 #define st_mtim st_mtimespec
  55 #define st_ctim st_ctimespec
  56 #endif
  57
  58 namespace HPHP {
  59
  60 //////////////////////////////////////////////////////////////////////
  61
  62 namespace {
  63
  64 //////////////////////////////////////////////////////////////////////
  65
  66 using OptLog = folly::Optional<StructuredLogEntry>;
  67
  68 struct LogTimer {
  69   LogTimer(const char* name, OptLog& ent)
  70     : m_name(name)
  71     , m_ent(ent)
  72     , m_start(m_ent ? Timer::GetThreadCPUTimeNanos() : -1)
  73   {}
  74   ~LogTimer() { stop(); }
  75
  76   void stop() {
  77     if (m_start == -1) return;
  78
  79     auto const elapsed = Timer::GetThreadCPUTimeNanos() - m_start;
  80     m_ent->setInt(m_name, elapsed / 1000);
  81     m_start = -1;
  82   }
  83
  84 private:
  85   const char* m_name;
  86   OptLog& m_ent;
  87   int64_t m_start;
  88 };
  89
  90 struct CachedUnit {
  91   CachedUnit() = default;
  92   explicit CachedUnit(Unit* unit, size_t rdsBitId)
  93     : unit(unit)
  94     , rdsBitId(rdsBitId)
  95   {}
  96
  97   Unit* unit{nullptr};  // null if there is no Unit for this path
  98   size_t rdsBitId{-1u}; // id of the RDS bit for whether the Unit is included
  99 };
 100
 101 //////////////////////////////////////////////////////////////////////
 102 // RepoAuthoritative mode unit caching
 103
 104 /*
 105  * In RepoAuthoritative mode, loaded units are never unloaded, we
 106  * don't support symlink chasing, you can't include urls, and files
 107  * are never changed, which makes the code here significantly simpler.
 108  * Because of this it pays to keep it separate from the other cases so
 109  * they don't need to be littered with RepoAuthoritative checks.
 110  */
 111
 112 using RepoUnitCache = RankedCHM<
 113   const StringData*,     // must be static
 114   CachedUnit,
 115   StringDataHashCompare,
 116   RankUnitCache
 117 >;
 118 RepoUnitCache s_repoUnitCache;
 119
 120 CachedUnit lookupUnitRepoAuth(const StringData* path) {
 121   path = makeStaticString(path);
 122
 123   RepoUnitCache::accessor acc;
 124   if (!s_repoUnitCache.insert(acc, path)) {
 125     return acc->second;
 126   }
 127
 128   try {
 129     /*
 130      * Insert path.  Find the Md5 for this path, and then the unit for
 131      * this Md5.  If either aren't found we return the
 132      * default-constructed cache entry.
 133      *
 134      * NB: we're holding the CHM lock on this bucket while we're doing
 135      * this.
 136      */
 137     MD5 md5;
 138     if (Repo::get().findFile(path->data(),
 139                              RuntimeOption::SourceRoot,
 140                              md5) == RepoStatus::error) {
 141       return acc->second;
 142     }
 143
 144     acc->second.unit = Repo::get().loadUnit(path->data(), md5).release();
 145     if (acc->second.unit) {
 146       acc->second.rdsBitId = rds::allocBit();
 147     }
 148   } catch (...) {
 149     s_repoUnitCache.erase(acc);
 150     throw;
 151   }
 152   return acc->second;
 153 }
 154
 155 //////////////////////////////////////////////////////////////////////
 156 // Non-repo mode unit caching
 157
 158 struct CachedUnitWithFree {
 159   CachedUnitWithFree() = delete;
 160   explicit CachedUnitWithFree(const CachedUnitWithFree&) = delete;
 161   CachedUnitWithFree& operator=(const CachedUnitWithFree&) = delete;
 162
 163   explicit CachedUnitWithFree(const CachedUnit& src) : cu(src) {}
 164   ~CachedUnitWithFree() {
 165     if (auto oldUnit = cu.unit) {
 166       Treadmill::enqueue([oldUnit] { delete oldUnit; });
 167     }
 168   }
 169   CachedUnit cu;
 170 };
 171
 172 struct CachedUnitNonRepo {
 173   std::shared_ptr<CachedUnitWithFree> cachedUnit;
 174 #ifdef _MSC_VER
 175   time_t mtime;
 176 #else
 177   struct timespec mtime;
 178   struct timespec ctime;
 179 #endif
 180   ino_t ino;
 181   dev_t devId;
 182 };
 183
 184 using NonRepoUnitCache = RankedCHM<
 185   const StringData*,     // must be static
 186   CachedUnitNonRepo,
 187   StringDataHashCompare,
 188   RankUnitCache
 189 >;
 190 NonRepoUnitCache s_nonRepoUnitCache;
 191
 192 // When running in remote unix server mode with UnixServerQuarantineUnits set,
 193 // we need to cache any unit generated from a file descriptor passed via the
 194 // client process in a special quarantined cached. Units in this cache may only
 195 // be loaded in unix server requests from the same user and are never used when
 196 // handling web requests.
 197 using PerUserCache = folly::AtomicHashMap<uid_t, NonRepoUnitCache*>;
 198 PerUserCache s_perUserUnitCaches(10);
 199
 200 #ifndef _MSC_VER
 201 int64_t timespecCompare(const struct timespec& l,
 202                         const struct timespec& r) {
 203   if (l.tv_sec != r.tv_sec) return l.tv_sec - r.tv_sec;
 204   return l.tv_nsec - r.tv_nsec;
 205 }
 206 #endif
 207
 208 uint64_t g_units_seen_count = 0;
 209
 210 bool stressUnitCache() {
 211   if (RuntimeOption::EvalStressUnitCacheFreq <= 0) return false;
 212   if (RuntimeOption::EvalStressUnitCacheFreq == 1) return true;
 213   return ++g_units_seen_count % RuntimeOption::EvalStressUnitCacheFreq == 0;
 214 }
 215
 216 bool isChanged(const CachedUnitNonRepo& cu, const struct stat& s) {
 217   // If the cached unit is null, we always need to consider it out of date (in
 218   // case someone created the file).  This case should only happen if something
 219   // successfully stat'd the file, but then it was gone by the time we tried to
 220   // open() it.
 221   return !cu.cachedUnit ||
 222          cu.cachedUnit->cu.unit == nullptr ||
 223 #ifdef _MSC_VER
 224          cu.mtime - s.st_mtime < 0 ||
 225 #else
 226          timespecCompare(cu.mtime, s.st_mtim) < 0 ||
 227          timespecCompare(cu.ctime, s.st_ctim) < 0 ||
 228 #endif
 229          cu.ino != s.st_ino ||
 230          cu.devId != s.st_dev ||
 231          stressUnitCache();
 232 }
 233
 234 folly::Optional<String> readFileAsString(Stream::Wrapper* w,
 235                                          const StringData* path) {
 236   if (w) {
 237     if (const auto f = w->open(StrNR(path), "r", 0, nullptr)) {
 238       return f->read();
 239     }
 240     return folly::none;
 241   }
 242   auto const fd = open(path->data(), O_RDONLY);
 243   if (fd < 0) return folly::none;
 244   auto file = req::make<PlainFile>(fd);
 245   return file->read();
 246 }
 247
 248 CachedUnit createUnitFromString(const char* path,
 249                                 const String& contents,
 250                                 Unit** releaseUnit,
 251                                 OptLog& ent) {
 252   auto const md5 = MD5{mangleUnitMd5(string_md5(contents.slice()))};
 253   // Try the repo; if it's not already there, invoke the compiler.
 254   if (auto unit = Repo::get().loadUnit(path, md5)) {
 255     return CachedUnit { unit.release(), rds::allocBit() };
 256   }
 257   LogTimer compileTimer("compile_ms", ent);
 258   auto const unit = compile_file(contents.data(), contents.size(), md5, path,
 259                                  releaseUnit);
 260   return CachedUnit { unit, rds::allocBit() };
 261 }
 262
 263 CachedUnit createUnitFromUrl(const StringData* const requestedPath) {
 264   auto const w = Stream::getWrapperFromURI(StrNR(requestedPath));
 265   if (!w) return CachedUnit{};
 266   auto const f = w->open(StrNR(requestedPath), "r", 0, nullptr);
 267   if (!f) return CachedUnit{};
 268   StringBuffer sb;
 269   sb.read(f.get());
 270   OptLog ent;
 271   return createUnitFromString(requestedPath->data(), sb.detach(), nullptr, ent);
 272 }
 273
 274 CachedUnit createUnitFromFile(const StringData* const path,
 275                               Unit** releaseUnit, Stream::Wrapper* w,
 276                               OptLog& ent) {
 277   auto const contents = readFileAsString(w, path);
 278   return contents
 279     ? createUnitFromString(path->data(), *contents, releaseUnit, ent)
 280     : CachedUnit{};
 281 }
 282
 283 // When running via the CLI server special access checks may need to be
 284 // performed, and in the event that the server is unable to load the file an
 285 // alternative per client cache may be used.
 286 NonRepoUnitCache& getNonRepoCache(const StringData* rpath,
 287                                   Stream::Wrapper*& w) {
 288   if (auto uc = get_cli_ucred()) {
 289     if (!(w = Stream::getWrapperFromURI(StrNR(rpath)))) {
 290       return s_nonRepoUnitCache;
 291     }
 292
 293     auto unit_check_quarantine = [&] () -> NonRepoUnitCache& {
 294       if (!RuntimeOption::EvalUnixServerQuarantineUnits) {
 295         return s_nonRepoUnitCache;
 296       }
 297       auto iter = s_perUserUnitCaches.find(uc->uid);
 298       if (iter != s_perUserUnitCaches.end()) return *iter->second;
 299       auto cache = new NonRepoUnitCache;
 300       auto res = s_perUserUnitCaches.insert(uc->uid, cache);
 301       if (!res.second) delete cache;
 302       return *res.first->second;
 303     };
 304
 305     // If the server cannot access rpath attempt to open the unit on the
 306     // client. When UnixServerQuarantineUnits is set store units opened by
 307     // clients in per UID caches which are never accessible by server web
 308     // requests.
 309     if (access(rpath->data(), R_OK) == -1) {
 310       return unit_check_quarantine();
 311     }
 312
 313     // When UnixServerVerifyExeAccess is set clients may not execute units if
 314     // they cannot read them, even when the server has access. To verify that
 315     // clients have access they are asked to open the file for read access,
 316     // and using fstat the server verifies that the file it sees is identical
 317     // to the unit opened by the client.
 318     if (RuntimeOption::EvalUnixServerVerifyExeAccess) {
 319       struct stat local, remote;
 320       auto remoteFile = w->open(StrNR(rpath), "r", 0, nullptr);
 321       if (!remoteFile ||
 322           fcntl(remoteFile->fd(), F_GETFL) != O_RDONLY ||
 323           fstat(remoteFile->fd(), &remote) != 0 ||
 324           stat(rpath->data(), &local) != 0 ||
 325           remote.st_dev != local.st_dev ||
 326           remote.st_ino != local.st_ino) {
 327         return unit_check_quarantine();
 328       }
 329     }
 330
 331     // When the server is able to read the file prefer to access it that way,
 332     // in all modes units loaded by the server are cached for all clients.
 333     w = nullptr;
 334   }
 335   return s_nonRepoUnitCache;
 336 }
 337
 338 CachedUnit loadUnitNonRepoAuth(StringData* requestedPath,
 339                                const struct stat& statInfo,
 340                                OptLog& ent) {
 341   LogTimer loadTime("load_ms", ent);
 342   if (strstr(requestedPath->data(), "://") != nullptr) {
 343     // URL-based units are not currently cached in memory, but the Repo still
 344     // caches them on disk.
 345     return createUnitFromUrl(requestedPath);
 346   }
 347
 348   // The string we're using as a key must be static, because we're using it as
 349   // a key in the cache (across requests).
 350   auto const path =
 351     makeStaticString(
 352       // XXX: it seems weird we have to do this even though we already ran
 353       // resolveVmInclude.
 354       (FileUtil::isAbsolutePath(requestedPath->toCppString())
 355        ?  String{requestedPath}
 356         : String(SourceRootInfo::GetCurrentSourceRoot()) + StrNR(requestedPath)
 357       ).get()
 358     );
 359
 360   auto const rpath = [&] () -> const StringData* {
 361     if (RuntimeOption::CheckSymLink) {
 362       std::string rp = StatCache::realpath(path->data());
 363       if (rp.size() != 0) {
 364         if (rp.size() != path->size() ||
 365             memcmp(rp.data(), path->data(), rp.size())) {
 366           return makeStaticString(rp);
 367         }
 368       }
 369     }
 370     return path;
 371   }();
 372
 373   Stream::Wrapper* w = nullptr;
 374   auto& cache = getNonRepoCache(rpath, w);
 375
 376   assert(
 377     !w || &cache != &s_nonRepoUnitCache ||
 378     !RuntimeOption::EvalUnixServerQuarantineUnits
 379   );
 380
 381   // Freeing a unit while holding the tbb lock would cause a rank violation when
 382   // recycle-tc is enabled as reclaiming dead functions requires that the code
 383   // and metadata locks be acquired.
 384   Unit* releaseUnit = nullptr;
 385   SCOPE_EXIT { if (releaseUnit) delete releaseUnit; };
 386
 387   auto const cuptr = [&] () -> std::shared_ptr<CachedUnitWithFree> {
 388     NonRepoUnitCache::accessor rpathAcc;
 389
 390     if (!cache.insert(rpathAcc, rpath)) {
 391       if (!isChanged(rpathAcc->second, statInfo)) {
 392         if (ent) ent->setStr("type", "cache_hit_writelock");
 393         return rpathAcc->second.cachedUnit;
 394       }
 395       if (ent) ent->setStr("type", "cache_stale");
 396     } else {
 397       if (ent) ent->setStr("type", "cache_miss");
 398     }
 399
 400     /*
 401      * NB: the new-unit creation path is here, and is done while holding the tbb
 402      * lock on s_nonRepoUnitCache.  This was originally done deliberately to
 403      * avoid wasting time in the compiler (during server startup, many requests
 404      * hit the same code initial paths that are shared, and would all be
 405      * compiling the same files).  It's not 100% clear if this is the best way
 406      * to handle that idea, though (tbb locks spin aggressively and are
 407      * expected to be low contention).
 408      */
 409
 410     auto const cu = createUnitFromFile(rpath, &releaseUnit, w, ent);
 411     rpathAcc->second.cachedUnit = std::make_shared<CachedUnitWithFree>(cu);
 412 #ifdef _MSC_VER
 413     rpathAcc->second.mtime      = statInfo.st_mtime;
 414 #else
 415     rpathAcc->second.mtime      = statInfo.st_mtim;
 416     rpathAcc->second.ctime      = statInfo.st_ctim;
 417 #endif
 418     rpathAcc->second.ino        = statInfo.st_ino;
 419     rpathAcc->second.devId      = statInfo.st_dev;
 420
 421     return rpathAcc->second.cachedUnit;
 422   }();
 423
 424   if (path != rpath) {
 425     NonRepoUnitCache::accessor pathAcc;
 426     cache.insert(pathAcc, path);
 427     pathAcc->second.cachedUnit = cuptr;
 428 #ifdef _MSC_VER
 429     pathAcc->second.mtime      = statInfo.st_mtime;
 430 #else
 431     pathAcc->second.mtime      = statInfo.st_mtim;
 432     pathAcc->second.ctime      = statInfo.st_ctim;
 433 #endif
 434     pathAcc->second.ino        = statInfo.st_ino;
 435     pathAcc->second.devId      = statInfo.st_dev;
 436   }
 437
 438   return cuptr->cu;
 439 }
 440
 441 CachedUnit lookupUnitNonRepoAuth(StringData* requestedPath,
 442                                  const struct stat& statInfo,
 443                                  OptLog& ent) {
 444   // Steady state, its probably already in the cache. Try that first
 445   {
 446     NonRepoUnitCache::const_accessor acc;
 447     if (s_nonRepoUnitCache.find(acc, requestedPath)) {
 448       if (!isChanged(acc->second, statInfo)) {
 449         if (ent) ent->setStr("type", "cache_hit_readlock");
 450         return acc->second.cachedUnit->cu;
 451       }
 452     }
 453   }
 454   return loadUnitNonRepoAuth(requestedPath, statInfo, ent);
 455 }
 456
 457 //////////////////////////////////////////////////////////////////////
 458 // resolveVmInclude callbacks
 459
 460 struct ResolveIncludeContext {
 461   String path;    // translated path of the file
 462   struct stat* s; // stat for the file
 463   bool allow_dir; // return true for dirs?
 464 };
 465
 466 bool findFile(const StringData* path, struct stat* s, bool allow_dir,
 467               Stream::Wrapper* w) {
 468   // We rely on this side-effect in RepoAuthoritative mode right now, since the
 469   // stat information is an output-param of resolveVmInclude, but we aren't
 470   // really going to call stat.
 471   s->st_mode = 0;
 472
 473   if (RuntimeOption::RepoAuthoritative) {
 474     return lookupUnitRepoAuth(path).unit != nullptr;
 475   }
 476
 477   if (StatCache::stat(path->data(), s) == 0) {
 478     // The call explicitly populates the struct for dirs, but returns false for
 479     // them because it is geared toward file includes.
 480     return allow_dir || !S_ISDIR(s->st_mode);
 481   }
 482
 483   if (w && w->stat(StrNR(path), s) == 0) {
 484     return allow_dir || !S_ISDIR(s->st_mode);
 485   }
 486   return false;
 487 }
 488
 489 bool findFileWrapper(const String& file, void* ctx) {
 490   auto const context = static_cast<ResolveIncludeContext*>(ctx);
 491   assert(context->path.isNull());
 492
 493   Stream::Wrapper* w = Stream::getWrapperFromURI(file);
 494   if (w && !w->isNormalFileStream()) {
 495     if (w->stat(file, context->s) == 0) {
 496       context->path = file;
 497       return true;
 498     }
 499   }
 500
 501   // handle file://
 502   if (StringUtil::IsFileUrl(file)) {
 503     return findFileWrapper(file.substr(7), ctx);
 504   }
 505
 506   if (!w) return false;
 507
 508   auto passW =
 509     RuntimeOption::RepoAuthoritative ||
 510     dynamic_cast<FileStreamWrapper*>(w) ||
 511     !w->isNormalFileStream()
 512       ? nullptr
 513       : w;
 514
 515   // TranslatePath() will canonicalize the path and also check
 516   // whether the file is in an allowed directory.
 517   String translatedPath = File::TranslatePathKeepRelative(file);
 518   if (!FileUtil::isAbsolutePath(file.toCppString())) {
 519     if (findFile(translatedPath.get(), context->s, context->allow_dir,
 520                  passW)) {
 521       context->path = translatedPath;
 522       return true;
 523     }
 524     return false;
 525   }
 526   if (RuntimeOption::SandboxMode || !RuntimeOption::AlwaysUseRelativePath) {
 527     if (findFile(translatedPath.get(), context->s, context->allow_dir, passW)) {
 528       context->path = translatedPath;
 529       return true;
 530     }
 531   }
 532   std::string server_root(SourceRootInfo::GetCurrentSourceRoot());
 533   if (server_root.empty()) {
 534     server_root = std::string(g_context->getCwd().data());
 535     if (server_root.empty() ||
 536         FileUtil::isDirSeparator(server_root[server_root.size() - 1])) {
 537       server_root += FileUtil::getDirSeparator();
 538     }
 539   }
 540   String rel_path(FileUtil::relativePath(server_root, translatedPath.data()));
 541   if (findFile(rel_path.get(), context->s, context->allow_dir, passW)) {
 542     context->path = rel_path;
 543     return true;
 544   }
 545   return false;
 546 }
 547
 548 void logLoad(
 549   StructuredLogEntry& ent,
 550   StringData* path,
 551   const char* cwd,
 552   String rpath,
 553   const CachedUnit& cu
 554 ) {
 555   ent.setStr("include_path", path->data());
 556   ent.setStr("current_dir", cwd);
 557   ent.setStr("resolved_path", rpath.data());
 558   if (auto const u = cu.unit) {
 559     const StringData* err;
 560     int line;
 561     if (u->compileTimeFatal(err, line)) {
 562       ent.setStr("result", "compile_fatal");
 563       ent.setStr("error", err->data());
 564     } else if (u->parseFatal(err, line)) {
 565       ent.setStr("result", "parse_fatal");
 566       ent.setStr("error", err->data());
 567     } else {
 568       ent.setStr("result", "success");
 569     }
 570
 571     ent.setStr("md5", u->md5().toString());
 572     ent.setStr("repo_sn", folly::to<std::string>(u->sn()));
 573     ent.setStr("repo_id", folly::to<std::string>(u->repoID()));
 574
 575     ent.setInt("bc_len", u->bclen());
 576     ent.setInt("num_litstrs", u->numLitstrs());
 577     ent.setInt("num_funcs", u->funcs().size());
 578     ent.setInt("num_classes", u->preclasses().size());
 579     ent.setInt("num_type_aliases", u->typeAliases().size());
 580   } else {
 581     ent.setStr("result", "file_not_found");
 582   }
 583
 584   switch (requestKind) {
 585   case RequestKind::Warmup: ent.setStr("request_kind", "warmup"); break;
 586   case RequestKind::Profile: ent.setStr("request_kind", "profile"); break;
 587   case RequestKind::Standard: ent.setStr("request_kind", "standard"); break;
 588   case RequestKind::NonVM: ent.setStr("request_kind", "nonVM"); break;
 589   }
 590   ent.setInt("request_count", requestCount());
 591
 592   StructuredLog::log("hhvm_unit_cache", ent);
 593 }
 594
 595 //////////////////////////////////////////////////////////////////////
 596
 597 CachedUnit checkoutFile(
 598   StringData* path,
 599   const struct stat& statInfo,
 600   OptLog& ent
 601 ) {
 602   return RuntimeOption::RepoAuthoritative
 603     ? lookupUnitRepoAuth(path)
 604     : lookupUnitNonRepoAuth(path, statInfo, ent);
 605 }
 606
 607 const std::string mangleUnitPHP7Options() {
 608   // As the list of options increases, we may want to do something smarter here?
 609   std::string s;
 610   s += (RuntimeOption::PHP7_IntSemantics ? '1' : '0') +
 611       (RuntimeOption::PHP7_LTR_assign ? '1' : '0') +
 612       (RuntimeOption::PHP7_NoHexNumerics ? '1' : '0') +
 613       (RuntimeOption::PHP7_Builtins ? '1' : '0') +
 614       (RuntimeOption::PHP7_ScalarTypes ? '1' : '0') +
 615       (RuntimeOption::PHP7_Substr ? '1' : '0') +
 616       (RuntimeOption::PHP7_UVS ? '1' : '0');
 617   return s;
 618 }
 619
 620 const std::string mangleAliasedNamespaces() {
 621   std::string s;
 622   s += folly::to<std::string>(RuntimeOption::AliasedNamespaces.size());
 623   s += '\0';
 624   for (auto& par : RuntimeOption::AliasedNamespaces) {
 625     s += par.first + '\0' + par.second + '\0';
 626   }
 627   return s;
 628 }
 629
 630 const std::string mangleExternCompilerVersions() {
 631   std::string s;
 632   if (HackcMode::kNever != hackc_mode()) {
 633     s += hackc_version();
 634   }
 635   return s;
 636 }
 637
 638 //////////////////////////////////////////////////////////////////////
 639
 640 } // end empty namespace
 641
 642 //////////////////////////////////////////////////////////////////////
 643
 644 std::string mangleUnitMd5(const std::string& fileMd5) {
 645   std::string t = fileMd5 + '\0'
 646     + (RuntimeOption::AssertEmitted ? '1' : '0')
 647     + (RuntimeOption::AutoprimeGenerators ? '1' : '0')
 648     + (RuntimeOption::EnableHipHopExperimentalSyntax ? '1' : '0')
 649     + (RuntimeOption::EnableHipHopSyntax ? '1' : '0')
 650     + (RuntimeOption::EnableXHP ? '1' : '0')
 651     + (RuntimeOption::EvalAllowHhas ? '1' : '0')
 652     + (RuntimeOption::EvalEmitSwitch ? '1' : '0')
 653     + (RuntimeOption::EvalEnableCallBuiltin ? '1' : '0')
 654     + (RuntimeOption::EvalHackArrCompatNotices ? '1' : '0')
 655     + (RuntimeOption::EvalHackArrCompatIsArrayNotices ? '1' : '0')
 656     + (RuntimeOption::EvalHackArrCompatPromoteNotices ? '1' : '0')
 657     + (RuntimeOption::EvalHackArrCompatTypeHintNotices ? '1' : '0')
 658     + (RuntimeOption::EvalHackArrCompatDVCmpNotices ? '1' : '0')
 659     + (RuntimeOption::EvalHackCompilerFallback ? '1' : '0')
 660     + (RuntimeOption::EvalJitEnableRenameFunction ? '1' : '0')
 661     + (RuntimeOption::EvalLoadFilepathFromUnitCache ? '1' : '0')
 662     + (RuntimeOption::IntsOverflowToInts ? '1' : '0')
 663     + (RuntimeOption::EvalReffinessInvariance ? '1' : '0')
 664     + (RuntimeOption::EvalCreateInOutWrapperFunctions ? '1' : '0')
 665     + std::to_string(RuntimeOption::EvalForbidDynamicCalls)
 666     + (RuntimeOption::EvalNoticeOnBuiltinDynamicCalls ? '1' : '0')
 667     + (RuntimeOption::EvalHackArrDVArrs ? '1' : '0')
 668     + (RuntimeOption::EvalDisableHphpcOpts ? '1' : '0')
 669     + (RuntimeOption::EvalUseMSRVForInOut ? '1' : '0')
 670     + (RuntimeOption::EvalAllowObjectDestructors ? '1' : '0')
 671     + (RuntimeOption::EvalUseExternCompilerForSystemLib ? '1' : '0')
 672     + RuntimeOption::EvalHackCompilerCommand + '\0'
 673     + mangleUnitPHP7Options()
 674     + mangleAliasedNamespaces()
 675     + mangleExternCompilerVersions();
 676   return string_md5(t);
 677 }
 678
 679 size_t numLoadedUnits() {
 680   if (RuntimeOption::RepoAuthoritative) {
 681     return s_repoUnitCache.size();
 682   }
 683   return s_nonRepoUnitCache.size();
 684 }
 685
 686 String resolveVmInclude(StringData* path,
 687                         const char* currentDir,
 688                         struct stat* s,
 689                         bool allow_dir /* = false */) {
 690   ResolveIncludeContext ctx;
 691   ctx.s = s;
 692   ctx.allow_dir = allow_dir;
 693   void* vpCtx = &ctx;
 694   resolve_include(String{path}, currentDir, findFileWrapper, vpCtx);
 695   // If resolve_include() could not find the file, return NULL
 696   return ctx.path;
 697 }
 698
 699 Unit* lookupUnit(StringData* path, const char* currentDir, bool* initial_opt) {
 700   bool init;
 701   bool& initial = initial_opt ? *initial_opt : init;
 702   initial = true;
 703
 704   OptLog ent;
 705   if (!RuntimeOption::RepoAuthoritative &&
 706       StructuredLog::coinflip(RuntimeOption::EvalLogUnitLoadRate)) {
 707     ent.emplace();
 708   }
 709
 710   LogTimer lookupTimer("lookup_ms", ent);
 711
 712   /*
 713    * NB: the m_evaledFiles map is only for the debugger, and could be omitted
 714    * in RepoAuthoritative mode, but currently isn't.
 715    */
 716
 717   struct stat s;
 718   auto const spath = resolveVmInclude(path, currentDir, &s);
 719   if (spath.isNull()) return nullptr;
 720
 721   auto const eContext = g_context.getNoCheck();
 722
 723   // Check if this file has already been included.
 724   auto it = eContext->m_evaledFiles.find(spath.get());
 725   if (it != end(eContext->m_evaledFiles)) {
 726     // In RepoAuthoritative mode we assume that the files are unchanged.
 727     initial = false;
 728     if (RuntimeOption::RepoAuthoritative ||
 729         (it->second.ts_sec > s.st_mtime) ||
 730         ((it->second.ts_sec == s.st_mtime) &&
 731          (it->second.ts_nsec >= s.st_mtim.tv_nsec))) {
 732       return it->second.unit;
 733     }
 734   }
 735
 736   // This file hasn't been included yet, so we need to parse the file
 737   auto const cunit = checkoutFile(spath.get(), s, ent);
 738   if (cunit.unit && initial_opt) {
 739     // if initial_opt is not set, this shouldn't be recorded as a
 740     // per request fetch of the file.
 741     if (rds::testAndSetBit(cunit.rdsBitId)) {
 742       initial = false;
 743     }
 744     // if parsing was successful, update the mappings for spath and
 745     // rpath (if it exists).
 746     eContext->m_evaledFilesOrder.push_back(cunit.unit->filepath());
 747     eContext->m_evaledFiles[spath.get()] =
 748       {cunit.unit, s.st_mtime, static_cast<unsigned long>(s.st_mtim.tv_nsec)};
 749     spath.get()->incRefCount();
 750     if (!cunit.unit->filepath()->same(spath.get())) {
 751       eContext->m_evaledFiles[cunit.unit->filepath()] =
 752         {cunit.unit, s.st_mtime, static_cast<unsigned long>(s.st_mtim.tv_nsec)};
 753     }
 754     if (g_system_profiler) {
 755       g_system_profiler->fileLoadCallBack(path->toCppString());
 756     }
 757     DEBUGGER_ATTACHED_ONLY(phpDebuggerFileLoadHook(cunit.unit));
 758   }
 759
 760   lookupTimer.stop();
 761   if (ent) logLoad(*ent, path, currentDir, spath, cunit);
 762   return cunit.unit;
 763 }
 764
 765 //////////////////////////////////////////////////////////////////////
 766
 767 void preloadRepo() {
 768   auto& repo = Repo::get();
 769   auto units = repo.enumerateUnits(RepoIdLocal, true, false);
 770   if (units.size() == 0) {
 771     units = repo.enumerateUnits(RepoIdCentral, true, false);
 772   }
 773   if (!units.size()) return;
 774
 775   std::vector<std::thread> workers;
 776   auto numWorkers = Process::GetCPUCount();
 777   // Compute a batch size that causes each thread to process approximately 16
 778   // batches.  Even if the batches are somewhat imbalanced in what they contain,
 779   // the straggler workers are very unlikey to take more than 10% longer than
 780   // the first worker to finish.
 781   size_t batchSize{std::max(units.size() / numWorkers / 16, size_t(1))};
 782   std::atomic<size_t> index{0};
 783   for (auto worker = 0; worker < numWorkers; ++worker) {
 784     workers.push_back(std::thread([&] {
 785       hphp_thread_init();
 786       hphp_session_init();
 787
 788       while (true) {
 789         auto begin = index.fetch_add(batchSize);
 790         auto end = std::min(begin + batchSize, units.size());
 791         if (begin >= end) break;
 792         auto unitCount = end - begin;
 793         for (auto i = size_t{0}; i < unitCount; ++i) {
 794           auto& kv = units[begin + i];
 795           try {
 796             lookupUnit(String(RuntimeOption::SourceRoot + kv.first).get(),
 797                        "", nullptr);
 798           } catch (...) {
 799             // swallow errors silently
 800           }
 801         }
 802       }
 803
 804       hphp_context_exit();
 805       hphp_session_exit();
 806       hphp_thread_exit();
 807     }));
 808   }
 809   for (auto& worker : workers) {
 810     worker.join();
 811   }
 812 }
 813
 814 //////////////////////////////////////////////////////////////////////
 815
 816 }