hphp/runtime/vm/repo.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #include "hphp/runtime/vm/repo.h"
  17
  18 #include <folly/Format.h>
  19 #include <folly/Singleton.h>
  20
  21 #include "hphp/hhbbc/options.h"
  22 #include "hphp/runtime/vm/blob-helper.h"
  23 #include "hphp/runtime/vm/repo-global-data.h"
  24 #include "hphp/runtime/server/xbox-server.h"
  25
  26 #include "hphp/util/assertions.h"
  27 #include "hphp/util/async-func.h"
  28 #include "hphp/util/build-info.h"
  29 #include "hphp/util/hugetlb.h"
  30 #include "hphp/util/logger.h"
  31 #include "hphp/util/process.h"
  32 #include "hphp/util/trace.h"
  33
  34 #include <grp.h>
  35 #include <pwd.h>
  36 #include <sys/stat.h>
  37
  38 namespace HPHP {
  39
  40 TRACE_SET_MOD(hhbc);
  41
  42 const char* Repo::kMagicProduct =
  43   "facebook.com HipHop Virtual Machine bytecode repository";
  44 const char* Repo::kSchemaPlaceholder = "%{schema}";
  45 const char* Repo::kDbs[RepoIdCount] = { "main",   // Central.
  46                                         "local"}; // Local.
  47 Repo::GlobalData Repo::s_globalData;
  48
  49 void initialize_repo() {
  50   if (!sqlite3_threadsafe()) {
  51     TRACE(0, "SQLite was compiled without thread support; aborting\n");
  52     abort();
  53   }
  54   if (sqlite3_config(SQLITE_CONFIG_MULTITHREAD) != SQLITE_OK) {
  55     TRACE(1, "Failed to set default SQLite multi-threading mode\n");
  56   }
  57   if (sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0) != SQLITE_OK) {
  58     TRACE(1, "Failed to disable SQLite memory statistics\n");
  59   }
  60 }
  61
  62 THREAD_LOCAL(Repo, t_dh);
  63
  64 Repo& Repo::get() {
  65   return *t_dh.get();
  66 }
  67
  68 void Repo::shutdown() {
  69   t_dh.destroy();
  70 }
  71
  72 static SimpleMutex s_lock;
  73 static std::atomic<unsigned> s_nRepos;
  74
  75 bool Repo::prefork() {
  76   if (num_1g_pages() > 0) {
  77     // We put data on the 1G huge pages, and we don't want to do COW upon
  78     // fork().  If you need to fork(), configure HHVM not to use 1G pages.
  79     return true;
  80   }
  81   if (!t_dh.isNull()) {
  82     t_dh.destroy();
  83   }
  84   s_lock.lock();
  85   XboxServer::Stop();
  86   if (s_nRepos > 0 || AsyncFuncImpl::count()) {
  87     XboxServer::Restart();
  88     s_lock.unlock();
  89     return true;
  90   }
  91   folly::SingletonVault::singleton()->destroyInstances();
  92   return false;
  93 }
  94
  95 void Repo::postfork(pid_t pid) {
  96   folly::SingletonVault::singleton()->reenableInstances();
  97   XboxServer::Restart();
  98   if (pid == 0) {
  99     Logger::ResetPid();
 100     new (&s_lock) SimpleMutex();
 101   } else {
 102     s_lock.unlock();
 103   }
 104 }
 105
 106 Repo::Repo()
 107   : RepoProxy(*this),
 108     m_insertFileHash{InsertFileHashStmt(*this, 0),
 109                      InsertFileHashStmt(*this, 1)},
 110     m_getFileHash{GetFileHashStmt(*this, 0), GetFileHashStmt(*this, 1)},
 111     m_dbc(nullptr), m_localReadable(false), m_localWritable(false),
 112     m_evalRepoId(-1), m_txDepth(0), m_rollback(false), m_beginStmt(*this),
 113     m_rollbackStmt(*this), m_commitStmt(*this), m_urp(*this), m_pcrp(*this),
 114     m_frp(*this), m_lsrp(*this) {
 115
 116   ++s_nRepos;
 117   connect();
 118 }
 119
 120 Repo::~Repo() {
 121   disconnect();
 122   --s_nRepos;
 123 }
 124
 125 std::string Repo::s_cliFile;
 126 void Repo::setCliFile(const std::string& cliFile) {
 127   assert(s_cliFile.empty());
 128   assert(t_dh.isNull());
 129   s_cliFile = cliFile;
 130 }
 131
 132 size_t Repo::stringLengthLimit() const {
 133   static const size_t limit = sqlite3_limit(m_dbc, SQLITE_LIMIT_LENGTH, -1);
 134   return limit;
 135 }
 136
 137 void Repo::loadGlobalData(bool allowFailure /* = false */,
 138                           bool readArrayTable /* = true */) {
 139   m_lsrp.load();
 140
 141   if (!RuntimeOption::RepoAuthoritative) return;
 142
 143   std::vector<std::string> failures;
 144
 145   /*
 146    * This should probably just go to the Local repo always, except
 147    * that our unit test suite is currently running RepoAuthoritative
 148    * tests with the compiled repo as the Central repo.
 149    */
 150   for (int repoId = RepoIdCount - 1; repoId >= 0; --repoId) {
 151     if (repoName(repoId).empty()) {
 152       // The repo wasn't loadable
 153       continue;
 154     }
 155     try {
 156       RepoStmt stmt(*this);
 157       const auto& tbl = table(repoId, "GlobalData");
 158       stmt.prepare(
 159         folly::format(
 160           "SELECT count(*), data from {};", tbl
 161         ).str()
 162       );
 163       RepoTxn txn(*this);
 164       RepoTxnQuery query(txn, stmt);
 165       query.step();
 166       if (!query.row()) {
 167         throw RepoExc("Can't find table %s", tbl.c_str());
 168       };
 169       int val;
 170       query.getInt(0, val);
 171       if (val == 0) {
 172         throw RepoExc("No rows in %s. Did you forget to compile that file with "
 173                       "this HHVM version?", tbl.c_str());
 174       }
 175       BlobDecoder decoder = query.getBlob(1);
 176       decoder(s_globalData);
 177       if (readArrayTable) {
 178         auto& arrayTypeTable = globalArrayTypeTable();
 179         decoder(arrayTypeTable);
 180         decoder.assertDone();
 181       }
 182       txn.commit();
 183     } catch (RepoExc& e) {
 184       failures.push_back(repoName(repoId) + ": "  + e.msg());
 185       continue;
 186     }
 187
 188     // TODO: this should probably read out the other elements of the global data
 189     // which control Option or RuntimeOption values -- the others are read out
 190     // in an inconsistent and ad-hoc manner. But I don't understand their uses
 191     // and interactions well enough to feel comfortable fixing now.
 192     RuntimeOption::EvalPromoteEmptyObject    = s_globalData.PromoteEmptyObject;
 193     RuntimeOption::EnableIntrinsicsExtension =
 194       s_globalData.EnableIntrinsicsExtension;
 195     HHBBC::options.ElideAutoloadInvokes     = s_globalData.ElideAutoloadInvokes;
 196     RuntimeOption::AutoprimeGenerators      = s_globalData.AutoprimeGenerators;
 197     RuntimeOption::EnableHipHopSyntax       = s_globalData.EnableHipHopSyntax;
 198     RuntimeOption::EvalHardTypeHints          = s_globalData.HardTypeHints;
 199     RuntimeOption::EvalUseHHBBC               = s_globalData.UsedHHBBC;
 200     RuntimeOption::PHP7_Builtins              = s_globalData.PHP7_Builtins;
 201     RuntimeOption::PHP7_IntSemantics          = s_globalData.PHP7_IntSemantics;
 202     RuntimeOption::PHP7_ScalarTypes           = s_globalData.PHP7_ScalarTypes;
 203     RuntimeOption::PHP7_Substr                = s_globalData.PHP7_Substr;
 204     RuntimeOption::DisallowDynamicVarEnvFuncs =
 205       s_globalData.DisallowDynamicVarEnvFuncs;
 206
 207     if (s_globalData.HardReturnTypeHints) {
 208       RuntimeOption::EvalCheckReturnTypeHints = 3;
 209     }
 210     if (s_globalData.ThisTypeHintLevel == 3) {
 211       RuntimeOption::EvalThisTypeHintLevel = s_globalData.ThisTypeHintLevel;
 212     }
 213
 214     if (RuntimeOption::ServerExecutionMode() &&
 215         RuntimeOption::EvalHackArrCompatNotices) {
 216       // Temporary until we verify Makefile changes work in prod
 217       Logger::Info(
 218         folly::sformat(
 219           "HackArrCompatNotices is {} in repo",
 220           s_globalData.HackArrCompatNotices ? "enabled" : "disabled")
 221       );
 222     }
 223
 224     return;
 225   }
 226
 227   if (allowFailure) return;
 228
 229   if (failures.empty()) {
 230     std::fprintf(stderr, "No repo was loadable. Check all the possible repo "
 231                  "locations (Repo.Central.Path, HHVM_REPO_CENTRAL_PATH, and "
 232                  "$HOME/.hhvm.hhbc) to make sure one of them is a valid "
 233                  "sqlite3 HHVM repo built with this exact HHVM version.\n");
 234   } else {
 235     // We should always have a global data section in RepoAuthoritative
 236     // mode, or the repo is messed up.
 237     std::fprintf(stderr, "Failed to load Repo::GlobalData:\n");
 238     for (auto& f : failures) {
 239       std::fprintf(stderr, "  %s\n", f.c_str());
 240     }
 241   }
 242
 243   assert(Process::IsInMainThread());
 244   exit(1);
 245 }
 246
 247 void Repo::saveGlobalData(GlobalData newData) {
 248   s_globalData = newData;
 249
 250   auto const repoId = repoIdForNewUnit(UnitOrigin::File);
 251   RepoStmt stmt(*this);
 252   stmt.prepare(
 253     folly::format(
 254       "INSERT INTO {} VALUES(@data);", table(repoId, "GlobalData")
 255     ).str()
 256   );
 257   RepoTxn txn(*this);
 258   RepoTxnQuery query(txn, stmt);
 259   BlobEncoder encoder;
 260   encoder(s_globalData);
 261   encoder(globalArrayTypeTable());
 262   query.bindBlob("@data", encoder, /* static */ true);
 263   query.exec();
 264
 265   // TODO(#3521039): we could just put the litstr table in the same
 266   // blob as the above and delete LitstrRepoProxy.
 267   LitstrTable::get().forEachNamedEntity(
 268     [this, &txn, repoId](int i, const NamedEntityPair& namedEntity) {
 269       lsrp().insertLitstr(repoId).insert(txn, i, namedEntity.first);
 270     });
 271
 272   txn.commit();
 273 }
 274
 275 std::unique_ptr<Unit> Repo::loadUnit(const std::string& name, const MD5& md5) {
 276   if (m_dbc == nullptr) {
 277     return nullptr;
 278   }
 279   return m_urp.load(name, md5);
 280 }
 281
 282 std::vector<std::pair<std::string,MD5>>
 283 Repo::enumerateUnits(int repoId, bool preloadOnly, bool warn) {
 284   std::vector<std::pair<std::string,MD5>> ret;
 285
 286   try {
 287     RepoStmt stmt(*this);
 288     stmt.prepare(preloadOnly ?
 289                  folly::sformat(
 290                    "SELECT path, {0}.md5 FROM {0} "
 291                    "LEFT JOIN {1} ON ({0}.md5={1}.md5) WHERE preload != 0 "
 292                    "ORDER BY preload DESC;",
 293                    table(repoId, "FileMd5"),
 294                    table(repoId, "Unit")) :
 295                  folly::sformat(
 296                    "SELECT path, md5 FROM {};",
 297                    table(repoId, "FileMd5"))
 298                 );
 299     RepoTxn txn(*this);
 300     RepoTxnQuery query(txn, stmt);
 301
 302     for (query.step(); query.row(); query.step()) {
 303       std::string path;
 304       MD5 md5;
 305
 306       query.getStdString(0, path);
 307       query.getMd5(1, md5);
 308
 309       ret.emplace_back(path, md5);
 310     }
 311
 312     txn.commit();
 313   } catch (RepoExc& e) {
 314     if (warn) {
 315       fprintf(stderr, "failed to enumerate units: %s\n", e.what());
 316     }
 317     // Ugh - the error is dropped.  At least we might have printed an error to
 318     // stderr.
 319   }
 320
 321   return ret;
 322 }
 323
 324 void Repo::InsertFileHashStmt::insert(RepoTxn& txn, const StringData* path,
 325                                       const MD5& md5) {
 326   if (!prepared()) {
 327     std::stringstream ssInsert;
 328     ssInsert << "INSERT INTO " << m_repo.table(m_repoId, "FileMd5")
 329              << " VALUES(@path, @md5);";
 330     txn.prepare(*this, ssInsert.str());
 331   }
 332   RepoTxnQuery query(txn, *this);
 333   query.bindStaticString("@path", path);
 334   query.bindMd5("@md5", md5);
 335   query.exec();
 336 }
 337
 338 RepoStatus Repo::GetFileHashStmt::get(const char *path, MD5& md5) {
 339   try {
 340     RepoTxn txn(m_repo);
 341     if (!prepared()) {
 342       std::stringstream ssSelect;
 343       ssSelect << "SELECT f.md5 FROM "
 344                << m_repo.table(m_repoId, "FileMd5")
 345                << " AS f, " << m_repo.table(m_repoId, "Unit")
 346                << " AS u WHERE path == @path AND f.md5 == u.md5"
 347                << " ORDER BY unitSn DESC LIMIT 1;";
 348       txn.prepare(*this, ssSelect.str());
 349     }
 350     RepoTxnQuery query(txn, *this);
 351     query.bindText("@path", path, strlen(path));
 352     query.step();
 353     if (!query.row()) {
 354       return RepoStatus::error;
 355     }
 356     query.getMd5(0, md5);
 357     txn.commit();
 358     return RepoStatus::success;
 359   } catch (RepoExc& re) {
 360     return RepoStatus::error;
 361   }
 362 }
 363
 364 RepoStatus Repo::findFile(const char *path, const std::string &root, MD5& md5) {
 365   if (m_dbc == nullptr) {
 366     return RepoStatus::error;
 367   }
 368   int repoId;
 369   for (repoId = RepoIdCount - 1; repoId >= 0; --repoId) {
 370     if (*path == '/' && !root.empty() &&
 371         !strncmp(root.c_str(), path, root.size()) &&
 372         (m_getFileHash[repoId].get(path + root.size(), md5) ==
 373          RepoStatus::success)) {
 374       TRACE(3, "Repo loaded file hash for '%s' from '%s'\n",
 375                path + root.size(), repoName(repoId).c_str());
 376       return RepoStatus::success;
 377     }
 378     if (m_getFileHash[repoId].get(path, md5) == RepoStatus::success) {
 379       TRACE(3, "Repo loaded file hash for '%s' from '%s'\n",
 380                 path, repoName(repoId).c_str());
 381       return RepoStatus::success;
 382     }
 383   }
 384   TRACE(3, "Repo file hash: error loading '%s'\n", path);
 385   return RepoStatus::error;
 386 }
 387
 388 RepoStatus Repo::insertMd5(UnitOrigin unitOrigin, UnitEmitter* ue,
 389                            RepoTxn& txn) {
 390   const StringData* path = ue->m_filepath;
 391   const MD5& md5 = ue->md5();
 392   int repoId = repoIdForNewUnit(unitOrigin);
 393   if (repoId == RepoIdInvalid) {
 394     return RepoStatus::error;
 395   }
 396   try {
 397     m_insertFileHash[repoId].insert(txn, path, md5);
 398     return RepoStatus::success;
 399   } catch (RepoExc& re) {
 400     TRACE(3, "Failed to commit md5 for '%s' to '%s': %s\n",
 401               path->data(), repoName(repoId).c_str(), re.msg().c_str());
 402     return RepoStatus::error;
 403   }
 404 }
 405
 406 void Repo::commitMd5(UnitOrigin unitOrigin, UnitEmitter* ue) {
 407   try {
 408     RepoTxn txn(*this);
 409     RepoStatus err = insertMd5(unitOrigin, ue, txn);
 410     if (err == RepoStatus::success) {
 411       txn.commit();
 412     }
 413   } catch (RepoExc& re) {
 414     int repoId = repoIdForNewUnit(unitOrigin);
 415     if (repoId != RepoIdInvalid) {
 416       TRACE(3, "Failed to commit md5 for '%s' to '%s': %s\n",
 417                ue->m_filepath->data(), repoName(repoId).c_str(),
 418                re.msg().c_str());
 419     }
 420   }
 421 }
 422
 423 std::string Repo::table(int repoId, const char* tablePrefix) {
 424   std::stringstream ss;
 425   ss << dbName(repoId) << "." << tablePrefix << "_" << repoSchemaId();
 426   return ss.str();
 427 }
 428
 429 void Repo::exec(const std::string& sQuery) {
 430   RepoStmt stmt(*this);
 431   stmt.prepare(sQuery);
 432   RepoQuery query(stmt);
 433   query.exec();
 434 }
 435
 436 void Repo::begin() {
 437   if (m_txDepth > 0) {
 438     m_txDepth++;
 439     return;
 440   }
 441   if (debug) {
 442     // Verify start state.
 443     always_assert(m_txDepth == 0);
 444     always_assert(!m_rollback);
 445     if (true) {
 446       // Bypass RepoQuery, in order to avoid triggering exceptions.
 447       int rc = sqlite3_step(m_rollbackStmt.get());
 448       switch (rc) {
 449       case SQLITE_DONE:
 450       case SQLITE_ROW:
 451         not_reached();
 452       default:
 453         break;
 454       }
 455     } else {
 456       bool rollbackFailed = false;
 457       try {
 458         RepoQuery query(m_rollbackStmt);
 459         query.exec();
 460       } catch (RepoExc& re) {
 461         rollbackFailed = true;
 462       }
 463       always_assert(rollbackFailed);
 464     }
 465   }
 466   RepoQuery query(m_beginStmt);
 467   query.exec();
 468   m_txDepth++;
 469 }
 470
 471 void Repo::txPop() {
 472   // We mix the concept of rollback with a normal commit so that if we try to
 473   // rollback an inner transaction we eventually end up rolling back the outer
 474   // transaction instead (Sqlite doesn't support rolling back partial
 475   // transactions).
 476   assert(m_txDepth > 0);
 477   if (m_txDepth > 1) {
 478     m_txDepth--;
 479     return;
 480   }
 481   if (!m_rollback) {
 482     RepoQuery query(m_commitStmt);
 483     query.exec();
 484   } else {
 485     // We're in the outermost transaction - so clear the rollback flag.
 486     m_rollback = false;
 487     RepoQuery query(m_rollbackStmt);
 488     try {
 489       query.exec();
 490     } catch (RepoExc& ex) {
 491       /*
 492        * Having a rollback fail is actually a normal, expected case,
 493        * so just swallow this.
 494        *
 495        * In particular, according to the docs, if we got an I/O error
 496        * while doing a commit, the rollback will often fail with "no
 497        * transaction in progress", because the commit will have
 498        * automatically been rolled back.  Recommended practice is
 499        * still to execute a rollback statement and ignore the error.
 500        */
 501       TRACE(4, "repo rollback failure: %s\n", ex.what());
 502     }
 503   }
 504   // Decrement depth after query execution, in case an exception occurs during
 505   // commit.  This allows for subsequent rollback of the failed commit.
 506   m_txDepth--;
 507 }
 508
 509 void Repo::rollback() {
 510   m_rollback = true;
 511   // NOTE: A try/catch isn't necessary - txPop() handles rollback as a nothrow.
 512   txPop();
 513 }
 514
 515 void Repo::commit() {
 516   txPop();
 517 }
 518
 519 RepoStatus Repo::insertUnit(UnitEmitter* ue, UnitOrigin unitOrigin,
 520                             RepoTxn& txn) {
 521   if (insertMd5(unitOrigin, ue, txn) == RepoStatus::error ||
 522       ue->insert(unitOrigin, txn) == RepoStatus::error) {
 523     return RepoStatus::error;
 524   }
 525   return RepoStatus::success;
 526 }
 527
 528 void Repo::commitUnit(UnitEmitter* ue, UnitOrigin unitOrigin) {
 529   if (!RuntimeOption::RepoCommit) return;
 530
 531   try {
 532     commitMd5(unitOrigin, ue);
 533     ue->commit(unitOrigin);
 534   } catch (const std::exception& e) {
 535     TRACE(0, "unexpected exception in commitUnit: %s\n",
 536           e.what());
 537     assert(false);
 538   }
 539 }
 540
 541 void Repo::connect() {
 542   initCentral();
 543   initLocal();
 544   if (!RuntimeOption::RepoEvalMode.compare("local")) {
 545     m_evalRepoId = (m_localWritable) ? RepoIdLocal : RepoIdCentral;
 546   } else if (!RuntimeOption::RepoEvalMode.compare("central")) {
 547     m_evalRepoId = RepoIdCentral;
 548   } else {
 549     assert(!RuntimeOption::RepoEvalMode.compare("readonly"));
 550     m_evalRepoId = RepoIdInvalid;
 551   }
 552   TRACE(1, "Repo.Eval.Mode=%s\n",
 553            (m_evalRepoId == RepoIdLocal)
 554            ? "local"
 555            : (m_evalRepoId == RepoIdCentral)
 556              ? "central"
 557              : "readonly");
 558 }
 559
 560 void Repo::disconnect() {
 561   if (m_dbc != nullptr) {
 562     sqlite3_close(m_dbc);
 563     m_dbc = nullptr;
 564     m_localReadable = false;
 565     m_localWritable = false;
 566     m_evalRepoId = RepoIdInvalid;
 567   }
 568 }
 569
 570 void Repo::initCentral() {
 571   std::string error;
 572
 573   assert(m_dbc == nullptr);
 574   auto tryPath = [this, &error](const char* path) {
 575     std::string subErr;
 576     if (openCentral(path, subErr) == RepoStatus::error) {
 577       folly::format(&error, "  {}\n", subErr.empty() ? path : subErr);
 578       return false;
 579     }
 580     return true;
 581   };
 582
 583   auto fail_no_repo = [&error] {
 584     error = "Failed to initialize central HHBC repository:\n" + error;
 585     // Database initialization failed; this is an unrecoverable state.
 586     Logger::Error("%s", error.c_str());
 587
 588     if (Process::IsInMainThread()) {
 589       exit(1);
 590     }
 591     always_assert_flog(false, "{}", error);
 592   };
 593
 594   // Try Repo.Central.Path
 595   if (!RuntimeOption::RepoCentralPath.empty() &&
 596       tryPath(RuntimeOption::RepoCentralPath.c_str())) {
 597     return;
 598   }
 599
 600   // Try HHVM_REPO_CENTRAL_PATH
 601   const char* HHVM_REPO_CENTRAL_PATH = getenv("HHVM_REPO_CENTRAL_PATH");
 602   if (HHVM_REPO_CENTRAL_PATH != nullptr &&
 603       tryPath(HHVM_REPO_CENTRAL_PATH)) {
 604     return;
 605   }
 606
 607   if (!RuntimeOption::RepoAllowFallbackPath) fail_no_repo();
 608
 609   // Try "$HOME/.hhvm.hhbc".
 610   char* HOME = getenv("HOME");
 611   if (HOME != nullptr) {
 612     std::string centralPath = HOME;
 613     centralPath += "/.hhvm.hhbc";
 614     if (tryPath(centralPath.c_str())) {
 615       return;
 616     }
 617   }
 618
 619 #ifndef _WIN32
 620   // Try the equivalent of "$HOME/.hhvm.hhbc", but look up the home directory
 621   // in the password database.
 622   {
 623     passwd pwbuf;
 624     passwd* pwbufp;
 625     long bufsize = sysconf(_SC_GETPW_R_SIZE_MAX);
 626     if (bufsize != -1) {
 627       auto buf = new char[bufsize];
 628       SCOPE_EXIT { delete[] buf; };
 629       if (!getpwuid_r(getuid(), &pwbuf, buf, size_t(bufsize), &pwbufp)
 630           && (HOME == nullptr || strcmp(HOME, pwbufp->pw_dir))) {
 631         std::string centralPath = pwbufp->pw_dir;
 632         centralPath += "/.hhvm.hhbc";
 633         if (tryPath(centralPath.c_str())) {
 634           return;
 635         }
 636       }
 637     }
 638   }
 639 #else // _WIN32
 640   // Try "$HOMEDRIVE$HOMEPATH/.hhvm.hhbc"
 641   char* HOMEDRIVE = getenv("HOMEDRIVE");
 642   if (HOMEDRIVE != nullptr) {
 643     char* HOMEPATH = getenv("HOMEPATH");
 644     if (HOMEPATH != nullptr) {
 645       std::string centralPath = HOMEDRIVE;
 646       centralPath += HOMEPATH;
 647       centralPath += "\\.hhvm.hhbc";
 648       if (tryPath(centralPath.c_str()))
 649         return;
 650     }
 651   }
 652 #endif
 653
 654   fail_no_repo();
 655 }
 656
 657 static int busyHandler(void* opaque, int nCalls) {
 658   Repo* repo UNUSED = static_cast<Repo*>(opaque);
 659   // yield to allow other threads access to the machine
 660   // spin-wait can starve other threads.
 661   usleep(1000 * nCalls);
 662   return 1; // Tell SQLite to retry.
 663 }
 664
 665 std::string Repo::insertSchema(const char* path) {
 666   assert(strstr(repoSchemaId().begin(), kSchemaPlaceholder) == nullptr);
 667   std::string result = path;
 668   size_t idx;
 669   if ((idx = result.find(kSchemaPlaceholder)) != std::string::npos) {
 670     result.replace(idx, strlen(kSchemaPlaceholder), repoSchemaId().begin());
 671   }
 672   TRACE(2, "Repo::%s() transformed %s into %s\n",
 673         __func__, path, result.c_str());
 674   return result;
 675 }
 676
 677 namespace {
 678 /*
 679  * Convert the permission bits from the given stat struct to an ls-style
 680  * rwxrwxrwx format.
 681  */
 682 std::string showPermissions(const struct stat& s) {
 683   static const std::pair<int, char> bits[] = {
 684     {S_IRUSR, 'r'}, {S_IWUSR, 'w'}, {S_IXUSR, 'x'},
 685     {S_IRGRP, 'r'}, {S_IWGRP, 'w'}, {S_IXGRP, 'x'},
 686     {S_IROTH, 'r'}, {S_IWOTH, 'w'}, {S_IXOTH, 'x'},
 687   };
 688   std::string ret;
 689   ret.reserve(sizeof(bits) / sizeof(bits[0]));
 690
 691   for (auto pair : bits) {
 692     ret += (s.st_mode & pair.first) ? pair.second : '-';
 693   }
 694   return ret;
 695 }
 696
 697 struct PasswdBuffer {
 698   explicit PasswdBuffer(int name)
 699     : size{sysconf(name)}
 700   {
 701     if (size == -1) size = 1024;
 702     data = std::make_unique<char[]>(size);
 703   }
 704
 705   long size;
 706   std::unique_ptr<char[]> data;
 707 };
 708
 709 /*
 710  * Return the name of the user with the given id.
 711  */
 712 std::string uidToName(uid_t uid) {
 713 #ifndef _WIN32
 714   auto buffer = PasswdBuffer{_SC_GETPW_R_SIZE_MAX};
 715   passwd pw;
 716   passwd* result;
 717
 718   auto err = getpwuid_r(uid, &pw, buffer.data.get(), buffer.size, &result);
 719   if (err != 0) return folly::errnoStr(errno).toStdString();
 720   if (result == nullptr) return "user does not exist";
 721   return pw.pw_name;
 722 #else
 723   return "<unsupported>";
 724 #endif
 725 }
 726
 727 /*
 728  * Return the uid of the user with the given name.
 729  */
 730 uid_t nameToUid(const std::string& name) {
 731 #ifndef _WIN32
 732   auto buffer = PasswdBuffer{_SC_GETPW_R_SIZE_MAX};
 733   passwd pw;
 734   passwd* result;
 735
 736   auto err = getpwnam_r(
 737     name.c_str(), &pw, buffer.data.get(), buffer.size, &result
 738   );
 739   if (err != 0 || result == nullptr) return -1;
 740   return pw.pw_uid;
 741 #else
 742   return -1;
 743 #endif
 744 }
 745
 746 /*
 747  * Return the name of the group with the given id.
 748  */
 749 std::string gidToName(gid_t gid) {
 750 #ifndef _WIN32
 751   auto buffer = PasswdBuffer{_SC_GETGR_R_SIZE_MAX};
 752   group grp;
 753   group* result;
 754
 755   auto err = getgrgid_r(gid, &grp, buffer.data.get(), buffer.size, &result);
 756   if (err != 0) return folly::errnoStr(errno).toStdString();
 757   if (result == nullptr) return "group does not exist";
 758   return grp.gr_name;
 759 #else
 760   return "<unsupported>";
 761 #endif
 762 }
 763
 764 /*
 765  * Return the gid of the group with the given name.
 766  */
 767 gid_t nameToGid(const std::string& name) {
 768 #ifndef _WIN32
 769   auto buffer = PasswdBuffer{_SC_GETGR_R_SIZE_MAX};
 770   group grp;
 771   group* result;
 772
 773   auto err = getgrnam_r(
 774     name.c_str(), &grp, buffer.data.get(), buffer.size, &result
 775   );
 776   if (err != 0 || result == nullptr) return -1;
 777   return grp.gr_gid;
 778 #else
 779   return -1;
 780 #endif
 781 }
 782
 783 void setCentralRepoFileMode(const std::string& path) {
 784   // These runtime options are all best-effort, so we don't care if any of the
 785   // operations fail.
 786
 787   if (auto const mode = RuntimeOption::RepoCentralFileMode) {
 788     chmod(path.c_str(), mode);
 789   }
 790
 791   if (!RuntimeOption::RepoCentralFileUser.empty()) {
 792     auto const uid = nameToUid(RuntimeOption::RepoCentralFileUser);
 793     chown(path.c_str(), uid, -1);
 794   }
 795
 796   if (!RuntimeOption::RepoCentralFileGroup.empty()) {
 797     auto const gid = nameToGid(RuntimeOption::RepoCentralFileGroup);
 798     chown(path.c_str(), -1, gid);
 799   }
 800 }
 801 }
 802
 803 RepoStatus Repo::openCentral(const char* rawPath, std::string& errorMsg) {
 804   std::string repoPath = insertSchema(rawPath);
 805   // SQLITE_OPEN_NOMUTEX specifies that the connection be opened such
 806   // that no mutexes are used to protect the database connection from other
 807   // threads.  However, multiple connections can still be used concurrently,
 808   // because SQLite as a whole is thread-safe.
 809   if (int err = sqlite3_open_v2(repoPath.c_str(), &m_dbc,
 810                                 SQLITE_OPEN_NOMUTEX |
 811                                 SQLITE_OPEN_READWRITE |
 812                                 SQLITE_OPEN_CREATE, nullptr)) {
 813     TRACE(1, "Repo::%s() failed to open candidate central repo '%s'\n",
 814              __func__, repoPath.c_str());
 815     errorMsg = folly::format("Failed to open {}: {} - {}",
 816                              repoPath, err, sqlite3_errmsg(m_dbc)).str();
 817     return RepoStatus::error;
 818   }
 819
 820   // Register a busy handler to avoid spurious SQLITE_BUSY errors.
 821   sqlite3_busy_handler(m_dbc, busyHandler, (void*)this);
 822   try {
 823     m_beginStmt.prepare("BEGIN TRANSACTION;");
 824     m_rollbackStmt.prepare("ROLLBACK;");
 825     m_commitStmt.prepare("COMMIT;");
 826     pragmas(RepoIdCentral);
 827   } catch (RepoExc& re) {
 828     TRACE(1, "Repo::%s() failed to initialize connection to canditate repo"
 829              " '%s': %s\n", __func__, repoPath.c_str(), re.what());
 830     errorMsg = folly::format("Failed to initialize connection to {}: {}",
 831                              repoPath, re.what()).str();
 832     return RepoStatus::error;
 833   }
 834
 835   // sqlite3_open_v2() will silently open in read-only mode if file permissions
 836   // prevent writing.  Therefore, tell initSchema() to verify that the database
 837   // is writable.
 838   bool centralWritable = true;
 839   if (initSchema(RepoIdCentral, centralWritable, errorMsg) == RepoStatus::error
 840       || !centralWritable) {
 841     TRACE(1, "Repo::initSchema() failed for candidate central repo '%s'\n",
 842              repoPath.c_str());
 843     struct stat repoStat;
 844     std::string statStr;
 845     if (stat(repoPath.c_str(), &repoStat) == 0) {
 846       statStr = folly::sformat("{} {}:{}",
 847                                showPermissions(repoStat),
 848                                uidToName(repoStat.st_uid),
 849                                gidToName(repoStat.st_gid));
 850     } else {
 851       statStr = folly::errnoStr(errno).toStdString();
 852     }
 853     errorMsg = folly::format("Failed to initialize schema in {}({}): {}",
 854                              repoPath, statStr, errorMsg).str();
 855     return RepoStatus::error;
 856   }
 857   m_centralRepo = repoPath;
 858   setCentralRepoFileMode(repoPath);
 859   TRACE(1, "Central repo: '%s'\n", m_centralRepo.c_str());
 860   return RepoStatus::success;
 861 }
 862
 863 void Repo::initLocal() {
 864   if (RuntimeOption::RepoLocalMode.compare("--")) {
 865     bool isWritable;
 866     if (!RuntimeOption::RepoLocalMode.compare("rw")) {
 867       isWritable = true;
 868     } else {
 869       assert(!RuntimeOption::RepoLocalMode.compare("r-"));
 870       isWritable = false;
 871     }
 872
 873     if (!RuntimeOption::RepoLocalPath.empty()) {
 874       attachLocal(RuntimeOption::RepoLocalPath.c_str(), isWritable);
 875     } else if (RuntimeOption::RepoAllowFallbackPath) {
 876       if (!RuntimeOption::ServerExecutionMode()) {
 877         std::string cliRepo = s_cliFile;
 878         if (!cliRepo.empty()) {
 879           cliRepo += ".hhbc";
 880         }
 881         attachLocal(cliRepo.c_str(), isWritable);
 882       } else {
 883         attachLocal("hhvm.hhbc", isWritable);
 884       }
 885     }
 886   }
 887 }
 888
 889 void Repo::attachLocal(const char* path, bool isWritable) {
 890   std::string repoPath = insertSchema(path);
 891   if (!isWritable) {
 892     // Make sure the repo exists before attaching it, in order to avoid
 893     // creating a read-only repo.
 894     struct stat buf;
 895     if (!strchr(repoPath.c_str(), ':') &&
 896         stat(repoPath.c_str(), &buf) != 0) {
 897       return;
 898     }
 899   }
 900   try {
 901     std::stringstream ssAttach;
 902     ssAttach << "ATTACH DATABASE '" << repoPath << "' as "
 903              << dbName(RepoIdLocal) << ";";
 904     exec(ssAttach.str());
 905     pragmas(RepoIdLocal);
 906   } catch (RepoExc& re) {
 907     // Failed to run pragmas on local DB - ignored
 908     return;
 909   }
 910
 911   std::string error;
 912   if (initSchema(RepoIdLocal, isWritable, error) == RepoStatus::error) {
 913     FTRACE(1, "Local repo {} failed to init schema: {}\n", repoPath, error);
 914     return;
 915   }
 916   m_localRepo = repoPath;
 917   m_localReadable = true;
 918   m_localWritable = isWritable;
 919   TRACE(1, "Local repo: '%s' (read%s)\n",
 920            m_localRepo.c_str(), m_localWritable ? "-write" : "-only");
 921 }
 922
 923 void Repo::pragmas(int repoId) {
 924   // Valid synchronous values: 0 (OFF), 1 (NORMAL), 2 (FULL).
 925   static const int synchronous = 0;
 926   setIntPragma(repoId, "synchronous", synchronous);
 927   setIntPragma(repoId, "cache_size", 20);
 928   // Valid journal_mode values: delete, truncate, persist, memory, wal, off.
 929   setTextPragma(repoId, "journal_mode", RuntimeOption::RepoJournal.c_str());
 930 }
 931
 932 void Repo::getIntPragma(int repoId, const char* name, int& val) {
 933   std::stringstream ssPragma;
 934   ssPragma << "PRAGMA " << dbName(repoId) << "." << name << ";";
 935   RepoStmt stmt(*this);
 936   stmt.prepare(ssPragma.str());
 937   RepoQuery query(stmt);
 938   query.step();
 939   query.getInt(0, val);
 940 }
 941
 942 void Repo::setIntPragma(int repoId, const char* name, int val) {
 943   // Pragma writes must be executed outside transactions, since they may change
 944   // transaction behavior.
 945   std::stringstream ssPragma;
 946   ssPragma << "PRAGMA " << dbName(repoId) << "." << name << " = " << val << ";";
 947   exec(ssPragma.str());
 948   if (debug) {
 949     // Verify that the pragma had the desired effect.
 950     int newval = -1;
 951     getIntPragma(repoId, name, newval);
 952     if (newval != val) {
 953       throw RepoExc("Unexpected PRAGMA %s.%s value: %d\n",
 954                     dbName(repoId), name, newval);
 955     }
 956   }
 957 }
 958
 959 void Repo::getTextPragma(int repoId, const char* name, std::string& val) {
 960   std::stringstream ssPragma;
 961   ssPragma << "PRAGMA " << dbName(repoId) << "." << name << ";";
 962   RepoStmt stmt(*this);
 963   stmt.prepare(ssPragma.str());
 964   RepoQuery query(stmt);
 965   const char* s;
 966   query.step();
 967   query.getText(0, s);
 968   val = s;
 969 }
 970
 971 void Repo::setTextPragma(int repoId, const char* name, const char* val) {
 972   // Pragma writes must be executed outside transactions, since they may change
 973   // transaction behavior.
 974   std::stringstream ssPragma;
 975   ssPragma <<
 976     "PRAGMA " << dbName(repoId) << "." << name << " = '" << val << "';";
 977   exec(ssPragma.str());
 978   if (debug) {
 979     // Verify that the pragma had the desired effect.
 980     std::string newval = "?";
 981     getTextPragma(repoId, name, newval);
 982     if (strcmp(newval.c_str(), val)) {
 983       throw RepoExc("Unexpected PRAGMA %s.%s value: %s\n",
 984                     dbName(repoId), name, newval.c_str());
 985     }
 986   }
 987 }
 988
 989 RepoStatus Repo::initSchema(int repoId, bool& isWritable,
 990                             std::string& errorMsg) {
 991   if (!schemaExists(repoId)) {
 992     if (createSchema(repoId, errorMsg) == RepoStatus::error) {
 993       // Check whether this failure is due to losing the schema
 994       // initialization race with another process.
 995       if (!schemaExists(repoId)) {
 996         return RepoStatus::error;
 997       }
 998     } else {
 999       // createSchema() successfully wrote to the database, so no further
1000       // verification is necessary.
1001       return RepoStatus::success;
1002     }
1003   }
1004   if (isWritable) {
1005     isWritable = writable(repoId);
1006   }
1007   return RepoStatus::success;
1008 }
1009
1010 bool Repo::schemaExists(int repoId) {
1011   try {
1012     RepoTxn txn(*this);
1013     std::stringstream ssSelect;
1014     ssSelect << "SELECT product FROM " << table(repoId, "magic") << ";";
1015     RepoStmt stmt(*this);
1016     // If the DB is 'new' and hasn't been initialized yet then we expect this
1017     // prepare() to fail.
1018     stmt.prepare(ssSelect.str());
1019     // This SHOULDN'T fail - we create the table under a transaction - so if it
1020     // exists then it should have our magic value.
1021     RepoTxnQuery query(txn, stmt);
1022     query.step();
1023     const char* text; /**/ query.getText(0, text);
1024     if (strcmp(kMagicProduct, text)) {
1025       return false;
1026     }
1027     txn.commit();
1028   } catch (RepoExc& re) {
1029     return false;
1030   }
1031   return true;
1032 }
1033
1034 RepoStatus Repo::createSchema(int repoId, std::string& errorMsg) {
1035   try {
1036     RepoTxn txn(*this);
1037     {
1038       std::stringstream ssCreate;
1039       ssCreate << "CREATE TABLE " << table(repoId, "magic")
1040                << "(product TEXT);";
1041       txn.exec(ssCreate.str());
1042
1043       std::stringstream ssInsert;
1044       ssInsert << "INSERT INTO " << table(repoId, "magic")
1045                << " VALUES('" << kMagicProduct << "');";
1046       txn.exec(ssInsert.str());
1047     }
1048     {
1049       std::stringstream ssCreate;
1050       ssCreate << "CREATE TABLE " << table(repoId, "FileMd5")
1051                << "(path TEXT, md5 BLOB, UNIQUE(path, md5));";
1052       txn.exec(ssCreate.str());
1053     }
1054     txn.exec(folly::format("CREATE TABLE {} (data BLOB);",
1055                            table(repoId, "GlobalData")).str());
1056     m_urp.createSchema(repoId, txn);
1057     m_pcrp.createSchema(repoId, txn);
1058     m_frp.createSchema(repoId, txn);
1059     m_lsrp.createSchema(repoId, txn);
1060
1061     txn.commit();
1062   } catch (RepoExc& re) {
1063     errorMsg = re.what();
1064     return RepoStatus::error;
1065   }
1066   return RepoStatus::success;
1067 }
1068
1069 bool Repo::writable(int repoId) {
1070   switch (sqlite3_db_readonly(m_dbc, dbName(repoId))) {
1071     case 0:  return true;
1072     case 1:  return false;
1073     case -1: return false;
1074     default: break;
1075   }
1076   always_assert(false);
1077 }
1078
1079 //////////////////////////////////////////////////////////////////////
1080
1081 void batchCommit(const std::vector<std::unique_ptr<UnitEmitter>>& ues) {
1082   auto& repo = Repo::get();
1083
1084   // Attempt batch commit.  This can legitimately fail due to multiple input
1085   // files having identical contents.
1086   bool err = false;
1087   {
1088     RepoTxn txn(repo);
1089
1090     for (auto& ue : ues) {
1091       if (repo.insertUnit(ue.get(), UnitOrigin::File, txn) ==
1092           RepoStatus::error) {
1093         err = true;
1094         break;
1095       }
1096     }
1097     if (!err) {
1098       txn.commit();
1099     }
1100   }
1101
1102   // Commit units individually if an error occurred during batch commit.
1103   if (err) {
1104     for (auto& ue : ues) {
1105       repo.commitUnit(ue.get(), UnitOrigin::File);
1106     }
1107   }
1108 }
1109
1110 //////////////////////////////////////////////////////////////////////
1111
1112 }