hphp/runtime/vm/repo.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #include "hphp/runtime/vm/repo.h"
  17
  18 #include <sstream>
  19
  20 #include <folly/Format.h>
  21 #include <folly/Singleton.h>
  22
  23 #include "hphp/hhbbc/options.h"
  24 #include "hphp/runtime/vm/blob-helper.h"
  25 #include "hphp/runtime/vm/repo-global-data.h"
  26 #include "hphp/runtime/server/xbox-server.h"
  27
  28 #include "hphp/util/assertions.h"
  29 #include "hphp/util/async-func.h"
  30 #include "hphp/util/build-info.h"
  31 #include "hphp/util/hugetlb.h"
  32 #include "hphp/util/logger.h"
  33 #include "hphp/util/process.h"
  34 #include "hphp/util/trace.h"
  35
  36 #include <grp.h>
  37 #include <pwd.h>
  38 #include <sys/stat.h>
  39
  40 namespace HPHP {
  41
  42 TRACE_SET_MOD(hhbc);
  43
  44 const char* Repo::kMagicProduct =
  45   "facebook.com HipHop Virtual Machine bytecode repository";
  46 const char* Repo::kDbs[RepoIdCount] = { "main",   // Central.
  47                                         "local"}; // Local.
  48 Repo::GlobalData Repo::s_globalData;
  49
  50 void initialize_repo() {
  51   if (!sqlite3_threadsafe()) {
  52     TRACE(0, "SQLite was compiled without thread support; aborting\n");
  53     abort();
  54   }
  55   if (sqlite3_config(SQLITE_CONFIG_MULTITHREAD) != SQLITE_OK) {
  56     TRACE(1, "Failed to set default SQLite multi-threading mode\n");
  57   }
  58   if (sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0) != SQLITE_OK) {
  59     TRACE(1, "Failed to disable SQLite memory statistics\n");
  60   }
  61 }
  62
  63 THREAD_LOCAL(Repo, t_dh);
  64
  65 Repo& Repo::get() {
  66   return *t_dh.get();
  67 }
  68
  69 void Repo::shutdown() {
  70   t_dh.destroy();
  71 }
  72
  73 static SimpleMutex s_lock;
  74 static std::atomic<unsigned> s_nRepos;
  75
  76 bool Repo::prefork() {
  77   if (num_1g_pages() > 0) {
  78     // We put data on the 1G huge pages, and we don't want to do COW upon
  79     // fork().  If you need to fork(), configure HHVM not to use 1G pages.
  80     return true;
  81   }
  82   if (!t_dh.isNull()) {
  83     t_dh.destroy();
  84   }
  85   s_lock.lock();
  86   XboxServer::Stop();
  87   if (s_nRepos > 0 || AsyncFuncImpl::count()) {
  88     XboxServer::Restart();
  89     s_lock.unlock();
  90     return true;
  91   }
  92   folly::SingletonVault::singleton()->destroyInstances();
  93   return false;
  94 }
  95
  96 void Repo::postfork(pid_t pid) {
  97   folly::SingletonVault::singleton()->reenableInstances();
  98   XboxServer::Restart();
  99   if (pid == 0) {
 100     Logger::ResetPid();
 101     new (&s_lock) SimpleMutex();
 102   } else {
 103     s_lock.unlock();
 104   }
 105 }
 106
 107 Repo::Repo()
 108   : RepoProxy(*this),
 109     m_insertFileHash{InsertFileHashStmt(*this, 0),
 110                      InsertFileHashStmt(*this, 1)},
 111     m_getFileHash{GetFileHashStmt(*this, 0), GetFileHashStmt(*this, 1)},
 112     m_dbc(nullptr), m_localReadable(false), m_localWritable(false),
 113     m_evalRepoId(-1), m_txDepth(0), m_rollback(false), m_beginStmt(*this),
 114     m_rollbackStmt(*this), m_commitStmt(*this), m_urp(*this), m_pcrp(*this),
 115     m_rrp(*this), m_frp(*this), m_lsrp(*this) {
 116
 117   ++s_nRepos;
 118   connect();
 119 }
 120
 121 Repo::~Repo() noexcept {
 122   disconnect();
 123   --s_nRepos;
 124 }
 125
 126 std::string Repo::s_cliFile;
 127 void Repo::setCliFile(const std::string& cliFile) {
 128   assertx(s_cliFile.empty());
 129   assertx(t_dh.isNull());
 130   s_cliFile = cliFile;
 131 }
 132
 133 size_t Repo::stringLengthLimit() const {
 134   static const size_t limit = sqlite3_limit(m_dbc, SQLITE_LIMIT_LENGTH, -1);
 135   return limit;
 136 }
 137
 138 bool Repo::hasGlobalData() {
 139   for (int repoId = RepoIdCount - 1; repoId >= 0; --repoId) {
 140     if (repoName(repoId).empty()) {
 141       // The repo wasn't loadable
 142       continue;
 143     }
 144
 145     RepoStmt stmt(*this);
 146     const auto& tbl = table(repoId, "GlobalData");
 147     stmt.prepare(
 148       folly::sformat(
 149         "SELECT count(*) FROM {};", tbl
 150       )
 151     );
 152     auto txn = RepoTxn{begin()};
 153     RepoTxnQuery query(txn, stmt);
 154     query.step();
 155
 156     if (!query.row()) {
 157       return false;
 158     }
 159
 160     int val;
 161     query.getInt(0, val);
 162     return val != 0;
 163   }
 164
 165   return false;
 166 }
 167
 168 void Repo::loadGlobalData(bool readArrayTable /* = true */) {
 169   if (readArrayTable) m_lsrp.load();
 170
 171   if (!RuntimeOption::RepoAuthoritative) return;
 172
 173   std::vector<std::string> failures;
 174
 175   /*
 176    * This should probably just go to the Local repo always, except
 177    * that our unit test suite is currently running RepoAuthoritative
 178    * tests with the compiled repo as the Central repo.
 179    */
 180   for (int repoId = RepoIdCount - 1; repoId >= 0; --repoId) {
 181     if (repoName(repoId).empty()) {
 182       // The repo wasn't loadable
 183       continue;
 184     }
 185     try {
 186       RepoStmt stmt(*this);
 187       const auto& tbl = table(repoId, "GlobalData");
 188       stmt.prepare(
 189         folly::sformat(
 190           "SELECT count(*), data FROM {};", tbl
 191         )
 192       );
 193       auto txn = RepoTxn{begin()};
 194       RepoTxnQuery query(txn, stmt);
 195       query.step();
 196       if (!query.row()) {
 197         throw RepoExc("Can't find table %s", tbl.c_str());
 198       };
 199       int val;
 200       query.getInt(0, val);
 201       if (val == 0) {
 202         throw RepoExc("No rows in %s. Did you forget to compile that file with "
 203                       "this HHVM version?", tbl.c_str());
 204       }
 205       BlobDecoder decoder = query.getBlob(1, true);
 206       decoder(s_globalData);
 207       FTRACE(1, "GlobalData loaded from '{}':\n", repoName(repoId));
 208       FTRACE(1, "{}", show(s_globalData));
 209       if (readArrayTable) {
 210         auto& arrayTypeTable = globalArrayTypeTable();
 211         decoder(arrayTypeTable);
 212         decoder(s_globalData.APCProfile);
 213         decoder(s_globalData.ConstantFunctions);
 214         decoder.assertDone();
 215       }
 216       txn.commit();
 217     } catch (RepoExc& e) {
 218       failures.push_back(repoName(repoId) + ": "  + e.msg());
 219       continue;
 220     }
 221
 222     // TODO: this should probably read out the other elements of the global data
 223     // which control Option or RuntimeOption values -- the others are read out
 224     // in an inconsistent and ad-hoc manner. But I don't understand their uses
 225     // and interactions well enough to feel comfortable fixing now.
 226     RuntimeOption::EvalPromoteEmptyObject    = s_globalData.PromoteEmptyObject;
 227     RuntimeOption::EnableIntrinsicsExtension =
 228       s_globalData.EnableIntrinsicsExtension;
 229     HHBBC::options.ElideAutoloadInvokes     = s_globalData.ElideAutoloadInvokes;
 230     RuntimeOption::EnableHipHopSyntax       = s_globalData.EnableHipHopSyntax;
 231     RuntimeOption::EvalUseHHBBC             = s_globalData.UsedHHBBC;
 232     RuntimeOption::PHP7_Builtins            = s_globalData.PHP7_Builtins;
 233     RuntimeOption::PHP7_IntSemantics        = s_globalData.PHP7_IntSemantics;
 234     RuntimeOption::PHP7_NoHexNumerics       = s_globalData.PHP7_NoHexNumerics;
 235     RuntimeOption::PHP7_ScalarTypes         = s_globalData.PHP7_ScalarTypes;
 236     RuntimeOption::PHP7_Substr              = s_globalData.PHP7_Substr;
 237     RuntimeOption::EvalReffinessInvariance  = s_globalData.ReffinessInvariance;
 238     RuntimeOption::EvalCheckPropTypeHints   = s_globalData.CheckPropTypeHints;
 239     RuntimeOption::EvalHackArrDVArrs        = s_globalData.HackArrDVArrs;
 240     RuntimeOption::EnableArgsInBacktraces = s_globalData.EnableArgsInBacktraces;
 241     RuntimeOption::EvalAbortBuildOnVerifyError =
 242       s_globalData.AbortBuildOnVerifyError;
 243     if (s_globalData.HardReturnTypeHints) {
 244       RuntimeOption::EvalCheckReturnTypeHints = 3;
 245     }
 246     if (s_globalData.ThisTypeHintLevel == 3) {
 247       RuntimeOption::EvalThisTypeHintLevel = s_globalData.ThisTypeHintLevel;
 248     }
 249     RuntimeOption::ConstantFunctions.clear();
 250     for (auto const& elm : s_globalData.ConstantFunctions) {
 251       RuntimeOption::ConstantFunctions.insert(elm);
 252     }
 253
 254     return;
 255   }
 256
 257   if (failures.empty()) {
 258     std::fprintf(stderr, "No repo was loadable. Check all the possible repo "
 259                  "locations (Repo.Central.Path, HHVM_REPO_CENTRAL_PATH, and "
 260                  "$HOME/.hhvm.hhbc) to make sure one of them is a valid "
 261                  "sqlite3 HHVM repo built with this exact HHVM version.\n");
 262   } else {
 263     // We should always have a global data section in RepoAuthoritative
 264     // mode, or the repo is messed up.
 265     std::fprintf(stderr, "Failed to load Repo::GlobalData:\n");
 266     for (auto& f : failures) {
 267       std::fprintf(stderr, "  %s\n", f.c_str());
 268     }
 269   }
 270
 271   assertx(Process::IsInMainThread());
 272   exit(1);
 273 }
 274
 275 void Repo::saveGlobalData(GlobalData newData) {
 276   s_globalData = newData;
 277
 278   auto const repoId = repoIdForNewUnit(UnitOrigin::File);
 279   RepoStmt stmt(*this);
 280   stmt.prepare(
 281     folly::format(
 282       "INSERT INTO {} VALUES(@data);", table(repoId, "GlobalData")
 283     ).str()
 284   );
 285   auto txn = RepoTxn{begin()};
 286   RepoTxnQuery query(txn, stmt);
 287   BlobEncoder encoder{true};
 288   encoder(s_globalData);
 289   encoder(globalArrayTypeTable());
 290   encoder(s_globalData.APCProfile);
 291   encoder(s_globalData.ConstantFunctions);
 292   query.bindBlob("@data", encoder, /* static */ true);
 293   query.exec();
 294
 295   // TODO(#3521039): we could just put the litstr table in the same
 296   // blob as the above and delete LitstrRepoProxy.
 297   LitstrTable::get().forEachLitstr(
 298     [this, &txn, repoId](int i, const StringData* name) {
 299       lsrp().insertLitstr(repoId).insert(txn, i, name);
 300     });
 301
 302   txn.commit();
 303 }
 304
 305 std::unique_ptr<Unit> Repo::loadUnit(const std::string& name, const SHA1& sha1,
 306                                      const Native::FuncTable& nativeFuncs) {
 307   if (m_dbc == nullptr) {
 308     return nullptr;
 309   }
 310   return m_urp.load(name, sha1, nativeFuncs);
 311 }
 312
 313 std::vector<std::pair<std::string,SHA1>>
 314 Repo::enumerateUnits(int repoId, bool warn) {
 315   std::vector<std::pair<std::string,SHA1>> ret;
 316
 317   try {
 318     RepoStmt stmt(*this);
 319     stmt.prepare(folly::sformat(
 320                    "SELECT path, sha1 FROM {};",
 321                    table(repoId, "FileSha1"))
 322                 );
 323     auto txn = RepoTxn{begin()};
 324     RepoTxnQuery query(txn, stmt);
 325
 326     for (query.step(); query.row(); query.step()) {
 327       std::string path;
 328       SHA1 sha1;
 329
 330       query.getStdString(0, path);
 331       query.getSha1(1, sha1);
 332
 333       ret.emplace_back(path, sha1);
 334     }
 335
 336     txn.commit();
 337   } catch (RepoExc& e) {
 338     if (warn) {
 339       fprintf(stderr, "failed to enumerate units: %s\n", e.what());
 340     }
 341     // Ugh - the error is dropped.  At least we might have printed an error to
 342     // stderr.
 343   }
 344
 345   return ret;
 346 }
 347
 348 void Repo::InsertFileHashStmt::insert(RepoTxn& txn, const StringData* path,
 349                                       const SHA1& sha1) {
 350   if (!prepared()) {
 351     auto insertQuery = folly::sformat(
 352       "INSERT INTO {} VALUES(@path, @sha1);",
 353       m_repo.table(m_repoId, "FileSha1"));
 354     txn.prepare(*this, insertQuery);
 355   }
 356   RepoTxnQuery query(txn, *this);
 357   query.bindStaticString("@path", path);
 358   query.bindSha1("@sha1", sha1);
 359   query.exec();
 360 }
 361
 362 RepoStatus Repo::GetFileHashStmt::get(const char *path, SHA1& sha1) {
 363   try {
 364     auto txn = RepoTxn{m_repo.begin()};
 365     if (!prepared()) {
 366       auto selectQuery = folly::sformat(
 367         "SELECT f.sha1 "
 368         "FROM {} AS f, {} AS u "
 369         "WHERE path == @path AND f.sha1 == u.sha1 "
 370         "ORDER BY unitSn DESC LIMIT 1;",
 371         m_repo.table(m_repoId, "FileSha1"),
 372         m_repo.table(m_repoId, "Unit"));
 373       txn.prepare(*this, selectQuery);
 374     }
 375     RepoTxnQuery query(txn, *this);
 376     query.bindText("@path", path, strlen(path));
 377     query.step();
 378     if (!query.row()) {
 379       return RepoStatus::error;
 380     }
 381     query.getSha1(0, sha1);
 382     txn.commit();
 383     return RepoStatus::success;
 384   } catch (RepoExc& re) {
 385     return RepoStatus::error;
 386   }
 387 }
 388
 389 RepoStatus Repo::findFile(const char *path, const std::string &root,
 390                           SHA1& sha1) {
 391   if (m_dbc == nullptr) {
 392     return RepoStatus::error;
 393   }
 394   int repoId;
 395   for (repoId = RepoIdCount - 1; repoId >= 0; --repoId) {
 396     if (*path == '/' && !root.empty() &&
 397         !strncmp(root.c_str(), path, root.size()) &&
 398         (m_getFileHash[repoId].get(path + root.size(), sha1) ==
 399          RepoStatus::success)) {
 400       TRACE(3, "Repo loaded file hash for '%s' from '%s'\n",
 401                path + root.size(), repoName(repoId).c_str());
 402       return RepoStatus::success;
 403     }
 404     if (m_getFileHash[repoId].get(path, sha1) == RepoStatus::success) {
 405       TRACE(3, "Repo loaded file hash for '%s' from '%s'\n",
 406                 path, repoName(repoId).c_str());
 407       return RepoStatus::success;
 408     }
 409   }
 410   TRACE(3, "Repo file hash: error loading '%s'\n", path);
 411   return RepoStatus::error;
 412 }
 413
 414 RepoStatus Repo::insertSha1(UnitOrigin unitOrigin, UnitEmitter* ue,
 415                            RepoTxn& txn) {
 416   const StringData* path = ue->m_filepath;
 417   const SHA1& sha1 = ue->sha1();
 418   int repoId = repoIdForNewUnit(unitOrigin);
 419   if (repoId == RepoIdInvalid) {
 420     return RepoStatus::error;
 421   }
 422   try {
 423     m_insertFileHash[repoId].insert(txn, path, sha1);
 424     return RepoStatus::success;
 425   } catch (RepoExc& re) {
 426     TRACE(3, "Failed to commit sha1 for '%s' to '%s': %s\n",
 427               path->data(), repoName(repoId).c_str(), re.msg().c_str());
 428     return RepoStatus::error;
 429   }
 430 }
 431
 432 void Repo::commitSha1(UnitOrigin unitOrigin, UnitEmitter* ue) {
 433   try {
 434     auto txn = RepoTxn{begin()};
 435     RepoStatus err = insertSha1(unitOrigin, ue, txn);
 436     if (err == RepoStatus::success) {
 437       txn.commit();
 438     }
 439   } catch (RepoExc& re) {
 440     int repoId = repoIdForNewUnit(unitOrigin);
 441     if (repoId != RepoIdInvalid) {
 442       TRACE(3, "Failed to commit sha1 for '%s' to '%s': %s\n",
 443                ue->m_filepath->data(), repoName(repoId).c_str(),
 444                re.msg().c_str());
 445     }
 446   }
 447 }
 448
 449 std::string Repo::table(int repoId, const char* tablePrefix) {
 450   return folly::sformat(
 451     "{}.{}_{}", dbName(repoId), tablePrefix, repoSchemaId());
 452 }
 453
 454 void Repo::exec(const std::string& sQuery) {
 455   RepoStmt stmt(*this);
 456   stmt.prepare(sQuery);
 457   RepoQuery query(stmt);
 458   query.exec();
 459 }
 460
 461 RepoTxn Repo::begin() {
 462   if (m_txDepth > 0) {
 463     m_txDepth++;
 464     return RepoTxn{*this};
 465   }
 466   if (debug) {
 467     // Verify start state.
 468     always_assert(m_txDepth == 0);
 469     always_assert(!m_rollback);
 470     if (true) {
 471       // Bypass RepoQuery, in order to avoid triggering exceptions.
 472       int rc = sqlite3_step(m_rollbackStmt.get());
 473       switch (rc) {
 474       case SQLITE_DONE:
 475       case SQLITE_ROW:
 476         not_reached();
 477       default:
 478         break;
 479       }
 480     } else {
 481       bool rollbackFailed = false;
 482       try {
 483         RepoQuery query(m_rollbackStmt);
 484         query.exec();
 485       } catch (RepoExc& re) {
 486         rollbackFailed = true;
 487       }
 488       always_assert(rollbackFailed);
 489     }
 490   }
 491   RepoQuery query(m_beginStmt);
 492   query.exec();
 493   m_txDepth++;
 494
 495   return RepoTxn(*this);
 496 }
 497
 498 void Repo::txPop() {
 499   // We mix the concept of rollback with a normal commit so that if we try to
 500   // rollback an inner transaction we eventually end up rolling back the outer
 501   // transaction instead (Sqlite doesn't support rolling back partial
 502   // transactions).
 503   assertx(m_txDepth > 0);
 504   if (m_txDepth > 1) {
 505     m_txDepth--;
 506     return;
 507   }
 508   if (!m_rollback) {
 509     RepoQuery query(m_commitStmt);
 510     query.exec();
 511   } else {
 512     // We're in the outermost transaction - so clear the rollback flag.
 513     m_rollback = false;
 514     RepoQuery query(m_rollbackStmt);
 515     try {
 516       query.exec();
 517     } catch (RepoExc& ex) {
 518       /*
 519        * Having a rollback fail is actually a normal, expected case,
 520        * so just swallow this.
 521        *
 522        * In particular, according to the docs, if we got an I/O error
 523        * while doing a commit, the rollback will often fail with "no
 524        * transaction in progress", because the commit will have
 525        * automatically been rolled back.  Recommended practice is
 526        * still to execute a rollback statement and ignore the error.
 527        */
 528       TRACE(4, "repo rollback failure: %s\n", ex.what());
 529     }
 530   }
 531   // Decrement depth after query execution, in case an exception occurs during
 532   // commit.  This allows for subsequent rollback of the failed commit.
 533   m_txDepth--;
 534 }
 535
 536 void Repo::rollback() {
 537   m_rollback = true;
 538   // NOTE: A try/catch isn't necessary - txPop() handles rollback as a nothrow.
 539   txPop();
 540 }
 541
 542 void Repo::commit() {
 543   txPop();
 544 }
 545
 546 RepoStatus Repo::insertUnit(UnitEmitter* ue, UnitOrigin unitOrigin,
 547                             RepoTxn& txn) {
 548   if (insertSha1(unitOrigin, ue, txn) == RepoStatus::error ||
 549       ue->insert(unitOrigin, txn) == RepoStatus::error) {
 550     return RepoStatus::error;
 551   }
 552   return RepoStatus::success;
 553 }
 554
 555 void Repo::commitUnit(UnitEmitter* ue, UnitOrigin unitOrigin) {
 556   if (!RuntimeOption::RepoCommit || ue->m_ICE) return;
 557
 558   try {
 559     commitSha1(unitOrigin, ue);
 560     ue->commit(unitOrigin);
 561   } catch (const std::exception& e) {
 562     TRACE(0, "unexpected exception in commitUnit: %s\n",
 563           e.what());
 564     assertx(false);
 565   }
 566 }
 567
 568 void Repo::connect() {
 569   initCentral();
 570   initLocal();
 571   if (!RuntimeOption::RepoEvalMode.compare("local")) {
 572     m_evalRepoId = (m_localWritable) ? RepoIdLocal : RepoIdCentral;
 573   } else if (!RuntimeOption::RepoEvalMode.compare("central")) {
 574     m_evalRepoId = RepoIdCentral;
 575   } else {
 576     assertx(!RuntimeOption::RepoEvalMode.compare("readonly"));
 577     m_evalRepoId = RepoIdInvalid;
 578   }
 579   TRACE(1, "Repo.Eval.Mode=%s\n",
 580            (m_evalRepoId == RepoIdLocal)
 581            ? "local"
 582            : (m_evalRepoId == RepoIdCentral)
 583              ? "central"
 584              : "readonly");
 585 }
 586
 587 void Repo::disconnect() noexcept {
 588   if (m_dbc != nullptr) {
 589     sqlite3_close(m_dbc);
 590     m_dbc = nullptr;
 591     m_localReadable = false;
 592     m_localWritable = false;
 593     m_evalRepoId = RepoIdInvalid;
 594   }
 595 }
 596
 597 void Repo::initCentral() {
 598   std::string error;
 599
 600   assertx(m_dbc == nullptr);
 601   auto tryPath = [this, &error](const char* path) {
 602     std::string subErr;
 603     if (openCentral(path, subErr) == RepoStatus::error) {
 604       folly::format(&error, "  {}\n", subErr.empty() ? path : subErr);
 605       return false;
 606     }
 607     return true;
 608   };
 609
 610   auto fail_no_repo = [&error] {
 611     error = "Failed to initialize central HHBC repository:\n" + error;
 612     // Database initialization failed; this is an unrecoverable state.
 613     Logger::Error(error);
 614
 615     if (Process::IsInMainThread()) {
 616       exit(1);
 617     }
 618     always_assert_flog(false, "{}", error);
 619   };
 620
 621   // Try Repo.Central.Path
 622   if (!RuntimeOption::RepoCentralPath.empty() &&
 623       tryPath(RuntimeOption::RepoCentralPath.c_str())) {
 624     return;
 625   }
 626
 627   // Try HHVM_REPO_CENTRAL_PATH
 628   const char* HHVM_REPO_CENTRAL_PATH = getenv("HHVM_REPO_CENTRAL_PATH");
 629   if (HHVM_REPO_CENTRAL_PATH != nullptr &&
 630       tryPath(HHVM_REPO_CENTRAL_PATH)) {
 631     return;
 632   }
 633
 634   if (!RuntimeOption::RepoAllowFallbackPath) fail_no_repo();
 635
 636   // Try "$HOME/.hhvm.hhbc".
 637   char* HOME = getenv("HOME");
 638   if (HOME != nullptr) {
 639     std::string centralPath = HOME;
 640     centralPath += "/.hhvm.hhbc";
 641     if (tryPath(centralPath.c_str())) {
 642       return;
 643     }
 644   }
 645
 646 #ifndef _WIN32
 647   // Try the equivalent of "$HOME/.hhvm.hhbc", but look up the home directory
 648   // in the password database.
 649   {
 650     passwd pwbuf;
 651     passwd* pwbufp;
 652     long bufsize = sysconf(_SC_GETPW_R_SIZE_MAX);
 653     if (bufsize != -1) {
 654       auto buf = new char[bufsize];
 655       SCOPE_EXIT { delete[] buf; };
 656       if (!getpwuid_r(getuid(), &pwbuf, buf, size_t(bufsize), &pwbufp)
 657           && (HOME == nullptr || strcmp(HOME, pwbufp->pw_dir))) {
 658         std::string centralPath = pwbufp->pw_dir;
 659         centralPath += "/.hhvm.hhbc";
 660         if (tryPath(centralPath.c_str())) {
 661           return;
 662         }
 663       }
 664     }
 665   }
 666 #else // _WIN32
 667   // Try "$HOMEDRIVE$HOMEPATH/.hhvm.hhbc"
 668   char* HOMEDRIVE = getenv("HOMEDRIVE");
 669   if (HOMEDRIVE != nullptr) {
 670     char* HOMEPATH = getenv("HOMEPATH");
 671     if (HOMEPATH != nullptr) {
 672       std::string centralPath = HOMEDRIVE;
 673       centralPath += HOMEPATH;
 674       centralPath += "\\.hhvm.hhbc";
 675       if (tryPath(centralPath.c_str()))
 676         return;
 677     }
 678   }
 679 #endif
 680
 681   fail_no_repo();
 682 }
 683
 684 namespace {
 685 /*
 686  * Convert the permission bits from the given stat struct to an ls-style
 687  * rwxrwxrwx format.
 688  */
 689 std::string showPermissions(const struct stat& s) {
 690   static const std::pair<int, char> bits[] = {
 691     {S_IRUSR, 'r'}, {S_IWUSR, 'w'}, {S_IXUSR, 'x'},
 692     {S_IRGRP, 'r'}, {S_IWGRP, 'w'}, {S_IXGRP, 'x'},
 693     {S_IROTH, 'r'}, {S_IWOTH, 'w'}, {S_IXOTH, 'x'},
 694   };
 695   std::string ret;
 696   ret.reserve(sizeof(bits) / sizeof(bits[0]));
 697
 698   for (auto pair : bits) {
 699     ret += (s.st_mode & pair.first) ? pair.second : '-';
 700   }
 701   return ret;
 702 }
 703
 704 struct PasswdBuffer {
 705   explicit PasswdBuffer(int name)
 706     : size{sysconf(name)}
 707   {
 708     if (size == -1) size = 1024;
 709     data = std::make_unique<char[]>(size);
 710   }
 711
 712   long size;
 713   std::unique_ptr<char[]> data;
 714 };
 715
 716 /*
 717  * Return the name of the user with the given id.
 718  */
 719 std::string uidToName(uid_t uid) {
 720 #ifndef _WIN32
 721   auto buffer = PasswdBuffer{_SC_GETPW_R_SIZE_MAX};
 722   passwd pw;
 723   passwd* result;
 724
 725   auto err = getpwuid_r(uid, &pw, buffer.data.get(), buffer.size, &result);
 726   if (err != 0) return folly::errnoStr(errno).toStdString();
 727   if (result == nullptr) return "user does not exist";
 728   return pw.pw_name;
 729 #else
 730   return "<unsupported>";
 731 #endif
 732 }
 733
 734 /*
 735  * Return the uid of the user with the given name.
 736  */
 737 uid_t nameToUid(const std::string& name) {
 738 #ifndef _WIN32
 739   auto buffer = PasswdBuffer{_SC_GETPW_R_SIZE_MAX};
 740   passwd pw;
 741   passwd* result;
 742
 743   auto err = getpwnam_r(
 744     name.c_str(), &pw, buffer.data.get(), buffer.size, &result
 745   );
 746   if (err != 0 || result == nullptr) return -1;
 747   return pw.pw_uid;
 748 #else
 749   return -1;
 750 #endif
 751 }
 752
 753 /*
 754  * Return the name of the group with the given id.
 755  */
 756 std::string gidToName(gid_t gid) {
 757 #ifndef _WIN32
 758   auto buffer = PasswdBuffer{_SC_GETGR_R_SIZE_MAX};
 759   group grp;
 760   group* result;
 761
 762   auto err = getgrgid_r(gid, &grp, buffer.data.get(), buffer.size, &result);
 763   if (err != 0) return folly::errnoStr(errno).toStdString();
 764   if (result == nullptr) return "group does not exist";
 765   return grp.gr_name;
 766 #else
 767   return "<unsupported>";
 768 #endif
 769 }
 770
 771 /*
 772  * Return the gid of the group with the given name.
 773  */
 774 gid_t nameToGid(const std::string& name) {
 775 #ifndef _WIN32
 776   auto buffer = PasswdBuffer{_SC_GETGR_R_SIZE_MAX};
 777   group grp;
 778   group* result;
 779
 780   auto err = getgrnam_r(
 781     name.c_str(), &grp, buffer.data.get(), buffer.size, &result
 782   );
 783   if (err != 0 || result == nullptr) return -1;
 784   return grp.gr_gid;
 785 #else
 786   return -1;
 787 #endif
 788 }
 789
 790 void setCentralRepoFileMode(const std::string& path) {
 791   // These runtime options are all best-effort, so we don't care if any of the
 792   // operations fail.
 793
 794   if (auto const mode = RuntimeOption::RepoCentralFileMode) {
 795     chmod(path.c_str(), mode);
 796   }
 797
 798   if (!RuntimeOption::RepoCentralFileUser.empty()) {
 799     auto const uid = nameToUid(RuntimeOption::RepoCentralFileUser);
 800     chown(path.c_str(), uid, -1);
 801   }
 802
 803   if (!RuntimeOption::RepoCentralFileGroup.empty()) {
 804     auto const gid = nameToGid(RuntimeOption::RepoCentralFileGroup);
 805     chown(path.c_str(), -1, gid);
 806   }
 807 }
 808 }
 809
 810 RepoStatus Repo::openCentral(const char* rawPath, std::string& errorMsg) {
 811   std::string repoPath = insertSchema(rawPath);
 812   // SQLITE_OPEN_NOMUTEX specifies that the connection be opened such
 813   // that no mutexes are used to protect the database connection from other
 814   // threads.  However, multiple connections can still be used concurrently,
 815   // because SQLite as a whole is thread-safe.
 816   if (int err = sqlite3_open_v2(repoPath.c_str(), &m_dbc,
 817                                 SQLITE_OPEN_NOMUTEX |
 818                                 SQLITE_OPEN_READWRITE |
 819                                 SQLITE_OPEN_CREATE, nullptr)) {
 820     TRACE(1, "Repo::%s() failed to open candidate central repo '%s'\n",
 821              __func__, repoPath.c_str());
 822     errorMsg = folly::format("Failed to open {}: {} - {}",
 823                              repoPath, err, sqlite3_errmsg(m_dbc)).str();
 824     return RepoStatus::error;
 825   }
 826
 827   if (RuntimeOption::RepoBusyTimeoutMS) {
 828     sqlite3_busy_timeout(m_dbc, RuntimeOption::RepoBusyTimeoutMS);
 829   }
 830   try {
 831     m_beginStmt.prepare("BEGIN TRANSACTION;");
 832     m_rollbackStmt.prepare("ROLLBACK;");
 833     m_commitStmt.prepare("COMMIT;");
 834     pragmas(RepoIdCentral);
 835   } catch (RepoExc& re) {
 836     TRACE(1, "Repo::%s() failed to initialize connection to canditate repo"
 837              " '%s': %s\n", __func__, repoPath.c_str(), re.what());
 838     errorMsg = folly::format("Failed to initialize connection to {}: {}",
 839                              repoPath, re.what()).str();
 840     return RepoStatus::error;
 841   }
 842
 843   // sqlite3_open_v2() will silently open in read-only mode if file permissions
 844   // prevent writing.  Therefore, tell initSchema() to verify that the database
 845   // is writable.
 846   bool centralWritable = true;
 847   if (initSchema(RepoIdCentral, centralWritable, errorMsg) == RepoStatus::error
 848       || !centralWritable) {
 849     TRACE(1, "Repo::initSchema() failed for candidate central repo '%s'\n",
 850              repoPath.c_str());
 851     struct stat repoStat;
 852     std::string statStr;
 853     if (stat(repoPath.c_str(), &repoStat) == 0) {
 854       statStr = folly::sformat("{} {}:{}",
 855                                showPermissions(repoStat),
 856                                uidToName(repoStat.st_uid),
 857                                gidToName(repoStat.st_gid));
 858     } else {
 859       statStr = folly::errnoStr(errno).toStdString();
 860     }
 861     errorMsg = folly::format("Failed to initialize schema in {}({}): {}",
 862                              repoPath, statStr, errorMsg).str();
 863     return RepoStatus::error;
 864   }
 865   m_centralRepo = repoPath;
 866   setCentralRepoFileMode(repoPath);
 867   TRACE(1, "Central repo: '%s'\n", m_centralRepo.c_str());
 868   return RepoStatus::success;
 869 }
 870
 871 void Repo::initLocal() {
 872   if (RuntimeOption::RepoLocalMode.compare("--")) {
 873     bool isWritable;
 874     if (!RuntimeOption::RepoLocalMode.compare("rw")) {
 875       isWritable = true;
 876     } else {
 877       assertx(!RuntimeOption::RepoLocalMode.compare("r-"));
 878       isWritable = false;
 879     }
 880
 881     if (!RuntimeOption::RepoLocalPath.empty()) {
 882       attachLocal(RuntimeOption::RepoLocalPath.c_str(), isWritable);
 883     } else if (RuntimeOption::RepoAllowFallbackPath) {
 884       if (!RuntimeOption::ServerExecutionMode()) {
 885         std::string cliRepo = s_cliFile;
 886         if (!cliRepo.empty()) {
 887           cliRepo += ".hhbc";
 888         }
 889         attachLocal(cliRepo.c_str(), isWritable);
 890       } else {
 891         attachLocal("hhvm.hhbc", isWritable);
 892       }
 893     }
 894   }
 895 }
 896
 897 void Repo::attachLocal(const char* path, bool isWritable) {
 898   std::string repoPath = insertSchema(path);
 899   if (!isWritable) {
 900     // Make sure the repo exists before attaching it, in order to avoid
 901     // creating a read-only repo.
 902     struct stat buf;
 903     if (!strchr(repoPath.c_str(), ':') &&
 904         stat(repoPath.c_str(), &buf) != 0) {
 905       return;
 906     }
 907   }
 908   try {
 909     auto attachQuery = folly::sformat(
 910       "ATTACH DATABASE '{}' as {};", repoPath, dbName(RepoIdLocal));
 911     exec(attachQuery);
 912     pragmas(RepoIdLocal);
 913   } catch (RepoExc& re) {
 914     // Failed to run pragmas on local DB - ignored
 915     return;
 916   }
 917
 918   std::string error;
 919   if (initSchema(RepoIdLocal, isWritable, error) == RepoStatus::error) {
 920     FTRACE(1, "Local repo {} failed to init schema: {}\n", repoPath, error);
 921     return;
 922   }
 923   m_localRepo = repoPath;
 924   m_localReadable = true;
 925   m_localWritable = isWritable;
 926   TRACE(1, "Local repo: '%s' (read%s)\n",
 927            m_localRepo.c_str(), m_localWritable ? "-write" : "-only");
 928 }
 929
 930 void Repo::pragmas(int repoId) {
 931   // Valid synchronous values: 0 (OFF), 1 (NORMAL), 2 (FULL).
 932   static const int synchronous = 0;
 933   setIntPragma(repoId, "synchronous", synchronous);
 934   setIntPragma(repoId, "cache_size", 20);
 935   // Valid journal_mode values: delete, truncate, persist, memory, wal, off.
 936   setTextPragma(repoId, "journal_mode", RuntimeOption::RepoJournal.c_str());
 937 }
 938
 939 void Repo::getIntPragma(int repoId, const char* name, int& val) {
 940   auto pragmaQuery = folly::sformat("PRAGMA {}.{};", dbName(repoId), name);
 941   RepoStmt stmt(*this);
 942   stmt.prepare(pragmaQuery);
 943   RepoQuery query(stmt);
 944   query.step();
 945   query.getInt(0, val);
 946 }
 947
 948 void Repo::setIntPragma(int repoId, const char* name, int val) {
 949   // Read first to see if a write can be avoided
 950   int oldval = -1;
 951   getIntPragma(repoId, name, oldval);
 952   if (val == oldval) return;
 953
 954   // Pragma writes must be executed outside transactions, since they may change
 955   // transaction behavior.
 956   auto pragmaQuery = folly::sformat(
 957     "PRAGMA {}.{} = {};", dbName(repoId), name, val);
 958   exec(pragmaQuery);
 959   if (debug) {
 960     // Verify that the pragma had the desired effect.
 961     int newval = -1;
 962     getIntPragma(repoId, name, newval);
 963     if (newval != val) {
 964       throw RepoExc("Unexpected PRAGMA %s.%s value: %d\n",
 965                     dbName(repoId), name, newval);
 966     }
 967   }
 968 }
 969
 970 void Repo::getTextPragma(int repoId, const char* name, std::string& val) {
 971   auto pragmaQuery = folly::sformat("PRAGMA {}.{};", dbName(repoId), name);
 972   RepoStmt stmt(*this);
 973   stmt.prepare(pragmaQuery);
 974   RepoQuery query(stmt);
 975   const char* s;
 976   query.step();
 977   query.getText(0, s);
 978   val = s;
 979 }
 980
 981 void Repo::setTextPragma(int repoId, const char* name, const char* val) {
 982   // Read first to see if a write can be avoided
 983   std::string oldval = "?";
 984   getTextPragma(repoId, name, oldval);
 985   if (!strcmp(oldval.c_str(), val)) return;
 986
 987   // Pragma writes must be executed outside transactions, since they may change
 988   // transaction behavior.
 989   auto pragmaQuery = folly::sformat(
 990     "PRAGMA {}.{} = {};", dbName(repoId), name, val);
 991   exec(pragmaQuery);
 992   if (debug) {
 993     // Verify that the pragma had the desired effect.
 994     std::string newval = "?";
 995     getTextPragma(repoId, name, newval);
 996     if (strcmp(newval.c_str(), val)) {
 997       throw RepoExc("Unexpected PRAGMA %s.%s value: %s\n",
 998                     dbName(repoId), name, newval.c_str());
 999     }
1000   }
1001 }
1002
1003 RepoStatus Repo::initSchema(int repoId, bool& isWritable,
1004                             std::string& errorMsg) {
1005   if (!schemaExists(repoId)) {
1006     if (createSchema(repoId, errorMsg) == RepoStatus::error) {
1007       // Check whether this failure is due to losing the schema
1008       // initialization race with another process.
1009       if (!schemaExists(repoId)) {
1010         return RepoStatus::error;
1011       }
1012     } else {
1013       // createSchema() successfully wrote to the database, so no further
1014       // verification is necessary.
1015       return RepoStatus::success;
1016     }
1017   }
1018   if (isWritable) {
1019     isWritable = writable(repoId);
1020   }
1021   return RepoStatus::success;
1022 }
1023
1024 bool Repo::schemaExists(int repoId) {
1025   try {
1026     auto txn = RepoTxn{begin()};
1027     auto selectQuery = folly::sformat(
1028       "SELECT product FROM {};", table(repoId, "magic"));
1029     RepoStmt stmt(*this);
1030     // If the DB is 'new' and hasn't been initialized yet then we expect this
1031     // prepare() to fail.
1032     stmt.prepare(selectQuery);
1033     // This SHOULDN'T fail - we create the table under a transaction - so if it
1034     // exists then it should have our magic value.
1035     RepoTxnQuery query(txn, stmt);
1036     query.step();
1037     const char* text; /**/ query.getText(0, text);
1038     if (strcmp(kMagicProduct, text)) {
1039       return false;
1040     }
1041     txn.commit();
1042   } catch (RepoExc& re) {
1043     return false;
1044   }
1045   return true;
1046 }
1047
1048 RepoStatus Repo::createSchema(int repoId, std::string& errorMsg) {
1049   try {
1050     auto txn = RepoTxn{begin()};
1051     {
1052       auto createQuery = folly::sformat(
1053         "CREATE TABLE {} (product TEXT);", table(repoId, "magic"));
1054       txn.exec(createQuery);
1055
1056       auto insertQuery = folly::sformat(
1057         "INSERT INTO {} VALUES('{}');", table(repoId, "magic"), kMagicProduct);
1058       txn.exec(insertQuery);
1059     }
1060     {
1061       auto createQuery = folly::sformat(
1062         "CREATE TABLE {} (path TEXT, sha1 BLOB, UNIQUE(path, sha1));",
1063         table(repoId, "FileSha1"));
1064       txn.exec(createQuery);
1065     }
1066     txn.exec(folly::sformat("CREATE TABLE {} (data BLOB);",
1067                            table(repoId, "GlobalData")));
1068     m_urp.createSchema(repoId, txn);
1069     m_pcrp.createSchema(repoId, txn);
1070     m_rrp.createSchema(repoId, txn);
1071     m_frp.createSchema(repoId, txn);
1072     m_lsrp.createSchema(repoId, txn);
1073
1074     txn.commit();
1075   } catch (RepoExc& re) {
1076     errorMsg = re.what();
1077     return RepoStatus::error;
1078   }
1079   return RepoStatus::success;
1080 }
1081
1082 bool Repo::writable(int repoId) {
1083   switch (sqlite3_db_readonly(m_dbc, dbName(repoId))) {
1084     case 0:  return true;
1085     case 1:  return false;
1086     case -1: return false;
1087     default: break;
1088   }
1089   always_assert(false);
1090 }
1091
1092 //////////////////////////////////////////////////////////////////////
1093
1094 void batchCommit(const std::vector<std::unique_ptr<UnitEmitter>>& ues) {
1095   auto& repo = Repo::get();
1096
1097   // Attempt batch commit.  This can legitimately fail due to multiple input
1098   // files having identical contents.
1099   bool err = false;
1100   {
1101     auto txn = RepoTxn{repo.begin()};
1102
1103     for (auto& ue : ues) {
1104       if (repo.insertUnit(ue.get(), UnitOrigin::File, txn) ==
1105           RepoStatus::error) {
1106         err = true;
1107         break;
1108       }
1109     }
1110     if (!err) {
1111       txn.commit();
1112     }
1113   }
1114
1115   // Commit units individually if an error occurred during batch commit.
1116   if (err) {
1117     for (auto& ue : ues) {
1118       repo.commitUnit(ue.get(), UnitOrigin::File);
1119     }
1120   }
1121 }
1122
1123 //////////////////////////////////////////////////////////////////////
1124
1125 }