hphp/runtime/vm/repo.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #include "hphp/runtime/vm/repo.h"
  17
  18 #include <sstream>
  19
  20 #include <folly/Format.h>
  21 #include <folly/Singleton.h>
  22
  23 #include "hphp/runtime/vm/blob-helper.h"
  24 #include "hphp/runtime/vm/repo-global-data.h"
  25 #include "hphp/runtime/server/xbox-server.h"
  26
  27 #include "hphp/util/assertions.h"
  28 #include "hphp/util/async-func.h"
  29 #include "hphp/util/build-info.h"
  30 #include "hphp/util/hugetlb.h"
  31 #include "hphp/util/logger.h"
  32 #include "hphp/util/process.h"
  33 #include "hphp/util/trace.h"
  34 #include "hphp/util/user-info.h"
  35
  36 #include <grp.h>
  37 #include <pwd.h>
  38 #include <sys/stat.h>
  39
  40 namespace HPHP {
  41
  42 TRACE_SET_MOD(hhbc);
  43
  44 const char* Repo::kMagicProduct =
  45   "facebook.com HipHop Virtual Machine bytecode repository";
  46 const char* Repo::kDbs[RepoIdCount] = { "main",   // Central.
  47                                         "local"}; // Local.
  48 Repo::GlobalData Repo::s_globalData;
  49
  50 void initialize_repo() {
  51   if (!sqlite3_threadsafe()) {
  52     TRACE(0, "SQLite was compiled without thread support; aborting\n");
  53     abort();
  54   }
  55   if (sqlite3_config(SQLITE_CONFIG_MULTITHREAD) != SQLITE_OK) {
  56     TRACE(1, "Failed to set default SQLite multi-threading mode\n");
  57   }
  58   if (sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0) != SQLITE_OK) {
  59     TRACE(1, "Failed to disable SQLite memory statistics\n");
  60   }
  61 }
  62
  63 THREAD_LOCAL(Repo, t_dh);
  64
  65 Repo& Repo::get() {
  66   return *t_dh.get();
  67 }
  68
  69 void Repo::shutdown() {
  70   t_dh.destroy();
  71 }
  72
  73 static SimpleMutex s_lock;
  74 static std::atomic<unsigned> s_nRepos;
  75
  76 bool Repo::prefork() {
  77   if (num_1g_pages() > 0) {
  78     // We put data on the 1G huge pages, and we don't want to do COW upon
  79     // fork().  If you need to fork(), configure HHVM not to use 1G pages.
  80     return true;
  81   }
  82   if (!t_dh.isNull()) {
  83     t_dh.destroy();
  84   }
  85   s_lock.lock();
  86   XboxServer::Stop();
  87   if (s_nRepos > 0 || AsyncFuncImpl::count()) {
  88     XboxServer::Restart();
  89     s_lock.unlock();
  90     return true;
  91   }
  92   folly::SingletonVault::singleton()->destroyInstances();
  93   return false;
  94 }
  95
  96 void Repo::postfork(pid_t pid) {
  97   folly::SingletonVault::singleton()->reenableInstances();
  98   XboxServer::Restart();
  99   if (pid == 0) {
 100     Logger::ResetPid();
 101     new (&s_lock) SimpleMutex();
 102   } else {
 103     s_lock.unlock();
 104   }
 105 }
 106
 107 Repo::Repo()
 108   : RepoProxy(*this),
 109     m_insertFileHash{InsertFileHashStmt(*this, 0),
 110                      InsertFileHashStmt(*this, 1)},
 111     m_getFileHash{GetFileHashStmt(*this, 0), GetFileHashStmt(*this, 1)},
 112     m_removeFileHash{RemoveFileHashStmt(*this, 0),
 113                      RemoveFileHashStmt(*this, 1)},
 114     m_dbc(nullptr), m_localReadable(false), m_localWritable(false),
 115     m_evalRepoId(-1), m_txDepth(0), m_rollback(false), m_beginStmt(*this),
 116     m_rollbackStmt(*this), m_commitStmt(*this), m_urp(*this), m_pcrp(*this),
 117     m_rrp(*this), m_frp(*this), m_lsrp(*this) {
 118
 119   ++s_nRepos;
 120   connect();
 121 }
 122
 123 Repo::~Repo() noexcept {
 124   disconnect();
 125   --s_nRepos;
 126 }
 127
 128 std::string Repo::s_cliFile;
 129 void Repo::setCliFile(const std::string& cliFile) {
 130   assertx(s_cliFile.empty());
 131   assertx(t_dh.isNull());
 132   s_cliFile = cliFile;
 133 }
 134
 135 size_t Repo::stringLengthLimit() const {
 136   static const size_t limit = sqlite3_limit(m_dbc, SQLITE_LIMIT_LENGTH, -1);
 137   return limit;
 138 }
 139
 140 bool Repo::hasGlobalData() {
 141   for (int repoId = RepoIdCount - 1; repoId >= 0; --repoId) {
 142     if (repoName(repoId).empty()) {
 143       // The repo wasn't loadable
 144       continue;
 145     }
 146
 147     RepoStmt stmt(*this);
 148     const auto& tbl = table(repoId, "GlobalData");
 149     stmt.prepare(
 150       folly::sformat(
 151         "SELECT count(*) FROM {};", tbl
 152       )
 153     );
 154     auto txn = RepoTxn{begin()};
 155     RepoTxnQuery query(txn, stmt);
 156     query.step();
 157
 158     if (!query.row()) {
 159       return false;
 160     }
 161
 162     int val;
 163     query.getInt(0, val);
 164     return val != 0;
 165   }
 166
 167   return false;
 168 }
 169
 170 void Repo::loadGlobalData(bool readArrayTable /* = true */) {
 171   if (readArrayTable) m_lsrp.load(RuntimeOption::RepoLitstrLazyLoad);
 172
 173   if (!RuntimeOption::RepoAuthoritative) return;
 174
 175   std::vector<std::string> failures;
 176
 177   /*
 178    * This should probably just go to the Local repo always, except
 179    * that our unit test suite is currently running RepoAuthoritative
 180    * tests with the compiled repo as the Central repo.
 181    */
 182   for (int repoId = RepoIdCount - 1; repoId >= 0; --repoId) {
 183     if (repoName(repoId).empty()) {
 184       // The repo wasn't loadable
 185       continue;
 186     }
 187     try {
 188       RepoStmt stmt(*this);
 189       const auto& tbl = table(repoId, "GlobalData");
 190       stmt.prepare(
 191         folly::sformat(
 192           "SELECT data FROM {} WHERE key = @key;", tbl
 193         )
 194       );
 195       auto txn = RepoTxn{begin()};
 196       {
 197         RepoTxnQuery query(txn, stmt);
 198         auto key = std::string("config");
 199         query.bindStdString("@key", key);
 200         query.step();
 201         if (!query.row()) {
 202           throw RepoExc("Can't find key = 'config' in %s", tbl.c_str());
 203         };
 204         BlobDecoder decoder = query.getBlob(0, true);
 205         decoder(s_globalData);
 206         FTRACE(1, "GlobalData loaded from '{}':\n", repoName(repoId));
 207         FTRACE(1, "{}", show(s_globalData));
 208       }
 209
 210       if (readArrayTable) {
 211         {
 212           RepoTxnQuery query(txn, stmt);
 213           auto key = std::string("arraytable");
 214           query.bindStdString("@key", key);
 215           query.step();
 216           if (!query.row()) {
 217             throw RepoExc("Can't find key = 'arraytable' in %s", tbl.c_str());
 218           };
 219           BlobDecoder decoder = query.getBlob(0, true);
 220           auto& arrayTypeTable = globalArrayTypeTable();
 221           decoder(arrayTypeTable);
 222           decoder(s_globalData.APCProfile);
 223           decoder(s_globalData.ConstantFunctions);
 224           decoder.assertDone();
 225         }
 226       }
 227       txn.commit();
 228     } catch (RepoExc& e) {
 229       failures.push_back(repoName(repoId) + ": "  + e.msg());
 230       continue;
 231     }
 232
 233     // TODO: this should probably read out the other elements of the global data
 234     // which control Option or RuntimeOption values -- the others are read out
 235     // in an inconsistent and ad-hoc manner. But I don't understand their uses
 236     // and interactions well enough to feel comfortable fixing now.
 237     RuntimeOption::EvalPromoteEmptyObject    = s_globalData.PromoteEmptyObject;
 238     RuntimeOption::EnableIntrinsicsExtension =
 239       s_globalData.EnableIntrinsicsExtension;
 240     RuntimeOption::PHP7_Builtins            = s_globalData.PHP7_Builtins;
 241     RuntimeOption::PHP7_NoHexNumerics       = s_globalData.PHP7_NoHexNumerics;
 242     RuntimeOption::PHP7_Substr              = s_globalData.PHP7_Substr;
 243     RuntimeOption::EvalCheckPropTypeHints   = s_globalData.CheckPropTypeHints;
 244     RuntimeOption::EvalHackArrDVArrs        = s_globalData.HackArrDVArrs;
 245     /*
 246      * We only should enable array provenance at runtime if it was enabled in
 247      * the repo AND we have logging enabled--otherwise it's pointless to do the
 248      * bookkeeping
 249      *
 250      * Also--just because array provenance wasn't enabled in the repo doesn't
 251      * mean it can't be explicitly enabled at runtime
 252      */
 253     RuntimeOption::EvalArrayProvenance = RuntimeOption::EvalArrayProvenance ||
 254       (s_globalData.ArrayProvenance && RuntimeOption::EvalLogArrayProvenance);
 255     RuntimeOption::EnableArgsInBacktraces = s_globalData.EnableArgsInBacktraces;
 256     RuntimeOption::EvalAbortBuildOnVerifyError =
 257       s_globalData.AbortBuildOnVerifyError;
 258     RuntimeOption::StrictArrayFillKeys      = s_globalData.StrictArrayFillKeys;
 259     if (s_globalData.HardReturnTypeHints) {
 260       RuntimeOption::EvalCheckReturnTypeHints = 3;
 261     }
 262     if (s_globalData.HardGenericsUB) {
 263       RuntimeOption::EvalEnforceGenericsUB = 2;
 264     }
 265
 266     RuntimeOption::EvalIsCompatibleClsMethType =
 267       s_globalData.IsCompatibleClsMethType;
 268     RuntimeOption::EvalEmitClsMethPointers = s_globalData.EmitClsMethPointers;
 269
 270     RuntimeOption::ConstantFunctions.clear();
 271     for (auto const& elm : s_globalData.ConstantFunctions) {
 272       RuntimeOption::ConstantFunctions.insert(elm);
 273     }
 274
 275     return;
 276   }
 277
 278   if (failures.empty()) {
 279     std::fprintf(stderr, "No repo was loadable. Check all the possible repo "
 280                  "locations (Repo.Central.Path, HHVM_REPO_CENTRAL_PATH, and "
 281                  "$HOME/.hhvm.hhbc) to make sure one of them is a valid "
 282                  "sqlite3 HHVM repo built with this exact HHVM version.\n");
 283   } else {
 284     // We should always have a global data section in RepoAuthoritative
 285     // mode, or the repo is messed up.
 286     std::fprintf(stderr, "Failed to load Repo::GlobalData:\n");
 287     for (auto& f : failures) {
 288       std::fprintf(stderr, "  %s\n", f.c_str());
 289     }
 290   }
 291
 292   assertx(Process::IsInMainThread());
 293   exit(1);
 294 }
 295
 296 void Repo::saveGlobalData(GlobalData newData) {
 297   s_globalData = newData;
 298
 299   auto const repoId = repoIdForNewUnit(UnitOrigin::File);
 300   RepoStmt stmt(*this);
 301   stmt.prepare(
 302     folly::format(
 303       "INSERT INTO {} VALUES(@key, @data);", table(repoId, "GlobalData")
 304     ).str()
 305   );
 306   auto txn = RepoTxn{begin()};
 307   {
 308     RepoTxnQuery query(txn, stmt);
 309     auto key = std::string("config");
 310     query.bindStdString("@key", key);
 311     BlobEncoder encoder{true};
 312     encoder(s_globalData);
 313     query.bindBlob("@data", encoder, /* static */ true);
 314     query.exec();
 315   }
 316
 317   {
 318     RepoTxnQuery query(txn, stmt);
 319     auto key = std::string("arraytable");
 320     query.bindStdString("@key", key);
 321     BlobEncoder encoder{true};
 322     encoder(globalArrayTypeTable());
 323     encoder(s_globalData.APCProfile);
 324     encoder(s_globalData.ConstantFunctions);
 325     query.bindBlob("@data", encoder, /* static */ true);
 326     query.exec();
 327   }
 328
 329   // TODO(#3521039): we could just put the litstr table in the same
 330   // blob as the above and delete LitstrRepoProxy.
 331   LitstrTable::get().setReading();
 332   LitstrRepoProxy::InsertLitstrStmt insertStmt{*this, repoId};
 333   LitstrTable::get().forEachLitstr(
 334     [this, &txn, &insertStmt](int i, const StringData* name) {
 335       insertStmt.insert(txn, i, name);
 336     });
 337
 338   txn.commit();
 339 }
 340
 341 std::unique_ptr<Unit> Repo::loadUnit(const folly::StringPiece name,
 342                                      const SHA1& sha1,
 343                                      const Native::FuncTable& nativeFuncs) {
 344   ARRPROV_USE_RUNTIME_LOCATION();
 345   if (m_dbc == nullptr) return nullptr;
 346   tracing::Block _{
 347     "repo-load-unit", [&] { return tracing::Props{}.add("name", name); }
 348   };
 349   return m_urp.load(name, sha1, nativeFuncs);
 350 }
 351
 352 void Repo::forgetUnit(const std::string& name) {
 353   if (m_dbc == nullptr) {
 354     return;
 355   }
 356
 357   auto const repoId = repoIdForNewUnit(UnitOrigin::File);
 358   auto txn = RepoTxn{begin()};
 359   m_removeFileHash[repoId].remove(txn, name);
 360   txn.commit();
 361 }
 362
 363 std::vector<std::pair<std::string,SHA1>>
 364 Repo::enumerateUnits(int repoId, bool warn) {
 365   std::vector<std::pair<std::string,SHA1>> ret;
 366
 367   try {
 368     RepoStmt stmt(*this);
 369     stmt.prepare(folly::sformat(
 370                    "SELECT path, sha1 FROM {};",
 371                    table(repoId, "FileSha1"))
 372                 );
 373     auto txn = RepoTxn{begin()};
 374     RepoTxnQuery query(txn, stmt);
 375
 376     for (query.step(); query.row(); query.step()) {
 377       std::string path;
 378       SHA1 sha1;
 379
 380       query.getStdString(0, path);
 381       query.getSha1(1, sha1);
 382
 383       ret.emplace_back(path, sha1);
 384     }
 385
 386     txn.commit();
 387   } catch (RepoExc& e) {
 388     if (warn) {
 389       fprintf(stderr, "failed to enumerate units: %s\n", e.what());
 390     }
 391     // Ugh - the error is dropped.  At least we might have printed an error to
 392     // stderr.
 393   }
 394
 395   return ret;
 396 }
 397
 398 void Repo::InsertFileHashStmt::insert(RepoTxn& txn, const StringData* path,
 399                                       const SHA1& sha1) {
 400   if (!prepared()) {
 401     auto insertQuery = folly::sformat(
 402       "INSERT INTO {} VALUES(@path, @sha1);",
 403       m_repo.table(m_repoId, "FileSha1"));
 404     txn.prepare(*this, insertQuery);
 405   }
 406   RepoTxnQuery query(txn, *this);
 407   query.bindStaticString("@path", path);
 408   query.bindSha1("@sha1", sha1);
 409   query.exec();
 410 }
 411
 412 RepoStatus Repo::GetFileHashStmt::get(const char *path, SHA1& sha1) {
 413   try {
 414     auto txn = RepoTxn{m_repo.begin()};
 415     if (!prepared()) {
 416       auto selectQuery = folly::sformat(
 417         "SELECT f.sha1 "
 418         "FROM {} AS f, {} AS u "
 419         "WHERE path == @path AND f.sha1 == u.sha1 "
 420         "ORDER BY unitSn DESC LIMIT 1;",
 421         m_repo.table(m_repoId, "FileSha1"),
 422         m_repo.table(m_repoId, "Unit"));
 423       txn.prepare(*this, selectQuery);
 424     }
 425     RepoTxnQuery query(txn, *this);
 426     query.bindText("@path", path, strlen(path));
 427     query.step();
 428     if (!query.row()) {
 429       return RepoStatus::error;
 430     }
 431     query.getSha1(0, sha1);
 432     txn.commit();
 433     return RepoStatus::success;
 434   } catch (RepoExc& re) {
 435     return RepoStatus::error;
 436   }
 437 }
 438
 439 void Repo::RemoveFileHashStmt::remove(RepoTxn& txn, const std::string& path) {
 440   if (!prepared()) {
 441     auto insertQuery = folly::sformat(
 442       "DELETE FROM {} WHERE path == @path;",
 443       m_repo.table(m_repoId, "FileSha1"));
 444     txn.prepare(*this, insertQuery);
 445   }
 446   RepoTxnQuery query(txn, *this);
 447   query.bindStdString("@path", path);
 448   query.exec();
 449 }
 450
 451 RepoStatus Repo::findFile(const char *path, const std::string &root,
 452                           SHA1& sha1) {
 453   tracing::Block _{
 454     "repo-find-file",
 455     [&] {
 456       return tracing::Props{}
 457         .add("path", path)
 458         .add("root", root);
 459     }
 460   };
 461
 462   if (m_dbc == nullptr) {
 463     return RepoStatus::error;
 464   }
 465   int repoId;
 466   for (repoId = RepoIdCount - 1; repoId >= 0; --repoId) {
 467     if (*path == '/' && !root.empty() &&
 468         !strncmp(root.c_str(), path, root.size()) &&
 469         (m_getFileHash[repoId].get(path + root.size(), sha1) ==
 470          RepoStatus::success)) {
 471       TRACE(3, "Repo loaded file hash for '%s' from '%s'\n",
 472                path + root.size(), repoName(repoId).c_str());
 473       return RepoStatus::success;
 474     }
 475     if (m_getFileHash[repoId].get(path, sha1) == RepoStatus::success) {
 476       TRACE(3, "Repo loaded file hash for '%s' from '%s'\n",
 477                 path, repoName(repoId).c_str());
 478       return RepoStatus::success;
 479     }
 480   }
 481   TRACE(3, "Repo file hash: error loading '%s'\n", path);
 482   return RepoStatus::error;
 483 }
 484
 485 RepoStatus Repo::insertSha1(UnitOrigin unitOrigin, UnitEmitter* ue,
 486                            RepoTxn& txn) {
 487   const StringData* path = ue->m_filepath;
 488   const SHA1& sha1 = ue->sha1();
 489   int repoId = repoIdForNewUnit(unitOrigin);
 490   if (repoId == RepoIdInvalid) {
 491     return RepoStatus::error;
 492   }
 493   try {
 494     m_insertFileHash[repoId].insert(txn, path, sha1);
 495     return RepoStatus::success;
 496   } catch (RepoExc& re) {
 497     TRACE(3, "Failed to commit sha1 for '%s' to '%s': %s\n",
 498               path->data(), repoName(repoId).c_str(), re.msg().c_str());
 499     return RepoStatus::error;
 500   }
 501 }
 502
 503 void Repo::commitSha1(UnitOrigin unitOrigin, UnitEmitter* ue) {
 504   try {
 505     auto txn = RepoTxn{begin()};
 506     RepoStatus err = insertSha1(unitOrigin, ue, txn);
 507     if (err == RepoStatus::success) {
 508       txn.commit();
 509     }
 510   } catch (RepoExc& re) {
 511     tracing::addPointNoTrace("sha1-commit-exn");
 512     int repoId = repoIdForNewUnit(unitOrigin);
 513     if (repoId != RepoIdInvalid) {
 514       TRACE(3, "Failed to commit sha1 for '%s' to '%s': %s\n",
 515                ue->m_filepath->data(), repoName(repoId).c_str(),
 516                re.msg().c_str());
 517     }
 518   }
 519 }
 520
 521 std::string Repo::table(int repoId, const char* tablePrefix) {
 522   return folly::sformat(
 523     "{}.{}_{}", dbName(repoId), tablePrefix, repoSchemaId());
 524 }
 525
 526 void Repo::exec(const std::string& sQuery) {
 527   RepoStmt stmt(*this);
 528   stmt.prepare(sQuery);
 529   RepoQuery query(stmt);
 530   query.exec();
 531 }
 532
 533 RepoTxn Repo::begin() {
 534   if (m_txDepth > 0) {
 535     m_txDepth++;
 536     return RepoTxn{*this};
 537   }
 538   if (debug) {
 539     // Verify start state.
 540     always_assert(m_txDepth == 0);
 541     always_assert(!m_rollback);
 542     if (true) {
 543       // Bypass RepoQuery, in order to avoid triggering exceptions.
 544       int rc = sqlite3_step(m_rollbackStmt.get());
 545       switch (rc) {
 546       case SQLITE_DONE:
 547       case SQLITE_ROW:
 548         not_reached();
 549       default:
 550         break;
 551       }
 552     } else {
 553       bool rollbackFailed = false;
 554       try {
 555         RepoQuery query(m_rollbackStmt);
 556         query.exec();
 557       } catch (RepoExc& re) {
 558         rollbackFailed = true;
 559       }
 560       always_assert(rollbackFailed);
 561     }
 562   }
 563   RepoQuery query(m_beginStmt);
 564   query.exec();
 565   m_txDepth++;
 566
 567   return RepoTxn(*this);
 568 }
 569
 570 void Repo::txPop() {
 571   // We mix the concept of rollback with a normal commit so that if we try to
 572   // rollback an inner transaction we eventually end up rolling back the outer
 573   // transaction instead (Sqlite doesn't support rolling back partial
 574   // transactions).
 575   assertx(m_txDepth > 0);
 576   if (m_txDepth > 1) {
 577     m_txDepth--;
 578     return;
 579   }
 580   if (!m_rollback) {
 581     RepoQuery query(m_commitStmt);
 582     query.exec();
 583   } else {
 584     // We're in the outermost transaction - so clear the rollback flag.
 585     m_rollback = false;
 586     RepoQuery query(m_rollbackStmt);
 587     try {
 588       query.exec();
 589     } catch (RepoExc& ex) {
 590       /*
 591        * Having a rollback fail is actually a normal, expected case,
 592        * so just swallow this.
 593        *
 594        * In particular, according to the docs, if we got an I/O error
 595        * while doing a commit, the rollback will often fail with "no
 596        * transaction in progress", because the commit will have
 597        * automatically been rolled back.  Recommended practice is
 598        * still to execute a rollback statement and ignore the error.
 599        */
 600       TRACE(4, "repo rollback failure: %s\n", ex.what());
 601     }
 602   }
 603   // Decrement depth after query execution, in case an exception occurs during
 604   // commit.  This allows for subsequent rollback of the failed commit.
 605   m_txDepth--;
 606 }
 607
 608 void Repo::rollback() {
 609   m_rollback = true;
 610   // NOTE: A try/catch isn't necessary - txPop() handles rollback as a nothrow.
 611   txPop();
 612 }
 613
 614 void Repo::commit() {
 615   txPop();
 616 }
 617
 618 RepoStatus Repo::insertUnit(UnitEmitter* ue, UnitOrigin unitOrigin,
 619                             RepoTxn& txn) {
 620   if (insertSha1(unitOrigin, ue, txn) == RepoStatus::error ||
 621       ue->insert(unitOrigin, txn) == RepoStatus::error) {
 622     return RepoStatus::error;
 623   }
 624   return RepoStatus::success;
 625 }
 626
 627 void Repo::commitUnit(UnitEmitter* ue, UnitOrigin unitOrigin) {
 628   if (!RuntimeOption::RepoCommit || ue->m_ICE) return;
 629
 630   tracing::Block _{
 631     "repo-commit-unit",
 632     [&] { return tracing::Props{}.add("filename", ue->m_filepath); }
 633   };
 634
 635   try {
 636     commitSha1(unitOrigin, ue);
 637     ue->commit(unitOrigin);
 638   } catch (const std::exception& e) {
 639     TRACE(0, "unexpected exception in commitUnit: %s\n",
 640           e.what());
 641     assertx(false);
 642   }
 643 }
 644
 645 void Repo::connect() {
 646   initCentral();
 647   initLocal();
 648   if (!RuntimeOption::RepoEvalMode.compare("local")) {
 649     m_evalRepoId = (m_localWritable) ? RepoIdLocal : RepoIdCentral;
 650   } else if (!RuntimeOption::RepoEvalMode.compare("central")) {
 651     m_evalRepoId = RepoIdCentral;
 652   } else {
 653     assertx(!RuntimeOption::RepoEvalMode.compare("readonly"));
 654     m_evalRepoId = RepoIdInvalid;
 655   }
 656   TRACE(1, "Repo.Eval.Mode=%s\n",
 657            (m_evalRepoId == RepoIdLocal)
 658            ? "local"
 659            : (m_evalRepoId == RepoIdCentral)
 660              ? "central"
 661              : "readonly");
 662 }
 663
 664 void Repo::disconnect() noexcept {
 665   if (m_dbc != nullptr) {
 666     sqlite3_close_v2(m_dbc);
 667     m_dbc = nullptr;
 668     m_localReadable = false;
 669     m_localWritable = false;
 670     m_evalRepoId = RepoIdInvalid;
 671   }
 672 }
 673
 674 void Repo::initCentral() {
 675   std::string error;
 676
 677   assertx(m_dbc == nullptr);
 678   auto tryPath = [this, &error](const char* path) {
 679     std::string subErr;
 680     if (openCentral(path, subErr) == RepoStatus::error) {
 681       folly::format(&error, "  {}\n", subErr.empty() ? path : subErr);
 682       return false;
 683     }
 684     return true;
 685   };
 686
 687   auto fail_no_repo = [&error] {
 688     error = "Failed to initialize central HHBC repository:\n" + error;
 689     // Database initialization failed; this is an unrecoverable state.
 690     Logger::Error(error);
 691
 692     if (Process::IsInMainThread()) {
 693       exit(1);
 694     }
 695     always_assert_flog(false, "{}", error);
 696   };
 697
 698   // Try Repo.Central.Path
 699   if (!RuntimeOption::RepoCentralPath.empty() &&
 700       tryPath(RuntimeOption::RepoCentralPath.c_str())) {
 701     return;
 702   }
 703
 704   // Try HHVM_REPO_CENTRAL_PATH
 705   const char* HHVM_REPO_CENTRAL_PATH = getenv("HHVM_REPO_CENTRAL_PATH");
 706   if (HHVM_REPO_CENTRAL_PATH != nullptr &&
 707       tryPath(HHVM_REPO_CENTRAL_PATH)) {
 708     return;
 709   }
 710
 711   if (!RuntimeOption::RepoAllowFallbackPath) fail_no_repo();
 712
 713   // Try "$HOME/.hhvm.hhbc".
 714   char* HOME = getenv("HOME");
 715   if (HOME != nullptr) {
 716     std::string centralPath = HOME;
 717     centralPath += "/.hhvm.hhbc";
 718     if (tryPath(centralPath.c_str())) {
 719       return;
 720     }
 721   }
 722
 723 #ifndef _WIN32
 724   // Try the equivalent of "$HOME/.hhvm.hhbc", but look up the home directory
 725   // in the password database.
 726   {
 727     auto buf = PasswdBuffer{};
 728     passwd* pw;
 729     auto err = getpwuid_r(getuid(), &buf.ent, buf.data.get(), buf.size, &pw);
 730     if (err == 0 && pw != nullptr &&
 731         (HOME == nullptr || strcmp(HOME, pw->pw_dir))) {
 732       std::string centralPath = pw->pw_dir;
 733       centralPath += "/.hhvm.hhbc";
 734       if (tryPath(centralPath.c_str())) {
 735         return;
 736       }
 737     }
 738   }
 739 #else // _WIN32
 740   // Try "$HOMEDRIVE$HOMEPATH/.hhvm.hhbc"
 741   char* HOMEDRIVE = getenv("HOMEDRIVE");
 742   if (HOMEDRIVE != nullptr) {
 743     char* HOMEPATH = getenv("HOMEPATH");
 744     if (HOMEPATH != nullptr) {
 745       std::string centralPath = HOMEDRIVE;
 746       centralPath += HOMEPATH;
 747       centralPath += "\\.hhvm.hhbc";
 748       if (tryPath(centralPath.c_str()))
 749         return;
 750     }
 751   }
 752 #endif
 753
 754   fail_no_repo();
 755 }
 756
 757 namespace {
 758 /*
 759  * Convert the permission bits from the given stat struct to an ls-style
 760  * rwxrwxrwx format.
 761  */
 762 std::string showPermissions(const struct stat& s) {
 763   static const std::pair<int, char> bits[] = {
 764     {S_IRUSR, 'r'}, {S_IWUSR, 'w'}, {S_IXUSR, 'x'},
 765     {S_IRGRP, 'r'}, {S_IWGRP, 'w'}, {S_IXGRP, 'x'},
 766     {S_IROTH, 'r'}, {S_IWOTH, 'w'}, {S_IXOTH, 'x'},
 767   };
 768   std::string ret;
 769   ret.reserve(sizeof(bits) / sizeof(bits[0]));
 770
 771   for (auto pair : bits) {
 772     ret += (s.st_mode & pair.first) ? pair.second : '-';
 773   }
 774   return ret;
 775 }
 776
 777 /*
 778  * Return the name of the user with the given id.
 779  */
 780 std::string uidToName(uid_t uid) {
 781 #ifndef _WIN32
 782   auto buf = PasswdBuffer{};
 783   passwd* pw;
 784
 785   auto err = getpwuid_r(uid, &buf.ent, buf.data.get(), buf.size, &pw);
 786   if (err != 0) return folly::toStdString(folly::errnoStr(errno));
 787   if (pw == nullptr) return "user does not exist";
 788   return pw->pw_name;
 789 #else
 790   return "<unsupported>";
 791 #endif
 792 }
 793
 794 /*
 795  * Return the uid of the user with the given name.
 796  */
 797 uid_t nameToUid(const std::string& name) {
 798 #ifndef _WIN32
 799   auto buf = PasswdBuffer{};
 800   passwd* pw;
 801
 802   auto err = getpwnam_r(name.c_str(), &buf.ent, buf.data.get(), buf.size, &pw);
 803   if (err != 0 || pw == nullptr) return -1;
 804   return pw->pw_uid;
 805 #else
 806   return -1;
 807 #endif
 808 }
 809
 810 /*
 811  * Return the name of the group with the given id.
 812  */
 813 std::string gidToName(gid_t gid) {
 814 #ifndef _WIN32
 815   auto buf = GroupBuffer{};
 816   group* grp;
 817
 818   auto err = getgrgid_r(gid, &buf.ent, buf.data.get(), buf.size, &grp);
 819   if (err != 0) return folly::toStdString(folly::errnoStr(errno));
 820   if (grp == nullptr) return "group does not exist";
 821   return grp->gr_name;
 822 #else
 823   return "<unsupported>";
 824 #endif
 825 }
 826
 827 /*
 828  * Return the gid of the group with the given name.
 829  */
 830 gid_t nameToGid(const std::string& name) {
 831 #ifndef _WIN32
 832   auto buf = GroupBuffer{};
 833   group* grp;
 834
 835   auto err = getgrnam_r(name.c_str(), &buf.ent, buf.data.get(), buf.size, &grp);
 836   if (err != 0 || grp == nullptr) return -1;
 837   return grp->gr_gid;
 838 #else
 839   return -1;
 840 #endif
 841 }
 842
 843 void setCentralRepoFileMode(const std::string& path) {
 844   // These runtime options are all best-effort, so we don't care if any of the
 845   // operations fail.
 846
 847   if (auto const mode = RuntimeOption::RepoCentralFileMode) {
 848     chmod(path.c_str(), mode);
 849   }
 850
 851   if (!RuntimeOption::RepoCentralFileUser.empty()) {
 852     auto const uid = nameToUid(RuntimeOption::RepoCentralFileUser);
 853     chown(path.c_str(), uid, -1);
 854   }
 855
 856   if (!RuntimeOption::RepoCentralFileGroup.empty()) {
 857     auto const gid = nameToGid(RuntimeOption::RepoCentralFileGroup);
 858     chown(path.c_str(), -1, gid);
 859   }
 860 }
 861 }
 862
 863 RepoStatus Repo::openCentral(const char* rawPath, std::string& errorMsg) {
 864   std::string repoPath = rawPath;
 865   replacePlaceholders(repoPath);
 866   // SQLITE_OPEN_NOMUTEX specifies that the connection be opened such
 867   // that no mutexes are used to protect the database connection from other
 868   // threads.  However, multiple connections can still be used concurrently,
 869   // because SQLite as a whole is thread-safe.
 870   if (int err = sqlite3_open_v2(repoPath.c_str(), &m_dbc,
 871                                 SQLITE_OPEN_NOMUTEX |
 872                                 SQLITE_OPEN_READWRITE |
 873                                 SQLITE_OPEN_CREATE, nullptr)) {
 874     TRACE(1, "Repo::%s() failed to open candidate central repo '%s'\n",
 875              __func__, repoPath.c_str());
 876     errorMsg = folly::format("Failed to open {}: {} - {}",
 877                              repoPath, err, sqlite3_errmsg(m_dbc)).str();
 878     return RepoStatus::error;
 879   }
 880
 881   if (RuntimeOption::RepoBusyTimeoutMS) {
 882     sqlite3_busy_timeout(m_dbc, RuntimeOption::RepoBusyTimeoutMS);
 883   }
 884   try {
 885     m_beginStmt.prepare("BEGIN TRANSACTION;");
 886     m_rollbackStmt.prepare("ROLLBACK;");
 887     m_commitStmt.prepare("COMMIT;");
 888     pragmas(RepoIdCentral);
 889   } catch (RepoExc& re) {
 890     TRACE(1, "Repo::%s() failed to initialize connection to canditate repo"
 891              " '%s': %s\n", __func__, repoPath.c_str(), re.what());
 892     errorMsg = folly::format("Failed to initialize connection to {}: {}",
 893                              repoPath, re.what()).str();
 894     return RepoStatus::error;
 895   }
 896
 897   // sqlite3_open_v2() will silently open in read-only mode if file permissions
 898   // prevent writing.  Therefore, tell initSchema() to verify that the database
 899   // is writable.
 900   bool centralWritable = true;
 901   if (initSchema(RepoIdCentral, centralWritable, errorMsg) == RepoStatus::error
 902       || !centralWritable) {
 903     TRACE(1, "Repo::initSchema() failed for candidate central repo '%s'\n",
 904              repoPath.c_str());
 905     struct stat repoStat;
 906     std::string statStr;
 907     if (stat(repoPath.c_str(), &repoStat) == 0) {
 908       statStr = folly::sformat("{} {}:{}",
 909                                showPermissions(repoStat),
 910                                uidToName(repoStat.st_uid),
 911                                gidToName(repoStat.st_gid));
 912     } else {
 913       statStr = folly::toStdString(folly::errnoStr(errno));
 914     }
 915     errorMsg = folly::format("Failed to initialize schema in {}({}): {}",
 916                              repoPath, statStr, errorMsg).str();
 917     return RepoStatus::error;
 918   }
 919   m_centralRepo = repoPath;
 920   setCentralRepoFileMode(repoPath);
 921   TRACE(1, "Central repo: '%s'\n", m_centralRepo.c_str());
 922   return RepoStatus::success;
 923 }
 924
 925 void Repo::initLocal() {
 926   if (RuntimeOption::RepoLocalMode.compare("--")) {
 927     bool isWritable;
 928     if (!RuntimeOption::RepoLocalMode.compare("rw")) {
 929       isWritable = true;
 930     } else {
 931       assertx(!RuntimeOption::RepoLocalMode.compare("r-"));
 932       isWritable = false;
 933     }
 934
 935     if (!RuntimeOption::RepoLocalPath.empty()) {
 936       attachLocal(RuntimeOption::RepoLocalPath.c_str(), isWritable);
 937     } else if (RuntimeOption::RepoAllowFallbackPath) {
 938       if (!RuntimeOption::ServerExecutionMode()) {
 939         std::string cliRepo = s_cliFile;
 940         if (!cliRepo.empty()) {
 941           cliRepo += ".hhbc";
 942         }
 943         attachLocal(cliRepo.c_str(), isWritable);
 944       } else {
 945         attachLocal("hhvm.hhbc", isWritable);
 946       }
 947     }
 948   }
 949 }
 950
 951 void Repo::attachLocal(const char* path, bool isWritable) {
 952   std::string repoPath = path;
 953   replacePlaceholders(repoPath);
 954   if (!isWritable) {
 955     // Make sure the repo exists before attaching it, in order to avoid
 956     // creating a read-only repo.
 957     struct stat buf;
 958     if (!strchr(repoPath.c_str(), ':') &&
 959         stat(repoPath.c_str(), &buf) != 0) {
 960       return;
 961     }
 962   }
 963   try {
 964     auto attachQuery = folly::sformat(
 965       "ATTACH DATABASE '{}' as {};", repoPath, dbName(RepoIdLocal));
 966     exec(attachQuery);
 967     pragmas(RepoIdLocal);
 968   } catch (RepoExc& re) {
 969     // Failed to run pragmas on local DB - ignored
 970     return;
 971   }
 972
 973   std::string error;
 974   if (initSchema(RepoIdLocal, isWritable, error) == RepoStatus::error) {
 975     FTRACE(1, "Local repo {} failed to init schema: {}\n", repoPath, error);
 976     return;
 977   }
 978   m_localRepo = repoPath;
 979   m_localReadable = true;
 980   m_localWritable = isWritable;
 981   TRACE(1, "Local repo: '%s' (read%s)\n",
 982            m_localRepo.c_str(), m_localWritable ? "-write" : "-only");
 983 }
 984
 985 void Repo::pragmas(int repoId) {
 986   // Valid synchronous values: 0 (OFF), 1 (NORMAL), 2 (FULL).
 987   static const int synchronous = 0;
 988   setIntPragma(repoId, "synchronous", synchronous);
 989   setIntPragma(repoId, "cache_size", 20);
 990   // Valid journal_mode values: delete, truncate, persist, memory, wal, off.
 991   setTextPragma(repoId, "journal_mode", RuntimeOption::RepoJournal.c_str());
 992 }
 993
 994 void Repo::getIntPragma(int repoId, const char* name, int& val) {
 995   auto pragmaQuery = folly::sformat("PRAGMA {}.{};", dbName(repoId), name);
 996   RepoStmt stmt(*this);
 997   stmt.prepare(pragmaQuery);
 998   RepoQuery query(stmt);
 999   query.step();
1000   query.getInt(0, val);
1001 }
1002
1003 void Repo::setIntPragma(int repoId, const char* name, int val) {
1004   // Read first to see if a write can be avoided
1005   int oldval = -1;
1006   getIntPragma(repoId, name, oldval);
1007   if (val == oldval) return;
1008
1009   // Pragma writes must be executed outside transactions, since they may change
1010   // transaction behavior.
1011   auto pragmaQuery = folly::sformat(
1012     "PRAGMA {}.{} = {};", dbName(repoId), name, val);
1013   exec(pragmaQuery);
1014   if (debug) {
1015     // Verify that the pragma had the desired effect.
1016     int newval = -1;
1017     getIntPragma(repoId, name, newval);
1018     if (newval != val) {
1019       throw RepoExc("Unexpected PRAGMA %s.%s value: %d\n",
1020                     dbName(repoId), name, newval);
1021     }
1022   }
1023 }
1024
1025 void Repo::getTextPragma(int repoId, const char* name, std::string& val) {
1026   auto pragmaQuery = folly::sformat("PRAGMA {}.{};", dbName(repoId), name);
1027   RepoStmt stmt(*this);
1028   stmt.prepare(pragmaQuery);
1029   RepoQuery query(stmt);
1030   const char* s;
1031   query.step();
1032   query.getText(0, s);
1033   val = s;
1034 }
1035
1036 void Repo::setTextPragma(int repoId, const char* name, const char* val) {
1037   // Read first to see if a write can be avoided
1038   std::string oldval = "?";
1039   getTextPragma(repoId, name, oldval);
1040   if (!strcmp(oldval.c_str(), val)) return;
1041   // Pragma writes must be executed outside transactions, since they may change
1042   // transaction behavior.
1043   auto pragmaQuery = folly::sformat(
1044     "PRAGMA {}.{} = {};", dbName(repoId), name, val);
1045   exec(pragmaQuery);
1046   if (debug) {
1047     // Verify that the pragma had the desired effect.
1048     std::string newval = "?";
1049     getTextPragma(repoId, name, newval);
1050     if (strcmp(newval.c_str(), val)) {
1051       // If the db is in memory, journal mode will stick at "memory"
1052       // unless its turned off. Ignore attempts to change it to
1053       // something else.
1054       if (!strcmp(name, "journal_mode") &&
1055           !strcmp(newval.c_str(), "memory") &&
1056           strcmp(val, "off")) {
1057         return;
1058       }
1059       throw RepoExc("Unexpected PRAGMA %s.%s value: %s (should be %s)\n",
1060                     dbName(repoId), name, newval.c_str(), val);
1061     }
1062   }
1063 }
1064
1065 RepoStatus Repo::initSchema(int repoId, bool& isWritable,
1066                             std::string& errorMsg) {
1067   if (!schemaExists(repoId)) {
1068     if (createSchema(repoId, errorMsg) == RepoStatus::error) {
1069       // Check whether this failure is due to losing the schema
1070       // initialization race with another process.
1071       if (!schemaExists(repoId)) {
1072         return RepoStatus::error;
1073       }
1074     } else {
1075       // createSchema() successfully wrote to the database, so no further
1076       // verification is necessary.
1077       return RepoStatus::success;
1078     }
1079   }
1080   if (isWritable) {
1081     isWritable = writable(repoId);
1082   }
1083   return RepoStatus::success;
1084 }
1085
1086 bool Repo::schemaExists(int repoId) {
1087   try {
1088     auto txn = RepoTxn{begin()};
1089     auto selectQuery = folly::sformat(
1090       "SELECT product FROM {};", table(repoId, "magic"));
1091     RepoStmt stmt(*this);
1092     // If the DB is 'new' and hasn't been initialized yet then we expect this
1093     // prepare() to fail.
1094     stmt.prepare(selectQuery);
1095     // This SHOULDN'T fail - we create the table under a transaction - so if it
1096     // exists then it should have our magic value.
1097     RepoTxnQuery query(txn, stmt);
1098     query.step();
1099     const char* text; /**/ query.getText(0, text);
1100     if (strcmp(kMagicProduct, text)) {
1101       return false;
1102     }
1103     txn.commit();
1104   } catch (RepoExc& re) {
1105     return false;
1106   }
1107   return true;
1108 }
1109
1110 RepoStatus Repo::createSchema(int repoId, std::string& errorMsg) {
1111   try {
1112     auto txn = RepoTxn{begin()};
1113     {
1114       auto createQuery = folly::sformat(
1115         "CREATE TABLE {} (product TEXT);", table(repoId, "magic"));
1116       txn.exec(createQuery);
1117
1118       auto insertQuery = folly::sformat(
1119         "INSERT INTO {} VALUES('{}');", table(repoId, "magic"), kMagicProduct);
1120       txn.exec(insertQuery);
1121     }
1122     {
1123       auto createQuery = folly::sformat(
1124         "CREATE TABLE {} (path TEXT, sha1 BLOB, UNIQUE(path, sha1));",
1125         table(repoId, "FileSha1"));
1126       txn.exec(createQuery);
1127     }
1128     txn.exec(folly::sformat("CREATE TABLE {} (key TEXT, data BLOB);",
1129                            table(repoId, "GlobalData")));
1130     m_urp.createSchema(repoId, txn);
1131     m_pcrp.createSchema(repoId, txn);
1132     m_rrp.createSchema(repoId, txn);
1133     m_frp.createSchema(repoId, txn);
1134     m_lsrp.createSchema(repoId, txn);
1135
1136     txn.commit();
1137   } catch (RepoExc& re) {
1138     errorMsg = re.what();
1139     return RepoStatus::error;
1140   }
1141   return RepoStatus::success;
1142 }
1143
1144 bool Repo::writable(int repoId) {
1145   switch (sqlite3_db_readonly(m_dbc, dbName(repoId))) {
1146     case 0:  return true;
1147     case 1:  return false;
1148     case -1: return false;
1149     default: break;
1150   }
1151   always_assert(false);
1152 }
1153
1154 //////////////////////////////////////////////////////////////////////
1155
1156 void batchCommit(const std::vector<std::unique_ptr<UnitEmitter>>& ues) {
1157   auto& repo = Repo::get();
1158
1159   // Attempt batch commit.  This can legitimately fail due to multiple input
1160   // files having identical contents.
1161   bool err = false;
1162   {
1163     auto txn = RepoTxn{repo.begin()};
1164
1165     for (auto& ue : ues) {
1166       if (repo.insertUnit(ue.get(), UnitOrigin::File, txn) ==
1167           RepoStatus::error) {
1168         err = true;
1169         break;
1170       }
1171     }
1172     if (!err) {
1173       txn.commit();
1174     }
1175   }
1176
1177   // Commit units individually if an error occurred during batch commit.
1178   if (err) {
1179     for (auto& ue : ues) {
1180       repo.commitUnit(ue.get(), UnitOrigin::File);
1181     }
1182   }
1183 }
1184
1185 //////////////////////////////////////////////////////////////////////
1186
1187 }