hphp/runtime/vm/repo.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #include "hphp/runtime/vm/repo.h"
  17
  18 #include <sstream>
  19
  20 #include <folly/Format.h>
  21 #include <folly/Singleton.h>
  22
  23 #include "hphp/runtime/base/autoload-handler.h"
  24 #include "hphp/runtime/base/repo-autoload-map.h"
  25
  26 #include "hphp/runtime/vm/blob-helper.h"
  27 #include "hphp/runtime/vm/repo-autoload-map-builder.h"
  28 #include "hphp/runtime/vm/repo-file.h"
  29 #include "hphp/runtime/vm/repo-global-data.h"
  30
  31 #include "hphp/runtime/server/xbox-server.h"
  32
  33 #include "hphp/util/assertions.h"
  34 #include "hphp/util/async-func.h"
  35 #include "hphp/util/build-info.h"
  36 #include "hphp/util/hugetlb.h"
  37 #include "hphp/util/logger.h"
  38 #include "hphp/util/process.h"
  39 #include "hphp/util/trace.h"
  40 #include "hphp/util/user-info.h"
  41
  42 #include <grp.h>
  43 #include <pwd.h>
  44 #include <sys/stat.h>
  45
  46 namespace HPHP {
  47
  48 TRACE_SET_MOD(hhbc);
  49
  50 const char* Repo::kMagicProduct =
  51   "facebook.com HipHop Virtual Machine bytecode repository";
  52 const char* Repo::kDbs[RepoIdCount] = { "main",   // Central.
  53                                         "local"}; // Local.
  54 RepoGlobalData Repo::s_globalData;
  55 bool Repo::s_deleteLocalOnFailure;
  56
  57 void initialize_repo() {
  58   if (!sqlite3_threadsafe()) {
  59     TRACE(0, "SQLite was compiled without thread support; aborting\n");
  60     abort();
  61   }
  62   if (sqlite3_config(SQLITE_CONFIG_MULTITHREAD) != SQLITE_OK) {
  63     TRACE(1, "Failed to set default SQLite multi-threading mode\n");
  64   }
  65   if (sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0) != SQLITE_OK) {
  66     TRACE(1, "Failed to disable SQLite memory statistics\n");
  67   }
  68 }
  69
  70 THREAD_LOCAL(Repo, t_dh);
  71
  72 Repo& Repo::get() {
  73   assertx(!RO::RepoAuthoritative);
  74   return *t_dh.get();
  75 }
  76
  77 void Repo::shutdown() {
  78   assertx(!RO::RepoAuthoritative);
  79   t_dh.destroy();
  80 }
  81
  82 static SimpleMutex s_lock;
  83 static std::atomic<unsigned> s_nRepos;
  84
  85 bool Repo::prefork() {
  86   if (num_1g_pages() > 0 || RuntimeOption::EvalFileBackedColdArena) {
  87     // We put data on shared pages, which won't work with fork().
  88     return true;
  89   }
  90   if (!t_dh.isNull()) {
  91     t_dh.destroy();
  92   }
  93   s_lock.lock();
  94   XboxServer::Stop();
  95   if (s_nRepos > 0 || AsyncFuncImpl::count()) {
  96     XboxServer::Restart();
  97     s_lock.unlock();
  98     return true;
  99   }
 100   folly::SingletonVault::singleton()->destroyInstances();
 101   return false;
 102 }
 103
 104 void Repo::postfork(pid_t pid) {
 105   folly::SingletonVault::singleton()->reenableInstances();
 106   RepoFile::postfork();
 107   XboxServer::Restart();
 108   if (pid == 0) {
 109     Logger::ResetPid();
 110     new (&s_lock) SimpleMutex();
 111   } else {
 112     s_lock.unlock();
 113   }
 114 }
 115
 116 Repo::Repo()
 117   : RepoProxy(*this),
 118     m_insertFileHash{InsertFileHashStmt(*this, 0),
 119                      InsertFileHashStmt(*this, 1)},
 120     m_getFileHash{GetFileHashStmt(*this, 0), GetFileHashStmt(*this, 1)},
 121     m_removeFileHash{RemoveFileHashStmt(*this, 0),
 122                      RemoveFileHashStmt(*this, 1)},
 123     m_getUnitPath{GetUnitPathStmt(*this, 0), GetUnitPathStmt(*this, 1)},
 124     m_getUnit{GetUnitStmt(*this, 0), GetUnitStmt(*this, 1)},
 125     m_dbc(nullptr), m_localWritable(false), m_centralWritable(false),
 126     m_txDepth(0), m_rollback(false), m_beginStmt(*this),
 127     m_rollbackStmt(*this), m_commitStmt(*this), m_urp(*this), m_pcrp(*this),
 128     m_rrp(*this), m_frp(*this), m_lsrp(*this), m_numOpenRepos(0) {
 129   assertx(!RO::RepoAuthoritative);
 130   ++s_nRepos;
 131   connect();
 132 }
 133
 134 Repo::~Repo() noexcept {
 135   disconnect();
 136   --s_nRepos;
 137 }
 138
 139 std::string Repo::s_cliFile;
 140 void Repo::setCliFile(const std::string& cliFile) {
 141   assertx(s_cliFile.empty());
 142   assertx(t_dh.isNull());
 143   s_cliFile = cliFile;
 144 }
 145
 146 size_t Repo::stringLengthLimit() const {
 147   static const size_t limit = sqlite3_limit(m_dbc, SQLITE_LIMIT_LENGTH, -1);
 148   assertx(limit > 8);
 149
 150   // In addition to restricting the size of text and blob fields, during an
 151   // INSERT or SELECT operation an entire row is encoded as blob, therefore,
 152   // SQLITE_LIMIT_LENGTH controls the maximum total size of a row. We currently
 153   // store litstrs in a table with an INTEGER key and a TEXT litstr, the size of
 154   // the litstr must leave room for an integer, which can require between 1 and
 155   // 8 bytes.
 156   return limit - 8;
 157 }
 158
 159 bool Repo::hasGlobalData() {
 160 for (int repoId = numOpenRepos() - 1; repoId >= 0; --repoId) {
 161     RepoStmt stmt(*this);
 162     const auto& tbl = table(repoId, "GlobalData");
 163     stmt.prepare(
 164       folly::sformat(
 165         "SELECT count(*) FROM {};", tbl
 166       )
 167     );
 168     auto txn = RepoTxn{begin()};
 169     RepoTxnQuery query(txn, stmt);
 170     query.step();
 171
 172     if (!query.row()) {
 173       return false;
 174     }
 175
 176     int val;
 177     query.getInt(0, val);
 178     return val != 0;
 179   }
 180
 181   return false;
 182 }
 183
 184 void Repo::loadGlobalData(bool readGlobalTables /* = true */) {
 185   if (readGlobalTables) m_lsrp.load(RuntimeOption::RepoLitstrLazyLoad);
 186
 187   if (!RuntimeOption::RepoAuthoritative) return;
 188
 189   std::vector<std::string> failures;
 190
 191   /*
 192    * This should probably just go to the Local repo always, except
 193    * that our unit test suite is currently running RepoAuthoritative
 194    * tests with the compiled repo as the Central repo.
 195    */
 196   for (int repoId = numOpenRepos() - 1; repoId >= 0; --repoId) {
 197     try {
 198       RepoStmt stmt(*this);
 199       const auto& tbl = table(repoId, "GlobalData");
 200       stmt.prepare(
 201         folly::sformat(
 202           "SELECT data FROM {} WHERE key = @key;", tbl
 203         )
 204       );
 205       auto txn = RepoTxn{begin()};
 206       {
 207         RepoTxnQuery query(txn, stmt);
 208         auto key = std::string("config");
 209         query.bindStdString("@key", key);
 210         query.step();
 211         if (!query.row()) {
 212           throw RepoExc("Can't find key = 'config' in %s", tbl.c_str());
 213         }
 214         BlobDecoder decoder = query.getBlob(0, true);
 215         decoder(s_globalData);
 216         FTRACE(1, "GlobalData loaded from '{}':\n", repoName(repoId));
 217         FTRACE(1, "{}", show(s_globalData));
 218       }
 219
 220       if (readGlobalTables) {
 221         {
 222           RepoTxnQuery query(txn, stmt);
 223           auto key = std::string("arraytable");
 224           query.bindStdString("@key", key);
 225           query.step();
 226           if (!query.row()) {
 227             throw RepoExc("Can't find key = 'arraytable' in %s", tbl.c_str());
 228           }
 229           BlobDecoder decoder = query.getBlob(0, true);
 230           auto& arrayTypeTable = globalArrayTypeTable();
 231           decoder(arrayTypeTable);
 232           decoder(s_globalData.ConstantFunctions);
 233           decoder.assertDone();
 234         }
 235
 236         {
 237           RepoTxnQuery query(txn, stmt);
 238           auto key = std::string("autoloadmap");
 239           query.bindStdString("@key", key);
 240           query.step();
 241           if (!query.row()) {
 242             throw RepoExc("Can't find key = 'autoloadmap' in %s", tbl.c_str());
 243           }
 244           BlobDecoder decoder = query.getBlob(0, true);
 245           AutoloadHandler::setRepoAutoloadMap(
 246             RepoAutoloadMapBuilder::serde(decoder)
 247           );
 248         }
 249       }
 250
 251       txn.commit();
 252     } catch (RepoExc& e) {
 253       failures.push_back(repoName(repoId) + ": "  + e.msg());
 254       continue;
 255     }
 256
 257     // TODO: this should probably read out the other elements of the global data
 258     // which control Option or RuntimeOption values -- the others are read out
 259     // in an inconsistent and ad-hoc manner. But I don't understand their uses
 260     // and interactions well enough to feel comfortable fixing now.
 261     RuntimeOption::EnableIntrinsicsExtension =
 262       s_globalData.EnableIntrinsicsExtension;
 263     RuntimeOption::PHP7_Builtins            = s_globalData.PHP7_Builtins;
 264     RuntimeOption::PHP7_NoHexNumerics       = s_globalData.PHP7_NoHexNumerics;
 265     RuntimeOption::PHP7_Substr              = s_globalData.PHP7_Substr;
 266     RuntimeOption::EvalCheckPropTypeHints   = s_globalData.CheckPropTypeHints;
 267     RuntimeOption::EvalHackArrDVArrs        = true; // TODO(kshaunak): Clean up.
 268
 269     always_assert(!RO::EvalArrayProvenance);
 270     always_assert(!RO::EvalLogArrayProvenance);
 271
 272     RuntimeOption::EnableArgsInBacktraces = s_globalData.EnableArgsInBacktraces;
 273     RuntimeOption::EvalAbortBuildOnVerifyError =
 274       s_globalData.AbortBuildOnVerifyError;
 275     RuntimeOption::StrictArrayFillKeys      = s_globalData.StrictArrayFillKeys;
 276    if (s_globalData.HardGenericsUB) {
 277       RuntimeOption::EvalEnforceGenericsUB = 2;
 278     }
 279
 280     RuntimeOption::EvalIsCompatibleClsMethType =
 281       s_globalData.IsCompatibleClsMethType;
 282     RuntimeOption::EvalEmitClassPointers = s_globalData.EmitClassPointers;
 283     RuntimeOption::EvalEmitClsMethPointers = s_globalData.EmitClsMethPointers;
 284     RO::EvalForbidDynamicCallsWithAttr =
 285       s_globalData.ForbidDynamicCallsWithAttr;
 286
 287     RO::EvalHackArrCompatIsVecDictNotices =
 288       s_globalData.HackArrCompatIsVecDictNotices;
 289
 290     RO::EvalRaiseClassConversionWarning =
 291       s_globalData.RaiseClassConversionWarning;
 292     RO::EvalClassPassesClassname =
 293       s_globalData.ClassPassesClassname;
 294     RO::EvalClassnameNotices =
 295       s_globalData.ClassnameNotices;
 296     RO::EvalRaiseClsMethConversionWarning =
 297       s_globalData.RaiseClsMethConversionWarning;
 298
 299     RO::EvalNoticeOnCoerceForStrConcat =
 300       s_globalData.NoticeOnCoerceForStrConcat;
 301
 302     RO::EvalNoticeOnCoerceForBitOp =
 303       s_globalData.NoticeOnCoerceForBitOp;
 304
 305     RuntimeOption::ConstantFunctions.clear();
 306     for (auto const& elm : s_globalData.ConstantFunctions) {
 307       auto result = RuntimeOption::ConstantFunctions.emplace(
 308         elm.first, make_tv<KindOfUninit>()
 309       );
 310       if (result.second) {
 311         tvAsVariant(result.first->second) = unserialize_from_string(
 312           elm.second, VariableUnserializer::Type::Internal
 313         );
 314         tvAsVariant(result.first->second).setEvalScalar();
 315       }
 316     }
 317     return;
 318   }
 319
 320   if (failures.empty()) {
 321     std::fprintf(stderr, "No repo was loadable. Check all the possible repo "
 322                  "locations (Repo.Central.Path, HHVM_REPO_CENTRAL_PATH, and "
 323                  "$HOME/.hhvm.hhbc) to make sure one of them is a valid "
 324                  "sqlite3 HHVM repo built with this exact HHVM version.\n");
 325   } else {
 326     // We should always have a global data section in RepoAuthoritative
 327     // mode, or the repo is messed up.
 328     std::fprintf(stderr, "Failed to load RepoGlobalData:\n");
 329     for (auto& f : failures) {
 330       std::fprintf(stderr, "  %s\n", f.c_str());
 331     }
 332   }
 333
 334   assertx(Process::IsInMainThread());
 335   exit(1);
 336 }
 337
 338 void Repo::saveGlobalData(RepoGlobalData&& newData,
 339                           const RepoAutoloadMapBuilder& autoloadMapBuilder) {
 340   s_globalData = std::move(newData);
 341
 342   auto const repoId = repoIdForNewUnit(UnitOrigin::File);
 343   RepoStmt stmt(*this);
 344   stmt.prepare(
 345     folly::format(
 346       "INSERT INTO {} VALUES(@key, @data);", table(repoId, "GlobalData")
 347     ).str()
 348   );
 349   auto txn = RepoTxn{begin()};
 350   {
 351     RepoTxnQuery query(txn, stmt);
 352     auto key = std::string("config");
 353     query.bindStdString("@key", key);
 354     BlobEncoder encoder{true};
 355     encoder(s_globalData);
 356     query.bindBlob("@data", encoder, /* static */ true);
 357     query.exec();
 358   }
 359
 360   {
 361     RepoTxnQuery query(txn, stmt);
 362     auto key = std::string("arraytable");
 363     query.bindStdString("@key", key);
 364     BlobEncoder encoder{true};
 365     encoder(globalArrayTypeTable());
 366     encoder(s_globalData.ConstantFunctions);
 367     query.bindBlob("@data", encoder, /* static */ true);
 368     query.exec();
 369   }
 370
 371   {
 372     RepoTxnQuery query(txn, stmt);
 373     auto key = std::string("autoloadmap");
 374     query.bindStdString("@key", key);
 375     BlobEncoder encoder{true};
 376     autoloadMapBuilder.serde(encoder);
 377     query.bindBlob("@data", encoder, /* static */ true);
 378     query.exec();
 379   }
 380
 381   // TODO(#3521039): we could just put the litstr table in the same
 382   // blob as the above and delete LitstrRepoProxy.
 383   LitstrTable::get().setReading();
 384   LitstrRepoProxy::InsertLitstrStmt insertStmt{*this, repoId};
 385   LitstrTable::get().forEachLitstr(
 386     [this, &txn, &insertStmt](int i, const StringData* name) {
 387       insertStmt.insert(txn, i, name);
 388     });
 389
 390   txn.commit();
 391 }
 392
 393 std::unique_ptr<Unit> Repo::loadUnit(const folly::StringPiece name,
 394                                      const SHA1& sha1,
 395                                      const Native::FuncTable& nativeFuncs) {
 396   if (m_dbc == nullptr) return nullptr;
 397   tracing::Block _{
 398     "repo-load-unit", [&] { return tracing::Props{}.add("name", name); }
 399   };
 400   return m_urp.load(name, sha1, nativeFuncs);
 401 }
 402
 403 void Repo::forgetUnit(const std::string& name) {
 404   if (m_dbc == nullptr) {
 405     return;
 406   }
 407
 408   auto const repoId = repoIdForNewUnit(UnitOrigin::File);
 409   auto txn = RepoTxn{begin()};
 410   m_removeFileHash[repoId].remove(txn, name);
 411   txn.commit();
 412 }
 413
 414 std::vector<std::pair<std::string,SHA1>>
 415 Repo::enumerateUnits(int repoId, bool warn) {
 416   std::vector<std::pair<std::string,SHA1>> ret;
 417
 418   try {
 419     RepoStmt stmt(*this);
 420     stmt.prepare(folly::sformat(
 421                    "SELECT path, sha1 FROM {};",
 422                    table(repoId, "FileSha1"))
 423                 );
 424     auto txn = RepoTxn{begin()};
 425     RepoTxnQuery query(txn, stmt);
 426
 427     for (query.step(); query.row(); query.step()) {
 428       std::string path;
 429       SHA1 sha1;
 430
 431       query.getStdString(0, path);
 432       query.getSha1(1, sha1);
 433
 434       ret.emplace_back(path, sha1);
 435     }
 436
 437     txn.commit();
 438   } catch (RepoExc& e) {
 439     if (warn) {
 440       fprintf(stderr, "failed to enumerate units: %s\n", e.what());
 441     }
 442     // Ugh - the error is dropped.  At least we might have printed an error to
 443     // stderr.
 444   }
 445
 446   return ret;
 447 }
 448
 449 void Repo::InsertFileHashStmt::insert(RepoTxn& txn, const StringData* path,
 450                                       const SHA1& sha1) {
 451   if (!prepared()) {
 452     auto insertQuery = folly::sformat(
 453       "INSERT INTO {} VALUES(@path, @sha1);",
 454       m_repo.table(m_repoId, "FileSha1"));
 455     txn.prepare(*this, insertQuery);
 456   }
 457   RepoTxnQuery query(txn, *this);
 458   query.bindStaticString("@path", path);
 459   query.bindSha1("@sha1", sha1);
 460   query.exec();
 461 }
 462
 463 RepoStatus Repo::GetFileHashStmt::get(const char *path, SHA1& sha1) {
 464   try {
 465     auto txn = RepoTxn{m_repo.begin()};
 466     if (!prepared()) {
 467       auto selectQuery = folly::sformat(
 468         "SELECT f.sha1 "
 469         "FROM {} AS f, {} AS u "
 470         "WHERE path == @path AND f.sha1 == u.sha1 "
 471         "ORDER BY unitSn DESC LIMIT 1;",
 472         m_repo.table(m_repoId, "FileSha1"),
 473         m_repo.table(m_repoId, "Unit"));
 474       txn.prepare(*this, selectQuery);
 475     }
 476     RepoTxnQuery query(txn, *this);
 477     query.bindText("@path", path, strlen(path));
 478     query.step();
 479     if (!query.row()) {
 480       return RepoStatus::error;
 481     }
 482     query.getSha1(0, sha1);
 483     txn.commit();
 484     return RepoStatus::success;
 485   } catch (RepoExc& re) {
 486     return RepoStatus::error;
 487   }
 488 }
 489
 490 void Repo::RemoveFileHashStmt::remove(RepoTxn& txn, const std::string& path) {
 491   if (!prepared()) {
 492     auto insertQuery = folly::sformat(
 493       "DELETE FROM {} WHERE path == @path;",
 494       m_repo.table(m_repoId, "FileSha1"));
 495     txn.prepare(*this, insertQuery);
 496   }
 497   RepoTxnQuery query(txn, *this);
 498   query.bindStdString("@path", path);
 499   query.exec();
 500 }
 501
 502 std::optional<String> Repo::GetUnitPathStmt::get(int64_t unitSn) {
 503   auto txn = RepoTxn{m_repo.begin()};
 504   if (!prepared()) {
 505     auto selectQuery = folly::sformat(
 506       "SELECT f.path "
 507       "FROM {} AS u, {} AS f "
 508       "WHERE u.unitSn == @unitSn AND f.sha1 == u.sha1",
 509       m_repo.table(m_repoId, "Unit"),
 510       m_repo.table(m_repoId, "FileSha1"));
 511     txn.prepare(*this, selectQuery);
 512   }
 513   RepoTxnQuery query(txn, *this);
 514   query.bindInt64("@unitSn", unitSn);
 515   query.step();
 516   if (!query.row()) {
 517     return {};
 518   }
 519
 520   StringData* spath; query.getStaticString(0, spath);
 521   auto path = String(spath);
 522   txn.commit();
 523   return path;
 524 }
 525
 526 std::optional<String> Repo::findPath(int64_t unitSn, const std::string& root) {
 527   if (m_dbc == nullptr) {
 528     return {};
 529   }
 530
 531   int repoId;
 532   for (repoId = numOpenRepos() - 1; repoId >= 0; --repoId) {
 533     auto maybeRelPath = m_getUnitPath[repoId].get(unitSn);
 534     if (maybeRelPath.has_value()) {
 535       auto relPath = maybeRelPath.value();
 536       String path;
 537       if (relPath[0] == '/') {
 538         path = relPath;
 539       } else {
 540         path = root + relPath;
 541       }
 542       TRACE(3, "Repo loaded file path for '%ld' from '%s'\n",
 543                 unitSn, repoName(repoId).c_str());
 544       return path;
 545     }
 546   }
 547   TRACE(3, "Repo file path: error loading '%ld'\n", unitSn);
 548   return {};
 549 }
 550
 551 RepoStatus Repo::GetUnitStmt::get(const char* path, int64_t& unitSn) {
 552   try {
 553     auto txn = RepoTxn{m_repo.begin()};
 554     if (!prepared()) {
 555       auto selectQuery = folly::sformat(
 556         "SELECT u.unitSn "
 557         "FROM {} AS f, {} AS u "
 558         "WHERE f.path == @path AND f.sha1 == u.sha1",
 559         m_repo.table(m_repoId, "FileSha1"),
 560         m_repo.table(m_repoId, "Unit"));
 561       txn.prepare(*this, selectQuery);
 562     }
 563     RepoTxnQuery query(txn, *this);
 564     query.bindText("@path", path, strlen(path));
 565     query.step();
 566     if (!query.row()) {
 567       return RepoStatus::error;
 568     }
 569     int unitSn_;               /**/ query.getInt(0, unitSn_);
 570     unitSn = unitSn_;
 571     txn.commit();
 572     return RepoStatus::success;
 573   } catch (RepoExc& re) {
 574     return RepoStatus::error;
 575   }
 576 }
 577
 578 RepoStatus Repo::findUnit(const char* path, const std::string& root,
 579                           int64_t& unitSn) {
 580   if (m_dbc == nullptr) {
 581     return RepoStatus::error;
 582   }
 583   int repoId;
 584   for (repoId = numOpenRepos() - 1; repoId >= 0; --repoId) {
 585     if (*path == '/' && !root.empty() &&
 586         !strncmp(root.c_str(), path, root.size()) &&
 587         (m_getUnit[repoId].get(path + root.size(), unitSn) ==
 588          RepoStatus::success)) {
 589       TRACE(3, "Repo loaded unit for '%s' from '%s'\n",
 590                path + root.size(), repoName(repoId).c_str());
 591       return RepoStatus::success;
 592     }
 593     if (m_getUnit[repoId].get(path, unitSn) == RepoStatus::success) {
 594       TRACE(3, "Repo loaded unit for '%s' from '%s'\n",
 595                 path, repoName(repoId).c_str());
 596       return RepoStatus::success;
 597     }
 598   }
 599   TRACE(3, "Repo unit: error loading '%s'\n", path);
 600   return RepoStatus::error;
 601 }
 602
 603 RepoStatus Repo::findFile(const char *path, const std::string& root,
 604                           SHA1& sha1) {
 605   tracing::Block _{
 606     "repo-find-file",
 607     [&] {
 608       return tracing::Props{}
 609         .add("path", path)
 610         .add("root", root);
 611     }
 612   };
 613
 614   if (m_dbc == nullptr) {
 615     return RepoStatus::error;
 616   }
 617   int repoId;
 618   for (repoId = numOpenRepos() - 1; repoId >= 0; --repoId) {
 619     if (*path == '/' && !root.empty() &&
 620         !strncmp(root.c_str(), path, root.size()) &&
 621         (m_getFileHash[repoId].get(path + root.size(), sha1) ==
 622          RepoStatus::success)) {
 623       TRACE(3, "Repo loaded file hash for '%s' from '%s'\n",
 624                path + root.size(), repoName(repoId).c_str());
 625       return RepoStatus::success;
 626     }
 627     if (m_getFileHash[repoId].get(path, sha1) == RepoStatus::success) {
 628       TRACE(3, "Repo loaded file hash for '%s' from '%s'\n",
 629                 path, repoName(repoId).c_str());
 630       return RepoStatus::success;
 631     }
 632   }
 633   TRACE(3, "Repo file hash: error loading '%s'\n", path);
 634   return RepoStatus::error;
 635 }
 636
 637 RepoStatus Repo::insertSha1(UnitOrigin unitOrigin, UnitEmitter* ue,
 638                            RepoTxn& txn) {
 639   const StringData* path = ue->m_filepath;
 640   const SHA1& sha1 = ue->sha1();
 641   int repoId = repoIdForNewUnit(unitOrigin);
 642   if (repoId == RepoIdInvalid) {
 643     return RepoStatus::error;
 644   }
 645   try {
 646     m_insertFileHash[repoId].insert(txn, path, sha1);
 647     return RepoStatus::success;
 648   } catch (RepoExc& re) {
 649     TRACE(3, "Failed to commit sha1 for '%s' to '%s': %s\n",
 650               path->data(), repoName(repoId).c_str(), re.msg().c_str());
 651     return RepoStatus::error;
 652   }
 653 }
 654
 655 void Repo::commitSha1(UnitOrigin unitOrigin, UnitEmitter* ue) {
 656   try {
 657     auto txn = RepoTxn{begin()};
 658     RepoStatus err = insertSha1(unitOrigin, ue, txn);
 659     if (err == RepoStatus::success) {
 660       txn.commit();
 661     }
 662   } catch (RepoExc& re) {
 663     tracing::addPointNoTrace("sha1-commit-exn");
 664     int repoId = repoIdForNewUnit(unitOrigin);
 665     if (repoId != RepoIdInvalid) {
 666       TRACE(3, "Failed to commit sha1 for '%s' to '%s': %s\n",
 667                ue->m_filepath->data(), repoName(repoId).c_str(),
 668                re.msg().c_str());
 669     }
 670   }
 671 }
 672
 673 std::string Repo::table(int repoId, const char* tablePrefix) {
 674   return folly::sformat(
 675     "{}.{}_{}", dbName(repoId), tablePrefix, repoSchemaId());
 676 }
 677
 678 void Repo::exec(const std::string& sQuery) {
 679   RepoStmt stmt(*this);
 680   stmt.prepare(sQuery);
 681   RepoQuery query(stmt);
 682   query.exec();
 683 }
 684
 685 RepoTxn Repo::begin() {
 686   if (m_txDepth > 0) {
 687     m_txDepth++;
 688     return RepoTxn{*this};
 689   }
 690   if (debug) {
 691     // Verify start state.
 692     always_assert(m_txDepth == 0);
 693     always_assert(!m_rollback);
 694     if (true) {
 695       // Bypass RepoQuery, in order to avoid triggering exceptions.
 696       int rc = sqlite3_step(m_rollbackStmt.get());
 697       switch (rc) {
 698       case SQLITE_DONE:
 699       case SQLITE_ROW:
 700         not_reached();
 701       default:
 702         break;
 703       }
 704     } else {
 705       bool rollbackFailed = false;
 706       try {
 707         RepoQuery query(m_rollbackStmt);
 708         query.exec();
 709       } catch (RepoExc& re) {
 710         rollbackFailed = true;
 711       }
 712       always_assert(rollbackFailed);
 713     }
 714   }
 715   RepoQuery query(m_beginStmt);
 716   query.exec();
 717   m_txDepth++;
 718
 719   return RepoTxn(*this);
 720 }
 721
 722 void Repo::txPop() {
 723   // We mix the concept of rollback with a normal commit so that if we try to
 724   // rollback an inner transaction we eventually end up rolling back the outer
 725   // transaction instead (Sqlite doesn't support rolling back partial
 726   // transactions).
 727   assertx(m_txDepth > 0);
 728   if (m_txDepth > 1) {
 729     m_txDepth--;
 730     return;
 731   }
 732   if (!m_rollback) {
 733     RepoQuery query(m_commitStmt);
 734     query.exec();
 735   } else {
 736     // We're in the outermost transaction - so clear the rollback flag.
 737     m_rollback = false;
 738     RepoQuery query(m_rollbackStmt);
 739     try {
 740       query.exec();
 741     } catch (RepoExc& ex) {
 742       /*
 743        * Having a rollback fail is actually a normal, expected case,
 744        * so just swallow this.
 745        *
 746        * In particular, according to the docs, if we got an I/O error
 747        * while doing a commit, the rollback will often fail with "no
 748        * transaction in progress", because the commit will have
 749        * automatically been rolled back.  Recommended practice is
 750        * still to execute a rollback statement and ignore the error.
 751        */
 752       TRACE(4, "repo rollback failure: %s\n", ex.what());
 753     }
 754   }
 755   // Decrement depth after query execution, in case an exception occurs during
 756   // commit.  This allows for subsequent rollback of the failed commit.
 757   m_txDepth--;
 758 }
 759
 760 void Repo::rollback() {
 761   m_rollback = true;
 762   // NOTE: A try/catch isn't necessary - txPop() handles rollback as a nothrow.
 763   txPop();
 764 }
 765
 766 void Repo::commit() {
 767   txPop();
 768 }
 769
 770 RepoStatus Repo::insertUnit(UnitEmitter* ue, UnitOrigin unitOrigin,
 771                             RepoTxn& txn, bool usePreAllocatedUnitSn) {
 772   if (insertSha1(unitOrigin, ue, txn) == RepoStatus::error ||
 773       ue->insert(unitOrigin, txn, usePreAllocatedUnitSn) == RepoStatus::error) {
 774     return RepoStatus::error;
 775   }
 776   return RepoStatus::success;
 777 }
 778
 779 void Repo::commitUnit(UnitEmitter* ue, UnitOrigin unitOrigin,
 780                       bool usePreAllocatedUnitSn) {
 781   if (!RuntimeOption::RepoCommit || ue->m_ICE) return;
 782
 783   tracing::Block _{
 784     "repo-commit-unit",
 785     [&] { return tracing::Props{}.add("filename", ue->m_filepath); }
 786   };
 787
 788   try {
 789     commitSha1(unitOrigin, ue);
 790     ue->commit(unitOrigin, usePreAllocatedUnitSn);
 791   } catch (const std::exception& e) {
 792     TRACE(0, "unexpected exception in commitUnit: %s\n",
 793           e.what());
 794     assertx(false);
 795   }
 796 }
 797
 798 void Repo::connect() {
 799   m_numOpenRepos = 0;
 800   if (initCentral()) {
 801     m_numOpenRepos++;
 802   }
 803   if (initLocal()) {
 804     m_numOpenRepos++;
 805   }
 806 }
 807
 808 void Repo::disconnect() noexcept {
 809   if (m_dbc != nullptr) {
 810     sqlite3_close_v2(m_dbc);
 811     m_dbc = nullptr;
 812     m_localWritable = false;
 813     m_centralWritable = false;
 814   }
 815   m_numOpenRepos = 0;
 816 }
 817
 818 bool Repo::initCentral() {
 819   std::string error;
 820
 821   if (RuntimeOption::RepoCentralMode == RepoMode::Closed &&
 822       RuntimeOption::RepoLocalMode == RepoMode::Closed) {
 823     return false;
 824   }
 825
 826   assertx(m_dbc == nullptr);
 827   auto tryPath = [this, &error](const char* path) {
 828     std::string subErr;
 829     if (openCentral(path, subErr) == RepoStatus::error) {
 830       folly::format(&error, "  {}\n", subErr.empty() ? path : subErr);
 831       return false;
 832     }
 833     return true;
 834   };
 835
 836   auto fail_no_repo = [&error] {
 837     error = "Failed to initialize central HHBC repository:\n" + error;
 838     // Database initialization failed; this is an unrecoverable state.
 839     Logger::Error(error);
 840
 841     if (Process::IsInMainThread()) {
 842       exit(1);
 843     }
 844     always_assert_flog(false, "{}", error);
 845   };
 846
 847   // Try Repo.Central.Path
 848   if (!RuntimeOption::RepoCentralPath.empty() &&
 849       tryPath(RuntimeOption::RepoCentralPath.c_str())) {
 850     return true;
 851   }
 852
 853   // Try HHVM_REPO_CENTRAL_PATH
 854   const char* HHVM_REPO_CENTRAL_PATH = getenv("HHVM_REPO_CENTRAL_PATH");
 855   if (HHVM_REPO_CENTRAL_PATH != nullptr &&
 856       tryPath(HHVM_REPO_CENTRAL_PATH)) {
 857     return true;
 858   }
 859
 860   if (!RuntimeOption::RepoAllowFallbackPath) fail_no_repo();
 861
 862   // Try "$HOME/.hhvm.hhbc".
 863   char* HOME = getenv("HOME");
 864   if (HOME != nullptr) {
 865     std::string centralPath = HOME;
 866     centralPath += "/.hhvm.hhbc";
 867     if (tryPath(centralPath.c_str())) {
 868       return true;
 869     }
 870   }
 871
 872 #ifndef _WIN32
 873   // Try the equivalent of "$HOME/.hhvm.hhbc", but look up the home directory
 874   // in the password database.
 875   {
 876     auto buf = PasswdBuffer{};
 877     passwd* pw;
 878     auto err = getpwuid_r(getuid(), &buf.ent, buf.data.get(), buf.size, &pw);
 879     if (err == 0 && pw != nullptr &&
 880         (HOME == nullptr || strcmp(HOME, pw->pw_dir))) {
 881       std::string centralPath = pw->pw_dir;
 882       centralPath += "/.hhvm.hhbc";
 883       if (tryPath(centralPath.c_str())) {
 884         return true;
 885       }
 886     }
 887   }
 888 #else // _WIN32
 889   // Try "$HOMEDRIVE$HOMEPATH/.hhvm.hhbc"
 890   char* HOMEDRIVE = getenv("HOMEDRIVE");
 891   if (HOMEDRIVE != nullptr) {
 892     char* HOMEPATH = getenv("HOMEPATH");
 893     if (HOMEPATH != nullptr) {
 894       std::string centralPath = HOMEDRIVE;
 895       centralPath += HOMEPATH;
 896       centralPath += "\\.hhvm.hhbc";
 897       if (tryPath(centralPath.c_str()))
 898         return true;
 899     }
 900   }
 901 #endif
 902
 903   fail_no_repo();
 904   return false;
 905 }
 906
 907 namespace {
 908 /*
 909  * Convert the permission bits from the given stat struct to an ls-style
 910  * rwxrwxrwx format.
 911  */
 912 std::string showPermissions(const struct stat& s) {
 913   static const std::pair<int, char> bits[] = {
 914     {S_IRUSR, 'r'}, {S_IWUSR, 'w'}, {S_IXUSR, 'x'},
 915     {S_IRGRP, 'r'}, {S_IWGRP, 'w'}, {S_IXGRP, 'x'},
 916     {S_IROTH, 'r'}, {S_IWOTH, 'w'}, {S_IXOTH, 'x'},
 917   };
 918   std::string ret;
 919   ret.reserve(sizeof(bits) / sizeof(bits[0]));
 920
 921   for (auto pair : bits) {
 922     ret += (s.st_mode & pair.first) ? pair.second : '-';
 923   }
 924   return ret;
 925 }
 926
 927 /*
 928  * Return the name of the user with the given id.
 929  */
 930 std::string uidToName(uid_t uid) {
 931 #ifndef _WIN32
 932   auto buf = PasswdBuffer{};
 933   passwd* pw;
 934
 935   auto err = getpwuid_r(uid, &buf.ent, buf.data.get(), buf.size, &pw);
 936   if (err != 0) return folly::errnoStr(errno);
 937   if (pw == nullptr) return "user does not exist";
 938   return pw->pw_name;
 939 #else
 940   return "<unsupported>";
 941 #endif
 942 }
 943
 944 /*
 945  * Return the uid of the user with the given name.
 946  */
 947 uid_t nameToUid(const std::string& name) {
 948 #ifndef _WIN32
 949   auto buf = PasswdBuffer{};
 950   passwd* pw;
 951
 952   auto err = getpwnam_r(name.c_str(), &buf.ent, buf.data.get(), buf.size, &pw);
 953   if (err != 0 || pw == nullptr) return -1;
 954   return pw->pw_uid;
 955 #else
 956   return -1;
 957 #endif
 958 }
 959
 960 /*
 961  * Return the name of the group with the given id.
 962  */
 963 std::string gidToName(gid_t gid) {
 964 #ifndef _WIN32
 965   auto buf = GroupBuffer{};
 966   group* grp;
 967
 968   auto err = getgrgid_r(gid, &buf.ent, buf.data.get(), buf.size, &grp);
 969   if (err != 0) return folly::errnoStr(errno);
 970   if (grp == nullptr) return "group does not exist";
 971   return grp->gr_name;
 972 #else
 973   return "<unsupported>";
 974 #endif
 975 }
 976
 977 /*
 978  * Return the gid of the group with the given name.
 979  */
 980 gid_t nameToGid(const std::string& name) {
 981 #ifndef _WIN32
 982   auto buf = GroupBuffer{};
 983   group* grp;
 984
 985   auto err = getgrnam_r(name.c_str(), &buf.ent, buf.data.get(), buf.size, &grp);
 986   if (err != 0 || grp == nullptr) return -1;
 987   return grp->gr_gid;
 988 #else
 989   return -1;
 990 #endif
 991 }
 992
 993 void setCentralRepoFileMode(const std::string& path) {
 994   // These runtime options are all best-effort, so we don't care if any of the
 995   // operations fail.
 996
 997   if (auto const mode = RuntimeOption::RepoCentralFileMode) {
 998     chmod(path.c_str(), mode);
 999   }
1000
1001   if (!RuntimeOption::RepoCentralFileUser.empty()) {
1002     auto const uid = nameToUid(RuntimeOption::RepoCentralFileUser);
1003     chown(path.c_str(), uid, -1);
1004   }
1005
1006   if (!RuntimeOption::RepoCentralFileGroup.empty()) {
1007     auto const gid = nameToGid(RuntimeOption::RepoCentralFileGroup);
1008     chown(path.c_str(), -1, gid);
1009   }
1010 }
1011 }
1012
1013 namespace {
1014   std::mutex s_sqliteCreateMutex;
1015 }
1016
1017 RepoStatus Repo::openCentral(const char* rawPath, std::string& errorMsg) {
1018   std::string repoPath = rawPath;
1019   replacePlaceholders(repoPath);
1020   // SQLITE_OPEN_NOMUTEX specifies that the connection be opened such
1021   // that no mutexes are used to protect the database connection from other
1022   // threads.  However, multiple connections can still be used concurrently,
1023   // because SQLite as a whole is thread-safe.
1024   // Because we attach the local repo to the central one if either of them
1025   // is readwrite we need to open the central one with readwrite.
1026   int flags = SQLITE_OPEN_NOMUTEX;
1027   if (RuntimeOption::RepoCentralMode == RepoMode::ReadWrite ||
1028       RuntimeOption::RepoLocalMode == RepoMode::ReadWrite) {
1029     flags |= SQLITE_OPEN_READWRITE;
1030     if (RuntimeOption::RepoCentralMode == RepoMode::ReadWrite) {
1031       flags |= SQLITE_OPEN_CREATE;
1032     }
1033   } else {
1034     flags |= SQLITE_OPEN_READONLY;
1035   }
1036
1037   {
1038     // sqlite can't handle concurrent SQLITE_OPEN_CREATEs
1039     std::lock_guard<std::mutex> l(s_sqliteCreateMutex);
1040     if (int err = sqlite3_open_v2(repoPath.c_str(), &m_dbc, flags, nullptr)) {
1041       TRACE(1, "Repo::%s() failed to open candidate central repo '%s'\n",
1042                __func__, repoPath.c_str());
1043       errorMsg = folly::format("Failed to open {}: {} - {}",
1044                                repoPath, err, sqlite3_errmsg(m_dbc)).str();
1045       return RepoStatus::error;
1046     }
1047   }
1048
1049   if (RuntimeOption::RepoBusyTimeoutMS) {
1050     sqlite3_busy_timeout(m_dbc, RuntimeOption::RepoBusyTimeoutMS);
1051   }
1052   try {
1053     m_beginStmt.prepare("BEGIN TRANSACTION;");
1054     m_rollbackStmt.prepare("ROLLBACK;");
1055     m_commitStmt.prepare("COMMIT;");
1056     pragmas(RepoIdCentral);
1057   } catch (RepoExc& re) {
1058     TRACE(1, "Repo::%s() failed to initialize connection to canditate repo"
1059              " '%s': %s\n", __func__, repoPath.c_str(), re.what());
1060     errorMsg = folly::format("Failed to initialize connection to {}: {}",
1061                              repoPath, re.what()).str();
1062     return RepoStatus::error;
1063   }
1064
1065   // sqlite3_open_v2() will silently open in read-only mode if file permissions
1066   // prevent writing.  Therefore, tell initSchema() to verify that the database
1067   // is writable.
1068   bool shouldBeWritable = RuntimeOption::RepoCentralMode == RepoMode::ReadWrite;
1069   bool centralWritable = shouldBeWritable;
1070   if (initSchema(RepoIdCentral, centralWritable, errorMsg) == RepoStatus::error
1071       || (!centralWritable && shouldBeWritable)) {
1072     TRACE(1, "Repo::initSchema() failed for candidate central repo '%s'\n",
1073              repoPath.c_str());
1074     struct stat repoStat;
1075     std::string statStr;
1076     if (stat(repoPath.c_str(), &repoStat) == 0) {
1077       statStr = folly::sformat("{} {}:{}",
1078                                showPermissions(repoStat),
1079                                uidToName(repoStat.st_uid),
1080                                gidToName(repoStat.st_gid));
1081     } else {
1082       statStr = folly::errnoStr(errno);
1083     }
1084     errorMsg = folly::format("Failed to initialize schema in {}({}): {}",
1085                              repoPath, statStr, errorMsg).str();
1086     return RepoStatus::error;
1087   }
1088   m_centralRepo = repoPath;
1089   m_centralWritable = centralWritable;
1090   setCentralRepoFileMode(repoPath);
1091   TRACE(1, "Central repo: '%s'\n", m_centralRepo.c_str());
1092   return RepoStatus::success;
1093 }
1094
1095 bool Repo::initLocal() {
1096   if (RuntimeOption::RepoLocalMode != RepoMode::Closed) {
1097     bool isWritable;
1098     if (RuntimeOption::RepoLocalMode == RepoMode::ReadWrite) {
1099       isWritable = true;
1100     } else {
1101       assertx(RuntimeOption::RepoLocalMode == RepoMode::ReadOnly);
1102       isWritable = false;
1103     }
1104
1105     if (!RuntimeOption::RepoLocalPath.empty()) {
1106       if (attachLocal(RuntimeOption::RepoLocalPath.c_str(), isWritable)) {
1107         return true;
1108       }
1109       if (!s_deleteLocalOnFailure) {
1110         return false;
1111       }
1112
1113       unlink(RuntimeOption::RepoLocalPath.c_str());
1114       s_deleteLocalOnFailure = false;
1115       Logger::Warning("Deleting local repo because it was corrupt");
1116       return attachLocal(RuntimeOption::RepoLocalPath.c_str(), true);
1117     } else if (RuntimeOption::RepoAllowFallbackPath) {
1118       if (!RuntimeOption::ServerExecutionMode()) {
1119         std::string cliRepo = s_cliFile;
1120         if (!cliRepo.empty()) {
1121           cliRepo += ".hhbc";
1122         }
1123         return attachLocal(cliRepo.c_str(), isWritable);
1124       } else {
1125         return attachLocal("hhvm.hhbc", isWritable);
1126       }
1127     }
1128   }
1129
1130   return false;
1131 }
1132
1133 bool Repo::attachLocal(const char* path, bool isWritable) {
1134   std::string repoPath = path;
1135   replacePlaceholders(repoPath);
1136   if (!isWritable) {
1137     // Make sure the repo exists before attaching it, in order to avoid
1138     // creating a read-only repo.
1139     struct stat buf;
1140     if (!strchr(repoPath.c_str(), ':') &&
1141         stat(repoPath.c_str(), &buf) != 0) {
1142       return false;
1143     }
1144   }
1145   try {
1146     m_localWritable = isWritable;
1147     auto attachQuery = folly::sformat(
1148       "ATTACH DATABASE '{}' as {};", repoPath, dbName(RepoIdLocal));
1149     exec(attachQuery);
1150     pragmas(RepoIdLocal);
1151   } catch (RepoExc& re) {
1152     // Failed to run pragmas on local DB - ignored
1153     m_localWritable = false;
1154     return false;
1155   }
1156
1157   std::string error;
1158   if (initSchema(RepoIdLocal, isWritable, error) == RepoStatus::error) {
1159     FTRACE(1, "Local repo {} failed to init schema: {}\n", repoPath, error);
1160     return false;
1161   }
1162   m_localRepo = repoPath;
1163   TRACE(1, "Local repo: '%s' (read%s)\n",
1164            m_localRepo.c_str(), m_localWritable ? "-write" : "-only");
1165   return true;
1166 }
1167
1168 void Repo::pragmas(int repoId) {
1169   // Valid synchronous values: 0 (OFF), 1 (NORMAL), 2 (FULL).
1170   static const int synchronous = 0;
1171   setIntPragma(repoId, "synchronous", synchronous);
1172   setIntPragma(repoId, "cache_size", 20);
1173   // Valid journal_mode values: delete, truncate, persist, memory, wal, off.
1174   setTextPragma(repoId, "journal_mode", RuntimeOption::RepoJournal.c_str());
1175 }
1176
1177 void Repo::getIntPragma(int repoId, const char* name, int& val) {
1178   auto pragmaQuery = folly::sformat("PRAGMA {}.{};", dbName(repoId), name);
1179   RepoStmt stmt(*this);
1180   stmt.prepare(pragmaQuery);
1181   RepoQuery query(stmt);
1182   query.step();
1183   query.getInt(0, val);
1184 }
1185
1186 void Repo::setIntPragma(int repoId, const char* name, int val) {
1187   // Read first to see if a write can be avoided
1188   int oldval = -1;
1189   getIntPragma(repoId, name, oldval);
1190   if (val == oldval) return;
1191
1192   // Pragma writes must be executed outside transactions, since they may change
1193   // transaction behavior.
1194   auto pragmaQuery = folly::sformat(
1195     "PRAGMA {}.{} = {};", dbName(repoId), name, val);
1196   exec(pragmaQuery);
1197   if (debug) {
1198     // Verify that the pragma had the desired effect.
1199     int newval = -1;
1200     getIntPragma(repoId, name, newval);
1201     if (newval != val) {
1202       throw RepoExc("Unexpected PRAGMA %s.%s value: %d\n",
1203                     dbName(repoId), name, newval);
1204     }
1205   }
1206 }
1207
1208 void Repo::getTextPragma(int repoId, const char* name, std::string& val) {
1209   auto pragmaQuery = folly::sformat("PRAGMA {}.{};", dbName(repoId), name);
1210   RepoStmt stmt(*this);
1211   stmt.prepare(pragmaQuery);
1212   RepoQuery query(stmt);
1213   const char* s;
1214   query.step();
1215   query.getText(0, s);
1216   val = s;
1217 }
1218
1219 void Repo::setTextPragma(int repoId, const char* name, const char* val) {
1220   // Read first to see if a write can be avoided
1221   std::string oldval = "?";
1222   getTextPragma(repoId, name, oldval);
1223   if (!strcmp(oldval.c_str(), val)) return;
1224   // Pragma writes must be executed outside transactions, since they may change
1225   // transaction behavior.
1226   auto pragmaQuery = folly::sformat(
1227     "PRAGMA {}.{} = {};", dbName(repoId), name, val);
1228   exec(pragmaQuery);
1229   if (debug) {
1230     // Verify that the pragma had the desired effect.
1231     std::string newval = "?";
1232     getTextPragma(repoId, name, newval);
1233     if (strcmp(newval.c_str(), val)) {
1234       // If the db is in memory, journal mode will stick at "memory"
1235       // unless its turned off. Ignore attempts to change it to
1236       // something else.
1237       if (!strcmp(name, "journal_mode") &&
1238           !strcmp(newval.c_str(), "memory") &&
1239           strcmp(val, "off")) {
1240         return;
1241       }
1242       throw RepoExc("Unexpected PRAGMA %s.%s value: %s (should be %s)\n",
1243                     dbName(repoId), name, newval.c_str(), val);
1244     }
1245   }
1246 }
1247
1248 RepoStatus Repo::initSchema(int repoId, bool& isWritable,
1249                             std::string& errorMsg) {
1250   if (!schemaExists(repoId)) {
1251     if (createSchema(repoId, errorMsg) == RepoStatus::error) {
1252       // Check whether this failure is due to losing the schema
1253       // initialization race with another process.
1254       if (!schemaExists(repoId)) {
1255         return RepoStatus::error;
1256       }
1257     } else {
1258       // createSchema() successfully wrote to the database, so no further
1259       // verification is necessary.
1260       return RepoStatus::success;
1261     }
1262   }
1263   if (isWritable) {
1264     isWritable = writable(repoId);
1265   }
1266   return RepoStatus::success;
1267 }
1268
1269 bool Repo::schemaExists(int repoId) {
1270   try {
1271     auto txn = RepoTxn{begin()};
1272     auto selectQuery = folly::sformat(
1273       "SELECT product FROM {};", table(repoId, "magic"));
1274     RepoStmt stmt(*this);
1275     // If the DB is 'new' and hasn't been initialized yet then we expect this
1276     // prepare() to fail.
1277     stmt.prepare(selectQuery);
1278     // This SHOULDN'T fail - we create the table under a transaction - so if it
1279     // exists then it should have our magic value.
1280     RepoTxnQuery query(txn, stmt);
1281     query.step();
1282     const char* text; /**/ query.getText(0, text);
1283     if (strcmp(kMagicProduct, text)) {
1284       return false;
1285     }
1286     txn.commit();
1287   } catch (RepoExc& re) {
1288     return false;
1289   }
1290   return true;
1291 }
1292
1293 RepoStatus Repo::createSchema(int repoId, std::string& errorMsg) {
1294   try {
1295     auto txn = RepoTxn{begin()};
1296     {
1297       auto createQuery = folly::sformat(
1298         "CREATE TABLE {} (product TEXT);", table(repoId, "magic"));
1299       txn.exec(createQuery);
1300
1301       auto insertQuery = folly::sformat(
1302         "INSERT INTO {} VALUES('{}');", table(repoId, "magic"), kMagicProduct);
1303       txn.exec(insertQuery);
1304     }
1305     {
1306       auto createQuery = folly::sformat(
1307         "CREATE TABLE {} (path TEXT, sha1 BLOB, UNIQUE(path, sha1));",
1308         table(repoId, "FileSha1"));
1309       txn.exec(createQuery);
1310
1311       auto indexQuery = folly::sformat(
1312         "CREATE INDEX {}_sha1_index ON {}_{} (sha1);",
1313         table(repoId, "FileSha1"),
1314         "FileSha1",
1315         repoSchemaId());
1316       txn.exec(indexQuery);
1317     }
1318     txn.exec(folly::sformat("CREATE TABLE {} (key TEXT, data BLOB);",
1319                            table(repoId, "GlobalData")));
1320     m_urp.createSchema(repoId, txn);
1321     m_pcrp.createSchema(repoId, txn);
1322     m_rrp.createSchema(repoId, txn);
1323     m_frp.createSchema(repoId, txn);
1324     m_lsrp.createSchema(repoId, txn);
1325
1326     txn.commit();
1327   } catch (RepoExc& re) {
1328     errorMsg = re.what();
1329     return RepoStatus::error;
1330   }
1331   return RepoStatus::success;
1332 }
1333
1334 bool Repo::writable(int repoId) {
1335   switch (sqlite3_db_readonly(m_dbc, dbName(repoId))) {
1336     case 0:  return true;
1337     case 1:  return false;
1338     case -1: return false;
1339     default: break;
1340   }
1341   always_assert(false);
1342 }
1343
1344 //////////////////////////////////////////////////////////////////////
1345
1346 bool batchCommitWithoutRetry(const std::vector<std::unique_ptr<UnitEmitter>>& ues,
1347                              bool usePreAllocatedUnitSn) {
1348   auto& repo = Repo::get();
1349
1350   bool err = false;
1351   {
1352     auto txn = RepoTxn{repo.begin()};
1353
1354     for (auto& ue : ues) {
1355       assertx(!usePreAllocatedUnitSn || ue->m_sn != -1);
1356       if (repo.insertUnit(ue.get(), UnitOrigin::File, txn, usePreAllocatedUnitSn) ==
1357           RepoStatus::error) {
1358         err = true;
1359         break;
1360       }
1361     }
1362     if (!err) {
1363       txn.commit();
1364     }
1365   }
1366
1367   return err;
1368 }
1369
1370 void batchCommit(const std::vector<std::unique_ptr<UnitEmitter>>& ues,
1371                  bool usePreAllocatedUnitSn) {
1372   auto& repo = Repo::get();
1373
1374   // Attempt batch commit.  This can legitimately fail due to multiple input
1375   // files having identical contents.
1376   bool err = batchCommitWithoutRetry(ues, usePreAllocatedUnitSn);
1377
1378   // Commit units individually if an error occurred during batch commit.
1379   if (err) {
1380     for (auto& ue : ues) {
1381       repo.commitUnit(ue.get(), UnitOrigin::File, usePreAllocatedUnitSn);
1382     }
1383   }
1384 }
1385
1386 //////////////////////////////////////////////////////////////////////
1387
1388 }