hphp/runtime/vm/unit-emitter.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/unit-emitter.h"
  18
  19 #include "hphp/compiler/builtin_symbols.h"
  20 #include "hphp/parser/location.h"
  21 #include "hphp/system/systemlib.h"
  22
  23 #include "hphp/runtime/base/array-data.h"
  24 #include "hphp/runtime/base/attr.h"
  25 #include "hphp/runtime/base/file-util.h"
  26 #include "hphp/runtime/base/runtime-option.h"
  27 #include "hphp/runtime/base/static-string-table.h"
  28 #include "hphp/runtime/base/typed-value.h"
  29 #include "hphp/runtime/base/repo-auth-type-array.h"
  30 #include "hphp/runtime/base/variable-serializer.h"
  31
  32 #include "hphp/runtime/ext/std/ext_std_variable.h"
  33
  34 #include "hphp/runtime/vm/blob-helper.h"
  35 #include "hphp/runtime/vm/disas.h"
  36 #include "hphp/runtime/vm/func.h"
  37 #include "hphp/runtime/vm/func-emitter.h"
  38 #include "hphp/runtime/vm/jit/perf-counters.h"
  39 #include "hphp/runtime/vm/litstr-table.h"
  40 #include "hphp/runtime/vm/native.h"
  41 #include "hphp/runtime/vm/preclass.h"
  42 #include "hphp/runtime/vm/preclass-emitter.h"
  43 #include "hphp/runtime/vm/record-emitter.h"
  44 #include "hphp/runtime/vm/repo.h"
  45 #include "hphp/runtime/vm/repo-autoload-map-builder.h"
  46 #include "hphp/runtime/vm/repo-helpers.h"
  47 #include "hphp/runtime/vm/unit.h"
  48 #include "hphp/runtime/vm/verifier/check.h"
  49
  50 #include "hphp/util/alloc.h"
  51 #include "hphp/util/logger.h"
  52 #include "hphp/util/read-only-arena.h"
  53 #include "hphp/util/sha1.h"
  54 #include "hphp/util/trace.h"
  55
  56 #include <boost/algorithm/string/predicate.hpp>
  57
  58 #include <folly/Memory.h>
  59 #include <folly/FileUtil.h>
  60
  61 #include <algorithm>
  62 #include <cstdio>
  63 #include <memory>
  64 #include <string>
  65 #include <utility>
  66 #include <vector>
  67
  68 namespace HPHP {
  69 ///////////////////////////////////////////////////////////////////////////////
  70
  71 TRACE_SET_MOD(hhbc);
  72
  73 using MergeKind = Unit::MergeKind;
  74
  75 ///////////////////////////////////////////////////////////////////////////////
  76
  77 using BytecodeArena = ReadOnlyArena<VMColdAllocator<char>>;
  78 static BytecodeArena& get_readonly_arena() {
  79   static BytecodeArena arena(RuntimeOption::EvalHHBCArenaChunkSize);
  80   return arena;
  81 }
  82
  83 /*
  84  * Export for the admin server.
  85  */
  86 size_t hhbc_arena_capacity() {
  87   if (!RuntimeOption::RepoAuthoritative) return 0;
  88   return get_readonly_arena().capacity();
  89 }
  90
  91 ///////////////////////////////////////////////////////////////////////////////
  92
  93 UnitEmitter::UnitEmitter(const SHA1& sha1,
  94                          const SHA1& bcSha1,
  95                          const Native::FuncTable& nativeFuncs,
  96                          bool useGlobalIds)
  97   : m_useGlobalIds(useGlobalIds)
  98   , m_mainReturn(make_tv<KindOfUninit>())
  99   , m_nativeFuncs(nativeFuncs)
 100   , m_sha1(sha1)
 101   , m_bcSha1(bcSha1)
 102   , m_bc((unsigned char*)malloc(BCMaxInit))
 103   , m_bclen(0)
 104   , m_bcmax(BCMaxInit)
 105   , m_nextFuncSn(0)
 106   , m_allClassesHoistable(true)
 107 {}
 108
 109 UnitEmitter::~UnitEmitter() {
 110   if (m_bc) free(m_bc);
 111
 112   for (auto& pce : m_pceVec) delete pce;
 113   for (auto& re : m_reVec) delete re;
 114 }
 115
 116
 117 ///////////////////////////////////////////////////////////////////////////////
 118 // Basic data.
 119
 120 void UnitEmitter::setBc(const unsigned char* bc, size_t bclen) {
 121   if (m_bc) {
 122     free(m_bc);
 123   }
 124   m_bc = (unsigned char*)malloc(bclen);
 125   m_bcmax = bclen;
 126   memcpy(m_bc, bc, bclen);
 127   m_bclen = bclen;
 128 }
 129
 130
 131 ///////////////////////////////////////////////////////////////////////////////
 132 // Litstrs and Arrays.
 133
 134 const StringData* UnitEmitter::lookupLitstr(Id id) const {
 135   if (!isUnitLitstrId(id)) {
 136     return LitstrTable::get().lookupLitstrId(id);
 137   }
 138   auto unitId = decodeUnitLitstrId(id);
 139   assertx(unitId < m_litstrs.size());
 140   return m_litstrs[unitId];
 141 }
 142
 143 const ArrayData* UnitEmitter::lookupArray(Id id) const {
 144   assertx(id < m_arrays.size());
 145   return m_arrays[id];
 146 }
 147
 148 const RepoAuthType::Array* UnitEmitter::lookupArrayType(Id id) const {
 149   return RuntimeOption::RepoAuthoritative
 150            ? globalArrayTypeTable().lookup(id)
 151            : m_arrayTypeTable.lookup(id);
 152 }
 153
 154 void UnitEmitter::repopulateArrayTypeTable(
 155   const ArrayTypeTable::Builder& builder) {
 156   m_arrayTypeTable.repopulate(builder);
 157 }
 158
 159 Id UnitEmitter::mergeLitstr(const StringData* litstr) {
 160   if (m_useGlobalIds) {
 161     return LitstrTable::get().mergeLitstr(litstr);
 162   }
 163   return encodeUnitLitstrId(mergeUnitLitstr(litstr));
 164 }
 165
 166 Id UnitEmitter::mergeUnitLitstr(const StringData* litstr) {
 167   auto it = m_litstr2id.find(litstr);
 168   if (it == m_litstr2id.end()) {
 169     const StringData* str = makeStaticString(litstr);
 170     Id id = m_litstrs.size();
 171     m_litstrs.push_back(str);
 172     m_litstr2id[str] = id;
 173     return id;
 174   } else {
 175     return it->second;
 176   }
 177 }
 178
 179 Id UnitEmitter::mergeArray(const ArrayData* a) {
 180   assertx(a->isStatic());
 181   auto const id = static_cast<Id>(m_arrays.size());
 182   m_array2id.emplace(a, id);
 183   m_arrays.push_back(a);
 184   return id;
 185 }
 186
 187
 188 ///////////////////////////////////////////////////////////////////////////////
 189 // FuncEmitters.
 190
 191 void UnitEmitter::initMain(int line1, int line2) {
 192   assertx(m_fes.size() == 0);
 193   StringData* name = staticEmptyString();
 194   FuncEmitter* pseudomain = newFuncEmitter(name);
 195   Attr attrs = AttrMayUseVV;
 196   pseudomain->init(line1, line2, 0, attrs, false, name);
 197 }
 198
 199 void UnitEmitter::addTrivialPseudoMain() {
 200   initMain(0, 0);
 201   auto const mfe = getMain();
 202   emitOp(OpInt);
 203   emitInt64(1);
 204   emitOp(OpRetC);
 205   mfe->maxStackCells = 1;
 206   mfe->finish(bcPos());
 207
 208   TypedValue mainReturn;
 209   mainReturn.m_data.num = 1;
 210   mainReturn.m_type = KindOfInt64;
 211   m_mainReturn = mainReturn;
 212   m_mergeOnly = true;
 213 }
 214
 215 FuncEmitter* UnitEmitter::newFuncEmitter(const StringData* name) {
 216   // The pseudomain comes first.
 217   assertx(m_fes.size() > 0 || !strcmp(name->data(), ""));
 218
 219   auto fe = std::make_unique<FuncEmitter>(*this, m_nextFuncSn++, m_fes.size(),
 220                                           name);
 221   m_fes.push_back(std::move(fe));
 222   return m_fes.back().get();
 223 }
 224
 225 FuncEmitter* UnitEmitter::newMethodEmitter(const StringData* name,
 226                                            PreClassEmitter* pce) {
 227   return new FuncEmitter(*this, m_nextFuncSn++, name, pce);
 228 }
 229
 230 void UnitEmitter::appendTopEmitter(std::unique_ptr<FuncEmitter>&& fe) {
 231   fe->setIds(m_nextFuncSn++, m_fes.size());
 232   m_fes.push_back(std::move(fe));
 233 }
 234
 235 Func* UnitEmitter::newFunc(const FuncEmitter* fe, Unit& unit,
 236                            const StringData* name, Attr attrs,
 237                            int numParams) {
 238   Func *func = nullptr;
 239   if (attrs & AttrIsMethCaller) {
 240     auto const pair = Func::getMethCallerNames(name);
 241     func = new (Func::allocFuncMem(numParams)) Func(
 242       unit, name, attrs, pair.first, pair.second);
 243   } else {
 244     func = new (Func::allocFuncMem(numParams)) Func(unit, name, attrs);
 245   }
 246   if (unit.m_extended) unit.getExtended()->m_funcTable.push_back(func);
 247   return func;
 248 }
 249
 250
 251 ///////////////////////////////////////////////////////////////////////////////
 252 // PreClassEmitters.
 253
 254 void UnitEmitter::addPreClassEmitter(PreClassEmitter* pce) {
 255   if (pce->hoistability() && m_hoistablePreClassSet.count(pce->name())) {
 256     pce->setHoistable(PreClass::Mergeable);
 257   }
 258   auto hoistable = pce->hoistability();
 259
 260   if (hoistable >= PreClass::MaybeHoistable) {
 261     m_hoistablePreClassSet.insert(pce->name());
 262     m_hoistablePceIdList.push_back(pce->id());
 263   } else {
 264     m_allClassesHoistable = false;
 265   }
 266   if (hoistable >= PreClass::Mergeable &&
 267       hoistable < PreClass::AlwaysHoistable) {
 268     if (m_returnSeen) {
 269       m_allClassesHoistable = false;
 270     } else {
 271       pushMergeableClass(pce);
 272     }
 273   }
 274 }
 275
 276 PreClassEmitter* UnitEmitter::newBarePreClassEmitter(
 277   const std::string& name,
 278   PreClass::Hoistable hoistable
 279 ) {
 280   auto pce = new PreClassEmitter(*this, m_pceVec.size(), name, hoistable);
 281   m_pceVec.push_back(pce);
 282   return pce;
 283 }
 284
 285 PreClassEmitter* UnitEmitter::newPreClassEmitter(
 286   const std::string& name,
 287   PreClass::Hoistable hoistable
 288 ) {
 289   PreClassEmitter* pce = newBarePreClassEmitter(name, hoistable);
 290   addPreClassEmitter(pce);
 291   return pce;
 292 }
 293
 294 RecordEmitter* UnitEmitter::newRecordEmitter(const std::string& name) {
 295   auto const re = new RecordEmitter(*this, m_reVec.size(), name);
 296   m_reVec.push_back(re);
 297   return re;
 298 }
 299
 300 Id UnitEmitter::pceId(folly::StringPiece clsName) {
 301   Id id = 0;
 302   for (auto p : m_pceVec) {
 303     if (p->name()->slice() == clsName) return id;
 304     id++;
 305   }
 306   return -1;
 307 }
 308
 309 ///////////////////////////////////////////////////////////////////////////////
 310 // Type aliases.
 311
 312 Id UnitEmitter::addTypeAlias(const TypeAlias& td) {
 313   Id id = m_typeAliases.size();
 314   m_typeAliases.push_back(td);
 315   return id;
 316 }
 317
 318 ///////////////////////////////////////////////////////////////////////////////
 319 // Constants.
 320
 321 Id UnitEmitter::addConstant(const Constant& c) {
 322   Id id = m_constants.size();
 323   TRACE(1, "Add Constant %d %s %d\n", id, c.name->data(), c.attrs);
 324   m_constants.push_back(c);
 325   return id;
 326 }
 327
 328 ///////////////////////////////////////////////////////////////////////////////
 329 // Source locations.
 330
 331 SourceLocTable UnitEmitter::createSourceLocTable() const {
 332   assertx(m_sourceLocTab.size() != 0);
 333   SourceLocTable locations;
 334   for (size_t i = 0; i < m_sourceLocTab.size(); ++i) {
 335     Offset endOff = i < m_sourceLocTab.size() - 1
 336       ? m_sourceLocTab[i + 1].first
 337       : m_bclen;
 338     locations.push_back(SourceLocEntry(endOff, m_sourceLocTab[i].second));
 339   }
 340   return locations;
 341 }
 342
 343 namespace {
 344
 345 using SrcLoc = std::vector<std::pair<Offset, SourceLoc>>;
 346
 347 /*
 348  * Create a LineTable from `srcLoc'.
 349  */
 350 LineTable createLineTable(const SrcLoc& srcLoc, Offset bclen) {
 351   LineTable lines;
 352   if (srcLoc.empty()) {
 353     return lines;
 354   }
 355
 356   auto prev = srcLoc.begin();
 357   for (auto it = prev + 1; it != srcLoc.end(); ++it) {
 358     if (prev->second.line1 != it->second.line1) {
 359       lines.push_back(LineEntry(it->first, prev->second.line1));
 360       prev = it;
 361     }
 362   }
 363
 364   lines.push_back(LineEntry(bclen, prev->second.line1));
 365   return lines;
 366 }
 367
 368 }
 369
 370 void UnitEmitter::recordSourceLocation(const Location::Range& sLoc,
 371                                        Offset start) {
 372   // Some byte codes, such as for the implicit "return 0" at the end of a
 373   // a source file do not have valid source locations. This check makes
 374   // sure we don't record a (dummy) source location in this case.
 375   if (start > 0 && sLoc.line0 == -1) return;
 376   SourceLoc newLoc(sLoc);
 377   if (!m_sourceLocTab.empty()) {
 378     if (m_sourceLocTab.back().second == newLoc) {
 379       // Combine into the interval already at the back of the vector.
 380       assertx(start >= m_sourceLocTab.back().first);
 381       return;
 382     }
 383     assertx(m_sourceLocTab.back().first < start &&
 384            "source location offsets must be added to UnitEmitter in "
 385            "increasing order");
 386   } else {
 387     // First record added should be for bytecode offset zero or very rarely one
 388     // when the source starts with a label and a Nop is inserted.
 389     assertx(start == 0 || start == 1);
 390   }
 391   m_sourceLocTab.push_back(std::make_pair(start, newLoc));
 392 }
 393
 394
 395 ///////////////////////////////////////////////////////////////////////////////
 396 // Mergeables.
 397
 398 void UnitEmitter::pushMergeableClass(PreClassEmitter* e) {
 399   m_mergeableStmts.push_back(std::make_pair(MergeKind::Class, e->id()));
 400 }
 401
 402 void UnitEmitter::pushMergeableId(Unit::MergeKind kind, const Id id) {
 403   m_mergeableStmts.push_back(std::make_pair(kind, id));
 404   m_allClassesHoistable = false;
 405 }
 406
 407 void UnitEmitter::insertMergeableId(Unit::MergeKind kind, int ix, const Id id) {
 408   assertx(size_t(ix) <= m_mergeableStmts.size());
 409   m_mergeableStmts.insert(m_mergeableStmts.begin() + ix,
 410                           std::make_pair(kind, id));
 411   m_allClassesHoistable = false;
 412 }
 413
 414 void UnitEmitter::pushMergeableRecord(const Id id) {
 415   m_mergeableStmts.push_back(std::make_pair(Unit::MergeKind::Record, id));
 416   m_allClassesHoistable = false;
 417 }
 418
 419 void UnitEmitter::insertMergeableRecord(int ix, const Id id) {
 420   assertx(size_t(ix) <= m_mergeableStmts.size());
 421   m_mergeableStmts.insert(m_mergeableStmts.begin() + ix,
 422                           std::make_pair(Unit::MergeKind::Record, id));
 423   m_allClassesHoistable = false;
 424 }
 425
 426 ///////////////////////////////////////////////////////////////////////////////
 427 // Initialization and execution.
 428
 429 void UnitEmitter::commit(UnitOrigin unitOrigin, bool usePreAllocatedUnitSn) {
 430   Repo& repo = Repo::get();
 431   try {
 432     auto txn = RepoTxn{repo.begin()};
 433     RepoStatus err = insert(unitOrigin, txn, usePreAllocatedUnitSn);
 434     if (err == RepoStatus::success) {
 435       txn.commit();
 436     }
 437   } catch (RepoExc& re) {
 438     tracing::addPointNoTrace("ue-commit-exn");
 439     int repoId = repo.repoIdForNewUnit(unitOrigin);
 440     if (repoId != RepoIdInvalid) {
 441       TRACE(3, "Failed to commit '%s' (%s) to '%s': %s\n",
 442                m_filepath->data(), m_sha1.toString().c_str(),
 443                repo.repoName(repoId).c_str(), re.msg().c_str());
 444     }
 445   }
 446 }
 447
 448 RepoStatus UnitEmitter::insert(UnitOrigin unitOrigin, RepoTxn& txn,
 449                                bool usePreAllocatedUnitSn) {
 450   Repo& repo = Repo::get();
 451   UnitRepoProxy& urp = repo.urp();
 452   int repoId = Repo::get().repoIdForNewUnit(unitOrigin);
 453   if (repoId == RepoIdInvalid) {
 454     return RepoStatus::error;
 455   }
 456   m_repoId = repoId;
 457
 458   try {
 459     {
 460       if (!m_sourceLocTab.empty()) {
 461         m_lineTable = createLineTable(m_sourceLocTab, m_bclen);
 462       }
 463       urp.insertUnit[repoId].insert(*this, txn, m_sn, m_sha1, m_bc,
 464                                     m_bclen, usePreAllocatedUnitSn);
 465     }
 466     int64_t usn = m_sn;
 467     urp.insertUnitLineTable[repoId].insert(txn, usn, m_lineTable);
 468     for (unsigned i = 0; i < m_litstrs.size(); ++i) {
 469       urp.insertUnitLitstr[repoId].insert(txn, usn, i, m_litstrs[i]);
 470     }
 471     for (unsigned i = 0; i < m_typeAliases.size(); ++i) {
 472       urp.insertUnitTypeAlias[repoId].insert(*this, txn, usn, i,
 473                                              m_typeAliases[i]);
 474     }
 475     for (unsigned i = 0; i < m_constants.size(); ++i) {
 476       urp.insertUnitConstant[repoId].insert(*this, txn, usn, i, m_constants[i]);
 477     }
 478     for (unsigned i = 0; i < m_arrays.size(); ++i) {
 479       // We check that arrays do not exceed a configurable maximum size in the
 480       // assembler, so just assume that they're okay here.
 481       MemoryManager::SuppressOOM so(*tl_heap);
 482
 483       auto const arr_str = [&]{
 484         VariableSerializer vs{VariableSerializer::Type::Internal};
 485         vs.setUnitFilename(m_filepath);
 486         return vs.serializeValue(
 487           VarNR(const_cast<ArrayData*>(m_arrays[i])), false
 488         ).toCppString();
 489       }();
 490
 491       urp.insertUnitArray[repoId].insert(txn, usn, i, arr_str);
 492     }
 493     urp.insertUnitArrayTypeTable[repoId].insert(txn, usn, *this);
 494     for (auto& fe : m_fes) {
 495       fe->commit(txn);
 496     }
 497     for (auto& pce : m_pceVec) {
 498       pce->commit(txn);
 499     }
 500     for (auto& re : m_reVec) {
 501       re->commit(txn);
 502     }
 503
 504     for (int i = 0, n = m_mergeableStmts.size(); i < n; i++) {
 505       switch (m_mergeableStmts[i].first) {
 506         case MergeKind::Done:
 507         case MergeKind::UniqueDefinedClass:
 508           not_reached();
 509         case MergeKind::Class:
 510           break;
 511         case MergeKind::TypeAlias:
 512         case MergeKind::Record:
 513         case MergeKind::Define: {
 514           urp.insertUnitMergeable[repoId].insert(
 515             txn, usn, i,
 516             m_mergeableStmts[i].first, m_mergeableStmts[i].second);
 517           break;
 518         }
 519       }
 520     }
 521     if (RuntimeOption::RepoDebugInfo) {
 522       for (size_t i = 0; i < m_sourceLocTab.size(); ++i) {
 523         SourceLoc& e = m_sourceLocTab[i].second;
 524         Offset endOff = i < m_sourceLocTab.size() - 1
 525                             ? m_sourceLocTab[i + 1].first
 526                             : m_bclen;
 527
 528         urp.insertUnitSourceLoc[repoId]
 529            .insert(txn, usn, endOff, e.line0, e.char0, e.line1, e.char1);
 530       }
 531     }
 532     return RepoStatus::success;
 533   } catch (RepoExc& re) {
 534     TRACE(3, "Failed to commit '%s' (%s) to '%s': %s\n",
 535              m_filepath->data(), m_sha1.toString().c_str(),
 536              repo.repoName(repoId).c_str(), re.msg().c_str());
 537     return RepoStatus::error;
 538   }
 539 }
 540
 541 ServiceData::ExportedTimeSeries* g_hhbc_size = ServiceData::createTimeSeries(
 542   "vm.hhbc-size",
 543   {ServiceData::StatsType::AVG,
 544    ServiceData::StatsType::SUM,
 545    ServiceData::StatsType::COUNT}
 546 );
 547
 548 static const unsigned char*
 549 allocateBCRegion(const unsigned char* bc, size_t bclen) {
 550   g_hhbc_size->addValue(bclen);
 551   if (RuntimeOption::RepoAuthoritative) {
 552     // In RepoAuthoritative, we assume we won't ever deallocate units
 553     // and that this is read-only, mostly cold data.  So we throw it
 554     // in a bump-allocator that's mprotect'd to prevent writes.
 555     return static_cast<const unsigned char*>(
 556       get_readonly_arena().allocate(bc, bclen)
 557     );
 558   }
 559   auto mem = static_cast<unsigned char*>(malloc(bclen));
 560   std::copy(bc, bc + bclen, mem);
 561   return mem;
 562 }
 563
 564 bool UnitEmitter::check(bool verbose) const {
 565   return Verifier::checkUnit(
 566     this,
 567     verbose ? Verifier::kVerbose : Verifier::kStderr
 568   );
 569 }
 570
 571 bool needs_extended_line_table() {
 572   return RuntimeOption::RepoDebugInfo &&
 573     (RuntimeOption::EvalDumpHhas ||
 574      RuntimeOption::EnableHphpdDebugger ||
 575      RuntimeOption::EnableVSDebugger ||
 576      RuntimeOption::EnableDebuggerServer);
 577 }
 578
 579 std::unique_ptr<Unit> UnitEmitter::create(bool saveLineTable) const {
 580   INC_TPC(unit_load);
 581
 582   tracing::BlockNoTrace _{"unit-create"};
 583
 584   static const bool kVerify = debug || RuntimeOption::EvalVerify ||
 585     RuntimeOption::EvalVerifyOnly || RuntimeOption::EvalFatalOnVerifyError;
 586   static const bool kVerifyVerboseSystem =
 587     getenv("HHVM_VERIFY_VERBOSE_SYSTEM");
 588   static const bool kVerifyVerbose =
 589     kVerifyVerboseSystem || getenv("HHVM_VERIFY_VERBOSE");
 590
 591   const bool isSystemLib = FileUtil::isSystemName(m_filepath->slice());
 592   const bool doVerify =
 593     kVerify || boost::ends_with(m_filepath->data(), ".hhas");
 594   if (doVerify) {
 595     auto const verbose = isSystemLib ? kVerifyVerboseSystem : kVerifyVerbose;
 596     if (!check(verbose)) {
 597       if (!verbose) {
 598         std::cerr << folly::format(
 599           "Verification failed for unit {}. Re-run with "
 600           "HHVM_VERIFY_VERBOSE{}=1 to see more details.\n",
 601           m_filepath->data(), isSystemLib ? "_SYSTEM" : ""
 602         );
 603       }
 604       if (RuntimeOption::EvalVerifyOnly) {
 605         if (!isSystemLib) {
 606           std::fflush(stdout);
 607           _Exit(1);
 608         }
 609       } else if (RuntimeOption::EvalFatalOnVerifyError) {
 610         return createFatalUnit(
 611           const_cast<StringData*>(m_filepath),
 612           m_sha1,
 613           FatalOp::Parse,
 614           makeStaticString("A bytecode verification error was detected")
 615         )->create(saveLineTable);
 616       }
 617     }
 618     if (!isSystemLib && RuntimeOption::EvalVerifyOnly) {
 619       std::fflush(stdout);
 620       _Exit(0);
 621     }
 622   }
 623
 624   std::unique_ptr<Unit> u {
 625     RuntimeOption::RepoAuthoritative && !RuntimeOption::SandboxMode &&
 626       m_litstrs.empty() && m_arrayTypeTable.empty() ?
 627     new Unit : new UnitExtended
 628   };
 629
 630   u->m_repoId = saveLineTable ? RepoIdInvalid : m_repoId;
 631   u->m_sn = m_sn;
 632   u->m_bc = allocateBCRegion(m_bc, m_bclen);
 633   u->m_bclen = m_bclen;
 634   u->m_filepath = m_filepath;
 635   u->m_mainReturn = m_mainReturn;
 636   u->m_mergeOnly = m_mergeOnly;
 637   u->m_isHHFile = m_isHHFile;
 638   u->m_dirpath = makeStaticString(FileUtil::dirname(StrNR{m_filepath}));
 639   u->m_sha1 = m_sha1;
 640   u->m_bcSha1 = m_bcSha1;
 641   u->m_arrays = m_arrays;
 642   for (auto const& pce : m_pceVec) {
 643     u->m_preClasses.push_back(PreClassPtr(pce->create(*u)));
 644   }
 645   for (auto const& re : m_reVec) {
 646     u->m_preRecords.push_back(PreRecordDescPtr(re->create(*u)));
 647   }
 648   u->m_typeAliases = m_typeAliases;
 649   u->m_constants = m_constants;
 650   u->m_metaData = m_metaData;
 651   u->m_fileAttributes = m_fileAttributes;
 652   u->m_ICE = m_ICE;
 653
 654   size_t ix = m_fes.size() + m_hoistablePceIdList.size();
 655   if (m_mergeOnly && !m_allClassesHoistable) {
 656     size_t extra = 0;
 657     for (auto& mergeable : m_mergeableStmts) {
 658       extra++;
 659       if (!RuntimeOption::RepoAuthoritative && SystemLib::s_inited) {
 660         if (mergeable.first != MergeKind::Class) {
 661           extra = 0;
 662           u->m_mergeOnly = false;
 663           break;
 664         }
 665       }
 666     }
 667     ix += extra;
 668   }
 669   Unit::MergeInfo *mi = Unit::MergeInfo::alloc(ix);
 670   u->m_mergeInfo.store(mi, std::memory_order_relaxed);
 671   ix = 0;
 672   for (auto& fe : m_fes) {
 673     auto const func = fe->create(*u);
 674     if (func->top()) {
 675       if (!mi->m_firstHoistableFunc) {
 676         mi->m_firstHoistableFunc = ix;
 677       }
 678     } else {
 679       assertx(!mi->m_firstHoistableFunc);
 680     }
 681     assertx(ix == fe->id());
 682     mi->mergeableObj(ix++) = func;
 683   }
 684   assertx(u->getMain(nullptr, false)->isPseudoMain());
 685   if (!mi->m_firstHoistableFunc) {
 686     mi->m_firstHoistableFunc =  ix;
 687   }
 688   mi->m_firstHoistablePreClass = ix;
 689   assertx(m_fes.size());
 690   for (auto& id : m_hoistablePceIdList) {
 691     mi->mergeableObj(ix++) = u->m_preClasses[id].get();
 692   }
 693   mi->m_firstMergeablePreClass = ix;
 694   if (u->m_mergeOnly && !m_allClassesHoistable) {
 695     for (auto& mergeable : m_mergeableStmts) {
 696       switch (mergeable.first) {
 697         case MergeKind::Class:
 698           mi->mergeableObj(ix++) = u->m_preClasses[mergeable.second].get();
 699           break;
 700         case MergeKind::Define:
 701           assertx(RuntimeOption::RepoAuthoritative);
 702         case MergeKind::Record:
 703         case MergeKind::TypeAlias:
 704           mi->mergeableObj(ix++) =
 705             (void*)((intptr_t(mergeable.second) << 3) + (int)mergeable.first);
 706           break;
 707         case MergeKind::Done:
 708         case MergeKind::UniqueDefinedClass:
 709           not_reached();
 710       }
 711     }
 712   }
 713   assertx(ix == mi->m_mergeablesSize);
 714   mi->mergeableObj(ix) = (void*)MergeKind::Done;
 715
 716   /*
 717    * What's going on is we're going to have a m_lineTable if this UnitEmitter
 718    * was loaded from the repo, and no m_sourceLocTab (it's demand-loaded by
 719    * unit.cpp because it's only used for the debugger).  Don't bother creating
 720    * the line table here, because we can retrieve it from the repo later.
 721    *
 722    * On the other hand, if this unit was just created by parsing a php file (or
 723    * whatnot) which was not committed to the repo, we'll have a m_sourceLocTab.
 724    * In this case we should populate m_lineTable (otherwise we might lose line
 725    * info altogether, since it may not be backed by a repo).
 726    */
 727   if (m_sourceLocTab.size() != 0) {
 728     stashLineTable(u.get(), createLineTable(m_sourceLocTab, m_bclen));
 729     // If the debugger is enabled, or we plan to dump hhas we will
 730     // need the extended line table information in the output, and if
 731     // we're not writing the repo, stashing it here is necessary for
 732     // it to make it through.
 733     if (needs_extended_line_table()) {
 734       stashExtendedLineTable(u.get(), createSourceLocTable());
 735     }
 736   } else if (saveLineTable) {
 737     stashLineTable(u.get(), m_lineTable);
 738   }
 739
 740   if (u->m_extended) {
 741     auto ux = u->getExtended();
 742     for (auto s : m_litstrs) {
 743       ux->m_namedInfo.emplace_back(LowStringPtr{s});
 744     }
 745     ux->m_arrayTypeTable = m_arrayTypeTable;
 746
 747     // Funcs can be recorded out of order when loading them from the
 748     // repo currently.  So sort 'em here.
 749     std::sort(ux->m_funcTable.begin(), ux->m_funcTable.end(),
 750               [] (const Func* a, const Func* b) {
 751                 return a->past() < b->past();
 752               });
 753   } else {
 754     assertx(!m_litstrs.size());
 755     assertx(m_arrayTypeTable.empty());
 756   }
 757
 758   if (RuntimeOption::EvalDumpHhas > 1 ||
 759     (SystemLib::s_inited && RuntimeOption::EvalDumpHhas == 1)) {
 760     auto const& hhaspath = RuntimeOption::EvalDumpHhasToFile;
 761     if (!hhaspath.empty()) {
 762       static std::atomic<bool> first_unit{true};
 763       auto const flags = O_WRONLY | O_CREAT | (first_unit ? O_TRUNC : O_APPEND);
 764       if (!folly::writeFile(disassemble(u.get()), hhaspath.c_str(), flags)) {
 765         Logger::Error("Failed to write hhas to %s", hhaspath.c_str());
 766         _Exit(1);
 767       }
 768       first_unit = false;
 769     } else {
 770       std::printf("%s", disassemble(u.get()).c_str());
 771       std::fflush(stdout);
 772     }
 773     if (SystemLib::s_inited) {
 774       _Exit(0);
 775     }
 776   }
 777
 778   if (RuntimeOption::EvalDumpBytecode) {
 779     // Dump human-readable bytecode.
 780     Trace::traceRelease("%s", u->toString().c_str());
 781   }
 782
 783   return u;
 784 }
 785
 786 template<class SerDe>
 787 void UnitEmitter::serdeMetaData(SerDe& sd) {
 788   sd(m_mainReturn)
 789     (m_mergeOnly)
 790     (m_isHHFile)
 791     (m_metaData)
 792     (m_fileAttributes)
 793     (m_symbol_refs)
 794     (m_bcSha1)
 795     ;
 796
 797   if (RuntimeOption::EvalLoadFilepathFromUnitCache) {
 798     /* May be different than the unit origin: e.g. for hhas files. */
 799     sd(m_filepath);
 800   }
 801 }
 802
 803 template <typename SerDe>
 804 void UnitEmitter::serde(SerDe& sd) {
 805   MemoryManager::SuppressOOM so(*tl_heap);
 806
 807   serdeMetaData(sd);
 808   // These are not touched by serdeMetaData:
 809   sd(m_returnSeen);
 810   sd(m_ICE);
 811   sd(m_allClassesHoistable);
 812
 813   auto const seq = [&] (auto const& c, auto const& r, auto const& w) {
 814     if constexpr (SerDe::deserializing) {
 815       size_t size;
 816       sd(size);
 817       for (size_t i = 0; i < size; ++i) r(sd, i);
 818     } else {
 819       sd(c.size());
 820       for (auto const& x : c) w(sd, x);
 821     }
 822   };
 823
 824   // Literal strings
 825   seq(
 826     m_litstrs,
 827     [&] (auto& sd, size_t i) {
 828       const StringData* s;
 829       sd(s);
 830       auto const id UNUSED = mergeUnitLitstr(s);
 831       assertx(id == i);
 832     },
 833     [&] (auto& sd, const StringData* s) { sd(s); }
 834   );
 835
 836   // Arrays
 837   seq(
 838     m_arrays,
 839     [&] (auto& sd, size_t i) {
 840       std::string key;
 841       sd(key);
 842
 843       auto v = [&]{
 844         VariableUnserializer vu{
 845           key.data(),
 846           key.size(),
 847           VariableUnserializer::Type::Internal
 848         };
 849         vu.setUnitFilename(m_filepath);
 850         return vu.unserialize();
 851       }();
 852       assertx(v.isArray());
 853       auto ad = v.detach().m_data.parr;
 854       ArrayData::GetScalarArray(&ad);
 855       auto const id DEBUG_ONLY = mergeArray(ad);
 856       assertx(id == i);
 857     },
 858     [&] (auto& sd, const ArrayData* a) {
 859       auto const str = [&]{
 860         VariableSerializer vs{VariableSerializer::Type::Internal};
 861         vs.setUnitFilename(m_filepath);
 862         return
 863           vs.serializeValue(VarNR(const_cast<ArrayData*>(a)), false)
 864           .toCppString();
 865       }();
 866       sd(str);
 867     }
 868   );
 869
 870   // HHBBC array types
 871   sd(m_arrayTypeTable);
 872
 873   // Pre-class emitters
 874   seq(
 875     m_pceVec,
 876     [&] (auto& sd, size_t i) {
 877       std::string name;
 878       int hoistable;
 879       sd(name);
 880       sd(hoistable);
 881       auto pce = newPreClassEmitter(name, (PreClass::Hoistable)hoistable);
 882       pce->serdeMetaData(sd);
 883       assertx(pce->id() == i);
 884     },
 885     [&] (auto& sd, PreClassEmitter* pce) {
 886       auto const nm = StripIdFromAnonymousClassName(pce->name()->slice());
 887       sd(nm.toString());
 888       sd((int)pce->hoistability());
 889       pce->serdeMetaData(sd);
 890     }
 891   );
 892
 893   // Record emitters
 894   seq(
 895     m_reVec,
 896     [&] (auto& sd, size_t i) {
 897       std::string name;
 898       sd(name);
 899       auto re = newRecordEmitter(name);
 900       re->serdeMetaData(sd);
 901       assertx(re->id() == i);
 902     },
 903     [&] (auto& sd, RecordEmitter* re) {
 904       auto const nm = StripIdFromAnonymousClassName(re->name()->slice());
 905       sd(nm.toString());
 906       re->serdeMetaData(sd);
 907     }
 908   );
 909
 910   // Type aliases
 911   seq(
 912     m_typeAliases,
 913     [&] (auto& sd, size_t i) {
 914       TypeAlias ta;
 915       sd(ta.name);
 916       sd(ta);
 917       auto const id UNUSED = addTypeAlias(ta);
 918       assertx(id == i);
 919     },
 920     [&] (auto& sd, const TypeAlias& ta) {
 921       sd(ta.name);
 922       sd(ta);
 923     }
 924   );
 925
 926   // Constants
 927   seq(
 928     m_constants,
 929     [&] (auto& sd, size_t i) {
 930       Constant cns;
 931       sd(cns.name);
 932       sd(cns);
 933       if (type(cns.val) == KindOfUninit) {
 934         cns.val.m_data.pcnt = reinterpret_cast<MaybeCountable*>(Unit::getCns);
 935       }
 936       auto const id UNUSED = addConstant(cns);
 937       assertx(id == i);
 938     },
 939     [&] (auto& sd, const Constant& cns) {
 940       sd(cns.name);
 941       sd(cns);
 942     }
 943   );
 944
 945   // Line table
 946   sd(
 947     m_lineTable,
 948     [&] (const LineEntry& prev, const LineEntry& curDelta) {
 949       if (SerDe::deserializing) {
 950         return LineEntry {
 951           curDelta.pastOffset() + prev.pastOffset(),
 952           curDelta.val() + prev.val()
 953         };
 954       } else {
 955         return LineEntry {
 956           curDelta.pastOffset() - prev.pastOffset(),
 957           curDelta.val() - prev.val()
 958         };
 959       }
 960     }
 961   );
 962
 963   // Mergeables
 964   sd(m_mergeableStmts);
 965
 966   // Func emitters (we cannot use seq for these because they come from
 967   // both the pre-class emitters and m_fes.
 968   if constexpr (SerDe::deserializing) {
 969     size_t total;
 970     sd(total);
 971     for (size_t i = 0; i < total; ++i) {
 972       Id pceId;
 973       const StringData* name;
 974       bool top;
 975       sd(pceId);
 976       sd(name);
 977       sd(top);
 978
 979       FuncEmitter* fe;
 980       if (pceId < 0) {
 981         fe = newFuncEmitter(name);
 982       } else {
 983         auto funcPce = pce(pceId);
 984         fe = newMethodEmitter(name, funcPce);
 985         auto const added UNUSED = funcPce->addMethod(fe);
 986         assertx(added);
 987       }
 988       assertx(fe->sn() == i);
 989       fe->top = top;
 990       fe->serdeMetaData(sd);
 991       fe->setEHTabIsSorted();
 992       fe->finish(fe->past);
 993     }
 994   } else {
 995     auto total = m_fes.size();
 996     for (auto const pce : m_pceVec) total += pce->methods().size();
 997     sd(total);
 998
 999     auto const write = [&] (FuncEmitter* fe, Id pceId) {
1000       sd(pceId);
1001       sd(fe->name);
1002       sd(fe->top);
1003       fe->serdeMetaData(sd);
1004     };
1005     for (auto const& fe : m_fes) write(fe.get(), -1);
1006     for (auto const pce : m_pceVec) {
1007       for (auto const fe : pce->methods()) write(fe, pce->id());
1008     }
1009   }
1010
1011   // Source location table
1012   sd(m_sourceLocTab);
1013
1014   // Bytecode
1015   if constexpr (SerDe::deserializing) {
1016     size_t size;
1017     sd(size);
1018     assertx(sd.remaining() <= size);
1019     setBc(sd.data(), size);
1020     sd.advance(size);
1021   } else {
1022     sd(m_bclen);
1023     sd.writeRaw((const char*)m_bc, m_bclen);
1024   }
1025 }
1026
1027 template void UnitEmitter::serde<>(BlobDecoder&);
1028 template void UnitEmitter::serde<>(BlobEncoder&);
1029
1030 ///////////////////////////////////////////////////////////////////////////////
1031 // UnitRepoProxy.
1032
1033 UnitRepoProxy::UnitRepoProxy(Repo& repo)
1034     : RepoProxy(repo)
1035 #define URP_OP(c, o) \
1036     , o{c##Stmt(repo, 0), c##Stmt(repo, 1)}
1037     URP_OPS
1038 #undef URP_OP
1039 {}
1040
1041 UnitRepoProxy::~UnitRepoProxy() {
1042 }
1043
1044 void UnitRepoProxy::createSchema(int repoId, RepoTxn& txn) {
1045   {
1046     auto createQuery = folly::sformat(
1047       "CREATE TABLE {} "
1048       "(unitSn INTEGER PRIMARY KEY, sha1 BLOB UNIQUE, globalids INTEGER,"
1049       " bc BLOB, data BLOB);",
1050       m_repo.table(repoId, "Unit"));
1051     txn.exec(createQuery);
1052   }
1053   {
1054     auto createQuery = folly::sformat(
1055       "CREATE TABLE {} "
1056       "(unitSn INTEGER, litstrId INTEGER, litstr TEXT,"
1057       " PRIMARY KEY (unitSn, litstrId));",
1058       m_repo.table(repoId, "UnitLitstr"));
1059     txn.exec(createQuery);
1060   }
1061   {
1062     auto createQuery = folly::sformat(
1063       "CREATE TABLE {} "
1064       "(unitSn INTEGER, typeAliasId INTEGER, name TEXT, data BLOB, "
1065       " PRIMARY KEY (unitSn, typeAliasId));",
1066       m_repo.table(repoId, "UnitTypeAlias"));
1067     txn.exec(createQuery);
1068   }
1069   {
1070     auto createQuery = folly::sformat(
1071       "CREATE TABLE {} "
1072       "(unitSn INTEGER, constantId INTEGER, name TEXT, data BLOB, "
1073       " PRIMARY KEY (unitSn, constantId));",
1074       m_repo.table(repoId, "UnitConstant"));
1075     txn.exec(createQuery);
1076   }
1077   {
1078     auto createQuery = folly::sformat(
1079       "CREATE TABLE {} "
1080       "(unitSn INTEGER, arrayId INTEGER, array BLOB, "
1081       " PRIMARY KEY (unitSn, arrayId));",
1082       m_repo.table(repoId, "UnitArray"));
1083     txn.exec(createQuery);
1084   }
1085   {
1086     auto createQuery = folly::sformat(
1087       "CREATE TABLE {} "
1088       "(unitSn INTEGER PRIMARY KEY, arrayTypeTable BLOB);",
1089       m_repo.table(repoId, "UnitArrayTypeTable"));
1090     txn.exec(createQuery);
1091   }
1092   {
1093     auto createQuery = folly::sformat(
1094       "CREATE TABLE {} "
1095       "(unitSn INTEGER, mergeableIx INTEGER, mergeableKind INTEGER, "
1096       " mergeableId INTEGER, "
1097       " PRIMARY KEY (unitSn, mergeableIx));",
1098       m_repo.table(repoId, "UnitMergeables"));
1099     txn.exec(createQuery);
1100   }
1101   {
1102     auto createQuery = folly::sformat(
1103       "CREATE TABLE {} "
1104       "(unitSn INTEGER, pastOffset INTEGER, line0 INTEGER,"
1105       " char0 INTEGER, line1 INTEGER, char1 INTEGER,"
1106       " PRIMARY KEY (unitSn, pastOffset));",
1107       m_repo.table(repoId, "UnitSourceLoc"));
1108     txn.exec(createQuery);
1109   }
1110   {
1111     auto createQuery = folly::sformat(
1112       "CREATE TABLE {} (unitSn INTEGER PRIMARY KEY, data BLOB);",
1113       m_repo.table(repoId, "UnitLineTable"));
1114     txn.exec(createQuery);
1115   }
1116 }
1117
1118 std::unique_ptr<UnitEmitter> UnitRepoProxy::loadEmitter(
1119     const folly::StringPiece name,
1120     const SHA1& sha1,
1121     const Native::FuncTable& nativeFuncs) {
1122   // We set useGlobalIds to false as a placeholder; it will be set
1123   // correctly by UnitRepoProxy::GetUnitStmt::get.
1124   auto ue = std::make_unique<UnitEmitter>(sha1, SHA1{}, nativeFuncs, false);
1125   if (!RuntimeOption::EvalLoadFilepathFromUnitCache) {
1126     ue->m_filepath = makeStaticString(name);
1127   }
1128   // Look for a repo that contains a unit with matching SHA1.
1129   int repoId;
1130   for (repoId = RepoIdCount - 1; repoId >= 0; --repoId) {
1131     if (getUnit[repoId].get(*ue, sha1) == RepoStatus::success) {
1132       break;
1133     }
1134   }
1135   if (repoId < 0) {
1136     TRACE(3, "No repo contains '%s' (0x%s)\n",
1137              name.data(), sha1.toString().c_str());
1138     return nullptr;
1139   }
1140   try {
1141     getUnitLitstrs[repoId].get(*ue);
1142     getUnitArrays[repoId].get(*ue);
1143     getUnitArrayTypeTable[repoId].get(*ue);
1144     m_repo.pcrp().getPreClasses[repoId].get(*ue);
1145     m_repo.rrp().getRecords[repoId].get(*ue);
1146     getUnitTypeAliases[repoId].get(*ue);
1147     getUnitConstants[repoId].get(*ue);
1148     getUnitMergeables[repoId].get(*ue);
1149     getUnitLineTable[repoId].get(ue->m_sn, ue->m_lineTable);
1150     m_repo.frp().getFuncs[repoId].get(*ue);
1151   } catch (RepoExc& re) {
1152     TRACE(0,
1153           "Repo error loading '%s' (0x%s) from '%s': %s\n",
1154           name.data(), sha1.toString().c_str(),
1155           m_repo.repoName(repoId).c_str(), re.msg().c_str());
1156     return nullptr;
1157   }
1158   TRACE(3, "Repo loaded '%s' (0x%s) from '%s'\n",
1159            name.data(), sha1.toString().c_str(),
1160            m_repo.repoName(repoId).c_str());
1161   return ue;
1162 }
1163
1164 std::unique_ptr<Unit>
1165 UnitRepoProxy::load(const folly::StringPiece name, const SHA1& sha1,
1166                     const Native::FuncTable& nativeFuncs) {
1167   ARRPROV_USE_RUNTIME_LOCATION();
1168   auto ue = loadEmitter(name, sha1, nativeFuncs);
1169   if (!ue) return nullptr;
1170
1171 #ifdef USE_JEMALLOC
1172   if (RuntimeOption::TrackPerUnitMemory) {
1173     size_t len = sizeof(uint64_t*);
1174     uint64_t* alloc;
1175     uint64_t* del;
1176     mallctl("thread.allocatedp", static_cast<void*>(&alloc), &len, nullptr, 0);
1177     mallctl("thread.deallocatedp", static_cast<void*>(&del), &len, nullptr, 0);
1178     auto before = *alloc;
1179     auto debefore = *del;
1180     std::unique_ptr<Unit> result = ue->create();
1181     auto after = *alloc;
1182     auto deafter = *del;
1183
1184     auto path = folly::sformat("/tmp/units-{}.map", getpid());
1185     auto change = (after - deafter) - (before - debefore);
1186     auto str = folly::sformat("{} {}\n", name, change);
1187     auto out = std::fopen(path.c_str(), "a");
1188     if (out) {
1189       std::fwrite(str.data(), str.size(), 1, out);
1190       std::fclose(out);
1191     }
1192
1193     return result;
1194   }
1195 #endif
1196
1197   auto unit = ue->create();
1198   if (BuiltinSymbols::s_systemAr) {
1199     assertx(ue->m_filepath->data()[0] == '/' &&
1200             ue->m_filepath->data()[1] == ':');
1201     BuiltinSymbols::RecordSystemlibFile(std::move(ue));
1202   }
1203   FTRACE(1, "Creating unit {} for `{}`\n", unit.get(), name);
1204   return unit;
1205 }
1206
1207 void UnitRepoProxy::InsertUnitStmt
1208                   ::insert(const UnitEmitter& ue,
1209                            RepoTxn& txn, int64_t& unitSn, const SHA1& sha1,
1210                            const unsigned char* bc, size_t bclen,
1211                            bool usePreAllocatedUnitSn) {
1212   BlobEncoder dataBlob{ue.useGlobalIds()};
1213
1214   if (!prepared()) {
1215     auto insertQuery = folly::sformat(
1216       "INSERT INTO {} VALUES(@unitSn, @sha1, @globalids, @bc, @data);",
1217       m_repo.table(m_repoId, "Unit"));
1218     txn.prepare(*this, insertQuery);
1219   }
1220   RepoTxnQuery query(txn, *this);
1221   query.bindSha1("@sha1", sha1);
1222   query.bindBool("@globalids", ue.m_useGlobalIds);
1223   query.bindBlob("@bc", (const void*)bc, bclen);
1224   const_cast<UnitEmitter&>(ue).serdeMetaData(dataBlob);
1225   query.bindBlob("@data", dataBlob, /* static */ true);
1226   if (!usePreAllocatedUnitSn) {
1227     query.bindNull("@unitSn");
1228   } else {
1229     always_assert(unitSn != -1);
1230     query.bindInt64("@unitSn", unitSn);
1231   }
1232   query.exec();
1233   if (!usePreAllocatedUnitSn) {
1234     unitSn = query.getInsertedRowid();
1235   }
1236 }
1237
1238 RepoStatus UnitRepoProxy::GetUnitStmt::get(UnitEmitter& ue, const SHA1& sha1) {
1239   try {
1240     auto txn = RepoTxn{m_repo.begin()};
1241     if (!prepared()) {
1242       auto selectQuery = folly::sformat(
1243         "SELECT unitSn, globalids, bc, data FROM {} WHERE sha1 == @sha1;",
1244         m_repo.table(m_repoId, "Unit"));
1245       txn.prepare(*this, selectQuery);
1246     }
1247     RepoTxnQuery query(txn, *this);
1248     query.bindSha1("@sha1", sha1);
1249     query.step();
1250     if (!query.row()) {
1251       return RepoStatus::error;
1252     }
1253     int64_t unitSn;                     /**/ query.getInt64(0, unitSn);
1254     bool useGlobalIds;                  /**/ query.getBool(1, useGlobalIds);
1255     const void* bc; size_t bclen;       /**/ query.getBlob(2, bc, bclen);
1256     BlobDecoder dataBlob =              /**/ query.getBlob(3, useGlobalIds);
1257
1258     ue.m_repoId = m_repoId;
1259     ue.m_sn = unitSn;
1260     ue.m_useGlobalIds = useGlobalIds;
1261     ue.setBc(static_cast<const unsigned char*>(bc), bclen);
1262     ue.serdeMetaData(dataBlob);
1263
1264     txn.commit();
1265   } catch (RepoExc& re) {
1266     return RepoStatus::error;
1267   }
1268   return RepoStatus::success;
1269 }
1270
1271 void UnitRepoProxy::InsertUnitLitstrStmt
1272                   ::insert(RepoTxn& txn, int64_t unitSn, Id litstrId,
1273                            const StringData* litstr) {
1274   if (!prepared()) {
1275     auto insertQuery = folly::sformat(
1276       "INSERT INTO {} VALUES(@unitSn, @litstrId, @litstr);",
1277       m_repo.table(m_repoId, "UnitLitstr"));
1278     txn.prepare(*this, insertQuery);
1279   }
1280   RepoTxnQuery query(txn, *this);
1281   query.bindInt64("@unitSn", unitSn);
1282   query.bindId("@litstrId", litstrId);
1283   query.bindStaticString("@litstr", litstr);
1284   query.exec();
1285 }
1286
1287 void UnitRepoProxy::GetUnitLitstrsStmt
1288                   ::get(UnitEmitter& ue) {
1289   auto txn = RepoTxn{m_repo.begin()};
1290   if (!prepared()) {
1291     auto selectQuery = folly::sformat(
1292       "SELECT litstrId, litstr FROM {} "
1293       " WHERE unitSn == @unitSn ORDER BY litstrId ASC;",
1294       m_repo.table(m_repoId, "UnitLitstr"));
1295     txn.prepare(*this, selectQuery);
1296   }
1297   RepoTxnQuery query(txn, *this);
1298   query.bindInt64("@unitSn", ue.m_sn);
1299   do {
1300     query.step();
1301     if (query.row()) {
1302       Id litstrId;        /**/ query.getId(0, litstrId);
1303       StringData* litstr; /**/ query.getStaticString(1, litstr);
1304       Id id UNUSED = ue.mergeUnitLitstr(litstr);
1305       assertx(id == litstrId);
1306     }
1307   } while (!query.done());
1308   txn.commit();
1309 }
1310
1311 void UnitRepoProxy::InsertUnitArrayTypeTableStmt::insert(
1312   RepoTxn& txn, int64_t unitSn, const UnitEmitter& ue) {
1313
1314   if (!prepared()) {
1315     auto insertQuery = folly::sformat(
1316       "INSERT INTO {} VALUES(@unitSn, @arrayTypeTable);",
1317       m_repo.table(m_repoId, "UnitArrayTypeTable"));
1318     txn.prepare(*this, insertQuery);
1319   }
1320   RepoTxnQuery query(txn, *this);
1321   query.bindInt64("@unitSn", unitSn);
1322   BlobEncoder dataBlob{ue.useGlobalIds()};
1323   dataBlob(ue.m_arrayTypeTable);
1324   query.bindBlob("@arrayTypeTable", dataBlob, /* static */ true);
1325   query.exec();
1326 }
1327
1328 void UnitRepoProxy::GetUnitArrayTypeTableStmt
1329                   ::get(UnitEmitter& ue) {
1330   auto txn = RepoTxn{m_repo.begin()};
1331   if (!prepared()) {
1332     auto selectQuery = folly::sformat(
1333       "SELECT unitSn, arrayTypeTable FROM {} WHERE unitSn == @unitSn;",
1334       m_repo.table(m_repoId, "UnitArrayTypeTable"));
1335     txn.prepare(*this, selectQuery);
1336   }
1337
1338   RepoTxnQuery query(txn, *this);
1339   query.bindInt64("@unitSn", ue.m_sn);
1340
1341   query.step();
1342   assertx(query.row());
1343   BlobDecoder dataBlob = query.getBlob(1, ue.useGlobalIds());
1344   dataBlob(ue.m_arrayTypeTable);
1345   dataBlob.assertDone();
1346   query.step();
1347   assertx(query.done());
1348
1349   txn.commit();
1350 }
1351
1352 void UnitRepoProxy::InsertUnitArrayStmt
1353                   ::insert(RepoTxn& txn, int64_t unitSn, Id arrayId,
1354                            const std::string& array) {
1355   if (!prepared()) {
1356     auto insertQuery = folly::sformat(
1357       "INSERT INTO {} VALUES(@unitSn, @arrayId, @array);",
1358       m_repo.table(m_repoId, "UnitArray"));
1359     txn.prepare(*this, insertQuery);
1360   }
1361   RepoTxnQuery query(txn, *this);
1362   query.bindInt64("@unitSn", unitSn);
1363   query.bindId("@arrayId", arrayId);
1364   query.bindStdString("@array", array);
1365   query.exec();
1366 }
1367
1368 void UnitRepoProxy::GetUnitArraysStmt
1369                   ::get(UnitEmitter& ue) {
1370   auto txn = RepoTxn{m_repo.begin()};
1371   if (!prepared()) {
1372     auto selectQuery = folly::sformat(
1373       "SELECT arrayId, array FROM {} "
1374       " WHERE unitSn == @unitSn ORDER BY arrayId ASC;",
1375       m_repo.table(m_repoId, "UnitArray"));
1376     txn.prepare(*this, selectQuery);
1377   }
1378   RepoTxnQuery query(txn, *this);
1379   query.bindInt64("@unitSn", ue.m_sn);
1380   do {
1381     query.step();
1382     if (query.row()) {
1383       // We check that arrays do not exceed a configurable maximum size in the
1384       // assembler, so just assume that they're okay here.
1385       MemoryManager::SuppressOOM so(*tl_heap);
1386
1387       Id arrayId;        /**/ query.getId(0, arrayId);
1388       std::string key;   /**/ query.getStdString(1, key);
1389
1390       Variant v = [&]{
1391         VariableUnserializer vu{
1392           key.data(),
1393           key.size(),
1394           VariableUnserializer::Type::Internal
1395         };
1396         vu.setUnitFilename(ue.m_filepath);
1397         return vu.unserialize();
1398       }();
1399       assertx(v.isArray());
1400       ArrayData* ad = v.detach().m_data.parr;
1401       ArrayData::GetScalarArray(&ad);
1402       Id id DEBUG_ONLY = ue.mergeArray(ad);
1403       assertx(id == arrayId);
1404     }
1405   } while (!query.done());
1406   txn.commit();
1407 }
1408
1409 void UnitRepoProxy::InsertUnitMergeableStmt
1410                   ::insert(RepoTxn& txn, int64_t unitSn,
1411                            int ix, Unit::MergeKind kind, Id id) {
1412   assertx(kind == MergeKind::TypeAlias ||
1413           kind == MergeKind::Define ||
1414           kind == MergeKind::Record);
1415   if (!prepared()) {
1416     auto insertQuery = folly::sformat(
1417       "INSERT INTO {} VALUES("
1418       " @unitSn, @mergeableIx, @mergeableKind, @mergeableId);",
1419       m_repo.table(m_repoId, "UnitMergeables"));
1420     txn.prepare(*this, insertQuery);
1421   }
1422
1423   RepoTxnQuery query(txn, *this);
1424   query.bindInt64("@unitSn", unitSn);
1425   query.bindInt("@mergeableIx", ix);
1426   query.bindInt("@mergeableKind", (int)kind);
1427   query.bindId("@mergeableId", id);
1428   query.exec();
1429 }
1430
1431 void UnitRepoProxy::GetUnitMergeablesStmt
1432                   ::get(UnitEmitter& ue) {
1433   auto txn = RepoTxn{m_repo.begin()};
1434   if (!prepared()) {
1435     auto selectQuery = folly::sformat(
1436       "SELECT mergeableIx, mergeableKind, mergeableId "
1437       "FROM {} "
1438       "WHERE unitSn == @unitSn ORDER BY mergeableIx ASC;",
1439       m_repo.table(m_repoId, "UnitMergeables"));
1440     txn.prepare(*this, selectQuery);
1441   }
1442   RepoTxnQuery query(txn, *this);
1443   query.bindInt64("@unitSn", ue.m_sn);
1444   do {
1445     query.step();
1446     if (query.row()) {
1447       int mergeableIx;           /**/ query.getInt(0, mergeableIx);
1448       int mergeableKind;         /**/ query.getInt(1, mergeableKind);
1449       Id mergeableId;            /**/ query.getInt(2, mergeableId);
1450
1451       auto k = MergeKind(mergeableKind);
1452
1453       if (UNLIKELY(!RuntimeOption::RepoAuthoritative)) {
1454         /*
1455          * We're using a repo generated in WholeProgram mode,
1456          * but we're not using it in RepoAuthoritative mode
1457          * (this is dodgy to start with). We're not going to
1458          * deal with requires at merge time, so drop them
1459          * here, and clear the mergeOnly flag for the unit.
1460          * The two exceptions are persistent constants and
1461          * TypeAliases which are allowed in systemlib.
1462          */
1463         if ((k != MergeKind::Define && k != MergeKind::TypeAlias)
1464             || SystemLib::s_inited) {
1465           ue.m_mergeOnly = false;
1466         }
1467       }
1468       switch (k) {
1469         case MergeKind::Define:
1470         case MergeKind::TypeAlias:
1471           ue.insertMergeableId(k, mergeableIx, mergeableId);
1472           break;
1473         case MergeKind::Record:
1474           ue.insertMergeableRecord(mergeableIx, mergeableId);
1475           break;
1476         default: break;
1477       }
1478     }
1479   } while (!query.done());
1480   txn.commit();
1481 }
1482
1483 void UnitRepoProxy::InsertUnitLineTableStmt
1484                   ::insert(RepoTxn& txn,
1485                            int64_t unitSn,
1486                            LineTable& lineTable) {
1487   if (!prepared()) {
1488     auto insertQuery = folly::sformat(
1489       "INSERT INTO {} VALUES(@unitSn, @data);",
1490       m_repo.table(m_repoId, "UnitLineTable"));
1491     txn.prepare(*this, insertQuery);
1492   }
1493
1494   BlobEncoder dataBlob{false};
1495   RepoTxnQuery query(txn, *this);
1496   query.bindInt64("@unitSn", unitSn);
1497   dataBlob(
1498     lineTable,
1499     [&](const LineEntry& prev, const LineEntry& cur) -> LineEntry {
1500       return LineEntry {
1501         cur.pastOffset() - prev.pastOffset(),
1502         cur.val() - prev.val()
1503       };
1504     }
1505   );
1506
1507   query.bindBlob("@data", dataBlob, /* static */ true);
1508   query.exec();
1509 }
1510
1511 void UnitRepoProxy::GetUnitLineTableStmt::get(int64_t unitSn,
1512                                               LineTable& lineTable) {
1513   auto txn = RepoTxn{m_repo.begin()};
1514   if (!prepared()) {
1515     auto selectQuery = folly::sformat(
1516       "SELECT data FROM {} WHERE unitSn == @unitSn;",
1517       m_repo.table(m_repoId, "UnitLineTable"));
1518     txn.prepare(*this, selectQuery);
1519   }
1520   RepoTxnQuery query(txn, *this);
1521   query.bindInt64("@unitSn", unitSn);
1522   query.step();
1523   if (query.row()) {
1524     BlobDecoder dataBlob = query.getBlob(0, false);
1525     dataBlob(
1526       lineTable,
1527       [&](const LineEntry& prev, const LineEntry& delta) -> LineEntry {
1528         return LineEntry {
1529           delta.pastOffset() + prev.pastOffset(),
1530           delta.val() + prev.val()
1531         };
1532       }
1533     );
1534   }
1535   txn.commit();
1536 }
1537
1538 void UnitRepoProxy::InsertUnitTypeAliasStmt
1539                   ::insert(const UnitEmitter& ue,
1540                            RepoTxn& txn,
1541                            int64_t unitSn,
1542                            Id typeAliasId,
1543                            const TypeAlias& typeAlias) {
1544   if (!prepared()) {
1545     auto insertQuery = folly::sformat(
1546       "INSERT INTO {} VALUES (@unitSn, @typeAliasId, @name, @data);",
1547       m_repo.table(m_repoId, "UnitTypeAlias"));
1548     txn.prepare(*this, insertQuery);
1549   }
1550
1551   BlobEncoder dataBlob{ue.useGlobalIds()};
1552   RepoTxnQuery query(txn, *this);
1553   query.bindInt64("@unitSn", unitSn);
1554   query.bindInt64("@typeAliasId", typeAliasId);
1555   query.bindStaticString("@name", typeAlias.name);
1556
1557   dataBlob(typeAlias);
1558   query.bindBlob("@data", dataBlob, /* static */ true);
1559   query.exec();
1560 }
1561
1562 void UnitRepoProxy::GetUnitTypeAliasesStmt::get(UnitEmitter& ue) {
1563   auto txn = RepoTxn{m_repo.begin()};
1564   if (!prepared()) {
1565     auto selectQuery = folly::sformat(
1566       "SELECT typeAliasId, name, data FROM {} WHERE unitSn == @unitSn;",
1567       m_repo.table(m_repoId, "UnitTypeAlias"));
1568     txn.prepare(*this, selectQuery);
1569   }
1570   RepoTxnQuery query(txn, *this);
1571
1572   query.bindInt64("@unitSn", ue.m_sn);
1573   do {
1574     query.step();
1575     if (query.row()) {
1576       TypeAlias ta;
1577       Id typeAliasId;        /**/ query.getId(0, typeAliasId);
1578       StringData *name;      /**/ query.getStaticString(1, name);
1579       ta.name = makeStaticString(name);
1580       BlobDecoder dataBlob = /**/ query.getBlob(2, ue.useGlobalIds());
1581       dataBlob(ta);
1582       Id id UNUSED = ue.addTypeAlias(ta);
1583       assertx(id == typeAliasId);
1584     }
1585   } while (!query.done());
1586   txn.commit();
1587 }
1588
1589 void UnitRepoProxy::InsertUnitConstantStmt
1590                   ::insert(const UnitEmitter& ue,
1591                            RepoTxn& txn,
1592                            int64_t unitSn,
1593                            Id constantId,
1594                            const Constant& constant) {
1595   if (!prepared()) {
1596     auto insertQuery = folly::sformat(
1597       "INSERT INTO {} VALUES (@unitSn, @constantId, @name, @data);",
1598       m_repo.table(m_repoId, "UnitConstant"));
1599     txn.prepare(*this, insertQuery);
1600   }
1601
1602   BlobEncoder dataBlob{ue.useGlobalIds()};
1603   RepoTxnQuery query(txn, *this);
1604   query.bindInt64("@unitSn", unitSn);
1605   query.bindInt64("@constantId", constantId);
1606   query.bindStaticString("@name", constant.name);
1607
1608   dataBlob(constant);
1609   query.bindBlob("@data", dataBlob, /* static */ true);
1610   query.exec();
1611 }
1612
1613 void UnitRepoProxy::GetUnitConstantsStmt::get(UnitEmitter& ue) {
1614   auto txn = RepoTxn{m_repo.begin()};
1615   if (!prepared()) {
1616     auto selectQuery = folly::sformat(
1617       "SELECT constantId, name, data FROM {} WHERE unitSn == @unitSn;",
1618       m_repo.table(m_repoId, "UnitConstant"));
1619     txn.prepare(*this, selectQuery);
1620   }
1621   RepoTxnQuery query(txn, *this);
1622
1623   query.bindInt64("@unitSn", ue.m_sn);
1624   do {
1625     query.step();
1626     if (query.row()) {
1627       Constant c;
1628       Id constantId;        /**/ query.getId(0, constantId);
1629       StringData *name;      /**/ query.getStaticString(1, name);
1630       c.name = makeStaticString(name);
1631       BlobDecoder dataBlob = /**/ query.getBlob(2, ue.useGlobalIds());
1632
1633       // We check that arrays do not exceed a configurable maximum size in the
1634       // assembler, so just assume that they're okay here.
1635       MemoryManager::SuppressOOM so(*tl_heap);
1636
1637       dataBlob(c);
1638       if (type(c.val) == KindOfUninit) {
1639         c.val.m_data.pcnt = reinterpret_cast<MaybeCountable*>(Unit::getCns);
1640       }
1641       Id id UNUSED = ue.addConstant(c);
1642       assertx(id == constantId);
1643     }
1644   } while (!query.done());
1645   txn.commit();
1646 }
1647
1648 void UnitRepoProxy::InsertUnitSourceLocStmt
1649                   ::insert(RepoTxn& txn, int64_t unitSn, Offset pastOffset,
1650                            int line0, int char0, int line1, int char1) {
1651   if (!prepared()) {
1652     auto insertQuery = folly::sformat(
1653       "INSERT INTO {} "
1654       "VALUES(@unitSn, @pastOffset, @line0, @char0, @line1, @char1);",
1655       m_repo.table(m_repoId, "UnitSourceLoc"));
1656     txn.prepare(*this, insertQuery);
1657   }
1658   RepoTxnQuery query(txn, *this);
1659   query.bindInt64("@unitSn", unitSn);
1660   query.bindOffset("@pastOffset", pastOffset);
1661   query.bindInt("@line0", line0);
1662   query.bindInt("@char0", char0);
1663   query.bindInt("@line1", line1);
1664   query.bindInt("@char1", char1);
1665   query.exec();
1666 }
1667
1668 RepoStatus
1669 UnitRepoProxy::GetSourceLocTabStmt::get(int64_t unitSn,
1670                                         SourceLocTable& sourceLocTab) {
1671   try {
1672     auto txn = RepoTxn{m_repo.begin()};
1673     if (!prepared()) {
1674       auto selectQuery = folly::sformat(
1675         "SELECT pastOffset, line0, char0, line1, char1 "
1676         "FROM {} "
1677         "WHERE unitSn == @unitSn "
1678         "ORDER BY pastOffset ASC;",
1679         m_repo.table(m_repoId, "UnitSourceLoc"));
1680       txn.prepare(*this, selectQuery);
1681     }
1682     RepoTxnQuery query(txn, *this);
1683     query.bindInt64("@unitSn", unitSn);
1684     do {
1685       query.step();
1686       if (!query.row()) {
1687         return RepoStatus::error;
1688       }
1689       Offset pastOffset;
1690       query.getOffset(0, pastOffset);
1691       SourceLoc sLoc;
1692       query.getInt(1, sLoc.line0);
1693       query.getInt(2, sLoc.char0);
1694       query.getInt(3, sLoc.line1);
1695       query.getInt(4, sLoc.char1);
1696       SourceLocEntry entry(pastOffset, sLoc);
1697       sourceLocTab.push_back(entry);
1698     } while (!query.done());
1699     txn.commit();
1700   } catch (RepoExc& re) {
1701     return RepoStatus::error;
1702   }
1703   return RepoStatus::success;
1704 }
1705
1706 std::unique_ptr<UnitEmitter>
1707 createFatalUnit(StringData* filename, const SHA1& sha1, FatalOp op,
1708                 StringData* err, Location::Range loc) {
1709   auto ue = std::make_unique<UnitEmitter>(sha1, SHA1{}, Native::s_noNativeFuncs,
1710                                           false);
1711   ue->m_filepath = filename;
1712   ue->m_isHHFile = true;
1713   ue->initMain(1, 1);
1714   ue->recordSourceLocation(loc, ue->bcPos());
1715   ue->emitOp(OpString);
1716   ue->emitInt32(ue->mergeLitstr(err));
1717   ue->emitOp(OpFatal);
1718   ue->emitByte(static_cast<uint8_t>(op));
1719   FuncEmitter* fe = ue->getMain();
1720   fe->maxStackCells = 1;
1721   fe->finish(ue->bcPos());
1722   return ue;
1723 }
1724
1725 ///////////////////////////////////////////////////////////////////////////////
1726 }