hphp/runtime/vm/unit-emitter.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2014 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/unit-emitter.h"
  18
  19 #include "hphp/compiler/option.h"
  20 #include "hphp/parser/location.h"
  21 #include "hphp/system/systemlib.h"
  22
  23 #include "hphp/runtime/base/types.h"
  24 #include "hphp/runtime/base/array-data.h"
  25 #include "hphp/runtime/base/attr.h"
  26 #include "hphp/runtime/base/file-util.h"
  27 #include "hphp/runtime/base/runtime-option.h"
  28 #include "hphp/runtime/base/static-string-table.h"
  29 #include "hphp/runtime/base/typed-value.h"
  30
  31 #include "hphp/runtime/ext/std/ext_std_variable.h"
  32
  33 #include "hphp/runtime/vm/blob-helper.h"
  34 #include "hphp/runtime/vm/disas.h"
  35 #include "hphp/runtime/vm/func.h"
  36 #include "hphp/runtime/vm/func-emitter.h"
  37 #include "hphp/runtime/vm/litstr-table.h"
  38 #include "hphp/runtime/vm/native.h"
  39 #include "hphp/runtime/vm/preclass.h"
  40 #include "hphp/runtime/vm/preclass-emitter.h"
  41 #include "hphp/runtime/vm/repo.h"
  42 #include "hphp/runtime/vm/repo-helpers.h"
  43 #include "hphp/runtime/vm/unit.h"
  44 #include "hphp/runtime/vm/verifier/check.h"
  45
  46 #include "hphp/util/md5.h"
  47 #include "hphp/util/read-only-arena.h"
  48 #include "hphp/util/trace.h"
  49
  50 #include <boost/algorithm/string/predicate.hpp>
  51
  52 #include <folly/Memory.h>
  53
  54 #include <algorithm>
  55 #include <cstdio>
  56 #include <memory>
  57 #include <sstream>
  58 #include <string>
  59 #include <utility>
  60 #include <vector>
  61
  62 namespace HPHP {
  63 ///////////////////////////////////////////////////////////////////////////////
  64
  65 TRACE_SET_MOD(hhbc);
  66
  67 using MergeKind = Unit::MergeKind;
  68
  69 ///////////////////////////////////////////////////////////////////////////////
  70
  71 static ReadOnlyArena& get_readonly_arena() {
  72   static ReadOnlyArena arena(RuntimeOption::EvalHHBCArenaChunkSize);
  73   return arena;
  74 }
  75
  76 /*
  77  * Export for the admin server.
  78  */
  79 size_t hhbc_arena_capacity() {
  80   if (!RuntimeOption::RepoAuthoritative) return 0;
  81   return get_readonly_arena().capacity();
  82 }
  83
  84 ///////////////////////////////////////////////////////////////////////////////
  85
  86 UnitEmitter::UnitEmitter(const MD5& md5)
  87   : m_mainReturn(make_tv<KindOfUninit>())
  88   , m_md5(md5)
  89   , m_bc((unsigned char*)malloc(BCMaxInit))
  90   , m_bclen(0)
  91   , m_bcmax(BCMaxInit)
  92   , m_nextFuncSn(0)
  93   , m_allClassesHoistable(true)
  94 {}
  95
  96 UnitEmitter::~UnitEmitter() {
  97   if (m_bc) free(m_bc);
  98
  99   for (auto& fe : m_fes) delete fe;
 100   for (auto& pce : m_pceVec) delete pce;
 101 }
 102
 103
 104 ///////////////////////////////////////////////////////////////////////////////
 105 // Basic data.
 106
 107 void UnitEmitter::setBc(const unsigned char* bc, size_t bclen) {
 108   if (m_bc) {
 109     free(m_bc);
 110   }
 111   m_bc = (unsigned char*)malloc(bclen);
 112   m_bcmax = bclen;
 113   memcpy(m_bc, bc, bclen);
 114   m_bclen = bclen;
 115 }
 116
 117
 118 ///////////////////////////////////////////////////////////////////////////////
 119 // Litstrs and Arrays.
 120
 121 const StringData* UnitEmitter::lookupLitstr(Id id) const {
 122   if (isGlobalLitstrId(id)) {
 123     return LitstrTable::get().lookupLitstrId(decodeGlobalLitstrId(id));
 124   }
 125   assert(id < m_litstrs.size());
 126   return m_litstrs[id];
 127 }
 128
 129 const ArrayData* UnitEmitter::lookupArray(Id id) const {
 130   assert(id < m_arrays.size());
 131   return m_arrays[id].array;
 132 }
 133
 134 Id UnitEmitter::mergeLitstr(const StringData* litstr) {
 135   if (Option::WholeProgram) {
 136     return encodeGlobalLitstrId(LitstrTable::get().mergeLitstr(litstr));
 137   }
 138   return mergeUnitLitstr(litstr);
 139 }
 140
 141 Id UnitEmitter::mergeUnitLitstr(const StringData* litstr) {
 142   auto it = m_litstr2id.find(litstr);
 143   if (it == m_litstr2id.end()) {
 144     const StringData* str = makeStaticString(litstr);
 145     Id id = m_litstrs.size();
 146     m_litstrs.push_back(str);
 147     m_litstr2id[str] = id;
 148     return id;
 149   } else {
 150     return it->second;
 151   }
 152 }
 153
 154 Id UnitEmitter::mergeArray(const ArrayData* a) {
 155   Variant v(const_cast<ArrayData*>(a));
 156   auto key = HHVM_FN(serialize)(v).toCppString();
 157   return mergeArray(a, key);
 158 }
 159
 160 Id UnitEmitter::mergeArray(const ArrayData* a, const std::string& key) {
 161   auto const it = m_array2id.find(key);
 162   if (it != m_array2id.end()) {
 163     return it->second;
 164   }
 165   a = ArrayData::GetScalarArray(const_cast<ArrayData*>(a), key);
 166   Id id = m_arrays.size();
 167   ArrayVecElm ave = {key, a};
 168   m_arrays.push_back(ave);
 169   m_array2id[key] = id;
 170   return id;
 171 }
 172
 173
 174 ///////////////////////////////////////////////////////////////////////////////
 175 // FuncEmitters.
 176
 177 void UnitEmitter::initMain(int line1, int line2) {
 178   assert(m_fes.size() == 0);
 179   StringData* name = makeStaticString("");
 180   FuncEmitter* pseudomain = newFuncEmitter(name);
 181   Attr attrs = AttrMayUseVV;
 182   pseudomain->init(line1, line2, 0, attrs, false, name);
 183 }
 184
 185 void UnitEmitter::addTrivialPseudoMain() {
 186   initMain(0, 0);
 187   auto const mfe = getMain();
 188   emitOp(OpInt);
 189   emitInt64(1);
 190   emitOp(OpRetC);
 191   mfe->maxStackCells = 1;
 192   mfe->finish(bcPos(), false);
 193   recordFunction(mfe);
 194
 195   TypedValue mainReturn;
 196   mainReturn.m_data.num = 1;
 197   mainReturn.m_type = KindOfInt64;
 198   m_mainReturn = mainReturn;
 199   m_mergeOnly = true;
 200 }
 201
 202 FuncEmitter* UnitEmitter::newFuncEmitter(const StringData* name) {
 203   // The pseudomain comes first.
 204   assert(m_fes.size() > 0 || !strcmp(name->data(), ""));
 205
 206   FuncEmitter* fe = new FuncEmitter(*this, m_nextFuncSn++, m_fes.size(), name);
 207   m_fes.push_back(fe);
 208   return fe;
 209 }
 210
 211 FuncEmitter* UnitEmitter::newMethodEmitter(const StringData* name,
 212                                            PreClassEmitter* pce) {
 213   return new FuncEmitter(*this, m_nextFuncSn++, name, pce);
 214 }
 215
 216 void UnitEmitter::appendTopEmitter(FuncEmitter* fe) {
 217   fe->setIds(m_nextFuncSn++, m_fes.size());
 218   m_fes.push_back(fe);
 219 }
 220
 221 void UnitEmitter::recordFunction(FuncEmitter* fe) {
 222   m_feTab.push_back(std::make_pair(fe->past, fe));
 223 }
 224
 225 Func* UnitEmitter::newFunc(const FuncEmitter* fe, Unit& unit,
 226                            PreClass* preClass, int line1, int line2,
 227                            Offset base, Offset past,
 228                            const StringData* name, Attr attrs, bool top,
 229                            const StringData* docComment, int numParams,
 230                            bool needsNextClonedClosure) {
 231   Func* f = new (Func::allocFuncMem(name, numParams,
 232                                     needsNextClonedClosure,
 233                                     !preClass))
 234     Func(unit, name, attrs);
 235   m_fMap[fe] = f;
 236   return f;
 237 }
 238
 239
 240 ///////////////////////////////////////////////////////////////////////////////
 241 // PreClassEmitters.
 242
 243 PreClassEmitter* UnitEmitter::newPreClassEmitter(
 244   const StringData* name,
 245   PreClass::Hoistable hoistable
 246 ) {
 247   if (hoistable && m_hoistablePreClassSet.count(name)) {
 248     hoistable = PreClass::Mergeable;
 249   }
 250
 251   PreClassEmitter* pce = new PreClassEmitter(*this, m_pceVec.size(),
 252                                              name, hoistable);
 253
 254   if (hoistable >= PreClass::MaybeHoistable) {
 255     m_hoistablePreClassSet.insert(name);
 256     if (hoistable == PreClass::ClosureHoistable) {
 257       // Closures should appear at the VERY top of the file, so if any class in
 258       // the same file tries to use them, they are already defined. We had a
 259       // fun race where one thread was autoloading a file, finished parsing the
 260       // class, then another thread came along and saw the class was already
 261       // loaded and ran it before the first thread had time to parse the
 262       // closure class.
 263       m_hoistablePceIdList.push_front(pce->id());
 264     } else {
 265       m_hoistablePceIdList.push_back(pce->id());
 266     }
 267   } else {
 268     m_allClassesHoistable = false;
 269   }
 270   if (hoistable >= PreClass::Mergeable &&
 271       hoistable < PreClass::AlwaysHoistable) {
 272     if (m_returnSeen) {
 273       m_allClassesHoistable = false;
 274     } else {
 275       pushMergeableClass(pce);
 276     }
 277   }
 278   m_pceVec.push_back(pce);
 279   return pce;
 280 }
 281
 282
 283 ///////////////////////////////////////////////////////////////////////////////
 284 // Type aliases.
 285
 286 Id UnitEmitter::addTypeAlias(const TypeAlias& td) {
 287   Id id = m_typeAliases.size();
 288   m_typeAliases.push_back(td);
 289   return id;
 290 }
 291
 292
 293 ///////////////////////////////////////////////////////////////////////////////
 294 // Source locations.
 295
 296 SourceLocTable UnitEmitter::createSourceLocTable() const {
 297   SourceLocTable locations;
 298   for (size_t i = 0; i < m_sourceLocTab.size(); ++i) {
 299     Offset endOff = i < m_sourceLocTab.size() - 1
 300       ? m_sourceLocTab[i + 1].first
 301       : m_bclen;
 302     locations.push_back(SourceLocEntry(endOff, m_sourceLocTab[i].second));
 303   }
 304   return locations;
 305 }
 306
 307 namespace {
 308
 309 using SrcLoc = std::vector<std::pair<Offset, SourceLoc>>;
 310
 311 /*
 312  * Create a LineTable from `srcLoc'.
 313  */
 314 LineTable createLineTable(SrcLoc& srcLoc, Offset bclen) {
 315   LineTable lines;
 316   for (size_t i = 0; i < srcLoc.size(); ++i) {
 317     Offset endOff = i < srcLoc.size() - 1 ? srcLoc[i + 1].first : bclen;
 318     lines.push_back(LineEntry(endOff, srcLoc[i].second.line1));
 319   }
 320   return lines;
 321 }
 322
 323 }
 324
 325 void UnitEmitter::recordSourceLocation(const Location* sLoc, Offset start) {
 326   // Some byte codes, such as for the implicit "return 0" at the end of a
 327   // a source file do not have valid source locations. This check makes
 328   // sure we don't record a (dummy) source location in this case.
 329   if (start > 0 && sLoc->line0 == 1 && sLoc->char0 == 1 &&
 330     sLoc->line1 == 1 && sLoc->char1 == 1 && strlen(sLoc->file) == 0) return;
 331   SourceLoc newLoc(*sLoc);
 332   if (!m_sourceLocTab.empty()) {
 333     if (m_sourceLocTab.back().second == newLoc) {
 334       // Combine into the interval already at the back of the vector.
 335       assert(start >= m_sourceLocTab.back().first);
 336       return;
 337     }
 338     assert(m_sourceLocTab.back().first < start &&
 339            "source location offsets must be added to UnitEmitter in "
 340            "increasing order");
 341   } else {
 342     // First record added should be for bytecode offset zero.
 343     assert(start == 0);
 344   }
 345   m_sourceLocTab.push_back(std::make_pair(start, newLoc));
 346 }
 347
 348
 349 ///////////////////////////////////////////////////////////////////////////////
 350 // Mergeables.
 351
 352 void UnitEmitter::pushMergeableClass(PreClassEmitter* e) {
 353   m_mergeableStmts.push_back(std::make_pair(MergeKind::Class, e->id()));
 354 }
 355
 356 void UnitEmitter::pushMergeableInclude(Unit::MergeKind kind,
 357                                        const StringData* unitName) {
 358   m_mergeableStmts.push_back(
 359     std::make_pair(kind, mergeLitstr(unitName)));
 360   m_allClassesHoistable = false;
 361 }
 362
 363 void UnitEmitter::insertMergeableInclude(int ix, Unit::MergeKind kind, Id id) {
 364   assert(size_t(ix) <= m_mergeableStmts.size());
 365   m_mergeableStmts.insert(m_mergeableStmts.begin() + ix,
 366                           std::make_pair(kind, id));
 367   m_allClassesHoistable = false;
 368 }
 369
 370 void UnitEmitter::pushMergeableDef(Unit::MergeKind kind,
 371                                    const StringData* name,
 372                                    const TypedValue& tv) {
 373   m_mergeableStmts.push_back(std::make_pair(kind, m_mergeableValues.size()));
 374   m_mergeableValues.push_back(std::make_pair(mergeLitstr(name), tv));
 375   m_allClassesHoistable = false;
 376 }
 377
 378 void UnitEmitter::insertMergeableDef(int ix, Unit::MergeKind kind,
 379                                      Id id, const TypedValue& tv) {
 380   assert(size_t(ix) <= m_mergeableStmts.size());
 381   m_mergeableStmts.insert(m_mergeableStmts.begin() + ix,
 382                           std::make_pair(kind, m_mergeableValues.size()));
 383   m_mergeableValues.push_back(std::make_pair(id, tv));
 384   m_allClassesHoistable = false;
 385 }
 386
 387
 388 ///////////////////////////////////////////////////////////////////////////////
 389 // Initialization and execution.
 390
 391 void UnitEmitter::commit(UnitOrigin unitOrigin) {
 392   Repo& repo = Repo::get();
 393   try {
 394     RepoTxn txn(repo);
 395     bool err = insert(unitOrigin, txn);
 396     if (!err) {
 397       txn.commit();
 398     }
 399   } catch (RepoExc& re) {
 400     int repoId = repo.repoIdForNewUnit(unitOrigin);
 401     if (repoId != RepoIdInvalid) {
 402       TRACE(3, "Failed to commit '%s' (0x%016" PRIx64 "%016" PRIx64 ") to '%s': %s\n",
 403                m_filepath->data(), m_md5.q[0], m_md5.q[1],
 404                repo.repoName(repoId).c_str(), re.msg().c_str());
 405     }
 406   }
 407 }
 408
 409 bool UnitEmitter::insert(UnitOrigin unitOrigin, RepoTxn& txn) {
 410   Repo& repo = Repo::get();
 411   UnitRepoProxy& urp = repo.urp();
 412   int repoId = Repo::get().repoIdForNewUnit(unitOrigin);
 413   if (repoId == RepoIdInvalid) {
 414     return true;
 415   }
 416   m_repoId = repoId;
 417
 418   try {
 419     {
 420       m_lineTable = createLineTable(m_sourceLocTab, m_bclen);
 421       urp.insertUnit[repoId].insert(*this, txn, m_sn, m_md5, m_bc,
 422                                     m_bclen);
 423     }
 424     int64_t usn = m_sn;
 425     urp.insertUnitLineTable(repoId, txn, usn, m_lineTable);
 426     for (unsigned i = 0; i < m_litstrs.size(); ++i) {
 427       urp.insertUnitLitstr[repoId].insert(txn, usn, i, m_litstrs[i]);
 428     }
 429     for (unsigned i = 0; i < m_arrays.size(); ++i) {
 430       urp.insertUnitArray[repoId].insert(txn, usn, i,
 431                                          m_arrays[i].serialized);
 432     }
 433     for (auto& fe : m_fes) {
 434       fe->commit(txn);
 435     }
 436     for (auto& pce : m_pceVec) {
 437       pce->commit(txn);
 438     }
 439
 440     for (int i = 0, n = m_mergeableStmts.size(); i < n; i++) {
 441       switch (m_mergeableStmts[i].first) {
 442         case MergeKind::Done:
 443         case MergeKind::UniqueDefinedClass:
 444           not_reached();
 445         case MergeKind::Class: break;
 446         case MergeKind::ReqDoc: {
 447           urp.insertUnitMergeable[repoId].insert(
 448             txn, usn, i,
 449             m_mergeableStmts[i].first, m_mergeableStmts[i].second, nullptr);
 450           break;
 451         }
 452         case MergeKind::Define:
 453         case MergeKind::PersistentDefine:
 454         case MergeKind::Global: {
 455           int ix = m_mergeableStmts[i].second;
 456           urp.insertUnitMergeable[repoId].insert(
 457             txn, usn, i,
 458             m_mergeableStmts[i].first,
 459             m_mergeableValues[ix].first, &m_mergeableValues[ix].second);
 460           break;
 461         }
 462       }
 463     }
 464     if (RuntimeOption::RepoDebugInfo) {
 465       for (size_t i = 0; i < m_sourceLocTab.size(); ++i) {
 466         SourceLoc& e = m_sourceLocTab[i].second;
 467         Offset endOff = i < m_sourceLocTab.size() - 1
 468                           ? m_sourceLocTab[i + 1].first
 469                           : m_bclen;
 470
 471         urp.insertUnitSourceLoc[repoId]
 472            .insert(txn, usn, endOff, e.line0, e.char0, e.line1, e.char1);
 473       }
 474     }
 475     return false;
 476   } catch (RepoExc& re) {
 477     TRACE(3, "Failed to commit '%s' (0x%016" PRIx64 "%016" PRIx64 ") to '%s': %s\n",
 478              m_filepath->data(), m_md5.q[0], m_md5.q[1],
 479              repo.repoName(repoId).c_str(), re.msg().c_str());
 480     return true;
 481   }
 482 }
 483
 484 static const unsigned char*
 485 allocateBCRegion(const unsigned char* bc, size_t bclen) {
 486   if (RuntimeOption::RepoAuthoritative) {
 487     // In RepoAuthoritative, we assume we won't ever deallocate units
 488     // and that this is read-only, mostly cold data.  So we throw it
 489     // in a bump-allocator that's mprotect'd to prevent writes.
 490     return static_cast<const unsigned char*>(
 491       get_readonly_arena().allocate(bc, bclen)
 492     );
 493   }
 494   auto mem = static_cast<unsigned char*>(malloc(bclen));
 495   std::copy(bc, bc + bclen, mem);
 496   return mem;
 497 }
 498
 499 std::unique_ptr<Unit> UnitEmitter::create() {
 500   auto u = folly::make_unique<Unit>();
 501   u->m_repoId = m_repoId;
 502   u->m_sn = m_sn;
 503   u->m_bc = allocateBCRegion(m_bc, m_bclen);
 504   u->m_bclen = m_bclen;
 505   u->m_filepath = m_filepath;
 506   u->m_mainReturn = m_mainReturn;
 507   u->m_mergeOnly = m_mergeOnly;
 508   u->m_isHHFile = m_isHHFile;
 509   {
 510     const std::string& dirname = FileUtil::safe_dirname(m_filepath->data(),
 511                                                         m_filepath->size());
 512     u->m_dirpath = makeStaticString(dirname);
 513   }
 514   u->m_md5 = m_md5;
 515   for (unsigned i = 0; i < m_litstrs.size(); ++i) {
 516     NamedEntityPair np;
 517     np.first = m_litstrs[i];
 518     np.second = nullptr;
 519     u->m_namedInfo.push_back(np);
 520   }
 521   u->m_arrays = [&]() -> std::vector<const ArrayData*> {
 522     auto ret = std::vector<const ArrayData*>{};
 523     for (unsigned i = 0; i < m_arrays.size(); ++i) {
 524       ret.push_back(m_arrays[i].array);
 525     }
 526     return ret;
 527   }();
 528   for (auto const& pce : m_pceVec) {
 529     u->m_preClasses.push_back(PreClassPtr(pce->create(*u)));
 530   }
 531   u->m_typeAliases = m_typeAliases;
 532
 533   size_t ix = m_fes.size() + m_hoistablePceIdList.size();
 534   if (m_mergeOnly && !m_allClassesHoistable) {
 535     size_t extra = 0;
 536     for (auto& mergeable : m_mergeableStmts) {
 537       extra++;
 538       if (!RuntimeOption::RepoAuthoritative && SystemLib::s_inited) {
 539         if (mergeable.first != MergeKind::Class) {
 540           extra = 0;
 541           u->m_mergeOnly = false;
 542           break;
 543         }
 544       } else {
 545         switch (mergeable.first) {
 546           case MergeKind::PersistentDefine:
 547           case MergeKind::Define:
 548           case MergeKind::Global:
 549             extra += sizeof(TypedValueAux) / sizeof(void*);
 550             break;
 551           default:
 552             break;
 553         }
 554       }
 555     }
 556     ix += extra;
 557   }
 558   Unit::MergeInfo *mi = Unit::MergeInfo::alloc(ix);
 559   u->m_mergeInfo = mi;
 560   ix = 0;
 561   for (auto& fe : m_fes) {
 562     Func* func = fe->create(*u);
 563     if (func->top()) {
 564       if (!mi->m_firstHoistableFunc) {
 565         mi->m_firstHoistableFunc = ix;
 566       }
 567     } else {
 568       assert(!mi->m_firstHoistableFunc);
 569     }
 570     mi->mergeableObj(ix++) = func;
 571   }
 572   assert(u->getMain()->isPseudoMain());
 573   if (!mi->m_firstHoistableFunc) {
 574     mi->m_firstHoistableFunc =  ix;
 575   }
 576   mi->m_firstHoistablePreClass = ix;
 577   assert(m_fes.size());
 578   for (auto& id : m_hoistablePceIdList) {
 579     mi->mergeableObj(ix++) = u->m_preClasses[id].get();
 580   }
 581   mi->m_firstMergeablePreClass = ix;
 582   if (u->m_mergeOnly && !m_allClassesHoistable) {
 583     for (auto& mergeable : m_mergeableStmts) {
 584       switch (mergeable.first) {
 585         case MergeKind::Class:
 586           mi->mergeableObj(ix++) = u->m_preClasses[mergeable.second].get();
 587           break;
 588         case MergeKind::ReqDoc: {
 589           assert(RuntimeOption::RepoAuthoritative);
 590           void* name = u->lookupLitstrId(mergeable.second);
 591           mi->mergeableObj(ix++) = (char*)name + (int)mergeable.first;
 592           break;
 593         }
 594         case MergeKind::Define:
 595         case MergeKind::Global:
 596           assert(RuntimeOption::RepoAuthoritative);
 597         case MergeKind::PersistentDefine: {
 598           void* name = u->lookupLitstrId
 599             (m_mergeableValues[mergeable.second].first);
 600           mi->mergeableObj(ix++) = (char*)name + (int)mergeable.first;
 601           auto& tv = m_mergeableValues[mergeable.second].second;
 602           auto* tva = (TypedValueAux*)mi->mergeableData(ix);
 603           tva->m_data = tv.m_data;
 604           tva->m_type = tv.m_type;
 605           // leave tva->m_aux uninitialized
 606           ix += sizeof(*tva) / sizeof(void*);
 607           assert(sizeof(*tva) % sizeof(void*) == 0);
 608           break;
 609         }
 610         case MergeKind::Done:
 611         case MergeKind::UniqueDefinedClass:
 612           not_reached();
 613       }
 614     }
 615   }
 616   assert(ix == mi->m_mergeablesSize);
 617   mi->mergeableObj(ix) = (void*)MergeKind::Done;
 618
 619   /*
 620    * What's going on is we're going to have a m_lineTable if this UnitEmitter
 621    * was loaded from the repo, and no m_sourceLocTab (it's demand-loaded by
 622    * unit.cpp because it's only used for the debugger).  Don't bother creating
 623    * the line table here, because we can retrieve it from the repo later.
 624    *
 625    * On the other hand, if this unit was just created by parsing a php file (or
 626    * whatnot) which was not committed to the repo, we'll have a m_sourceLocTab.
 627    * In this case we should populate m_lineTable (otherwise we might lose line
 628    * info altogether, since it may not be backed by a repo).
 629    */
 630   if (m_sourceLocTab.size() != 0) {
 631     stashLineTable(u.get(), createLineTable(m_sourceLocTab, m_bclen));
 632   }
 633
 634   for (size_t i = 0; i < m_feTab.size(); ++i) {
 635     assert(m_feTab[i].second->past == m_feTab[i].first);
 636     assert(m_fMap.find(m_feTab[i].second) != m_fMap.end());
 637     u->m_funcTable.push_back(
 638       FuncEntry(m_feTab[i].first, m_fMap.find(m_feTab[i].second)->second));
 639   }
 640
 641   // Funcs can be recorded out of order when loading them from the
 642   // repo currently.  So sort 'em here.
 643   std::sort(u->m_funcTable.begin(), u->m_funcTable.end());
 644
 645   m_fMap.clear();
 646
 647   if (RuntimeOption::EvalDumpBytecode) {
 648     // Dump human-readable bytecode.
 649     Trace::traceRelease("%s", u->toString().c_str());
 650   }
 651   if (RuntimeOption::EvalDumpHhas && SystemLib::s_inited) {
 652     std::printf("%s", disassemble(u.get()).c_str());
 653     std::fflush(stdout);
 654     _Exit(0);
 655   }
 656
 657   static const bool kVerify = debug || getenv("HHVM_VERIFY");
 658   static const bool kVerifyVerboseSystem =
 659     getenv("HHVM_VERIFY_VERBOSE_SYSTEM");
 660   static const bool kVerifyVerbose =
 661     kVerifyVerboseSystem || getenv("HHVM_VERIFY_VERBOSE");
 662
 663   const bool isSystemLib = u->filepath()->empty() ||
 664     boost::ends_with(u->filepath()->data(), "systemlib.php");
 665   const bool doVerify =
 666     kVerify || boost::ends_with(u->filepath()->data(), "hhas");
 667   if (doVerify) {
 668     Verifier::checkUnit(
 669       u.get(),
 670       isSystemLib ? kVerifyVerboseSystem : kVerifyVerbose
 671     );
 672   }
 673
 674   return u;
 675 }
 676
 677 template<class SerDe>
 678 void UnitEmitter::serdeMetaData(SerDe& sd) {
 679   sd(m_mainReturn)
 680     (m_mergeOnly)
 681     (m_isHHFile)
 682     (m_typeAliases)
 683     (m_preloadPriority)
 684     ;
 685 }
 686
 687
 688 ///////////////////////////////////////////////////////////////////////////////
 689 // UnitRepoProxy.
 690
 691 UnitRepoProxy::UnitRepoProxy(Repo& repo)
 692     : RepoProxy(repo)
 693 #define URP_OP(c, o) \
 694     , o{c##Stmt(repo, 0), c##Stmt(repo, 1)}
 695     URP_OPS
 696 #undef URP_OP
 697 {}
 698
 699 UnitRepoProxy::~UnitRepoProxy() {
 700 }
 701
 702 void UnitRepoProxy::createSchema(int repoId, RepoTxn& txn) {
 703   {
 704     std::stringstream ssCreate;
 705     ssCreate << "CREATE TABLE " << m_repo.table(repoId, "Unit")
 706              << "(unitSn INTEGER PRIMARY KEY, md5 BLOB, bc BLOB, data BLOB, "
 707                 "UNIQUE (md5));";
 708     txn.exec(ssCreate.str());
 709   }
 710   {
 711     std::stringstream ssCreate;
 712     ssCreate << "CREATE TABLE " << m_repo.table(repoId, "UnitLitstr")
 713              << "(unitSn INTEGER, litstrId INTEGER, litstr TEXT,"
 714                 " PRIMARY KEY (unitSn, litstrId));";
 715     txn.exec(ssCreate.str());
 716   }
 717   {
 718     std::stringstream ssCreate;
 719     ssCreate << "CREATE TABLE " << m_repo.table(repoId, "UnitArray")
 720              << "(unitSn INTEGER, arrayId INTEGER, array BLOB,"
 721                 " PRIMARY KEY (unitSn, arrayId));";
 722     txn.exec(ssCreate.str());
 723   }
 724   {
 725     std::stringstream ssCreate;
 726     ssCreate << "CREATE TABLE " << m_repo.table(repoId, "UnitMergeables")
 727              << "(unitSn INTEGER, mergeableIx INTEGER,"
 728                 " mergeableKind INTEGER, mergeableId INTEGER,"
 729                 " mergeableValue BLOB,"
 730                 " PRIMARY KEY (unitSn, mergeableIx));";
 731     txn.exec(ssCreate.str());
 732   }
 733   {
 734     std::stringstream ssCreate;
 735     ssCreate << "CREATE TABLE " << m_repo.table(repoId, "UnitSourceLoc")
 736              << "(unitSn INTEGER, pastOffset INTEGER, line0 INTEGER,"
 737                 " char0 INTEGER, line1 INTEGER, char1 INTEGER,"
 738                 " PRIMARY KEY (unitSn, pastOffset));";
 739     txn.exec(ssCreate.str());
 740   }
 741   {
 742     std::stringstream ssCreate;
 743     ssCreate << "CREATE TABLE " << m_repo.table(repoId, "UnitLineTable")
 744              << "(unitSn INTEGER PRIMARY KEY, data BLOB);";
 745     txn.exec(ssCreate.str());
 746   }
 747 }
 748
 749 bool UnitRepoProxy::loadHelper(UnitEmitter& ue,
 750                                const std::string& name,
 751                                const MD5& md5) {
 752   ue.m_filepath = makeStaticString(name);
 753   // Look for a repo that contains a unit with matching MD5.
 754   int repoId;
 755   for (repoId = RepoIdCount - 1; repoId >= 0; --repoId) {
 756     if (!getUnit[repoId].get(ue, md5)) {
 757       break;
 758     }
 759   }
 760   if (repoId < 0) {
 761     TRACE(3, "No repo contains '%s' (0x%016" PRIx64  "%016" PRIx64 ")\n",
 762              name.c_str(), md5.q[0], md5.q[1]);
 763     return false;
 764   }
 765   try {
 766     getUnitLitstrs[repoId].get(ue);
 767     getUnitArrays[repoId].get(ue);
 768     m_repo.pcrp().getPreClasses[repoId].get(ue);
 769     getUnitMergeables[repoId].get(ue);
 770     getUnitLineTable(repoId, ue.m_sn, ue.m_lineTable);
 771     m_repo.frp().getFuncs[repoId].get(ue);
 772   } catch (RepoExc& re) {
 773     TRACE(0,
 774           "Repo error loading '%s' (0x%016" PRIx64 "%016"
 775           PRIx64 ") from '%s': %s\n",
 776           name.c_str(), md5.q[0], md5.q[1], m_repo.repoName(repoId).c_str(),
 777           re.msg().c_str());
 778     return false;
 779   }
 780   TRACE(3, "Repo loaded '%s' (0x%016" PRIx64 "%016" PRIx64 ") from '%s'\n",
 781            name.c_str(), md5.q[0], md5.q[1], m_repo.repoName(repoId).c_str());
 782   return true;
 783 }
 784
 785 std::unique_ptr<UnitEmitter>
 786 UnitRepoProxy::loadEmitter(const std::string& name, const MD5& md5) {
 787   auto ue = folly::make_unique<UnitEmitter>(md5);
 788   if (!loadHelper(*ue, name, md5)) ue.reset();
 789   return ue;
 790 }
 791
 792 std::unique_ptr<Unit>
 793 UnitRepoProxy::load(const std::string& name, const MD5& md5) {
 794   UnitEmitter ue(md5);
 795   if (!loadHelper(ue, name, md5)) return nullptr;
 796   return ue.create();
 797 }
 798
 799 void UnitRepoProxy::InsertUnitStmt
 800                   ::insert(const UnitEmitter& ue,
 801                            RepoTxn& txn, int64_t& unitSn, const MD5& md5,
 802                            const unsigned char* bc, size_t bclen) {
 803   BlobEncoder dataBlob;
 804
 805   if (!prepared()) {
 806     std::stringstream ssInsert;
 807     ssInsert << "INSERT INTO " << m_repo.table(m_repoId, "Unit")
 808              << " VALUES(NULL, @md5, @bc, @data);";
 809     txn.prepare(*this, ssInsert.str());
 810   }
 811   RepoTxnQuery query(txn, *this);
 812   query.bindMd5("@md5", md5);
 813   query.bindBlob("@bc", (const void*)bc, bclen);
 814   const_cast<UnitEmitter&>(ue).serdeMetaData(dataBlob);
 815   query.bindBlob("@data", dataBlob, /* static */ true);
 816   query.exec();
 817   unitSn = query.getInsertedRowid();
 818 }
 819
 820 bool UnitRepoProxy::GetUnitStmt
 821                   ::get(UnitEmitter& ue, const MD5& md5) {
 822   try {
 823     RepoTxn txn(m_repo);
 824     if (!prepared()) {
 825       std::stringstream ssSelect;
 826       ssSelect << "SELECT unitSn,bc,data FROM "
 827                << m_repo.table(m_repoId, "Unit")
 828                << " WHERE md5 == @md5;";
 829       txn.prepare(*this, ssSelect.str());
 830     }
 831     RepoTxnQuery query(txn, *this);
 832     query.bindMd5("@md5", md5);
 833     query.step();
 834     if (!query.row()) {
 835       return true;
 836     }
 837     int64_t unitSn;                     /**/ query.getInt64(0, unitSn);
 838     const void* bc; size_t bclen;       /**/ query.getBlob(1, bc, bclen);
 839     BlobDecoder dataBlob =              /**/ query.getBlob(2);
 840
 841     ue.m_repoId = m_repoId;
 842     ue.m_sn = unitSn;
 843     ue.setBc(static_cast<const unsigned char*>(bc), bclen);
 844     ue.serdeMetaData(dataBlob);
 845
 846     txn.commit();
 847   } catch (RepoExc& re) {
 848     return true;
 849   }
 850   return false;
 851 }
 852
 853 void UnitRepoProxy::InsertUnitLitstrStmt
 854                   ::insert(RepoTxn& txn, int64_t unitSn, Id litstrId,
 855                            const StringData* litstr) {
 856   if (!prepared()) {
 857     std::stringstream ssInsert;
 858     ssInsert << "INSERT INTO " << m_repo.table(m_repoId, "UnitLitstr")
 859              << " VALUES(@unitSn, @litstrId, @litstr);";
 860     txn.prepare(*this, ssInsert.str());
 861   }
 862   RepoTxnQuery query(txn, *this);
 863   query.bindInt64("@unitSn", unitSn);
 864   query.bindId("@litstrId", litstrId);
 865   query.bindStaticString("@litstr", litstr);
 866   query.exec();
 867 }
 868
 869 void UnitRepoProxy::GetUnitLitstrsStmt
 870                   ::get(UnitEmitter& ue) {
 871   RepoTxn txn(m_repo);
 872   if (!prepared()) {
 873     std::stringstream ssSelect;
 874     ssSelect << "SELECT litstrId,litstr FROM "
 875              << m_repo.table(m_repoId, "UnitLitstr")
 876              << " WHERE unitSn == @unitSn ORDER BY litstrId ASC;";
 877     txn.prepare(*this, ssSelect.str());
 878   }
 879   RepoTxnQuery query(txn, *this);
 880   query.bindInt64("@unitSn", ue.m_sn);
 881   do {
 882     query.step();
 883     if (query.row()) {
 884       Id litstrId;        /**/ query.getId(0, litstrId);
 885       StringData* litstr; /**/ query.getStaticString(1, litstr);
 886       Id id UNUSED = ue.mergeUnitLitstr(litstr);
 887       assert(id == litstrId);
 888     }
 889   } while (!query.done());
 890   txn.commit();
 891 }
 892
 893 void UnitRepoProxy::InsertUnitArrayStmt
 894                   ::insert(RepoTxn& txn, int64_t unitSn, Id arrayId,
 895                            const std::string& array) {
 896   if (!prepared()) {
 897     std::stringstream ssInsert;
 898     ssInsert << "INSERT INTO " << m_repo.table(m_repoId, "UnitArray")
 899              << " VALUES(@unitSn, @arrayId, @array);";
 900     txn.prepare(*this, ssInsert.str());
 901   }
 902   RepoTxnQuery query(txn, *this);
 903   query.bindInt64("@unitSn", unitSn);
 904   query.bindId("@arrayId", arrayId);
 905   query.bindStdString("@array", array);
 906   query.exec();
 907 }
 908
 909 void UnitRepoProxy::GetUnitArraysStmt
 910                   ::get(UnitEmitter& ue) {
 911   RepoTxn txn(m_repo);
 912   if (!prepared()) {
 913     std::stringstream ssSelect;
 914     ssSelect << "SELECT arrayId,array FROM "
 915              << m_repo.table(m_repoId, "UnitArray")
 916              << " WHERE unitSn == @unitSn ORDER BY arrayId ASC;";
 917     txn.prepare(*this, ssSelect.str());
 918   }
 919   RepoTxnQuery query(txn, *this);
 920   query.bindInt64("@unitSn", ue.m_sn);
 921   do {
 922     query.step();
 923     if (query.row()) {
 924       Id arrayId;        /**/ query.getId(0, arrayId);
 925       std::string key;   /**/ query.getStdString(1, key);
 926       Variant v = unserialize_from_buffer(key.data(), key.size());
 927       Id id UNUSED = ue.mergeArray(v.asArrRef().get(), key);
 928       assert(id == arrayId);
 929     }
 930   } while (!query.done());
 931   txn.commit();
 932 }
 933
 934 void UnitRepoProxy::InsertUnitMergeableStmt
 935                   ::insert(RepoTxn& txn, int64_t unitSn,
 936                            int ix, Unit::MergeKind kind, Id id,
 937                            TypedValue* value) {
 938   if (!prepared()) {
 939     std::stringstream ssInsert;
 940     ssInsert << "INSERT INTO " << m_repo.table(m_repoId, "UnitMergeables")
 941              << " VALUES(@unitSn, @mergeableIx, @mergeableKind,"
 942                 " @mergeableId, @mergeableValue);";
 943     txn.prepare(*this, ssInsert.str());
 944   }
 945
 946   RepoTxnQuery query(txn, *this);
 947   query.bindInt64("@unitSn", unitSn);
 948   query.bindInt("@mergeableIx", ix);
 949   query.bindInt("@mergeableKind", (int)kind);
 950   query.bindId("@mergeableId", id);
 951   if (value) {
 952     assert(kind == MergeKind::Define ||
 953            kind == MergeKind::PersistentDefine ||
 954            kind == MergeKind::Global);
 955     query.bindTypedValue("@mergeableValue", *value);
 956   } else {
 957     assert(kind == MergeKind::ReqDoc);
 958     query.bindNull("@mergeableValue");
 959   }
 960   query.exec();
 961 }
 962
 963 void UnitRepoProxy::GetUnitMergeablesStmt
 964                   ::get(UnitEmitter& ue) {
 965   RepoTxn txn(m_repo);
 966   if (!prepared()) {
 967     std::stringstream ssSelect;
 968     ssSelect << "SELECT mergeableIx,mergeableKind,mergeableId,mergeableValue"
 969                 " FROM "
 970              << m_repo.table(m_repoId, "UnitMergeables")
 971              << " WHERE unitSn == @unitSn ORDER BY mergeableIx ASC;";
 972     txn.prepare(*this, ssSelect.str());
 973   }
 974   RepoTxnQuery query(txn, *this);
 975   query.bindInt64("@unitSn", ue.m_sn);
 976   do {
 977     query.step();
 978     if (query.row()) {
 979       int mergeableIx;           /**/ query.getInt(0, mergeableIx);
 980       int mergeableKind;         /**/ query.getInt(1, mergeableKind);
 981       Id mergeableId;            /**/ query.getInt(2, mergeableId);
 982
 983       auto k = MergeKind(mergeableKind);
 984
 985       if (UNLIKELY(!RuntimeOption::RepoAuthoritative)) {
 986         /*
 987          * We're using a repo generated in WholeProgram mode,
 988          * but we're not using it in RepoAuthoritative mode
 989          * (this is dodgy to start with). We're not going to
 990          * deal with requires at merge time, so drop them
 991          * here, and clear the mergeOnly flag for the unit.
 992          * The one exception is persistent constants are allowed in systemlib.
 993          */
 994         if (k != MergeKind::PersistentDefine || SystemLib::s_inited) {
 995           ue.m_mergeOnly = false;
 996         }
 997       }
 998       switch (k) {
 999         case MergeKind::ReqDoc:
1000           ue.insertMergeableInclude(mergeableIx, k, mergeableId);
1001           break;
1002         case MergeKind::PersistentDefine:
1003         case MergeKind::Define:
1004         case MergeKind::Global: {
1005           TypedValue mergeableValue; /**/ query.getTypedValue(3,
1006                                                               mergeableValue);
1007           ue.insertMergeableDef(mergeableIx, k, mergeableId, mergeableValue);
1008           break;
1009         }
1010         default: break;
1011       }
1012     }
1013   } while (!query.done());
1014   txn.commit();
1015 }
1016
1017 void UnitRepoProxy::insertUnitLineTable(int repoId,
1018                                         RepoTxn& txn,
1019                                         int64_t unitSn,
1020                                         LineTable& lineTable) {
1021   RepoStmt stmt(m_repo);
1022   stmt.prepare(
1023     folly::format(
1024       "INSERT INTO {} VALUES(@unitSn, @data);",
1025       m_repo.table(repoId, "UnitLineTable")
1026     ).str());
1027
1028   RepoTxnQuery query(txn, stmt);
1029   BlobEncoder dataBlob;
1030   dataBlob.encode(lineTable);
1031   query.bindInt64("@unitSn", unitSn);
1032   query.bindBlob("@data", dataBlob, /* static */ true);
1033   query.exec();
1034 }
1035
1036 void UnitRepoProxy::getUnitLineTable(int repoId,
1037                                      int64_t unitSn,
1038                                      LineTable& lineTable) {
1039   RepoStmt stmt(m_repo);
1040   stmt.prepare(
1041     folly::format(
1042       "SELECT data FROM {} WHERE unitSn == @unitSn;",
1043       m_repo.table(repoId, "UnitLineTable")
1044     ).str());
1045
1046   RepoTxn txn(m_repo);
1047   RepoTxnQuery query(txn, stmt);
1048   query.bindInt64("@unitSn", unitSn);
1049   query.step();
1050   if (query.row()) {
1051     BlobDecoder dataBlob = query.getBlob(0);
1052     dataBlob.decode(lineTable);
1053   }
1054   txn.commit();
1055 }
1056
1057 void UnitRepoProxy::InsertUnitSourceLocStmt
1058                   ::insert(RepoTxn& txn, int64_t unitSn, Offset pastOffset,
1059                            int line0, int char0, int line1, int char1) {
1060   if (!prepared()) {
1061     std::stringstream ssInsert;
1062     ssInsert << "INSERT INTO " << m_repo.table(m_repoId, "UnitSourceLoc")
1063              << " VALUES(@unitSn, @pastOffset, @line0, @char0, @line1,"
1064                 " @char1);";
1065     txn.prepare(*this, ssInsert.str());
1066   }
1067   RepoTxnQuery query(txn, *this);
1068   query.bindInt64("@unitSn", unitSn);
1069   query.bindOffset("@pastOffset", pastOffset);
1070   query.bindInt("@line0", line0);
1071   query.bindInt("@char0", char0);
1072   query.bindInt("@line1", line1);
1073   query.bindInt("@char1", char1);
1074   query.exec();
1075 }
1076
1077 bool UnitRepoProxy::GetSourceLocTabStmt
1078      ::get(int64_t unitSn, SourceLocTable& sourceLocTab) {
1079   try {
1080     RepoTxn txn(m_repo);
1081     if (!prepared()) {
1082       std::stringstream ssSelect;
1083       ssSelect << "SELECT pastOffset,line0,char0,line1,char1 FROM "
1084                << m_repo.table(m_repoId, "UnitSourceLoc")
1085                << " WHERE unitSn == @unitSn"
1086                   " ORDER BY pastOffset ASC;";
1087       txn.prepare(*this, ssSelect.str());
1088     }
1089     RepoTxnQuery query(txn, *this);
1090     query.bindInt64("@unitSn", unitSn);
1091     do {
1092       query.step();
1093       if (!query.row()) {
1094         return true;
1095       }
1096       Offset pastOffset;
1097       query.getOffset(0, pastOffset);
1098       SourceLoc sLoc;
1099       query.getInt(1, sLoc.line0);
1100       query.getInt(2, sLoc.char0);
1101       query.getInt(3, sLoc.line1);
1102       query.getInt(4, sLoc.char1);
1103       SourceLocEntry entry(pastOffset, sLoc);
1104       sourceLocTab.push_back(entry);
1105     } while (!query.done());
1106     txn.commit();
1107   } catch (RepoExc& re) {
1108     return true;
1109   }
1110   return false;
1111 }
1112
1113 ///////////////////////////////////////////////////////////////////////////////
1114 }