hphp/compiler/compiler.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/compiler/compiler.h"
  18
  19 #include "hphp/compiler/option.h"
  20 #include "hphp/compiler/package.h"
  21
  22 #include "hphp/hhbbc/hhbbc.h"
  23 #include "hphp/hhbbc/options.h"
  24
  25 #include "hphp/runtime/base/config.h"
  26 #include "hphp/runtime/base/file-util.h"
  27 #include "hphp/runtime/base/ini-setting.h"
  28 #include "hphp/runtime/base/preg.h"
  29 #include "hphp/runtime/base/program-functions.h"
  30 #include "hphp/runtime/base/variable-serializer.h"
  31 #include "hphp/runtime/version.h"
  32
  33 #include "hphp/runtime/vm/disas.h"
  34 #include "hphp/runtime/vm/preclass-emitter.h"
  35 #include "hphp/runtime/vm/repo-autoload-map-builder.h"
  36 #include "hphp/runtime/vm/repo-global-data.h"
  37 #include "hphp/runtime/vm/type-alias-emitter.h"
  38 #include "hphp/runtime/vm/unit-emitter.h"
  39
  40 #include "hphp/util/async-func.h"
  41 #include "hphp/util/build-info.h"
  42 #include "hphp/util/current-executable.h"
  43 #include "hphp/util/exception.h"
  44 #include "hphp/util/hdf.h"
  45 #include "hphp/util/job-queue.h"
  46 #include "hphp/util/logger.h"
  47 #include "hphp/util/process.h"
  48 #include "hphp/util/process-exec.h"
  49 #include "hphp/util/rds-local.h"
  50 #include "hphp/util/text-util.h"
  51 #include "hphp/util/timer.h"
  52 #ifndef _MSC_VER
  53 #include "hphp/util/light-process.h"
  54 #endif
  55
  56 #include "hphp/hhvm/process-init.h"
  57
  58 #include <sys/types.h>
  59 #ifndef _MSC_VER
  60 #include <sys/wait.h>
  61 #include <dlfcn.h>
  62 #endif
  63
  64 #include <boost/algorithm/string/replace.hpp>
  65 #include <boost/program_options/options_description.hpp>
  66 #include <boost/program_options/positional_options.hpp>
  67 #include <boost/program_options/variables_map.hpp>
  68 #include <boost/program_options/parsers.hpp>
  69
  70 #include <exception>
  71 #include <filesystem>
  72 #include <fstream>
  73
  74 #include <folly/portability/SysStat.h>
  75
  76 using namespace boost::program_options;
  77
  78 namespace HPHP {
  79
  80 using namespace extern_worker;
  81
  82 ///////////////////////////////////////////////////////////////////////////////
  83
  84 namespace {
  85
  86 ///////////////////////////////////////////////////////////////////////////////
  87
  88 struct CompilerOptions {
  89   std::string outputDir;
  90   std::vector<std::string> config;
  91   std::vector<std::string> confStrings;
  92   std::vector<std::string> iniStrings;
  93   std::string inputDir;
  94   std::vector<std::string> inputs;
  95   std::string inputList;
  96   std::vector<std::string> modules;
  97   std::vector<std::string> excludeDirs;
  98   std::vector<std::string> excludeFiles;
  99   std::vector<std::string> excludePatterns;
 100   std::vector<std::string> excludeStaticDirs;
 101   std::vector<std::string> excludeStaticFiles;
 102   std::vector<std::string> excludeStaticPatterns;
 103   std::vector<std::string> cfiles;
 104   std::vector<std::string> cmodules;
 105   std::string push_phases;
 106   std::string matched_overrides;
 107   bool parseOnDemand;
 108   int logLevel;
 109   std::string filecache;
 110   bool coredump;
 111 };
 112
 113 ///////////////////////////////////////////////////////////////////////////////
 114
 115 void applyBuildOverrides(IniSetting::Map& ini,
 116                          Hdf& config,
 117                          CompilerOptions& po) {
 118   std::string push_phases = Config::GetString(ini, config, "Build.PushPhases");
 119   po.push_phases = push_phases;
 120   // convert push phases to newline-separated, to make matching them less
 121   // error-prone.
 122   replaceAll(push_phases, ",", "\n");
 123   bool loggedOnce = false;
 124
 125   for (Hdf hdf = config["Overrides"].firstChild();
 126        hdf.exists();
 127        hdf = hdf.next()) {
 128     if (!loggedOnce) {
 129       Logger::Info(folly::sformat(
 130                        "Matching build overrides using: push_phases='{}'",
 131                        po.push_phases));
 132       loggedOnce = true;
 133     }
 134     if (Config::matchHdfPattern(push_phases, ini, hdf, "push_phase" , "m")) {
 135       Logger::Info(folly::sformat("Matched override: {}", hdf.getName()));
 136       folly::format(
 137         &po.matched_overrides,
 138         "{}{}",
 139         po.matched_overrides.empty() ? "" : ",",
 140         hdf.getName()
 141       );
 142
 143       if (hdf.exists("clear")) {
 144         std::vector<std::string> list;
 145         hdf["clear"].configGet(list);
 146         for (auto const& s : list) {
 147           config.remove(s);
 148         }
 149       }
 150       config.copy(hdf["overwrite"]);
 151       // no break here, so we can continue to match more overrides
 152     }
 153     hdf["overwrite"].setVisited(); // avoid lint complaining
 154     if (hdf.exists("clear")) {
 155       // when the tier does not match, "clear" is not accessed
 156       // mark it visited, so the linter does not complain
 157       hdf["clear"].setVisited();
 158     }
 159   }
 160 }
 161
 162 bool addAutoloadQueryToPackage(Package& package, const std::string& queryStr) {
 163   try {
 164     auto query = folly::parseJson(queryStr);
 165     if (!query.isObject()) {
 166       Logger::FError("Autoload.Query is not a JSON Object");
 167       return false;
 168     }
 169     auto expr = query["expression"];
 170     for (auto& term : expr) {
 171       if (term.isArray() && term[0] == "dirname") {
 172         Logger::FInfo("adding autoload dir {}", term[1].asString());
 173         package.addDirectory(term[1].asString());
 174       }
 175     }
 176     return true;
 177   } catch (const folly::json::parse_error& e) {
 178     Logger::FError("Error JSON-parsing Autoload.Query = \"{}\": {}",
 179                    queryStr, e.what());
 180     return false;
 181   }
 182 }
 183
 184 void addInputsToPackage(Package& package, const CompilerOptions& po) {
 185   if (po.modules.empty() && po.inputs.empty() && po.inputList.empty()) {
 186     package.addDirectory("/");
 187   } else {
 188     for (auto const& module : po.modules) {
 189       package.addDirectory(module);
 190     }
 191     for (auto const& cmodule : po.cmodules) {
 192       package.addStaticDirectory(cmodule);
 193     }
 194     for (auto const& cfile : po.cfiles) {
 195       package.addStaticFile(cfile);
 196     }
 197     for (auto const& input : po.inputs) {
 198       package.addSourceFile(input);
 199     }
 200     if (!po.inputList.empty()) {
 201       package.addInputList(po.inputList);
 202     }
 203   }
 204 }
 205
 206 void genText(const UnitEmitter& ue, const std::string& outputPath) {
 207   assertx(Option::GenerateTextHHBC || Option::GenerateHhasHHBC);
 208
 209   auto const unit = ue.create();
 210
 211   auto const basePath = [&] {
 212     auto fullPath = outputPath;
 213     if (!fullPath.empty() &&
 214         !FileUtil::isDirSeparator(fullPath[fullPath.size() - 1])) {
 215       fullPath += FileUtil::getDirSeparator();
 216     }
 217
 218     auto const fileName = "php/" + unit->filepath()->toCppString();
 219     if (fileName.size() > 4 &&
 220         fileName.substr(fileName.length() - 4) == ".php") {
 221       fullPath += fileName.substr(0, fileName.length() - 4);
 222     } else {
 223       fullPath += fileName;
 224     }
 225
 226     for (auto pos = outputPath.size(); pos < fullPath.size(); pos++) {
 227       if (FileUtil::isDirSeparator(fullPath[pos])) {
 228         mkdir(fullPath.substr(0, pos).c_str(), 0777);
 229       }
 230     }
 231     return fullPath;
 232   }();
 233
 234   if (Option::GenerateTextHHBC) {
 235     auto const fullPath = basePath + ".hhbc.txt";
 236     std::ofstream f(fullPath.c_str());
 237     if (!f) {
 238       Logger::Error("Unable to open %s for write", fullPath.c_str());
 239     } else {
 240       f << "Hash: " << ue.sha1().toString() << std::endl;
 241       f << unit->toString();
 242       f.close();
 243     }
 244   }
 245
 246   if (Option::GenerateHhasHHBC) {
 247     auto const fullPath = basePath + ".hhas";
 248     std::ofstream f(fullPath.c_str());
 249     if (!f) {
 250       Logger::Error("Unable to open %s for write", fullPath.c_str());
 251     } else {
 252       f << disassemble(unit.get());
 253       f.close();
 254     }
 255   }
 256 }
 257
 258 /*
 259  * It's an invariant that symbols in the repo must be Unique and
 260  * Persistent. Verify all relevant symbols are unique and set the
 261  * appropriate Attrs.
 262  */
 263 struct SymbolSets {
 264   SymbolSets() {
 265     // These aren't stored in the repo, but we still need to check for
 266     // collisions against them, so put them in the maps.
 267     for (auto const& kv : Native::getConstants()) {
 268       assertx(kv.second.m_type != KindOfUninit ||
 269               kv.second.dynamic());
 270       add(constants, kv.first, nullptr, "constant");
 271     }
 272   }
 273
 274   // For local parses, where we have an UnitEmitter
 275   void add(UnitEmitter& ue) {
 276     // Verify uniqueness of symbols and set Attrs appropriately.
 277     auto const path = ue.m_filepath;
 278
 279     add(units, path, path, "unit");
 280
 281     for (size_t n = 0; n < ue.numPreClasses(); ++n) {
 282       auto pce = ue.pce(n);
 283       pce->setAttrs(pce->attrs() | AttrUnique | AttrPersistent);
 284       if (pce->attrs() & AttrEnum) add(enums, pce->name(), path, "enum");
 285       add(classes, pce->name(), path, "class", typeAliases);
 286     }
 287     for (auto& fe : ue.fevec()) {
 288       if (fe->attrs & AttrIsMethCaller) {
 289         if (addNoFail(funcs, fe->name, path, "function")) {
 290           fe->attrs |= AttrUnique | AttrPersistent;
 291         }
 292       } else {
 293         fe->attrs |= AttrUnique | AttrPersistent;
 294         add(funcs, fe->name, path, "function");
 295       }
 296     }
 297     for (auto& te : ue.typeAliases()) {
 298       te->setAttrs(te->attrs() | AttrUnique | AttrPersistent);
 299       add(typeAliases, te->name(), path, "type alias", classes);
 300     }
 301     for (auto& c : ue.constants()) {
 302       c.attrs |= AttrUnique | AttrPersistent;
 303       add(constants, c.name, path, "constant");
 304     }
 305     for (auto& m : ue.modules()) {
 306       m.attrs |= AttrUnique | AttrPersistent;
 307       add(modules, m.name, path, "module");
 308     }
 309   }
 310
 311   // For remote parses, where we don't have an UnitEmitter
 312   void add(const Package::ParseMeta::Definitions& d, const StringData* path) {
 313     add(units, path, path, "unit");
 314
 315     for (auto const& c : d.m_classes) {
 316       add(classes, c, path, "class", typeAliases);
 317     }
 318     for (auto const& e : d.m_enums) {
 319       add(enums, e, path, "enum");
 320       add(classes, e, path, "class", typeAliases);
 321     }
 322     for (auto const& f : d.m_funcs) {
 323       add(funcs, f, path, "function");
 324     }
 325     for (auto const& m : d.m_methCallers) {
 326       addNoFail(funcs, m, path, "function");
 327     }
 328     for (auto const& a : d.m_typeAliases) {
 329       add(typeAliases, a, path, "type alias", classes);
 330     }
 331     for (auto const& c : d.m_constants) {
 332       add(constants, c, path, "constant");
 333     }
 334     for (auto const& m : d.m_modules) {
 335       add(modules, m, path, "module");
 336     }
 337   }
 338
 339   struct NonUnique : std::runtime_error {
 340     using std::runtime_error::runtime_error;
 341   };
 342
 343 private:
 344   template <typename T>
 345   void add(T& map,
 346            const StringData* name,
 347            const StringData* unit,
 348            const char* type) {
 349     assertx(name->isStatic());
 350     assertx(!unit || unit->isStatic());
 351     auto const ret = map.emplace(name, unit);
 352     if (!ret.second) return fail(name, unit, ret.first->second, type);
 353   }
 354
 355   template <typename T>
 356   bool addNoFail(T& map,
 357                  const StringData* name,
 358                  const StringData* unit,
 359                  const char* type) {
 360     assertx(name->isStatic());
 361     assertx(!unit || unit->isStatic());
 362     return map.emplace(name, unit).second;
 363   }
 364
 365   template <typename T, typename E>
 366   void add(T& map,
 367            const StringData* name,
 368            const StringData* unit,
 369            const char* type,
 370            const E& other) {
 371     assertx(name->isStatic());
 372     assertx(!unit || unit->isStatic());
 373     auto const it = other.find(name);
 374     if (it != other.end()) return fail(name, unit, it->second, "symbol");
 375     add(map, name, unit, type);
 376   }
 377
 378   [[noreturn]]
 379   void fail(const StringData* name,
 380             const StringData* unit1,
 381             const StringData* unit2,
 382             const char* type) {
 383     auto const filename = [] (const StringData* u) {
 384       if (!u) return "BUILTIN";
 385       return u->data();
 386     };
 387
 388     throw NonUnique{
 389       folly::sformat(
 390         "More than one {} with the name {}. In {} and {}",
 391         type,
 392         name,
 393         filename(unit1),
 394         filename(unit2)
 395       )
 396     };
 397   }
 398
 399   using IMap = folly_concurrent_hash_map_simd<
 400     const StringData*,
 401     const StringData*,
 402     string_data_hash,
 403     string_data_isame
 404   >;
 405   using Map = folly_concurrent_hash_map_simd<
 406     const StringData*,
 407     const StringData*,
 408     string_data_hash,
 409     string_data_same
 410   >;
 411
 412   IMap enums;
 413   IMap classes;
 414   IMap funcs;
 415   IMap typeAliases;
 416   Map constants;
 417   Map modules;
 418   Map units;
 419 };
 420
 421 RepoGlobalData getGlobalData() {
 422   auto const now = std::chrono::high_resolution_clock::now();
 423   auto const nanos =
 424     std::chrono::duration_cast<std::chrono::nanoseconds>(
 425       now.time_since_epoch()
 426     );
 427
 428   auto gd                        = RepoGlobalData{};
 429   gd.Signature                   = nanos.count();
 430   gd.CheckPropTypeHints          = RuntimeOption::EvalCheckPropTypeHints;
 431   gd.PHP7_NoHexNumerics          = RuntimeOption::PHP7_NoHexNumerics;
 432   gd.PHP7_Substr                 = RuntimeOption::PHP7_Substr;
 433   gd.PHP7_Builtins               = RuntimeOption::PHP7_Builtins;
 434   gd.HardGenericsUB              = RuntimeOption::EvalEnforceGenericsUB >= 2;
 435   gd.EnableIntrinsicsExtension   = RuntimeOption::EnableIntrinsicsExtension;
 436   gd.ForbidDynamicCallsToFunc    = RuntimeOption::EvalForbidDynamicCallsToFunc;
 437   gd.ForbidDynamicCallsWithAttr  =
 438     RuntimeOption::EvalForbidDynamicCallsWithAttr;
 439   gd.ForbidDynamicCallsToClsMeth =
 440     RuntimeOption::EvalForbidDynamicCallsToClsMeth;
 441   gd.ForbidDynamicCallsToInstMeth =
 442     RuntimeOption::EvalForbidDynamicCallsToInstMeth;
 443   gd.ForbidDynamicConstructs     = RuntimeOption::EvalForbidDynamicConstructs;
 444   gd.LogKnownMethodsAsDynamicCalls =
 445     RuntimeOption::EvalLogKnownMethodsAsDynamicCalls;
 446   gd.EnableArgsInBacktraces      = RuntimeOption::EnableArgsInBacktraces;
 447   gd.NoticeOnBuiltinDynamicCalls =
 448     RuntimeOption::EvalNoticeOnBuiltinDynamicCalls;
 449   gd.InitialNamedEntityTableSize =
 450     RuntimeOption::EvalInitialNamedEntityTableSize;
 451   gd.InitialStaticStringTableSize =
 452     RuntimeOption::EvalInitialStaticStringTableSize;
 453   gd.HackArrCompatSerializeNotices =
 454     RuntimeOption::EvalHackArrCompatSerializeNotices;
 455   gd.AbortBuildOnVerifyError = RuntimeOption::EvalAbortBuildOnVerifyError;
 456   gd.EmitClassPointers = RuntimeOption::EvalEmitClassPointers;
 457   gd.EmitClsMethPointers = RuntimeOption::EvalEmitClsMethPointers;
 458   gd.IsVecNotices = RuntimeOption::EvalIsVecNotices;
 459   gd.RaiseClassConversionWarning =
 460     RuntimeOption::EvalRaiseClassConversionWarning;
 461   gd.ClassPassesClassname = RuntimeOption::EvalClassPassesClassname;
 462   gd.ClassnameNotices = RuntimeOption::EvalClassnameNotices;
 463   gd.ClassIsStringNotices = RuntimeOption::EvalClassIsStringNotices;
 464   gd.StrictArrayFillKeys = RuntimeOption::StrictArrayFillKeys;
 465   gd.TraitConstantInterfaceBehavior =
 466     RuntimeOption::EvalTraitConstantInterfaceBehavior;
 467   gd.BuildMayNoticeOnMethCallerHelperIsObject =
 468     RO::EvalBuildMayNoticeOnMethCallerHelperIsObject;
 469   gd.DiamondTraitMethods = RuntimeOption::EvalDiamondTraitMethods;
 470   gd.EvalCoeffectEnforcementLevels = RO::EvalCoeffectEnforcementLevels;
 471   gd.EmitBespokeTypeStructures = RO::EvalEmitBespokeTypeStructures;
 472
 473   if (Option::ConstFoldFileBC) {
 474     gd.SourceRootForFileBC.emplace(RO::SourceRoot);
 475   }
 476
 477   for (auto const& elm : RuntimeOption::ConstantFunctions) {
 478     auto const s = internal_serialize(tvAsCVarRef(elm.second));
 479     gd.ConstantFunctions.emplace_back(elm.first, s.toCppString());
 480   }
 481   std::sort(gd.ConstantFunctions.begin(), gd.ConstantFunctions.end());
 482
 483   return gd;
 484 }
 485
 486 void setCoredumps(const CompilerOptions& po) {
 487   if (!po.coredump) return;
 488 #ifdef _MSC_VER
 489 /**
 490  * Windows actually does core dump size and control at a system, not an app
 491  * level. So we do nothing here and are at the mercy of Dr. Watson.
 492  */
 493 #elif defined(__APPLE__) || defined(__FreeBSD__)
 494   struct rlimit rl;
 495   getrlimit(RLIMIT_CORE, &rl);
 496   rl.rlim_cur = 80000000LL;
 497   if (rl.rlim_max < rl.rlim_cur) {
 498     rl.rlim_max = rl.rlim_cur;
 499   }
 500   setrlimit(RLIMIT_CORE, &rl);
 501 #else
 502   struct rlimit64 rl;
 503   getrlimit64(RLIMIT_CORE, &rl);
 504   rl.rlim_cur = 8000000000LL;
 505   if (rl.rlim_max < rl.rlim_cur) {
 506     rl.rlim_max = rl.rlim_cur;
 507   }
 508   setrlimit64(RLIMIT_CORE, &rl);
 509 #endif
 510 }
 511
 512 int prepareOptions(CompilerOptions &po, int argc, char **argv) {
 513   options_description desc("HipHop Compiler for PHP Usage:\n\n"
 514                            "\thphp <options> <inputs>\n\n"
 515                            "Options");
 516
 517   std::vector<std::string> formats;
 518
 519   desc.add_options()
 520     ("help", "display this message")
 521     ("version", "display version number")
 522     ("format,f", value<std::vector<std::string>>(&formats)->composing(),
 523      "HHBC Output format: binary (default) | hhas | text")
 524     ("input-dir", value<std::string>(&po.inputDir), "input directory")
 525     ("inputs,i", value<std::vector<std::string>>(&po.inputs)->composing(),
 526      "input file names")
 527     ("input-list", value<std::string>(&po.inputList),
 528      "file containing list of file names, one per line")
 529     ("module", value<std::vector<std::string>>(&po.modules)->composing(),
 530      "directories containing all input files")
 531     ("exclude-dir",
 532      value<std::vector<std::string>>(&po.excludeDirs)->composing(),
 533      "directories to exclude from the input")
 534     ("exclude-file",
 535      value<std::vector<std::string>>(&po.excludeFiles)->composing(),
 536      "files to exclude from the input, even if parse-on-demand finds it")
 537     ("exclude-pattern",
 538      value<std::vector<std::string>>(&po.excludePatterns)->composing(),
 539      "regex (in 'find' command's regex command line option format) of files "
 540      "or directories to exclude from the input, even if parse-on-demand finds "
 541      "it")
 542     ("exclude-static-pattern",
 543      value<std::vector<std::string>>(&po.excludeStaticPatterns)->composing(),
 544      "regex (in 'find' command's regex command line option format) of files "
 545      "or directories to exclude from static content cache")
 546     ("exclude-static-dir",
 547      value<std::vector<std::string>>(&po.excludeStaticDirs)->composing(),
 548      "directories to exclude from static content cache")
 549     ("exclude-static-file",
 550      value<std::vector<std::string>>(&po.excludeStaticFiles)->composing(),
 551      "files to exclude from static content cache")
 552     ("cfile", value<std::vector<std::string>>(&po.cfiles)->composing(),
 553      "extra static files forced to include without exclusion checking")
 554     ("cmodule", value<std::vector<std::string>>(&po.cmodules)->composing(),
 555      "extra directories for static files without exclusion checking")
 556     ("parse-on-demand", value<bool>(&po.parseOnDemand)->default_value(true),
 557      "whether to parse files that are not specified from command line")
 558     ("output-dir,o", value<std::string>(&po.outputDir), "output directory")
 559     ("config,c", value<std::vector<std::string>>(&po.config)->composing(),
 560      "config file name")
 561     ("config-value,v",
 562      value<std::vector<std::string>>(&po.confStrings)->composing(),
 563      "individual configuration string in a format of name=value, where "
 564      "name can be any valid configuration for a config file")
 565     ("define,d", value<std::vector<std::string>>(&po.iniStrings)->composing(),
 566      "define an ini setting in the same format ( foo[=bar] ) as provided in a "
 567      ".ini file")
 568     ("log,l",
 569      value<int>(&po.logLevel)->default_value(-1),
 570      "-1: (default); 0: no logging; 1: errors only; 2: warnings and errors; "
 571      "3: informational as well; 4: really verbose.")
 572     ("file-cache",
 573      value<std::string>(&po.filecache),
 574      "if specified, generate a static file cache with this file name")
 575     ("coredump",
 576      value<bool>(&po.coredump)->default_value(false),
 577      "turn on coredump")
 578     ("compiler-id", "display the git hash for the compiler id")
 579     ("repo-schema", "display the repo schema id used by this app")
 580     ;
 581
 582   positional_options_description p;
 583   p.add("inputs", -1);
 584   variables_map vm;
 585   try {
 586     auto opts = command_line_parser(argc, argv).options(desc)
 587                                                .positional(p).run();
 588     try {
 589       store(opts, vm);
 590       notify(vm);
 591 #if defined(BOOST_VERSION) && BOOST_VERSION >= 105000 && BOOST_VERSION <= 105400
 592     } catch (const error_with_option_name &e) {
 593       std::string wrong_name = e.get_option_name();
 594       std::string right_name = get_right_option_name(opts, wrong_name);
 595       std::string message = e.what();
 596       if (right_name != "") {
 597         boost::replace_all(message, wrong_name, right_name);
 598       }
 599       Logger::Error("Error in command line: %s", message.c_str());
 600       std::cout << desc << "\n";
 601       return -1;
 602 #endif
 603     } catch (const error& e) {
 604       Logger::Error("Error in command line: %s", e.what());
 605       std::cout << desc << "\n";
 606       return -1;
 607     }
 608   } catch (const unknown_option& e) {
 609     Logger::Error("Error in command line: %s", e.what());
 610     std::cout << desc << "\n";
 611     return -1;
 612   } catch (const error& e) {
 613     Logger::Error("Error in command line: %s", e.what());
 614     std::cout << desc << "\n";
 615     return -1;
 616   } catch (...) {
 617     Logger::Error("Error in command line parsing.");
 618     std::cout << desc << "\n";
 619     return -1;
 620   }
 621   if (argc <= 1 || vm.count("help")) {
 622     std::cout << desc << "\n";
 623     return 1;
 624   }
 625   if (vm.count("version")) {
 626     std::cout << "HipHop Repo Compiler";
 627     std::cout << " " << HHVM_VERSION;
 628     std::cout << " (" << (debug ? "dbg" : "rel") << ")\n";
 629     std::cout << "Compiler: " << compilerId() << "\n";
 630     std::cout << "Repo schema: " << repoSchemaId() << "\n";
 631     return 1;
 632   }
 633
 634   if (vm.count("compiler-id")) {
 635     std::cout << compilerId() << "\n";
 636     return 1;
 637   }
 638
 639   if (vm.count("repo-schema")) {
 640     std::cout << repoSchemaId() << "\n";
 641     return 1;
 642   }
 643
 644   if (po.outputDir.empty()) {
 645     Logger::Error("Error in command line: output-dir must be provided.");
 646     std::cout << desc << "\n";
 647     return -1;
 648   }
 649
 650   // log level
 651   if (po.logLevel != -1) {
 652     Logger::LogLevel = (Logger::LogLevelType)po.logLevel;
 653   } else {
 654     Logger::LogLevel = Logger::LogInfo;
 655   }
 656   Logger::Escape = false;
 657   Logger::AlwaysEscapeLog = false;
 658
 659   if (!formats.empty()) {
 660     for (auto const& format : formats) {
 661       if (format == "text") {
 662         Option::GenerateTextHHBC = true;
 663       } else if (format == "hhas") {
 664         Option::GenerateHhasHHBC = true;
 665       } else if (format == "binary") {
 666         Option::GenerateBinaryHHBC = true;
 667       } else {
 668         Logger::Error("Unknown format for HHBC target: %s", format.c_str());
 669         std::cout << desc << "\n";
 670         return -1;
 671       }
 672     }
 673   } else {
 674     Option::GenerateBinaryHHBC = true;
 675   }
 676
 677   tl_heap.getCheck();
 678   IniSetting::Map ini = IniSetting::Map::object;
 679   Hdf config;
 680   for (auto const& file : po.config) {
 681     Config::ParseConfigFile(file, ini, config);
 682   }
 683   for (auto const& iniString : po.iniStrings) {
 684     Config::ParseIniString(iniString, ini);
 685   }
 686   for (auto const& confString : po.confStrings) {
 687     Config::ParseHdfString(confString, config);
 688   }
 689   applyBuildOverrides(ini, config, po);
 690   Hdf runtime = config["Runtime"];
 691   // The configuration command line strings were already processed above
 692   // Don't process them again.
 693   //
 694   // Note that some options depends on RepoAuthoritative, we thus
 695   // set/unset them here. We restore it to false since we need
 696   // compile_systemlib_string to actually parse the file instead of
 697   // trying to load it from repo (which is the case when
 698   // RepoAuthoritative is true).
 699   RuntimeOption::RepoAuthoritative = true;
 700   // Set RepoPath to satisfy assertions (we need a path set in
 701   // RepoAuthoritative). It will never actually be used.
 702   RuntimeOption::RepoPath = "/tmp/dummy.hhbc";
 703   // We don't want debug info in repo builds, since we don't support attaching
 704   // a debugger in repo authoritative mode, but we want the default for debug
 705   // info to be true so that it's present in sandboxes. Override that default
 706   // here, since we only get here when building for repo authoritative mode.
 707   RuntimeOption::RepoDebugInfo = false;
 708   RuntimeOption::Load(ini, runtime);
 709   Option::Load(ini, config);
 710   RuntimeOption::RepoAuthoritative = false;
 711   RuntimeOption::RepoPath = "";
 712   RuntimeOption::EvalJit = false;
 713   RuntimeOption::EvalLowStaticArrays = false;
 714
 715   std::vector<std::string> badnodes;
 716   config.lint(badnodes);
 717   for (auto const& badnode : badnodes) {
 718     Logger::Error("Possible bad config node: %s", badnode.c_str());
 719   }
 720
 721   // we need to initialize pcre cache table very early
 722   pcre_init();
 723
 724   if (po.inputDir.empty()) po.inputDir = '.';
 725   po.inputDir = FileUtil::normalizeDir(po.inputDir);
 726
 727   for (auto const& dir : po.excludeDirs) {
 728     Option::PackageExcludeDirs.insert(FileUtil::normalizeDir(dir));
 729   }
 730   for (auto const& file : po.excludeFiles) {
 731     Option::PackageExcludeFiles.insert(file);
 732   }
 733   for (auto const& pattern : po.excludePatterns) {
 734     Option::PackageExcludePatterns.insert(
 735       format_pattern(pattern, true /* prefixSlash */));
 736   }
 737   for (auto const& dir : po.excludeStaticDirs) {
 738     Option::PackageExcludeStaticDirs.insert(FileUtil::normalizeDir(dir));
 739   }
 740   for (auto const& file : po.excludeStaticFiles) {
 741     Option::PackageExcludeStaticFiles.insert(file);
 742   }
 743   for (auto const& pattern : po.excludeStaticPatterns) {
 744     Option::PackageExcludeStaticPatterns.insert(
 745       format_pattern(pattern, true /* prefixSlash */));
 746   }
 747
 748   return 0;
 749 }
 750
 751 ///////////////////////////////////////////////////////////////////////////////
 752
 753 Options makeExternWorkerOptions() {
 754   Options options;
 755   options
 756     .setUseCase(Option::ExternWorkerUseCase)
 757     .setUseSubprocess(Option::ExternWorkerForceSubprocess
 758                       ? Options::UseSubprocess::Always
 759                       : Options::UseSubprocess::Fallback)
 760     .setCacheExecs(Option::ExternWorkerUseExecCache)
 761     .setCleanup(Option::ExternWorkerCleanup)
 762     .setUseEdenFS(RO::EvalUseEdenFS)
 763     .setUseRichClient(Option::ExternWorkerUseRichClient)
 764     .setUseZippyRichClient(Option::ExternWorkerUseZippyRichClient)
 765     .setUseP2P(Option::ExternWorkerUseP2P)
 766     .setVerboseLogging(Option::ExternWorkerVerboseLogging);
 767   if (Option::ExternWorkerTimeoutSecs > 0) {
 768     options.setTimeout(std::chrono::seconds{Option::ExternWorkerTimeoutSecs});
 769   }
 770   if (!Option::ExternWorkerWorkingDir.empty()) {
 771     options.setWorkingDir(Option::ExternWorkerWorkingDir);
 772   }
 773   if (Option::ExternWorkerThrottleRetries >= 0) {
 774     options.setThrottleRetries(Option::ExternWorkerThrottleRetries);
 775   }
 776   if (Option::ExternWorkerThrottleBaseWaitMSecs >= 0) {
 777     options.setThrottleBaseWait(
 778       std::chrono::milliseconds{Option::ExternWorkerThrottleBaseWaitMSecs}
 779     );
 780   }
 781   return options;
 782 }
 783
 784 void logPhaseStats(const std::string& phase, const Package& package,
 785     extern_worker::Client& client, StructuredLogEntry& sample, int64_t micros)
 786 {
 787   auto const& stats = client.getStats();
 788   Logger::FInfo(
 789     "  {}: total package files {:,}\n"
 790     "  Execs: {:,} total, {:,} cache-hits, {:,} optimistically, {:,} fallback\n"
 791     "  Files: {:,} total, {:,} read, {:,} queried, {:,} uploaded ({:,} bytes), {:,} fallback\n"
 792     "  Blobs: {:,} total, {:,} queried, {:,} uploaded ({:,} bytes), {:,} fallback\n"
 793     "  Cpu: {:,} usec usage, {:,} allocated cores\n"
 794     "  Mem: {:,} max used, {:,} reserved\n"
 795     "  {:,} downloads ({:,} bytes), {:,} throttles",
 796     phase,
 797     package.getTotalFiles(),
 798     stats.execs.load(),
 799     stats.execCacheHits.load(),
 800     stats.optimisticExecs.load(),
 801     stats.execFallbacks.load(),
 802     stats.files.load(),
 803     stats.filesRead.load(),
 804     stats.filesQueried.load(),
 805     stats.filesUploaded.load(),
 806     stats.fileBytesUploaded.load(),
 807     stats.fileFallbacks.load(),
 808     stats.blobs.load(),
 809     stats.blobsQueried.load(),
 810     stats.blobsUploaded.load(),
 811     stats.blobBytesUploaded.load(),
 812     stats.blobFallbacks.load(),
 813     stats.execCpuUsec.load(),
 814     stats.execAllocatedCores.load(),
 815     stats.execMaxUsedMem.load(),
 816     stats.execReservedMem.load(),
 817     stats.downloads.load(),
 818     stats.bytesDownloaded.load(),
 819     stats.throttles.load()
 820   );
 821   sample.setInt(phase + "_total_files", package.getTotalFiles());
 822
 823   sample.setInt(phase + "_micros", micros);
 824   if (auto const t = package.inputsTime()) {
 825     sample.setInt(
 826       phase + "_input_micros",
 827       std::chrono::duration_cast<std::chrono::microseconds>(*t).count()
 828     );
 829   }
 830   if (auto const t = package.ondemandTime()) {
 831     sample.setInt(
 832       phase + "_ondemand_micros",
 833       std::chrono::duration_cast<std::chrono::microseconds>(*t).count()
 834     );
 835   }
 836
 837   sample.setInt(phase + "_total_execs", stats.execs.load());
 838   sample.setInt(phase + "_cache_hits", stats.execCacheHits.load());
 839   sample.setInt(phase + "_optimistically", stats.optimisticExecs.load());
 840   sample.setInt(phase + "_fallbacks", stats.execFallbacks.load());
 841
 842   sample.setInt(phase + "_total_files", stats.files.load());
 843   sample.setInt(phase + "_file_reads", stats.filesRead.load());
 844   sample.setInt(phase + "_file_queries", stats.filesQueried.load());
 845   sample.setInt(phase + "_file_stores", stats.filesUploaded.load());
 846   sample.setInt(phase + "_file_stores_bytes", stats.fileBytesUploaded.load());
 847   sample.setInt(phase + "_file_fallbacks", stats.fileFallbacks.load());
 848
 849   sample.setInt(phase + "_total_blobs", stats.blobs.load());
 850   sample.setInt(phase + "_blob_queries", stats.blobsQueried.load());
 851   sample.setInt(phase + "_blob_stores", stats.blobsUploaded.load());
 852   sample.setInt(phase + "_blob_stores_bytes", stats.blobBytesUploaded.load());
 853   sample.setInt(phase + "_blob_fallbacks", stats.blobFallbacks.load());
 854
 855   sample.setInt(phase + "_total_loads", stats.downloads.load());
 856   sample.setInt(phase + "_total_loads_bytes", stats.bytesDownloaded.load());
 857   sample.setInt(phase + "_throttles", stats.throttles.load());
 858
 859   sample.setInt(phase + "_exec_cpu_usec", stats.execCpuUsec.load());
 860   sample.setInt(phase + "_exec_allocated_cores", stats.execAllocatedCores.load());
 861   sample.setInt(phase + "_exec_max_used_mem", stats.execMaxUsedMem.load());
 862   sample.setInt(phase + "_exec_reserved_mem", stats.execReservedMem.load());
 863
 864   sample.setStr(phase + "_fellback",
 865     client.fellback() ? "true" : "false"
 866   );
 867 }
 868
 869 // Compute a UnitIndex by parsing decls for all autoload-eligible files.
 870 // If no Autoload.Query is specified by RepoOptions, this just indexes
 871 // the input files.
 872 bool computeIndex(
 873     const CompilerOptions& po,
 874     StructuredLogEntry& sample,
 875     coro::TicketExecutor& executor,
 876     extern_worker::Client& client,
 877     UnitIndex& index
 878 ) {
 879   auto const onIndex = [&] (std::string&& rpath, Package::IndexMeta&& meta) {
 880     auto const interned_rpath = makeStaticString(rpath);
 881     auto insert = [&](auto const& names, auto& map, const char* kind) {
 882       for (auto name : names) {
 883         auto const ret = map.emplace(name, interned_rpath);
 884         if (!ret.second) {
 885           Logger::FWarning("Duplicate {} {} in {} and {}",
 886               kind, name, ret.first->first, interned_rpath
 887           );
 888         }
 889       }
 890     };
 891     insert(meta.types, index.types, "type");
 892     insert(meta.funcs, index.funcs, "function");
 893     insert(meta.constants, index.constants, "constant");
 894   };
 895
 896   Package indexPackage{po.inputDir, false, executor, client};
 897   Timer indexTimer(Timer::WallTime, "indexing");
 898
 899   auto const& repoFlags = RepoOptions::forFile(po.inputDir.c_str()).flags();
 900   auto const queryStr = repoFlags.autoloadQuery();
 901   if (!queryStr.empty() && po.parseOnDemand) {
 902     // Index the files specified by Autoload.Query
 903     if (!addAutoloadQueryToPackage(indexPackage, queryStr)) return false;
 904   } else {
 905     // index just the input files
 906     addInputsToPackage(indexPackage, po);
 907   }
 908
 909   if (!coro::wait(indexPackage.index(onIndex))) return false;
 910
 911   logPhaseStats("index", indexPackage, client, sample,
 912                 indexTimer.getMicroSeconds());
 913   Logger::FInfo("index size: types={:,} funcs={:,} constants={:,}",
 914       index.types.size(),
 915       index.funcs.size(),
 916       index.constants.size()
 917   );
 918   client.resetStats();
 919
 920   return true;
 921 }
 922
 923 ///////////////////////////////////////////////////////////////////////////////
 924
 925 // Parses a file and produces an UnitEmitter. Used when we're not
 926 // going to run HHBBC.
 927 struct ParseJob {
 928   static std::string name() { return "hphpc-parse"; }
 929
 930   static void init(const Package::Config& config,
 931                    Package::FileMetaVec meta) {
 932     Package::parseInit(config, std::move(meta));
 933   }
 934   static Package::ParseMetaVec fini() {
 935     return Package::parseFini();
 936   }
 937
 938   static UnitEmitterSerdeWrapper run(const std::string& contents,
 939                                      const RepoOptionsFlags& flags) {
 940     return Package::parseRun(contents, flags);
 941   }
 942 };
 943
 944 using WPI = HHBBC::WholeProgramInput;
 945
 946 // Parses a file (as ParseJob does), but then hands the UnitEmitter
 947 // off to HHBBC to produce a WholeProgramInput key and value. This is
 948 // for when we are going to run HHBBC.
 949 struct ParseForHHBBCJob {
 950   static std::string name() { return "hphpc-parse-for-hhbbc"; }
 951
 952   static void init(const Package::Config& config,
 953                    const HHBBC::Config& hhbbcConfig,
 954                    Package::FileMetaVec meta) {
 955     Package::parseInit(config, std::move(meta));
 956     HHBBC::options = hhbbcConfig.o;
 957     hhbbcConfig.gd.load(true);
 958   }
 959   static std::tuple<Package::ParseMetaVec, std::vector<WPI::Key>> fini() {
 960     return std::make_tuple(Package::parseFini(), std::move(s_inputKeys));
 961   }
 962
 963   static Variadic<WPI::Value> run(const std::string& contents,
 964                                   const RepoOptionsFlags& flags) {
 965     auto wrapper = Package::parseRun(contents, flags);
 966     if (!wrapper.m_ue) return {};
 967
 968     std::vector<WPI::Value> values;
 969     for (auto& [key, value] : WPI::make(std::move(wrapper.m_ue))) {
 970       s_inputKeys.emplace_back(std::move(key));
 971       values.emplace_back(std::move(value));
 972     }
 973     return Variadic<WPI::Value>{std::move(values)};
 974   }
 975
 976   static std::vector<WPI::Key> s_inputKeys;
 977 };
 978
 979 std::vector<WPI::Key> ParseForHHBBCJob::s_inputKeys;
 980
 981 Job<ParseJob> s_parseJob;
 982 Job<ParseForHHBBCJob> s_parseForHHBBCJob;
 983
 984 ///////////////////////////////////////////////////////////////////////////////
 985
 986 bool process(const CompilerOptions &po) {
 987 #ifndef _MSC_VER
 988   LightProcess::Initialize(RuntimeOption::LightProcessFilePrefix,
 989                            RuntimeOption::LightProcessCount,
 990                            RuntimeOption::EvalRecordSubprocessTimes,
 991                            {});
 992 #endif
 993
 994   setCoredumps(po);
 995
 996   register_process_init();
 997
 998   StructuredLogEntry sample;
 999   sample.setStr("debug", debug ? "true" : "false");
1000   sample.setStr("use_case", Option::ExternWorkerUseCase);
1001   sample.setInt("use_rich_client", Option::ExternWorkerUseRichClient);
1002   sample.setInt("use_zippy_rich_client",
1003       Option::ExternWorkerUseZippyRichClient);
1004   sample.setInt("use_p2p", Option::ExternWorkerUseP2P);
1005   sample.setInt("force_subprocess", Option::ExternWorkerForceSubprocess);
1006   sample.setInt("use_exec_cache", Option::ExternWorkerUseExecCache);
1007   sample.setInt("timeout_secs", Option::ExternWorkerTimeoutSecs);
1008   sample.setInt("cleanup", Option::ExternWorkerCleanup);
1009   sample.setInt("throttle_retries", Option::ExternWorkerThrottleRetries);
1010   sample.setInt("throttle_base_wait_ms",
1011                 Option::ExternWorkerThrottleBaseWaitMSecs);
1012   sample.setStr("working_dir", Option::ExternWorkerWorkingDir);
1013   sample.setInt("parser_group_size", Option::ParserGroupSize);
1014   sample.setInt("parser_dir_group_size_limit", Option::ParserDirGroupSizeLimit);
1015   sample.setInt("parser_thread_count", Option::ParserThreadCount);
1016   sample.setInt("parser_optimistic_store", Option::ParserOptimisticStore);
1017   sample.setInt("parser_async_cleanup", Option::ParserAsyncCleanup);
1018   sample.setStr("push_phases", po.push_phases);
1019   sample.setStr("matched_overrides", po.matched_overrides);
1020   sample.setStr("use_hhbbc", RO::EvalUseHHBBC ? "true" : "false");
1021
1022   // Track the unit-emitters created for system during
1023   // hphp_process_init().
1024   SystemLib::keepRegisteredUnitEmitters(true);
1025   hphp_process_init();
1026   SCOPE_EXIT { hphp_process_exit(); };
1027   SystemLib::keepRegisteredUnitEmitters(false);
1028
1029   auto const outputFile = po.outputDir + "/hhvm.hhbc";
1030   unlink(outputFile.c_str());
1031
1032   auto executor = std::make_unique<coro::TicketExecutor>(
1033     "HPHPcWorker",
1034     0,
1035     size_t(Option::ParserThreadCount <= 0 ? 1 : Option::ParserThreadCount),
1036     [] {
1037       hphp_thread_init();
1038       hphp_session_init(Treadmill::SessionKind::CompilerEmit);
1039     },
1040     [] {
1041       hphp_context_exit();
1042       hphp_session_exit();
1043       hphp_thread_exit();
1044     },
1045     std::chrono::minutes{15}
1046   );
1047   auto client =
1048     std::make_unique<Client>(executor->sticky(), makeExternWorkerOptions());
1049
1050   sample.setStr("extern_worker_impl", client->implName());
1051
1052   UnitIndex index;
1053   if (!computeIndex(po, sample, *executor, *client, index)) return false;
1054
1055   auto package = std::make_unique<Package>(
1056     po.inputDir,
1057     po.parseOnDemand,
1058     *executor,
1059     *client
1060   );
1061
1062   // Always used, but we can clear it early to save memory.
1063   Optional<SymbolSets> unique;
1064   unique.emplace();
1065
1066   Optional<RepoAutoloadMapBuilder> autoload;
1067   Optional<RepoFileBuilder> repo;
1068   std::atomic<uint32_t> nextSn{0};
1069   std::mutex repoLock;
1070
1071   // HHBBC specific state (if we're going to run it).
1072   Optional<WPI> hhbbcInputs;
1073   Optional<coro::AsyncValue<Ref<HHBBC::Config>>> hhbbcConfig;
1074   if (RO::EvalUseHHBBC) {
1075     hhbbcInputs.emplace();
1076     // We want to do this as early as possible
1077     hhbbcConfig.emplace(
1078       [&client] () {
1079         return client->store(HHBBC::Config::get(getGlobalData()));
1080       },
1081       executor->sticky()
1082     );
1083   }
1084
1085   // Emit a fully processed unit (either processed by HHBBC or not).
1086   auto const emit = [&] (std::unique_ptr<UnitEmitter> ue) {
1087     assertx(ue);
1088     assertx(Option::GenerateBinaryHHBC ||
1089             Option::GenerateTextHHBC ||
1090             Option::GenerateHhasHHBC);
1091
1092     if (Option::GenerateTextHHBC || Option::GenerateHhasHHBC) {
1093       genText(*ue, po.outputDir);
1094     }
1095
1096     if (!Option::GenerateBinaryHHBC) return;
1097
1098     if (!RO::EvalUseHHBBC) {
1099       // HHBBC assigns m_sn and the SHA1, but we have to do it ourself
1100       // if we're not running it.
1101       auto const sn = nextSn++;
1102       ue->m_symbol_refs.clear();
1103       ue->m_sn = sn;
1104       ue->setSha1(SHA1 { sn });
1105       unique->add(*ue);
1106     }
1107
1108     autoload->addUnit(*ue);
1109     RepoFileBuilder::EncodedUE encoded{*ue};
1110     std::scoped_lock<std::mutex> _{repoLock};
1111     repo->add(encoded);
1112   };
1113
1114   // Process unit-emitters produced locally (usually systemlib stuff).
1115   auto const local = [&] (Package::UEVec ues) -> coro::Task<void> {
1116     if (RO::EvalUseHHBBC) {
1117       // If we're using HHBBC, turn them into WholeProgramInput
1118       // key/values (after checking uniqueness), upload the values,
1119       // and store them in the WholeProgramInput.
1120       std::vector<WPI::Key> keys;
1121       std::vector<WPI::Value> values;
1122
1123       for (auto& ue : ues) {
1124         unique->add(*ue);
1125         for (auto& [key, value] : WPI::make(std::move(ue))) {
1126           keys.emplace_back(std::move(key));
1127           values.emplace_back(std::move(value));
1128         }
1129       }
1130
1131       if (keys.empty()) HPHP_CORO_RETURN_VOID;
1132       auto valueRefs = HPHP_CORO_AWAIT(client->storeMulti(std::move(values)));
1133
1134       auto const numKeys = keys.size();
1135       assertx(valueRefs.size() == numKeys);
1136
1137       for (size_t i = 0; i < numKeys; ++i) {
1138         hhbbcInputs->add(std::move(keys[i]), std::move(valueRefs[i]));
1139       }
1140       HPHP_CORO_RETURN_VOID;
1141     }
1142
1143     // Otherwise just emit it
1144     for (auto& ue : ues) emit(std::move(ue));
1145     HPHP_CORO_RETURN_VOID;
1146   };
1147
1148   // Parse a group of files remotely
1149   auto const remote = [&] (const Ref<Package::Config>& config,
1150                            Ref<Package::FileMetaVec> fileMetas,
1151                            std::vector<Package::FileData> files,
1152                            bool optimistic)
1153     -> coro::Task<Package::ParseMetaVec> {
1154     if (RO::EvalUseHHBBC) {
1155       // Run the HHBBC parse job, which produces WholeProgramInput
1156       // key/values.
1157       auto hhbbcConfigRef = HPHP_CORO_AWAIT(hhbbcConfig->getCopy());
1158       auto [inputValueRefs, metaRefs] = HPHP_CORO_AWAIT(
1159         client->exec(
1160           s_parseForHHBBCJob,
1161           std::make_tuple(
1162             config,
1163             std::move(hhbbcConfigRef),
1164             std::move(fileMetas)
1165           ),
1166           std::move(files),
1167           optimistic
1168         )
1169       );
1170
1171       // The parse metadata and the keys are loaded, but the values
1172       // are kept as Refs.
1173       auto [parseMetas, inputKeys] =
1174         HPHP_CORO_AWAIT(client->load(std::move(metaRefs)));
1175
1176       // We don't have unit-emitters to do uniqueness checking, but
1177       // the parse metadata has the definitions we can use instead.
1178       for (auto const& p : parseMetas) {
1179         if (!p.m_filepath) continue;
1180         unique->add(p.m_definitions, p.m_filepath);
1181       }
1182
1183       auto const numKeys = inputKeys.size();
1184       size_t keyIdx = 0;
1185       for (auto& v : inputValueRefs) {
1186         for (auto& r : v) {
1187           always_assert(keyIdx < numKeys);
1188           hhbbcInputs->add(std::move(inputKeys[keyIdx]), std::move(r));
1189           ++keyIdx;
1190         }
1191       }
1192       always_assert(keyIdx == numKeys);
1193
1194       HPHP_CORO_MOVE_RETURN(parseMetas);
1195     }
1196
1197     // Otherwise, do a "normal" (non-HHBBC parse job), load the
1198     // unit-emitters and parse metadata, and emit the unit-emitters.
1199     auto [wrapperRefs, metaRefs] = HPHP_CORO_AWAIT(
1200       client->exec(
1201         s_parseJob,
1202         std::make_tuple(
1203           config,
1204           std::move(fileMetas)
1205         ),
1206         std::move(files),
1207         optimistic
1208       )
1209     );
1210
1211     auto [wrappers, parseMetas] = HPHP_CORO_AWAIT(coro::collect(
1212       client->load(std::move(wrapperRefs)),
1213       client->load(std::move(metaRefs))
1214     ));
1215
1216     for (auto& wrapper : wrappers) {
1217       if (!wrapper.m_ue) continue;
1218       emit(std::move(wrapper.m_ue));
1219     }
1220     HPHP_CORO_MOVE_RETURN(parseMetas);
1221   };
1222
1223   {
1224     Timer parseTimer(Timer::WallTime, "parsing");
1225     addInputsToPackage(*package, po);
1226
1227     if (!RO::EvalUseHHBBC && Option::GenerateBinaryHHBC) {
1228       autoload.emplace();
1229       repo.emplace(outputFile);
1230     }
1231     if (!coro::wait(package->parse(index, remote, local))) return false;
1232
1233     logPhaseStats("parse", *package, *client, sample,
1234                   parseTimer.getMicroSeconds());
1235   }
1236
1237   // We don't need the ondemand/autoload index after this point.
1238   index.clear();
1239
1240   std::thread fileCache{
1241     [&, package = std::move(package)] () mutable {
1242       SCOPE_EXIT { package.reset(); };
1243       if (po.filecache.empty()) return;
1244       Timer _{Timer::WallTime, "saving file cache..."};
1245       HphpSessionAndThread session{Treadmill::SessionKind::CompilerEmit};
1246       package->getFileCache()->save(po.filecache.c_str());
1247       struct stat sb;
1248       stat(po.filecache.c_str(), &sb);
1249       Logger::Info("%" PRId64" MB %s saved",
1250                    (int64_t)sb.st_size/(1024*1024), po.filecache.c_str());
1251     }
1252   };
1253   SCOPE_EXIT { fileCache.join(); };
1254
1255   std::thread asyncDispose;
1256   SCOPE_EXIT { if (asyncDispose.joinable()) asyncDispose.join(); };
1257   auto const dispose = [&] (std::unique_ptr<coro::TicketExecutor> e,
1258                             std::unique_ptr<Client> c) {
1259     if (!Option::ParserAsyncCleanup) {
1260       // If we don't want to cleanup asynchronously, do so now.
1261       c.reset();
1262       e.reset();
1263       return;
1264     }
1265     // All the thread does is reset the unique_ptr to run the dtor.
1266     asyncDispose = std::thread{
1267       [e = std::move(e), c = std::move(c)] () mutable {
1268         c.reset();
1269         e.reset();
1270       }
1271     };
1272   };
1273
1274   auto const finish = [&] {
1275     if (!Option::GenerateBinaryHHBC) return true;
1276     Timer _{Timer::WallTime, "finalizing repo"};
1277     repo->finish(getGlobalData(), *autoload);
1278     return true;
1279   };
1280   if (!RO::EvalUseHHBBC) {
1281     dispose(std::move(executor), std::move(client));
1282     return finish();
1283   }
1284
1285   // We don't need these anymore, and since they can consume a lot of
1286   // memory, free them before doing anything else.
1287   unique.reset();
1288   hhbbcConfig.reset();
1289
1290   assertx(!autoload.has_value());
1291   assertx(!repo.has_value());
1292   if (Option::GenerateBinaryHHBC) {
1293     autoload.emplace();
1294     repo.emplace(outputFile);
1295   }
1296
1297   if (Option::ConstFoldFileBC) {
1298     HHBBC::options.SourceRootForFileBC = RO::SourceRoot;
1299   }
1300
1301   Timer timer{Timer::WallTime, "running HHBBC"};
1302   HphpSession session{Treadmill::SessionKind::HHBBC};
1303   HHBBC::whole_program(
1304     std::move(*hhbbcInputs),
1305     std::move(executor),
1306     std::move(client),
1307     emit,
1308     dispose,
1309     sample,
1310     Option::ParserThreadCount > 0 ? Option::ParserThreadCount : 0
1311   );
1312   return finish();
1313 }
1314
1315 ///////////////////////////////////////////////////////////////////////////////
1316
1317 }
1318
1319 ///////////////////////////////////////////////////////////////////////////////
1320
1321 int compiler_main(int argc, char **argv) {
1322   try {
1323     rds::local::init();
1324     SCOPE_EXIT { rds::local::fini(); };
1325
1326     CompilerOptions po;
1327     auto const ret = prepareOptions(po, argc, argv);
1328     if (ret == 1) return 0; // --help
1329     if (ret != 0) return ret; // command line error
1330
1331     Timer totalTimer(Timer::WallTime, "running hphp");
1332     mkdir(po.outputDir.c_str(), 0777);
1333     if (!process(po)) {
1334       Logger::Error("hphp failed");
1335       return -1;
1336     } else {
1337       Logger::Info("all files saved in %s ...", po.outputDir.c_str());
1338       return 0;
1339     }
1340   } catch (const Exception& e) {
1341     Logger::Error("Exception: %s", e.getMessage().c_str());
1342   } catch (const std::exception& e) {
1343     Logger::Error("std::exception: %s", e.what());
1344   } catch (...) {
1345     Logger::Error("(non-standard exception \"%s\" was thrown)",
1346                   current_exception_name().c_str());
1347   }
1348   return -1;
1349 }
1350
1351 ///////////////////////////////////////////////////////////////////////////////
1352
1353 }