Make HHBBC validate that all symbols are unique
[hiphop-php.git] / hphp / hhbbc / main.cpp
blobb74855815fd4fdb49c96e49b89aaf8992c4d974b
1 /*
2 +----------------------------------------------------------------------+
3 | HipHop for PHP |
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
16 #include "hphp/hhbbc/hhbbc.h"
18 #include <stdexcept>
19 #include <iostream>
20 #include <cstdlib>
21 #include <string>
22 #include <memory>
23 #include <cstdint>
24 #include <algorithm>
25 #include <exception>
26 #include <utility>
27 #include <vector>
29 #include <boost/program_options.hpp>
30 #include <boost/filesystem.hpp>
32 #include <folly/ScopeGuard.h>
33 #include <folly/String.h>
34 #include <folly/portability/Unistd.h>
36 #include "hphp/runtime/base/ini-setting.h"
37 #include "hphp/runtime/base/runtime-option.h"
38 #include "hphp/runtime/base/vm-worker.h"
39 #include "hphp/hhvm/process-init.h"
40 #include "hphp/runtime/vm/native.h"
41 #include "hphp/runtime/vm/repo.h"
42 #include "hphp/runtime/vm/repo-autoload-map-builder.h"
43 #include "hphp/runtime/vm/repo-global-data.h"
44 #include "hphp/runtime/vm/treadmill.h"
46 #include "hphp/hhbbc/misc.h"
47 #include "hphp/hhbbc/options.h"
48 #include "hphp/hhbbc/stats.h"
49 #include "hphp/hhbbc/parallel.h"
50 #include "hphp/hhbbc/representation.h"
52 #include "hphp/util/rds-local.h"
53 #include "hphp/util/logger.h"
55 namespace HPHP { namespace HHBBC {
57 namespace {
59 namespace fs = boost::filesystem;
61 //////////////////////////////////////////////////////////////////////
63 std::string output_repo;
64 std::string input_repo;
65 std::string hack_compiler_extract_path;
66 bool logging = true;
67 bool print_bytecode_stats_and_exit = false;
70 //////////////////////////////////////////////////////////////////////
72 template<class SinglePassReadableRange>
73 MethodMap make_method_map(SinglePassReadableRange& range) {
74 auto ret = MethodMap{};
75 for (auto& str : range) {
76 std::vector<std::string> parts;
77 folly::split("::", str, parts);
78 if (parts.size() != 2) {
79 ret[""].insert(str);
80 continue;
82 ret[parts[0]].insert(parts[1]);
84 return ret;
87 template<class SinglePassReadableRange>
88 OpcodeSet make_bytecode_map(SinglePassReadableRange& bcs) {
89 if (bcs.empty()) return {};
90 hphp_fast_map<std::string,Op> bcmap;
91 for (auto i = 0; i < Op_count; i++) {
92 auto const op = static_cast<Op>(i);
93 bcmap[opcodeToName(op)] = op;
95 OpcodeSet oset;
96 for (auto& n : bcs) {
97 if (bcmap.count(n)) oset.insert(bcmap[n]);
99 return oset;
102 void parse_options(int argc, char** argv) {
103 namespace po = boost::program_options;
105 auto const defaultThreadCount =
106 std::max<long>(sysconf(_SC_NPROCESSORS_ONLN) - 1, 1);
108 std::vector<std::string> interceptable_fns;
109 std::vector<std::string> trace_fns;
110 std::vector<std::string> trace_bcs;
111 bool no_logging = false;
112 bool no_cores = false;
114 po::options_description basic("Options");
115 basic.add_options()
116 ("help", "display help message")
117 ("output,o",
118 po::value(&output_repo)->default_value("hhvm.hhbbc"),
119 "output hhbc repo path")
120 ("input",
121 po::value(&input_repo)->default_value("hhvm.hhbc"),
122 "input hhbc repo path")
123 ("stats-file",
124 po::value(&options.stats_file)->default_value(""),
125 "stats file path")
126 ("no-optimizations",
127 po::bool_switch(&options.NoOptimizations),
128 "turn off all optimizations")
129 ("no-logging",
130 po::bool_switch(&no_logging),
131 "turn off logging")
132 ("no-cores",
133 po::bool_switch(&no_cores),
134 "turn off core dumps (useful when running lots of tests in parallel)")
135 ("extended-stats",
136 po::bool_switch(&options.extendedStats),
137 "Spend time to produce extra stats")
138 ("profile-memory",
139 po::value(&options.profileMemory)->default_value(""),
140 "If non-empty, dump jemalloc memory profiles at key points")
141 ("parallel-num-threads",
142 po::value(&parallel::num_threads)->default_value(defaultThreadCount),
143 "Number of threads to use for parallelism")
144 ("parallel-final-threads",
145 po::value(&parallel::final_threads)->default_value(
146 parallel::final_threads),
147 "Number of threads to use for the final pass")
148 ("parallel-work-size",
149 po::value(&parallel::work_chunk)->default_value(120),
150 "Work unit size for parallelism")
151 ("trace",
152 po::value(&trace_fns)->composing(),
153 "Add a function to increase tracing level on (for debugging)")
154 ("trace-bytecode",
155 po::value(&trace_bcs)->composing(),
156 "Add a bytecode to trace (for debugging)")
157 ("hack-compiler-extract-path",
158 po::value(&hack_compiler_extract_path)->default_value(""),
159 "hack compiler extract path")
162 // Some extra esoteric options that aren't exposed in --help for
163 // now.
164 po::options_description extended("Extended Options");
165 extended.add_options()
166 ("analyze-func-wlimit", po::value(&options.analyzeFuncWideningLimit))
167 ("analyze-class-wlimit", po::value(&options.analyzeClassWideningLimit))
168 ("return-refine-limit", po::value(&options.returnTypeRefineLimit))
169 ("public-sprop-refine-limit", po::value(&options.publicSPropRefineLimit))
170 ("bytecode-stats", po::bool_switch(&print_bytecode_stats_and_exit))
173 po::options_description oflags("Optimization Flags");
174 oflags.add_options()
175 ("context-sensitive-interp", po::value(&options.ContextSensitiveInterp))
176 ("remove-dead-blocks", po::value(&options.RemoveDeadBlocks))
177 ("constant-prop", po::value(&options.ConstantProp))
178 ("constant-fold-builtins", po::value(&options.ConstantFoldBuiltins))
179 ("local-dce", po::value(&options.LocalDCE))
180 ("global-dce", po::value(&options.GlobalDCE))
181 ("remove-unused-local-names", po::value(&options.RemoveUnusedLocalNames))
182 ("compact-local-slots", po::value(&options.CompactLocalSlots))
183 ("insert-assertions", po::value(&options.InsertAssertions))
184 ("insert-stack-assertions", po::value(&options.InsertStackAssertions))
185 ("filter-assertions", po::value(&options.FilterAssertions))
186 ("strength-reduce", po::value(&options.StrengthReduce))
187 ("func-families", po::value(&options.FuncFamilies))
188 ("hard-private-prop", po::value(&options.HardPrivatePropInference))
189 ("analyze-pseudomains", po::value(&options.AnalyzePseudomains))
190 ("analyze-public-statics", po::value(&options.AnalyzePublicStatics))
193 po::options_description all;
194 all.add(basic).add(extended).add(oflags);
196 po::positional_options_description pd;
197 pd.add("input", 1);
199 po::variables_map vm;
200 po::store(
201 po::command_line_parser(argc, argv)
202 .options(all)
203 .positional(pd)
204 .extra_parser(
205 [&] (const std::string& s) -> std::pair<std::string,std::string> {
206 if (s.find("-f") == 0) {
207 if (s.substr(2, 3) == "no-") {
208 return { s.substr(5), "false" };
210 return { s.substr(2), "true" };
212 return {};
214 .run(),
217 po::notify(vm);
219 if (vm.count("help")) {
220 std::cout << basic << "\n"
221 "Individual optimizations may be turned on and off using gcc-style -fflag\n"
222 "and -fno-flag arguments. The various optimization flags are documented\n"
223 "in the code.\n";
224 std::exit(0);
227 if (no_cores) {
228 struct rlimit rl{};
229 setrlimit(RLIMIT_CORE, &rl);
232 if (!options.ConstantProp) options.ConstantFoldBuiltins = false;
234 options.TraceFunctions = make_method_map(trace_fns);
235 options.TraceBytecodes = make_bytecode_map(trace_bcs);
237 if (!options.profileMemory.empty()) {
238 mallctlWrite("prof.active", true);
239 mallctlWrite("prof.thread_active_init", true);
242 logging = !no_logging;
245 UNUSED void validate_options() {
246 if (parallel::work_chunk <= 10 || parallel::num_threads < 1) {
247 std::cerr << "Invalid parallelism configuration.\n";
248 std::exit(1);
251 if (options.AnalyzePublicStatics && !options.AnalyzePseudomains) {
252 std::cerr << "-fanalyze-public-statics requires -fanalyze-pseudomains\n";
253 std::exit(1);
256 if (options.RemoveUnusedLocalNames && !options.GlobalDCE) {
257 std::cerr << "-fremove-unused-local-names requires -fglobal-dce\n";
258 std::exit(1);
261 if (options.CompactLocalSlots && !options.GlobalDCE) {
262 std::cerr << "-fcompact-local-slots requires -fglobal-dce\n";
263 std::exit(1);
267 //////////////////////////////////////////////////////////////////////
269 void open_repo(const std::string& path) {
270 RuntimeOption::RepoCentralPath = path;
271 // Make sure the changes take effect
272 Repo::shutdown();
273 Repo::get();
276 template<typename F>
277 std::vector<SString> load_input(F&& fun) {
278 trace_time timer("load units");
280 open_repo(input_repo);
281 Repo::get().loadGlobalData();
282 SCOPE_EXIT { Repo::shutdown(); };
284 auto const& gd = Repo::get().global();
285 // When running hhbbc, these option is loaded from GD, and will override CLI.
286 // When running hhvm, these option is not loaded from GD, but read from CLI.
287 RO::EvalJitEnableRenameFunction = gd.EnableRenameFunction;
288 RO::EvalHackArrCompatNotices =
289 RO::EvalHackArrCompatCheckCompare =
290 RO::EvalHackArrCompatCheckArrayPlus =
291 RO::EvalHackArrCompatCheckArrayKeyCast =
292 gd.HackArrCompatNotices;
293 RO::EvalForbidDynamicCallsToFunc = gd.ForbidDynamicCallsToFunc;
294 RO::EvalForbidDynamicCallsToClsMeth =
295 gd.ForbidDynamicCallsToClsMeth;
296 RO::EvalForbidDynamicCallsToInstMeth =
297 gd.ForbidDynamicCallsToInstMeth;
298 RO::EvalForbidDynamicConstructs = gd.ForbidDynamicConstructs;
299 RO::EvalForbidDynamicCallsWithAttr =
300 gd.ForbidDynamicCallsWithAttr;
301 RO::EvalLogKnownMethodsAsDynamicCalls =
302 gd.LogKnownMethodsAsDynamicCalls;
303 RO::EvalNoticeOnBuiltinDynamicCalls =
304 gd.NoticeOnBuiltinDynamicCalls;
305 RO::EvalHackArrCompatIsArrayNotices =
306 gd.HackArrCompatIsArrayNotices;
307 RO::EvalHackArrCompatTypeHintNotices =
308 gd.HackArrCompatTypeHintNotices;
309 RO::EvalHackArrCompatDVCmpNotices =
310 gd.HackArrCompatDVCmpNotices;
311 RO::EvalHackArrCompatSerializeNotices =
312 gd.HackArrCompatSerializeNotices;
313 RO::EvalHackArrCompatSpecialization = gd.HackArrCompatSpecialization;
314 RO::EvalHackArrDVArrs = gd.HackArrDVArrs;
315 RO::EvalAbortBuildOnVerifyError = gd.AbortBuildOnVerifyError;
316 RO::EnableArgsInBacktraces = gd.EnableArgsInBacktraces;
317 RO::EvalEmitClsMethPointers = gd.EmitClsMethPointers;
318 RO::EvalIsVecNotices = gd.IsVecNotices;
319 RO::EvalIsCompatibleClsMethType = gd.IsCompatibleClsMethType;
320 RO::EvalArrayProvenance =
321 RO::EvalArrProvHackArrays =
322 RO::EvalArrProvDVArrays =
323 gd.ArrayProvenance;
324 RO::StrictArrayFillKeys = gd.StrictArrayFillKeys;
325 RO::EvalEnableFuncStringInterop = gd.EnableFuncStringInterop;
326 if (gd.HardGenericsUB) {
327 RO::EvalEnforceGenericsUB = 2;
328 } else {
329 RO::EvalEnforceGenericsUB = 1;
332 auto const units = Repo::get().enumerateUnits(RepoIdCentral, true);
333 auto const size = units.size();
334 fun(size, nullptr);
335 parallel::for_each(
336 units,
337 [&] (const std::pair<std::string,SHA1>& kv) {
338 fun(
339 size,
340 Repo::get().urp().loadEmitter(
341 kv.first, kv.second, Native::s_noNativeFuncs
346 return Repo().get().global().APCProfile;
349 void write_units(UnitEmitterQueue& ueq,
350 RepoAutoloadMapBuilder& autoloadMapBuilder) {
351 folly::Optional<trace_time> timer;
353 RuntimeOption::RepoCommit = true;
354 RuntimeOption::RepoEvalMode = "local";
355 RuntimeOption::RepoDebugInfo = false; // Don't record UnitSourceLoc
356 open_repo(output_repo);
357 SCOPE_EXIT { Repo::shutdown(); };
359 std::vector<std::unique_ptr<UnitEmitter>> ues;
360 while (auto ue = ueq.pop()) {
361 if (!timer) timer.emplace("writing output repo");
362 autoloadMapBuilder.addUnit(*ue);
363 ues.push_back(std::move(ue));
364 if (ues.size() == 8) {
365 auto const DEBUG_ONLY err = batchCommitWithoutRetry(ues, true);
366 always_assert(!err);
367 ues.clear();
371 auto const DEBUG_ONLY err = batchCommitWithoutRetry(ues, true);
372 always_assert(!err);
373 ues.clear();
376 void write_global_data(
377 std::unique_ptr<ArrayTypeTable::Builder>& arrTable,
378 std::vector<SString> apcProfile,
379 const RepoAutoloadMapBuilder& autoloadMapBuilder) {
381 auto const now = std::chrono::high_resolution_clock::now();
382 auto const nanos =
383 std::chrono::duration_cast<std::chrono::nanoseconds>(
384 now.time_since_epoch()
387 auto gd = Repo::GlobalData{};
388 gd.Signature = nanos.count();
389 gd.HardGenericsUB = RuntimeOption::EvalEnforceGenericsUB >= 2;
390 gd.HardReturnTypeHints = RuntimeOption::EvalCheckReturnTypeHints >= 3;
391 gd.CheckPropTypeHints = RuntimeOption::EvalCheckPropTypeHints;
392 gd.HardPrivatePropInference = options.HardPrivatePropInference;
393 gd.PHP7_NoHexNumerics = RuntimeOption::PHP7_NoHexNumerics;
394 gd.PHP7_Substr = RuntimeOption::PHP7_Substr;
395 gd.PHP7_Builtins = RuntimeOption::PHP7_Builtins;
396 gd.EnableRenameFunction = RuntimeOption::EvalJitEnableRenameFunction;
397 gd.HackArrCompatNotices = RuntimeOption::EvalHackArrCompatNotices;
398 gd.EnableIntrinsicsExtension = RuntimeOption::EnableIntrinsicsExtension;
399 gd.APCProfile = std::move(apcProfile);
400 gd.ForbidDynamicCallsToFunc = RuntimeOption::EvalForbidDynamicCallsToFunc;
401 gd.ForbidDynamicCallsToClsMeth =
402 RuntimeOption::EvalForbidDynamicCallsToClsMeth;
403 gd.ForbidDynamicCallsToInstMeth =
404 RuntimeOption::EvalForbidDynamicCallsToInstMeth;
405 gd.ForbidDynamicConstructs = RuntimeOption::EvalForbidDynamicConstructs;
406 gd.ForbidDynamicCallsWithAttr =
407 RuntimeOption::EvalForbidDynamicCallsWithAttr;
408 gd.LogKnownMethodsAsDynamicCalls =
409 RuntimeOption::EvalLogKnownMethodsAsDynamicCalls;
410 gd.AbortBuildOnVerifyError = RuntimeOption::EvalAbortBuildOnVerifyError;
411 gd.EnableArgsInBacktraces = RuntimeOption::EnableArgsInBacktraces;
412 gd.NoticeOnBuiltinDynamicCalls =
413 RuntimeOption::EvalNoticeOnBuiltinDynamicCalls;
414 gd.HackArrCompatIsArrayNotices =
415 RuntimeOption::EvalHackArrCompatIsArrayNotices;
416 gd.HackArrCompatTypeHintNotices =
417 RuntimeOption::EvalHackArrCompatTypeHintNotices;
418 gd.HackArrCompatDVCmpNotices =
419 RuntimeOption::EvalHackArrCompatDVCmpNotices;
420 gd.HackArrCompatSerializeNotices =
421 RuntimeOption::EvalHackArrCompatSerializeNotices;
422 gd.HackArrCompatSpecialization =
423 RuntimeOption::EvalHackArrCompatSpecialization;
424 gd.HackArrDVArrs = RuntimeOption::EvalHackArrDVArrs;
425 gd.InitialNamedEntityTableSize =
426 RuntimeOption::EvalInitialNamedEntityTableSize;
427 gd.InitialStaticStringTableSize =
428 RuntimeOption::EvalInitialStaticStringTableSize;
429 gd.EmitClsMethPointers = RuntimeOption::EvalEmitClsMethPointers;
430 gd.IsVecNotices = RuntimeOption::EvalIsVecNotices;
431 gd.IsCompatibleClsMethType = RuntimeOption::EvalIsCompatibleClsMethType;
432 gd.ArrayProvenance = RuntimeOption::EvalArrayProvenance;
433 gd.StrictArrayFillKeys = RuntimeOption::StrictArrayFillKeys;
434 gd.EnableFuncStringInterop = RO::EvalEnableFuncStringInterop;
436 for (auto const& elm : RuntimeOption::ConstantFunctions) {
437 gd.ConstantFunctions.push_back(elm);
440 if (arrTable) globalArrayTypeTable().repopulate(*arrTable);
441 // NOTE: There's no way to tell if saveGlobalData() fails for some reason.
442 Repo::get().saveGlobalData(std::move(gd), autoloadMapBuilder);
445 void compile_repo() {
446 auto program = make_program();
448 auto apcProfile = load_input(
449 [&] (size_t size, std::unique_ptr<UnitEmitter> ue) {
450 if (!ue) {
451 if (logging) {
452 std::cout << folly::format("{} units\n", size);
454 return;
456 add_unit_to_program(ue.get(), *program);
460 UnitEmitterQueue ueq;
461 std::unique_ptr<ArrayTypeTable::Builder> arrTable;
462 std::exception_ptr wp_thread_ex = nullptr;
463 VMWorker wp_thread(
464 [&] {
465 HphpSession _{Treadmill::SessionKind::CompileRepo};
466 Trace::BumpRelease bumper(Trace::hhbbc_time, -1, logging);
467 try {
468 whole_program(std::move(program), ueq, arrTable);
469 } catch (...) {
470 wp_thread_ex = std::current_exception();
471 ueq.push(nullptr);
475 wp_thread.start();
477 RepoAutoloadMapBuilder autoloadMapBuilder;
478 write_units(ueq, autoloadMapBuilder);
479 write_global_data(arrTable, apcProfile, autoloadMapBuilder);
482 wp_thread.waitForEnd();
483 if (wp_thread_ex) {
484 rethrow_exception(wp_thread_ex);
488 void print_repo_bytecode_stats() {
489 std::array<std::atomic<uint64_t>,Op_count> op_counts{};
491 auto const input = load_input(
492 [&] (size_t, std::unique_ptr<UnitEmitter> ue) {
493 if (!ue) return;
494 auto pc = ue->bc();
495 auto const end = pc + ue->bcPos();
496 for (; pc < end; pc += instrLen(pc)) {
497 auto &opc = op_counts[static_cast<uint16_t>(peek_op(pc))];
498 opc.fetch_add(1, std::memory_order_relaxed);
503 for (auto i = uint32_t{}; i < op_counts.size(); ++i) {
504 std::cout << folly::format(
505 "{: <20} {}\n",
506 opcodeToName(static_cast<Op>(i)),
507 op_counts[i].load(std::memory_order_relaxed)
512 //////////////////////////////////////////////////////////////////////
516 int main(int argc, char** argv) try {
517 parse_options(argc, argv);
519 if (!print_bytecode_stats_and_exit && fs::exists(output_repo)) {
520 std::cout << "output repo already exists; removing it\n";
521 if (unlink(output_repo.c_str())) {
522 std::cerr << "failed to unlink output repo: "
523 << strerror(errno) << '\n';
524 return 1;
527 if (!fs::exists(input_repo)) {
528 std::cerr << "input repo `" << input_repo << "' not found\n";
529 return 1;
532 initialize_repo();
534 // We need to set this flag so Repo::global will let us access it.
535 RuntimeOption::RepoAuthoritative = true;
537 LitstrTable::init();
538 RuntimeOption::RepoLocalMode = "--";
539 RuntimeOption::RepoEvalMode = "readonly";
540 open_repo(input_repo);
541 Repo::get().loadGlobalData(false);
542 LitstrTable::fini();
543 auto const& gd = Repo::get().global();
544 if (gd.InitialNamedEntityTableSize) {
545 RuntimeOption::EvalInitialNamedEntityTableSize =
546 gd.InitialNamedEntityTableSize;
548 if (gd.InitialStaticStringTableSize) {
549 RuntimeOption::EvalInitialStaticStringTableSize =
550 gd.InitialStaticStringTableSize;
553 rds::local::init();
554 SCOPE_EXIT { rds::local::fini(); };
556 Hdf config;
557 IniSetting::Map ini = IniSetting::Map::object;
558 RuntimeOption::Load(ini, config);
559 RuntimeOption::RepoLocalPath = "/tmp/hhbbc.repo";
560 RuntimeOption::RepoCentralPath = input_repo;
561 RuntimeOption::RepoLocalMode = "--";
562 RuntimeOption::RepoJournal = "memory";
563 RuntimeOption::RepoCommit = false;
564 RuntimeOption::EvalJit = false;
566 if (!hack_compiler_extract_path.empty()) {
567 RuntimeOption::EvalHackCompilerExtractPath = hack_compiler_extract_path;
570 register_process_init();
572 hphp_process_init();
573 LitstrTable::get().setWriting();
574 SCOPE_EXIT { hphp_process_exit(); };
576 Repo::shutdown();
578 if (print_bytecode_stats_and_exit) {
579 print_repo_bytecode_stats();
580 return 0;
583 Trace::BumpRelease bumper(Trace::hhbbc_time, -1, logging);
584 compile_repo(); // NOTE: errors ignored
585 return 0;
588 catch (std::exception& e) {
589 Logger::Error("std::exception: %s", e.what());
590 return 1;
593 //////////////////////////////////////////////////////////////////////