When using hhbbc let the index decide unitSn instead of sqlite
[hiphop-php.git] / hphp / hhbbc / main.cpp
blobe1d066794e2cba1f98378afabf0f33cfb3e482d9
1 /*
2 +----------------------------------------------------------------------+
3 | HipHop for PHP |
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
16 #include "hphp/hhbbc/hhbbc.h"
18 #include <stdexcept>
19 #include <iostream>
20 #include <cstdlib>
21 #include <string>
22 #include <memory>
23 #include <cstdint>
24 #include <algorithm>
25 #include <exception>
26 #include <utility>
27 #include <vector>
29 #include <boost/program_options.hpp>
30 #include <boost/filesystem.hpp>
32 #include <folly/ScopeGuard.h>
33 #include <folly/String.h>
34 #include <folly/portability/Unistd.h>
36 #include "hphp/runtime/base/ini-setting.h"
37 #include "hphp/runtime/base/runtime-option.h"
38 #include "hphp/runtime/base/vm-worker.h"
39 #include "hphp/hhvm/process-init.h"
40 #include "hphp/runtime/vm/native.h"
41 #include "hphp/runtime/vm/repo.h"
42 #include "hphp/runtime/vm/repo-autoload-map-builder.h"
43 #include "hphp/runtime/vm/repo-global-data.h"
44 #include "hphp/runtime/vm/treadmill.h"
46 #include "hphp/hhbbc/misc.h"
47 #include "hphp/hhbbc/options.h"
48 #include "hphp/hhbbc/stats.h"
49 #include "hphp/hhbbc/parallel.h"
50 #include "hphp/hhbbc/representation.h"
52 #include "hphp/util/rds-local.h"
54 namespace HPHP { namespace HHBBC {
56 namespace {
58 namespace fs = boost::filesystem;
60 //////////////////////////////////////////////////////////////////////
62 std::string output_repo;
63 std::string input_repo;
64 std::string hack_compiler_extract_path;
65 bool logging = true;
66 bool print_bytecode_stats_and_exit = false;
69 //////////////////////////////////////////////////////////////////////
71 template<class SinglePassReadableRange>
72 MethodMap make_method_map(SinglePassReadableRange& range) {
73 auto ret = MethodMap{};
74 for (auto& str : range) {
75 std::vector<std::string> parts;
76 folly::split("::", str, parts);
77 if (parts.size() != 2) {
78 ret[""].insert(str);
79 continue;
81 ret[parts[0]].insert(parts[1]);
83 return ret;
86 template<class SinglePassReadableRange>
87 OpcodeSet make_bytecode_map(SinglePassReadableRange& bcs) {
88 if (bcs.empty()) return {};
89 hphp_fast_map<std::string,Op> bcmap;
90 for (auto i = 0; i < Op_count; i++) {
91 auto const op = static_cast<Op>(i);
92 bcmap[opcodeToName(op)] = op;
94 OpcodeSet oset;
95 for (auto& n : bcs) {
96 if (bcmap.count(n)) oset.insert(bcmap[n]);
98 return oset;
101 void parse_options(int argc, char** argv) {
102 namespace po = boost::program_options;
104 auto const defaultThreadCount =
105 std::max<long>(sysconf(_SC_NPROCESSORS_ONLN) - 1, 1);
107 std::vector<std::string> interceptable_fns;
108 std::vector<std::string> trace_fns;
109 std::vector<std::string> trace_bcs;
110 bool no_logging = false;
111 bool no_cores = false;
113 po::options_description basic("Options");
114 basic.add_options()
115 ("help", "display help message")
116 ("output,o",
117 po::value(&output_repo)->default_value("hhvm.hhbbc"),
118 "output hhbc repo path")
119 ("input",
120 po::value(&input_repo)->default_value("hhvm.hhbc"),
121 "input hhbc repo path")
122 ("stats-file",
123 po::value(&options.stats_file)->default_value(""),
124 "stats file path")
125 ("no-optimizations",
126 po::bool_switch(&options.NoOptimizations),
127 "turn off all optimizations")
128 ("no-logging",
129 po::bool_switch(&no_logging),
130 "turn off logging")
131 ("no-cores",
132 po::bool_switch(&no_cores),
133 "turn off core dumps (useful when running lots of tests in parallel)")
134 ("extended-stats",
135 po::bool_switch(&options.extendedStats),
136 "Spend time to produce extra stats")
137 ("profile-memory",
138 po::value(&options.profileMemory)->default_value(""),
139 "If non-empty, dump jemalloc memory profiles at key points")
140 ("parallel-num-threads",
141 po::value(&parallel::num_threads)->default_value(defaultThreadCount),
142 "Number of threads to use for parallelism")
143 ("parallel-final-threads",
144 po::value(&parallel::final_threads)->default_value(
145 parallel::final_threads),
146 "Number of threads to use for the final pass")
147 ("parallel-work-size",
148 po::value(&parallel::work_chunk)->default_value(120),
149 "Work unit size for parallelism")
150 ("trace",
151 po::value(&trace_fns)->composing(),
152 "Add a function to increase tracing level on (for debugging)")
153 ("trace-bytecode",
154 po::value(&trace_bcs)->composing(),
155 "Add a bytecode to trace (for debugging)")
156 ("hack-compiler-extract-path",
157 po::value(&hack_compiler_extract_path)->default_value(""),
158 "hack compiler extract path")
161 // Some extra esoteric options that aren't exposed in --help for
162 // now.
163 po::options_description extended("Extended Options");
164 extended.add_options()
165 ("analyze-func-wlimit", po::value(&options.analyzeFuncWideningLimit))
166 ("analyze-class-wlimit", po::value(&options.analyzeClassWideningLimit))
167 ("return-refine-limit", po::value(&options.returnTypeRefineLimit))
168 ("public-sprop-refine-limit", po::value(&options.publicSPropRefineLimit))
169 ("bytecode-stats", po::bool_switch(&print_bytecode_stats_and_exit))
172 po::options_description oflags("Optimization Flags");
173 oflags.add_options()
174 ("context-sensitive-interp", po::value(&options.ContextSensitiveInterp))
175 ("remove-dead-blocks", po::value(&options.RemoveDeadBlocks))
176 ("constant-prop", po::value(&options.ConstantProp))
177 ("constant-fold-builtins", po::value(&options.ConstantFoldBuiltins))
178 ("local-dce", po::value(&options.LocalDCE))
179 ("global-dce", po::value(&options.GlobalDCE))
180 ("remove-unused-local-names", po::value(&options.RemoveUnusedLocalNames))
181 ("compact-local-slots", po::value(&options.CompactLocalSlots))
182 ("insert-assertions", po::value(&options.InsertAssertions))
183 ("insert-stack-assertions", po::value(&options.InsertStackAssertions))
184 ("filter-assertions", po::value(&options.FilterAssertions))
185 ("strength-reduce", po::value(&options.StrengthReduce))
186 ("func-families", po::value(&options.FuncFamilies))
187 ("hard-private-prop", po::value(&options.HardPrivatePropInference))
188 ("analyze-pseudomains", po::value(&options.AnalyzePseudomains))
189 ("analyze-public-statics", po::value(&options.AnalyzePublicStatics))
192 po::options_description all;
193 all.add(basic).add(extended).add(oflags);
195 po::positional_options_description pd;
196 pd.add("input", 1);
198 po::variables_map vm;
199 po::store(
200 po::command_line_parser(argc, argv)
201 .options(all)
202 .positional(pd)
203 .extra_parser(
204 [&] (const std::string& s) -> std::pair<std::string,std::string> {
205 if (s.find("-f") == 0) {
206 if (s.substr(2, 3) == "no-") {
207 return { s.substr(5), "false" };
209 return { s.substr(2), "true" };
211 return {};
213 .run(),
216 po::notify(vm);
218 if (vm.count("help")) {
219 std::cout << basic << "\n"
220 "Individual optimizations may be turned on and off using gcc-style -fflag\n"
221 "and -fno-flag arguments. The various optimization flags are documented\n"
222 "in the code.\n";
223 std::exit(0);
226 if (no_cores) {
227 struct rlimit rl{};
228 setrlimit(RLIMIT_CORE, &rl);
231 if (!options.ConstantProp) options.ConstantFoldBuiltins = false;
233 options.TraceFunctions = make_method_map(trace_fns);
234 options.TraceBytecodes = make_bytecode_map(trace_bcs);
236 if (!options.profileMemory.empty()) {
237 mallctlWrite("prof.active", true);
238 mallctlWrite("prof.thread_active_init", true);
241 logging = !no_logging;
244 UNUSED void validate_options() {
245 if (parallel::work_chunk <= 10 || parallel::num_threads < 1) {
246 std::cerr << "Invalid parallelism configuration.\n";
247 std::exit(1);
250 if (options.AnalyzePublicStatics && !options.AnalyzePseudomains) {
251 std::cerr << "-fanalyze-public-statics requires -fanalyze-pseudomains\n";
252 std::exit(1);
255 if (options.RemoveUnusedLocalNames && !options.GlobalDCE) {
256 std::cerr << "-fremove-unused-local-names requires -fglobal-dce\n";
257 std::exit(1);
260 if (options.CompactLocalSlots && !options.GlobalDCE) {
261 std::cerr << "-fcompact-local-slots requires -fglobal-dce\n";
262 std::exit(1);
266 //////////////////////////////////////////////////////////////////////
268 void open_repo(const std::string& path) {
269 RuntimeOption::RepoCentralPath = path;
270 // Make sure the changes take effect
271 Repo::shutdown();
272 Repo::get();
275 template<typename F>
276 std::vector<SString> load_input(F&& fun) {
277 trace_time timer("load units");
279 open_repo(input_repo);
280 Repo::get().loadGlobalData();
281 SCOPE_EXIT { Repo::shutdown(); };
283 auto const& gd = Repo::get().global();
284 // When running hhbbc, these option is loaded from GD, and will override CLI.
285 // When running hhvm, these option is not loaded from GD, but read from CLI.
286 RO::EvalJitEnableRenameFunction = gd.EnableRenameFunction;
287 RO::EvalHackArrCompatNotices =
288 RO::EvalHackArrCompatCheckCompare =
289 RO::EvalHackArrCompatCheckArrayPlus =
290 RO::EvalHackArrCompatCheckArrayKeyCast =
291 gd.HackArrCompatNotices;
292 RO::EvalForbidDynamicCallsToFunc = gd.ForbidDynamicCallsToFunc;
293 RO::EvalForbidDynamicCallsToClsMeth =
294 gd.ForbidDynamicCallsToClsMeth;
295 RO::EvalForbidDynamicCallsToInstMeth =
296 gd.ForbidDynamicCallsToInstMeth;
297 RO::EvalForbidDynamicConstructs = gd.ForbidDynamicConstructs;
298 RO::EvalForbidDynamicCallsWithAttr =
299 gd.ForbidDynamicCallsWithAttr;
300 RO::EvalLogKnownMethodsAsDynamicCalls =
301 gd.LogKnownMethodsAsDynamicCalls;
302 RO::EvalNoticeOnBuiltinDynamicCalls =
303 gd.NoticeOnBuiltinDynamicCalls;
304 RO::EvalHackArrCompatIsArrayNotices =
305 gd.HackArrCompatIsArrayNotices;
306 RO::EvalHackArrCompatTypeHintNotices =
307 gd.HackArrCompatTypeHintNotices;
308 RO::EvalHackArrCompatDVCmpNotices =
309 gd.HackArrCompatDVCmpNotices;
310 RO::EvalHackArrCompatHackArrCmpNotices =
311 gd.HackArrCompatHackArrCmpNotices;
312 RO::EvalHackArrCompatSerializeNotices =
313 gd.HackArrCompatSerializeNotices;
314 RO::EvalHackArrDVArrs = gd.HackArrDVArrs;
315 RO::EvalHackArrEmptyBasedBoolEqCmp = gd.HackArrEmptyBasedBoolEqCmp;
316 RO::EvalAbortBuildOnVerifyError = gd.AbortBuildOnVerifyError;
317 RO::EnableArgsInBacktraces = gd.EnableArgsInBacktraces;
318 RO::EvalEmitClsMethPointers = gd.EmitClsMethPointers;
319 RO::EvalIsVecNotices = gd.IsVecNotices;
320 RO::EvalIsCompatibleClsMethType = gd.IsCompatibleClsMethType;
321 RO::EvalArrayProvenance =
322 RO::EvalArrProvHackArrays =
323 RO::EvalArrProvDVArrays =
324 gd.ArrayProvenance;
325 RO::StrictArrayFillKeys = gd.StrictArrayFillKeys;
326 if (gd.HardGenericsUB) RO::EvalEnforceGenericsUB = 2;
328 auto const units = Repo::get().enumerateUnits(RepoIdCentral, true);
329 auto const size = units.size();
330 fun(size, nullptr);
331 parallel::for_each(
332 units,
333 [&] (const std::pair<std::string,SHA1>& kv) {
334 fun(
335 size,
336 Repo::get().urp().loadEmitter(
337 kv.first, kv.second, Native::s_noNativeFuncs
342 return Repo().get().global().APCProfile;
345 void write_units(UnitEmitterQueue& ueq) {
346 folly::Optional<trace_time> timer;
348 RuntimeOption::RepoCommit = true;
349 RuntimeOption::RepoEvalMode = "local";
350 RuntimeOption::RepoDebugInfo = false; // Don't record UnitSourceLoc
351 open_repo(output_repo);
352 SCOPE_EXIT { Repo::shutdown(); };
354 std::vector<std::unique_ptr<UnitEmitter>> ues;
355 while (auto ue = ueq.pop()) {
356 if (!timer) timer.emplace("writing output repo");
357 ues.push_back(std::move(ue));
358 if (ues.size() == 8) {
359 auto const DEBUG_ONLY err = batchCommitWithoutRetry(ues, true);
360 always_assert(!err);
361 ues.clear();
365 auto const DEBUG_ONLY err = batchCommitWithoutRetry(ues, true);
366 always_assert(!err);
367 ues.clear();
370 void write_global_data(
371 std::unique_ptr<ArrayTypeTable::Builder>& arrTable,
372 std::vector<SString> apcProfile) {
374 auto const now = std::chrono::high_resolution_clock::now();
375 auto const nanos =
376 std::chrono::duration_cast<std::chrono::nanoseconds>(
377 now.time_since_epoch()
380 auto gd = Repo::GlobalData{};
381 gd.Signature = nanos.count();
382 gd.HardGenericsUB = RuntimeOption::EvalEnforceGenericsUB >= 2;
383 gd.HardReturnTypeHints = RuntimeOption::EvalCheckReturnTypeHints >= 3;
384 gd.CheckPropTypeHints = RuntimeOption::EvalCheckPropTypeHints;
385 gd.HardPrivatePropInference = options.HardPrivatePropInference;
386 gd.PHP7_NoHexNumerics = RuntimeOption::PHP7_NoHexNumerics;
387 gd.PHP7_Substr = RuntimeOption::PHP7_Substr;
388 gd.PHP7_Builtins = RuntimeOption::PHP7_Builtins;
389 gd.PromoteEmptyObject = RuntimeOption::EvalPromoteEmptyObject;
390 gd.EnableRenameFunction = RuntimeOption::EvalJitEnableRenameFunction;
391 gd.HackArrCompatNotices = RuntimeOption::EvalHackArrCompatNotices;
392 gd.EnableIntrinsicsExtension = RuntimeOption::EnableIntrinsicsExtension;
393 gd.APCProfile = std::move(apcProfile);
394 gd.ForbidDynamicCallsToFunc = RuntimeOption::EvalForbidDynamicCallsToFunc;
395 gd.ForbidDynamicCallsToClsMeth =
396 RuntimeOption::EvalForbidDynamicCallsToClsMeth;
397 gd.ForbidDynamicCallsToInstMeth =
398 RuntimeOption::EvalForbidDynamicCallsToInstMeth;
399 gd.ForbidDynamicConstructs = RuntimeOption::EvalForbidDynamicConstructs;
400 gd.ForbidDynamicCallsWithAttr =
401 RuntimeOption::EvalForbidDynamicCallsWithAttr;
402 gd.LogKnownMethodsAsDynamicCalls =
403 RuntimeOption::EvalLogKnownMethodsAsDynamicCalls;
404 gd.AbortBuildOnVerifyError = RuntimeOption::EvalAbortBuildOnVerifyError;
405 gd.EnableArgsInBacktraces = RuntimeOption::EnableArgsInBacktraces;
406 gd.NoticeOnBuiltinDynamicCalls =
407 RuntimeOption::EvalNoticeOnBuiltinDynamicCalls;
408 gd.HackArrCompatIsArrayNotices =
409 RuntimeOption::EvalHackArrCompatIsArrayNotices;
410 gd.HackArrCompatTypeHintNotices =
411 RuntimeOption::EvalHackArrCompatTypeHintNotices;
412 gd.HackArrCompatDVCmpNotices =
413 RuntimeOption::EvalHackArrCompatDVCmpNotices;
414 gd.HackArrCompatHackArrCmpNotices =
415 RuntimeOption::EvalHackArrCompatHackArrCmpNotices;
416 gd.HackArrCompatSerializeNotices =
417 RuntimeOption::EvalHackArrCompatSerializeNotices;
418 gd.HackArrDVArrs = RuntimeOption::EvalHackArrDVArrs;
419 gd.HackArrEmptyBasedBoolEqCmp = RuntimeOption::EvalHackArrEmptyBasedBoolEqCmp;
420 gd.InitialNamedEntityTableSize =
421 RuntimeOption::EvalInitialNamedEntityTableSize;
422 gd.InitialStaticStringTableSize =
423 RuntimeOption::EvalInitialStaticStringTableSize;
424 gd.EmitClsMethPointers = RuntimeOption::EvalEmitClsMethPointers;
425 gd.IsVecNotices = RuntimeOption::EvalIsVecNotices;
426 gd.IsCompatibleClsMethType = RuntimeOption::EvalIsCompatibleClsMethType;
427 gd.ArrayProvenance = RuntimeOption::EvalArrayProvenance;
428 gd.StrictArrayFillKeys = RuntimeOption::StrictArrayFillKeys;
430 for (auto const& elm : RuntimeOption::ConstantFunctions) {
431 gd.ConstantFunctions.push_back(elm);
434 globalArrayTypeTable().repopulate(*arrTable);
435 // NOTE: There's no way to tell if saveGlobalData() fails for some reason.
436 Repo::get().saveGlobalData(std::move(gd));
439 void compile_repo() {
440 auto program = make_program();
442 auto apcProfile = load_input(
443 [&] (size_t size, std::unique_ptr<UnitEmitter> ue) {
444 if (!ue) {
445 if (logging) {
446 std::cout << folly::format("{} units\n", size);
448 return;
450 add_unit_to_program(ue.get(), *program);
454 UnitEmitterQueue ueq;
455 std::unique_ptr<ArrayTypeTable::Builder> arrTable;
456 VMWorker wp_thread(
457 [&] {
458 HphpSession _{Treadmill::SessionKind::CompileRepo};
459 Trace::BumpRelease bumper(Trace::hhbbc_time, -1, logging);
460 whole_program(std::move(program), ueq, arrTable);
463 wp_thread.start();
465 auto guard = RepoAutoloadMapBuilder::collect();
466 write_units(ueq);
467 write_global_data(arrTable, apcProfile);
469 wp_thread.waitForEnd();
472 void print_repo_bytecode_stats() {
473 std::array<std::atomic<uint64_t>,Op_count> op_counts{};
475 auto const input = load_input(
476 [&] (size_t, std::unique_ptr<UnitEmitter> ue) {
477 if (!ue) return;
478 auto pc = ue->bc();
479 auto const end = pc + ue->bcPos();
480 for (; pc < end; pc += instrLen(pc)) {
481 auto &opc = op_counts[static_cast<uint16_t>(peek_op(pc))];
482 opc.fetch_add(1, std::memory_order_relaxed);
487 for (auto i = uint32_t{}; i < op_counts.size(); ++i) {
488 std::cout << folly::format(
489 "{: <20} {}\n",
490 opcodeToName(static_cast<Op>(i)),
491 op_counts[i].load(std::memory_order_relaxed)
496 //////////////////////////////////////////////////////////////////////
500 int main(int argc, char** argv) try {
501 parse_options(argc, argv);
503 if (!print_bytecode_stats_and_exit && fs::exists(output_repo)) {
504 std::cout << "output repo already exists; removing it\n";
505 if (unlink(output_repo.c_str())) {
506 std::cerr << "failed to unlink output repo: "
507 << strerror(errno) << '\n';
508 return 1;
511 if (!fs::exists(input_repo)) {
512 std::cerr << "input repo `" << input_repo << "' not found\n";
513 return 1;
516 initialize_repo();
518 // We need to set this flag so Repo::global will let us access it.
519 RuntimeOption::RepoAuthoritative = true;
521 LitstrTable::init();
522 RuntimeOption::RepoLocalMode = "--";
523 RuntimeOption::RepoEvalMode = "readonly";
524 open_repo(input_repo);
525 Repo::get().loadGlobalData(false);
526 LitstrTable::fini();
527 auto const& gd = Repo::get().global();
528 if (gd.InitialNamedEntityTableSize) {
529 RuntimeOption::EvalInitialNamedEntityTableSize =
530 gd.InitialNamedEntityTableSize;
532 if (gd.InitialStaticStringTableSize) {
533 RuntimeOption::EvalInitialStaticStringTableSize =
534 gd.InitialStaticStringTableSize;
537 rds::local::init();
538 SCOPE_EXIT { rds::local::fini(); };
540 Hdf config;
541 IniSetting::Map ini = IniSetting::Map::object;
542 RuntimeOption::Load(ini, config);
543 RuntimeOption::RepoLocalPath = "/tmp/hhbbc.repo";
544 RuntimeOption::RepoCentralPath = input_repo;
545 RuntimeOption::RepoLocalMode = "--";
546 RuntimeOption::RepoJournal = "memory";
547 RuntimeOption::RepoCommit = false;
548 RuntimeOption::EvalJit = false;
550 if (!hack_compiler_extract_path.empty()) {
551 RuntimeOption::EvalHackCompilerExtractPath = hack_compiler_extract_path;
554 register_process_init();
556 hphp_process_init();
557 SCOPE_EXIT { hphp_process_exit(); };
559 Repo::shutdown();
561 if (print_bytecode_stats_and_exit) {
562 print_repo_bytecode_stats();
563 return 0;
566 Trace::BumpRelease bumper(Trace::hhbbc_time, -1, logging);
567 compile_repo(); // NOTE: errors ignored
568 return 0;
571 catch (std::exception& e) {
572 std::cerr << e.what() << '\n';
573 return 1;
576 //////////////////////////////////////////////////////////////////////