2 +----------------------------------------------------------------------+
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
16 #include "hphp/hhbbc/hhbbc.h"
29 #include <boost/program_options.hpp>
30 #include <boost/filesystem.hpp>
32 #include <folly/ScopeGuard.h>
33 #include <folly/String.h>
34 #include <folly/portability/Unistd.h>
36 #include "hphp/runtime/base/ini-setting.h"
37 #include "hphp/runtime/base/runtime-option.h"
38 #include "hphp/runtime/base/vm-worker.h"
39 #include "hphp/hhvm/process-init.h"
40 #include "hphp/runtime/vm/native.h"
41 #include "hphp/runtime/vm/repo.h"
42 #include "hphp/runtime/vm/repo-autoload-map-builder.h"
43 #include "hphp/runtime/vm/repo-global-data.h"
44 #include "hphp/runtime/vm/treadmill.h"
46 #include "hphp/hhbbc/misc.h"
47 #include "hphp/hhbbc/options.h"
48 #include "hphp/hhbbc/stats.h"
49 #include "hphp/hhbbc/parallel.h"
50 #include "hphp/hhbbc/representation.h"
52 #include "hphp/util/rds-local.h"
53 #include "hphp/util/logger.h"
55 namespace HPHP
{ namespace HHBBC
{
59 namespace fs
= boost::filesystem
;
61 //////////////////////////////////////////////////////////////////////
63 std::string output_repo
;
64 std::string input_repo
;
65 std::string hack_compiler_extract_path
;
67 bool print_bytecode_stats_and_exit
= false;
70 //////////////////////////////////////////////////////////////////////
72 template<class SinglePassReadableRange
>
73 MethodMap
make_method_map(SinglePassReadableRange
& range
) {
74 auto ret
= MethodMap
{};
75 for (auto& str
: range
) {
76 std::vector
<std::string
> parts
;
77 folly::split("::", str
, parts
);
78 if (parts
.size() != 2) {
82 ret
[parts
[0]].insert(parts
[1]);
87 template<class SinglePassReadableRange
>
88 OpcodeSet
make_bytecode_map(SinglePassReadableRange
& bcs
) {
89 if (bcs
.empty()) return {};
90 hphp_fast_map
<std::string
,Op
> bcmap
;
91 for (auto i
= 0; i
< Op_count
; i
++) {
92 auto const op
= static_cast<Op
>(i
);
93 bcmap
[opcodeToName(op
)] = op
;
97 if (bcmap
.count(n
)) oset
.insert(bcmap
[n
]);
102 void parse_options(int argc
, char** argv
) {
103 namespace po
= boost::program_options
;
105 auto const defaultThreadCount
=
106 std::max
<long>(sysconf(_SC_NPROCESSORS_ONLN
) - 1, 1);
108 std::vector
<std::string
> interceptable_fns
;
109 std::vector
<std::string
> trace_fns
;
110 std::vector
<std::string
> trace_bcs
;
111 bool no_logging
= false;
112 bool no_cores
= false;
114 po::options_description
basic("Options");
116 ("help", "display help message")
118 po::value(&output_repo
)->default_value("hhvm.hhbbc"),
119 "output hhbc repo path")
121 po::value(&input_repo
)->default_value("hhvm.hhbc"),
122 "input hhbc repo path")
124 po::value(&options
.stats_file
)->default_value(""),
127 po::bool_switch(&options
.NoOptimizations
),
128 "turn off all optimizations")
130 po::bool_switch(&no_logging
),
133 po::bool_switch(&no_cores
),
134 "turn off core dumps (useful when running lots of tests in parallel)")
136 po::bool_switch(&options
.extendedStats
),
137 "Spend time to produce extra stats")
139 po::value(&options
.profileMemory
)->default_value(""),
140 "If non-empty, dump jemalloc memory profiles at key points")
141 ("parallel-num-threads",
142 po::value(¶llel::num_threads
)->default_value(defaultThreadCount
),
143 "Number of threads to use for parallelism")
144 ("parallel-final-threads",
145 po::value(¶llel::final_threads
)->default_value(
146 parallel::final_threads
),
147 "Number of threads to use for the final pass")
148 ("parallel-work-size",
149 po::value(¶llel::work_chunk
)->default_value(120),
150 "Work unit size for parallelism")
152 po::value(&trace_fns
)->composing(),
153 "Add a function to increase tracing level on (for debugging)")
155 po::value(&trace_bcs
)->composing(),
156 "Add a bytecode to trace (for debugging)")
157 ("hack-compiler-extract-path",
158 po::value(&hack_compiler_extract_path
)->default_value(""),
159 "hack compiler extract path")
162 // Some extra esoteric options that aren't exposed in --help for
164 po::options_description
extended("Extended Options");
165 extended
.add_options()
166 ("analyze-func-wlimit", po::value(&options
.analyzeFuncWideningLimit
))
167 ("analyze-class-wlimit", po::value(&options
.analyzeClassWideningLimit
))
168 ("return-refine-limit", po::value(&options
.returnTypeRefineLimit
))
169 ("public-sprop-refine-limit", po::value(&options
.publicSPropRefineLimit
))
170 ("bytecode-stats", po::bool_switch(&print_bytecode_stats_and_exit
))
173 po::options_description
oflags("Optimization Flags");
175 ("context-sensitive-interp", po::value(&options
.ContextSensitiveInterp
))
176 ("remove-dead-blocks", po::value(&options
.RemoveDeadBlocks
))
177 ("constant-prop", po::value(&options
.ConstantProp
))
178 ("constant-fold-builtins", po::value(&options
.ConstantFoldBuiltins
))
179 ("local-dce", po::value(&options
.LocalDCE
))
180 ("global-dce", po::value(&options
.GlobalDCE
))
181 ("remove-unused-local-names", po::value(&options
.RemoveUnusedLocalNames
))
182 ("compact-local-slots", po::value(&options
.CompactLocalSlots
))
183 ("insert-assertions", po::value(&options
.InsertAssertions
))
184 ("insert-stack-assertions", po::value(&options
.InsertStackAssertions
))
185 ("filter-assertions", po::value(&options
.FilterAssertions
))
186 ("strength-reduce", po::value(&options
.StrengthReduce
))
187 ("func-families", po::value(&options
.FuncFamilies
))
188 ("hard-private-prop", po::value(&options
.HardPrivatePropInference
))
189 ("analyze-pseudomains", po::value(&options
.AnalyzePseudomains
))
190 ("analyze-public-statics", po::value(&options
.AnalyzePublicStatics
))
193 po::options_description all
;
194 all
.add(basic
).add(extended
).add(oflags
);
196 po::positional_options_description pd
;
199 po::variables_map vm
;
201 po::command_line_parser(argc
, argv
)
205 [&] (const std::string
& s
) -> std::pair
<std::string
,std::string
> {
206 if (s
.find("-f") == 0) {
207 if (s
.substr(2, 3) == "no-") {
208 return { s
.substr(5), "false" };
210 return { s
.substr(2), "true" };
219 if (vm
.count("help")) {
220 std::cout
<< basic
<< "\n"
221 "Individual optimizations may be turned on and off using gcc-style -fflag\n"
222 "and -fno-flag arguments. The various optimization flags are documented\n"
229 setrlimit(RLIMIT_CORE
, &rl
);
232 if (!options
.ConstantProp
) options
.ConstantFoldBuiltins
= false;
234 options
.TraceFunctions
= make_method_map(trace_fns
);
235 options
.TraceBytecodes
= make_bytecode_map(trace_bcs
);
237 if (!options
.profileMemory
.empty()) {
238 mallctlWrite("prof.active", true);
239 mallctlWrite("prof.thread_active_init", true);
242 logging
= !no_logging
;
245 UNUSED
void validate_options() {
246 if (parallel::work_chunk
<= 10 || parallel::num_threads
< 1) {
247 std::cerr
<< "Invalid parallelism configuration.\n";
251 if (options
.AnalyzePublicStatics
&& !options
.AnalyzePseudomains
) {
252 std::cerr
<< "-fanalyze-public-statics requires -fanalyze-pseudomains\n";
256 if (options
.RemoveUnusedLocalNames
&& !options
.GlobalDCE
) {
257 std::cerr
<< "-fremove-unused-local-names requires -fglobal-dce\n";
261 if (options
.CompactLocalSlots
&& !options
.GlobalDCE
) {
262 std::cerr
<< "-fcompact-local-slots requires -fglobal-dce\n";
267 //////////////////////////////////////////////////////////////////////
269 void open_repo(const std::string
& path
) {
270 RuntimeOption::RepoCentralPath
= path
;
271 // Make sure the changes take effect
277 std::vector
<SString
> load_input(F
&& fun
) {
278 trace_time
timer("load units");
280 open_repo(input_repo
);
281 Repo::get().loadGlobalData();
282 SCOPE_EXIT
{ Repo::shutdown(); };
284 auto const& gd
= Repo::get().global();
285 // When running hhbbc, these option is loaded from GD, and will override CLI.
286 // When running hhvm, these option is not loaded from GD, but read from CLI.
287 RO::EvalJitEnableRenameFunction
= gd
.EnableRenameFunction
;
288 RO::EvalHackArrCompatNotices
=
289 RO::EvalHackArrCompatCheckCompare
=
290 RO::EvalHackArrCompatCheckArrayPlus
=
291 RO::EvalHackArrCompatCheckArrayKeyCast
=
292 gd
.HackArrCompatNotices
;
293 RO::EvalForbidDynamicCallsToFunc
= gd
.ForbidDynamicCallsToFunc
;
294 RO::EvalForbidDynamicCallsToClsMeth
=
295 gd
.ForbidDynamicCallsToClsMeth
;
296 RO::EvalForbidDynamicCallsToInstMeth
=
297 gd
.ForbidDynamicCallsToInstMeth
;
298 RO::EvalForbidDynamicConstructs
= gd
.ForbidDynamicConstructs
;
299 RO::EvalForbidDynamicCallsWithAttr
=
300 gd
.ForbidDynamicCallsWithAttr
;
301 RO::EvalLogKnownMethodsAsDynamicCalls
=
302 gd
.LogKnownMethodsAsDynamicCalls
;
303 RO::EvalNoticeOnBuiltinDynamicCalls
=
304 gd
.NoticeOnBuiltinDynamicCalls
;
305 RO::EvalHackArrCompatIsArrayNotices
=
306 gd
.HackArrCompatIsArrayNotices
;
307 RO::EvalHackArrCompatTypeHintNotices
=
308 gd
.HackArrCompatTypeHintNotices
;
309 RO::EvalHackArrCompatDVCmpNotices
=
310 gd
.HackArrCompatDVCmpNotices
;
311 RO::EvalHackArrCompatSerializeNotices
=
312 gd
.HackArrCompatSerializeNotices
;
313 RO::EvalHackArrCompatSpecialization
= gd
.HackArrCompatSpecialization
;
314 RO::EvalHackArrDVArrs
= gd
.HackArrDVArrs
;
315 RO::EvalAbortBuildOnVerifyError
= gd
.AbortBuildOnVerifyError
;
316 RO::EnableArgsInBacktraces
= gd
.EnableArgsInBacktraces
;
317 RO::EvalEmitClsMethPointers
= gd
.EmitClsMethPointers
;
318 RO::EvalIsVecNotices
= gd
.IsVecNotices
;
319 RO::EvalIsCompatibleClsMethType
= gd
.IsCompatibleClsMethType
;
320 RO::EvalArrayProvenance
=
321 RO::EvalArrProvHackArrays
=
322 RO::EvalArrProvDVArrays
=
324 RO::StrictArrayFillKeys
= gd
.StrictArrayFillKeys
;
325 RO::EvalEnableFuncStringInterop
= gd
.EnableFuncStringInterop
;
326 if (gd
.HardGenericsUB
) {
327 RO::EvalEnforceGenericsUB
= 2;
329 RO::EvalEnforceGenericsUB
= 1;
332 auto const units
= Repo::get().enumerateUnits(RepoIdCentral
, true);
333 auto const size
= units
.size();
337 [&] (const std::pair
<std::string
,SHA1
>& kv
) {
340 Repo::get().urp().loadEmitter(
341 kv
.first
, kv
.second
, Native::s_noNativeFuncs
346 return Repo().get().global().APCProfile
;
349 void write_units(UnitEmitterQueue
& ueq
,
350 RepoAutoloadMapBuilder
& autoloadMapBuilder
) {
351 folly::Optional
<trace_time
> timer
;
353 RuntimeOption::RepoCommit
= true;
354 RuntimeOption::RepoEvalMode
= "local";
355 RuntimeOption::RepoDebugInfo
= false; // Don't record UnitSourceLoc
356 open_repo(output_repo
);
357 SCOPE_EXIT
{ Repo::shutdown(); };
359 std::vector
<std::unique_ptr
<UnitEmitter
>> ues
;
360 while (auto ue
= ueq
.pop()) {
361 if (!timer
) timer
.emplace("writing output repo");
362 autoloadMapBuilder
.addUnit(*ue
);
363 ues
.push_back(std::move(ue
));
364 if (ues
.size() == 8) {
365 auto const DEBUG_ONLY err
= batchCommitWithoutRetry(ues
, true);
371 auto const DEBUG_ONLY err
= batchCommitWithoutRetry(ues
, true);
376 void write_global_data(
377 std::unique_ptr
<ArrayTypeTable::Builder
>& arrTable
,
378 std::vector
<SString
> apcProfile
,
379 const RepoAutoloadMapBuilder
& autoloadMapBuilder
) {
381 auto const now
= std::chrono::high_resolution_clock::now();
383 std::chrono::duration_cast
<std::chrono::nanoseconds
>(
384 now
.time_since_epoch()
387 auto gd
= Repo::GlobalData
{};
388 gd
.Signature
= nanos
.count();
389 gd
.HardGenericsUB
= RuntimeOption::EvalEnforceGenericsUB
>= 2;
390 gd
.HardReturnTypeHints
= RuntimeOption::EvalCheckReturnTypeHints
>= 3;
391 gd
.CheckPropTypeHints
= RuntimeOption::EvalCheckPropTypeHints
;
392 gd
.HardPrivatePropInference
= options
.HardPrivatePropInference
;
393 gd
.PHP7_NoHexNumerics
= RuntimeOption::PHP7_NoHexNumerics
;
394 gd
.PHP7_Substr
= RuntimeOption::PHP7_Substr
;
395 gd
.PHP7_Builtins
= RuntimeOption::PHP7_Builtins
;
396 gd
.EnableRenameFunction
= RuntimeOption::EvalJitEnableRenameFunction
;
397 gd
.HackArrCompatNotices
= RuntimeOption::EvalHackArrCompatNotices
;
398 gd
.EnableIntrinsicsExtension
= RuntimeOption::EnableIntrinsicsExtension
;
399 gd
.APCProfile
= std::move(apcProfile
);
400 gd
.ForbidDynamicCallsToFunc
= RuntimeOption::EvalForbidDynamicCallsToFunc
;
401 gd
.ForbidDynamicCallsToClsMeth
=
402 RuntimeOption::EvalForbidDynamicCallsToClsMeth
;
403 gd
.ForbidDynamicCallsToInstMeth
=
404 RuntimeOption::EvalForbidDynamicCallsToInstMeth
;
405 gd
.ForbidDynamicConstructs
= RuntimeOption::EvalForbidDynamicConstructs
;
406 gd
.ForbidDynamicCallsWithAttr
=
407 RuntimeOption::EvalForbidDynamicCallsWithAttr
;
408 gd
.LogKnownMethodsAsDynamicCalls
=
409 RuntimeOption::EvalLogKnownMethodsAsDynamicCalls
;
410 gd
.AbortBuildOnVerifyError
= RuntimeOption::EvalAbortBuildOnVerifyError
;
411 gd
.EnableArgsInBacktraces
= RuntimeOption::EnableArgsInBacktraces
;
412 gd
.NoticeOnBuiltinDynamicCalls
=
413 RuntimeOption::EvalNoticeOnBuiltinDynamicCalls
;
414 gd
.HackArrCompatIsArrayNotices
=
415 RuntimeOption::EvalHackArrCompatIsArrayNotices
;
416 gd
.HackArrCompatTypeHintNotices
=
417 RuntimeOption::EvalHackArrCompatTypeHintNotices
;
418 gd
.HackArrCompatDVCmpNotices
=
419 RuntimeOption::EvalHackArrCompatDVCmpNotices
;
420 gd
.HackArrCompatSerializeNotices
=
421 RuntimeOption::EvalHackArrCompatSerializeNotices
;
422 gd
.HackArrCompatSpecialization
=
423 RuntimeOption::EvalHackArrCompatSpecialization
;
424 gd
.HackArrDVArrs
= RuntimeOption::EvalHackArrDVArrs
;
425 gd
.InitialNamedEntityTableSize
=
426 RuntimeOption::EvalInitialNamedEntityTableSize
;
427 gd
.InitialStaticStringTableSize
=
428 RuntimeOption::EvalInitialStaticStringTableSize
;
429 gd
.EmitClsMethPointers
= RuntimeOption::EvalEmitClsMethPointers
;
430 gd
.IsVecNotices
= RuntimeOption::EvalIsVecNotices
;
431 gd
.IsCompatibleClsMethType
= RuntimeOption::EvalIsCompatibleClsMethType
;
432 gd
.ArrayProvenance
= RuntimeOption::EvalArrayProvenance
;
433 gd
.StrictArrayFillKeys
= RuntimeOption::StrictArrayFillKeys
;
434 gd
.EnableFuncStringInterop
= RO::EvalEnableFuncStringInterop
;
436 for (auto const& elm
: RuntimeOption::ConstantFunctions
) {
437 gd
.ConstantFunctions
.push_back(elm
);
440 if (arrTable
) globalArrayTypeTable().repopulate(*arrTable
);
441 // NOTE: There's no way to tell if saveGlobalData() fails for some reason.
442 Repo::get().saveGlobalData(std::move(gd
), autoloadMapBuilder
);
445 void compile_repo() {
446 auto program
= make_program();
448 auto apcProfile
= load_input(
449 [&] (size_t size
, std::unique_ptr
<UnitEmitter
> ue
) {
452 std::cout
<< folly::format("{} units\n", size
);
456 add_unit_to_program(ue
.get(), *program
);
460 UnitEmitterQueue ueq
;
461 std::unique_ptr
<ArrayTypeTable::Builder
> arrTable
;
462 std::exception_ptr wp_thread_ex
= nullptr;
465 HphpSession _
{Treadmill::SessionKind::CompileRepo
};
466 Trace::BumpRelease
bumper(Trace::hhbbc_time
, -1, logging
);
468 whole_program(std::move(program
), ueq
, arrTable
);
470 wp_thread_ex
= std::current_exception();
477 RepoAutoloadMapBuilder autoloadMapBuilder
;
478 write_units(ueq
, autoloadMapBuilder
);
479 write_global_data(arrTable
, apcProfile
, autoloadMapBuilder
);
482 wp_thread
.waitForEnd();
484 rethrow_exception(wp_thread_ex
);
488 void print_repo_bytecode_stats() {
489 std::array
<std::atomic
<uint64_t>,Op_count
> op_counts
{};
491 auto const input
= load_input(
492 [&] (size_t, std::unique_ptr
<UnitEmitter
> ue
) {
495 auto const end
= pc
+ ue
->bcPos();
496 for (; pc
< end
; pc
+= instrLen(pc
)) {
497 auto &opc
= op_counts
[static_cast<uint16_t>(peek_op(pc
))];
498 opc
.fetch_add(1, std::memory_order_relaxed
);
503 for (auto i
= uint32_t{}; i
< op_counts
.size(); ++i
) {
504 std::cout
<< folly::format(
506 opcodeToName(static_cast<Op
>(i
)),
507 op_counts
[i
].load(std::memory_order_relaxed
)
512 //////////////////////////////////////////////////////////////////////
516 int main(int argc
, char** argv
) try {
517 parse_options(argc
, argv
);
519 if (!print_bytecode_stats_and_exit
&& fs::exists(output_repo
)) {
520 std::cout
<< "output repo already exists; removing it\n";
521 if (unlink(output_repo
.c_str())) {
522 std::cerr
<< "failed to unlink output repo: "
523 << strerror(errno
) << '\n';
527 if (!fs::exists(input_repo
)) {
528 std::cerr
<< "input repo `" << input_repo
<< "' not found\n";
534 // We need to set this flag so Repo::global will let us access it.
535 RuntimeOption::RepoAuthoritative
= true;
538 RuntimeOption::RepoLocalMode
= "--";
539 RuntimeOption::RepoEvalMode
= "readonly";
540 open_repo(input_repo
);
541 Repo::get().loadGlobalData(false);
543 auto const& gd
= Repo::get().global();
544 if (gd
.InitialNamedEntityTableSize
) {
545 RuntimeOption::EvalInitialNamedEntityTableSize
=
546 gd
.InitialNamedEntityTableSize
;
548 if (gd
.InitialStaticStringTableSize
) {
549 RuntimeOption::EvalInitialStaticStringTableSize
=
550 gd
.InitialStaticStringTableSize
;
554 SCOPE_EXIT
{ rds::local::fini(); };
557 IniSetting::Map ini
= IniSetting::Map::object
;
558 RuntimeOption::Load(ini
, config
);
559 RuntimeOption::RepoLocalPath
= "/tmp/hhbbc.repo";
560 RuntimeOption::RepoCentralPath
= input_repo
;
561 RuntimeOption::RepoLocalMode
= "--";
562 RuntimeOption::RepoJournal
= "memory";
563 RuntimeOption::RepoCommit
= false;
564 RuntimeOption::EvalJit
= false;
566 if (!hack_compiler_extract_path
.empty()) {
567 RuntimeOption::EvalHackCompilerExtractPath
= hack_compiler_extract_path
;
570 register_process_init();
573 LitstrTable::get().setWriting();
574 SCOPE_EXIT
{ hphp_process_exit(); };
578 if (print_bytecode_stats_and_exit
) {
579 print_repo_bytecode_stats();
583 Trace::BumpRelease
bumper(Trace::hhbbc_time
, -1, logging
);
584 compile_repo(); // NOTE: errors ignored
588 catch (std::exception
& e
) {
589 Logger::Error("std::exception: %s", e
.what());
593 //////////////////////////////////////////////////////////////////////