2 +----------------------------------------------------------------------+
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
17 #include "hphp/compiler/compiler.h"
19 #include "hphp/compiler/analysis/analysis_result.h"
20 #include "hphp/compiler/analysis/emitter.h"
21 #include "hphp/compiler/builtin_symbols.h"
22 #include "hphp/compiler/option.h"
23 #include "hphp/compiler/package.h"
25 #include "hphp/hhbbc/hhbbc.h"
26 #include "hphp/runtime/base/config.h"
27 #include "hphp/runtime/base/file-util.h"
28 #include "hphp/runtime/base/ini-setting.h"
29 #include "hphp/runtime/base/array-provenance.h"
30 #include "hphp/runtime/base/program-functions.h"
31 #include "hphp/runtime/version.h"
32 #include "hphp/runtime/vm/unit-parser.h"
33 #include "hphp/system/systemlib.h"
35 #include "hphp/util/async-func.h"
36 #include "hphp/util/build-info.h"
37 #include "hphp/util/current-executable.h"
38 #include "hphp/util/exception.h"
39 #include "hphp/util/hdf.h"
40 #include "hphp/util/logger.h"
41 #include "hphp/util/process.h"
42 #include "hphp/util/process-exec.h"
43 #include "hphp/util/rds-local.h"
44 #include "hphp/util/text-util.h"
45 #include "hphp/util/timer.h"
47 #include "hphp/util/light-process.h"
50 #include "hphp/hhvm/process-init.h"
52 #include <sys/types.h>
58 #include <boost/algorithm/string/replace.hpp>
59 #include <boost/program_options/options_description.hpp>
60 #include <boost/program_options/positional_options.hpp>
61 #include <boost/program_options/variables_map.hpp>
62 #include <boost/program_options/parsers.hpp>
63 #include <boost/filesystem.hpp>
67 #include <folly/portability/SysStat.h>
69 using namespace boost::program_options
;
73 ///////////////////////////////////////////////////////////////////////////////
75 struct CompilerOptions
{
77 std::string outputDir
;
78 std::vector
<std::string
> config
;
79 std::string configDir
;
80 std::vector
<std::string
> confStrings
;
81 std::vector
<std::string
> iniStrings
;
83 std::vector
<std::string
> inputs
;
84 std::string inputList
;
85 std::vector
<std::string
> includePaths
;
86 std::vector
<std::string
> modules
;
87 std::vector
<std::string
> excludeDirs
;
88 std::vector
<std::string
> excludeFiles
;
89 std::vector
<std::string
> excludePatterns
;
90 std::vector
<std::string
> excludeStaticDirs
;
91 std::vector
<std::string
> excludeStaticFiles
;
92 std::vector
<std::string
> excludeStaticPatterns
;
93 std::vector
<std::string
> fmodules
;
94 std::vector
<std::string
> ffiles
;
95 std::vector
<std::string
> cfiles
;
96 std::vector
<std::string
> cmodules
;
99 std::string programArgs
;
104 std::string filecache
;
108 ///////////////////////////////////////////////////////////////////////////////
110 struct AsyncFileCacheSaver
: AsyncFunc
<AsyncFileCacheSaver
> {
111 AsyncFileCacheSaver(Package
*package
, const char *name
)
112 : AsyncFunc
<AsyncFileCacheSaver
>(this, &AsyncFileCacheSaver::saveCache
),
113 m_package(package
), m_name(name
) {
117 Timer
timer(Timer::WallTime
, "saving file cache...");
118 m_package
->getFileCache()->save(m_name
);
122 Logger::Info("%" PRId64
" MB %s saved",
123 (int64_t)sb
.st_size
/(1024*1024), m_name
);
131 ///////////////////////////////////////////////////////////////////////////////
132 // forward declarations
134 int prepareOptions(CompilerOptions
&po
, int argc
, char **argv
);
135 void createOutputDirectory(CompilerOptions
&po
);
136 int process(const CompilerOptions
&po
);
137 int phpTarget(const CompilerOptions
&po
, AnalysisResultPtr ar
);
138 void hhbcTargetInit(const CompilerOptions
&po
, AnalysisResultPtr ar
);
139 int hhbcTarget(const CompilerOptions
&po
, AnalysisResultPtr
&& ar
,
140 AsyncFileCacheSaver
&fcThread
);
143 ///////////////////////////////////////////////////////////////////////////////
145 int compiler_main(int argc
, char **argv
) {
149 SCOPE_EXIT
{ rds::local::fini(); };
151 int ret
= prepareOptions(po
, argc
, argv
);
152 if (ret
== 1) return 0; // --help
153 if (ret
== -1) return -1; // command line error
155 Timer
totalTimer(Timer::WallTime
, "running hphp");
156 createOutputDirectory(po
);
157 if (ret
== 0) ret
= process(po
);
159 Logger::Error("hphp failed");
161 Logger::Info("all files saved in %s ...", po
.outputDir
.c_str());
164 } catch (Exception
& e
) {
165 Logger::Error("Exception: %s", e
.getMessage().c_str());
166 } catch (std::exception
& e
) {
167 Logger::Error("std::exception: %s", e
.what());
169 Logger::Error("(non-standard exception \"%s\" was thrown)",
170 current_exception_name().c_str());
175 ///////////////////////////////////////////////////////////////////////////////
179 void applyBuildOverrides(IniSetting::Map
& ini
, Hdf
& config
) {
180 std::string push_phases
= Config::GetString(ini
, config
, "Build.PushPhases");
181 // convert push phases to newline-separated, to make matching them less
183 replaceAll(push_phases
, ",", "\n");
184 bool loggedOnce
= false;
186 for (Hdf hdf
= config
["Overrides"].firstChild();
190 Logger::Info(folly::sformat(
191 "Matching build overrides using: push_phases='{}'",
195 if (Config::matchHdfPattern(push_phases
, ini
, hdf
, "push_phase" , "m")) {
196 Logger::Info(folly::sformat("Matched override: {}", hdf
.getName()));
198 if (hdf
.exists("clear")) {
199 std::vector
<std::string
> list
;
200 hdf
["clear"].configGet(list
);
201 for (auto const& s
: list
) {
205 config
.copy(hdf
["overwrite"]);
206 // no break here, so we can continue to match more overrides
208 hdf
["overwrite"].setVisited(); // avoid lint complaining
209 if (hdf
.exists("clear")) {
210 // when the tier does not match, "clear" is not accessed
211 // mark it visited, so the linter does not complain
212 hdf
["clear"].setVisited();
219 int prepareOptions(CompilerOptions
&po
, int argc
, char **argv
) {
220 options_description
desc("HipHop Compiler for PHP Usage:\n\n"
221 "\thphp <options> <inputs>\n\n"
230 ("help", "display this message")
231 ("version", "display version number")
232 ("target,t", value
<std::string
>(&dummy4
)->default_value("hhbc"),
233 "hhbc") // TODO: T115189426 remove this
234 ("format,f", value
<std::string
>(&po
.format
),
235 "hhbc: binary (default) | hhas | text")
236 ("input-dir", value
<std::string
>(&po
.inputDir
), "input directory")
237 ("program", value
<std::string
>(&po
.program
)->default_value("program"),
238 "final program name to use")
239 ("args", value
<std::string
>(&po
.programArgs
), "program arguments")
240 ("inputs,i", value
<std::vector
<std::string
>>(&po
.inputs
),
242 ("input-list", value
<std::string
>(&po
.inputList
),
243 "file containing list of file names, one per line")
245 value
<std::vector
<std::string
>>(&po
.includePaths
)->composing(),
246 "a list of full paths to search for files being included in includes "
247 "or requires but cannot be found assuming relative paths")
248 ("module", value
<std::vector
<std::string
>>(&po
.modules
)->composing(),
249 "directories containing all input files")
251 value
<std::vector
<std::string
>>(&po
.excludeDirs
)->composing(),
252 "directories to exclude from the input")
253 ("fmodule", value
<std::vector
<std::string
>>(&po
.fmodules
)->composing(),
254 "same with module, except no exclusion checking is performed, so these "
255 "modules are forced to be included")
256 ("ffile", value
<std::vector
<std::string
>>(&po
.ffiles
)->composing(),
257 "extra PHP files forced to include without exclusion checking")
259 value
<std::vector
<std::string
>>(&po
.excludeFiles
)->composing(),
260 "files to exclude from the input, even if parse-on-demand finds it")
262 value
<std::vector
<std::string
>>(&po
.excludePatterns
)->composing(),
263 "regex (in 'find' command's regex command line option format) of files "
264 "or directories to exclude from the input, even if parse-on-demand finds "
266 ("exclude-static-pattern",
267 value
<std::vector
<std::string
>>(&po
.excludeStaticPatterns
)->composing(),
268 "regex (in 'find' command's regex command line option format) of files "
269 "or directories to exclude from static content cache")
270 ("exclude-static-dir",
271 value
<std::vector
<std::string
>>(&po
.excludeStaticDirs
)->composing(),
272 "directories to exclude from static content cache")
273 ("exclude-static-file",
274 value
<std::vector
<std::string
>>(&po
.excludeStaticFiles
)->composing(),
275 "files to exclude from static content cache")
276 ("cfile", value
<std::vector
<std::string
>>(&po
.cfiles
)->composing(),
277 "extra static files forced to include without exclusion checking")
278 ("cmodule", value
<std::vector
<std::string
>>(&po
.cmodules
)->composing(),
279 "extra directories for static files without exclusion checking")
280 ("parse-on-demand", value
<bool>(&po
.parseOnDemand
)->default_value(true),
281 "whether to parse files that are not specified from command line")
282 ("branch", value
<std::string
>(&po
.branch
), "SVN branch")
283 ("revision", value
<int>(&po
.revision
), "SVN revision")
284 ("output-dir,o", value
<std::string
>(&po
.outputDir
), "output directory")
285 ("sync-dir", value
<std::string
>(&dummy3
), // TODO: T115189426 remove this
286 "Files will be created in this directory first, then sync with output "
287 "directory without overwriting identical files. Great for incremental "
288 "compilation and build.")
289 ("gen-stats", value
<bool>(&dummy2
)->default_value(false), // TODO: T115189426 remove this
290 "whether to generate code errors")
291 ("keep-tempdir,k", value
<bool>(&po
.keepTempDir
)->default_value(false),
292 "whether to keep the temporary directory")
293 ("config,c", value
<std::vector
<std::string
>>(&po
.config
)->composing(),
295 ("config-dir", value
<std::string
>(&po
.configDir
),
296 "root directory configuration is based on (for example, "
297 "excluded directories may be relative path in configuration.")
299 value
<std::vector
<std::string
>>(&po
.confStrings
)->composing(),
300 "individual configuration string in a format of name=value, where "
301 "name can be any valid configuration for a config file")
302 ("define,d", value
<std::vector
<std::string
>>(&po
.iniStrings
)->composing(),
303 "define an ini setting in the same format ( foo[=bar] ) as provided in a "
306 value
<int>(&po
.logLevel
)->default_value(-1),
307 "-1: (default); 0: no logging; 1: errors only; 2: warnings and errors; "
308 "3: informational as well; 4: really verbose.")
310 value
<bool>(&dummy
)->default_value(true), // TODO: T115189426 remove this
311 "force to ignore code generation errors and continue compilations")
313 value
<std::string
>(&po
.filecache
),
314 "if specified, generate a static file cache with this file name")
316 value
<bool>(&po
.coredump
)->default_value(false),
318 ("compiler-id", "display the git hash for the compiler id")
319 ("repo-schema", "display the repo schema id used by this app")
322 positional_options_description p
;
326 auto opts
= command_line_parser(argc
, argv
).options(desc
)
327 .positional(p
).run();
331 #if defined(BOOST_VERSION) && BOOST_VERSION >= 105000 && BOOST_VERSION <= 105400
332 } catch (const error_with_option_name
&e
) {
333 std::string wrong_name
= e
.get_option_name();
334 std::string right_name
= get_right_option_name(opts
, wrong_name
);
335 std::string message
= e
.what();
336 if (right_name
!= "") {
337 boost::replace_all(message
, wrong_name
, right_name
);
339 Logger::Error("Error in command line: %s", message
.c_str());
340 cout
<< desc
<< "\n";
343 } catch (const error
& e
) {
344 Logger::Error("Error in command line: %s", e
.what());
345 cout
<< desc
<< "\n";
348 } catch (const unknown_option
& e
) {
349 Logger::Error("Error in command line: %s", e
.what());
350 cout
<< desc
<< "\n";
352 } catch (const error
& e
) {
353 Logger::Error("Error in command line: %s", e
.what());
354 cout
<< desc
<< "\n";
357 Logger::Error("Error in command line parsing.");
358 cout
<< desc
<< "\n";
361 if (argc
<= 1 || vm
.count("help")) {
362 cout
<< desc
<< "\n";
365 if (vm
.count("version")) {
366 cout
<< "HipHop Repo Compiler";
367 cout
<< " " << HHVM_VERSION
;
368 cout
<< " (" << (debug
? "dbg" : "rel") << ")\n";
369 cout
<< "Compiler: " << compilerId() << "\n";
370 cout
<< "Repo schema: " << repoSchemaId() << "\n";
374 if (vm
.count("compiler-id")) {
375 cout
<< compilerId() << "\n";
379 if (vm
.count("repo-schema")) {
380 cout
<< repoSchemaId() << "\n";
384 if (po
.program
== "program") {
385 po
.program
= "hhvm.hhbc";
389 if (po
.logLevel
!= -1) {
390 Logger::LogLevel
= (Logger::LogLevelType
)po
.logLevel
;
392 Logger::LogLevel
= Logger::LogInfo
;
396 IniSetting::Map ini
= IniSetting::Map::object
;
398 for (auto const& file
: po
.config
) {
399 Config::ParseConfigFile(file
, ini
, config
);
401 for (auto const& iniString
: po
.iniStrings
) {
402 Config::ParseIniString(iniString
, ini
);
404 for (auto const& confString
: po
.confStrings
) {
405 Config::ParseHdfString(confString
, config
);
407 applyBuildOverrides(ini
, config
);
408 Hdf runtime
= config
["Runtime"];
409 // The configuration command line strings were already processed above
410 // Don't process them again.
412 // Note that some options depends on RepoAuthoritative, we thus set/unset them
413 // here. If we reach this code, we are invoking hhvm --hphp, which is
414 // supposed to be in repo mode only. But we are restoring it to false since
415 // we need compile_systemlib_string to actually parse the file instead of
416 // trying to load it from repo (which is the case when RepoAuthoritative is
418 RuntimeOption::RepoAuthoritative
= true;
419 // Set RepoPath to satisfy assertions (we need a path set in
420 // RepoAuthoritative). It will never actually be used.
421 RuntimeOption::RepoPath
= "/tmp/dummy.hhbc";
422 // We don't want debug info in repo builds, since we don't support attaching
423 // a debugger in repo authoritative mode, but we want the default for debug
424 // info to be true so that it's present in sandboxes. Override that default
425 // here, since we only get here when building for repo authoritative mode.
426 RuntimeOption::RepoDebugInfo
= false;
427 RuntimeOption::Load(ini
, runtime
);
428 Option::Load(ini
, config
);
429 RuntimeOption::RepoAuthoritative
= false;
430 RuntimeOption::RepoPath
= "";
431 RuntimeOption::EvalJit
= false;
433 std::vector
<std::string
> badnodes
;
434 config
.lint(badnodes
);
435 for (auto const& badnode
: badnodes
) {
436 Logger::Error("Possible bad config node: %s", badnode
.c_str());
439 // we need to initialize pcre cache table very early
442 if (po
.inputDir
.empty()) {
445 po
.inputDir
= FileUtil::normalizeDir(po
.inputDir
);
446 if (po
.configDir
.empty()) {
447 po
.configDir
= po
.inputDir
;
449 po
.configDir
= FileUtil::normalizeDir(po
.configDir
);
450 Option::RootDirectory
= po
.configDir
;
451 Option::IncludeSearchPaths
= po
.includePaths
;
453 for (auto const& dir
: po
.excludeDirs
) {
454 Option::PackageExcludeDirs
.insert(FileUtil::normalizeDir(dir
));
456 for (auto const& file
: po
.excludeFiles
) {
457 Option::PackageExcludeFiles
.insert(file
);
459 for (auto const& pattern
: po
.excludePatterns
) {
460 Option::PackageExcludePatterns
.insert(
461 format_pattern(pattern
, true /* prefixSlash */));
463 for (auto const& dir
: po
.excludeStaticDirs
) {
464 Option::PackageExcludeStaticDirs
.insert(FileUtil::normalizeDir(dir
));
466 for (auto const& file
: po
.excludeStaticFiles
) {
467 Option::PackageExcludeStaticFiles
.insert(file
);
469 for (auto const& pattern
: po
.excludeStaticPatterns
) {
470 Option::PackageExcludeStaticPatterns
.insert(
471 format_pattern(pattern
, true /* prefixSlash */));
474 Option::ProgramName
= po
.program
;
476 if (po
.format
.empty()) po
.format
= "binary";
480 ///////////////////////////////////////////////////////////////////////////////
482 int process(const CompilerOptions
&po
) {
484 LightProcess::Initialize(RuntimeOption::LightProcessFilePrefix
,
485 RuntimeOption::LightProcessCount
,
486 RuntimeOption::EvalRecordSubprocessTimes
,
493 * Windows actually does core dump size and control at a system, not an app
494 * level. So we do nothing here and are at the mercy of Dr. Watson.
496 #elif defined(__APPLE__) || defined(__FreeBSD__)
498 getrlimit(RLIMIT_CORE
, &rl
);
499 rl
.rlim_cur
= 80000000LL;
500 if (rl
.rlim_max
< rl
.rlim_cur
) {
501 rl
.rlim_max
= rl
.rlim_cur
;
503 setrlimit(RLIMIT_CORE
, &rl
);
506 getrlimit64(RLIMIT_CORE
, &rl
);
507 rl
.rlim_cur
= 8000000000LL;
508 if (rl
.rlim_max
< rl
.rlim_cur
) {
509 rl
.rlim_max
= rl
.rlim_cur
;
511 setrlimit64(RLIMIT_CORE
, &rl
);
515 register_process_init();
517 Timer
timer(Timer::WallTime
);
519 Package package
{po
.inputDir
.c_str()};
520 AnalysisResultPtr ar
= package
.getAnalysisResult();
522 hhbcTargetInit(po
, ar
);
524 BuiltinSymbols::s_systemAr
= ar
;
526 SCOPE_EXIT
{ hphp_process_exit(); };
527 BuiltinSymbols::s_systemAr
.reset();
529 package
.createAsyncState();
531 // We need to do this manually, and not rely on ~Package, because
532 // it has to be done *before* we call hphp_process_exit().
533 if (auto clearer
= package
.clearAsyncState()) clearer
->join();
536 // This should be set before parsing anything
537 RuntimeOption::EvalLowStaticArrays
= false;
540 LitstrTable::get().setWriting();
541 LitarrayTable::init();
542 LitarrayTable::get().setWriting();
545 Timer
timer2(Timer::WallTime
, "parsing");
546 ar
->setPackage(&package
);
547 ar
->setParseOnDemand(po
.parseOnDemand
);
548 if (!po
.parseOnDemand
) {
549 ar
->setParseOnDemandDirs(Option::ParseOnDemandDirs
);
551 if (po
.modules
.empty() && po
.fmodules
.empty() &&
552 po
.ffiles
.empty() && po
.inputs
.empty() && po
.inputList
.empty()) {
553 package
.addAllFiles();
555 for (auto const& module
: po
.modules
) {
556 package
.addDirectory(module
);
558 for (auto const& fmodule
: po
.fmodules
) {
559 package
.addDirectory(fmodule
);
561 for (auto const& ffile
: po
.ffiles
) {
562 package
.addSourceFile(ffile
);
564 for (auto const& cmodule
: po
.cmodules
) {
565 package
.addStaticDirectory(cmodule
);
567 for (auto const& cfile
: po
.cfiles
) {
568 package
.addStaticFile(cfile
);
570 for (auto const& input
: po
.inputs
) {
571 package
.addSourceFile(input
);
573 if (!po
.inputList
.empty()) {
574 package
.addInputList(po
.inputList
);
577 if (!package
.parse()) return 1;
580 "{} files parsed, {} cached, {} files read, {} files stored",
581 package
.getTotalParses(),
582 package
.getParseCacheHits(),
583 package
.getFileReads(),
584 package
.getFileStores()
588 // Start asynchronously destroying the async state, since it may
589 // take a long time. We'll do it in the background while the rest of
590 // the compile pipeline runs.
591 auto clearer
= package
.clearAsyncState();
592 SCOPE_EXIT
{ if (clearer
) clearer
->join(); };
595 AsyncFileCacheSaver
fileCacheThread(&package
, po
.filecache
.c_str());
596 if (!po
.filecache
.empty()) {
597 fileCacheThread
.start();
600 ar
->setFinish([&po
,&timer
,&package
](AnalysisResultPtr res
) {
605 if (!po
.filecache
.empty()) {
606 fileCacheThread
.waitForEnd();
610 return hhbcTarget(po
, std::move(ar
), fileCacheThread
);
613 ///////////////////////////////////////////////////////////////////////////////
615 void hhbcTargetInit(const CompilerOptions
&po
, AnalysisResultPtr ar
) {
616 ar
->setOutputPath(po
.outputDir
);
617 // Propagate relevant compiler-specific options to the runtime.
618 RuntimeOption::RepoPath
= ar
->getOutputPath() + '/' + po
.program
;
619 unlink(RuntimeOption::RepoPath
.c_str());
622 int hhbcTarget(const CompilerOptions
&po
, AnalysisResultPtr
&& ar
,
623 AsyncFileCacheSaver
&fcThread
) {
626 const char *type
= 0;
627 if (po
.format
.find("text") != std::string::npos
) {
628 Option::GenerateTextHHBC
= true;
629 type
= "creating text HHBC files";
632 if (po
.format
.find("hhas") != std::string::npos
) {
633 Option::GenerateHhasHHBC
= true;
634 type
= "creating hhas HHBC files";
637 if (po
.format
.find("binary") != std::string::npos
) {
638 Option::GenerateBinaryHHBC
= true;
639 type
= "creating binary HHBC files";
643 if (formatCount
== 0) {
644 Logger::Error("Unknown format for HHBC target: %s", po
.format
.c_str());
648 unlink(RuntimeOption::RepoPath
.c_str());
649 /* without this, emitClass allows classes with interfaces to be
651 SystemLib::s_inited
= true;
653 // the function is only invoked in hhvm --hphp, which is supposed to be in
654 // repo mode only. we are not setting it earlier in `compiler_main` since we
655 // want systemlib to be built without repo-auth == true, or otherwise,
656 // `compile_systemlib_string` will try to load systemlib from repo, while we
658 RuntimeOption::RepoAuthoritative
= true;
660 Timer
timer(Timer::WallTime
, type
);
661 Compiler::emitAllHHBC(std::move(ar
));
666 ///////////////////////////////////////////////////////////////////////////////
668 void createOutputDirectory(CompilerOptions
&po
) {
669 if (po
.outputDir
.empty()) {
670 const char *t
= getenv("TEMP");
674 std::string temp
= t
;
675 temp
+= "/hphp_XXXXXX";
676 std::vector
<char> path(begin(temp
), end(temp
));
677 path
.push_back('\0');
678 po
.outputDir
= mkdtemp(&path
[0]);
679 Logger::Info("creating temporary directory %s ...", po
.outputDir
.c_str());
681 mkdir(po
.outputDir
.c_str(), 0777);
684 ///////////////////////////////////////////////////////////////////////////////