From 6ce5c63a59c4076b6c2b58ce07692d5f8225e0ba Mon Sep 17 00:00:00 2001 From: Rafael Auler Date: Thu, 2 Aug 2018 14:24:57 -0700 Subject: [PATCH] Control number of threads using a runtime option Summary: The threaded implementation of debug-parser maintains a separate state of libdwarf/libelf for each thread. This lib maps the full binary into memory, which can be quite large and, depending on the number of threads and hhvm binary size, will exhaust all available memory and crash with an allocation error. Reviewed By: markw65 Differential Revision: D9127053 fbshipit-source-id: c25034cf8dd4e11b04c322d511221982f010fb74 --- hphp/tools/debug-parser/debug-parser-dwarf.cpp | 14 +++++--------- hphp/tools/debug-parser/debug-parser.cpp | 8 +++++--- hphp/tools/debug-parser/debug-parser.h | 8 +++++++- hphp/tools/type-info-gens/gen-member-reflection.cpp | 17 +++++++++++++++-- hphp/tools/type-info-gens/gen-type-scanners.cpp | 2 +- 5 files changed, 33 insertions(+), 16 deletions(-) diff --git a/hphp/tools/debug-parser/debug-parser-dwarf.cpp b/hphp/tools/debug-parser/debug-parser-dwarf.cpp index 2f209f2814d..798f3bb0a75 100644 --- a/hphp/tools/debug-parser/debug-parser-dwarf.cpp +++ b/hphp/tools/debug-parser/debug-parser-dwarf.cpp @@ -167,7 +167,7 @@ struct Scope { */ struct TypeParserImpl : TypeParser { - explicit TypeParserImpl(const std::string& filename); + explicit TypeParserImpl(const std::string& filename, int num_threads); Object getObject(ObjectTypeKey key) override; @@ -265,11 +265,7 @@ ObjectTypeName Scope::name() const { return ObjectTypeName{std::move(str), linkage()}; } -// Number of threads to use for building up state. More isn't necessarily -// better. -constexpr size_t kNumThreads = 24; - -TypeParserImpl::TypeParserImpl(const std::string& filename) +TypeParserImpl::TypeParserImpl(const std::string& filename, int num_threads) : m_dwarf{filename} { // Processing each compiliation unit is very expensive, as it involves walking @@ -387,7 +383,7 @@ TypeParserImpl::TypeParserImpl(const std::string& filename) // Fire up the thread pool Context context{filename, m_states, m_state_map}; HPHP::JobQueueDispatcher dispatcher{ - kNumThreads, kNumThreads, 0, false, &context + num_threads, num_threads, 0, false, &context }; dispatcher.start(); @@ -1864,8 +1860,8 @@ private: } std::unique_ptr -make_dwarf_type_parser(const std::string& filename) { - return std::make_unique(filename); +make_dwarf_type_parser(const std::string& filename, int num_threads) { + return std::make_unique(filename, num_threads); } std::unique_ptr make_dwarf_printer(const std::string& filename) { diff --git a/hphp/tools/debug-parser/debug-parser.cpp b/hphp/tools/debug-parser/debug-parser.cpp index 096dbbc79f3..5cd7b665957 100644 --- a/hphp/tools/debug-parser/debug-parser.cpp +++ b/hphp/tools/debug-parser/debug-parser.cpp @@ -106,10 +106,12 @@ std::string Type::toString() const { * control which implementation gets returned. */ -std::unique_ptr TypeParser::make(const std::string& filename) { +std::unique_ptr TypeParser::make(const std::string& filename, + int num_threads) { #if defined(__linux__) || defined(__FreeBSD__) - std::unique_ptr make_dwarf_type_parser(const std::string&); - return make_dwarf_type_parser(filename); + std::unique_ptr make_dwarf_type_parser(const std::string&, + int); + return make_dwarf_type_parser(filename, num_threads); #else return nullptr; #endif diff --git a/hphp/tools/debug-parser/debug-parser.h b/hphp/tools/debug-parser/debug-parser.h index b22e52a4ccd..66681d01b4f 100644 --- a/hphp/tools/debug-parser/debug-parser.h +++ b/hphp/tools/debug-parser/debug-parser.h @@ -481,7 +481,13 @@ struct TypeParser { // of the parser. Run the parser on the given filename, which may be an // executable or some form of object file. If the platform doesn't have a // supported debug info parser, this function will return null. - static std::unique_ptr make(const std::string& filename); + // The number of threads controls parallelism when building up state if the + // implementation supports it. More isn't necessarily better, and the dwarf + // implementation will allocate memory proportional to the size of the input + // binary and the number of threads. If the binary is very large, it may + // exhaust memory resources in some systems. + static std::unique_ptr make(const std::string& filename, + int num_threads); // Iterate over the list of all object types defined in the file. This is safe // to call from multiple threads concurrently. diff --git a/hphp/tools/type-info-gens/gen-member-reflection.cpp b/hphp/tools/type-info-gens/gen-member-reflection.cpp index 29eb5a3eebe..c4793da2b4e 100644 --- a/hphp/tools/type-info-gens/gen-member-reflection.cpp +++ b/hphp/tools/type-info-gens/gen-member-reflection.cpp @@ -129,6 +129,8 @@ void generate_entry(const Object& object, std::ostream& o, << " }"; } +size_t NumThreads = 24; + void generate(const std::string& source_executable, std::ostream& o) { o << "#include \n"; o << "#include \n\n"; @@ -148,7 +150,7 @@ void generate(const std::string& source_executable, std::ostream& o) { #undef X }; - auto const parser = TypeParser::make(source_executable); + auto const parser = TypeParser::make(source_executable, NumThreads); auto first = true; parser->forEachObject( @@ -194,7 +196,8 @@ int main(int argc, char** argv) { "filename to read debug-info from") ("output_file", po::value()->required(), - "filename of generated code"); + "filename of generated code") + ("num_threads", po::value(), "number of parallel threads"); try { po::variables_map vm; @@ -206,6 +209,16 @@ int main(int argc, char** argv) { return 1; } + if (vm.count("num_threads")) { + auto n = vm["num_threads"].as(); + if (n > 0) { + NumThreads = n; + } else { + std::cerr << "\nIllegal num_threads=" << n << "\n"; + return 1; + } + } + po::notify(vm); auto const source_executable = vm["source_file"].as(); diff --git a/hphp/tools/type-info-gens/gen-type-scanners.cpp b/hphp/tools/type-info-gens/gen-type-scanners.cpp index af4e6888867..f5dd695a1c7 100644 --- a/hphp/tools/type-info-gens/gen-type-scanners.cpp +++ b/hphp/tools/type-info-gens/gen-type-scanners.cpp @@ -629,7 +629,7 @@ Generator::Generator(const std::string& filename, bool skip) { // runtime. if (skip) return; - m_parser = TypeParser::make(filename); + m_parser = TypeParser::make(filename, NumThreads); tbb::concurrent_vector indexer_types; tbb::concurrent_vector collectable_markers; -- 2.11.4.GIT