hphp/runtime/vm/unit-parser.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/unit-parser.h"
  18
  19 #include <cinttypes>
  20 #include <condition_variable>
  21 #include <fstream>
  22 #include <iterator>
  23 #include <memory>
  24 #include <mutex>
  25 #include <signal.h>
  26 #include <sstream>
  27 #include <stdio.h>
  28 #include <string>
  29 #include <sys/types.h>
  30 #include <sys/wait.h>
  31
  32 #include <folly/compression/Zstd.h>
  33 #include <folly/DynamicConverter.h>
  34 #include <folly/json.h>
  35 #include <folly/FileUtil.h>
  36 #include <folly/system/ThreadName.h>
  37
  38 #include "hphp/hack/src/hackc/ffi_bridge/compiler_ffi.rs"
  39 #include "hphp/hack/src/hackc/hhbc-ast.h"
  40 #include "hphp/hack/src/parser/ffi_bridge/parser_ffi.rs"
  41 #include "hphp/runtime/base/autoload-map.h"
  42 #include "hphp/runtime/base/autoload-handler.h"
  43 #include "hphp/runtime/base/file-stream-wrapper.h"
  44 #include "hphp/runtime/base/ini-setting.h"
  45 #include "hphp/runtime/base/stream-wrapper-registry.h"
  46 #include "hphp/runtime/base/unit-cache.h"
  47 #include "hphp/runtime/vm/disas.h"
  48 #include "hphp/runtime/vm/native.h"
  49 #include "hphp/runtime/vm/decl-provider.h"
  50 #include "hphp/runtime/vm/hackc-translator.h"
  51 #include "hphp/runtime/vm/unit-emitter.h"
  52 #include "hphp/runtime/vm/unit-gen-helpers.h"
  53 #include "hphp/util/atomic-vector.h"
  54 #include "hphp/util/embedded-data.h"
  55 #include "hphp/util/gzip.h"
  56 #include "hphp/util/hackc-log.h"
  57 #include "hphp/util/light-process.h"
  58 #include "hphp/util/logger.h"
  59 #include "hphp/util/match.h"
  60 #include "hphp/util/sha1.h"
  61 #include "hphp/util/struct-log.h"
  62 #include "hphp/util/timer.h"
  63 #include "hphp/zend/zend-strtod.h"
  64
  65 namespace HPHP {
  66
  67 TRACE_SET_MOD(unit_parse);
  68
  69 UnitEmitterCacheHook g_unit_emitter_cache_hook = nullptr;
  70 static std::string s_misc_config;
  71
  72 namespace {
  73
  74 struct CompileException : Exception {
  75   template<class... A>
  76   explicit CompileException(A&&... args)
  77     : Exception(folly::sformat(std::forward<A>(args)...))
  78   {}
  79 };
  80
  81 [[noreturn]] void throwErrno(const char* what) {
  82   throw CompileException("{}: {}", what, folly::errnoStr(errno));
  83 }
  84
  85 CompilerResult assemble_string_handle_errors(const char* code,
  86                                              const std::string& hhas,
  87                                              const char* filename,
  88                                              const SHA1& sha1,
  89                                              const Native::FuncTable& nativeFuncs,
  90                                              bool& internal_error,
  91                                              CompileAbortMode mode) {
  92   try {
  93     return assemble_string(hhas.c_str(),
  94                            hhas.length(),
  95                            filename,
  96                            sha1,
  97                            nativeFuncs,
  98                            false);  /* swallow errors */
  99   } catch (const FatalErrorException&) {
 100     throw;
 101   } catch (const TranslationFatal& ex) {
 102     // Assembler returned an error when building this unit
 103     if (mode >= CompileAbortMode::VerifyErrors) internal_error = true;
 104     return ex.what();
 105   } catch (const AssemblerUnserializationError& ex) {
 106     // Variable unserializer threw when called from the assembler, treat it
 107     // as an internal error.
 108     internal_error = true;
 109     return ex.what();
 110   } catch (const AssemblerError& ex) {
 111     if (mode >= CompileAbortMode::VerifyErrors) internal_error = true;
 112
 113     if (RuntimeOption::EvalHackCompilerVerboseErrors) {
 114       auto const msg = folly::sformat(
 115         "{}\n"
 116         "========== PHP Source ==========\n"
 117         "{}\n"
 118         "========== HackC Result ==========\n"
 119         "{}\n",
 120         ex.what(),
 121         code,
 122         hhas
 123       );
 124       Logger::FError("HackC Generated a bad unit: {}", msg);
 125       return msg;
 126     } else {
 127       return ex.what();
 128     }
 129   } catch (const std::exception& ex) {
 130     internal_error = true;
 131     return ex.what();
 132   }
 133 }
 134
 135 ////////////////////////////////////////////////////////////////////////////////
 136
 137 struct ConfigBuilder {
 138   template<typename T>
 139   ConfigBuilder& addField(folly::StringPiece key, const T& data) {
 140     if (!m_config.isObject()) {
 141       m_config = folly::dynamic::object();
 142     }
 143
 144     m_config[key] = folly::dynamic::object(
 145       "global_value", folly::toDynamic(data));
 146
 147     return *this;
 148   }
 149
 150   std::string toString() const {
 151     return m_config.isNull() ? "" : folly::toJson(m_config);
 152   }
 153
 154  private:
 155   folly::dynamic m_config{nullptr};
 156 };
 157
 158 CompilerResult hackc_compile(
 159   const char* code,
 160   const char* filename,
 161   const SHA1& sha1,
 162   const Native::FuncTable& nativeFuncs,
 163   bool isSystemLib,
 164   bool forDebuggerEval,
 165   bool& internal_error,
 166   const RepoOptionsFlags& options,
 167   CompileAbortMode mode
 168 ) {
 169   // Create DeclProvider. Returns nullptr if disabled or too early.
 170   auto aliased_namespaces = options.getAliasedNamespacesConfig();
 171   auto provider = HhvmDeclProvider::create(options);
 172
 173   uint8_t flags = make_env_flags(
 174     isSystemLib,                    // is_systemlib
 175     false,                          // is_evaled
 176     forDebuggerEval,                // for_debugger_eval
 177     true,                           // dump_symbol_refs
 178     false                           // disable_toplevel_elaboration
 179   );
 180
 181   NativeEnv const native_env{
 182     reinterpret_cast<uint64_t>(provider.get()),
 183     reinterpret_cast<uint64_t>(provider ? &hhvm_decl_provider_get_decl : nullptr),
 184     filename,
 185     aliased_namespaces,
 186     s_misc_config,
 187     RuntimeOption::EvalEmitClassPointers,
 188     RuntimeOption::CheckIntOverflow,
 189     options.getCompilerFlags(),
 190     options.getParserFlags(),
 191     flags
 192   };
 193
 194   // Invoke hackc, producing a rust Vec<u8> containing HHAS.
 195   rust::Vec<uint8_t> hhas_vec = [&] {
 196     tracing::Block _{
 197       "hackc",
 198       [&] {
 199         return tracing::Props{}
 200           .add("filename", filename ? filename : "")
 201           .add("code_size", strlen(code));
 202       }
 203     };
 204     return hackc_compile_from_text_cpp_ffi(native_env, code);
 205   }();
 206   auto const hhas = std::string(hhas_vec.begin(), hhas_vec.end());
 207
 208   // Assemble HHAS into a UnitEmitter, or a std::string if there were errors.
 209   auto res = assemble_string_handle_errors(code,
 210                                            hhas,
 211                                            filename,
 212                                            sha1,
 213                                            nativeFuncs,
 214                                            internal_error,
 215                                            mode);
 216
 217   if (RO::EvalTranslateHackC) {
 218     rust::Box<HackCUnitWrapper> unit_wrapped =
 219       hackc_compile_unit_from_text_cpp_ffi(native_env, code);
 220     rust::Vec<uint8_t> hhbc {hackc_unit_to_string_cpp_ffi(native_env, *unit_wrapped)};
 221     std::string hhasString(hhbc.begin(), hhbc.end());
 222
 223     auto const assemblerOut = [&]() -> std::string {
 224       if (auto ue = boost::get<std::unique_ptr<UnitEmitter>>(&res)) {
 225         (*ue)->finish();
 226         return disassemble((*ue)->create().get(), true);
 227       }
 228       return boost::get<std::string>(res);
 229     }();
 230     const hackc::hhbc::HackCUnit* unit = hackCUnitRaw(unit_wrapped);
 231     try {
 232       auto const ue = unitEmitterFromHackCUnit(*unit,
 233                                                filename,
 234                                                sha1,
 235                                                nativeFuncs,
 236                                                hhasString);
 237       auto const hackCTranslatorOut = disassemble(ue->create().get(), true);
 238
 239       if (hackCTranslatorOut.length() != assemblerOut.length()) {
 240         Logger::FError("HackC Translator incorrect length: {}\n", filename);
 241       }
 242
 243       UNUSED auto start_of_line = 0;
 244       for(int i = 0; i < hackCTranslatorOut.length(); i++) {
 245         if (hackCTranslatorOut[i] == '\n' || assemblerOut[i] == '\n') {
 246           start_of_line = i;
 247         }
 248         if (hackCTranslatorOut[i] != assemblerOut[i]) {
 249           while (i < hackCTranslatorOut.length() &&
 250                  hackCTranslatorOut[i] != '\n' && assemblerOut[i] != '\n') {
 251             i++;
 252           }
 253           Logger::FError("HackC Translator incorrect: {}\n", filename);
 254           ITRACE(3, "HackC Translator: {}\n\nassembler: {}\n\n",
 255             hackCTranslatorOut.substr(start_of_line,i),
 256             assemblerOut.substr(start_of_line,i)
 257           );
 258           ITRACE(4, "HackC Translator:\n{}\n", hackCTranslatorOut);
 259           ITRACE(4, "assembler:\n{}\n", assemblerOut);
 260           break;
 261         }
 262       }
 263     } catch (const TranslationFatal& ex) {
 264       auto const err = ex.what();
 265       if (std::strcmp(err, assemblerOut.c_str())) {
 266         Logger::FError("HackC Translator incorrect error: {}\n", filename);
 267       }
 268       ITRACE(4, "HackC Translator Err: {}\n", err);
 269     }
 270   }
 271
 272   return res;
 273 }
 274 ////////////////////////////////////////////////////////////////////////////////
 275 }
 276
 277 CompilerAbort::CompilerAbort(const std::string& filename,
 278                              const std::string& error)
 279   : std::runtime_error{
 280       folly::sformat(
 281         "Encountered an internal error while processing HHAS for {}, "
 282         "bailing because Eval.AbortBuildOnCompilerError is set\n\n{}",
 283         filename, error
 284       )
 285     }
 286 {
 287 }
 288
 289 void compilers_start() {
 290   // Some configs, like IncludeRoots, can't easily be Config::Bind(ed), so here
 291   // we create a place to dump miscellaneous config values HackC might want.
 292   s_misc_config = []() -> std::string {
 293     if (RuntimeOption::EvalHackCompilerInheritConfig) {
 294       return ConfigBuilder()
 295         .addField("hhvm.include_roots", RuntimeOption::IncludeRoots)
 296         .toString();
 297     }
 298     return "";
 299   }();
 300 }
 301
 302 ParseFactsResult extract_facts(
 303   const std::string& filename,
 304   const std::string& code,
 305   const RepoOptionsFlags& options
 306 ) {
 307   auto const get_facts = [&](const std::string& source_text) -> ParseFactsResult {
 308     try {
 309       std::int32_t decl_flags = options.getDeclFlags();
 310       rust::Box<DeclParserOptions> decl_opts =
 311         hackc_create_direct_decl_parse_options(
 312           decl_flags,
 313           options.getAliasedNamespacesConfig());
 314       DeclResult decls = hackc_direct_decl_parse(*decl_opts, filename, source_text);
 315       FactsResult facts = hackc_decls_to_facts_cpp_ffi(decl_flags, decls, source_text);
 316       rust::String facts_as_json = hackc_facts_to_json_cpp_ffi(facts, source_text);
 317       return FactsJSONString { std::string(facts_as_json) };
 318     } catch (const std::exception& e) {
 319       return FactsJSONString { "" }; // Swallow errors from HackC
 320     }
 321   };
 322
 323   if (!code.empty()) {
 324     return get_facts(code);
 325   } else {
 326     auto w = Stream::getWrapperFromURI(StrNR(filename));
 327     if (!(w && dynamic_cast<FileStreamWrapper*>(w))) {
 328       throwErrno("Failed to extract facts: Could not get FileStreamWrapper.");
 329     }
 330     const auto f = w->open(StrNR(filename), "r", 0, nullptr);
 331     if (!f) throwErrno("Failed to extract facts: Could not read source code.");
 332     auto const str = f->read();
 333     return get_facts(str.get()->toCppString());
 334   }
 335 }
 336
 337 FfpResult ffp_parse_file(
 338   const std::string& contents,
 339   const RepoOptionsFlags& options
 340 ) {
 341   auto const env = options.getParserEnvironment();
 342   auto const parse_tree = hackc_parse_positioned_full_trivia_cpp_ffi(contents, env);
 343   return FfpJSONString {  std::string(parse_tree) };
 344 }
 345
 346 std::unique_ptr<UnitCompiler>
 347 UnitCompiler::create(LazyUnitContentsLoader& loader,
 348                      const char* filename,
 349                      const Native::FuncTable& nativeFuncs,
 350                      bool isSystemLib,
 351                      bool forDebuggerEval) {
 352   auto make = [&loader, &nativeFuncs, filename, isSystemLib, forDebuggerEval] {
 353     return std::make_unique<HackcUnitCompiler>(
 354       loader,
 355       filename,
 356       nativeFuncs,
 357       isSystemLib,
 358       forDebuggerEval
 359     );
 360   };
 361
 362   if (g_unit_emitter_cache_hook && !forDebuggerEval) {
 363     return std::make_unique<CacheUnitCompiler>(
 364       loader,
 365       filename,
 366       nativeFuncs,
 367       isSystemLib,
 368       false,
 369       std::move(make)
 370     );
 371   } else {
 372     return make();
 373   }
 374 }
 375
 376 std::unique_ptr<UnitEmitter> HackcUnitCompiler::compile(
 377     bool& cacheHit,
 378     CompileAbortMode mode) {
 379   auto ice = false;
 380   cacheHit = false;
 381
 382   auto res = hackc_compile(m_loader.contents().data(),
 383                            m_filename,
 384                            m_loader.sha1(),
 385                            m_nativeFuncs,
 386                            m_isSystemLib,
 387                            m_forDebuggerEval,
 388                            ice,
 389                            m_loader.options(),
 390                            mode);
 391   auto unitEmitter = match<std::unique_ptr<UnitEmitter>>(
 392     res,
 393     [&] (std::unique_ptr<UnitEmitter>& ue) {
 394       ue->finish();
 395       return std::move(ue);
 396     },
 397     [&] (std::string& err) {
 398       switch (mode) {
 399       case CompileAbortMode::Never:
 400         break;
 401       case CompileAbortMode::AllErrorsNull: {
 402         auto ue = std::unique_ptr<UnitEmitter>{};
 403         ue->finish();
 404         return ue;
 405       }
 406       case CompileAbortMode::OnlyICE:
 407       case CompileAbortMode::VerifyErrors:
 408       case CompileAbortMode::AllErrors:
 409         // run_compiler will promote errors to ICE as appropriate based on mode
 410         if (ice) throw CompilerAbort{m_filename, err};
 411       }
 412       return createFatalUnit(
 413         makeStaticString(m_filename),
 414         m_loader.sha1(),
 415         FatalOp::Runtime,
 416         err
 417       );
 418     }
 419   );
 420
 421   if (unitEmitter) unitEmitter->m_ICE = ice;
 422   return unitEmitter;
 423 }
 424
 425 std::unique_ptr<UnitEmitter>
 426 CacheUnitCompiler::compile(bool& cacheHit, CompileAbortMode mode) {
 427   assertx(g_unit_emitter_cache_hook);
 428   cacheHit = true;
 429   return g_unit_emitter_cache_hook(
 430     m_filename,
 431     m_loader.sha1(),
 432     m_loader.fileLength(),
 433     [&] (bool wantsICE) {
 434       if (!m_fallback) m_fallback = m_makeFallback();
 435       assertx(m_fallback);
 436       return m_fallback->compile(
 437         cacheHit,
 438         wantsICE ? mode : CompileAbortMode::AllErrorsNull
 439       );
 440     },
 441     m_nativeFuncs
 442   );
 443 }
 444
 445 ////////////////////////////////////////////////////////////////////////////////
 446 }