hphp/compiler/analysis/emitter.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/compiler/analysis/emitter.h"
  18
  19 #include "hphp/compiler/analysis/analysis_result.h"
  20 #include "hphp/compiler/builtin_symbols.h"
  21 #include "hphp/compiler/option.h"
  22 #include "hphp/hhbbc/hhbbc.h"
  23
  24 #include "hphp/runtime/base/execution-context.h"
  25 #include "hphp/runtime/base/program-functions.h"
  26 #include "hphp/runtime/base/runtime-option.h"
  27 #include "hphp/runtime/base/request-info.h"
  28 #include "hphp/runtime/vm/disas.h"
  29 #include "hphp/runtime/vm/extern-compiler.h"
  30 #include "hphp/runtime/vm/repo.h"
  31 #include "hphp/runtime/vm/repo-autoload-map-builder.h"
  32 #include "hphp/runtime/vm/repo-file.h"
  33 #include "hphp/runtime/vm/repo-global-data.h"
  34 #include "hphp/runtime/vm/treadmill.h"
  35 #include "hphp/runtime/vm/type-alias-emitter.h"
  36 #include "hphp/runtime/vm/unit.h"
  37 #include "hphp/util/job-queue.h"
  38 #include "hphp/util/logger.h"
  39
  40 #include <folly/ScopeGuard.h>
  41
  42 #include <exception>
  43 #include <fstream>
  44 #include <future>
  45 #include <iostream>
  46 #include <vector>
  47
  48 namespace HPHP {
  49
  50 namespace Compiler {
  51 ///////////////////////////////////////////////////////////////////////////////
  52
  53 namespace {
  54
  55 void genText(UnitEmitter* ue, const std::string& outputPath) {
  56   std::unique_ptr<Unit> unit(ue->create());
  57   auto const basePath = AnalysisResult::prepareFile(
  58     outputPath.c_str(),
  59     Option::UserFilePrefix + unit->filepath()->toCppString(),
  60     true, false);
  61
  62   if (Option::GenerateTextHHBC) {
  63     auto const fullPath = basePath + ".hhbc.txt";
  64
  65     std::ofstream f(fullPath.c_str());
  66     if (!f) {
  67       Logger::Error("Unable to open %s for write", fullPath.c_str());
  68     } else {
  69       f << "Hash: " << ue->sha1().toString() << std::endl;
  70       f << unit->toString();
  71       f.close();
  72     }
  73   }
  74
  75   if (Option::GenerateHhasHHBC) {
  76     auto const fullPath = basePath + ".hhas";
  77
  78     std::ofstream f(fullPath.c_str());
  79     if (!f) {
  80       Logger::Error("Unable to open %s for write", fullPath.c_str());
  81     } else {
  82       f << disassemble(unit.get());
  83       f.close();
  84     }
  85   }
  86 }
  87
  88 class GenTextWorker
  89     : public JobQueueWorker<UnitEmitter*, const std::string*, true, true> {
  90  public:
  91   void doJob(JobType job) override {
  92     try {
  93       genText(job, *m_context);
  94     } catch (Exception& e) {
  95       Logger::Error(e.getMessage());
  96     } catch (...) {
  97       Logger::Error("Fatal: An unexpected exception was thrown");
  98     }
  99   }
 100   void onThreadEnter() override {
 101     g_context.getCheck();
 102   }
 103   void onThreadExit() override {
 104     hphp_memory_cleanup();
 105   }
 106 };
 107
 108 void genText(const std::vector<std::unique_ptr<UnitEmitter>>& ues,
 109              const std::string& outputPath) {
 110   if (!ues.size()) return;
 111
 112   Timer timer(Timer::WallTime, "Generating text bytcode");
 113   if (ues.size() > Option::ParserThreadCount && Option::ParserThreadCount > 1) {
 114     JobQueueDispatcher<GenTextWorker> dispatcher {
 115       Option::ParserThreadCount,
 116       Option::ParserThreadCount,
 117       0, false, &outputPath
 118     };
 119     dispatcher.start();
 120     for (auto& ue : ues) {
 121       dispatcher.enqueue(ue.get());
 122     }
 123     dispatcher.waitEmpty();
 124   } else {
 125     for (auto& ue : ues) {
 126       genText(ue.get(), outputPath);
 127     }
 128   }
 129 }
 130
 131 RepoGlobalData getGlobalData() {
 132   auto const now = std::chrono::high_resolution_clock::now();
 133   auto const nanos =
 134     std::chrono::duration_cast<std::chrono::nanoseconds>(
 135       now.time_since_epoch()
 136     );
 137
 138   auto gd                        = RepoGlobalData{};
 139   gd.Signature                   = nanos.count();
 140   gd.CheckPropTypeHints          = RuntimeOption::EvalCheckPropTypeHints;
 141   gd.HardPrivatePropInference    = true;
 142   gd.PHP7_NoHexNumerics          = RuntimeOption::PHP7_NoHexNumerics;
 143   gd.PHP7_Substr                 = RuntimeOption::PHP7_Substr;
 144   gd.PHP7_Builtins               = RuntimeOption::PHP7_Builtins;
 145   gd.HardGenericsUB              = RuntimeOption::EvalEnforceGenericsUB >= 2;
 146   gd.HackArrCompatNotices        = RuntimeOption::EvalHackArrCompatNotices;
 147   gd.EnableIntrinsicsExtension   = RuntimeOption::EnableIntrinsicsExtension;
 148   gd.ForbidDynamicCallsToFunc    = RuntimeOption::EvalForbidDynamicCallsToFunc;
 149   gd.ForbidDynamicCallsWithAttr  =
 150     RuntimeOption::EvalForbidDynamicCallsWithAttr;
 151   gd.ForbidDynamicCallsToClsMeth =
 152     RuntimeOption::EvalForbidDynamicCallsToClsMeth;
 153   gd.ForbidDynamicCallsToInstMeth =
 154     RuntimeOption::EvalForbidDynamicCallsToInstMeth;
 155   gd.ForbidDynamicConstructs     = RuntimeOption::EvalForbidDynamicConstructs;
 156   gd.LogKnownMethodsAsDynamicCalls =
 157     RuntimeOption::EvalLogKnownMethodsAsDynamicCalls;
 158   gd.EnableArgsInBacktraces      = RuntimeOption::EnableArgsInBacktraces;
 159   gd.NoticeOnBuiltinDynamicCalls =
 160     RuntimeOption::EvalNoticeOnBuiltinDynamicCalls;
 161   gd.InitialNamedEntityTableSize =
 162     RuntimeOption::EvalInitialNamedEntityTableSize;
 163   gd.InitialStaticStringTableSize =
 164     RuntimeOption::EvalInitialStaticStringTableSize;
 165   gd.HackArrCompatIsVecDictNotices =
 166     RuntimeOption::EvalHackArrCompatIsVecDictNotices;
 167   gd.HackArrCompatSerializeNotices =
 168     RuntimeOption::EvalHackArrCompatSerializeNotices;
 169   gd.AbortBuildOnVerifyError = RuntimeOption::EvalAbortBuildOnVerifyError;
 170   gd.EmitClassPointers = RuntimeOption::EvalEmitClassPointers;
 171   gd.EmitClsMethPointers = RuntimeOption::EvalEmitClsMethPointers;
 172   gd.IsVecNotices = RuntimeOption::EvalIsVecNotices;
 173   gd.IsCompatibleClsMethType = RuntimeOption::EvalIsCompatibleClsMethType;
 174   gd.RaiseClassConversionWarning =
 175     RuntimeOption::EvalRaiseClassConversionWarning;
 176   gd.ClassPassesClassname = RuntimeOption::EvalClassPassesClassname;
 177   gd.ClassnameNotices = RuntimeOption::EvalClassnameNotices;
 178   gd.ClassIsStringNotices = RuntimeOption::EvalClassIsStringNotices;
 179   gd.RaiseClsMethConversionWarning =
 180     RuntimeOption::EvalRaiseClsMethConversionWarning;
 181   gd.StrictArrayFillKeys = RuntimeOption::StrictArrayFillKeys;
 182   gd.NoticeOnCoerceForStrConcat =
 183     RuntimeOption::EvalNoticeOnCoerceForStrConcat;
 184   gd.NoticeOnCoerceForBitOp =
 185     RuntimeOption::EvalNoticeOnCoerceForBitOp;
 186
 187   for (auto const& elm : RuntimeOption::ConstantFunctions) {
 188     auto const s = internal_serialize(tvAsCVarRef(elm.second));
 189     gd.ConstantFunctions.emplace_back(elm.first, s.toCppString());
 190   }
 191   return gd;
 192 }
 193
 194 /*
 195  * It's an invariant that symbols in the repo must be Unique and
 196  * Persistent. Normally HHBBC verifies this for us, but if we're not
 197  * using HHBBC and writing directly to the repo, we must do it
 198  * ourself. Verify all relevant symbols are unique and set the
 199  * appropriate Attrs.
 200  *
 201  * We use a common set of verification functions exported from HHBBC
 202  * (to keep error messages identical), so we need store the data in a
 203  * certain way it expects.
 204  */
 205 struct SymbolSets {
 206   struct Unit {
 207     const StringData* filename;
 208   };
 209   struct Data {
 210     const StringData* name;
 211     std::unique_ptr<Unit> unit;
 212     Attr attrs;
 213   };
 214
 215   using IMap = hphp_fast_map<
 216     const StringData*,
 217     std::unique_ptr<Data>,
 218     string_data_hash,
 219     string_data_isame
 220   >;
 221   using Map = hphp_fast_map<
 222     const StringData*,
 223     std::unique_ptr<Data>,
 224     string_data_hash,
 225     string_data_same
 226   >;
 227
 228   IMap enums;
 229   IMap classes;
 230   IMap funcs;
 231   IMap typeAliases;
 232   IMap records;
 233   Map constants;
 234
 235   static std::unique_ptr<Data> make(const UnitEmitter* ue,
 236                                     const StringData* name,
 237                                     Attr attrs) {
 238     assertx(name->isStatic());
 239     assertx(!ue || ue->m_filepath->isStatic());
 240     std::unique_ptr<Unit> unit;
 241     if (ue) {
 242       unit = std::make_unique<SymbolSets::Unit>();
 243       unit->filename = ue->m_filepath;
 244     }
 245     auto data = std::make_unique<SymbolSets::Data>();
 246     data->name = name;
 247     data->unit = std::move(unit);
 248     data->attrs = attrs;
 249     return data;
 250   }
 251
 252   SymbolSets() {
 253     // These aren't stored in the repo, but we still need to check for
 254     // collisions against them, so put them in the maps.
 255     for (auto const& kv : Native::getConstants()) {
 256       assertx(kv.second.m_type != KindOfUninit ||
 257               kv.second.dynamic());
 258       HHBBC::add_symbol(
 259         constants,
 260         make(nullptr, kv.first, AttrUnique | AttrPersistent),
 261         "constant"
 262       );
 263     }
 264   }
 265 };
 266
 267 void writeUnit(UnitEmitter& ue,
 268                RepoFileBuilder& repoBuilder,
 269                RepoAutoloadMapBuilder& autoloadMapBuilder,
 270                SymbolSets& sets) {
 271   // Verify uniqueness of symbols, set Attrs, then write to actual
 272   // repo.
 273   auto const make = [&] (const StringData* name, Attr attrs) {
 274     return SymbolSets::make(&ue, name, attrs);
 275   };
 276
 277   for (size_t n = 0; n < ue.numPreClasses(); ++n) {
 278     auto pce = ue.pce(n);
 279     pce->setAttrs(pce->attrs() | AttrUnique | AttrPersistent);
 280     if (pce->attrs() & AttrEnum) {
 281       HHBBC::add_symbol(sets.enums, make(pce->name(), pce->attrs()), "enum");
 282     }
 283     HHBBC::add_symbol(sets.classes, make(pce->name(), pce->attrs()), "class",
 284                       sets.records, sets.typeAliases);
 285   }
 286   for (auto& fe : ue.fevec()) {
 287     // Dedup meth_caller wrappers
 288     if (fe->attrs & AttrIsMethCaller && sets.funcs.count(fe->name)) continue;
 289     fe->attrs |= AttrUnique | AttrPersistent;
 290     HHBBC::add_symbol(sets.funcs, make(fe->name, fe->attrs), "function");
 291   }
 292   for (auto& te : ue.typeAliases()) {
 293     te->setAttrs(te->attrs() | AttrUnique | AttrPersistent);
 294     HHBBC::add_symbol(sets.typeAliases, make(te->name(), te->attrs()),
 295                       "type alias", sets.classes, sets.records);
 296   }
 297   for (auto& c : ue.constants()) {
 298     c.attrs |= AttrUnique | AttrPersistent;
 299     HHBBC::add_symbol(sets.constants, make(c.name, c.attrs), "constant");
 300   }
 301   for (size_t n = 0; n < ue.numRecords(); ++n) {
 302     auto const re = ue.re(n);
 303     re->setAttrs(re->attrs() | AttrUnique | AttrPersistent);
 304     HHBBC::add_symbol(sets.records, make(re->name(), re->attrs()), "record",
 305                       sets.classes, sets.typeAliases);
 306   }
 307
 308   autoloadMapBuilder.addUnit(ue);
 309   repoBuilder.add(ue);
 310 }
 311
 312 }
 313
 314 /*
 315  * This is the entry point for offline bytecode generation.
 316  */
 317 void emitAllHHBC(AnalysisResultPtr&& ar) {
 318   auto ues = ar->getHhasFiles();
 319   decltype(ues) ues_to_print;
 320   auto const outputPath = ar->getOutputPath();
 321
 322   std::thread wp_thread;
 323   std::future<void> fut;
 324
 325   auto unexpectedException = [&] (const char* what) {
 326     if (wp_thread.joinable()) {
 327       Logger::Error("emitAllHHBC exited via an exception "
 328                     "before wp_thread was joined: %s", what);
 329     }
 330     throw;
 331   };
 332
 333   try {
 334     {
 335       SCOPE_EXIT {
 336         genText(ues_to_print, outputPath);
 337       };
 338
 339       folly::Optional<RepoAutoloadMapBuilder> autoloadMapBuilder;
 340       folly::Optional<RepoFileBuilder> repoBuilder;
 341       folly::Optional<SymbolSets> symbolSets;
 342       if (Option::GenerateBinaryHHBC) {
 343         autoloadMapBuilder.emplace();
 344         repoBuilder.emplace(RuntimeOption::RepoCentralPath);
 345         symbolSets.emplace();
 346       }
 347
 348       auto program = std::move(ar->program());
 349       if (!program.get()) {
 350         uint32_t id = 0;
 351         for (auto& ue : ues) {
 352           ue->m_symbol_refs.clear();
 353           ue->m_sn = id;
 354           ue->setSha1(SHA1 { id });
 355           if (repoBuilder) {
 356             try {
 357               writeUnit(*ue, *repoBuilder, *autoloadMapBuilder, *symbolSets);
 358             } catch (const HHBBC::NonUniqueSymbolException&) {
 359               ar->setFinish({});
 360               throw;
 361             }
 362           }
 363           if (Option::GenerateTextHHBC || Option::GenerateHhasHHBC) {
 364             ues_to_print.emplace_back(std::move(ue));
 365           }
 366           id++;
 367         }
 368
 369         ar->finish();
 370         ar.reset();
 371
 372         if (repoBuilder) {
 373           Timer finalizeTime(Timer::WallTime, "finalizing repo");
 374           repoBuilder->finish(getGlobalData(), *autoloadMapBuilder);
 375         }
 376         return;
 377       }
 378
 379       assertx(ues.size() == 0);
 380
 381       ar->finish();
 382       ar.reset();
 383
 384       HHBBC::UnitEmitterQueue ueq{
 385         repoBuilder ? &*autoloadMapBuilder : nullptr,
 386         Option::GenerateTextHHBC || Option::GenerateHhasHHBC
 387       };
 388
 389       RuntimeOption::EvalJit = false; // For HHBBC to invoke builtins.
 390       std::unique_ptr<ArrayTypeTable::Builder> arrTable;
 391       std::promise<void> arrTableReady;
 392       fut = arrTableReady.get_future();
 393
 394       wp_thread = std::thread(
 395         [program = std::move(program), &ueq, &arrTable, &arrTableReady]
 396         () mutable {
 397           Timer timer(Timer::WallTime, "running HHBBC");
 398           HphpSessionAndThread _(Treadmill::SessionKind::CompilerEmit);
 399           try {
 400             // We rely on this function to provide a value to arrTable
 401             HHBBC::whole_program(
 402               std::move(program), ueq, arrTable,
 403               Option::ParserThreadCount > 0 ? Option::ParserThreadCount : 0,
 404               &arrTableReady);
 405           } catch (...) {
 406             arrTableReady.set_exception(std::current_exception());
 407             ueq.finish();
 408           }
 409         }
 410       );
 411
 412       folly::Optional<Timer> commitTime;
 413       while (auto encoded = ueq.pop()) {
 414         if (!commitTime) {
 415           commitTime.emplace(Timer::WallTime, "committing units to repo");
 416         }
 417         if (repoBuilder) repoBuilder->add(*encoded);
 418         if (Option::GenerateTextHHBC || Option::GenerateHhasHHBC) {
 419           if (auto ue = ueq.popUnitEmitter()) {
 420             ues_to_print.emplace_back(std::move(ue));
 421           }
 422         }
 423       }
 424
 425       fut.wait();
 426       Timer finalizeTime(Timer::WallTime, "finalizing repo");
 427       if (arrTable) globalArrayTypeTable().repopulate(*arrTable);
 428       if (repoBuilder) {
 429         repoBuilder->finish(getGlobalData(), *autoloadMapBuilder);
 430       }
 431     }
 432
 433     wp_thread.join();
 434     fut.get(); // Exception thrown here if it holds one, otherwise no-op.
 435   } catch (std::exception& ex) {
 436     unexpectedException(ex.what());
 437   } catch (const Object& o) {
 438     unexpectedException("Object");
 439   } catch (...) {
 440     unexpectedException("non-standard-exception");
 441   }
 442 }
 443
 444 extern "C" {
 445
 446 /**
 447  * This is the entry point from the runtime; i.e. online bytecode generation.
 448  * The 'filename' parameter may be NULL if there is no file associated with
 449  * the source code.
 450  *
 451  * Before being actually used, hphp_compiler_parse must be called with
 452  * a NULL `code' parameter to do initialization.
 453  */
 454
 455 Unit* hphp_compiler_parse(const char* code, int codeLen, const SHA1& sha1,
 456                           const char* filename,
 457                           const Native::FuncTable& nativeFuncs,
 458                           Unit** releaseUnit, bool forDebuggerEval,
 459                           const RepoOptions& options) {
 460   if (UNLIKELY(!code)) {
 461     // Do initialization when code is null; see above.
 462     Option::RecordErrors = false;
 463     Option::WholeProgram = false;
 464     TypeConstraint tc;
 465     return nullptr;
 466   }
 467
 468   tracing::Block _{
 469     "parse",
 470     [&] {
 471       return tracing::Props{}
 472         .add("filename", filename ? filename : "")
 473         .add("code_size", codeLen);
 474     }
 475   };
 476
 477   // Do not count memory used during parsing/emitting towards OOM.
 478   MemoryManager::SuppressOOM so(*tl_heap);
 479
 480   SCOPE_ASSERT_DETAIL("hphp_compiler_parse") { return filename; };
 481   std::unique_ptr<Unit> unit;
 482   SCOPE_EXIT {
 483     if (unit && releaseUnit) *releaseUnit = unit.release();
 484   };
 485
 486   // We don't want to invoke the JIT when trying to run PHP code.
 487   auto const prevFolding = RID().getJitFolding();
 488   RID().setJitFolding(true);
 489   SCOPE_EXIT { RID().setJitFolding(prevFolding); };
 490
 491   try {
 492     UnitOrigin unitOrigin = UnitOrigin::File;
 493     if (!filename) {
 494       filename = "";
 495       unitOrigin = UnitOrigin::Eval;
 496     }
 497
 498     std::unique_ptr<UnitEmitter> ue;
 499     // Check if this file contains raw hip hop bytecode instead of
 500     // php.  This is dictated by a special file extension.
 501     if (RuntimeOption::EvalAllowHhas) {
 502       if (const char* dot = strrchr(filename, '.')) {
 503         const char hhbc_ext[] = "hhas";
 504         if (!strcmp(dot + 1, hhbc_ext)) {
 505           ue = assemble_string(code, codeLen, filename, sha1, nativeFuncs);
 506         }
 507       }
 508     }
 509
 510     // If ue != nullptr then we assembled it above, so don't feed it into
 511     // the extern compiler
 512     if (!ue) {
 513       auto uc = UnitCompiler::create(code, codeLen, filename, sha1,
 514                                      nativeFuncs, forDebuggerEval, options);
 515       assertx(uc);
 516       try {
 517         tracing::BlockNoTrace _{"unit-compiler-run"};
 518         bool ignore;
 519         ue = uc->compile(ignore);
 520       } catch (const BadCompilerException& exc) {
 521         Logger::Error("Bad external compiler: %s", exc.what());
 522         return nullptr;
 523       }
 524     }
 525
 526     // NOTE: Repo errors are ignored!
 527     if (!RO::RepoAuthoritative) {
 528       Repo::get().commitUnit(ue.get(), unitOrigin, false);
 529     }
 530
 531     if (RO::EvalStressUnitSerde) ue = ue->stressSerde();
 532     unit = ue->create();
 533     if (BuiltinSymbols::s_systemAr) {
 534       assertx(ue->m_filepath->data()[0] == '/' &&
 535               ue->m_filepath->data()[1] == ':');
 536       BuiltinSymbols::s_systemAr->addHhasFile(std::move(ue));
 537     } else {
 538       ue.reset();
 539
 540       if (unit->sn() == -1 && !RO::RepoAuthoritative && RO::RepoCommit) {
 541         // the unit was not committed to the Repo, probably because
 542         // another thread did it first. Try to use the winner.
 543         auto u = Repo::get().loadUnit(filename ? filename : "",
 544                                       sha1,
 545                                       nativeFuncs);
 546         if (u != nullptr) {
 547           return u.release();
 548         }
 549       }
 550     }
 551
 552     return unit.release();
 553   } catch (const std::exception&) {
 554     // extern "C" function should not be throwing exceptions...
 555     return nullptr;
 556   }
 557 }
 558
 559 } // extern "C"
 560
 561 ///////////////////////////////////////////////////////////////////////////////
 562 }
 563 }