mono/mini/llvm-jit.cpp

   1 //
   2 // jit-llvm.cpp: Support code for using LLVM as a JIT backend
   3 //
   4 // (C) 2009-2011 Novell, Inc.
   5 // Copyright 2011-2015 Xamarin, Inc (http://www.xamarin.com)
   6 //
   7 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
   8 //
   9
  10 #include "config.h"
  11
  12 #include <llvm-c/Core.h>
  13 #include <llvm-c/ExecutionEngine.h>
  14
  15 #include "mini-llvm-cpp.h"
  16 #include "mini-runtime.h"
  17 #include "llvm-jit.h"
  18
  19 #if defined(MONO_ARCH_LLVM_JIT_SUPPORTED) && !defined(MONO_CROSS_COMPILE) && LLVM_API_VERSION > 600
  20
  21 #include <llvm/ADT/SmallVector.h>
  22 #include <llvm/Support/raw_ostream.h>
  23 #include <llvm/Support/Host.h>
  24 #include <llvm/Support/Memory.h>
  25 #include <llvm/Support/TargetSelect.h>
  26 #include <llvm/IR/Mangler.h>
  27 #include "llvm/IR/LegacyPassManager.h"
  28 #include "llvm/IR/LegacyPassNameParser.h"
  29 #include <llvm/ExecutionEngine/ExecutionEngine.h>
  30 #include "llvm/ExecutionEngine/Orc/CompileUtils.h"
  31 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
  32 #include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
  33 #include "llvm/ExecutionEngine/RTDyldMemoryManager.h"
  34 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
  35 #include "llvm/ExecutionEngine/JITSymbol.h"
  36 #include "llvm/Transforms/Scalar.h"
  37
  38 #if LLVM_API_VERSION >= 800
  39 #include "llvm/CodeGen/BuiltinGCs.h"
  40 #else
  41 #include "llvm/CodeGen/GCs.h"
  42 #endif
  43
  44 #include <cstdlib>
  45
  46 #include <mono/utils/mono-dl.h>
  47
  48 using namespace llvm;
  49 using namespace llvm::orc;
  50
  51 extern cl::opt<bool> EnableMonoEH;
  52 extern cl::opt<std::string> MonoEHFrameSymbol;
  53
  54 void
  55 mono_llvm_set_unhandled_exception_handler (void)
  56 {
  57 }
  58
  59 // noop function that merely ensures that certain symbols are not eliminated
  60 // from the resulting binary.
  61 static void
  62 link_gc () {
  63 #if LLVM_API_VERSION >= 800
  64         llvm::linkAllBuiltinGCs();
  65 #else
  66         llvm::linkCoreCLRGC(); // Mono uses built-in "coreclr" GCStrategy
  67 #endif
  68 }
  69
  70 template <typename T>
  71 static std::vector<T> singletonSet(T t) {
  72   std::vector<T> Vec;
  73   Vec.push_back(std::move(t));
  74   return Vec;
  75 }
  76
  77 #ifdef __MINGW32__
  78
  79 #include <stddef.h>
  80 extern void *memset(void *, int, size_t);
  81 void bzero (void *to, size_t count) { memset (to, 0, count); }
  82
  83 #endif
  84
  85 static MonoNativeTlsKey current_cfg_tls_id;
  86
  87 static unsigned char *
  88 alloc_code (LLVMValueRef function, int size)
  89 {
  90         auto cfg = (MonoCompile *)mono_native_tls_get_value (current_cfg_tls_id);
  91         g_assert (cfg);
  92         return (unsigned char *)mono_mem_manager_code_reserve (cfg->mem_manager, size);
  93 }
  94
  95 class MonoJitMemoryManager : public RTDyldMemoryManager
  96 {
  97 public:
  98         ~MonoJitMemoryManager() override;
  99
 100         uint8_t *allocateDataSection(uintptr_t Size,
 101                                                                  unsigned Alignment,
 102                                                                  unsigned SectionID,
 103                                                                  StringRef SectionName,
 104                                                                  bool IsReadOnly) override;
 105
 106         uint8_t *allocateCodeSection(uintptr_t Size,
 107                                                                  unsigned Alignment,
 108                                                                  unsigned SectionID,
 109                                                                  StringRef SectionName) override;
 110
 111         bool finalizeMemory(std::string *ErrMsg = nullptr) override;
 112 private:
 113         SmallVector<sys::MemoryBlock, 16> PendingCodeMem;
 114 };
 115
 116 MonoJitMemoryManager::~MonoJitMemoryManager()
 117 {
 118 }
 119
 120 uint8_t *
 121 MonoJitMemoryManager::allocateDataSection(uintptr_t Size,
 122                                                                                   unsigned Alignment,
 123                                                                                   unsigned SectionID,
 124                                                                                   StringRef SectionName,
 125                                                                                   bool IsReadOnly)
 126 {
 127         uint8_t *res;
 128
 129         // FIXME: Use a mempool
 130         if (Alignment == 0)
 131                 Alignment = 16;
 132         res = (uint8_t*)malloc (Size + Alignment);
 133         res = (uint8_t*)ALIGN_PTR_TO(res, Alignment);
 134         assert (res);
 135         g_assert (GPOINTER_TO_UINT (res) % Alignment == 0);
 136         memset (res, 0, Size);
 137         return res;
 138 }
 139
 140 uint8_t *
 141 MonoJitMemoryManager::allocateCodeSection(uintptr_t Size,
 142                                                                                   unsigned Alignment,
 143                                                                                   unsigned SectionID,
 144                                                                                   StringRef SectionName)
 145 {
 146         uint8_t *mem = alloc_code (NULL, Size);
 147         PendingCodeMem.push_back (sys::MemoryBlock ((void *)mem, Size));
 148         return mem;
 149 }
 150
 151 bool
 152 MonoJitMemoryManager::finalizeMemory(std::string *ErrMsg)
 153 {
 154         for (sys::MemoryBlock &Block : PendingCodeMem) {
 155 #if LLVM_API_VERSION >= 900
 156                 sys::Memory::InvalidateInstructionCache (Block.base (), Block.allocatedSize ());
 157 #else
 158                 sys::Memory::InvalidateInstructionCache (Block.base (), Block.size ());
 159 #endif
 160         }
 161         PendingCodeMem.clear ();
 162         return false;
 163 }
 164
 165 #if defined(TARGET_AMD64) || defined(TARGET_X86)
 166 #define NO_CALL_FRAME_OPT " -no-x86-call-frame-opt"
 167 #else
 168 #define NO_CALL_FRAME_OPT ""
 169 #endif
 170
 171 // The OptimizationList is automatically populated with registered Passes by the
 172 // PassNameParser.
 173 //
 174 static cl::list<const PassInfo*, bool, PassNameParser>
 175 PassList(cl::desc("Optimizations available:"));
 176
 177 static void
 178 init_function_pass_manager (legacy::FunctionPassManager &fpm)
 179 {
 180         auto reg = PassRegistry::getPassRegistry ();
 181         for (size_t i = 0; i < PassList.size(); i++) {
 182                 Pass *pass = PassList[i]->getNormalCtor()();
 183                 if (pass->getPassKind () == llvm::PT_Function || pass->getPassKind () == llvm::PT_Loop) {
 184                         fpm.add (pass);
 185                 } else {
 186                         auto info = reg->getPassInfo (pass->getPassID());
 187                         auto name = info->getPassArgument ();
 188                         printf("Opt pass is ignored: %.*s\n", (int) name.size(), name.data());
 189                 }
 190         }
 191         // -place-safepoints pass is mandatory
 192         fpm.add (createPlaceSafepointsPass ());
 193
 194         fpm.doInitialization();
 195 }
 196
 197 #if LLVM_API_VERSION >= 900
 198
 199 struct MonoLLVMJIT {
 200         std::shared_ptr<MonoJitMemoryManager> mmgr;
 201         ExecutionSession execution_session;
 202         std::map<VModuleKey, std::shared_ptr<SymbolResolver>> resolvers;
 203         TargetMachine *target_machine;
 204         LegacyRTDyldObjectLinkingLayer object_layer;
 205         LegacyIRCompileLayer<decltype(object_layer), SimpleCompiler> compile_layer;
 206         DataLayout data_layout;
 207         legacy::FunctionPassManager fpm;
 208
 209         MonoLLVMJIT (TargetMachine *tm, Module *pgo_module)
 210                 : mmgr (std::make_shared<MonoJitMemoryManager>())
 211                 , target_machine (tm)
 212                 , object_layer (
 213                         AcknowledgeORCv1Deprecation, execution_session,
 214                         [this] (VModuleKey k) {
 215                                 return LegacyRTDyldObjectLinkingLayer::Resources{
 216                                         mmgr, resolvers[k] };
 217                         })
 218                 , compile_layer (
 219                         AcknowledgeORCv1Deprecation, object_layer,
 220                         SimpleCompiler{*target_machine})
 221                 , data_layout (target_machine->createDataLayout())
 222                 , fpm (pgo_module)
 223         {
 224                 compile_layer.setNotifyCompiled ([] (VModuleKey, std::unique_ptr<Module> module) {
 225                         module.release ();
 226                 });
 227                 init_function_pass_manager (fpm);
 228         }
 229
 230         VModuleKey
 231         add_module (std::unique_ptr<Module> m)
 232         {
 233                 auto k = execution_session.allocateVModule();
 234                 auto lookup_name = [this] (const std::string &namestr) {
 235                         auto jit_sym = compile_layer.findSymbol(namestr, false);
 236                         if (jit_sym) {
 237                                 return jit_sym;
 238                         }
 239                         auto namebuf = namestr.c_str();
 240                         JITSymbolFlags flags{};
 241                         if (!strcmp(namebuf, "___bzero")) {
 242                                 return JITSymbol{(uint64_t)(gssize)(void*)bzero, flags};
 243                         }
 244                         auto current = mono_dl_open (NULL, 0, NULL);
 245                         g_assert (current);
 246                         auto name = namebuf[0] == '_' ? namebuf + 1 : namebuf;
 247                         void *sym = nullptr;
 248                         auto err = mono_dl_symbol (current, name, &sym);
 249                         if (!sym) {
 250                                 outs () << "R: " << namestr << " " << err << "\n";
 251                         }
 252                         assert (sym);
 253                         return JITSymbol{(uint64_t)(gssize)sym, flags};
 254                 };
 255                 auto on_error = [] (Error err) {
 256                         outs () << "R2: " << err << "\n";
 257                         assert (0);
 258                 };
 259                 auto resolver = createLegacyLookupResolver (execution_session,
 260                         lookup_name, on_error);
 261                 resolvers[k] = std::move (resolver);
 262                 auto err = compile_layer.addModule (k, std::move(m));
 263                 if (err) {
 264                         outs () << "addModule error: " << err << "\n";
 265                         assert (0);
 266                 }
 267                 return k;
 268         }
 269
 270         std::string
 271         mangle (llvm::StringRef name)
 272         {
 273                 std::string ret;
 274                 raw_string_ostream out{ret};
 275                 Mangler::getNameWithPrefix (out, name, data_layout);
 276                 return ret;
 277         }
 278
 279         std::string
 280         mangle (const GlobalValue *gv)
 281         {
 282                 std::string ret;
 283                 raw_string_ostream out{ret};
 284                 Mangler{}.getNameWithPrefix (out, gv, false);
 285                 return ret;
 286         }
 287
 288         gpointer
 289         compile (
 290                 Function *func, int nvars, LLVMValueRef *callee_vars,
 291                 gpointer *callee_addrs, gpointer *eh_frame)
 292         {
 293                 auto module = func->getParent ();
 294                 module->setDataLayout (data_layout);
 295                 fpm.run (*func);
 296                 // The lifetime of this module is managed by Mono, not LLVM, so
 297                 // the `unique_ptr` created here will be released in the
 298                 // NotifyCompiled callback.
 299                 auto k = add_module (std::unique_ptr<Module>(module));
 300                 auto bodysym = compile_layer.findSymbolIn (k, mangle (func), false);
 301                 auto bodyaddr = bodysym.getAddress ();
 302                 assert (bodyaddr);
 303                 for (int i = 0; i < nvars; ++i) {
 304                         auto var = unwrap<GlobalVariable> (callee_vars[i]);
 305                         auto sym = compile_layer.findSymbolIn (k, mangle (var->getName ()), true);
 306                         auto addr = sym.getAddress ();
 307                         g_assert ((bool)addr);
 308                         callee_addrs[i] = (gpointer)addr.get ();
 309                 }
 310                 auto ehsym = compile_layer.findSymbolIn (k, "mono_eh_frame", false);
 311                 auto ehaddr = ehsym.getAddress ();
 312                 g_assert ((bool)ehaddr);
 313                 *eh_frame = (gpointer)ehaddr.get ();
 314                 return (gpointer)bodyaddr.get ();
 315         }
 316 };
 317
 318 static MonoLLVMJIT *
 319 make_mono_llvm_jit (TargetMachine *target_machine, llvm::Module *pgo_module)
 320 {
 321         return new MonoLLVMJIT{target_machine, pgo_module};
 322 }
 323
 324 #elif LLVM_API_VERSION > 600
 325
 326 class MonoLLVMJIT {
 327 public:
 328         /* We use our own trampoline infrastructure instead of the Orc one */
 329         typedef RTDyldObjectLinkingLayer ObjLayerT;
 330         typedef IRCompileLayer<ObjLayerT, SimpleCompiler> CompileLayerT;
 331         typedef CompileLayerT::ModuleHandleT ModuleHandleT;
 332
 333         MonoLLVMJIT (TargetMachine *TM, MonoJitMemoryManager *mm)
 334                 : TM(TM), ObjectLayer([=] { return std::shared_ptr<RuntimeDyld::MemoryManager> (mm); }),
 335                   CompileLayer (ObjectLayer, SimpleCompiler (*TM)),
 336                   modules(),
 337                   fpm (nullptr)
 338         {
 339                 init_function_pass_manager (fpm);
 340         }
 341
 342         ModuleHandleT addModule(Function *F, std::shared_ptr<Module> M) {
 343                 auto Resolver = createLambdaResolver(
 344                       [&](const std::string &Name) {
 345                                                   const char *name = Name.c_str ();
 346                                                   JITSymbolFlags flags = JITSymbolFlags ();
 347                                                   if (!strcmp (name, "___bzero"))
 348                                                           return JITSymbol((uint64_t)(gssize)(void*)bzero, flags);
 349
 350                                                   MonoDl *current;
 351                                                   char *err;
 352                                                   void *symbol;
 353                                                   current = mono_dl_open (NULL, 0, NULL);
 354                                                   g_assert (current);
 355                                                   if (name [0] == '_')
 356                                                           err = mono_dl_symbol (current, name + 1, &symbol);
 357                                                   else
 358                                                           err = mono_dl_symbol (current, name, &symbol);
 359                                                   mono_dl_close (current);
 360                                                   if (!symbol)
 361                                                           outs () << "R: " << Name << "\n";
 362                                                   assert (symbol);
 363                                                   return JITSymbol((uint64_t)(gssize)symbol, flags);
 364                       },
 365                       [](const std::string &S) {
 366                                                   outs () << "R2: " << S << "\n";
 367                                                   assert (0);
 368                                                   return nullptr;
 369                                           } );
 370
 371                 auto m = CompileLayer.addModule(M, std::move(Resolver));
 372                 g_assert (!!m);
 373                 return m.get ();
 374         }
 375
 376         std::string mangle(const std::string &Name) {
 377                 std::string MangledName;
 378                 {
 379                         raw_string_ostream MangledNameStream(MangledName);
 380                         Mangler::getNameWithPrefix(MangledNameStream, Name,
 381                                                                            TM->createDataLayout());
 382                 }
 383                 return MangledName;
 384         }
 385
 386         std::string mangle(const GlobalValue *GV) {
 387                 std::string MangledName;
 388                 {
 389                         Mangler Mang;
 390
 391                         raw_string_ostream MangledNameStream(MangledName);
 392                         Mang.getNameWithPrefix(MangledNameStream, GV, false);
 393                 }
 394                 return MangledName;
 395         }
 396
 397         gpointer compile (Function *F, int nvars, LLVMValueRef *callee_vars, gpointer *callee_addrs, gpointer *eh_frame) {
 398                 F->getParent ()->setDataLayout (TM->createDataLayout ());
 399                 fpm.run(*F);
 400                 // TODO: run module wide optimizations, e.g. remove dead globals/functions
 401                 // Orc uses a shared_ptr to refer to modules so we have to save them ourselves to keep a ref
 402                 std::shared_ptr<Module> m (F->getParent ());
 403                 modules.push_back (m);
 404                 auto ModuleHandle = addModule (F, m);
 405                 auto BodySym = CompileLayer.findSymbolIn(ModuleHandle, mangle (F), false);
 406                 auto BodyAddr = BodySym.getAddress();
 407                 assert (BodyAddr);
 408
 409                 for (int i = 0; i < nvars; ++i) {
 410                         GlobalVariable *var = unwrap<GlobalVariable>(callee_vars [i]);
 411
 412                         auto sym = CompileLayer.findSymbolIn (ModuleHandle, mangle (var->getName ()), true);
 413                         auto addr = sym.getAddress ();
 414                         g_assert ((bool)addr);
 415                         callee_addrs [i] = (gpointer)addr.get ();
 416                 }
 417
 418                 auto ehsym = CompileLayer.findSymbolIn(ModuleHandle, "mono_eh_frame", false);
 419                 auto ehaddr = ehsym.getAddress ();
 420                 g_assert ((bool)ehaddr);
 421                 *eh_frame = (gpointer)ehaddr.get ();
 422                 return (gpointer)BodyAddr.get ();
 423         }
 424
 425 private:
 426         TargetMachine *TM;
 427         ObjLayerT ObjectLayer;
 428         CompileLayerT CompileLayer;
 429         std::vector<std::shared_ptr<Module>> modules;
 430         legacy::FunctionPassManager fpm;
 431 };
 432
 433 static MonoJitMemoryManager *mono_mm;
 434
 435 static MonoLLVMJIT *
 436 make_mono_llvm_jit (TargetMachine *target_machine, llvm::Module *)
 437 {
 438         mono_mm = new MonoJitMemoryManager ();
 439         return new MonoLLVMJIT(target_machine, mono_mm);
 440 }
 441
 442 #endif
 443
 444 static llvm::Module *dummy_pgo_module = nullptr;
 445 static MonoLLVMJIT *jit;
 446
 447 static void
 448 init_passes_and_options ()
 449 {
 450         PassRegistry &registry = *PassRegistry::getPassRegistry();
 451         initializeCore(registry);
 452         initializeScalarOpts(registry);
 453         initializeInstCombine(registry);
 454         initializeTarget(registry);
 455         initializeLoopIdiomRecognizeLegacyPassPass(registry);
 456
 457         // FIXME: find optimal mono specific order of passes
 458         // see https://llvm.org/docs/Frontend/PerformanceTips.html#pass-ordering
 459         // the following order is based on a stripped version of "OPT -O2"
 460         const char *default_opts = " -simplifycfg -sroa -lower-expect -instcombine -sroa -jump-threading -loop-rotate -licm -simplifycfg -lcssa -loop-idiom -indvars -loop-deletion -gvn -memcpyopt -sccp -bdce -instcombine -dse -simplifycfg -enable-implicit-null-checks -sroa -instcombine" NO_CALL_FRAME_OPT;
 461         const char *opts = g_getenv ("MONO_LLVM_OPT");
 462         if (opts == NULL)
 463                 opts = default_opts;
 464         else if (opts[0] == '+') // Append passes to the default order if starts with '+', overwrite otherwise
 465                 opts = g_strdup_printf ("%s %s", default_opts, opts + 1);
 466         else if (opts[0] != ' ') // pass order has to start with a leading whitespace
 467                 opts = g_strdup_printf (" %s", opts);
 468
 469         char **args = g_strsplit (opts, " ", -1);
 470         llvm::cl::ParseCommandLineOptions (g_strv_length (args), args, "");
 471         g_strfreev (args);
 472 }
 473
 474 void
 475 mono_llvm_jit_init ()
 476 {
 477         if (jit != nullptr) return;
 478
 479         link_gc ();
 480
 481         mono_native_tls_alloc (&current_cfg_tls_id, NULL);
 482
 483         InitializeNativeTarget ();
 484         InitializeNativeTargetAsmPrinter();
 485
 486         EnableMonoEH = true;
 487         MonoEHFrameSymbol = "mono_eh_frame";
 488         EngineBuilder EB;
 489
 490         if (mono_use_fast_math) {
 491                 TargetOptions opts;
 492                 opts.NoInfsFPMath = true;
 493                 opts.NoNaNsFPMath = true;
 494                 opts.NoSignedZerosFPMath = true;
 495                 opts.NoTrappingFPMath = true;
 496                 opts.UnsafeFPMath = true;
 497                 opts.AllowFPOpFusion = FPOpFusion::Fast;
 498                 EB.setTargetOptions (opts);
 499         }
 500
 501         EB.setOptLevel (CodeGenOpt::Aggressive);
 502         EB.setMCPU (sys::getHostCPUName ());
 503
 504 #ifdef TARGET_AMD64
 505         EB.setMArch ("x86-64");
 506 #elif TARGET_X86
 507         EB.setMArch ("x86");
 508 #elif TARGET_ARM64
 509         EB.setMArch ("aarch64");
 510 #elif TARGET_ARM
 511         EB.setMArch ("arm");
 512 #else
 513         g_assert_not_reached ();
 514 #endif
 515
 516         auto TM = EB.selectTarget ();
 517         assert (TM);
 518         dummy_pgo_module = unwrap (LLVMModuleCreateWithName("dummy-pgo-module"));
 519         init_passes_and_options ();
 520         jit = make_mono_llvm_jit (TM, dummy_pgo_module);
 521 }
 522
 523 MonoEERef
 524 mono_llvm_create_ee (LLVMExecutionEngineRef *ee)
 525 {
 526         return NULL;
 527 }
 528
 529 /*
 530  * mono_llvm_compile_method:
 531  *
 532  *   Compile METHOD to native code. Compute the addresses of the variables in CALLEE_VARS and store them into
 533  * CALLEE_ADDRS. Return the EH frame address in EH_FRAME.
 534  */
 535 gpointer
 536 mono_llvm_compile_method (MonoEERef mono_ee, MonoCompile *cfg, LLVMValueRef method, int nvars, LLVMValueRef *callee_vars, gpointer *callee_addrs, gpointer *eh_frame)
 537 {
 538         mono_native_tls_set_value (current_cfg_tls_id, cfg);
 539         auto ret = jit->compile (unwrap<Function> (method), nvars, callee_vars, callee_addrs, eh_frame);
 540         mono_native_tls_set_value (current_cfg_tls_id, nullptr);
 541         return ret;
 542 }
 543
 544 void
 545 mono_llvm_dispose_ee (MonoEERef *eeref)
 546 {
 547 }
 548
 549 #else /* MONO_CROSS_COMPILE or LLVM_API_VERSION < 600 */
 550
 551 void
 552 mono_llvm_set_unhandled_exception_handler (void)
 553 {
 554 }
 555
 556 void
 557 mono_llvm_jit_init ()
 558 {
 559 }
 560
 561 MonoEERef
 562 mono_llvm_create_ee (LLVMExecutionEngineRef *ee)
 563 {
 564         g_error ("LLVM JIT not supported on this platform.");
 565         return NULL;
 566 }
 567
 568 gpointer
 569 mono_llvm_compile_method (MonoEERef mono_ee, MonoCompile *cfg, LLVMValueRef method, int nvars, LLVMValueRef *callee_vars, gpointer *callee_addrs, gpointer *eh_frame)
 570 {
 571         g_assert_not_reached ();
 572         return NULL;
 573 }
 574
 575 void
 576 mono_llvm_dispose_ee (MonoEERef *eeref)
 577 {
 578         g_assert_not_reached ();
 579 }
 580
 581 #endif /* !MONO_CROSS_COMPILE */