lib/Support/RegisterPasses.cpp

   1 //===------ RegisterPasses.cpp - Add the Polly Passes to default passes  --===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file composes the individual LLVM-IR passes provided by Polly to a
  11 // functional polyhedral optimizer. The polyhedral optimizer is automatically
  12 // made available to LLVM based compilers by loading the Polly shared library
  13 // into such a compiler.
  14 //
  15 // The Polly optimizer is made available by executing a static constructor that
  16 // registers the individual Polly passes in the LLVM pass manager builder. The
  17 // passes are registered such that the default behaviour of the compiler is not
  18 // changed, but that the flag '-polly' provided at optimization level '-O3'
  19 // enables additional polyhedral optimizations.
  20 //===----------------------------------------------------------------------===//
  21
  22 #include "polly/RegisterPasses.h"
  23 #include "polly/Canonicalization.h"
  24 #include "polly/CodeGen/CodeGeneration.h"
  25 #include "polly/CodeGen/CodegenCleanup.h"
  26 #include "polly/CodeGen/IslAst.h"
  27 #include "polly/CodeGen/PPCGCodeGeneration.h"
  28 #include "polly/CodePreparation.h"
  29 #include "polly/DeLICM.h"
  30 #include "polly/DependenceInfo.h"
  31 #include "polly/FlattenSchedule.h"
  32 #include "polly/ForwardOpTree.h"
  33 #include "polly/LinkAllPasses.h"
  34 #include "polly/Options.h"
  35 #include "polly/PolyhedralInfo.h"
  36 #include "polly/ScopDetection.h"
  37 #include "polly/ScopInfo.h"
  38 #include "polly/Simplify.h"
  39 #include "polly/Support/DumpModulePass.h"
  40 #include "llvm/Analysis/CFGPrinter.h"
  41 #include "llvm/IR/LegacyPassManager.h"
  42 #include "llvm/IR/Verifier.h"
  43 #include "llvm/Passes/PassBuilder.h"
  44 #include "llvm/Support/TargetSelect.h"
  45 #include "llvm/Transforms/IPO.h"
  46 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
  47 #include "llvm/Transforms/Scalar.h"
  48 #include "llvm/Transforms/Vectorize.h"
  49
  50 using namespace llvm;
  51 using namespace polly;
  52
  53 cl::OptionCategory PollyCategory("Polly Options",
  54                                  "Configure the polly loop optimizer");
  55
  56 static cl::opt<bool>
  57     PollyEnabled("polly", cl::desc("Enable the polly optimizer (only at -O3)"),
  58                  cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
  59
  60 static cl::opt<bool> PollyDetectOnly(
  61     "polly-only-scop-detection",
  62     cl::desc("Only run scop detection, but no other optimizations"),
  63     cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
  64
  65 enum PassPositionChoice {
  66   POSITION_EARLY,
  67   POSITION_AFTER_LOOPOPT,
  68   POSITION_BEFORE_VECTORIZER
  69 };
  70
  71 enum OptimizerChoice { OPTIMIZER_NONE, OPTIMIZER_ISL };
  72
  73 static cl::opt<PassPositionChoice> PassPosition(
  74     "polly-position", cl::desc("Where to run polly in the pass pipeline"),
  75     cl::values(
  76         clEnumValN(POSITION_EARLY, "early", "Before everything"),
  77         clEnumValN(POSITION_AFTER_LOOPOPT, "after-loopopt",
  78                    "After the loop optimizer (but within the inline cycle)"),
  79         clEnumValN(POSITION_BEFORE_VECTORIZER, "before-vectorizer",
  80                    "Right before the vectorizer")),
  81     cl::Hidden, cl::init(POSITION_EARLY), cl::ZeroOrMore,
  82     cl::cat(PollyCategory));
  83
  84 static cl::opt<OptimizerChoice>
  85     Optimizer("polly-optimizer", cl::desc("Select the scheduling optimizer"),
  86               cl::values(clEnumValN(OPTIMIZER_NONE, "none", "No optimizer"),
  87                          clEnumValN(OPTIMIZER_ISL, "isl",
  88                                     "The isl scheduling optimizer")),
  89               cl::Hidden, cl::init(OPTIMIZER_ISL), cl::ZeroOrMore,
  90               cl::cat(PollyCategory));
  91
  92 enum CodeGenChoice { CODEGEN_FULL, CODEGEN_AST, CODEGEN_NONE };
  93 static cl::opt<CodeGenChoice> CodeGeneration(
  94     "polly-code-generation", cl::desc("How much code-generation to perform"),
  95     cl::values(clEnumValN(CODEGEN_FULL, "full", "AST and IR generation"),
  96                clEnumValN(CODEGEN_AST, "ast", "Only AST generation"),
  97                clEnumValN(CODEGEN_NONE, "none", "No code generation")),
  98     cl::Hidden, cl::init(CODEGEN_FULL), cl::ZeroOrMore, cl::cat(PollyCategory));
  99
 100 enum TargetChoice { TARGET_CPU, TARGET_GPU, TARGET_HYBRID };
 101 static cl::opt<TargetChoice>
 102     Target("polly-target", cl::desc("The hardware to target"),
 103            cl::values(clEnumValN(TARGET_CPU, "cpu", "generate CPU code")
 104 #ifdef GPU_CODEGEN
 105                           ,
 106                       clEnumValN(TARGET_GPU, "gpu", "generate GPU code"),
 107                       clEnumValN(TARGET_HYBRID, "hybrid",
 108                                  "generate GPU code (preferably) or CPU code")
 109 #endif
 110                           ),
 111            cl::init(TARGET_CPU), cl::ZeroOrMore, cl::cat(PollyCategory));
 112
 113 #ifdef GPU_CODEGEN
 114 static cl::opt<GPURuntime> GPURuntimeChoice(
 115     "polly-gpu-runtime", cl::desc("The GPU Runtime API to target"),
 116     cl::values(clEnumValN(GPURuntime::CUDA, "libcudart",
 117                           "use the CUDA Runtime API"),
 118                clEnumValN(GPURuntime::OpenCL, "libopencl",
 119                           "use the OpenCL Runtime API")),
 120     cl::init(GPURuntime::CUDA), cl::ZeroOrMore, cl::cat(PollyCategory));
 121
 122 static cl::opt<GPUArch>
 123     GPUArchChoice("polly-gpu-arch", cl::desc("The GPU Architecture to target"),
 124                   cl::values(clEnumValN(GPUArch::NVPTX64, "nvptx64",
 125                                         "target NVIDIA 64-bit architecture"),
 126                              clEnumValN(GPUArch::SPIR32, "spir32",
 127                                         "target SPIR 32-bit architecture"),
 128                              clEnumValN(GPUArch::SPIR64, "spir64",
 129                                         "target SPIR 64-bit architecture")),
 130                   cl::init(GPUArch::NVPTX64), cl::ZeroOrMore,
 131                   cl::cat(PollyCategory));
 132 #endif
 133
 134 VectorizerChoice polly::PollyVectorizerChoice;
 135 static cl::opt<polly::VectorizerChoice, true> Vectorizer(
 136     "polly-vectorizer", cl::desc("Select the vectorization strategy"),
 137     cl::values(
 138         clEnumValN(polly::VECTORIZER_NONE, "none", "No Vectorization"),
 139         clEnumValN(polly::VECTORIZER_POLLY, "polly",
 140                    "Polly internal vectorizer"),
 141         clEnumValN(
 142             polly::VECTORIZER_STRIPMINE, "stripmine",
 143             "Strip-mine outer loops for the loop-vectorizer to trigger")),
 144     cl::location(PollyVectorizerChoice), cl::init(polly::VECTORIZER_NONE),
 145     cl::ZeroOrMore, cl::cat(PollyCategory));
 146
 147 static cl::opt<bool> ImportJScop(
 148     "polly-import",
 149     cl::desc("Import the polyhedral description of the detected Scops"),
 150     cl::Hidden, cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
 151
 152 static cl::opt<bool> ExportJScop(
 153     "polly-export",
 154     cl::desc("Export the polyhedral description of the detected Scops"),
 155     cl::Hidden, cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
 156
 157 static cl::opt<bool> DeadCodeElim("polly-run-dce",
 158                                   cl::desc("Run the dead code elimination"),
 159                                   cl::Hidden, cl::init(false), cl::ZeroOrMore,
 160                                   cl::cat(PollyCategory));
 161
 162 static cl::opt<bool> PollyViewer(
 163     "polly-show",
 164     cl::desc("Highlight the code regions that will be optimized in a "
 165              "(CFG BBs and LLVM-IR instructions)"),
 166     cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
 167
 168 static cl::opt<bool> PollyOnlyViewer(
 169     "polly-show-only",
 170     cl::desc("Highlight the code regions that will be optimized in "
 171              "a (CFG only BBs)"),
 172     cl::init(false), cl::cat(PollyCategory));
 173
 174 static cl::opt<bool>
 175     PollyPrinter("polly-dot", cl::desc("Enable the Polly DOT printer in -O3"),
 176                  cl::Hidden, cl::value_desc("Run the Polly DOT printer at -O3"),
 177                  cl::init(false), cl::cat(PollyCategory));
 178
 179 static cl::opt<bool> PollyOnlyPrinter(
 180     "polly-dot-only",
 181     cl::desc("Enable the Polly DOT printer in -O3 (no BB content)"), cl::Hidden,
 182     cl::value_desc("Run the Polly DOT printer at -O3 (no BB content"),
 183     cl::init(false), cl::cat(PollyCategory));
 184
 185 static cl::opt<bool>
 186     CFGPrinter("polly-view-cfg",
 187                cl::desc("Show the Polly CFG right after code generation"),
 188                cl::Hidden, cl::init(false), cl::cat(PollyCategory));
 189
 190 static cl::opt<bool>
 191     EnablePolyhedralInfo("polly-enable-polyhedralinfo",
 192                          cl::desc("Enable polyhedral interface of Polly"),
 193                          cl::Hidden, cl::init(false), cl::cat(PollyCategory));
 194
 195 static cl::opt<bool>
 196     EnableForwardOpTree("polly-enable-optree",
 197                         cl::desc("Enable operand tree forwarding"), cl::Hidden,
 198                         cl::init(false), cl::cat(PollyCategory));
 199
 200 static cl::opt<bool>
 201     DumpBefore("polly-dump-before",
 202                cl::desc("Dump module before Polly transformations into a file "
 203                         "suffixed with \"-before\""),
 204                cl::init(false), cl::cat(PollyCategory));
 205
 206 static cl::list<std::string> DumpBeforeFile(
 207     "polly-dump-before-file",
 208     cl::desc("Dump module before Polly transformations to the given file"),
 209     cl::cat(PollyCategory));
 210
 211 static cl::opt<bool>
 212     DumpAfter("polly-dump-after",
 213               cl::desc("Dump module after Polly transformations into a file "
 214                        "suffixed with \"-after\""),
 215               cl::init(false), cl::cat(PollyCategory));
 216
 217 static cl::list<std::string> DumpAfterFile(
 218     "polly-dump-after-file",
 219     cl::desc("Dump module after Polly transformations to the given file"),
 220     cl::ZeroOrMore, cl::cat(PollyCategory));
 221
 222 static cl::opt<bool>
 223     EnableDeLICM("polly-enable-delicm",
 224                  cl::desc("Eliminate scalar loop carried dependences"),
 225                  cl::Hidden, cl::init(false), cl::cat(PollyCategory));
 226
 227 static cl::opt<bool>
 228     EnableSimplify("polly-enable-simplify",
 229                    cl::desc("Simplify SCoP after optimizations"),
 230                    cl::init(false), cl::cat(PollyCategory));
 231
 232 static cl::opt<bool> EnablePruneUnprofitable(
 233     "polly-enable-prune-unprofitable",
 234     cl::desc("Bail out on unprofitable SCoPs before rescheduling"), cl::Hidden,
 235     cl::init(true), cl::cat(PollyCategory));
 236
 237 namespace polly {
 238 void initializePollyPasses(PassRegistry &Registry) {
 239   initializeCodeGenerationPass(Registry);
 240
 241 #ifdef GPU_CODEGEN
 242   initializePPCGCodeGenerationPass(Registry);
 243   LLVMInitializeNVPTXTarget();
 244   LLVMInitializeNVPTXTargetInfo();
 245   LLVMInitializeNVPTXTargetMC();
 246   LLVMInitializeNVPTXAsmPrinter();
 247 #endif
 248   initializeCodePreparationPass(Registry);
 249   initializeDeadCodeElimPass(Registry);
 250   initializeDependenceInfoPass(Registry);
 251   initializeDependenceInfoWrapperPassPass(Registry);
 252   initializeJSONExporterPass(Registry);
 253   initializeJSONImporterPass(Registry);
 254   initializeIslAstInfoWrapperPassPass(Registry);
 255   initializeIslScheduleOptimizerPass(Registry);
 256   initializePollyCanonicalizePass(Registry);
 257   initializePolyhedralInfoPass(Registry);
 258   initializeScopDetectionWrapperPassPass(Registry);
 259   initializeScopInfoRegionPassPass(Registry);
 260   initializeScopInfoWrapperPassPass(Registry);
 261   initializeCodegenCleanupPass(Registry);
 262   initializeFlattenSchedulePass(Registry);
 263   initializeForwardOpTreePass(Registry);
 264   initializeDeLICMPass(Registry);
 265   initializeSimplifyPass(Registry);
 266   initializeDumpModulePass(Registry);
 267   initializePruneUnprofitablePass(Registry);
 268 }
 269
 270 /// Register Polly passes such that they form a polyhedral optimizer.
 271 ///
 272 /// The individual Polly passes are registered in the pass manager such that
 273 /// they form a full polyhedral optimizer. The flow of the optimizer starts with
 274 /// a set of preparing transformations that canonicalize the LLVM-IR such that
 275 /// the LLVM-IR is easier for us to understand and to optimizes. On the
 276 /// canonicalized LLVM-IR we first run the ScopDetection pass, which detects
 277 /// static control flow regions. Those regions are then translated by the
 278 /// ScopInfo pass into a polyhedral representation. As a next step, a scheduling
 279 /// optimizer is run on the polyhedral representation and finally the optimized
 280 /// polyhedral representation is code generated back to LLVM-IR.
 281 ///
 282 /// Besides this core functionality, we optionally schedule passes that provide
 283 /// a graphical view of the scops (Polly[Only]Viewer, Polly[Only]Printer), that
 284 /// allow the export/import of the polyhedral representation
 285 /// (JSCON[Exporter|Importer]) or that show the cfg after code generation.
 286 ///
 287 /// For certain parts of the Polly optimizer, several alternatives are provided:
 288 ///
 289 /// As scheduling optimizer we support the isl scheduling optimizer
 290 /// (http://freecode.com/projects/isl).
 291 /// It is also possible to run Polly with no optimizer. This mode is mainly
 292 /// provided to analyze the run and compile time changes caused by the
 293 /// scheduling optimizer.
 294 ///
 295 /// Polly supports the isl internal code generator.
 296 void registerPollyPasses(llvm::legacy::PassManagerBase &PM) {
 297   if (DumpBefore)
 298     PM.add(polly::createDumpModulePass("-before", true));
 299   for (auto &Filename : DumpBeforeFile)
 300     PM.add(polly::createDumpModulePass(Filename, false));
 301
 302   PM.add(polly::createScopDetectionWrapperPassPass());
 303
 304   if (PollyDetectOnly)
 305     return;
 306
 307   if (PollyViewer)
 308     PM.add(polly::createDOTViewerPass());
 309   if (PollyOnlyViewer)
 310     PM.add(polly::createDOTOnlyViewerPass());
 311   if (PollyPrinter)
 312     PM.add(polly::createDOTPrinterPass());
 313   if (PollyOnlyPrinter)
 314     PM.add(polly::createDOTOnlyPrinterPass());
 315
 316   PM.add(polly::createScopInfoRegionPassPass());
 317   if (EnablePolyhedralInfo)
 318     PM.add(polly::createPolyhedralInfoPass());
 319
 320   if (EnableForwardOpTree)
 321     PM.add(polly::createForwardOpTreePass());
 322   if (EnableDeLICM)
 323     PM.add(polly::createDeLICMPass());
 324   if (EnableSimplify)
 325     PM.add(polly::createSimplifyPass());
 326
 327   if (ImportJScop)
 328     PM.add(polly::createJSONImporterPass());
 329
 330   if (DeadCodeElim)
 331     PM.add(polly::createDeadCodeElimPass());
 332
 333   if (EnablePruneUnprofitable)
 334     PM.add(polly::createPruneUnprofitablePass());
 335
 336 #ifdef GPU_CODEGEN
 337   if (Target == TARGET_HYBRID)
 338     PM.add(
 339         polly::createPPCGCodeGenerationPass(GPUArchChoice, GPURuntimeChoice));
 340 #endif
 341   if (Target == TARGET_CPU || Target == TARGET_HYBRID)
 342     switch (Optimizer) {
 343     case OPTIMIZER_NONE:
 344       break; /* Do nothing */
 345
 346     case OPTIMIZER_ISL:
 347       PM.add(polly::createIslScheduleOptimizerPass());
 348       break;
 349     }
 350
 351   if (ExportJScop)
 352     PM.add(polly::createJSONExporterPass());
 353
 354   if (Target == TARGET_CPU || Target == TARGET_HYBRID)
 355     switch (CodeGeneration) {
 356     case CODEGEN_AST:
 357       PM.add(polly::createIslAstInfoWrapperPassPass());
 358       break;
 359     case CODEGEN_FULL:
 360       PM.add(polly::createCodeGenerationPass());
 361       break;
 362     case CODEGEN_NONE:
 363       break;
 364     }
 365 #ifdef GPU_CODEGEN
 366   else
 367     PM.add(
 368         polly::createPPCGCodeGenerationPass(GPUArchChoice, GPURuntimeChoice));
 369 #endif
 370
 371   // FIXME: This dummy ModulePass keeps some programs from miscompiling,
 372   // probably some not correctly preserved analyses. It acts as a barrier to
 373   // force all analysis results to be recomputed.
 374   PM.add(createBarrierNoopPass());
 375
 376   if (DumpAfter)
 377     PM.add(polly::createDumpModulePass("-after", true));
 378   for (auto &Filename : DumpAfterFile)
 379     PM.add(polly::createDumpModulePass(Filename, false));
 380
 381   if (CFGPrinter)
 382     PM.add(llvm::createCFGPrinterLegacyPassPass());
 383 }
 384
 385 static bool shouldEnablePolly() {
 386   if (PollyOnlyPrinter || PollyPrinter || PollyOnlyViewer || PollyViewer)
 387     PollyTrackFailures = true;
 388
 389   if (PollyOnlyPrinter || PollyPrinter || PollyOnlyViewer || PollyViewer ||
 390       ExportJScop || ImportJScop)
 391     PollyEnabled = true;
 392
 393   return PollyEnabled;
 394 }
 395
 396 static void
 397 registerPollyEarlyAsPossiblePasses(const llvm::PassManagerBuilder &Builder,
 398                                    llvm::legacy::PassManagerBase &PM) {
 399   if (!polly::shouldEnablePolly())
 400     return;
 401
 402   if (PassPosition != POSITION_EARLY)
 403     return;
 404
 405   registerCanonicalicationPasses(PM);
 406   polly::registerPollyPasses(PM);
 407 }
 408
 409 static void
 410 registerPollyLoopOptimizerEndPasses(const llvm::PassManagerBuilder &Builder,
 411                                     llvm::legacy::PassManagerBase &PM) {
 412   if (!polly::shouldEnablePolly())
 413     return;
 414
 415   if (PassPosition != POSITION_AFTER_LOOPOPT)
 416     return;
 417
 418   PM.add(polly::createCodePreparationPass());
 419   polly::registerPollyPasses(PM);
 420   PM.add(createCodegenCleanupPass());
 421 }
 422
 423 static void
 424 registerPollyScalarOptimizerLatePasses(const llvm::PassManagerBuilder &Builder,
 425                                        llvm::legacy::PassManagerBase &PM) {
 426   if (!polly::shouldEnablePolly())
 427     return;
 428
 429   if (PassPosition != POSITION_BEFORE_VECTORIZER)
 430     return;
 431
 432   PM.add(polly::createCodePreparationPass());
 433   polly::registerPollyPasses(PM);
 434   PM.add(createCodegenCleanupPass());
 435 }
 436
 437 static void buildDefaultPollyPipeline(FunctionPassManager &PM,
 438                                       PassBuilder::OptimizationLevel Level) {
 439   if (!polly::shouldEnablePolly())
 440     return;
 441   PassBuilder PB;
 442   ScopPassManager SPM;
 443
 444   // TODO add utility passes for the various command line options, once they're
 445   // ported
 446   assert(!DumpBefore && "This option is not implemented");
 447   assert(DumpBeforeFile.empty() && "This option is not implemented");
 448
 449   if (PollyDetectOnly)
 450     return;
 451
 452   assert(!PollyViewer && "This option is not implemented");
 453   assert(!PollyOnlyViewer && "This option is not implemented");
 454   assert(!PollyPrinter && "This option is not implemented");
 455   assert(!PollyOnlyPrinter && "This option is not implemented");
 456   assert(!EnablePolyhedralInfo && "This option is not implemented");
 457   assert(!EnableDeLICM && "This option is not implemented");
 458   assert(!EnableSimplify && "This option is not implemented");
 459   assert(!ImportJScop && "This option is not implemented");
 460   assert(!DeadCodeElim && "This option is not implemented");
 461   assert(!EnablePruneUnprofitable && "This option is not implemented");
 462   if (Target == TARGET_CPU || Target == TARGET_HYBRID)
 463     switch (Optimizer) {
 464     case OPTIMIZER_NONE:
 465       break; /* Do nothing */
 466     case OPTIMIZER_ISL:
 467       assert("ISL optimizer is not implemented");
 468       break;
 469     }
 470
 471   assert(!ExportJScop && "This option is not implemented");
 472
 473   if (Target == TARGET_CPU || Target == TARGET_HYBRID) {
 474     switch (CodeGeneration) {
 475     case CODEGEN_FULL:
 476       SPM.addPass(polly::CodeGenerationPass());
 477       break;
 478     case CODEGEN_AST:
 479     default: // Does it actually make sense to distinguish IslAst codegen?
 480       break;
 481     }
 482   }
 483 #ifdef GPU_CODEGEN
 484   else
 485     assert("Hybrid Target with GPU support is not implemented");
 486 #endif
 487
 488   PM.addPass(CodePreparationPass());
 489   PM.addPass(createFunctionToScopPassAdaptor(std::move(SPM)));
 490   PM.addPass(PB.buildFunctionSimplificationPipeline(
 491       Level, PassBuilder::ThinLTOPhase::None)); // Cleanup
 492
 493   assert(!DumpAfter && "This option is not implemented");
 494   assert(DumpAfterFile.empty() && "This option is not implemented");
 495
 496   if (CFGPrinter)
 497     PM.addPass(llvm::CFGPrinterPass());
 498 }
 499
 500 /// Register Polly to be available as an optimizer
 501 ///
 502 ///
 503 /// We can currently run Polly at three different points int the pass manager.
 504 /// a) very early, b) after the canonicalizing loop transformations and c) right
 505 /// before the vectorizer.
 506 ///
 507 /// The default is currently a), to register Polly such that it runs as early as
 508 /// possible. This has several implications:
 509 ///
 510 ///   1) We need to schedule more canonicalization passes
 511 ///
 512 ///   As nothing is run before Polly, it is necessary to run a set of preparing
 513 ///   transformations before Polly to canonicalize the LLVM-IR and to allow
 514 ///   Polly to detect and understand the code.
 515 ///
 516 ///   2) LICM and LoopIdiom pass have not yet been run
 517 ///
 518 ///   Loop invariant code motion as well as the loop idiom recognition pass make
 519 ///   it more difficult for Polly to transform code. LICM may introduce
 520 ///   additional data dependences that are hard to eliminate and the loop idiom
 521 ///   recognition pass may introduce calls to memset that we currently do not
 522 ///   understand. By running Polly early enough (meaning before these passes) we
 523 ///   avoid difficulties that may be introduced by these passes.
 524 ///
 525 ///   3) We get the full -O3 optimization sequence after Polly
 526 ///
 527 ///   The LLVM-IR that is generated by Polly has been optimized on a high level,
 528 ///   but it may be rather inefficient on the lower/scalar level. By scheduling
 529 ///   Polly before all other passes, we have the full sequence of -O3
 530 ///   optimizations behind us, such that inefficiencies on the low level can
 531 ///   be optimized away.
 532 ///
 533 /// We are currently evaluating the benefit or running Polly at position b) or
 534 /// c). b) is likely too early as it interacts with the inliner. c) is nice
 535 /// as everything is fully inlined and canonicalized, but we need to be able
 536 /// to handle LICMed code to make it useful.
 537 static llvm::RegisterStandardPasses RegisterPollyOptimizerEarly(
 538     llvm::PassManagerBuilder::EP_ModuleOptimizerEarly,
 539     registerPollyEarlyAsPossiblePasses);
 540
 541 static llvm::RegisterStandardPasses
 542     RegisterPollyOptimizerLoopEnd(llvm::PassManagerBuilder::EP_LoopOptimizerEnd,
 543                                   registerPollyLoopOptimizerEndPasses);
 544
 545 static llvm::RegisterStandardPasses RegisterPollyOptimizerScalarLate(
 546     llvm::PassManagerBuilder::EP_VectorizerStart,
 547     registerPollyScalarOptimizerLatePasses);
 548
 549 static OwningScopAnalysisManagerFunctionProxy
 550 createScopAnalyses(FunctionAnalysisManager &FAM) {
 551   OwningScopAnalysisManagerFunctionProxy Proxy;
 552 #define SCOP_ANALYSIS(NAME, CREATE_PASS)                                       \
 553   Proxy.getManager().registerPass([] { return CREATE_PASS; });
 554
 555 #include "PollyPasses.def"
 556
 557   Proxy.getManager().registerPass(
 558       [&FAM] { return FunctionAnalysisManagerScopProxy(FAM); });
 559   return Proxy;
 560 }
 561
 562 static void registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
 563 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
 564   FAM.registerPass([] { return CREATE_PASS; });
 565
 566 #include "PollyPasses.def"
 567
 568   FAM.registerPass([&FAM] { return createScopAnalyses(FAM); });
 569 }
 570
 571 static bool
 572 parseFunctionPipeline(StringRef Name, FunctionPassManager &FPM,
 573                       ArrayRef<PassBuilder::PipelineElement> Pipeline) {
 574   if (parseAnalysisUtilityPasses<OwningScopAnalysisManagerFunctionProxy>(
 575           "polly-scop-analyses", Name, FPM))
 576     return true;
 577
 578 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
 579   if (parseAnalysisUtilityPasses<                                              \
 580           std::remove_reference<decltype(CREATE_PASS)>::type>(NAME, Name,      \
 581                                                               FPM))            \
 582     return true;
 583
 584 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
 585   if (Name == NAME) {                                                          \
 586     FPM.addPass(CREATE_PASS);                                                  \
 587     return true;                                                               \
 588   }
 589
 590 #include "PollyPasses.def"
 591   return false;
 592 }
 593
 594 static bool parseScopPass(StringRef Name, ScopPassManager &SPM) {
 595 #define SCOP_ANALYSIS(NAME, CREATE_PASS)                                       \
 596   if (parseAnalysisUtilityPasses<                                              \
 597           std::remove_reference<decltype(CREATE_PASS)>::type>(NAME, Name,      \
 598                                                               SPM))            \
 599     return true;
 600
 601 #define SCOP_PASS(NAME, CREATE_PASS)                                           \
 602   if (Name == NAME) {                                                          \
 603     SPM.addPass(CREATE_PASS);                                                  \
 604     return true;                                                               \
 605   }
 606
 607 #include "PollyPasses.def"
 608
 609   return false;
 610 }
 611
 612 static bool parseScopPipeline(StringRef Name, FunctionPassManager &FPM,
 613                               ArrayRef<PassBuilder::PipelineElement> Pipeline) {
 614   if (Name != "scop")
 615     return false;
 616   if (!Pipeline.empty()) {
 617     ScopPassManager SPM;
 618     for (const auto &E : Pipeline)
 619       if (!parseScopPass(E.Name, SPM))
 620         return false;
 621     FPM.addPass(createFunctionToScopPassAdaptor(std::move(SPM)));
 622   }
 623   return true;
 624 }
 625
 626 static bool isScopPassName(StringRef Name) {
 627 #define SCOP_ANALYSIS(NAME, CREATE_PASS)                                       \
 628   if (Name == "require<" NAME ">")                                             \
 629     return true;                                                               \
 630   if (Name == "invalidate<" NAME ">")                                          \
 631     return true;
 632
 633 #define SCOP_PASS(NAME, CREATE_PASS)                                           \
 634   if (Name == NAME)                                                            \
 635     return true;
 636
 637 #include "PollyPasses.def"
 638
 639   return false;
 640 }
 641
 642 static bool
 643 parseTopLevelPipeline(ModulePassManager &MPM,
 644                       ArrayRef<PassBuilder::PipelineElement> Pipeline,
 645                       bool VerifyEachPass, bool DebugLogging) {
 646   std::vector<PassBuilder::PipelineElement> FullPipeline;
 647   StringRef FirstName = Pipeline.front().Name;
 648
 649   if (!isScopPassName(FirstName))
 650     return false;
 651
 652   FunctionPassManager FPM(DebugLogging);
 653   ScopPassManager SPM(DebugLogging);
 654
 655   for (auto &Element : Pipeline) {
 656     auto &Name = Element.Name;
 657     auto &InnerPipeline = Element.InnerPipeline;
 658     if (!InnerPipeline.empty()) // Scop passes don't have inner pipelines
 659       return false;
 660     if (!parseScopPass(Name, SPM))
 661       return false;
 662   }
 663
 664   FPM.addPass(createFunctionToScopPassAdaptor(std::move(SPM)));
 665   if (VerifyEachPass)
 666     FPM.addPass(VerifierPass());
 667   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 668   if (VerifyEachPass)
 669     MPM.addPass(VerifierPass());
 670
 671   return true;
 672 }
 673
 674 void RegisterPollyPasses(PassBuilder &PB) {
 675   PB.registerAnalysisRegistrationCallback(registerFunctionAnalyses);
 676   PB.registerPipelineParsingCallback(parseFunctionPipeline);
 677   PB.registerPipelineParsingCallback(parseScopPipeline);
 678   PB.registerParseTopLevelPipelineCallback(parseTopLevelPipeline);
 679
 680   if (PassPosition == POSITION_BEFORE_VECTORIZER)
 681     PB.registerVectorizerStartEPCallback(buildDefaultPollyPipeline);
 682   // FIXME else Error?
 683 }
 684 } // namespace polly