1 //===- Parsing, selection, and construction of pass pipelines -------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// This file provides the implementation of the PassBuilder based on our
11 /// static pass registry as well as related functionality. It also provides
12 /// helpers to aid in analyzing, debugging, and testing passes and pass
15 //===----------------------------------------------------------------------===//
17 #include "llvm/Passes/PassBuilder.h"
18 #include "llvm/ADT/StringSwitch.h"
19 #include "llvm/Analysis/AliasAnalysis.h"
20 #include "llvm/Analysis/AliasAnalysisEvaluator.h"
21 #include "llvm/Analysis/AssumptionCache.h"
22 #include "llvm/Analysis/BasicAliasAnalysis.h"
23 #include "llvm/Analysis/BlockFrequencyInfo.h"
24 #include "llvm/Analysis/BranchProbabilityInfo.h"
25 #include "llvm/Analysis/CFGPrinter.h"
26 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
27 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
28 #include "llvm/Analysis/CGSCCPassManager.h"
29 #include "llvm/Analysis/CallGraph.h"
30 #include "llvm/Analysis/DemandedBits.h"
31 #include "llvm/Analysis/DependenceAnalysis.h"
32 #include "llvm/Analysis/DominanceFrontier.h"
33 #include "llvm/Analysis/GlobalsModRef.h"
34 #include "llvm/Analysis/IVUsers.h"
35 #include "llvm/Analysis/LazyCallGraph.h"
36 #include "llvm/Analysis/LazyValueInfo.h"
37 #include "llvm/Analysis/LoopAccessAnalysis.h"
38 #include "llvm/Analysis/LoopCacheAnalysis.h"
39 #include "llvm/Analysis/LoopInfo.h"
40 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
41 #include "llvm/Analysis/MemorySSA.h"
42 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
43 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
44 #include "llvm/Analysis/PhiValues.h"
45 #include "llvm/Analysis/PostDominators.h"
46 #include "llvm/Analysis/ProfileSummaryInfo.h"
47 #include "llvm/Analysis/RegionInfo.h"
48 #include "llvm/Analysis/ScalarEvolution.h"
49 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
50 #include "llvm/Analysis/ScopedNoAliasAA.h"
51 #include "llvm/Analysis/StackSafetyAnalysis.h"
52 #include "llvm/Analysis/TargetLibraryInfo.h"
53 #include "llvm/Analysis/TargetTransformInfo.h"
54 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
55 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
56 #include "llvm/CodeGen/UnreachableBlockElim.h"
57 #include "llvm/IR/Dominators.h"
58 #include "llvm/IR/IRPrintingPasses.h"
59 #include "llvm/IR/PassManager.h"
60 #include "llvm/IR/SafepointIRVerifier.h"
61 #include "llvm/IR/Verifier.h"
62 #include "llvm/Support/Debug.h"
63 #include "llvm/Support/FormatVariadic.h"
64 #include "llvm/Support/Regex.h"
65 #include "llvm/Target/TargetMachine.h"
66 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
67 #include "llvm/Transforms/IPO/AlwaysInliner.h"
68 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
69 #include "llvm/Transforms/IPO/Attributor.h"
70 #include "llvm/Transforms/IPO/CalledValuePropagation.h"
71 #include "llvm/Transforms/IPO/ConstantMerge.h"
72 #include "llvm/Transforms/IPO/CrossDSOCFI.h"
73 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
74 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
75 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
76 #include "llvm/Transforms/IPO/FunctionAttrs.h"
77 #include "llvm/Transforms/IPO/FunctionImport.h"
78 #include "llvm/Transforms/IPO/GlobalDCE.h"
79 #include "llvm/Transforms/IPO/GlobalOpt.h"
80 #include "llvm/Transforms/IPO/GlobalSplit.h"
81 #include "llvm/Transforms/IPO/HotColdSplitting.h"
82 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
83 #include "llvm/Transforms/IPO/Inliner.h"
84 #include "llvm/Transforms/IPO/Internalize.h"
85 #include "llvm/Transforms/IPO/LowerTypeTests.h"
86 #include "llvm/Transforms/IPO/PartialInlining.h"
87 #include "llvm/Transforms/IPO/SCCP.h"
88 #include "llvm/Transforms/IPO/SampleProfile.h"
89 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
90 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
91 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
92 #include "llvm/Transforms/InstCombine/InstCombine.h"
93 #include "llvm/Transforms/Instrumentation.h"
94 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
95 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
96 #include "llvm/Transforms/Instrumentation/CGProfile.h"
97 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
98 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
99 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
100 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
101 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
102 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
103 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
104 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
105 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
106 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
107 #include "llvm/Transforms/Scalar/ADCE.h"
108 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
109 #include "llvm/Transforms/Scalar/BDCE.h"
110 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
111 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
112 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
113 #include "llvm/Transforms/Scalar/DCE.h"
114 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
115 #include "llvm/Transforms/Scalar/DivRemPairs.h"
116 #include "llvm/Transforms/Scalar/EarlyCSE.h"
117 #include "llvm/Transforms/Scalar/Float2Int.h"
118 #include "llvm/Transforms/Scalar/GVN.h"
119 #include "llvm/Transforms/Scalar/GuardWidening.h"
120 #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
121 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
122 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
123 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
124 #include "llvm/Transforms/Scalar/JumpThreading.h"
125 #include "llvm/Transforms/Scalar/LICM.h"
126 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
127 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
128 #include "llvm/Transforms/Scalar/LoopDeletion.h"
129 #include "llvm/Transforms/Scalar/LoopDistribute.h"
130 #include "llvm/Transforms/Scalar/LoopFuse.h"
131 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
132 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
133 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
134 #include "llvm/Transforms/Scalar/LoopPassManager.h"
135 #include "llvm/Transforms/Scalar/LoopPredication.h"
136 #include "llvm/Transforms/Scalar/LoopRotation.h"
137 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
138 #include "llvm/Transforms/Scalar/LoopSink.h"
139 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
140 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
141 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
142 #include "llvm/Transforms/Scalar/LowerAtomic.h"
143 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
144 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
145 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
146 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
147 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
148 #include "llvm/Transforms/Scalar/MergeICmps.h"
149 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
150 #include "llvm/Transforms/Scalar/NaryReassociate.h"
151 #include "llvm/Transforms/Scalar/NewGVN.h"
152 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
153 #include "llvm/Transforms/Scalar/Reassociate.h"
154 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
155 #include "llvm/Transforms/Scalar/SCCP.h"
156 #include "llvm/Transforms/Scalar/SROA.h"
157 #include "llvm/Transforms/Scalar/Scalarizer.h"
158 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
159 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
160 #include "llvm/Transforms/Scalar/Sink.h"
161 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
162 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
163 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
164 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
165 #include "llvm/Transforms/Utils/AddDiscriminators.h"
166 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
167 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
168 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
169 #include "llvm/Transforms/Utils/LCSSA.h"
170 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
171 #include "llvm/Transforms/Utils/LoopSimplify.h"
172 #include "llvm/Transforms/Utils/LowerInvoke.h"
173 #include "llvm/Transforms/Utils/Mem2Reg.h"
174 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
175 #include "llvm/Transforms/Utils/SymbolRewriter.h"
176 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
177 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
178 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
180 using namespace llvm
;
182 static cl::opt
<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
183 cl::ReallyHidden
, cl::init(4));
185 RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
186 cl::Hidden
, cl::ZeroOrMore
,
187 cl::desc("Run Partial inlinining pass"));
190 RunNewGVN("enable-npm-newgvn", cl::init(false),
191 cl::Hidden
, cl::ZeroOrMore
,
192 cl::desc("Run NewGVN instead of GVN"));
194 static cl::opt
<bool> EnableGVNHoist(
195 "enable-npm-gvn-hoist", cl::init(false), cl::Hidden
,
196 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
198 static cl::opt
<bool> EnableGVNSink(
199 "enable-npm-gvn-sink", cl::init(false), cl::Hidden
,
200 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
202 static cl::opt
<bool> EnableUnrollAndJam(
203 "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden
,
204 cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
206 static cl::opt
<bool> EnableSyntheticCounts(
207 "enable-npm-synthetic-counts", cl::init(false), cl::Hidden
, cl::ZeroOrMore
,
208 cl::desc("Run synthetic function entry count generation "
211 static Regex
DefaultAliasRegex(
212 "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
214 // This option is used in simplifying testing SampleFDO optimizations for
217 EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden
,
218 cl::desc("Enable control height reduction optimization (CHR)"));
220 PipelineTuningOptions::PipelineTuningOptions() {
221 LoopInterleaving
= EnableLoopInterleaving
;
222 LoopVectorization
= EnableLoopVectorization
;
223 SLPVectorization
= RunSLPVectorization
;
224 LoopUnrolling
= true;
225 ForgetAllSCEVInLoopUnroll
= ForgetSCEVInLoopUnroll
;
226 LicmMssaOptCap
= SetLicmMssaOptCap
;
227 LicmMssaNoAccForPromotionCap
= SetLicmMssaNoAccForPromotionCap
;
230 extern cl::opt
<bool> EnableHotColdSplit
;
231 extern cl::opt
<bool> EnableOrderFileInstrumentation
;
233 extern cl::opt
<bool> FlattenedProfileUsed
;
235 static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level
) {
237 case PassBuilder::O0
:
238 case PassBuilder::O1
:
239 case PassBuilder::O2
:
240 case PassBuilder::O3
:
243 case PassBuilder::Os
:
244 case PassBuilder::Oz
:
247 llvm_unreachable("Invalid optimization level!");
252 /// No-op module pass which does nothing.
253 struct NoOpModulePass
{
254 PreservedAnalyses
run(Module
&M
, ModuleAnalysisManager
&) {
255 return PreservedAnalyses::all();
257 static StringRef
name() { return "NoOpModulePass"; }
260 /// No-op module analysis.
261 class NoOpModuleAnalysis
: public AnalysisInfoMixin
<NoOpModuleAnalysis
> {
262 friend AnalysisInfoMixin
<NoOpModuleAnalysis
>;
263 static AnalysisKey Key
;
267 Result
run(Module
&, ModuleAnalysisManager
&) { return Result(); }
268 static StringRef
name() { return "NoOpModuleAnalysis"; }
271 /// No-op CGSCC pass which does nothing.
272 struct NoOpCGSCCPass
{
273 PreservedAnalyses
run(LazyCallGraph::SCC
&C
, CGSCCAnalysisManager
&,
274 LazyCallGraph
&, CGSCCUpdateResult
&UR
) {
275 return PreservedAnalyses::all();
277 static StringRef
name() { return "NoOpCGSCCPass"; }
280 /// No-op CGSCC analysis.
281 class NoOpCGSCCAnalysis
: public AnalysisInfoMixin
<NoOpCGSCCAnalysis
> {
282 friend AnalysisInfoMixin
<NoOpCGSCCAnalysis
>;
283 static AnalysisKey Key
;
287 Result
run(LazyCallGraph::SCC
&, CGSCCAnalysisManager
&, LazyCallGraph
&G
) {
290 static StringRef
name() { return "NoOpCGSCCAnalysis"; }
293 /// No-op function pass which does nothing.
294 struct NoOpFunctionPass
{
295 PreservedAnalyses
run(Function
&F
, FunctionAnalysisManager
&) {
296 return PreservedAnalyses::all();
298 static StringRef
name() { return "NoOpFunctionPass"; }
301 /// No-op function analysis.
302 class NoOpFunctionAnalysis
: public AnalysisInfoMixin
<NoOpFunctionAnalysis
> {
303 friend AnalysisInfoMixin
<NoOpFunctionAnalysis
>;
304 static AnalysisKey Key
;
308 Result
run(Function
&, FunctionAnalysisManager
&) { return Result(); }
309 static StringRef
name() { return "NoOpFunctionAnalysis"; }
312 /// No-op loop pass which does nothing.
313 struct NoOpLoopPass
{
314 PreservedAnalyses
run(Loop
&L
, LoopAnalysisManager
&,
315 LoopStandardAnalysisResults
&, LPMUpdater
&) {
316 return PreservedAnalyses::all();
318 static StringRef
name() { return "NoOpLoopPass"; }
321 /// No-op loop analysis.
322 class NoOpLoopAnalysis
: public AnalysisInfoMixin
<NoOpLoopAnalysis
> {
323 friend AnalysisInfoMixin
<NoOpLoopAnalysis
>;
324 static AnalysisKey Key
;
328 Result
run(Loop
&, LoopAnalysisManager
&, LoopStandardAnalysisResults
&) {
331 static StringRef
name() { return "NoOpLoopAnalysis"; }
334 AnalysisKey
NoOpModuleAnalysis::Key
;
335 AnalysisKey
NoOpCGSCCAnalysis::Key
;
336 AnalysisKey
NoOpFunctionAnalysis::Key
;
337 AnalysisKey
NoOpLoopAnalysis::Key
;
339 } // End anonymous namespace.
341 void PassBuilder::invokePeepholeEPCallbacks(
342 FunctionPassManager
&FPM
, PassBuilder::OptimizationLevel Level
) {
343 for (auto &C
: PeepholeEPCallbacks
)
347 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager
&MAM
) {
348 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
349 MAM.registerPass([&] { return CREATE_PASS; });
350 #include "PassRegistry.def"
352 for (auto &C
: ModuleAnalysisRegistrationCallbacks
)
356 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager
&CGAM
) {
357 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
358 CGAM.registerPass([&] { return CREATE_PASS; });
359 #include "PassRegistry.def"
361 for (auto &C
: CGSCCAnalysisRegistrationCallbacks
)
365 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager
&FAM
) {
366 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
367 FAM.registerPass([&] { return CREATE_PASS; });
368 #include "PassRegistry.def"
370 for (auto &C
: FunctionAnalysisRegistrationCallbacks
)
374 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager
&LAM
) {
375 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
376 LAM.registerPass([&] { return CREATE_PASS; });
377 #include "PassRegistry.def"
379 for (auto &C
: LoopAnalysisRegistrationCallbacks
)
384 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level
,
387 assert(Level
!= O0
&& "Must request optimizations!");
388 FunctionPassManager
FPM(DebugLogging
);
390 // Form SSA out of local memory accesses after breaking apart aggregates into
394 // Catch trivial redundancies
395 FPM
.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
397 // Hoisting of scalars and load expressions.
399 FPM
.addPass(GVNHoistPass());
401 // Global value numbering based sinking.
403 FPM
.addPass(GVNSinkPass());
404 FPM
.addPass(SimplifyCFGPass());
407 // Speculative execution if the target has divergent branches; otherwise nop.
408 FPM
.addPass(SpeculativeExecutionPass());
410 // Optimize based on known information about branches, and cleanup afterward.
411 FPM
.addPass(JumpThreadingPass());
412 FPM
.addPass(CorrelatedValuePropagationPass());
413 FPM
.addPass(SimplifyCFGPass());
415 FPM
.addPass(AggressiveInstCombinePass());
416 FPM
.addPass(InstCombinePass());
418 if (!isOptimizingForSize(Level
))
419 FPM
.addPass(LibCallsShrinkWrapPass());
421 invokePeepholeEPCallbacks(FPM
, Level
);
423 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
424 // using the size value profile. Don't perform this when optimizing for size.
425 if (PGOOpt
&& PGOOpt
->Action
== PGOOptions::IRUse
&&
426 !isOptimizingForSize(Level
))
427 FPM
.addPass(PGOMemOPSizeOpt());
429 FPM
.addPass(TailCallElimPass());
430 FPM
.addPass(SimplifyCFGPass());
432 // Form canonically associated expression trees, and simplify the trees using
433 // basic mathematical properties. For example, this will form (nearly)
434 // minimal multiplication trees.
435 FPM
.addPass(ReassociatePass());
437 // Add the primary loop simplification pipeline.
438 // FIXME: Currently this is split into two loop pass pipelines because we run
439 // some function passes in between them. These can and should be removed
440 // and/or replaced by scheduling the loop pass equivalents in the correct
441 // positions. But those equivalent passes aren't powerful enough yet.
442 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
443 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
444 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
445 // `LoopInstSimplify`.
446 LoopPassManager
LPM1(DebugLogging
), LPM2(DebugLogging
);
448 // Simplify the loop body. We do this initially to clean up after other loop
449 // passes run, either when iterating on a loop or on inner loops with
450 // implications on the outer loop.
451 LPM1
.addPass(LoopInstSimplifyPass());
452 LPM1
.addPass(LoopSimplifyCFGPass());
454 // Rotate Loop - disable header duplication at -Oz
455 LPM1
.addPass(LoopRotatePass(Level
!= Oz
));
456 LPM1
.addPass(LICMPass(PTO
.LicmMssaOptCap
, PTO
.LicmMssaNoAccForPromotionCap
));
457 LPM1
.addPass(SimpleLoopUnswitchPass());
458 LPM2
.addPass(IndVarSimplifyPass());
459 LPM2
.addPass(LoopIdiomRecognizePass());
461 for (auto &C
: LateLoopOptimizationsEPCallbacks
)
464 LPM2
.addPass(LoopDeletionPass());
465 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
466 // because it changes IR to makes profile annotation in back compile
468 if ((Phase
!= ThinLTOPhase::PreLink
|| !PGOOpt
||
469 PGOOpt
->Action
!= PGOOptions::SampleUse
) &&
472 LoopFullUnrollPass(Level
, false, PTO
.ForgetAllSCEVInLoopUnroll
));
474 for (auto &C
: LoopOptimizerEndEPCallbacks
)
477 // We provide the opt remark emitter pass for LICM to use. We only need to do
478 // this once as it is immutable.
479 FPM
.addPass(RequireAnalysisPass
<OptimizationRemarkEmitterAnalysis
, Function
>());
480 FPM
.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1
), DebugLogging
));
481 FPM
.addPass(SimplifyCFGPass());
482 FPM
.addPass(InstCombinePass());
483 FPM
.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2
), DebugLogging
));
485 // Eliminate redundancies.
487 // These passes add substantial compile time so skip them at O1.
488 FPM
.addPass(MergedLoadStoreMotionPass());
490 FPM
.addPass(NewGVNPass());
495 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
496 FPM
.addPass(MemCpyOptPass());
498 // Sparse conditional constant propagation.
499 // FIXME: It isn't clear why we do this *after* loop passes rather than
501 FPM
.addPass(SCCPPass());
503 // Delete dead bit computations (instcombine runs after to fold away the dead
504 // computations, and then ADCE will run later to exploit any new DCE
505 // opportunities that creates).
506 FPM
.addPass(BDCEPass());
508 // Run instcombine after redundancy and dead bit elimination to exploit
509 // opportunities opened up by them.
510 FPM
.addPass(InstCombinePass());
511 invokePeepholeEPCallbacks(FPM
, Level
);
513 // Re-consider control flow based optimizations after redundancy elimination,
515 FPM
.addPass(JumpThreadingPass());
516 FPM
.addPass(CorrelatedValuePropagationPass());
517 FPM
.addPass(DSEPass());
518 FPM
.addPass(createFunctionToLoopPassAdaptor(
519 LICMPass(PTO
.LicmMssaOptCap
, PTO
.LicmMssaNoAccForPromotionCap
),
522 for (auto &C
: ScalarOptimizerLateEPCallbacks
)
525 // Finally, do an expensive DCE pass to catch all the dead code exposed by
526 // the simplifications and basic cleanup after all the simplifications.
527 FPM
.addPass(ADCEPass());
528 FPM
.addPass(SimplifyCFGPass());
529 FPM
.addPass(InstCombinePass());
530 invokePeepholeEPCallbacks(FPM
, Level
);
532 if (EnableCHR
&& Level
== O3
&& PGOOpt
&&
533 (PGOOpt
->Action
== PGOOptions::IRUse
||
534 PGOOpt
->Action
== PGOOptions::SampleUse
))
535 FPM
.addPass(ControlHeightReductionPass());
540 void PassBuilder::addPGOInstrPasses(ModulePassManager
&MPM
, bool DebugLogging
,
541 PassBuilder::OptimizationLevel Level
,
542 bool RunProfileGen
, bool IsCS
,
543 std::string ProfileFile
,
544 std::string ProfileRemappingFile
) {
545 assert(Level
!= O0
&& "Not expecting O0 here!");
546 // Generally running simplification passes and the inliner with an high
547 // threshold results in smaller executables, but there may be cases where
548 // the size grows, so let's be conservative here and skip this simplification
549 // at -Os/Oz. We will not do this inline for context sensistive PGO (when
551 if (!isOptimizingForSize(Level
) && !IsCS
) {
554 // In the old pass manager, this is a cl::opt. Should still this be one?
555 IP
.DefaultThreshold
= 75;
557 // FIXME: The hint threshold has the same value used by the regular inliner.
558 // This should probably be lowered after performance testing.
559 // FIXME: this comment is cargo culted from the old pass manager, revisit).
560 IP
.HintThreshold
= 325;
562 CGSCCPassManager
CGPipeline(DebugLogging
);
564 CGPipeline
.addPass(InlinerPass(IP
));
566 FunctionPassManager FPM
;
568 FPM
.addPass(EarlyCSEPass()); // Catch trivial redundancies.
569 FPM
.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
570 FPM
.addPass(InstCombinePass()); // Combine silly sequences.
571 invokePeepholeEPCallbacks(FPM
, Level
);
573 CGPipeline
.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM
)));
575 MPM
.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline
)));
577 // Delete anything that is now dead to make sure that we don't instrument
578 // dead code. Instrumentation can end up keeping dead code around and
579 // dramatically increase code size.
580 MPM
.addPass(GlobalDCEPass());
583 if (!RunProfileGen
) {
584 assert(!ProfileFile
.empty() && "Profile use expecting a profile file!");
585 MPM
.addPass(PGOInstrumentationUse(ProfileFile
, ProfileRemappingFile
, IsCS
));
586 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
587 // RequireAnalysisPass for PSI before subsequent non-module passes.
588 MPM
.addPass(RequireAnalysisPass
<ProfileSummaryAnalysis
, Module
>());
592 // Perform PGO instrumentation.
593 MPM
.addPass(PGOInstrumentationGen(IsCS
));
595 FunctionPassManager FPM
;
596 FPM
.addPass(createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging
));
597 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(FPM
)));
599 // Add the profile lowering pass.
600 InstrProfOptions Options
;
601 if (!ProfileFile
.empty())
602 Options
.InstrProfileOutput
= ProfileFile
;
603 // Do counter promotion at Level greater than O0.
604 Options
.DoCounterPromotion
= true;
605 Options
.UseBFIInPromotion
= IsCS
;
606 MPM
.addPass(InstrProfiling(Options
, IsCS
));
609 void PassBuilder::addPGOInstrPassesForO0(ModulePassManager
&MPM
,
610 bool DebugLogging
, bool RunProfileGen
,
611 bool IsCS
, std::string ProfileFile
,
612 std::string ProfileRemappingFile
) {
613 if (!RunProfileGen
) {
614 assert(!ProfileFile
.empty() && "Profile use expecting a profile file!");
615 MPM
.addPass(PGOInstrumentationUse(ProfileFile
, ProfileRemappingFile
, IsCS
));
616 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
617 // RequireAnalysisPass for PSI before subsequent non-module passes.
618 MPM
.addPass(RequireAnalysisPass
<ProfileSummaryAnalysis
, Module
>());
622 // Perform PGO instrumentation.
623 MPM
.addPass(PGOInstrumentationGen(IsCS
));
624 // Add the profile lowering pass.
625 InstrProfOptions Options
;
626 if (!ProfileFile
.empty())
627 Options
.InstrProfileOutput
= ProfileFile
;
628 // Do not do counter promotion at O0.
629 Options
.DoCounterPromotion
= false;
630 Options
.UseBFIInPromotion
= IsCS
;
631 MPM
.addPass(InstrProfiling(Options
, IsCS
));
635 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level
) {
636 auto O3
= PassBuilder::O3
;
637 unsigned OptLevel
= Level
> O3
? 2 : Level
;
638 unsigned SizeLevel
= Level
> O3
? Level
- O3
: 0;
639 return getInlineParams(OptLevel
, SizeLevel
);
643 PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level
,
646 ModulePassManager
MPM(DebugLogging
);
648 bool HasSampleProfile
= PGOOpt
&& (PGOOpt
->Action
== PGOOptions::SampleUse
);
650 // In ThinLTO mode, when flattened profile is used, all the available
651 // profile information will be annotated in PreLink phase so there is
652 // no need to load the profile again in PostLink.
653 bool LoadSampleProfile
=
655 !(FlattenedProfileUsed
&& Phase
== ThinLTOPhase::PostLink
);
657 // During the ThinLTO backend phase we perform early indirect call promotion
658 // here, before globalopt. Otherwise imported available_externally functions
659 // look unreferenced and are removed. If we are going to load the sample
660 // profile then defer until later.
661 // TODO: See if we can move later and consolidate with the location where
662 // we perform ICP when we are loading a sample profile.
663 // TODO: We pass HasSampleProfile (whether there was a sample profile file
664 // passed to the compile) to the SamplePGO flag of ICP. This is used to
665 // determine whether the new direct calls are annotated with prof metadata.
666 // Ideally this should be determined from whether the IR is annotated with
667 // sample profile, and not whether the a sample profile was provided on the
668 // command line. E.g. for flattened profiles where we will not be reloading
669 // the sample profile in the ThinLTO backend, we ideally shouldn't have to
670 // provide the sample profile file.
671 if (Phase
== ThinLTOPhase::PostLink
&& !LoadSampleProfile
)
672 MPM
.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile
));
674 // Do basic inference of function attributes from known properties of system
675 // libraries and other oracles.
676 MPM
.addPass(InferFunctionAttrsPass());
678 // Create an early function pass manager to cleanup the output of the
680 FunctionPassManager
EarlyFPM(DebugLogging
);
681 EarlyFPM
.addPass(SimplifyCFGPass());
682 EarlyFPM
.addPass(SROA());
683 EarlyFPM
.addPass(EarlyCSEPass());
684 EarlyFPM
.addPass(LowerExpectIntrinsicPass());
686 EarlyFPM
.addPass(CallSiteSplittingPass());
688 // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
689 // to convert bitcast to direct calls so that they can be inlined during the
690 // profile annotation prepration step.
691 // More details about SamplePGO design can be found in:
692 // https://research.google.com/pubs/pub45290.html
693 // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
694 if (LoadSampleProfile
)
695 EarlyFPM
.addPass(InstCombinePass());
696 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM
)));
698 if (LoadSampleProfile
) {
699 // Annotate sample profile right after early FPM to ensure freshness of
701 MPM
.addPass(SampleProfileLoaderPass(PGOOpt
->ProfileFile
,
702 PGOOpt
->ProfileRemappingFile
,
703 Phase
== ThinLTOPhase::PreLink
));
704 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
705 // RequireAnalysisPass for PSI before subsequent non-module passes.
706 MPM
.addPass(RequireAnalysisPass
<ProfileSummaryAnalysis
, Module
>());
707 // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
708 // for the profile annotation to be accurate in the ThinLTO backend.
709 if (Phase
!= ThinLTOPhase::PreLink
)
710 // We perform early indirect call promotion here, before globalopt.
711 // This is important for the ThinLTO backend phase because otherwise
712 // imported available_externally functions look unreferenced and are
714 MPM
.addPass(PGOIndirectCallPromotion(Phase
== ThinLTOPhase::PostLink
,
715 true /* SamplePGO */));
718 // Interprocedural constant propagation now that basic cleanup has occurred
719 // and prior to optimizing globals.
720 // FIXME: This position in the pipeline hasn't been carefully considered in
721 // years, it should be re-analyzed.
722 MPM
.addPass(IPSCCPPass());
724 // Attach metadata to indirect call sites indicating the set of functions
725 // they may target at run-time. This should follow IPSCCP.
726 MPM
.addPass(CalledValuePropagationPass());
728 // Optimize globals to try and fold them into constants.
729 MPM
.addPass(GlobalOptPass());
731 // Promote any localized globals to SSA registers.
732 // FIXME: Should this instead by a run of SROA?
733 // FIXME: We should probably run instcombine and simplify-cfg afterward to
734 // delete control flows that are dead once globals have been folded to
736 MPM
.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
738 // Remove any dead arguments exposed by cleanups and constand folding
740 MPM
.addPass(DeadArgumentEliminationPass());
742 // Create a small function pass pipeline to cleanup after all the global
744 FunctionPassManager
GlobalCleanupPM(DebugLogging
);
745 GlobalCleanupPM
.addPass(InstCombinePass());
746 invokePeepholeEPCallbacks(GlobalCleanupPM
, Level
);
748 GlobalCleanupPM
.addPass(SimplifyCFGPass());
749 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM
)));
751 // Add all the requested passes for instrumentation PGO, if requested.
752 if (PGOOpt
&& Phase
!= ThinLTOPhase::PostLink
&&
753 (PGOOpt
->Action
== PGOOptions::IRInstr
||
754 PGOOpt
->Action
== PGOOptions::IRUse
)) {
755 addPGOInstrPasses(MPM
, DebugLogging
, Level
,
756 /* RunProfileGen */ PGOOpt
->Action
== PGOOptions::IRInstr
,
757 /* IsCS */ false, PGOOpt
->ProfileFile
,
758 PGOOpt
->ProfileRemappingFile
);
759 MPM
.addPass(PGOIndirectCallPromotion(false, false));
761 if (PGOOpt
&& Phase
!= ThinLTOPhase::PostLink
&&
762 PGOOpt
->CSAction
== PGOOptions::CSIRInstr
)
763 MPM
.addPass(PGOInstrumentationGenCreateVar(PGOOpt
->CSProfileGenFile
));
765 // Synthesize function entry counts for non-PGO compilation.
766 if (EnableSyntheticCounts
&& !PGOOpt
)
767 MPM
.addPass(SyntheticCountsPropagation());
769 // Require the GlobalsAA analysis for the module so we can query it within
770 // the CGSCC pipeline.
771 MPM
.addPass(RequireAnalysisPass
<GlobalsAA
, Module
>());
773 // Require the ProfileSummaryAnalysis for the module so we can query it within
775 MPM
.addPass(RequireAnalysisPass
<ProfileSummaryAnalysis
, Module
>());
777 // Now begin the main postorder CGSCC pipeline.
778 // FIXME: The current CGSCC pipeline has its origins in the legacy pass
779 // manager and trying to emulate its precise behavior. Much of this doesn't
780 // make a lot of sense and we should revisit the core CGSCC structure.
781 CGSCCPassManager
MainCGPipeline(DebugLogging
);
783 // Note: historically, the PruneEH pass was run first to deduce nounwind and
784 // generally clean up exception handling overhead. It isn't clear this is
785 // valuable as the inliner doesn't currently care whether it is inlining an
788 // Run the inliner first. The theory is that we are walking bottom-up and so
789 // the callees have already been fully optimized, and we want to inline them
790 // into the callers so that our optimizations can reflect that.
791 // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
792 // because it makes profile annotation in the backend inaccurate.
793 InlineParams IP
= getInlineParamsFromOptLevel(Level
);
794 if (Phase
== ThinLTOPhase::PreLink
&& PGOOpt
&&
795 PGOOpt
->Action
== PGOOptions::SampleUse
)
796 IP
.HotCallSiteThreshold
= 0;
797 MainCGPipeline
.addPass(InlinerPass(IP
));
799 // Now deduce any function attributes based in the current code.
800 MainCGPipeline
.addPass(PostOrderFunctionAttrsPass());
802 // When at O3 add argument promotion to the pass pipeline.
803 // FIXME: It isn't at all clear why this should be limited to O3.
805 MainCGPipeline
.addPass(ArgumentPromotionPass());
807 // Lastly, add the core function simplification pipeline nested inside the
809 MainCGPipeline
.addPass(createCGSCCToFunctionPassAdaptor(
810 buildFunctionSimplificationPipeline(Level
, Phase
, DebugLogging
)));
812 for (auto &C
: CGSCCOptimizerLateEPCallbacks
)
813 C(MainCGPipeline
, Level
);
815 // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
816 // to detect when we devirtualize indirect calls and iterate the SCC passes
817 // in that case to try and catch knock-on inlining or function attrs
818 // opportunities. Then we add it to the module pipeline by walking the SCCs
819 // in postorder (or bottom-up).
821 createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
822 std::move(MainCGPipeline
), MaxDevirtIterations
)));
827 ModulePassManager
PassBuilder::buildModuleOptimizationPipeline(
828 OptimizationLevel Level
, bool DebugLogging
, bool LTOPreLink
) {
829 ModulePassManager
MPM(DebugLogging
);
831 // Optimize globals now that the module is fully simplified.
832 MPM
.addPass(GlobalOptPass());
833 MPM
.addPass(GlobalDCEPass());
835 // Run partial inlining pass to partially inline functions that have
837 if (RunPartialInlining
)
838 MPM
.addPass(PartialInlinerPass());
840 // Remove avail extern fns and globals definitions since we aren't compiling
841 // an object file for later LTO. For LTO we want to preserve these so they
842 // are eligible for inlining at link-time. Note if they are unreferenced they
843 // will be removed by GlobalDCE later, so this only impacts referenced
844 // available externally globals. Eventually they will be suppressed during
845 // codegen, but eliminating here enables more opportunity for GlobalDCE as it
846 // may make globals referenced by available external functions dead and saves
847 // running remaining passes on the eliminated functions. These should be
848 // preserved during prelinking for link-time inlining decisions.
850 MPM
.addPass(EliminateAvailableExternallyPass());
852 if (EnableOrderFileInstrumentation
)
853 MPM
.addPass(InstrOrderFilePass());
855 // Do RPO function attribute inference across the module to forward-propagate
856 // attributes where applicable.
857 // FIXME: Is this really an optimization rather than a canonicalization?
858 MPM
.addPass(ReversePostOrderFunctionAttrsPass());
860 // Do a post inline PGO instrumentation and use pass. This is a context
861 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
862 // cross-module inline has not been done yet. The context sensitive
863 // instrumentation is after all the inlines are done.
864 if (!LTOPreLink
&& PGOOpt
) {
865 if (PGOOpt
->CSAction
== PGOOptions::CSIRInstr
)
866 addPGOInstrPasses(MPM
, DebugLogging
, Level
, /* RunProfileGen */ true,
867 /* IsCS */ true, PGOOpt
->CSProfileGenFile
,
868 PGOOpt
->ProfileRemappingFile
);
869 else if (PGOOpt
->CSAction
== PGOOptions::CSIRUse
)
870 addPGOInstrPasses(MPM
, DebugLogging
, Level
, /* RunProfileGen */ false,
871 /* IsCS */ true, PGOOpt
->ProfileFile
,
872 PGOOpt
->ProfileRemappingFile
);
875 // Re-require GloblasAA here prior to function passes. This is particularly
876 // useful as the above will have inlined, DCE'ed, and function-attr
877 // propagated everything. We should at this point have a reasonably minimal
878 // and richly annotated call graph. By computing aliasing and mod/ref
879 // information for all local globals here, the late loop passes and notably
880 // the vectorizer will be able to use them to help recognize vectorizable
881 // memory operations.
882 MPM
.addPass(RequireAnalysisPass
<GlobalsAA
, Module
>());
884 FunctionPassManager
OptimizePM(DebugLogging
);
885 OptimizePM
.addPass(Float2IntPass());
886 // FIXME: We need to run some loop optimizations to re-rotate loops after
887 // simplify-cfg and others undo their rotation.
889 // Optimize the loop execution. These passes operate on entire loop nests
890 // rather than on each loop in an inside-out manner, and so they are actually
893 for (auto &C
: VectorizerStartEPCallbacks
)
894 C(OptimizePM
, Level
);
896 // First rotate loops that may have been un-rotated by prior passes.
898 createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging
));
900 // Distribute loops to allow partial vectorization. I.e. isolate dependences
901 // into separate loop that would otherwise inhibit vectorization. This is
902 // currently only performed for loops marked with the metadata
903 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
904 OptimizePM
.addPass(LoopDistributePass());
906 // Now run the core loop vectorizer.
907 OptimizePM
.addPass(LoopVectorizePass(
908 LoopVectorizeOptions(!PTO
.LoopInterleaving
, !PTO
.LoopVectorization
)));
910 // Eliminate loads by forwarding stores from the previous iteration to loads
911 // of the current iteration.
912 OptimizePM
.addPass(LoopLoadEliminationPass());
914 // Cleanup after the loop optimization passes.
915 OptimizePM
.addPass(InstCombinePass());
917 // Now that we've formed fast to execute loop structures, we do further
918 // optimizations. These are run afterward as they might block doing complex
919 // analyses and transforms such as what are needed for loop vectorization.
921 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
922 // GVN, loop transforms, and others have already run, so it's now better to
923 // convert to more optimized IR using more aggressive simplify CFG options.
924 // The extra sinking transform can create larger basic blocks, so do this
925 // before SLP vectorization.
926 OptimizePM
.addPass(SimplifyCFGPass(SimplifyCFGOptions().
927 forwardSwitchCondToPhi(true).
928 convertSwitchToLookupTable(true).
929 needCanonicalLoops(false).
930 sinkCommonInsts(true)));
932 // Optimize parallel scalar instruction chains into SIMD instructions.
933 if (PTO
.SLPVectorization
)
934 OptimizePM
.addPass(SLPVectorizerPass());
936 OptimizePM
.addPass(InstCombinePass());
938 // Unroll small loops to hide loop backedge latency and saturate any parallel
939 // execution resources of an out-of-order processor. We also then need to
940 // clean up redundancies and loop invariant code.
941 // FIXME: It would be really good to use a loop-integrated instruction
942 // combiner for cleanup here so that the unrolling and LICM can be pipelined
943 // across the loop nests.
944 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
945 if (EnableUnrollAndJam
) {
947 createFunctionToLoopPassAdaptor(LoopUnrollAndJamPass(Level
)));
949 if (PTO
.LoopUnrolling
)
950 OptimizePM
.addPass(LoopUnrollPass(
951 LoopUnrollOptions(Level
, false, PTO
.ForgetAllSCEVInLoopUnroll
)));
952 OptimizePM
.addPass(WarnMissedTransformationsPass());
953 OptimizePM
.addPass(InstCombinePass());
954 OptimizePM
.addPass(RequireAnalysisPass
<OptimizationRemarkEmitterAnalysis
, Function
>());
955 OptimizePM
.addPass(createFunctionToLoopPassAdaptor(
956 LICMPass(PTO
.LicmMssaOptCap
, PTO
.LicmMssaNoAccForPromotionCap
),
959 // Now that we've vectorized and unrolled loops, we may have more refined
960 // alignment information, try to re-derive it here.
961 OptimizePM
.addPass(AlignmentFromAssumptionsPass());
963 // Split out cold code. Splitting is done late to avoid hiding context from
964 // other optimizations and inadvertently regressing performance. The tradeoff
965 // is that this has a higher code size cost than splitting early.
966 if (EnableHotColdSplit
&& !LTOPreLink
)
967 MPM
.addPass(HotColdSplittingPass());
969 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
970 // canonicalization pass that enables other optimizations. As a result,
971 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
973 OptimizePM
.addPass(LoopSinkPass());
975 // And finally clean up LCSSA form before generating code.
976 OptimizePM
.addPass(InstSimplifyPass());
978 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
979 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
980 // flattening of blocks.
981 OptimizePM
.addPass(DivRemPairsPass());
983 // LoopSink (and other loop passes since the last simplifyCFG) might have
984 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
985 OptimizePM
.addPass(SimplifyCFGPass());
987 // Optimize PHIs by speculating around them when profitable. Note that this
988 // pass needs to be run after any PRE or similar pass as it is essentially
989 // inserting redundancies into the program. This even includes SimplifyCFG.
990 OptimizePM
.addPass(SpeculateAroundPHIsPass());
992 for (auto &C
: OptimizerLastEPCallbacks
)
993 C(OptimizePM
, Level
);
995 // Add the core optimizing pipeline.
996 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM
)));
998 MPM
.addPass(CGProfilePass());
1000 // Now we need to do some global optimization transforms.
1001 // FIXME: It would seem like these should come first in the optimization
1002 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1004 MPM
.addPass(GlobalDCEPass());
1005 MPM
.addPass(ConstantMergePass());
1011 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level
,
1012 bool DebugLogging
, bool LTOPreLink
) {
1013 assert(Level
!= O0
&& "Must request optimizations for the default pipeline!");
1015 ModulePassManager
MPM(DebugLogging
);
1017 // Force any function attributes we want the rest of the pipeline to observe.
1018 MPM
.addPass(ForceFunctionAttrsPass());
1020 // Apply module pipeline start EP callback.
1021 for (auto &C
: PipelineStartEPCallbacks
)
1024 if (PGOOpt
&& PGOOpt
->SamplePGOSupport
)
1025 MPM
.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1027 // Add the core simplification pipeline.
1028 MPM
.addPass(buildModuleSimplificationPipeline(Level
, ThinLTOPhase::None
,
1031 // Now add the optimization pipeline.
1032 MPM
.addPass(buildModuleOptimizationPipeline(Level
, DebugLogging
, LTOPreLink
));
1038 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level
,
1039 bool DebugLogging
) {
1040 assert(Level
!= O0
&& "Must request optimizations for the default pipeline!");
1042 ModulePassManager
MPM(DebugLogging
);
1044 // Force any function attributes we want the rest of the pipeline to observe.
1045 MPM
.addPass(ForceFunctionAttrsPass());
1047 if (PGOOpt
&& PGOOpt
->SamplePGOSupport
)
1048 MPM
.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1050 // Apply module pipeline start EP callback.
1051 for (auto &C
: PipelineStartEPCallbacks
)
1054 // If we are planning to perform ThinLTO later, we don't bloat the code with
1055 // unrolling/vectorization/... now. Just simplify the module as much as we
1057 MPM
.addPass(buildModuleSimplificationPipeline(Level
, ThinLTOPhase::PreLink
,
1060 // Run partial inlining pass to partially inline functions that have
1062 // FIXME: It isn't clear whether this is really the right place to run this
1063 // in ThinLTO. Because there is another canonicalization and simplification
1064 // phase that will run after the thin link, running this here ends up with
1065 // less information than will be available later and it may grow functions in
1066 // ways that aren't beneficial.
1067 if (RunPartialInlining
)
1068 MPM
.addPass(PartialInlinerPass());
1070 // Reduce the size of the IR as much as possible.
1071 MPM
.addPass(GlobalOptPass());
1076 ModulePassManager
PassBuilder::buildThinLTODefaultPipeline(
1077 OptimizationLevel Level
, bool DebugLogging
,
1078 const ModuleSummaryIndex
*ImportSummary
) {
1079 ModulePassManager
MPM(DebugLogging
);
1081 if (ImportSummary
) {
1082 // These passes import type identifier resolutions for whole-program
1083 // devirtualization and CFI. They must run early because other passes may
1084 // disturb the specific instruction patterns that these passes look for,
1085 // creating dependencies on resolutions that may not appear in the summary.
1087 // For example, GVN may transform the pattern assume(type.test) appearing in
1088 // two basic blocks into assume(phi(type.test, type.test)), which would
1089 // transform a dependency on a WPD resolution into a dependency on a type
1090 // identifier resolution for CFI.
1092 // Also, WPD has access to more precise information than ICP and can
1093 // devirtualize more effectively, so it should operate on the IR first.
1095 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1096 // metadata and intrinsics.
1097 MPM
.addPass(WholeProgramDevirtPass(nullptr, ImportSummary
));
1098 MPM
.addPass(LowerTypeTestsPass(nullptr, ImportSummary
));
1104 // Force any function attributes we want the rest of the pipeline to observe.
1105 MPM
.addPass(ForceFunctionAttrsPass());
1107 // Add the core simplification pipeline.
1108 MPM
.addPass(buildModuleSimplificationPipeline(Level
, ThinLTOPhase::PostLink
,
1111 // Now add the optimization pipeline.
1112 MPM
.addPass(buildModuleOptimizationPipeline(Level
, DebugLogging
));
1118 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level
,
1119 bool DebugLogging
) {
1120 assert(Level
!= O0
&& "Must request optimizations for the default pipeline!");
1121 // FIXME: We should use a customized pre-link pipeline!
1122 return buildPerModuleDefaultPipeline(Level
, DebugLogging
,
1123 /* LTOPreLink */true);
1127 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level
, bool DebugLogging
,
1128 ModuleSummaryIndex
*ExportSummary
) {
1129 ModulePassManager
MPM(DebugLogging
);
1132 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1133 // metadata and intrinsics.
1134 MPM
.addPass(WholeProgramDevirtPass(ExportSummary
, nullptr));
1135 MPM
.addPass(LowerTypeTestsPass(ExportSummary
, nullptr));
1139 if (PGOOpt
&& PGOOpt
->Action
== PGOOptions::SampleUse
) {
1140 // Load sample profile before running the LTO optimization pipeline.
1141 MPM
.addPass(SampleProfileLoaderPass(PGOOpt
->ProfileFile
,
1142 PGOOpt
->ProfileRemappingFile
,
1143 false /* ThinLTOPhase::PreLink */));
1144 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1145 // RequireAnalysisPass for PSI before subsequent non-module passes.
1146 MPM
.addPass(RequireAnalysisPass
<ProfileSummaryAnalysis
, Module
>());
1149 // Remove unused virtual tables to improve the quality of code generated by
1150 // whole-program devirtualization and bitset lowering.
1151 MPM
.addPass(GlobalDCEPass());
1153 // Force any function attributes we want the rest of the pipeline to observe.
1154 MPM
.addPass(ForceFunctionAttrsPass());
1156 // Do basic inference of function attributes from known properties of system
1157 // libraries and other oracles.
1158 MPM
.addPass(InferFunctionAttrsPass());
1161 FunctionPassManager
EarlyFPM(DebugLogging
);
1162 EarlyFPM
.addPass(CallSiteSplittingPass());
1163 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM
)));
1165 // Indirect call promotion. This should promote all the targets that are
1166 // left by the earlier promotion pass that promotes intra-module targets.
1167 // This two-step promotion is to save the compile time. For LTO, it should
1168 // produce the same result as if we only do promotion here.
1169 MPM
.addPass(PGOIndirectCallPromotion(
1170 true /* InLTO */, PGOOpt
&& PGOOpt
->Action
== PGOOptions::SampleUse
));
1171 // Propagate constants at call sites into the functions they call. This
1172 // opens opportunities for globalopt (and inlining) by substituting function
1173 // pointers passed as arguments to direct uses of functions.
1174 MPM
.addPass(IPSCCPPass());
1176 // Attach metadata to indirect call sites indicating the set of functions
1177 // they may target at run-time. This should follow IPSCCP.
1178 MPM
.addPass(CalledValuePropagationPass());
1181 // Now deduce any function attributes based in the current code.
1182 MPM
.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1183 PostOrderFunctionAttrsPass()));
1185 // Do RPO function attribute inference across the module to forward-propagate
1186 // attributes where applicable.
1187 // FIXME: Is this really an optimization rather than a canonicalization?
1188 MPM
.addPass(ReversePostOrderFunctionAttrsPass());
1190 // Use in-range annotations on GEP indices to split globals where beneficial.
1191 MPM
.addPass(GlobalSplitPass());
1193 // Run whole program optimization of virtual call when the list of callees
1195 MPM
.addPass(WholeProgramDevirtPass(ExportSummary
, nullptr));
1197 // Stop here at -O1.
1199 // The LowerTypeTestsPass needs to run to lower type metadata and the
1200 // type.test intrinsics. The pass does nothing if CFI is disabled.
1201 MPM
.addPass(LowerTypeTestsPass(ExportSummary
, nullptr));
1205 // Optimize globals to try and fold them into constants.
1206 MPM
.addPass(GlobalOptPass());
1208 // Promote any localized globals to SSA registers.
1209 MPM
.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1211 // Linking modules together can lead to duplicate global constant, only
1212 // keep one copy of each constant.
1213 MPM
.addPass(ConstantMergePass());
1215 // Remove unused arguments from functions.
1216 MPM
.addPass(DeadArgumentEliminationPass());
1218 // Reduce the code after globalopt and ipsccp. Both can open up significant
1219 // simplification opportunities, and both can propagate functions through
1220 // function pointers. When this happens, we often have to resolve varargs
1221 // calls, etc, so let instcombine do this.
1222 FunctionPassManager
PeepholeFPM(DebugLogging
);
1224 PeepholeFPM
.addPass(AggressiveInstCombinePass());
1225 PeepholeFPM
.addPass(InstCombinePass());
1226 invokePeepholeEPCallbacks(PeepholeFPM
, Level
);
1228 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM
)));
1230 // Note: historically, the PruneEH pass was run first to deduce nounwind and
1231 // generally clean up exception handling overhead. It isn't clear this is
1232 // valuable as the inliner doesn't currently care whether it is inlining an
1233 // invoke or a call.
1234 // Run the inliner now.
1235 MPM
.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1236 InlinerPass(getInlineParamsFromOptLevel(Level
))));
1238 // Optimize globals again after we ran the inliner.
1239 MPM
.addPass(GlobalOptPass());
1241 // Garbage collect dead functions.
1242 // FIXME: Add ArgumentPromotion pass after once it's ported.
1243 MPM
.addPass(GlobalDCEPass());
1245 FunctionPassManager
FPM(DebugLogging
);
1246 // The IPO Passes may leave cruft around. Clean up after them.
1247 FPM
.addPass(InstCombinePass());
1248 invokePeepholeEPCallbacks(FPM
, Level
);
1250 FPM
.addPass(JumpThreadingPass());
1252 // Do a post inline PGO instrumentation and use pass. This is a context
1253 // sensitive PGO pass.
1255 if (PGOOpt
->CSAction
== PGOOptions::CSIRInstr
)
1256 addPGOInstrPasses(MPM
, DebugLogging
, Level
, /* RunProfileGen */ true,
1257 /* IsCS */ true, PGOOpt
->CSProfileGenFile
,
1258 PGOOpt
->ProfileRemappingFile
);
1259 else if (PGOOpt
->CSAction
== PGOOptions::CSIRUse
)
1260 addPGOInstrPasses(MPM
, DebugLogging
, Level
, /* RunProfileGen */ false,
1261 /* IsCS */ true, PGOOpt
->ProfileFile
,
1262 PGOOpt
->ProfileRemappingFile
);
1266 FPM
.addPass(SROA());
1268 // LTO provides additional opportunities for tailcall elimination due to
1269 // link-time inlining, and visibility of nocapture attribute.
1270 FPM
.addPass(TailCallElimPass());
1272 // Run a few AA driver optimizations here and now to cleanup the code.
1273 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(FPM
)));
1275 MPM
.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1276 PostOrderFunctionAttrsPass()));
1277 // FIXME: here we run IP alias analysis in the legacy PM.
1279 FunctionPassManager MainFPM
;
1281 // FIXME: once we fix LoopPass Manager, add LICM here.
1282 // FIXME: once we provide support for enabling MLSM, add it here.
1284 MainFPM
.addPass(NewGVNPass());
1286 MainFPM
.addPass(GVN());
1288 // Remove dead memcpy()'s.
1289 MainFPM
.addPass(MemCpyOptPass());
1291 // Nuke dead stores.
1292 MainFPM
.addPass(DSEPass());
1294 // FIXME: at this point, we run a bunch of loop passes:
1295 // indVarSimplify, loopDeletion, loopInterchange, loopUnroll,
1296 // loopVectorize. Enable them once the remaining issue with LPM
1299 MainFPM
.addPass(InstCombinePass());
1300 MainFPM
.addPass(SimplifyCFGPass());
1301 MainFPM
.addPass(SCCPPass());
1302 MainFPM
.addPass(InstCombinePass());
1303 MainFPM
.addPass(BDCEPass());
1305 // FIXME: We may want to run SLPVectorizer here.
1306 // After vectorization, assume intrinsics may tell us more
1307 // about pointer alignments.
1309 MainFPM
.add(AlignmentFromAssumptionsPass());
1312 // FIXME: Conditionally run LoadCombine here, after it's ported
1313 // (in case we still have this pass, given its questionable usefulness).
1315 MainFPM
.addPass(InstCombinePass());
1316 invokePeepholeEPCallbacks(MainFPM
, Level
);
1317 MainFPM
.addPass(JumpThreadingPass());
1318 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM
)));
1320 // Create a function that performs CFI checks for cross-DSO calls with
1321 // targets in the current module.
1322 MPM
.addPass(CrossDSOCFIPass());
1324 // Lower type metadata and the type.test intrinsic. This pass supports
1325 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1326 // to be run at link time if CFI is enabled. This pass does nothing if
1328 MPM
.addPass(LowerTypeTestsPass(ExportSummary
, nullptr));
1330 // Enable splitting late in the FullLTO post-link pipeline. This is done in
1331 // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1332 if (EnableHotColdSplit
)
1333 MPM
.addPass(HotColdSplittingPass());
1335 // Add late LTO optimization passes.
1336 // Delete basic blocks, which optimization passes may have killed.
1337 MPM
.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1339 // Drop bodies of available eternally objects to improve GlobalDCE.
1340 MPM
.addPass(EliminateAvailableExternallyPass());
1342 // Now that we have optimized the program, discard unreachable functions.
1343 MPM
.addPass(GlobalDCEPass());
1345 // FIXME: Maybe enable MergeFuncs conditionally after it's ported.
1349 AAManager
PassBuilder::buildDefaultAAPipeline() {
1352 // The order in which these are registered determines their priority when
1355 // First we register the basic alias analysis that provides the majority of
1356 // per-function local AA logic. This is a stateless, on-demand local set of
1358 AA
.registerFunctionAnalysis
<BasicAA
>();
1360 // Next we query fast, specialized alias analyses that wrap IR-embedded
1361 // information about aliasing.
1362 AA
.registerFunctionAnalysis
<ScopedNoAliasAA
>();
1363 AA
.registerFunctionAnalysis
<TypeBasedAA
>();
1365 // Add support for querying global aliasing information when available.
1366 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1367 // analysis, all that the `AAManager` can do is query for any *cached*
1368 // results from `GlobalsAA` through a readonly proxy.
1369 AA
.registerModuleAnalysis
<GlobalsAA
>();
1374 static Optional
<int> parseRepeatPassName(StringRef Name
) {
1375 if (!Name
.consume_front("repeat<") || !Name
.consume_back(">"))
1378 if (Name
.getAsInteger(0, Count
) || Count
<= 0)
1383 static Optional
<int> parseDevirtPassName(StringRef Name
) {
1384 if (!Name
.consume_front("devirt<") || !Name
.consume_back(">"))
1387 if (Name
.getAsInteger(0, Count
) || Count
<= 0)
1392 static bool checkParametrizedPassName(StringRef Name
, StringRef PassName
) {
1393 if (!Name
.consume_front(PassName
))
1395 // normal pass name w/o parameters == default parameters
1398 return Name
.startswith("<") && Name
.endswith(">");
1403 /// This performs customized parsing of pass name with parameters.
1405 /// We do not need parametrization of passes in textual pipeline very often,
1406 /// yet on a rare occasion ability to specify parameters right there can be
1409 /// \p Name - parameterized specification of a pass from a textual pipeline
1410 /// is a string in a form of :
1411 /// PassName '<' parameter-list '>'
1413 /// Parameter list is being parsed by the parser callable argument, \p Parser,
1414 /// It takes a string-ref of parameters and returns either StringError or a
1415 /// parameter list in a form of a custom parameters type, all wrapped into
1416 /// Expected<> template class.
1418 template <typename ParametersParseCallableT
>
1419 auto parsePassParameters(ParametersParseCallableT
&&Parser
, StringRef Name
,
1420 StringRef PassName
) -> decltype(Parser(StringRef
{})) {
1421 using ParametersT
= typename
decltype(Parser(StringRef
{}))::value_type
;
1423 StringRef Params
= Name
;
1424 if (!Params
.consume_front(PassName
)) {
1426 "unable to strip pass name from parametrized pass specification");
1429 return ParametersT
{};
1430 if (!Params
.consume_front("<") || !Params
.consume_back(">")) {
1431 assert(false && "invalid format for parametrized pass name");
1434 Expected
<ParametersT
> Result
= Parser(Params
);
1435 assert((Result
|| Result
.template errorIsA
<StringError
>()) &&
1436 "Pass parameter parser can only return StringErrors.");
1437 return std::move(Result
);
1440 /// Parser of parameters for LoopUnroll pass.
1441 Expected
<LoopUnrollOptions
> parseLoopUnrollOptions(StringRef Params
) {
1442 LoopUnrollOptions UnrollOpts
;
1443 while (!Params
.empty()) {
1444 StringRef ParamName
;
1445 std::tie(ParamName
, Params
) = Params
.split(';');
1446 int OptLevel
= StringSwitch
<int>(ParamName
)
1452 if (OptLevel
>= 0) {
1453 UnrollOpts
.setOptLevel(OptLevel
);
1457 bool Enable
= !ParamName
.consume_front("no-");
1458 if (ParamName
== "partial") {
1459 UnrollOpts
.setPartial(Enable
);
1460 } else if (ParamName
== "peeling") {
1461 UnrollOpts
.setPeeling(Enable
);
1462 } else if (ParamName
== "profile-peeling") {
1463 UnrollOpts
.setProfileBasedPeeling(Enable
);
1464 } else if (ParamName
== "runtime") {
1465 UnrollOpts
.setRuntime(Enable
);
1466 } else if (ParamName
== "upperbound") {
1467 UnrollOpts
.setUpperBound(Enable
);
1469 return make_error
<StringError
>(
1470 formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName
).str(),
1471 inconvertibleErrorCode());
1477 Expected
<MemorySanitizerOptions
> parseMSanPassOptions(StringRef Params
) {
1478 MemorySanitizerOptions Result
;
1479 while (!Params
.empty()) {
1480 StringRef ParamName
;
1481 std::tie(ParamName
, Params
) = Params
.split(';');
1483 if (ParamName
== "recover") {
1484 Result
.Recover
= true;
1485 } else if (ParamName
== "kernel") {
1486 Result
.Kernel
= true;
1487 } else if (ParamName
.consume_front("track-origins=")) {
1488 if (ParamName
.getAsInteger(0, Result
.TrackOrigins
))
1489 return make_error
<StringError
>(
1490 formatv("invalid argument to MemorySanitizer pass track-origins "
1491 "parameter: '{0}' ",
1494 inconvertibleErrorCode());
1496 return make_error
<StringError
>(
1497 formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName
)
1499 inconvertibleErrorCode());
1505 /// Parser of parameters for SimplifyCFG pass.
1506 Expected
<SimplifyCFGOptions
> parseSimplifyCFGOptions(StringRef Params
) {
1507 SimplifyCFGOptions Result
;
1508 while (!Params
.empty()) {
1509 StringRef ParamName
;
1510 std::tie(ParamName
, Params
) = Params
.split(';');
1512 bool Enable
= !ParamName
.consume_front("no-");
1513 if (ParamName
== "forward-switch-cond") {
1514 Result
.forwardSwitchCondToPhi(Enable
);
1515 } else if (ParamName
== "switch-to-lookup") {
1516 Result
.convertSwitchToLookupTable(Enable
);
1517 } else if (ParamName
== "keep-loops") {
1518 Result
.needCanonicalLoops(Enable
);
1519 } else if (ParamName
== "sink-common-insts") {
1520 Result
.sinkCommonInsts(Enable
);
1521 } else if (Enable
&& ParamName
.consume_front("bonus-inst-threshold=")) {
1522 APInt BonusInstThreshold
;
1523 if (ParamName
.getAsInteger(0, BonusInstThreshold
))
1524 return make_error
<StringError
>(
1525 formatv("invalid argument to SimplifyCFG pass bonus-threshold "
1526 "parameter: '{0}' ",
1528 inconvertibleErrorCode());
1529 Result
.bonusInstThreshold(BonusInstThreshold
.getSExtValue());
1531 return make_error
<StringError
>(
1532 formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName
).str(),
1533 inconvertibleErrorCode());
1539 /// Parser of parameters for LoopVectorize pass.
1540 Expected
<LoopVectorizeOptions
> parseLoopVectorizeOptions(StringRef Params
) {
1541 LoopVectorizeOptions Opts
;
1542 while (!Params
.empty()) {
1543 StringRef ParamName
;
1544 std::tie(ParamName
, Params
) = Params
.split(';');
1546 bool Enable
= !ParamName
.consume_front("no-");
1547 if (ParamName
== "interleave-forced-only") {
1548 Opts
.setInterleaveOnlyWhenForced(Enable
);
1549 } else if (ParamName
== "vectorize-forced-only") {
1550 Opts
.setVectorizeOnlyWhenForced(Enable
);
1552 return make_error
<StringError
>(
1553 formatv("invalid LoopVectorize parameter '{0}' ", ParamName
).str(),
1554 inconvertibleErrorCode());
1560 Expected
<bool> parseLoopUnswitchOptions(StringRef Params
) {
1561 bool Result
= false;
1562 while (!Params
.empty()) {
1563 StringRef ParamName
;
1564 std::tie(ParamName
, Params
) = Params
.split(';');
1566 bool Enable
= !ParamName
.consume_front("no-");
1567 if (ParamName
== "nontrivial") {
1570 return make_error
<StringError
>(
1571 formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName
)
1573 inconvertibleErrorCode());
1580 /// Tests whether a pass name starts with a valid prefix for a default pipeline
1582 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name
) {
1583 return Name
.startswith("default") || Name
.startswith("thinlto") ||
1584 Name
.startswith("lto");
1587 /// Tests whether registered callbacks will accept a given pass name.
1589 /// When parsing a pipeline text, the type of the outermost pipeline may be
1590 /// omitted, in which case the type is automatically determined from the first
1591 /// pass name in the text. This may be a name that is handled through one of the
1592 /// callbacks. We check this through the oridinary parsing callbacks by setting
1593 /// up a dummy PassManager in order to not force the client to also handle this
1595 template <typename PassManagerT
, typename CallbacksT
>
1596 static bool callbacksAcceptPassName(StringRef Name
, CallbacksT
&Callbacks
) {
1597 if (!Callbacks
.empty()) {
1598 PassManagerT DummyPM
;
1599 for (auto &CB
: Callbacks
)
1600 if (CB(Name
, DummyPM
, {}))
1606 template <typename CallbacksT
>
1607 static bool isModulePassName(StringRef Name
, CallbacksT
&Callbacks
) {
1608 // Manually handle aliases for pre-configured pipeline fragments.
1609 if (startsWithDefaultPipelineAliasPrefix(Name
))
1610 return DefaultAliasRegex
.match(Name
);
1612 // Explicitly handle pass manager names.
1613 if (Name
== "module")
1615 if (Name
== "cgscc")
1617 if (Name
== "function")
1620 // Explicitly handle custom-parsed pass names.
1621 if (parseRepeatPassName(Name
))
1624 #define MODULE_PASS(NAME, CREATE_PASS) \
1627 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
1628 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1630 #include "PassRegistry.def"
1632 return callbacksAcceptPassName
<ModulePassManager
>(Name
, Callbacks
);
1635 template <typename CallbacksT
>
1636 static bool isCGSCCPassName(StringRef Name
, CallbacksT
&Callbacks
) {
1637 // Explicitly handle pass manager names.
1638 if (Name
== "cgscc")
1640 if (Name
== "function")
1643 // Explicitly handle custom-parsed pass names.
1644 if (parseRepeatPassName(Name
))
1646 if (parseDevirtPassName(Name
))
1649 #define CGSCC_PASS(NAME, CREATE_PASS) \
1652 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
1653 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1655 #include "PassRegistry.def"
1657 return callbacksAcceptPassName
<CGSCCPassManager
>(Name
, Callbacks
);
1660 template <typename CallbacksT
>
1661 static bool isFunctionPassName(StringRef Name
, CallbacksT
&Callbacks
) {
1662 // Explicitly handle pass manager names.
1663 if (Name
== "function")
1668 // Explicitly handle custom-parsed pass names.
1669 if (parseRepeatPassName(Name
))
1672 #define FUNCTION_PASS(NAME, CREATE_PASS) \
1675 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
1676 if (checkParametrizedPassName(Name, NAME)) \
1678 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
1679 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1681 #include "PassRegistry.def"
1683 return callbacksAcceptPassName
<FunctionPassManager
>(Name
, Callbacks
);
1686 template <typename CallbacksT
>
1687 static bool isLoopPassName(StringRef Name
, CallbacksT
&Callbacks
) {
1688 // Explicitly handle pass manager names.
1692 // Explicitly handle custom-parsed pass names.
1693 if (parseRepeatPassName(Name
))
1696 #define LOOP_PASS(NAME, CREATE_PASS) \
1699 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
1700 if (checkParametrizedPassName(Name, NAME)) \
1702 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
1703 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1705 #include "PassRegistry.def"
1707 return callbacksAcceptPassName
<LoopPassManager
>(Name
, Callbacks
);
1710 Optional
<std::vector
<PassBuilder::PipelineElement
>>
1711 PassBuilder::parsePipelineText(StringRef Text
) {
1712 std::vector
<PipelineElement
> ResultPipeline
;
1714 SmallVector
<std::vector
<PipelineElement
> *, 4> PipelineStack
= {
1717 std::vector
<PipelineElement
> &Pipeline
= *PipelineStack
.back();
1718 size_t Pos
= Text
.find_first_of(",()");
1719 Pipeline
.push_back({Text
.substr(0, Pos
), {}});
1721 // If we have a single terminating name, we're done.
1722 if (Pos
== Text
.npos
)
1725 char Sep
= Text
[Pos
];
1726 Text
= Text
.substr(Pos
+ 1);
1728 // Just a name ending in a comma, continue.
1732 // Push the inner pipeline onto the stack to continue processing.
1733 PipelineStack
.push_back(&Pipeline
.back().InnerPipeline
);
1737 assert(Sep
== ')' && "Bogus separator!");
1738 // When handling the close parenthesis, we greedily consume them to avoid
1739 // empty strings in the pipeline.
1741 // If we try to pop the outer pipeline we have unbalanced parentheses.
1742 if (PipelineStack
.size() == 1)
1745 PipelineStack
.pop_back();
1746 } while (Text
.consume_front(")"));
1748 // Check if we've finished parsing.
1752 // Otherwise, the end of an inner pipeline always has to be followed by
1753 // a comma, and then we can continue.
1754 if (!Text
.consume_front(","))
1758 if (PipelineStack
.size() > 1)
1759 // Unbalanced paretheses.
1762 assert(PipelineStack
.back() == &ResultPipeline
&&
1763 "Wrong pipeline at the bottom of the stack!");
1764 return {std::move(ResultPipeline
)};
1767 Error
PassBuilder::parseModulePass(ModulePassManager
&MPM
,
1768 const PipelineElement
&E
,
1769 bool VerifyEachPass
, bool DebugLogging
) {
1770 auto &Name
= E
.Name
;
1771 auto &InnerPipeline
= E
.InnerPipeline
;
1773 // First handle complex passes like the pass managers which carry pipelines.
1774 if (!InnerPipeline
.empty()) {
1775 if (Name
== "module") {
1776 ModulePassManager
NestedMPM(DebugLogging
);
1777 if (auto Err
= parseModulePassPipeline(NestedMPM
, InnerPipeline
,
1778 VerifyEachPass
, DebugLogging
))
1780 MPM
.addPass(std::move(NestedMPM
));
1781 return Error::success();
1783 if (Name
== "cgscc") {
1784 CGSCCPassManager
CGPM(DebugLogging
);
1785 if (auto Err
= parseCGSCCPassPipeline(CGPM
, InnerPipeline
, VerifyEachPass
,
1788 MPM
.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM
)));
1789 return Error::success();
1791 if (Name
== "function") {
1792 FunctionPassManager
FPM(DebugLogging
);
1793 if (auto Err
= parseFunctionPassPipeline(FPM
, InnerPipeline
,
1794 VerifyEachPass
, DebugLogging
))
1796 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(FPM
)));
1797 return Error::success();
1799 if (auto Count
= parseRepeatPassName(Name
)) {
1800 ModulePassManager
NestedMPM(DebugLogging
);
1801 if (auto Err
= parseModulePassPipeline(NestedMPM
, InnerPipeline
,
1802 VerifyEachPass
, DebugLogging
))
1804 MPM
.addPass(createRepeatedPass(*Count
, std::move(NestedMPM
)));
1805 return Error::success();
1808 for (auto &C
: ModulePipelineParsingCallbacks
)
1809 if (C(Name
, MPM
, InnerPipeline
))
1810 return Error::success();
1812 // Normal passes can't have pipelines.
1813 return make_error
<StringError
>(
1814 formatv("invalid use of '{0}' pass as module pipeline", Name
).str(),
1815 inconvertibleErrorCode());
1819 // Manually handle aliases for pre-configured pipeline fragments.
1820 if (startsWithDefaultPipelineAliasPrefix(Name
)) {
1821 SmallVector
<StringRef
, 3> Matches
;
1822 if (!DefaultAliasRegex
.match(Name
, &Matches
))
1823 return make_error
<StringError
>(
1824 formatv("unknown default pipeline alias '{0}'", Name
).str(),
1825 inconvertibleErrorCode());
1827 assert(Matches
.size() == 3 && "Must capture two matched strings!");
1829 OptimizationLevel L
= StringSwitch
<OptimizationLevel
>(Matches
[2])
1837 // Add instrumentation PGO passes -- at O0 we can still do PGO.
1838 if (PGOOpt
&& Matches
[1] != "thinlto" &&
1839 (PGOOpt
->Action
== PGOOptions::IRInstr
||
1840 PGOOpt
->Action
== PGOOptions::IRUse
))
1841 addPGOInstrPassesForO0(
1843 /* RunProfileGen */ (PGOOpt
->Action
== PGOOptions::IRInstr
),
1844 /* IsCS */ false, PGOOpt
->ProfileFile
,
1845 PGOOpt
->ProfileRemappingFile
);
1846 // Do nothing else at all!
1847 return Error::success();
1850 if (Matches
[1] == "default") {
1851 MPM
.addPass(buildPerModuleDefaultPipeline(L
, DebugLogging
));
1852 } else if (Matches
[1] == "thinlto-pre-link") {
1853 MPM
.addPass(buildThinLTOPreLinkDefaultPipeline(L
, DebugLogging
));
1854 } else if (Matches
[1] == "thinlto") {
1855 MPM
.addPass(buildThinLTODefaultPipeline(L
, DebugLogging
, nullptr));
1856 } else if (Matches
[1] == "lto-pre-link") {
1857 MPM
.addPass(buildLTOPreLinkDefaultPipeline(L
, DebugLogging
));
1859 assert(Matches
[1] == "lto" && "Not one of the matched options!");
1860 MPM
.addPass(buildLTODefaultPipeline(L
, DebugLogging
, nullptr));
1862 return Error::success();
1865 // Finally expand the basic registered passes from the .inc file.
1866 #define MODULE_PASS(NAME, CREATE_PASS) \
1867 if (Name == NAME) { \
1868 MPM.addPass(CREATE_PASS); \
1869 return Error::success(); \
1871 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
1872 if (Name == "require<" NAME ">") { \
1874 RequireAnalysisPass< \
1875 std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \
1876 return Error::success(); \
1878 if (Name == "invalidate<" NAME ">") { \
1879 MPM.addPass(InvalidateAnalysisPass< \
1880 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
1881 return Error::success(); \
1883 #include "PassRegistry.def"
1885 for (auto &C
: ModulePipelineParsingCallbacks
)
1886 if (C(Name
, MPM
, InnerPipeline
))
1887 return Error::success();
1888 return make_error
<StringError
>(
1889 formatv("unknown module pass '{0}'", Name
).str(),
1890 inconvertibleErrorCode());
1893 Error
PassBuilder::parseCGSCCPass(CGSCCPassManager
&CGPM
,
1894 const PipelineElement
&E
, bool VerifyEachPass
,
1895 bool DebugLogging
) {
1896 auto &Name
= E
.Name
;
1897 auto &InnerPipeline
= E
.InnerPipeline
;
1899 // First handle complex passes like the pass managers which carry pipelines.
1900 if (!InnerPipeline
.empty()) {
1901 if (Name
== "cgscc") {
1902 CGSCCPassManager
NestedCGPM(DebugLogging
);
1903 if (auto Err
= parseCGSCCPassPipeline(NestedCGPM
, InnerPipeline
,
1904 VerifyEachPass
, DebugLogging
))
1906 // Add the nested pass manager with the appropriate adaptor.
1907 CGPM
.addPass(std::move(NestedCGPM
));
1908 return Error::success();
1910 if (Name
== "function") {
1911 FunctionPassManager
FPM(DebugLogging
);
1912 if (auto Err
= parseFunctionPassPipeline(FPM
, InnerPipeline
,
1913 VerifyEachPass
, DebugLogging
))
1915 // Add the nested pass manager with the appropriate adaptor.
1916 CGPM
.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM
)));
1917 return Error::success();
1919 if (auto Count
= parseRepeatPassName(Name
)) {
1920 CGSCCPassManager
NestedCGPM(DebugLogging
);
1921 if (auto Err
= parseCGSCCPassPipeline(NestedCGPM
, InnerPipeline
,
1922 VerifyEachPass
, DebugLogging
))
1924 CGPM
.addPass(createRepeatedPass(*Count
, std::move(NestedCGPM
)));
1925 return Error::success();
1927 if (auto MaxRepetitions
= parseDevirtPassName(Name
)) {
1928 CGSCCPassManager
NestedCGPM(DebugLogging
);
1929 if (auto Err
= parseCGSCCPassPipeline(NestedCGPM
, InnerPipeline
,
1930 VerifyEachPass
, DebugLogging
))
1933 createDevirtSCCRepeatedPass(std::move(NestedCGPM
), *MaxRepetitions
));
1934 return Error::success();
1937 for (auto &C
: CGSCCPipelineParsingCallbacks
)
1938 if (C(Name
, CGPM
, InnerPipeline
))
1939 return Error::success();
1941 // Normal passes can't have pipelines.
1942 return make_error
<StringError
>(
1943 formatv("invalid use of '{0}' pass as cgscc pipeline", Name
).str(),
1944 inconvertibleErrorCode());
1947 // Now expand the basic registered passes from the .inc file.
1948 #define CGSCC_PASS(NAME, CREATE_PASS) \
1949 if (Name == NAME) { \
1950 CGPM.addPass(CREATE_PASS); \
1951 return Error::success(); \
1953 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
1954 if (Name == "require<" NAME ">") { \
1955 CGPM.addPass(RequireAnalysisPass< \
1956 std::remove_reference<decltype(CREATE_PASS)>::type, \
1957 LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \
1958 CGSCCUpdateResult &>()); \
1959 return Error::success(); \
1961 if (Name == "invalidate<" NAME ">") { \
1962 CGPM.addPass(InvalidateAnalysisPass< \
1963 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
1964 return Error::success(); \
1966 #include "PassRegistry.def"
1968 for (auto &C
: CGSCCPipelineParsingCallbacks
)
1969 if (C(Name
, CGPM
, InnerPipeline
))
1970 return Error::success();
1971 return make_error
<StringError
>(
1972 formatv("unknown cgscc pass '{0}'", Name
).str(),
1973 inconvertibleErrorCode());
1976 Error
PassBuilder::parseFunctionPass(FunctionPassManager
&FPM
,
1977 const PipelineElement
&E
,
1978 bool VerifyEachPass
, bool DebugLogging
) {
1979 auto &Name
= E
.Name
;
1980 auto &InnerPipeline
= E
.InnerPipeline
;
1982 // First handle complex passes like the pass managers which carry pipelines.
1983 if (!InnerPipeline
.empty()) {
1984 if (Name
== "function") {
1985 FunctionPassManager
NestedFPM(DebugLogging
);
1986 if (auto Err
= parseFunctionPassPipeline(NestedFPM
, InnerPipeline
,
1987 VerifyEachPass
, DebugLogging
))
1989 // Add the nested pass manager with the appropriate adaptor.
1990 FPM
.addPass(std::move(NestedFPM
));
1991 return Error::success();
1993 if (Name
== "loop") {
1994 LoopPassManager
LPM(DebugLogging
);
1995 if (auto Err
= parseLoopPassPipeline(LPM
, InnerPipeline
, VerifyEachPass
,
1998 // Add the nested pass manager with the appropriate adaptor.
2000 createFunctionToLoopPassAdaptor(std::move(LPM
), DebugLogging
));
2001 return Error::success();
2003 if (auto Count
= parseRepeatPassName(Name
)) {
2004 FunctionPassManager
NestedFPM(DebugLogging
);
2005 if (auto Err
= parseFunctionPassPipeline(NestedFPM
, InnerPipeline
,
2006 VerifyEachPass
, DebugLogging
))
2008 FPM
.addPass(createRepeatedPass(*Count
, std::move(NestedFPM
)));
2009 return Error::success();
2012 for (auto &C
: FunctionPipelineParsingCallbacks
)
2013 if (C(Name
, FPM
, InnerPipeline
))
2014 return Error::success();
2016 // Normal passes can't have pipelines.
2017 return make_error
<StringError
>(
2018 formatv("invalid use of '{0}' pass as function pipeline", Name
).str(),
2019 inconvertibleErrorCode());
2022 // Now expand the basic registered passes from the .inc file.
2023 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2024 if (Name == NAME) { \
2025 FPM.addPass(CREATE_PASS); \
2026 return Error::success(); \
2028 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2029 if (checkParametrizedPassName(Name, NAME)) { \
2030 auto Params = parsePassParameters(PARSER, Name, NAME); \
2032 return Params.takeError(); \
2033 FPM.addPass(CREATE_PASS(Params.get())); \
2034 return Error::success(); \
2036 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2037 if (Name == "require<" NAME ">") { \
2039 RequireAnalysisPass< \
2040 std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \
2041 return Error::success(); \
2043 if (Name == "invalidate<" NAME ">") { \
2044 FPM.addPass(InvalidateAnalysisPass< \
2045 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2046 return Error::success(); \
2048 #include "PassRegistry.def"
2050 for (auto &C
: FunctionPipelineParsingCallbacks
)
2051 if (C(Name
, FPM
, InnerPipeline
))
2052 return Error::success();
2053 return make_error
<StringError
>(
2054 formatv("unknown function pass '{0}'", Name
).str(),
2055 inconvertibleErrorCode());
2058 Error
PassBuilder::parseLoopPass(LoopPassManager
&LPM
, const PipelineElement
&E
,
2059 bool VerifyEachPass
, bool DebugLogging
) {
2060 StringRef Name
= E
.Name
;
2061 auto &InnerPipeline
= E
.InnerPipeline
;
2063 // First handle complex passes like the pass managers which carry pipelines.
2064 if (!InnerPipeline
.empty()) {
2065 if (Name
== "loop") {
2066 LoopPassManager
NestedLPM(DebugLogging
);
2067 if (auto Err
= parseLoopPassPipeline(NestedLPM
, InnerPipeline
,
2068 VerifyEachPass
, DebugLogging
))
2070 // Add the nested pass manager with the appropriate adaptor.
2071 LPM
.addPass(std::move(NestedLPM
));
2072 return Error::success();
2074 if (auto Count
= parseRepeatPassName(Name
)) {
2075 LoopPassManager
NestedLPM(DebugLogging
);
2076 if (auto Err
= parseLoopPassPipeline(NestedLPM
, InnerPipeline
,
2077 VerifyEachPass
, DebugLogging
))
2079 LPM
.addPass(createRepeatedPass(*Count
, std::move(NestedLPM
)));
2080 return Error::success();
2083 for (auto &C
: LoopPipelineParsingCallbacks
)
2084 if (C(Name
, LPM
, InnerPipeline
))
2085 return Error::success();
2087 // Normal passes can't have pipelines.
2088 return make_error
<StringError
>(
2089 formatv("invalid use of '{0}' pass as loop pipeline", Name
).str(),
2090 inconvertibleErrorCode());
2093 // Now expand the basic registered passes from the .inc file.
2094 #define LOOP_PASS(NAME, CREATE_PASS) \
2095 if (Name == NAME) { \
2096 LPM.addPass(CREATE_PASS); \
2097 return Error::success(); \
2099 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2100 if (checkParametrizedPassName(Name, NAME)) { \
2101 auto Params = parsePassParameters(PARSER, Name, NAME); \
2103 return Params.takeError(); \
2104 LPM.addPass(CREATE_PASS(Params.get())); \
2105 return Error::success(); \
2107 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2108 if (Name == "require<" NAME ">") { \
2109 LPM.addPass(RequireAnalysisPass< \
2110 std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \
2111 LoopAnalysisManager, LoopStandardAnalysisResults &, \
2113 return Error::success(); \
2115 if (Name == "invalidate<" NAME ">") { \
2116 LPM.addPass(InvalidateAnalysisPass< \
2117 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2118 return Error::success(); \
2120 #include "PassRegistry.def"
2122 for (auto &C
: LoopPipelineParsingCallbacks
)
2123 if (C(Name
, LPM
, InnerPipeline
))
2124 return Error::success();
2125 return make_error
<StringError
>(formatv("unknown loop pass '{0}'", Name
).str(),
2126 inconvertibleErrorCode());
2129 bool PassBuilder::parseAAPassName(AAManager
&AA
, StringRef Name
) {
2130 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2131 if (Name == NAME) { \
2132 AA.registerModuleAnalysis< \
2133 std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2136 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2137 if (Name == NAME) { \
2138 AA.registerFunctionAnalysis< \
2139 std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2142 #include "PassRegistry.def"
2144 for (auto &C
: AAParsingCallbacks
)
2150 Error
PassBuilder::parseLoopPassPipeline(LoopPassManager
&LPM
,
2151 ArrayRef
<PipelineElement
> Pipeline
,
2152 bool VerifyEachPass
,
2153 bool DebugLogging
) {
2154 for (const auto &Element
: Pipeline
) {
2155 if (auto Err
= parseLoopPass(LPM
, Element
, VerifyEachPass
, DebugLogging
))
2157 // FIXME: No verifier support for Loop passes!
2159 return Error::success();
2162 Error
PassBuilder::parseFunctionPassPipeline(FunctionPassManager
&FPM
,
2163 ArrayRef
<PipelineElement
> Pipeline
,
2164 bool VerifyEachPass
,
2165 bool DebugLogging
) {
2166 for (const auto &Element
: Pipeline
) {
2168 parseFunctionPass(FPM
, Element
, VerifyEachPass
, DebugLogging
))
2171 FPM
.addPass(VerifierPass());
2173 return Error::success();
2176 Error
PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager
&CGPM
,
2177 ArrayRef
<PipelineElement
> Pipeline
,
2178 bool VerifyEachPass
,
2179 bool DebugLogging
) {
2180 for (const auto &Element
: Pipeline
) {
2181 if (auto Err
= parseCGSCCPass(CGPM
, Element
, VerifyEachPass
, DebugLogging
))
2183 // FIXME: No verifier support for CGSCC passes!
2185 return Error::success();
2188 void PassBuilder::crossRegisterProxies(LoopAnalysisManager
&LAM
,
2189 FunctionAnalysisManager
&FAM
,
2190 CGSCCAnalysisManager
&CGAM
,
2191 ModuleAnalysisManager
&MAM
) {
2192 MAM
.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM
); });
2193 MAM
.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM
); });
2194 CGAM
.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM
); });
2195 FAM
.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM
); });
2196 FAM
.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM
); });
2197 FAM
.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM
); });
2198 LAM
.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM
); });
2201 Error
PassBuilder::parseModulePassPipeline(ModulePassManager
&MPM
,
2202 ArrayRef
<PipelineElement
> Pipeline
,
2203 bool VerifyEachPass
,
2204 bool DebugLogging
) {
2205 for (const auto &Element
: Pipeline
) {
2206 if (auto Err
= parseModulePass(MPM
, Element
, VerifyEachPass
, DebugLogging
))
2209 MPM
.addPass(VerifierPass());
2211 return Error::success();
2214 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2215 // FIXME: Should this routine accept a TargetMachine or require the caller to
2216 // pre-populate the analysis managers with target-specific stuff?
2217 Error
PassBuilder::parsePassPipeline(ModulePassManager
&MPM
,
2218 StringRef PipelineText
,
2219 bool VerifyEachPass
, bool DebugLogging
) {
2220 auto Pipeline
= parsePipelineText(PipelineText
);
2221 if (!Pipeline
|| Pipeline
->empty())
2222 return make_error
<StringError
>(
2223 formatv("invalid pipeline '{0}'", PipelineText
).str(),
2224 inconvertibleErrorCode());
2226 // If the first name isn't at the module layer, wrap the pipeline up
2228 StringRef FirstName
= Pipeline
->front().Name
;
2230 if (!isModulePassName(FirstName
, ModulePipelineParsingCallbacks
)) {
2231 if (isCGSCCPassName(FirstName
, CGSCCPipelineParsingCallbacks
)) {
2232 Pipeline
= {{"cgscc", std::move(*Pipeline
)}};
2233 } else if (isFunctionPassName(FirstName
,
2234 FunctionPipelineParsingCallbacks
)) {
2235 Pipeline
= {{"function", std::move(*Pipeline
)}};
2236 } else if (isLoopPassName(FirstName
, LoopPipelineParsingCallbacks
)) {
2237 Pipeline
= {{"function", {{"loop", std::move(*Pipeline
)}}}};
2239 for (auto &C
: TopLevelPipelineParsingCallbacks
)
2240 if (C(MPM
, *Pipeline
, VerifyEachPass
, DebugLogging
))
2241 return Error::success();
2243 // Unknown pass or pipeline name!
2244 auto &InnerPipeline
= Pipeline
->front().InnerPipeline
;
2245 return make_error
<StringError
>(
2246 formatv("unknown {0} name '{1}'",
2247 (InnerPipeline
.empty() ? "pass" : "pipeline"), FirstName
)
2249 inconvertibleErrorCode());
2254 parseModulePassPipeline(MPM
, *Pipeline
, VerifyEachPass
, DebugLogging
))
2256 return Error::success();
2259 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
2260 Error
PassBuilder::parsePassPipeline(CGSCCPassManager
&CGPM
,
2261 StringRef PipelineText
,
2262 bool VerifyEachPass
, bool DebugLogging
) {
2263 auto Pipeline
= parsePipelineText(PipelineText
);
2264 if (!Pipeline
|| Pipeline
->empty())
2265 return make_error
<StringError
>(
2266 formatv("invalid pipeline '{0}'", PipelineText
).str(),
2267 inconvertibleErrorCode());
2269 StringRef FirstName
= Pipeline
->front().Name
;
2270 if (!isCGSCCPassName(FirstName
, CGSCCPipelineParsingCallbacks
))
2271 return make_error
<StringError
>(
2272 formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName
,
2275 inconvertibleErrorCode());
2278 parseCGSCCPassPipeline(CGPM
, *Pipeline
, VerifyEachPass
, DebugLogging
))
2280 return Error::success();
2283 // Primary pass pipeline description parsing routine for a \c
2284 // FunctionPassManager
2285 Error
PassBuilder::parsePassPipeline(FunctionPassManager
&FPM
,
2286 StringRef PipelineText
,
2287 bool VerifyEachPass
, bool DebugLogging
) {
2288 auto Pipeline
= parsePipelineText(PipelineText
);
2289 if (!Pipeline
|| Pipeline
->empty())
2290 return make_error
<StringError
>(
2291 formatv("invalid pipeline '{0}'", PipelineText
).str(),
2292 inconvertibleErrorCode());
2294 StringRef FirstName
= Pipeline
->front().Name
;
2295 if (!isFunctionPassName(FirstName
, FunctionPipelineParsingCallbacks
))
2296 return make_error
<StringError
>(
2297 formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName
,
2300 inconvertibleErrorCode());
2302 if (auto Err
= parseFunctionPassPipeline(FPM
, *Pipeline
, VerifyEachPass
,
2305 return Error::success();
2308 // Primary pass pipeline description parsing routine for a \c LoopPassManager
2309 Error
PassBuilder::parsePassPipeline(LoopPassManager
&CGPM
,
2310 StringRef PipelineText
,
2311 bool VerifyEachPass
, bool DebugLogging
) {
2312 auto Pipeline
= parsePipelineText(PipelineText
);
2313 if (!Pipeline
|| Pipeline
->empty())
2314 return make_error
<StringError
>(
2315 formatv("invalid pipeline '{0}'", PipelineText
).str(),
2316 inconvertibleErrorCode());
2319 parseLoopPassPipeline(CGPM
, *Pipeline
, VerifyEachPass
, DebugLogging
))
2322 return Error::success();
2325 Error
PassBuilder::parseAAPipeline(AAManager
&AA
, StringRef PipelineText
) {
2326 // If the pipeline just consists of the word 'default' just replace the AA
2327 // manager with our default one.
2328 if (PipelineText
== "default") {
2329 AA
= buildDefaultAAPipeline();
2330 return Error::success();
2333 while (!PipelineText
.empty()) {
2335 std::tie(Name
, PipelineText
) = PipelineText
.split(',');
2336 if (!parseAAPassName(AA
, Name
))
2337 return make_error
<StringError
>(
2338 formatv("unknown alias analysis name '{0}'", Name
).str(),
2339 inconvertibleErrorCode());
2342 return Error::success();