361 void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM,  in invokeOptimizerEarlyEPCallbacks()  argument
364     C(MPM, Level);  in invokeOptimizerEarlyEPCallbacks()
366 void PassBuilder::invokeOptimizerLastEPCallbacks(ModulePassManager &MPM,  in invokeOptimizerLastEPCallbacks()  argument
369     C(MPM, Level);  in invokeOptimizerLastEPCallbacks()
372     ModulePassManager &MPM, OptimizationLevel Level) {  in invokeFullLinkTimeOptimizationEarlyEPCallbacks()  argument
374     C(MPM, Level);  in invokeFullLinkTimeOptimizationEarlyEPCallbacks()
377     ModulePassManager &MPM, OptimizationLevel Level) {  in invokeFullLinkTimeOptimizationLastEPCallbacks()  argument
379     C(MPM, Level);  in invokeFullLinkTimeOptimizationLastEPCallbacks()
381 void PassBuilder::invokePipelineStartEPCallbacks(ModulePassManager &MPM,  in invokePipelineStartEPCallbacks()  argument
384     C(MPM, Level);  in invokePipelineStartEPCallbacks()
387     ModulePassManager &MPM, OptimizationLevel Level) {  in invokePipelineEarlySimplificationEPCallbacks()  argument
389     C(MPM, Level);  in invokePipelineEarlySimplificationEPCallbacks()
393 static void addAnnotationRemarksPass(ModulePassManager &MPM) {  in addAnnotationRemarksPass()  argument
394   MPM.addPass(createModuleToFunctionPassAdaptor(AnnotationRemarksPass()));  in addAnnotationRemarksPass()
766 void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {  in addRequiredLTOPreLinkPasses()  argument
767   MPM.addPass(CanonicalizeAliasesPass());  in addRequiredLTOPreLinkPasses()
768   MPM.addPass(NameAnonGlobalPass());  in addRequiredLTOPreLinkPasses()
771 void PassBuilder::addPreInlinerPasses(ModulePassManager &MPM,  in addPreInlinerPasses()  argument
802   MPM.addPass(std::move(MIWP));  in addPreInlinerPasses()
807   MPM.addPass(GlobalDCEPass());  in addPreInlinerPasses()
810 void PassBuilder::addPostPGOLoopRotation(ModulePassManager &MPM,  in addPostPGOLoopRotation()  argument
814     MPM.addPass(createModuleToFunctionPassAdaptor(  in addPostPGOLoopRotation()
824 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,  in addPGOInstrPasses()  argument
834     MPM.addPass(  in addPGOInstrPasses()
838     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());  in addPGOInstrPasses()
843   MPM.addPass(PGOInstrumentationGen(IsCS));  in addPGOInstrPasses()
845   addPostPGOLoopRotation(MPM, Level);  in addPGOInstrPasses()
860   MPM.addPass(InstrProfilingLoweringPass(Options, IsCS));  in addPGOInstrPasses()
864     ModulePassManager &MPM, bool RunProfileGen, bool IsCS,  in addPGOInstrPassesForO0()  argument
869     MPM.addPass(  in addPGOInstrPassesForO0()
873     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());  in addPGOInstrPassesForO0()
878   MPM.addPass(PGOInstrumentationGen(IsCS));  in addPGOInstrPassesForO0()
887   MPM.addPass(InstrProfilingLoweringPass(Options, IsCS));  in addPGOInstrPassesForO0()
993   ModulePassManager MPM;  in buildModuleInlinerPipeline()  local
1017   MPM.addPass(ModuleInlinerPass(IP, UseInlineAdvisor, Phase));  in buildModuleInlinerPipeline()
1019   MPM.addPass(createModuleToFunctionPassAdaptor(  in buildModuleInlinerPipeline()
1023   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(  in buildModuleInlinerPipeline()
1026   return MPM;  in buildModuleInlinerPipeline()
1038   ModulePassManager MPM;  in buildModuleSimplificationPipeline()  local
1044     MPM.addPass(SampleProfileProbePass(TM));  in buildModuleSimplificationPipeline()
1070     MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));  in buildModuleSimplificationPipeline()
1078     MPM.addPass(InferFunctionAttrsPass());  in buildModuleSimplificationPipeline()
1079     MPM.addPass(CoroEarlyPass());  in buildModuleSimplificationPipeline()
1092     MPM.addPass(createModuleToFunctionPassAdaptor(  in buildModuleSimplificationPipeline()
1099     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,  in buildModuleSimplificationPipeline()
1103     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());  in buildModuleSimplificationPipeline()
1111       MPM.addPass(  in buildModuleSimplificationPipeline()
1117   MPM.addPass(OpenMPOptPass());  in buildModuleSimplificationPipeline()
1120     MPM.addPass(AttributorPass());  in buildModuleSimplificationPipeline()
1126     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));  in buildModuleSimplificationPipeline()
1128   invokePipelineEarlySimplificationEPCallbacks(MPM, Level);  in buildModuleSimplificationPipeline()
1134   MPM.addPass(IPSCCPPass(  in buildModuleSimplificationPipeline()
1142   MPM.addPass(CalledValuePropagationPass());  in buildModuleSimplificationPipeline()
1145   MPM.addPass(GlobalOptPass());  in buildModuleSimplificationPipeline()
1156   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM),  in buildModuleSimplificationPipeline()
1178     addPreInlinerPasses(MPM, Level, Phase);  in buildModuleSimplificationPipeline()
1182     addPGOInstrPasses(MPM, Level,  in buildModuleSimplificationPipeline()
1188     MPM.addPass(PGOInstrumentationGen(false));  in buildModuleSimplificationPipeline()
1189     addPostPGOLoopRotation(MPM, Level);  in buildModuleSimplificationPipeline()
1190     MPM.addPass(PGOCtxProfLoweringPass());  in buildModuleSimplificationPipeline()
1194     MPM.addPass(PGOIndirectCallPromotion(false, false));  in buildModuleSimplificationPipeline()
1197     MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile,  in buildModuleSimplificationPipeline()
1201     MPM.addPass(MemProfUsePass(PGOOpt->MemoryProfile, PGOOpt->FS));  in buildModuleSimplificationPipeline()
1205     MPM.addPass(SyntheticCountsPropagation());  in buildModuleSimplificationPipeline()
1208     MPM.addPass(PGOForceFunctionAttrsPass(PGOOpt->ColdOptType));  in buildModuleSimplificationPipeline()
1210   MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true));  in buildModuleSimplificationPipeline()
1213     MPM.addPass(buildModuleInlinerPipeline(Level, Phase));  in buildModuleSimplificationPipeline()
1215     MPM.addPass(buildInlinerPipeline(Level, Phase));  in buildModuleSimplificationPipeline()
1219   MPM.addPass(DeadArgumentEliminationPass());  in buildModuleSimplificationPipeline()
1221   MPM.addPass(CoroCleanupPass());  in buildModuleSimplificationPipeline()
1224   MPM.addPass(GlobalOptPass());  in buildModuleSimplificationPipeline()
1225   MPM.addPass(GlobalDCEPass());  in buildModuleSimplificationPipeline()
1227   return MPM;  in buildModuleSimplificationPipeline()
1379   ModulePassManager MPM;  in buildModuleOptimizationPipeline()  local
1384     MPM.addPass(PartialInlinerPass());  in buildModuleOptimizationPipeline()
1396     MPM.addPass(EliminateAvailableExternallyPass());  in buildModuleOptimizationPipeline()
1399     MPM.addPass(InstrOrderFilePass());  in buildModuleOptimizationPipeline()
1404   MPM.addPass(ReversePostOrderFunctionAttrsPass());  in buildModuleOptimizationPipeline()
1412       addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,  in buildModuleOptimizationPipeline()
1417       addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,  in buildModuleOptimizationPipeline()
1431     MPM.addPass(RecomputeGlobalsAAPass());  in buildModuleOptimizationPipeline()
1433   invokeOptimizerEarlyEPCallbacks(MPM, Level);  in buildModuleOptimizationPipeline()
1523   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM),  in buildModuleOptimizationPipeline()
1526   invokeOptimizerLastEPCallbacks(MPM, Level);  in buildModuleOptimizationPipeline()
1532     MPM.addPass(HotColdSplittingPass());  in buildModuleOptimizationPipeline()
1539     MPM.addPass(IROutlinerPass());  in buildModuleOptimizationPipeline()
1545   MPM.addPass(GlobalDCEPass());  in buildModuleOptimizationPipeline()
1546   MPM.addPass(ConstantMergePass());  in buildModuleOptimizationPipeline()
1551     MPM.addPass(MergeFunctionsPass());  in buildModuleOptimizationPipeline()
1554     MPM.addPass(CGProfilePass(LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink ||  in buildModuleOptimizationPipeline()
1561     MPM.addPass(RelLookupTableConverterPass());  in buildModuleOptimizationPipeline()
1563   return MPM;  in buildModuleOptimizationPipeline()
1572   ModulePassManager MPM;  in buildPerModuleDefaultPipeline()  local
1575   MPM.addPass(Annotation2MetadataPass());  in buildPerModuleDefaultPipeline()
1578   MPM.addPass(ForceFunctionAttrsPass());  in buildPerModuleDefaultPipeline()
1581     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));  in buildPerModuleDefaultPipeline()
1584   invokePipelineStartEPCallbacks(MPM, Level);  in buildPerModuleDefaultPipeline()
1590   MPM.addPass(buildModuleSimplificationPipeline(Level, LTOPhase));  in buildPerModuleDefaultPipeline()
1593   MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPhase));  in buildPerModuleDefaultPipeline()
1597     MPM.addPass(PseudoProbeUpdatePass());  in buildPerModuleDefaultPipeline()
1600   addAnnotationRemarksPass(MPM);  in buildPerModuleDefaultPipeline()
1603     addRequiredLTOPreLinkPasses(MPM);  in buildPerModuleDefaultPipeline()
1604   return MPM;  in buildPerModuleDefaultPipeline()
1610   ModulePassManager MPM;  in buildFatLTODefaultPipeline()  local
1612     MPM.addPass(buildThinLTOPreLinkDefaultPipeline(Level));  in buildFatLTODefaultPipeline()
1614     MPM.addPass(buildLTOPreLinkDefaultPipeline(Level));  in buildFatLTODefaultPipeline()
1615   MPM.addPass(EmbedBitcodePass(ThinLTO, EmitSummary));  in buildFatLTODefaultPipeline()
1619     MPM.addPass(buildThinLTODefaultPipeline(Level, /*ImportSummary=*/nullptr));  in buildFatLTODefaultPipeline()
1622     MPM.addPass(  in buildFatLTODefaultPipeline()
1625     addAnnotationRemarksPass(MPM);  in buildFatLTODefaultPipeline()
1627   return MPM;  in buildFatLTODefaultPipeline()
1635   ModulePassManager MPM;  in buildThinLTOPreLinkDefaultPipeline()  local
1638   MPM.addPass(Annotation2MetadataPass());  in buildThinLTOPreLinkDefaultPipeline()
1641   MPM.addPass(ForceFunctionAttrsPass());  in buildThinLTOPreLinkDefaultPipeline()
1644     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));  in buildThinLTOPreLinkDefaultPipeline()
1647   invokePipelineStartEPCallbacks(MPM, Level);  in buildThinLTOPreLinkDefaultPipeline()
1652   MPM.addPass(buildModuleSimplificationPipeline(  in buildThinLTOPreLinkDefaultPipeline()
1663     MPM.addPass(PartialInlinerPass());  in buildThinLTOPreLinkDefaultPipeline()
1667     MPM.addPass(PseudoProbeUpdatePass());  in buildThinLTOPreLinkDefaultPipeline()
1672   invokeOptimizerEarlyEPCallbacks(MPM, Level);  in buildThinLTOPreLinkDefaultPipeline()
1673   invokeOptimizerLastEPCallbacks(MPM, Level);  in buildThinLTOPreLinkDefaultPipeline()
1676   addAnnotationRemarksPass(MPM);  in buildThinLTOPreLinkDefaultPipeline()
1678   addRequiredLTOPreLinkPasses(MPM);  in buildThinLTOPreLinkDefaultPipeline()
1680   return MPM;  in buildThinLTOPreLinkDefaultPipeline()
1685   ModulePassManager MPM;  in buildThinLTODefaultPipeline()  local
1691       MPM.addPass(MemProfContextDisambiguation(ImportSummary));  in buildThinLTODefaultPipeline()
1708     MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));  in buildThinLTODefaultPipeline()
1709     MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));  in buildThinLTODefaultPipeline()
1715     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));  in buildThinLTODefaultPipeline()
1719     MPM.addPass(EliminateAvailableExternallyPass());  in buildThinLTODefaultPipeline()
1720     MPM.addPass(GlobalDCEPass());  in buildThinLTODefaultPipeline()
1721     return MPM;  in buildThinLTODefaultPipeline()
1725   MPM.addPass(buildModuleSimplificationPipeline(  in buildThinLTODefaultPipeline()
1729   MPM.addPass(buildModuleOptimizationPipeline(  in buildThinLTODefaultPipeline()
1733   addAnnotationRemarksPass(MPM);  in buildThinLTODefaultPipeline()
1735   return MPM;  in buildThinLTODefaultPipeline()
1748   ModulePassManager MPM;  in buildLTODefaultPipeline()  local
1750   invokeFullLinkTimeOptimizationEarlyEPCallbacks(MPM, Level);  in buildLTODefaultPipeline()
1754   MPM.addPass(CrossDSOCFIPass());  in buildLTODefaultPipeline()
1759     MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));  in buildLTODefaultPipeline()
1760     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));  in buildLTODefaultPipeline()
1763     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));  in buildLTODefaultPipeline()
1765     invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);  in buildLTODefaultPipeline()
1768     addAnnotationRemarksPass(MPM);  in buildLTODefaultPipeline()
1770     return MPM;  in buildLTODefaultPipeline()
1775     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,  in buildLTODefaultPipeline()
1780     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());  in buildLTODefaultPipeline()
1784   MPM.addPass(OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));  in buildLTODefaultPipeline()
1788   MPM.addPass(GlobalDCEPass(/*InLTOPostLink=*/true));  in buildLTODefaultPipeline()
1792   MPM.addPass(InferFunctionAttrsPass());  in buildLTODefaultPipeline()
1795     MPM.addPass(createModuleToFunctionPassAdaptor(  in buildLTODefaultPipeline()
1802     MPM.addPass(PGOIndirectCallPromotion(  in buildLTODefaultPipeline()
1808     MPM.addPass(IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/  in buildLTODefaultPipeline()
1814     MPM.addPass(CalledValuePropagationPass());  in buildLTODefaultPipeline()
1818   MPM.addPass(  in buildLTODefaultPipeline()
1824   MPM.addPass(ReversePostOrderFunctionAttrsPass());  in buildLTODefaultPipeline()
1827   MPM.addPass(GlobalSplitPass());  in buildLTODefaultPipeline()
1831   MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));  in buildLTODefaultPipeline()
1837     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));  in buildLTODefaultPipeline()
1841     MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));  in buildLTODefaultPipeline()
1843     invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);  in buildLTODefaultPipeline()
1846     addAnnotationRemarksPass(MPM);  in buildLTODefaultPipeline()
1848     return MPM;  in buildLTODefaultPipeline()
1852   MPM.addPass(GlobalOptPass());  in buildLTODefaultPipeline()
1855   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));  in buildLTODefaultPipeline()
1859   MPM.addPass(ConstantMergePass());  in buildLTODefaultPipeline()
1862   MPM.addPass(DeadArgumentEliminationPass());  in buildLTODefaultPipeline()
1874   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM),  in buildLTODefaultPipeline()
1883     MPM.addPass(ModuleInlinerPass(getInlineParamsFromOptLevel(Level),  in buildLTODefaultPipeline()
1887     MPM.addPass(ModuleInlinerWrapperPass(  in buildLTODefaultPipeline()
1898     MPM.addPass(MemProfContextDisambiguation());  in buildLTODefaultPipeline()
1901   MPM.addPass(GlobalOptPass());  in buildLTODefaultPipeline()
1904   MPM.addPass(OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));  in buildLTODefaultPipeline()
1907   MPM.addPass(GlobalDCEPass(/*InLTOPostLink=*/true));  in buildLTODefaultPipeline()
1911   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(ArgumentPromotionPass()));  in buildLTODefaultPipeline()
1927       addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,  in buildLTODefaultPipeline()
1932       addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,  in buildLTODefaultPipeline()
1946   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM),  in buildLTODefaultPipeline()
1949   MPM.addPass(  in buildLTODefaultPipeline()
1955     MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());  in buildLTODefaultPipeline()
1958     MPM.addPass(  in buildLTODefaultPipeline()
2002   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(  in buildLTODefaultPipeline()
2007   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM),  in buildLTODefaultPipeline()
2014   MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));  in buildLTODefaultPipeline()
2017   MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));  in buildLTODefaultPipeline()
2021     MPM.addPass(HotColdSplittingPass());  in buildLTODefaultPipeline()
2042   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(LateFPM)));  in buildLTODefaultPipeline()
2045   MPM.addPass(EliminateAvailableExternallyPass());  in buildLTODefaultPipeline()
2048   MPM.addPass(GlobalDCEPass(/*InLTOPostLink=*/true));  in buildLTODefaultPipeline()
2051     MPM.addPass(MergeFunctionsPass());  in buildLTODefaultPipeline()
2054     MPM.addPass(CGProfilePass(/*InLTOPostLink=*/true));  in buildLTODefaultPipeline()
2056   invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);  in buildLTODefaultPipeline()
2059   addAnnotationRemarksPass(MPM);  in buildLTODefaultPipeline()
2061   return MPM;  in buildLTODefaultPipeline()
2069   ModulePassManager MPM;  in buildO0DefaultPipeline()  local
2076     MPM.addPass(SampleProfileProbePass(TM));  in buildO0DefaultPipeline()
2081         MPM,  in buildO0DefaultPipeline()
2087   MPM.addPass(createModuleToFunctionPassAdaptor(  in buildO0DefaultPipeline()
2090   invokePipelineStartEPCallbacks(MPM, Level);  in buildO0DefaultPipeline()
2093     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));  in buildO0DefaultPipeline()
2095   invokePipelineEarlySimplificationEPCallbacks(MPM, Level);  in buildO0DefaultPipeline()
2101   MPM.addPass(AlwaysInlinerPass(  in buildO0DefaultPipeline()
2105     MPM.addPass(MergeFunctionsPass());  in buildO0DefaultPipeline()
2108     MPM.addPass(  in buildO0DefaultPipeline()
2115       MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));  in buildO0DefaultPipeline()
2121       MPM.addPass(createModuleToFunctionPassAdaptor(  in buildO0DefaultPipeline()
2129       MPM.addPass(createModuleToFunctionPassAdaptor(  in buildO0DefaultPipeline()
2137       MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));  in buildO0DefaultPipeline()
2140   invokeOptimizerEarlyEPCallbacks(MPM, Level);  in buildO0DefaultPipeline()
2146       MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));  in buildO0DefaultPipeline()
2156   MPM.addPass(CoroConditionalWrapper(std::move(CoroPM)));  in buildO0DefaultPipeline()
2158   invokeOptimizerLastEPCallbacks(MPM, Level);  in buildO0DefaultPipeline()
2161     addRequiredLTOPreLinkPasses(MPM);  in buildO0DefaultPipeline()
2163   MPM.addPass(createModuleToFunctionPassAdaptor(AnnotationRemarksPass()));  in buildO0DefaultPipeline()
2165   return MPM;  in buildO0DefaultPipeline()