Lines Matching +full:atomic +full:- +full:threshold +full:- +full:us
1 //===- Construction of pass pipelines -------------------------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
15 //===----------------------------------------------------------------------===//
146 "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
147 cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
149 "Heuristics-based inliner version"),
151 "Use development mode (runtime-loadable model)"),
153 "Use release mode (AOT-compiled model)")));
156 "enable-npm-synthetic-counts", cl::Hidden,
162 EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
166 static cl::opt<bool> EnableModuleInliner("enable-module-inliner",
171 "mandatory-inlining-first", cl::init(false), cl::Hidden,
172 cl::desc("Perform mandatory inlinings module-wide, before performing "
176 "eagerly-invalidate-analyses", cl::init(true), cl::Hidden,
180 "enable-merge-functions", cl::init(false), cl::Hidden,
184 "enable-post-pgo-loop-rotation", cl::init(true), cl::Hidden,
188 "enable-global-analyses", cl::init(true), cl::Hidden,
189 cl::desc("Enable inter-procedural analyses"));
192 RunPartialInlining("enable-partial-inlining", cl::init(false), cl::Hidden,
196 "extra-vectorizer-passes", cl::init(false), cl::Hidden,
199 static cl::opt<bool> RunNewGVN("enable-newgvn", cl::init(false), cl::Hidden,
203 "enable-loopinterchange", cl::init(false), cl::Hidden,
206 static cl::opt<bool> EnableUnrollAndJam("enable-unroll-and-jam",
210 static cl::opt<bool> EnableLoopFlatten("enable-loop-flatten", cl::init(false),
215 // optimization at Oz, since loop-idiom recognition can then recognize things
220 "enable-loop-header-duplication", cl::init(false), cl::Hidden,
224 EnableDFAJumpThreading("enable-dfa-jump-thread",
230 "enable-pgo-force-function-attrs",
235 EnableHotColdSplit("hot-cold-split",
236 cl::desc("Enable hot-cold splitting pass"));
238 static cl::opt<bool> EnableIROutliner("ir-outliner", cl::init(false),
243 DisablePreInliner("disable-preinline", cl::init(false), cl::Hidden,
244 cl::desc("Disable pre-instrumentation inliner"));
247 "preinline-threshold", cl::Hidden, cl::init(75),
248 cl::desc("Control the amount of inlining in pre-instrumentation inliner "
252 EnableGVNHoist("enable-gvn-hoist",
256 EnableGVNSink("enable-gvn-sink",
260 "enable-jump-table-to-switch",
266 EnableCHR("enable-chr", cl::init(true), cl::Hidden,
270 "flattened-profile-used", cl::init(false), cl::Hidden,
275 "enable-order-file-instrumentation", cl::init(false), cl::Hidden,
279 EnableMatrix("enable-matrix", cl::init(false), cl::Hidden,
283 "enable-constraint-elimination", cl::init(true), cl::Hidden,
288 "attributor-enable", cl::Hidden, cl::init(AttributorRunOption::NONE),
289 cl::desc("Enable the attributor inter-procedural deduction pass"),
293 "enable module-wide attributor runs"),
300 "enable-sampled-instrumentation", cl::init(false), cl::Hidden,
303 "enable-loop-versioning-licm", cl::init(false), cl::Hidden,
323 InlinerThreshold = -1; in PipelineTuningOptions()
418 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */)); in buildO1FunctionSimplificationPipeline()
488 PGOOpt->Action != PGOOptions::SampleUse) in buildO1FunctionSimplificationPipeline()
564 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */)); in buildFunctionSimplificationPipeline()
602 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse && in buildFunctionSimplificationPipeline()
644 // Disable header duplication in loop rotation at -Oz. in buildFunctionSimplificationPipeline()
679 PGOOpt->Action != PGOOptions::SampleUse) in buildFunctionSimplificationPipeline()
728 // Re-consider control flow based optimizations after redundancy elimination, in buildFunctionSimplificationPipeline()
781 // FIXME: The hint threshold has the same value used by the regular inliner in addPreInlinerPasses()
813 // Disable header duplication in loop rotation at -Oz. in addPostPGOLoopRotation()
837 // RequireAnalysisPass for PSI before subsequent non-module passes. in addPGOInstrPasses()
859 Options.Atomic = AtomicCounterUpdate; in addPGOInstrPasses()
872 // RequireAnalysisPass for PSI before subsequent non-module passes. in addPGOInstrPassesForO0()
886 Options.Atomic = AtomicCounterUpdate; in addPGOInstrPassesForO0()
898 if (PTO.InlinerThreshold == -1) in buildInlinerPipeline()
902 // For PreLinkThinLTO + SamplePGO, set hot-caller threshold to 0 to in buildInlinerPipeline()
909 PGOOpt->Action == PGOOptions::SampleUse) in buildInlinerPipeline()
958 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if in buildInlinerPipeline()
996 // For PreLinkThinLTO + SamplePGO, set hot-caller threshold to 0 to in buildModuleInlinerPipeline()
1003 PGOOpt->Action == PGOOptions::SampleUse) in buildModuleInlinerPipeline()
1011 // inlining is processed in bottom-up order. in buildModuleInlinerPipeline()
1012 // While in module inliner, the inlining order is a priority-based order in buildModuleInlinerPipeline()
1042 if (PGOOpt && PGOOpt->PseudoProbeForProfiling && in buildModuleSimplificationPipeline()
1046 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse); in buildModuleSimplificationPipeline()
1099 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile, in buildModuleSimplificationPipeline()
1100 PGOOpt->ProfileRemappingFile, Phase)); in buildModuleSimplificationPipeline()
1102 // RequireAnalysisPass for PSI before subsequent non-module passes. in buildModuleSimplificationPipeline()
1116 // (quick!) no-op if there are no OpenMP runtime calls present in the module. in buildModuleSimplificationPipeline()
1133 // years, it should be re-analyzed. in buildModuleSimplificationPipeline()
1141 // they may target at run-time. This should follow IPSCCP. in buildModuleSimplificationPipeline()
1159 // We already asserted this happens in non-FullLTOPostLink earlier. in buildModuleSimplificationPipeline()
1163 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRInstr; in buildModuleSimplificationPipeline()
1165 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRUse; in buildModuleSimplificationPipeline()
1166 const bool IsMemprofUse = IsPGOPreLink && !PGOOpt->MemoryProfile.empty(); in buildModuleSimplificationPipeline()
1184 /*IsCS=*/false, PGOOpt->AtomicCounterUpdate, in buildModuleSimplificationPipeline()
1185 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, in buildModuleSimplificationPipeline()
1186 PGOOpt->FS); in buildModuleSimplificationPipeline()
1196 if (IsPGOPreLink && PGOOpt->CSAction == PGOOptions::CSIRInstr) in buildModuleSimplificationPipeline()
1197 MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile, in buildModuleSimplificationPipeline()
1201 MPM.addPass(MemProfUsePass(PGOOpt->MemoryProfile, PGOOpt->FS)); in buildModuleSimplificationPipeline()
1203 // Synthesize function entry counts for non-PGO compilation. in buildModuleSimplificationPipeline()
1208 MPM.addPass(PGOForceFunctionAttrsPass(PGOOpt->ColdOptType)); in buildModuleSimplificationPipeline()
1241 // parallel execution resources of an out-of-order processor. We also then in addVectorPasses()
1243 // FIXME: It would be really good to use a loop-integrated instruction in addVectorPasses()
1255 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have in addVectorPasses()
1256 // become constant-offset, thus enabling SROA and alloca promotion. Do so. in addVectorPasses()
1258 // or SimplifyCFG passes scheduled after us, that would cleanup in addVectorPasses()
1276 // common computations, hoist loop-invariant aspects out of any outer loop, in addVectorPasses()
1332 // parallel execution resources of an out-of-order processor. We also then in addVectorPasses()
1334 // FIXME: It would be really good to use a loop-integrated instruction in addVectorPasses()
1347 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have in addVectorPasses()
1348 // become constant-offset, thus enabling SROA and alloca promotion. Do so. in addVectorPasses()
1350 // or SimplifyCFG passes scheduled after us, that would cleanup in addVectorPasses()
1363 // 2. It helps to clean up some loop-invariant code created by the loop in addVectorPasses()
1371 // alignment information, try to re-derive it here. in addVectorPasses()
1388 // are eligible for inlining at link-time. Note if they are unreferenced they in buildModuleOptimizationPipeline()
1394 // preserved during prelinking for link-time inlining decisions. in buildModuleOptimizationPipeline()
1401 // Do RPO function attribute inference across the module to forward-propagate in buildModuleOptimizationPipeline()
1408 // cross-module inline has not been done yet. The context sensitive in buildModuleOptimizationPipeline()
1411 if (PGOOpt->CSAction == PGOOptions::CSIRInstr) in buildModuleOptimizationPipeline()
1413 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate, in buildModuleOptimizationPipeline()
1414 PGOOpt->CSProfileGenFile, PGOOpt->ProfileRemappingFile, in buildModuleOptimizationPipeline()
1415 PGOOpt->FS); in buildModuleOptimizationPipeline()
1416 else if (PGOOpt->CSAction == PGOOptions::CSIRUse) in buildModuleOptimizationPipeline()
1418 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate, in buildModuleOptimizationPipeline()
1419 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, in buildModuleOptimizationPipeline()
1420 PGOOpt->FS); in buildModuleOptimizationPipeline()
1423 // Re-compute GlobalsAA here prior to function passes. This is particularly in buildModuleOptimizationPipeline()
1424 // useful as the above will have inlined, DCE'ed, and function-attr in buildModuleOptimizationPipeline()
1439 // size. Other optimizations which runs later might get benefit of no-alias in buildModuleOptimizationPipeline()
1464 // FIXME: We need to run some loop optimizations to re-rotate loops after in buildModuleOptimizationPipeline()
1468 // rather than on each loop in an inside-out manner, and so they are actually in buildModuleOptimizationPipeline()
1474 // First rotate loops that may have been un-rotated by prior passes. in buildModuleOptimizationPipeline()
1475 // Disable header duplication at -Oz. in buildModuleOptimizationPipeline()
1490 // llvm.loop.distribute=true or when -enable-loop-distribute is specified. in buildModuleOptimizationPipeline()
1493 // Populates the VFABI attribute with the scalar-to-vector mappings in buildModuleOptimizationPipeline()
1509 // passes to avoid re-sinking, but before SimplifyCFG because it can allow in buildModuleOptimizationPipeline()
1517 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG. in buildModuleOptimizationPipeline()
1559 // Until the issue fixed, disable this pass during pre-linking phase. in buildModuleOptimizationPipeline()
1580 if (PGOOpt && PGOOpt->DebugInfoForProfiling) in buildPerModuleDefaultPipeline()
1595 if (PGOOpt && PGOOpt->PseudoProbeForProfiling && in buildPerModuleDefaultPipeline()
1596 PGOOpt->Action == PGOOptions::SampleUse) in buildPerModuleDefaultPipeline()
1617 // Use the ThinLTO post-link pipeline with sample profiling in buildFatLTODefaultPipeline()
1618 if (ThinLTO && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) in buildFatLTODefaultPipeline()
1643 if (PGOOpt && PGOOpt->DebugInfoForProfiling) in buildThinLTOPreLinkDefaultPipeline()
1665 if (PGOOpt && PGOOpt->PseudoProbeForProfiling && in buildThinLTOPreLinkDefaultPipeline()
1666 PGOOpt->Action == PGOOptions::SampleUse) in buildThinLTOPreLinkDefaultPipeline()
1671 // callbacks there in case of in-process ThinLTO called by linker. in buildThinLTOPreLinkDefaultPipeline()
1693 // These passes import type identifier resolutions for whole-program in buildThinLTODefaultPipeline()
1706 // The WPD and LowerTypeTest passes need to run at -O0 to lower type in buildThinLTODefaultPipeline()
1740 // FIXME: We should use a customized pre-link pipeline! in buildLTOPreLinkDefaultPipeline()
1752 // Create a function that performs CFI checks for cross-DSO calls with targets in buildLTODefaultPipeline()
1757 // The WPD and LowerTypeTest passes need to run at -O0 to lower type in buildLTODefaultPipeline()
1773 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) { in buildLTODefaultPipeline()
1775 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile, in buildLTODefaultPipeline()
1776 PGOOpt->ProfileRemappingFile, in buildLTODefaultPipeline()
1779 // RequireAnalysisPass for PSI before subsequent non-module passes. in buildLTODefaultPipeline()
1783 // Try to run OpenMP optimizations, quick no-op if no OpenMP metadata present. in buildLTODefaultPipeline()
1787 // whole-program devirtualization and bitset lowering. in buildLTODefaultPipeline()
1799 // left by the earlier promotion pass that promotes intra-module targets. in buildLTODefaultPipeline()
1800 // This two-step promotion is to save the compile time. For LTO, it should in buildLTODefaultPipeline()
1803 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)); in buildLTODefaultPipeline()
1813 // they may target at run-time. This should follow IPSCCP. in buildLTODefaultPipeline()
1821 // Do RPO function attribute inference across the module to forward-propagate in buildLTODefaultPipeline()
1826 // Use in-range annotations on GEP indices to split globals where beneficial. in buildLTODefaultPipeline()
1833 // Stop here at -O1. in buildLTODefaultPipeline()
1926 if (PGOOpt->CSAction == PGOOptions::CSIRInstr) in buildLTODefaultPipeline()
1928 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate, in buildLTODefaultPipeline()
1929 PGOOpt->CSProfileGenFile, PGOOpt->ProfileRemappingFile, in buildLTODefaultPipeline()
1930 PGOOpt->FS); in buildLTODefaultPipeline()
1931 else if (PGOOpt->CSAction == PGOOptions::CSIRUse) in buildLTODefaultPipeline()
1933 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate, in buildLTODefaultPipeline()
1934 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, in buildLTODefaultPipeline()
1935 PGOOpt->FS); in buildLTODefaultPipeline()
1942 // link-time inlining, and visibility of nocapture attribute. in buildLTODefaultPipeline()
2011 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs in buildLTODefaultPipeline()
2019 // Enable splitting late in the FullLTO post-link pipeline. in buildLTODefaultPipeline()
2033 // passes to avoid re-sinking, but before SimplifyCFG because it can allow in buildLTODefaultPipeline()
2075 if (PGOOpt && PGOOpt->PseudoProbeForProfiling) in buildO0DefaultPipeline()
2078 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr || in buildO0DefaultPipeline()
2079 PGOOpt->Action == PGOOptions::IRUse)) in buildO0DefaultPipeline()
2082 /*RunProfileGen=*/(PGOOpt->Action == PGOOptions::IRInstr), in buildO0DefaultPipeline()
2083 /*IsCS=*/false, PGOOpt->AtomicCounterUpdate, PGOOpt->ProfileFile, in buildO0DefaultPipeline()
2084 PGOOpt->ProfileRemappingFile, PGOOpt->FS); in buildO0DefaultPipeline()
2092 if (PGOOpt && PGOOpt->DebugInfoForProfiling) in buildO0DefaultPipeline()
2175 // per-function local AA logic. This is a stateless, on-demand local set of in buildDefaultAAPipeline()
2179 // Next we query fast, specialized alias analyses that wrap IR-embedded in buildDefaultAAPipeline()
2191 // Add target-specific alias analyses. in buildDefaultAAPipeline()
2193 TM->registerDefaultAliasAnalyses(AA); in buildDefaultAAPipeline()