1 //===- HotColdSplitting.cpp -- Outline Cold Regions -------------*- C++ -*-===//
10 /// The goal of hot/cold splitting is to improve the memory locality of code.
11 /// The splitting pass does this by identifying cold blocks and moving them into
14 /// When the splitting pass finds a cold block (referred to as "the sink"), it
15 /// grows a maximal cold region around that block. The maximal region contains
17 /// cold as the sink. Once a region is found, it's split out of the original
63 STATISTIC(NumColdRegionsFound, "Number of cold regions found.");
64 STATISTIC(NumColdRegionsOutlined, "Number of cold regions outlined.");
68 static cl::opt<bool> EnableStaticAnalysis("hot-cold-static-analysis",
73                        cl::desc("Base penalty for splitting cold code (as a "
77     "enable-cold-section", cl::init(false), cl::Hidden,
78     cl::desc("Enable placement of extracted cold functions"
79              " into a separate section after hot-cold splitting."));
82     ColdSectionName("hotcoldsplit-cold-section-name", cl::init("__llvm_cold"),
84                     cl::desc("Name for the section containing cold functions "
85                              "extracted by hot-cold splitting."));
92     "hotcoldsplit-cold-probability-denom", cl::init(100), cl::Hidden,
93     cl::desc("Divisor of cold branch probability."
100 /// A no successor, non-return block probably ends in unreachable and is cold.
143   // The block is cold if it calls/invokes a cold function. However, do not  in unlikelyExecuted()
144   // mark sanitizer traps as cold.  in unlikelyExecuted()
147       if (CB->hasFnAttr(Attribute::Cold) &&  in unlikelyExecuted()
151   // The block is cold if it has an unreachable terminator, unless it's  in unlikelyExecuted()
192 /// Mark \p F cold. Based on this assumption, also optimize it for minimum size.
194 /// module has profile data), set entry count to 0 to ensure treated as cold.
197   assert(!F.hasOptNone() && "Can't mark this cold");  in markFunctionCold()
199   if (!F.hasFnAttribute(Attribute::Cold)) {  in markFunctionCold()
200     F.addFnAttr(Attribute::Cold);  in markFunctionCold()
219 /// Check whether \p F is inherently cold.
221   if (F.hasFnAttribute(Attribute::Cold))  in isFunctionCold()
224   if (F.getCallingConv() == CallingConv::Cold)  in isFunctionCold()
241     // Find cold blocks of successors of BB during a reverse postorder traversal.  in isBasicBlockCold()
244     // A statically cold BB would be known before it is visited  in isBasicBlockCold()
256 // Returns false if the function should not be considered for hot-cold split
266   // should not be considered cold, as the function may be a trampoline.  in shouldOutlineFrom()
411 // Split the single \p EntryPoint cold region. \p CE is the region code
423       OutF->setCallingConv(CallingConv::Cold);  in extractColdRegion()
424       CI->setCallingConv(CallingConv::Cold);  in extractColdRegion()
441              << ore::NV("Original", OrigF) << " split cold code into "  in extractColdRegion()
475   /// Whether the entire function is cold.
522       // If the predecessor is cold and has no predecessors, the entire  in create()
523       // function must be cold.  in create()
530       // predecessor (or any of its predecessors) cold.  in create()
549     // If the sink can be added to the cold region, do so. It's considered as  in create()
552     // Otherwise, split cold sink-successor blocks using a separate region.  in create()
578       // any of its successors) cold.  in create()
603   /// Whether the entire function containing this region is cold.
641   // The set of cold blocks outlined.  in outlineColdRegions()
644   // The set of cold blocks cannot be outlined.  in outlineColdRegions()
647   // Set of cold blocks obtained with RPOT.  in outlineColdRegions()
679   // Find all cold regions.  in outlineColdRegions()
693       dbgs() << "Found a cold block:\n";  in outlineColdRegions()
708         LLVM_DEBUG(dbgs() << "Entire function is cold\n");  in outlineColdRegions()
715           dbgs() << "Hot/cold splitting attempting to outline these blocks:\n";  in outlineColdRegions()
725             /* Suffix */ "cold." + std::to_string(OutlinedFunctionID));  in outlineColdRegions()
741               dbgs() << "  contains cold block:" << Block->getName() << "\n";  in outlineColdRegions()
748           // The cold block region cannot be outlined.  in outlineColdRegions()
752               // Will skip this cold block in the loop to save the compile time  in outlineColdRegions()
764   // Outline single-entry cold regions, splitting up larger regions as needed.  in outlineColdRegions()
789     // Detect inherently cold functions and mark them as such.  in run()