1 //===-- MachineFunctionSplitter.cpp - Split machine functions //-----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // \file 10 // Uses profile information to split out cold blocks. 11 // 12 // This pass splits out cold machine basic blocks from the parent function. This 13 // implementation leverages the basic block section framework. Blocks marked 14 // cold by this pass are grouped together in a separate section prefixed with 15 // ".text.unlikely.*". The linker can then group these together as a cold 16 // section. The split part of the function is a contiguous region identified by 17 // the symbol "foo.cold". Grouping all cold blocks across functions together 18 // decreases fragmentation and improves icache and itlb utilization. Note that 19 // the overall changes to the binary size are negligible; only a small number of 20 // additional jump instructions may be introduced. 21 // 22 // For the original RFC of this pass please see 23 // https://groups.google.com/d/msg/llvm-dev/RUegaMg-iqc/wFAVxa6fCgAJ 24 //===----------------------------------------------------------------------===// 25 26 #include "llvm/ADT/SmallVector.h" 27 #include "llvm/ADT/Statistic.h" 28 #include "llvm/Analysis/ProfileSummaryInfo.h" 29 #include "llvm/CodeGen/BasicBlockSectionUtils.h" 30 #include "llvm/CodeGen/MachineBasicBlock.h" 31 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" 32 #include "llvm/CodeGen/MachineFunction.h" 33 #include "llvm/CodeGen/MachineFunctionPass.h" 34 #include "llvm/CodeGen/MachineModuleInfo.h" 35 #include "llvm/CodeGen/Passes.h" 36 #include "llvm/IR/Function.h" 37 #include "llvm/IR/Module.h" 38 #include "llvm/InitializePasses.h" 39 #include "llvm/Support/CommandLine.h" 40 41 using namespace llvm; 42 43 // FIXME: This cutoff value is CPU dependent and should be moved to 44 // TargetTransformInfo once we consider enabling this on other platforms. 45 // The value is expressed as a ProfileSummaryInfo integer percentile cutoff. 46 // Defaults to 999950, i.e. all blocks colder than 99.995 percentile are split. 47 // The default was empirically determined to be optimal when considering cutoff 48 // values between 99%-ile to 100%-ile with respect to iTLB and icache metrics on 49 // Intel CPUs. 50 static cl::opt<unsigned> 51 PercentileCutoff("mfs-psi-cutoff", 52 cl::desc("Percentile profile summary cutoff used to " 53 "determine cold blocks. Unused if set to zero."), 54 cl::init(999950), cl::Hidden); 55 56 static cl::opt<unsigned> ColdCountThreshold( 57 "mfs-count-threshold", 58 cl::desc( 59 "Minimum number of times a block must be executed to be retained."), 60 cl::init(1), cl::Hidden); 61 62 namespace { 63 64 class MachineFunctionSplitter : public MachineFunctionPass { 65 public: 66 static char ID; 67 MachineFunctionSplitter() : MachineFunctionPass(ID) { 68 initializeMachineFunctionSplitterPass(*PassRegistry::getPassRegistry()); 69 } 70 71 StringRef getPassName() const override { 72 return "Machine Function Splitter Transformation"; 73 } 74 75 void getAnalysisUsage(AnalysisUsage &AU) const override; 76 77 bool runOnMachineFunction(MachineFunction &F) override; 78 }; 79 } // end anonymous namespace 80 81 static bool isColdBlock(const MachineBasicBlock &MBB, 82 const MachineBlockFrequencyInfo *MBFI, 83 ProfileSummaryInfo *PSI) { 84 Optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB); 85 if (!Count.hasValue()) 86 return true; 87 88 if (PercentileCutoff > 0) { 89 return PSI->isColdCountNthPercentile(PercentileCutoff, *Count); 90 } 91 return (*Count < ColdCountThreshold); 92 } 93 94 bool MachineFunctionSplitter::runOnMachineFunction(MachineFunction &MF) { 95 // TODO: We only target functions with profile data. Static information may 96 // also be considered but we don't see performance improvements yet. 97 if (!MF.getFunction().hasProfileData()) 98 return false; 99 100 // TODO: We don't split functions where a section attribute has been set 101 // since the split part may not be placed in a contiguous region. It may also 102 // be more beneficial to augment the linker to ensure contiguous layout of 103 // split functions within the same section as specified by the attribute. 104 if (MF.getFunction().hasSection() || 105 MF.getFunction().hasFnAttribute("implicit-section-name")) 106 return false; 107 108 // We don't want to proceed further for cold functions 109 // or functions of unknown hotness. Lukewarm functions have no prefix. 110 Optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix(); 111 if (SectionPrefix.hasValue() && 112 (SectionPrefix.getValue().equals("unlikely") || 113 SectionPrefix.getValue().equals("unknown"))) { 114 return false; 115 } 116 117 // Renumbering blocks here preserves the order of the blocks as 118 // sortBasicBlocksAndUpdateBranches uses the numeric identifier to sort 119 // blocks. Preserving the order of blocks is essential to retaining decisions 120 // made by prior passes such as MachineBlockPlacement. 121 MF.RenumberBlocks(); 122 MF.setBBSectionsType(BasicBlockSection::Preset); 123 auto *MBFI = &getAnalysis<MachineBlockFrequencyInfo>(); 124 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); 125 126 SmallVector<MachineBasicBlock *, 2> LandingPads; 127 for (auto &MBB : MF) { 128 if (MBB.isEntryBlock()) 129 continue; 130 131 if (MBB.isEHPad()) 132 LandingPads.push_back(&MBB); 133 else if (isColdBlock(MBB, MBFI, PSI)) 134 MBB.setSectionID(MBBSectionID::ColdSectionID); 135 } 136 137 // We only split out eh pads if all of them are cold. 138 bool HasHotLandingPads = false; 139 for (const MachineBasicBlock *LP : LandingPads) { 140 if (!isColdBlock(*LP, MBFI, PSI)) 141 HasHotLandingPads = true; 142 } 143 if (!HasHotLandingPads) { 144 for (MachineBasicBlock *LP : LandingPads) 145 LP->setSectionID(MBBSectionID::ColdSectionID); 146 } 147 148 auto Comparator = [](const MachineBasicBlock &X, const MachineBasicBlock &Y) { 149 return X.getSectionID().Type < Y.getSectionID().Type; 150 }; 151 llvm::sortBasicBlocksAndUpdateBranches(MF, Comparator); 152 153 return true; 154 } 155 156 void MachineFunctionSplitter::getAnalysisUsage(AnalysisUsage &AU) const { 157 AU.addRequired<MachineModuleInfoWrapperPass>(); 158 AU.addRequired<MachineBlockFrequencyInfo>(); 159 AU.addRequired<ProfileSummaryInfoWrapperPass>(); 160 } 161 162 char MachineFunctionSplitter::ID = 0; 163 INITIALIZE_PASS(MachineFunctionSplitter, "machine-function-splitter", 164 "Split machine functions using profile information", false, 165 false) 166 167 MachineFunctionPass *llvm::createMachineFunctionSplitterPass() { 168 return new MachineFunctionSplitter(); 169 } 170