1 //===- StaticDataSplitter.cpp ---------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // The pass uses branch profile data to assign hotness based section qualifiers 10 // for the following types of static data: 11 // - Jump tables 12 // - Module-internal global variables 13 // - Constant pools 14 // 15 // For the original RFC of this pass please see 16 // https://discourse.llvm.org/t/rfc-profile-guided-static-data-partitioning/83744 17 18 #include "llvm/ADT/Statistic.h" 19 #include "llvm/Analysis/ProfileSummaryInfo.h" 20 #include "llvm/Analysis/StaticDataProfileInfo.h" 21 #include "llvm/CodeGen/MBFIWrapper.h" 22 #include "llvm/CodeGen/MachineBasicBlock.h" 23 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" 24 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" 25 #include "llvm/CodeGen/MachineConstantPool.h" 26 #include "llvm/CodeGen/MachineFunction.h" 27 #include "llvm/CodeGen/MachineFunctionPass.h" 28 #include "llvm/CodeGen/MachineJumpTableInfo.h" 29 #include "llvm/CodeGen/Passes.h" 30 #include "llvm/IR/GlobalVariable.h" 31 #include "llvm/InitializePasses.h" 32 #include "llvm/Pass.h" 33 #include "llvm/Target/TargetLoweringObjectFile.h" 34 35 using namespace llvm; 36 37 #define DEBUG_TYPE "static-data-splitter" 38 39 STATISTIC(NumHotJumpTables, "Number of hot jump tables seen."); 40 STATISTIC(NumColdJumpTables, "Number of cold jump tables seen."); 41 STATISTIC(NumUnknownJumpTables, 42 "Number of jump tables with unknown hotness. They are from functions " 43 "without profile information."); 44 45 class StaticDataSplitter : public MachineFunctionPass { 46 const MachineBranchProbabilityInfo *MBPI = nullptr; 47 const MachineBlockFrequencyInfo *MBFI = nullptr; 48 const ProfileSummaryInfo *PSI = nullptr; 49 StaticDataProfileInfo *SDPI = nullptr; 50 51 // If the global value is a local linkage global variable, return it. 52 // Otherwise, return nullptr. 53 const GlobalVariable *getLocalLinkageGlobalVariable(const GlobalValue *GV); 54 55 // Returns true if the global variable is in one of {.rodata, .bss, .data, 56 // .data.rel.ro} sections. 57 bool inStaticDataSection(const GlobalVariable &GV, const TargetMachine &TM); 58 59 // Returns the constant if the operand refers to a global variable or constant 60 // that gets lowered to static data sections. Otherwise, return nullptr. 61 const Constant *getConstant(const MachineOperand &Op, const TargetMachine &TM, 62 const MachineConstantPool *MCP); 63 64 // Use profiles to partition static data. 65 bool partitionStaticDataWithProfiles(MachineFunction &MF); 66 67 // Update LLVM statistics for a machine function with profiles. 68 void updateStatsWithProfiles(const MachineFunction &MF); 69 70 // Update LLVM statistics for a machine function without profiles. 71 void updateStatsWithoutProfiles(const MachineFunction &MF); 72 73 void annotateStaticDataWithoutProfiles(const MachineFunction &MF); 74 75 public: 76 static char ID; 77 78 StaticDataSplitter() : MachineFunctionPass(ID) { 79 initializeStaticDataSplitterPass(*PassRegistry::getPassRegistry()); 80 } 81 82 StringRef getPassName() const override { return "Static Data Splitter"; } 83 84 void getAnalysisUsage(AnalysisUsage &AU) const override { 85 MachineFunctionPass::getAnalysisUsage(AU); 86 AU.addRequired<MachineBranchProbabilityInfoWrapperPass>(); 87 AU.addRequired<MachineBlockFrequencyInfoWrapperPass>(); 88 AU.addRequired<ProfileSummaryInfoWrapperPass>(); 89 AU.addRequired<StaticDataProfileInfoWrapperPass>(); 90 // This pass does not modify any required analysis results except 91 // StaticDataProfileInfoWrapperPass, but StaticDataProfileInfoWrapperPass 92 // is made an immutable pass that it won't be re-scheduled by pass manager 93 // anyway. So mark setPreservesAll() here for faster compile time. 94 AU.setPreservesAll(); 95 } 96 97 bool runOnMachineFunction(MachineFunction &MF) override; 98 }; 99 100 bool StaticDataSplitter::runOnMachineFunction(MachineFunction &MF) { 101 MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI(); 102 MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI(); 103 PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); 104 105 SDPI = &getAnalysis<StaticDataProfileInfoWrapperPass>() 106 .getStaticDataProfileInfo(); 107 108 const bool ProfileAvailable = PSI && PSI->hasProfileSummary() && MBFI && 109 MF.getFunction().hasProfileData(); 110 111 if (!ProfileAvailable) { 112 annotateStaticDataWithoutProfiles(MF); 113 updateStatsWithoutProfiles(MF); 114 return false; 115 } 116 117 bool Changed = partitionStaticDataWithProfiles(MF); 118 119 updateStatsWithProfiles(MF); 120 return Changed; 121 } 122 123 const Constant * 124 StaticDataSplitter::getConstant(const MachineOperand &Op, 125 const TargetMachine &TM, 126 const MachineConstantPool *MCP) { 127 if (!Op.isGlobal() && !Op.isCPI()) 128 return nullptr; 129 130 if (Op.isGlobal()) { 131 // Find global variables with local linkage. 132 const GlobalVariable *GV = getLocalLinkageGlobalVariable(Op.getGlobal()); 133 // Skip 'llvm.'-prefixed global variables conservatively because they are 134 // often handled specially, and skip those not in static data 135 // sections. 136 if (!GV || GV->getName().starts_with("llvm.") || 137 !inStaticDataSection(*GV, TM)) 138 return nullptr; 139 return GV; 140 } 141 assert(Op.isCPI() && "Op must be constant pool index in this branch"); 142 int CPI = Op.getIndex(); 143 if (CPI == -1) 144 return nullptr; 145 146 assert(MCP != nullptr && "Constant pool info is not available."); 147 const MachineConstantPoolEntry &CPE = MCP->getConstants()[CPI]; 148 149 if (CPE.isMachineConstantPoolEntry()) 150 return nullptr; 151 152 return CPE.Val.ConstVal; 153 } 154 155 bool StaticDataSplitter::partitionStaticDataWithProfiles(MachineFunction &MF) { 156 // If any of the static data (jump tables, global variables, constant pools) 157 // are captured by the analysis, set `Changed` to true. Note this pass won't 158 // invalidate any analysis pass (see `getAnalysisUsage` above), so the main 159 // purpose of tracking and conveying the change (to pass manager) is 160 // informative as opposed to invalidating any analysis results. As an example 161 // of where this information is useful, `PMDataManager::dumpPassInfo` will 162 // only dump pass info if a local change happens, otherwise a pass appears as 163 // "skipped". 164 bool Changed = false; 165 166 MachineJumpTableInfo *MJTI = MF.getJumpTableInfo(); 167 168 // Jump table could be used by either terminating instructions or 169 // non-terminating ones, so we walk all instructions and use 170 // `MachineOperand::isJTI()` to identify jump table operands. 171 // Similarly, `MachineOperand::isCPI()` is used to identify constant pool 172 // usages in the same loop. 173 for (const auto &MBB : MF) { 174 std::optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB); 175 for (const MachineInstr &I : MBB) { 176 for (const MachineOperand &Op : I.operands()) { 177 if (!Op.isJTI() && !Op.isGlobal() && !Op.isCPI()) 178 continue; 179 180 if (Op.isJTI()) { 181 assert(MJTI != nullptr && "Jump table info is not available."); 182 const int JTI = Op.getIndex(); 183 // This is not a source block of jump table. 184 if (JTI == -1) 185 continue; 186 187 auto Hotness = MachineFunctionDataHotness::Hot; 188 189 // Hotness is based on source basic block hotness. 190 // TODO: PSI APIs are about instruction hotness. Introduce API for 191 // data access hotness. 192 if (Count && PSI->isColdCount(*Count)) 193 Hotness = MachineFunctionDataHotness::Cold; 194 195 Changed |= MJTI->updateJumpTableEntryHotness(JTI, Hotness); 196 } else if (const Constant *C = 197 getConstant(Op, MF.getTarget(), MF.getConstantPool())) { 198 SDPI->addConstantProfileCount(C, Count); 199 Changed = true; 200 } 201 } 202 } 203 } 204 return Changed; 205 } 206 207 const GlobalVariable * 208 StaticDataSplitter::getLocalLinkageGlobalVariable(const GlobalValue *GV) { 209 // LLVM IR Verifier requires that a declaration must have valid declaration 210 // linkage, and local linkages are not among the valid ones. So there is no 211 // need to check GV is not a declaration here. 212 return (GV && GV->hasLocalLinkage()) ? dyn_cast<GlobalVariable>(GV) : nullptr; 213 } 214 215 bool StaticDataSplitter::inStaticDataSection(const GlobalVariable &GV, 216 const TargetMachine &TM) { 217 218 SectionKind Kind = TargetLoweringObjectFile::getKindForGlobal(&GV, TM); 219 return Kind.isData() || Kind.isReadOnly() || Kind.isReadOnlyWithRel() || 220 Kind.isBSS(); 221 } 222 223 void StaticDataSplitter::updateStatsWithProfiles(const MachineFunction &MF) { 224 if (!AreStatisticsEnabled()) 225 return; 226 227 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) { 228 for (const auto &JumpTable : MJTI->getJumpTables()) { 229 if (JumpTable.Hotness == MachineFunctionDataHotness::Hot) { 230 ++NumHotJumpTables; 231 } else { 232 assert(JumpTable.Hotness == MachineFunctionDataHotness::Cold && 233 "A jump table is either hot or cold when profile information is " 234 "available."); 235 ++NumColdJumpTables; 236 } 237 } 238 } 239 } 240 241 void StaticDataSplitter::annotateStaticDataWithoutProfiles( 242 const MachineFunction &MF) { 243 for (const auto &MBB : MF) 244 for (const MachineInstr &I : MBB) 245 for (const MachineOperand &Op : I.operands()) 246 if (const Constant *C = 247 getConstant(Op, MF.getTarget(), MF.getConstantPool())) 248 SDPI->addConstantProfileCount(C, std::nullopt); 249 } 250 251 void StaticDataSplitter::updateStatsWithoutProfiles(const MachineFunction &MF) { 252 if (!AreStatisticsEnabled()) 253 return; 254 255 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) { 256 NumUnknownJumpTables += MJTI->getJumpTables().size(); 257 } 258 } 259 260 char StaticDataSplitter::ID = 0; 261 262 INITIALIZE_PASS_BEGIN(StaticDataSplitter, DEBUG_TYPE, "Split static data", 263 false, false) 264 INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass) 265 INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass) 266 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) 267 INITIALIZE_PASS_DEPENDENCY(StaticDataProfileInfoWrapperPass) 268 INITIALIZE_PASS_END(StaticDataSplitter, DEBUG_TYPE, "Split static data", false, 269 false) 270 271 MachineFunctionPass *llvm::createStaticDataSplitterPass() { 272 return new StaticDataSplitter(); 273 } 274