1 //===------- X86InsertPrefetch.cpp - Insert cache prefetch hints ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass applies cache prefetch instructions based on a profile. The pass 10 // assumes DiscriminateMemOps ran immediately before, to ensure debug info 11 // matches the one used at profile generation time. The profile is encoded in 12 // afdo format (text or binary). It contains prefetch hints recommendations. 13 // Each recommendation is made in terms of debug info locations, a type (i.e. 14 // nta, t{0|1|2}) and a delta. The debug info identifies an instruction with a 15 // memory operand (see X86DiscriminateMemOps). The prefetch will be made for 16 // a location at that memory operand + the delta specified in the 17 // recommendation. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "X86.h" 22 #include "X86InstrBuilder.h" 23 #include "X86InstrInfo.h" 24 #include "X86MachineFunctionInfo.h" 25 #include "X86Subtarget.h" 26 #include "llvm/CodeGen/MachineFunctionPass.h" 27 #include "llvm/CodeGen/MachineModuleInfo.h" 28 #include "llvm/IR/DebugInfoMetadata.h" 29 #include "llvm/ProfileData/SampleProf.h" 30 #include "llvm/ProfileData/SampleProfReader.h" 31 #include "llvm/Support/VirtualFileSystem.h" 32 #include "llvm/Transforms/IPO/SampleProfile.h" 33 using namespace llvm; 34 using namespace sampleprof; 35 36 static cl::opt<std::string> 37 PrefetchHintsFile("prefetch-hints-file", 38 cl::desc("Path to the prefetch hints profile. See also " 39 "-x86-discriminate-memops"), 40 cl::Hidden); 41 namespace { 42 43 class X86InsertPrefetch : public MachineFunctionPass { 44 void getAnalysisUsage(AnalysisUsage &AU) const override; 45 bool doInitialization(Module &) override; 46 47 bool runOnMachineFunction(MachineFunction &MF) override; 48 struct PrefetchInfo { 49 unsigned InstructionID; 50 int64_t Delta; 51 }; 52 typedef SmallVectorImpl<PrefetchInfo> Prefetches; 53 bool findPrefetchInfo(const FunctionSamples *Samples, const MachineInstr &MI, 54 Prefetches &prefetches) const; 55 56 public: 57 static char ID; 58 X86InsertPrefetch(const std::string &PrefetchHintsFilename); 59 StringRef getPassName() const override { 60 return "X86 Insert Cache Prefetches"; 61 } 62 63 private: 64 std::string Filename; 65 std::unique_ptr<SampleProfileReader> Reader; 66 }; 67 68 using PrefetchHints = SampleRecord::CallTargetMap; 69 70 // Return any prefetching hints for the specified MachineInstruction. The hints 71 // are returned as pairs (name, delta). 72 ErrorOr<PrefetchHints> getPrefetchHints(const FunctionSamples *TopSamples, 73 const MachineInstr &MI) { 74 if (const auto &Loc = MI.getDebugLoc()) 75 if (const auto *Samples = TopSamples->findFunctionSamples(Loc)) 76 return Samples->findCallTargetMapAt(FunctionSamples::getOffset(Loc), 77 Loc->getBaseDiscriminator()); 78 return std::error_code(); 79 } 80 81 // The prefetch instruction can't take memory operands involving vector 82 // registers. 83 bool IsMemOpCompatibleWithPrefetch(const MachineInstr &MI, int Op) { 84 Register BaseReg = MI.getOperand(Op + X86::AddrBaseReg).getReg(); 85 Register IndexReg = MI.getOperand(Op + X86::AddrIndexReg).getReg(); 86 return (BaseReg == 0 || 87 X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg) || 88 X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg)) && 89 (IndexReg == 0 || 90 X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg) || 91 X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg)); 92 } 93 94 } // end anonymous namespace 95 96 //===----------------------------------------------------------------------===// 97 // Implementation 98 //===----------------------------------------------------------------------===// 99 100 char X86InsertPrefetch::ID = 0; 101 102 X86InsertPrefetch::X86InsertPrefetch(const std::string &PrefetchHintsFilename) 103 : MachineFunctionPass(ID), Filename(PrefetchHintsFilename) {} 104 105 /// Return true if the provided MachineInstruction has cache prefetch hints. In 106 /// that case, the prefetch hints are stored, in order, in the Prefetches 107 /// vector. 108 bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples, 109 const MachineInstr &MI, 110 Prefetches &Prefetches) const { 111 assert(Prefetches.empty() && 112 "Expected caller passed empty PrefetchInfo vector."); 113 static constexpr std::pair<StringLiteral, unsigned> HintTypes[] = { 114 {"_nta_", X86::PREFETCHNTA}, 115 {"_t0_", X86::PREFETCHT0}, 116 {"_t1_", X86::PREFETCHT1}, 117 {"_t2_", X86::PREFETCHT2}, 118 }; 119 static const char *SerializedPrefetchPrefix = "__prefetch"; 120 121 const ErrorOr<PrefetchHints> T = getPrefetchHints(TopSamples, MI); 122 if (!T) 123 return false; 124 int16_t max_index = -1; 125 // Convert serialized prefetch hints into PrefetchInfo objects, and populate 126 // the Prefetches vector. 127 for (const auto &S_V : *T) { 128 StringRef Name = S_V.getKey(); 129 if (Name.consume_front(SerializedPrefetchPrefix)) { 130 int64_t D = static_cast<int64_t>(S_V.second); 131 unsigned IID = 0; 132 for (const auto &HintType : HintTypes) { 133 if (Name.startswith(HintType.first)) { 134 Name = Name.drop_front(HintType.first.size()); 135 IID = HintType.second; 136 break; 137 } 138 } 139 if (IID == 0) 140 return false; 141 uint8_t index = 0; 142 Name.consumeInteger(10, index); 143 144 if (index >= Prefetches.size()) 145 Prefetches.resize(index + 1); 146 Prefetches[index] = {IID, D}; 147 max_index = std::max(max_index, static_cast<int16_t>(index)); 148 } 149 } 150 assert(max_index + 1 >= 0 && 151 "Possible overflow: max_index + 1 should be positive."); 152 assert(static_cast<size_t>(max_index + 1) == Prefetches.size() && 153 "The number of prefetch hints received should match the number of " 154 "PrefetchInfo objects returned"); 155 return !Prefetches.empty(); 156 } 157 158 bool X86InsertPrefetch::doInitialization(Module &M) { 159 if (Filename.empty()) 160 return false; 161 162 LLVMContext &Ctx = M.getContext(); 163 // TODO: Propagate virtual file system into LLVM targets. 164 auto FS = vfs::getRealFileSystem(); 165 ErrorOr<std::unique_ptr<SampleProfileReader>> ReaderOrErr = 166 SampleProfileReader::create(Filename, Ctx, *FS); 167 if (std::error_code EC = ReaderOrErr.getError()) { 168 std::string Msg = "Could not open profile: " + EC.message(); 169 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg, 170 DiagnosticSeverity::DS_Warning)); 171 return false; 172 } 173 Reader = std::move(ReaderOrErr.get()); 174 Reader->read(); 175 return true; 176 } 177 178 void X86InsertPrefetch::getAnalysisUsage(AnalysisUsage &AU) const { 179 AU.setPreservesAll(); 180 MachineFunctionPass::getAnalysisUsage(AU); 181 } 182 183 bool X86InsertPrefetch::runOnMachineFunction(MachineFunction &MF) { 184 if (!Reader) 185 return false; 186 const FunctionSamples *Samples = Reader->getSamplesFor(MF.getFunction()); 187 if (!Samples) 188 return false; 189 190 bool Changed = false; 191 192 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); 193 SmallVector<PrefetchInfo, 4> Prefetches; 194 for (auto &MBB : MF) { 195 for (auto MI = MBB.instr_begin(); MI != MBB.instr_end();) { 196 auto Current = MI; 197 ++MI; 198 199 int Offset = X86II::getMemoryOperandNo(Current->getDesc().TSFlags); 200 if (Offset < 0) 201 continue; 202 unsigned Bias = X86II::getOperandBias(Current->getDesc()); 203 int MemOpOffset = Offset + Bias; 204 // FIXME(mtrofin): ORE message when the recommendation cannot be taken. 205 if (!IsMemOpCompatibleWithPrefetch(*Current, MemOpOffset)) 206 continue; 207 Prefetches.clear(); 208 if (!findPrefetchInfo(Samples, *Current, Prefetches)) 209 continue; 210 assert(!Prefetches.empty() && 211 "The Prefetches vector should contain at least a value if " 212 "findPrefetchInfo returned true."); 213 for (auto &PrefInfo : Prefetches) { 214 unsigned PFetchInstrID = PrefInfo.InstructionID; 215 int64_t Delta = PrefInfo.Delta; 216 const MCInstrDesc &Desc = TII->get(PFetchInstrID); 217 MachineInstr *PFetch = 218 MF.CreateMachineInstr(Desc, Current->getDebugLoc(), true); 219 MachineInstrBuilder MIB(MF, PFetch); 220 221 static_assert(X86::AddrBaseReg == 0 && X86::AddrScaleAmt == 1 && 222 X86::AddrIndexReg == 2 && X86::AddrDisp == 3 && 223 X86::AddrSegmentReg == 4, 224 "Unexpected change in X86 operand offset order."); 225 226 // This assumes X86::AddBaseReg = 0, {...}ScaleAmt = 1, etc. 227 // FIXME(mtrofin): consider adding a: 228 // MachineInstrBuilder::set(unsigned offset, op). 229 MIB.addReg(Current->getOperand(MemOpOffset + X86::AddrBaseReg).getReg()) 230 .addImm( 231 Current->getOperand(MemOpOffset + X86::AddrScaleAmt).getImm()) 232 .addReg( 233 Current->getOperand(MemOpOffset + X86::AddrIndexReg).getReg()) 234 .addImm(Current->getOperand(MemOpOffset + X86::AddrDisp).getImm() + 235 Delta) 236 .addReg(Current->getOperand(MemOpOffset + X86::AddrSegmentReg) 237 .getReg()); 238 239 if (!Current->memoperands_empty()) { 240 MachineMemOperand *CurrentOp = *(Current->memoperands_begin()); 241 MIB.addMemOperand(MF.getMachineMemOperand( 242 CurrentOp, CurrentOp->getOffset() + Delta, CurrentOp->getSize())); 243 } 244 245 // Insert before Current. This is because Current may clobber some of 246 // the registers used to describe the input memory operand. 247 MBB.insert(Current, PFetch); 248 Changed = true; 249 } 250 } 251 } 252 return Changed; 253 } 254 255 FunctionPass *llvm::createX86InsertPrefetchPass() { 256 return new X86InsertPrefetch(PrefetchHintsFile); 257 } 258