1 //===- X86DiscriminateMemOps.cpp - Unique IDs for Mem Ops -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// This pass aids profile-driven cache prefetch insertion by ensuring all
10 /// instructions that have a memory operand are distinguishible from each other.
11 ///
12 //===----------------------------------------------------------------------===//
13
14 #include "X86.h"
15 #include "X86InstrBuilder.h"
16 #include "X86InstrInfo.h"
17 #include "X86MachineFunctionInfo.h"
18 #include "X86Subtarget.h"
19 #include "llvm/CodeGen/MachineFunctionPass.h"
20 #include "llvm/CodeGen/MachineModuleInfo.h"
21 #include "llvm/IR/DebugInfoMetadata.h"
22 #include "llvm/ProfileData/SampleProf.h"
23 #include "llvm/ProfileData/SampleProfReader.h"
24 #include "llvm/Support/Debug.h"
25 #include "llvm/Transforms/IPO/SampleProfile.h"
26 #include <optional>
27 using namespace llvm;
28
29 #define DEBUG_TYPE "x86-discriminate-memops"
30
31 static cl::opt<bool> EnableDiscriminateMemops(
32 DEBUG_TYPE, cl::init(false),
33 cl::desc("Generate unique debug info for each instruction with a memory "
34 "operand. Should be enabled for profile-driven cache prefetching, "
35 "both in the build of the binary being profiled, as well as in "
36 "the build of the binary consuming the profile."),
37 cl::Hidden);
38
39 static cl::opt<bool> BypassPrefetchInstructions(
40 "x86-bypass-prefetch-instructions", cl::init(true),
41 cl::desc("When discriminating instructions with memory operands, ignore "
42 "prefetch instructions. This ensures the other memory operand "
43 "instructions have the same identifiers after inserting "
44 "prefetches, allowing for successive insertions."),
45 cl::Hidden);
46
47 namespace {
48
49 using Location = std::pair<StringRef, unsigned>;
50
diToLocation(const DILocation * Loc)51 Location diToLocation(const DILocation *Loc) {
52 return std::make_pair(Loc->getFilename(), Loc->getLine());
53 }
54
55 /// Ensure each instruction having a memory operand has a distinct <LineNumber,
56 /// Discriminator> pair.
updateDebugInfo(MachineInstr * MI,const DILocation * Loc)57 void updateDebugInfo(MachineInstr *MI, const DILocation *Loc) {
58 DebugLoc DL(Loc);
59 MI->setDebugLoc(DL);
60 }
61
62 class X86DiscriminateMemOps : public MachineFunctionPass {
63 bool runOnMachineFunction(MachineFunction &MF) override;
getPassName() const64 StringRef getPassName() const override {
65 return "X86 Discriminate Memory Operands";
66 }
67
68 public:
69 static char ID;
70
71 /// Default construct and initialize the pass.
72 X86DiscriminateMemOps();
73 };
74
IsPrefetchOpcode(unsigned Opcode)75 bool IsPrefetchOpcode(unsigned Opcode) {
76 return Opcode == X86::PREFETCHNTA || Opcode == X86::PREFETCHT0 ||
77 Opcode == X86::PREFETCHT1 || Opcode == X86::PREFETCHT2 ||
78 Opcode == X86::PREFETCHIT0 || Opcode == X86::PREFETCHIT1;
79 }
80 } // end anonymous namespace
81
82 //===----------------------------------------------------------------------===//
83 // Implementation
84 //===----------------------------------------------------------------------===//
85
86 char X86DiscriminateMemOps::ID = 0;
87
88 /// Default construct and initialize the pass.
X86DiscriminateMemOps()89 X86DiscriminateMemOps::X86DiscriminateMemOps() : MachineFunctionPass(ID) {}
90
runOnMachineFunction(MachineFunction & MF)91 bool X86DiscriminateMemOps::runOnMachineFunction(MachineFunction &MF) {
92 if (!EnableDiscriminateMemops)
93 return false;
94
95 DISubprogram *FDI = MF.getFunction().getSubprogram();
96 if (!FDI || !FDI->getUnit()->getDebugInfoForProfiling())
97 return false;
98
99 // Have a default DILocation, if we find instructions with memops that don't
100 // have any debug info.
101 const DILocation *ReferenceDI =
102 DILocation::get(FDI->getContext(), FDI->getLine(), 0, FDI);
103 assert(ReferenceDI && "ReferenceDI should not be nullptr");
104 DenseMap<Location, unsigned> MemOpDiscriminators;
105 MemOpDiscriminators[diToLocation(ReferenceDI)] = 0;
106
107 // Figure out the largest discriminator issued for each Location. When we
108 // issue new discriminators, we can thus avoid issuing discriminators
109 // belonging to instructions that don't have memops. This isn't a requirement
110 // for the goals of this pass, however, it avoids unnecessary ambiguity.
111 for (auto &MBB : MF) {
112 for (auto &MI : MBB) {
113 const auto &DI = MI.getDebugLoc();
114 if (!DI)
115 continue;
116 if (BypassPrefetchInstructions && IsPrefetchOpcode(MI.getDesc().Opcode))
117 continue;
118 Location Loc = diToLocation(DI);
119 MemOpDiscriminators[Loc] =
120 std::max(MemOpDiscriminators[Loc], DI->getBaseDiscriminator());
121 }
122 }
123
124 // Keep track of the discriminators seen at each Location. If an instruction's
125 // DebugInfo has a Location and discriminator we've already seen, replace its
126 // discriminator with a new one, to guarantee uniqueness.
127 DenseMap<Location, DenseSet<unsigned>> Seen;
128
129 bool Changed = false;
130 for (auto &MBB : MF) {
131 for (auto &MI : MBB) {
132 if (X86II::getMemoryOperandNo(MI.getDesc().TSFlags) < 0)
133 continue;
134 if (BypassPrefetchInstructions && IsPrefetchOpcode(MI.getDesc().Opcode))
135 continue;
136 const DILocation *DI = MI.getDebugLoc();
137 bool HasDebug = DI;
138 if (!HasDebug) {
139 DI = ReferenceDI;
140 }
141 Location L = diToLocation(DI);
142 DenseSet<unsigned> &Set = Seen[L];
143 const std::pair<DenseSet<unsigned>::iterator, bool> TryInsert =
144 Set.insert(DI->getBaseDiscriminator());
145 if (!TryInsert.second || !HasDebug) {
146 unsigned BF, DF, CI = 0;
147 DILocation::decodeDiscriminator(DI->getDiscriminator(), BF, DF, CI);
148 std::optional<unsigned> EncodedDiscriminator =
149 DILocation::encodeDiscriminator(MemOpDiscriminators[L] + 1, DF, CI);
150
151 if (!EncodedDiscriminator) {
152 // FIXME(mtrofin): The assumption is that this scenario is infrequent/OK
153 // not to support. If evidence points otherwise, we can explore synthesizeing
154 // unique DIs by adding fake line numbers, or by constructing 64 bit
155 // discriminators.
156 LLVM_DEBUG(dbgs() << "Unable to create a unique discriminator "
157 "for instruction with memory operand in: "
158 << DI->getFilename() << " Line: " << DI->getLine()
159 << " Column: " << DI->getColumn()
160 << ". This is likely due to a large macro expansion. \n");
161 continue;
162 }
163 // Since we were able to encode, bump the MemOpDiscriminators.
164 ++MemOpDiscriminators[L];
165 DI = DI->cloneWithDiscriminator(*EncodedDiscriminator);
166 assert(DI && "DI should not be nullptr");
167 updateDebugInfo(&MI, DI);
168 Changed = true;
169 std::pair<DenseSet<unsigned>::iterator, bool> MustInsert =
170 Set.insert(DI->getBaseDiscriminator());
171 (void)MustInsert; // Silence warning in release build.
172 assert(MustInsert.second && "New discriminator shouldn't be present in set");
173 }
174
175 // Bump the reference DI to avoid cramming discriminators on line 0.
176 // FIXME(mtrofin): pin ReferenceDI on blocks or first instruction with DI
177 // in a block. It's more consistent than just relying on the last memop
178 // instruction we happened to see.
179 ReferenceDI = DI;
180 }
181 }
182 return Changed;
183 }
184
createX86DiscriminateMemOpsPass()185 FunctionPass *llvm::createX86DiscriminateMemOpsPass() {
186 return new X86DiscriminateMemOps();
187 }
188