1 //===-- lib/CodeGen/MachineInstrBundle.cpp --------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/CodeGen/MachineInstrBundle.h"
10 #include "llvm/ADT/SetVector.h"
11 #include "llvm/ADT/SmallSet.h"
12 #include "llvm/ADT/SmallVector.h"
13 #include "llvm/CodeGen/MachineFunctionPass.h"
14 #include "llvm/CodeGen/MachineInstrBuilder.h"
15 #include "llvm/CodeGen/Passes.h"
16 #include "llvm/CodeGen/TargetInstrInfo.h"
17 #include "llvm/CodeGen/TargetRegisterInfo.h"
18 #include "llvm/CodeGen/TargetSubtargetInfo.h"
19 #include "llvm/InitializePasses.h"
20 #include "llvm/Pass.h"
21 #include "llvm/PassRegistry.h"
22 #include <utility>
23 using namespace llvm;
24
25 namespace {
26 class UnpackMachineBundles : public MachineFunctionPass {
27 public:
28 static char ID; // Pass identification
UnpackMachineBundles(std::function<bool (const MachineFunction &)> Ftor=nullptr)29 UnpackMachineBundles(
30 std::function<bool(const MachineFunction &)> Ftor = nullptr)
31 : MachineFunctionPass(ID), PredicateFtor(std::move(Ftor)) {
32 initializeUnpackMachineBundlesPass(*PassRegistry::getPassRegistry());
33 }
34
35 bool runOnMachineFunction(MachineFunction &MF) override;
36
37 private:
38 std::function<bool(const MachineFunction &)> PredicateFtor;
39 };
40 } // end anonymous namespace
41
42 char UnpackMachineBundles::ID = 0;
43 char &llvm::UnpackMachineBundlesID = UnpackMachineBundles::ID;
44 INITIALIZE_PASS(UnpackMachineBundles, "unpack-mi-bundles",
45 "Unpack machine instruction bundles", false, false)
46
runOnMachineFunction(MachineFunction & MF)47 bool UnpackMachineBundles::runOnMachineFunction(MachineFunction &MF) {
48 if (PredicateFtor && !PredicateFtor(MF))
49 return false;
50
51 bool Changed = false;
52 for (MachineBasicBlock &MBB : MF) {
53 for (MachineBasicBlock::instr_iterator MII = MBB.instr_begin(),
54 MIE = MBB.instr_end(); MII != MIE; ) {
55 MachineInstr *MI = &*MII;
56
57 // Remove BUNDLE instruction and the InsideBundle flags from bundled
58 // instructions.
59 if (MI->isBundle()) {
60 while (++MII != MIE && MII->isBundledWithPred()) {
61 MII->unbundleFromPred();
62 for (MachineOperand &MO : MII->operands()) {
63 if (MO.isReg() && MO.isInternalRead())
64 MO.setIsInternalRead(false);
65 }
66 }
67 MI->eraseFromParent();
68
69 Changed = true;
70 continue;
71 }
72
73 ++MII;
74 }
75 }
76
77 return Changed;
78 }
79
80 FunctionPass *
createUnpackMachineBundles(std::function<bool (const MachineFunction &)> Ftor)81 llvm::createUnpackMachineBundles(
82 std::function<bool(const MachineFunction &)> Ftor) {
83 return new UnpackMachineBundles(std::move(Ftor));
84 }
85
86 namespace {
87 class FinalizeMachineBundles : public MachineFunctionPass {
88 public:
89 static char ID; // Pass identification
FinalizeMachineBundles()90 FinalizeMachineBundles() : MachineFunctionPass(ID) {
91 initializeFinalizeMachineBundlesPass(*PassRegistry::getPassRegistry());
92 }
93
94 bool runOnMachineFunction(MachineFunction &MF) override;
95 };
96 } // end anonymous namespace
97
98 char FinalizeMachineBundles::ID = 0;
99 char &llvm::FinalizeMachineBundlesID = FinalizeMachineBundles::ID;
100 INITIALIZE_PASS(FinalizeMachineBundles, "finalize-mi-bundles",
101 "Finalize machine instruction bundles", false, false)
102
runOnMachineFunction(MachineFunction & MF)103 bool FinalizeMachineBundles::runOnMachineFunction(MachineFunction &MF) {
104 return llvm::finalizeBundles(MF);
105 }
106
107 /// Return the first found DebugLoc that has a DILocation, given a range of
108 /// instructions. The search range is from FirstMI to LastMI (exclusive). If no
109 /// DILocation is found, then an empty location is returned.
getDebugLoc(MachineBasicBlock::instr_iterator FirstMI,MachineBasicBlock::instr_iterator LastMI)110 static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI,
111 MachineBasicBlock::instr_iterator LastMI) {
112 for (auto MII = FirstMI; MII != LastMI; ++MII)
113 if (MII->getDebugLoc())
114 return MII->getDebugLoc();
115 return DebugLoc();
116 }
117
118 /// finalizeBundle - Finalize a machine instruction bundle which includes
119 /// a sequence of instructions starting from FirstMI to LastMI (exclusive).
120 /// This routine adds a BUNDLE instruction to represent the bundle, it adds
121 /// IsInternalRead markers to MachineOperands which are defined inside the
122 /// bundle, and it copies externally visible defs and uses to the BUNDLE
123 /// instruction.
finalizeBundle(MachineBasicBlock & MBB,MachineBasicBlock::instr_iterator FirstMI,MachineBasicBlock::instr_iterator LastMI)124 void llvm::finalizeBundle(MachineBasicBlock &MBB,
125 MachineBasicBlock::instr_iterator FirstMI,
126 MachineBasicBlock::instr_iterator LastMI) {
127 assert(FirstMI != LastMI && "Empty bundle?");
128 MIBundleBuilder Bundle(MBB, FirstMI, LastMI);
129
130 MachineFunction &MF = *MBB.getParent();
131 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
132 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
133
134 MachineInstrBuilder MIB =
135 BuildMI(MF, getDebugLoc(FirstMI, LastMI), TII->get(TargetOpcode::BUNDLE));
136 Bundle.prepend(MIB);
137
138 SmallSetVector<Register, 32> LocalDefs;
139 SmallSet<Register, 8> DeadDefSet;
140 SmallSet<Register, 16> KilledDefSet;
141 SmallSetVector<Register, 8> ExternUses;
142 SmallSet<Register, 8> KilledUseSet;
143 SmallSet<Register, 8> UndefUseSet;
144 for (auto MII = FirstMI; MII != LastMI; ++MII) {
145 // Debug instructions have no effects to track.
146 if (MII->isDebugInstr())
147 continue;
148
149 for (MachineOperand &MO : MII->all_uses()) {
150 Register Reg = MO.getReg();
151 if (!Reg)
152 continue;
153
154 if (LocalDefs.contains(Reg)) {
155 MO.setIsInternalRead();
156 if (MO.isKill()) {
157 // Internal def is now killed.
158 KilledDefSet.insert(Reg);
159 }
160 } else {
161 if (ExternUses.insert(Reg)) {
162 if (MO.isUndef())
163 UndefUseSet.insert(Reg);
164 }
165 if (MO.isKill()) {
166 // External def is now killed.
167 KilledUseSet.insert(Reg);
168 }
169 }
170 }
171
172 for (MachineOperand &MO : MII->all_defs()) {
173 Register Reg = MO.getReg();
174 if (!Reg)
175 continue;
176
177 if (LocalDefs.insert(Reg)) {
178 if (MO.isDead())
179 DeadDefSet.insert(Reg);
180 } else {
181 // Re-defined inside the bundle, it's no longer killed.
182 KilledDefSet.erase(Reg);
183 if (!MO.isDead()) {
184 // Previously defined but dead.
185 DeadDefSet.erase(Reg);
186 }
187 }
188
189 if (!MO.isDead() && Reg.isPhysical())
190 LocalDefs.insert_range(TRI->subregs(Reg));
191 }
192
193 // Set FrameSetup/FrameDestroy for the bundle. If any of the instructions
194 // got the property, then also set it on the bundle.
195 if (MII->getFlag(MachineInstr::FrameSetup))
196 MIB.setMIFlag(MachineInstr::FrameSetup);
197 if (MII->getFlag(MachineInstr::FrameDestroy))
198 MIB.setMIFlag(MachineInstr::FrameDestroy);
199 }
200
201 for (Register Reg : LocalDefs) {
202 // If it's not live beyond end of the bundle, mark it dead.
203 bool isDead = DeadDefSet.contains(Reg) || KilledDefSet.contains(Reg);
204 MIB.addReg(Reg, getDefRegState(true) | getDeadRegState(isDead) |
205 getImplRegState(true));
206 }
207
208 for (Register Reg : ExternUses) {
209 bool isKill = KilledUseSet.contains(Reg);
210 bool isUndef = UndefUseSet.contains(Reg);
211 MIB.addReg(Reg, getKillRegState(isKill) | getUndefRegState(isUndef) |
212 getImplRegState(true));
213 }
214 }
215
216 /// finalizeBundle - Same functionality as the previous finalizeBundle except
217 /// the last instruction in the bundle is not provided as an input. This is
218 /// used in cases where bundles are pre-determined by marking instructions
219 /// with 'InsideBundle' marker. It returns the MBB instruction iterator that
220 /// points to the end of the bundle.
221 MachineBasicBlock::instr_iterator
finalizeBundle(MachineBasicBlock & MBB,MachineBasicBlock::instr_iterator FirstMI)222 llvm::finalizeBundle(MachineBasicBlock &MBB,
223 MachineBasicBlock::instr_iterator FirstMI) {
224 MachineBasicBlock::instr_iterator E = MBB.instr_end();
225 MachineBasicBlock::instr_iterator LastMI = std::next(FirstMI);
226 while (LastMI != E && LastMI->isInsideBundle())
227 ++LastMI;
228 finalizeBundle(MBB, FirstMI, LastMI);
229 return LastMI;
230 }
231
232 /// finalizeBundles - Finalize instruction bundles in the specified
233 /// MachineFunction. Return true if any bundles are finalized.
finalizeBundles(MachineFunction & MF)234 bool llvm::finalizeBundles(MachineFunction &MF) {
235 bool Changed = false;
236 for (MachineBasicBlock &MBB : MF) {
237 MachineBasicBlock::instr_iterator MII = MBB.instr_begin();
238 MachineBasicBlock::instr_iterator MIE = MBB.instr_end();
239 if (MII == MIE)
240 continue;
241 assert(!MII->isInsideBundle() &&
242 "First instr cannot be inside bundle before finalization!");
243
244 for (++MII; MII != MIE; ) {
245 if (!MII->isInsideBundle())
246 ++MII;
247 else {
248 MII = finalizeBundle(MBB, std::prev(MII));
249 Changed = true;
250 }
251 }
252 }
253
254 return Changed;
255 }
256
AnalyzeVirtRegInBundle(MachineInstr & MI,Register Reg,SmallVectorImpl<std::pair<MachineInstr *,unsigned>> * Ops)257 VirtRegInfo llvm::AnalyzeVirtRegInBundle(
258 MachineInstr &MI, Register Reg,
259 SmallVectorImpl<std::pair<MachineInstr *, unsigned>> *Ops) {
260 VirtRegInfo RI = {false, false, false};
261 for (MIBundleOperands O(MI); O.isValid(); ++O) {
262 MachineOperand &MO = *O;
263 if (!MO.isReg() || MO.getReg() != Reg)
264 continue;
265
266 // Remember each (MI, OpNo) that refers to Reg.
267 if (Ops)
268 Ops->push_back(std::make_pair(MO.getParent(), O.getOperandNo()));
269
270 // Both defs and uses can read virtual registers.
271 if (MO.readsReg()) {
272 RI.Reads = true;
273 if (MO.isDef())
274 RI.Tied = true;
275 }
276
277 // Only defs can write.
278 if (MO.isDef())
279 RI.Writes = true;
280 else if (!RI.Tied &&
281 MO.getParent()->isRegTiedToDefOperand(O.getOperandNo()))
282 RI.Tied = true;
283 }
284 return RI;
285 }
286
287 std::pair<LaneBitmask, LaneBitmask>
AnalyzeVirtRegLanesInBundle(const MachineInstr & MI,Register Reg,const MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI)288 llvm::AnalyzeVirtRegLanesInBundle(const MachineInstr &MI, Register Reg,
289 const MachineRegisterInfo &MRI,
290 const TargetRegisterInfo &TRI) {
291
292 LaneBitmask UseMask, DefMask;
293
294 for (const MachineOperand &MO : const_mi_bundle_ops(MI)) {
295 if (!MO.isReg() || MO.getReg() != Reg)
296 continue;
297
298 unsigned SubReg = MO.getSubReg();
299 if (SubReg == 0 && MO.isUse() && !MO.isUndef())
300 UseMask |= MRI.getMaxLaneMaskForVReg(Reg);
301
302 LaneBitmask SubRegMask = TRI.getSubRegIndexLaneMask(SubReg);
303 if (MO.isDef()) {
304 if (!MO.isUndef())
305 UseMask |= ~SubRegMask;
306 DefMask |= SubRegMask;
307 } else if (!MO.isUndef())
308 UseMask |= SubRegMask;
309 }
310
311 return {UseMask, DefMask};
312 }
313
AnalyzePhysRegInBundle(const MachineInstr & MI,Register Reg,const TargetRegisterInfo * TRI)314 PhysRegInfo llvm::AnalyzePhysRegInBundle(const MachineInstr &MI, Register Reg,
315 const TargetRegisterInfo *TRI) {
316 bool AllDefsDead = true;
317 PhysRegInfo PRI = {false, false, false, false, false, false, false, false};
318
319 assert(Reg.isPhysical() && "analyzePhysReg not given a physical register!");
320 for (const MachineOperand &MO : const_mi_bundle_ops(MI)) {
321 if (MO.isRegMask() && MO.clobbersPhysReg(Reg)) {
322 PRI.Clobbered = true;
323 continue;
324 }
325
326 if (!MO.isReg())
327 continue;
328
329 Register MOReg = MO.getReg();
330 if (!MOReg || !MOReg.isPhysical())
331 continue;
332
333 if (!TRI->regsOverlap(MOReg, Reg))
334 continue;
335
336 bool Covered = TRI->isSuperRegisterEq(Reg, MOReg);
337 if (MO.readsReg()) {
338 PRI.Read = true;
339 if (Covered) {
340 PRI.FullyRead = true;
341 if (MO.isKill())
342 PRI.Killed = true;
343 }
344 } else if (MO.isDef()) {
345 PRI.Defined = true;
346 if (Covered)
347 PRI.FullyDefined = true;
348 if (!MO.isDead())
349 AllDefsDead = false;
350 }
351 }
352
353 if (AllDefsDead) {
354 if (PRI.FullyDefined || PRI.Clobbered)
355 PRI.DeadDef = true;
356 else if (PRI.Defined)
357 PRI.PartialDeadDef = true;
358 }
359
360 return PRI;
361 }
362