xref: /freebsd/contrib/llvm-project/llvm/lib/Target/ARM/MLxExpansionPass.cpp (revision 2f513db72b034fd5ef7f080b11be5c711c15186a)
1 //===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of
10 // multiple and add / sub instructions) when special VMLx hazards are detected.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "ARM.h"
15 #include "ARMBaseInstrInfo.h"
16 #include "ARMSubtarget.h"
17 #include "llvm/ADT/SmallPtrSet.h"
18 #include "llvm/ADT/Statistic.h"
19 #include "llvm/CodeGen/MachineFunctionPass.h"
20 #include "llvm/CodeGen/MachineInstr.h"
21 #include "llvm/CodeGen/MachineInstrBuilder.h"
22 #include "llvm/CodeGen/MachineRegisterInfo.h"
23 #include "llvm/CodeGen/TargetRegisterInfo.h"
24 #include "llvm/Support/CommandLine.h"
25 #include "llvm/Support/Debug.h"
26 #include "llvm/Support/raw_ostream.h"
27 using namespace llvm;
28 
29 #define DEBUG_TYPE "mlx-expansion"
30 
31 static cl::opt<bool>
32 ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden);
33 static cl::opt<unsigned>
34 ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden);
35 
36 STATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded");
37 
38 namespace {
39   struct MLxExpansion : public MachineFunctionPass {
40     static char ID;
41     MLxExpansion() : MachineFunctionPass(ID) {}
42 
43     bool runOnMachineFunction(MachineFunction &Fn) override;
44 
45     StringRef getPassName() const override {
46       return "ARM MLA / MLS expansion pass";
47     }
48 
49   private:
50     const ARMBaseInstrInfo *TII;
51     const TargetRegisterInfo *TRI;
52     MachineRegisterInfo *MRI;
53 
54     bool isLikeA9;
55     bool isSwift;
56     unsigned MIIdx;
57     MachineInstr* LastMIs[4];
58     SmallPtrSet<MachineInstr*, 4> IgnoreStall;
59 
60     void clearStack();
61     void pushStack(MachineInstr *MI);
62     MachineInstr *getAccDefMI(MachineInstr *MI) const;
63     unsigned getDefReg(MachineInstr *MI) const;
64     bool hasLoopHazard(MachineInstr *MI) const;
65     bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
66     bool FindMLxHazard(MachineInstr *MI);
67     void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
68                                 unsigned MulOpc, unsigned AddSubOpc,
69                                 bool NegAcc, bool HasLane);
70     bool ExpandFPMLxInstructions(MachineBasicBlock &MBB);
71   };
72   char MLxExpansion::ID = 0;
73 }
74 
75 void MLxExpansion::clearStack() {
76   std::fill(LastMIs, LastMIs + 4, nullptr);
77   MIIdx = 0;
78 }
79 
80 void MLxExpansion::pushStack(MachineInstr *MI) {
81   LastMIs[MIIdx] = MI;
82   if (++MIIdx == 4)
83     MIIdx = 0;
84 }
85 
86 MachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const {
87   // Look past COPY and INSERT_SUBREG instructions to find the
88   // real definition MI. This is important for _sfp instructions.
89   unsigned Reg = MI->getOperand(1).getReg();
90   if (TargetRegisterInfo::isPhysicalRegister(Reg))
91     return nullptr;
92 
93   MachineBasicBlock *MBB = MI->getParent();
94   MachineInstr *DefMI = MRI->getVRegDef(Reg);
95   while (true) {
96     if (DefMI->getParent() != MBB)
97       break;
98     if (DefMI->isCopyLike()) {
99       Reg = DefMI->getOperand(1).getReg();
100       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
101         DefMI = MRI->getVRegDef(Reg);
102         continue;
103       }
104     } else if (DefMI->isInsertSubreg()) {
105       Reg = DefMI->getOperand(2).getReg();
106       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
107         DefMI = MRI->getVRegDef(Reg);
108         continue;
109       }
110     }
111     break;
112   }
113   return DefMI;
114 }
115 
116 unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
117   unsigned Reg = MI->getOperand(0).getReg();
118   if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
119       !MRI->hasOneNonDBGUse(Reg))
120     return Reg;
121 
122   MachineBasicBlock *MBB = MI->getParent();
123   MachineInstr *UseMI = &*MRI->use_instr_nodbg_begin(Reg);
124   if (UseMI->getParent() != MBB)
125     return Reg;
126 
127   while (UseMI->isCopy() || UseMI->isInsertSubreg()) {
128     Reg = UseMI->getOperand(0).getReg();
129     if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
130         !MRI->hasOneNonDBGUse(Reg))
131       return Reg;
132     UseMI = &*MRI->use_instr_nodbg_begin(Reg);
133     if (UseMI->getParent() != MBB)
134       return Reg;
135   }
136 
137   return Reg;
138 }
139 
140 /// hasLoopHazard - Check whether an MLx instruction is chained to itself across
141 /// a single-MBB loop.
142 bool MLxExpansion::hasLoopHazard(MachineInstr *MI) const {
143   unsigned Reg = MI->getOperand(1).getReg();
144   if (TargetRegisterInfo::isPhysicalRegister(Reg))
145     return false;
146 
147   MachineBasicBlock *MBB = MI->getParent();
148   MachineInstr *DefMI = MRI->getVRegDef(Reg);
149   while (true) {
150 outer_continue:
151     if (DefMI->getParent() != MBB)
152       break;
153 
154     if (DefMI->isPHI()) {
155       for (unsigned i = 1, e = DefMI->getNumOperands(); i < e; i += 2) {
156         if (DefMI->getOperand(i + 1).getMBB() == MBB) {
157           unsigned SrcReg = DefMI->getOperand(i).getReg();
158           if (TargetRegisterInfo::isVirtualRegister(SrcReg)) {
159             DefMI = MRI->getVRegDef(SrcReg);
160             goto outer_continue;
161           }
162         }
163       }
164     } else if (DefMI->isCopyLike()) {
165       Reg = DefMI->getOperand(1).getReg();
166       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
167         DefMI = MRI->getVRegDef(Reg);
168         continue;
169       }
170     } else if (DefMI->isInsertSubreg()) {
171       Reg = DefMI->getOperand(2).getReg();
172       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
173         DefMI = MRI->getVRegDef(Reg);
174         continue;
175       }
176     }
177 
178     break;
179   }
180 
181   return DefMI == MI;
182 }
183 
184 bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
185   // FIXME: Detect integer instructions properly.
186   const MCInstrDesc &MCID = MI->getDesc();
187   unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
188   if (MI->mayStore())
189     return false;
190   unsigned Opcode = MCID.getOpcode();
191   if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
192     return false;
193   if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
194     return MI->readsRegister(Reg, TRI);
195   return false;
196 }
197 
198 static bool isFpMulInstruction(unsigned Opcode) {
199   switch (Opcode) {
200   case ARM::VMULS:
201   case ARM::VMULfd:
202   case ARM::VMULfq:
203   case ARM::VMULD:
204   case ARM::VMULslfd:
205   case ARM::VMULslfq:
206     return true;
207   default:
208     return false;
209   }
210 }
211 
212 bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
213   if (NumExpand >= ExpandLimit)
214     return false;
215 
216   if (ForceExapnd)
217     return true;
218 
219   MachineInstr *DefMI = getAccDefMI(MI);
220   if (TII->isFpMLxInstruction(DefMI->getOpcode())) {
221     // r0 = vmla
222     // r3 = vmla r0, r1, r2
223     // takes 16 - 17 cycles
224     //
225     // r0 = vmla
226     // r4 = vmul r1, r2
227     // r3 = vadd r0, r4
228     // takes about 14 - 15 cycles even with vmul stalling for 4 cycles.
229     IgnoreStall.insert(DefMI);
230     return true;
231   }
232 
233   // On Swift, we mostly care about hazards from multiplication instructions
234   // writing the accumulator and the pipelining of loop iterations by out-of-
235   // order execution.
236   if (isSwift)
237     return isFpMulInstruction(DefMI->getOpcode()) || hasLoopHazard(MI);
238 
239   if (IgnoreStall.count(MI))
240     return false;
241 
242   // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the
243   // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall
244   // preserves the in-order retirement of the instructions.
245   // Look at the next few instructions, if *most* of them can cause hazards,
246   // then the scheduler can't *fix* this, we'd better break up the VMLA.
247   unsigned Limit1 = isLikeA9 ? 1 : 4;
248   unsigned Limit2 = isLikeA9 ? 1 : 4;
249   for (unsigned i = 1; i <= 4; ++i) {
250     int Idx = ((int)MIIdx - i + 4) % 4;
251     MachineInstr *NextMI = LastMIs[Idx];
252     if (!NextMI)
253       continue;
254 
255     if (TII->canCauseFpMLxStall(NextMI->getOpcode())) {
256       if (i <= Limit1)
257         return true;
258     }
259 
260     // Look for VMLx RAW hazard.
261     if (i <= Limit2 && hasRAWHazard(getDefReg(MI), NextMI))
262       return true;
263   }
264 
265   return false;
266 }
267 
268 /// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair
269 /// of MUL + ADD / SUB instructions.
270 void
271 MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
272                                      unsigned MulOpc, unsigned AddSubOpc,
273                                      bool NegAcc, bool HasLane) {
274   unsigned DstReg = MI->getOperand(0).getReg();
275   bool DstDead = MI->getOperand(0).isDead();
276   unsigned AccReg = MI->getOperand(1).getReg();
277   unsigned Src1Reg = MI->getOperand(2).getReg();
278   unsigned Src2Reg = MI->getOperand(3).getReg();
279   bool Src1Kill = MI->getOperand(2).isKill();
280   bool Src2Kill = MI->getOperand(3).isKill();
281   unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0;
282   unsigned NextOp = HasLane ? 5 : 4;
283   ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm();
284   unsigned PredReg = MI->getOperand(++NextOp).getReg();
285 
286   const MCInstrDesc &MCID1 = TII->get(MulOpc);
287   const MCInstrDesc &MCID2 = TII->get(AddSubOpc);
288   const MachineFunction &MF = *MI->getParent()->getParent();
289   unsigned TmpReg = MRI->createVirtualRegister(
290                       TII->getRegClass(MCID1, 0, TRI, MF));
291 
292   MachineInstrBuilder MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID1, TmpReg)
293     .addReg(Src1Reg, getKillRegState(Src1Kill))
294     .addReg(Src2Reg, getKillRegState(Src2Kill));
295   if (HasLane)
296     MIB.addImm(LaneImm);
297   MIB.addImm(Pred).addReg(PredReg);
298 
299   MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID2)
300     .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead));
301 
302   if (NegAcc) {
303     bool AccKill = MRI->hasOneNonDBGUse(AccReg);
304     MIB.addReg(TmpReg, getKillRegState(true))
305        .addReg(AccReg, getKillRegState(AccKill));
306   } else {
307     MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true));
308   }
309   MIB.addImm(Pred).addReg(PredReg);
310 
311   LLVM_DEBUG({
312     dbgs() << "Expanding: " << *MI;
313     dbgs() << "  to:\n";
314     MachineBasicBlock::iterator MII = MI;
315     MII = std::prev(MII);
316     MachineInstr &MI2 = *MII;
317     MII = std::prev(MII);
318     MachineInstr &MI1 = *MII;
319     dbgs() << "    " << MI1;
320     dbgs() << "    " << MI2;
321   });
322 
323   MI->eraseFromParent();
324   ++NumExpand;
325 }
326 
327 bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) {
328   bool Changed = false;
329 
330   clearStack();
331   IgnoreStall.clear();
332 
333   unsigned Skip = 0;
334   MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend();
335   while (MII != E) {
336     MachineInstr *MI = &*MII++;
337 
338     if (MI->isPosition() || MI->isImplicitDef() || MI->isCopy())
339       continue;
340 
341     const MCInstrDesc &MCID = MI->getDesc();
342     if (MI->isBarrier()) {
343       clearStack();
344       Skip = 0;
345       continue;
346     }
347 
348     unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
349     if (Domain == ARMII::DomainGeneral) {
350       if (++Skip == 2)
351         // Assume dual issues of non-VFP / NEON instructions.
352         pushStack(nullptr);
353     } else {
354       Skip = 0;
355 
356       unsigned MulOpc, AddSubOpc;
357       bool NegAcc, HasLane;
358       if (!TII->isFpMLxInstruction(MCID.getOpcode(),
359                                    MulOpc, AddSubOpc, NegAcc, HasLane) ||
360           !FindMLxHazard(MI))
361         pushStack(MI);
362       else {
363         ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane);
364         Changed = true;
365       }
366     }
367   }
368 
369   return Changed;
370 }
371 
372 bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
373   if (skipFunction(Fn.getFunction()))
374     return false;
375 
376   TII = static_cast<const ARMBaseInstrInfo *>(Fn.getSubtarget().getInstrInfo());
377   TRI = Fn.getSubtarget().getRegisterInfo();
378   MRI = &Fn.getRegInfo();
379   const ARMSubtarget *STI = &Fn.getSubtarget<ARMSubtarget>();
380   if (!STI->expandMLx())
381     return false;
382   isLikeA9 = STI->isLikeA9() || STI->isSwift();
383   isSwift = STI->isSwift();
384 
385   bool Modified = false;
386   for (MachineBasicBlock &MBB : Fn)
387     Modified |= ExpandFPMLxInstructions(MBB);
388 
389   return Modified;
390 }
391 
392 FunctionPass *llvm::createMLxExpansionPass() {
393   return new MLxExpansion();
394 }
395