1 //
2 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3 // See https://llvm.org/LICENSE.txt for license information.
4 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5 //
6 //===----------------------------------------------------------------------===//
7 //
8 // This file contains a pass that performs optimization on SIMD instructions
9 // with high latency by splitting them into more efficient series of
10 // instructions.
11 //
12 // 1. Rewrite certain SIMD instructions with vector element due to their
13 // inefficiency on some targets.
14 //
15 // For example:
16 // fmla v0.4s, v1.4s, v2.s[1]
17 //
18 // Is rewritten into:
19 // dup v3.4s, v2.s[1]
20 // fmla v0.4s, v1.4s, v3.4s
21 //
22 // 2. Rewrite interleaved memory access instructions due to their
23 // inefficiency on some targets.
24 //
25 // For example:
26 // st2 {v0.4s, v1.4s}, addr
27 //
28 // Is rewritten into:
29 // zip1 v2.4s, v0.4s, v1.4s
30 // zip2 v3.4s, v0.4s, v1.4s
31 // stp q2, q3, addr
32 //
33 //===----------------------------------------------------------------------===//
34
35 #include "AArch64InstrInfo.h"
36 #include "llvm/ADT/SmallVector.h"
37 #include "llvm/ADT/Statistic.h"
38 #include "llvm/ADT/StringRef.h"
39 #include "llvm/CodeGen/MachineBasicBlock.h"
40 #include "llvm/CodeGen/MachineFunction.h"
41 #include "llvm/CodeGen/MachineFunctionPass.h"
42 #include "llvm/CodeGen/MachineInstr.h"
43 #include "llvm/CodeGen/MachineInstrBuilder.h"
44 #include "llvm/CodeGen/MachineOperand.h"
45 #include "llvm/CodeGen/MachineRegisterInfo.h"
46 #include "llvm/CodeGen/TargetInstrInfo.h"
47 #include "llvm/CodeGen/TargetSchedule.h"
48 #include "llvm/CodeGen/TargetSubtargetInfo.h"
49 #include "llvm/MC/MCInstrDesc.h"
50 #include "llvm/MC/MCSchedule.h"
51 #include "llvm/Pass.h"
52 #include <unordered_map>
53 #include <map>
54
55 using namespace llvm;
56
57 #define DEBUG_TYPE "aarch64-simdinstr-opt"
58
59 STATISTIC(NumModifiedInstr,
60 "Number of SIMD instructions modified");
61
62 #define AARCH64_VECTOR_BY_ELEMENT_OPT_NAME \
63 "AArch64 SIMD instructions optimization pass"
64
65 namespace {
66
67 struct AArch64SIMDInstrOpt : public MachineFunctionPass {
68 static char ID;
69
70 const TargetInstrInfo *TII;
71 MachineRegisterInfo *MRI;
72 TargetSchedModel SchedModel;
73
74 // The two maps below are used to cache decisions instead of recomputing:
75 // This is used to cache instruction replacement decisions within function
76 // units and across function units.
77 std::map<std::pair<unsigned, std::string>, bool> SIMDInstrTable;
78 // This is used to cache the decision of whether to leave the interleaved
79 // store instructions replacement pass early or not for a particular target.
80 std::unordered_map<std::string, bool> InterlEarlyExit;
81
82 typedef enum {
83 VectorElem,
84 Interleave
85 } Subpass;
86
87 // Instruction represented by OrigOpc is replaced by instructions in ReplOpc.
88 struct InstReplInfo {
89 unsigned OrigOpc;
90 std::vector<unsigned> ReplOpc;
91 const TargetRegisterClass RC;
92 };
93
94 #define RuleST2(OpcOrg, OpcR0, OpcR1, OpcR2, RC) \
95 {OpcOrg, {OpcR0, OpcR1, OpcR2}, RC}
96 #define RuleST4(OpcOrg, OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, \
97 OpcR7, OpcR8, OpcR9, RC) \
98 {OpcOrg, \
99 {OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, OpcR7, OpcR8, OpcR9}, RC}
100
101 // The Instruction Replacement Table:
102 std::vector<InstReplInfo> IRT = {
103 // ST2 instructions
104 RuleST2(AArch64::ST2Twov2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
105 AArch64::STPQi, AArch64::FPR128RegClass),
106 RuleST2(AArch64::ST2Twov4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
107 AArch64::STPQi, AArch64::FPR128RegClass),
108 RuleST2(AArch64::ST2Twov2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
109 AArch64::STPDi, AArch64::FPR64RegClass),
110 RuleST2(AArch64::ST2Twov8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
111 AArch64::STPQi, AArch64::FPR128RegClass),
112 RuleST2(AArch64::ST2Twov4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
113 AArch64::STPDi, AArch64::FPR64RegClass),
114 RuleST2(AArch64::ST2Twov16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
115 AArch64::STPQi, AArch64::FPR128RegClass),
116 RuleST2(AArch64::ST2Twov8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
117 AArch64::STPDi, AArch64::FPR64RegClass),
118 // ST4 instructions
119 RuleST4(AArch64::ST4Fourv2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
120 AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::ZIP1v2i64,
121 AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
122 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
123 RuleST4(AArch64::ST4Fourv4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
124 AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::ZIP1v4i32,
125 AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
126 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
127 RuleST4(AArch64::ST4Fourv2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
128 AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::ZIP1v2i32,
129 AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
130 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass),
131 RuleST4(AArch64::ST4Fourv8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
132 AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::ZIP1v8i16,
133 AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
134 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
135 RuleST4(AArch64::ST4Fourv4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
136 AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::ZIP1v4i16,
137 AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
138 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass),
139 RuleST4(AArch64::ST4Fourv16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
140 AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::ZIP1v16i8,
141 AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
142 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
143 RuleST4(AArch64::ST4Fourv8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
144 AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::ZIP1v8i8,
145 AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
146 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass)
147 };
148
149 // A costly instruction is replaced in this work by N efficient instructions
150 // The maximum of N is currently 10 and it is for ST4 case.
151 static const unsigned MaxNumRepl = 10;
152
AArch64SIMDInstrOpt__anon4abb80d70111::AArch64SIMDInstrOpt153 AArch64SIMDInstrOpt() : MachineFunctionPass(ID) {}
154
155 /// Based only on latency of instructions, determine if it is cost efficient
156 /// to replace the instruction InstDesc by the instructions stored in the
157 /// array InstDescRepl.
158 /// Return true if replacement is expected to be faster.
159 bool shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc,
160 SmallVectorImpl<const MCInstrDesc*> &ReplInstrMCID);
161
162 /// Determine if we need to exit the instruction replacement optimization
163 /// passes early. This makes sure that no compile time is spent in this pass
164 /// for targets with no need for any of these optimizations.
165 /// Return true if early exit of the pass is recommended.
166 bool shouldExitEarly(MachineFunction *MF, Subpass SP);
167
168 /// Check whether an equivalent DUP instruction has already been
169 /// created or not.
170 /// Return true when the DUP instruction already exists. In this case,
171 /// DestReg will point to the destination of the already created DUP.
172 bool reuseDUP(MachineInstr &MI, unsigned DupOpcode, unsigned SrcReg,
173 unsigned LaneNumber, unsigned *DestReg) const;
174
175 /// Certain SIMD instructions with vector element operand are not efficient.
176 /// Rewrite them into SIMD instructions with vector operands. This rewrite
177 /// is driven by the latency of the instructions.
178 /// Return true if the SIMD instruction is modified.
179 bool optimizeVectElement(MachineInstr &MI);
180
181 /// Process The REG_SEQUENCE instruction, and extract the source
182 /// operands of the ST2/4 instruction from it.
183 /// Example of such instructions.
184 /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1;
185 /// Return true when the instruction is processed successfully.
186 bool processSeqRegInst(MachineInstr *DefiningMI, unsigned* StReg,
187 unsigned* StRegKill, unsigned NumArg) const;
188
189 /// Load/Store Interleaving instructions are not always beneficial.
190 /// Replace them by ZIP instructionand classical load/store.
191 /// Return true if the SIMD instruction is modified.
192 bool optimizeLdStInterleave(MachineInstr &MI);
193
194 /// Return the number of useful source registers for this
195 /// instruction (2 for ST2 and 4 for ST4).
196 unsigned determineSrcReg(MachineInstr &MI) const;
197
198 bool runOnMachineFunction(MachineFunction &Fn) override;
199
getPassName__anon4abb80d70111::AArch64SIMDInstrOpt200 StringRef getPassName() const override {
201 return AARCH64_VECTOR_BY_ELEMENT_OPT_NAME;
202 }
203 };
204
205 char AArch64SIMDInstrOpt::ID = 0;
206
207 } // end anonymous namespace
208
209 INITIALIZE_PASS(AArch64SIMDInstrOpt, "aarch64-simdinstr-opt",
210 AARCH64_VECTOR_BY_ELEMENT_OPT_NAME, false, false)
211
212 /// Based only on latency of instructions, determine if it is cost efficient
213 /// to replace the instruction InstDesc by the instructions stored in the
214 /// array InstDescRepl.
215 /// Return true if replacement is expected to be faster.
216 bool AArch64SIMDInstrOpt::
shouldReplaceInst(MachineFunction * MF,const MCInstrDesc * InstDesc,SmallVectorImpl<const MCInstrDesc * > & InstDescRepl)217 shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc,
218 SmallVectorImpl<const MCInstrDesc*> &InstDescRepl) {
219 // Check if replacement decision is already available in the cached table.
220 // if so, return it.
221 std::string Subtarget = std::string(SchedModel.getSubtargetInfo()->getCPU());
222 auto InstID = std::make_pair(InstDesc->getOpcode(), Subtarget);
223 auto It = SIMDInstrTable.find(InstID);
224 if (It != SIMDInstrTable.end())
225 return It->second;
226
227 unsigned SCIdx = InstDesc->getSchedClass();
228 const MCSchedClassDesc *SCDesc =
229 SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx);
230
231 // If a target does not define resources for the instructions
232 // of interest, then return false for no replacement.
233 const MCSchedClassDesc *SCDescRepl;
234 if (!SCDesc->isValid() || SCDesc->isVariant())
235 {
236 SIMDInstrTable[InstID] = false;
237 return false;
238 }
239 for (const auto *IDesc : InstDescRepl)
240 {
241 SCDescRepl = SchedModel.getMCSchedModel()->getSchedClassDesc(
242 IDesc->getSchedClass());
243 if (!SCDescRepl->isValid() || SCDescRepl->isVariant())
244 {
245 SIMDInstrTable[InstID] = false;
246 return false;
247 }
248 }
249
250 // Replacement cost.
251 unsigned ReplCost = 0;
252 for (const auto *IDesc :InstDescRepl)
253 ReplCost += SchedModel.computeInstrLatency(IDesc->getOpcode());
254
255 if (SchedModel.computeInstrLatency(InstDesc->getOpcode()) > ReplCost)
256 {
257 SIMDInstrTable[InstID] = true;
258 return true;
259 }
260 else
261 {
262 SIMDInstrTable[InstID] = false;
263 return false;
264 }
265 }
266
267 /// Determine if we need to exit this pass for a kind of instruction replacement
268 /// early. This makes sure that no compile time is spent in this pass for
269 /// targets with no need for any of these optimizations beyond performing this
270 /// check.
271 /// Return true if early exit of this pass for a kind of instruction
272 /// replacement is recommended for a target.
shouldExitEarly(MachineFunction * MF,Subpass SP)273 bool AArch64SIMDInstrOpt::shouldExitEarly(MachineFunction *MF, Subpass SP) {
274 const MCInstrDesc* OriginalMCID;
275 SmallVector<const MCInstrDesc*, MaxNumRepl> ReplInstrMCID;
276
277 switch (SP) {
278 // For this optimization, check by comparing the latency of a representative
279 // instruction to that of the replacement instructions.
280 // TODO: check for all concerned instructions.
281 case VectorElem:
282 OriginalMCID = &TII->get(AArch64::FMLAv4i32_indexed);
283 ReplInstrMCID.push_back(&TII->get(AArch64::DUPv4i32lane));
284 ReplInstrMCID.push_back(&TII->get(AArch64::FMLAv4f32));
285 if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID))
286 return false;
287 break;
288
289 // For this optimization, check for all concerned instructions.
290 case Interleave:
291 std::string Subtarget =
292 std::string(SchedModel.getSubtargetInfo()->getCPU());
293 auto It = InterlEarlyExit.find(Subtarget);
294 if (It != InterlEarlyExit.end())
295 return It->second;
296
297 for (auto &I : IRT) {
298 OriginalMCID = &TII->get(I.OrigOpc);
299 for (auto &Repl : I.ReplOpc)
300 ReplInstrMCID.push_back(&TII->get(Repl));
301 if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) {
302 InterlEarlyExit[Subtarget] = false;
303 return false;
304 }
305 ReplInstrMCID.clear();
306 }
307 InterlEarlyExit[Subtarget] = true;
308 break;
309 }
310
311 return true;
312 }
313
314 /// Check whether an equivalent DUP instruction has already been
315 /// created or not.
316 /// Return true when the DUP instruction already exists. In this case,
317 /// DestReg will point to the destination of the already created DUP.
reuseDUP(MachineInstr & MI,unsigned DupOpcode,unsigned SrcReg,unsigned LaneNumber,unsigned * DestReg) const318 bool AArch64SIMDInstrOpt::reuseDUP(MachineInstr &MI, unsigned DupOpcode,
319 unsigned SrcReg, unsigned LaneNumber,
320 unsigned *DestReg) const {
321 for (MachineBasicBlock::iterator MII = MI, MIE = MI.getParent()->begin();
322 MII != MIE;) {
323 MII--;
324 MachineInstr *CurrentMI = &*MII;
325
326 if (CurrentMI->getOpcode() == DupOpcode &&
327 CurrentMI->getNumOperands() == 3 &&
328 CurrentMI->getOperand(1).getReg() == SrcReg &&
329 CurrentMI->getOperand(2).getImm() == LaneNumber) {
330 *DestReg = CurrentMI->getOperand(0).getReg();
331 return true;
332 }
333 }
334
335 return false;
336 }
337
338 /// Certain SIMD instructions with vector element operand are not efficient.
339 /// Rewrite them into SIMD instructions with vector operands. This rewrite
340 /// is driven by the latency of the instructions.
341 /// The instruction of concerns are for the time being FMLA, FMLS, FMUL,
342 /// and FMULX and hence they are hardcoded.
343 ///
344 /// For example:
345 /// fmla v0.4s, v1.4s, v2.s[1]
346 ///
347 /// Is rewritten into
348 /// dup v3.4s, v2.s[1] // DUP not necessary if redundant
349 /// fmla v0.4s, v1.4s, v3.4s
350 ///
351 /// Return true if the SIMD instruction is modified.
optimizeVectElement(MachineInstr & MI)352 bool AArch64SIMDInstrOpt::optimizeVectElement(MachineInstr &MI) {
353 const MCInstrDesc *MulMCID, *DupMCID;
354 const TargetRegisterClass *RC = &AArch64::FPR128RegClass;
355
356 switch (MI.getOpcode()) {
357 default:
358 return false;
359
360 // 4X32 instructions
361 case AArch64::FMLAv4i32_indexed:
362 DupMCID = &TII->get(AArch64::DUPv4i32lane);
363 MulMCID = &TII->get(AArch64::FMLAv4f32);
364 break;
365 case AArch64::FMLSv4i32_indexed:
366 DupMCID = &TII->get(AArch64::DUPv4i32lane);
367 MulMCID = &TII->get(AArch64::FMLSv4f32);
368 break;
369 case AArch64::FMULXv4i32_indexed:
370 DupMCID = &TII->get(AArch64::DUPv4i32lane);
371 MulMCID = &TII->get(AArch64::FMULXv4f32);
372 break;
373 case AArch64::FMULv4i32_indexed:
374 DupMCID = &TII->get(AArch64::DUPv4i32lane);
375 MulMCID = &TII->get(AArch64::FMULv4f32);
376 break;
377
378 // 2X64 instructions
379 case AArch64::FMLAv2i64_indexed:
380 DupMCID = &TII->get(AArch64::DUPv2i64lane);
381 MulMCID = &TII->get(AArch64::FMLAv2f64);
382 break;
383 case AArch64::FMLSv2i64_indexed:
384 DupMCID = &TII->get(AArch64::DUPv2i64lane);
385 MulMCID = &TII->get(AArch64::FMLSv2f64);
386 break;
387 case AArch64::FMULXv2i64_indexed:
388 DupMCID = &TII->get(AArch64::DUPv2i64lane);
389 MulMCID = &TII->get(AArch64::FMULXv2f64);
390 break;
391 case AArch64::FMULv2i64_indexed:
392 DupMCID = &TII->get(AArch64::DUPv2i64lane);
393 MulMCID = &TII->get(AArch64::FMULv2f64);
394 break;
395
396 // 2X32 instructions
397 case AArch64::FMLAv2i32_indexed:
398 RC = &AArch64::FPR64RegClass;
399 DupMCID = &TII->get(AArch64::DUPv2i32lane);
400 MulMCID = &TII->get(AArch64::FMLAv2f32);
401 break;
402 case AArch64::FMLSv2i32_indexed:
403 RC = &AArch64::FPR64RegClass;
404 DupMCID = &TII->get(AArch64::DUPv2i32lane);
405 MulMCID = &TII->get(AArch64::FMLSv2f32);
406 break;
407 case AArch64::FMULXv2i32_indexed:
408 RC = &AArch64::FPR64RegClass;
409 DupMCID = &TII->get(AArch64::DUPv2i32lane);
410 MulMCID = &TII->get(AArch64::FMULXv2f32);
411 break;
412 case AArch64::FMULv2i32_indexed:
413 RC = &AArch64::FPR64RegClass;
414 DupMCID = &TII->get(AArch64::DUPv2i32lane);
415 MulMCID = &TII->get(AArch64::FMULv2f32);
416 break;
417 }
418
419 SmallVector<const MCInstrDesc*, 2> ReplInstrMCID;
420 ReplInstrMCID.push_back(DupMCID);
421 ReplInstrMCID.push_back(MulMCID);
422 if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()),
423 ReplInstrMCID))
424 return false;
425
426 const DebugLoc &DL = MI.getDebugLoc();
427 MachineBasicBlock &MBB = *MI.getParent();
428 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
429
430 // Get the operands of the current SIMD arithmetic instruction.
431 Register MulDest = MI.getOperand(0).getReg();
432 Register SrcReg0 = MI.getOperand(1).getReg();
433 unsigned Src0IsKill = getKillRegState(MI.getOperand(1).isKill());
434 Register SrcReg1 = MI.getOperand(2).getReg();
435 unsigned Src1IsKill = getKillRegState(MI.getOperand(2).isKill());
436 unsigned DupDest;
437
438 // Instructions of interest have either 4 or 5 operands.
439 if (MI.getNumOperands() == 5) {
440 Register SrcReg2 = MI.getOperand(3).getReg();
441 unsigned Src2IsKill = getKillRegState(MI.getOperand(3).isKill());
442 unsigned LaneNumber = MI.getOperand(4).getImm();
443 // Create a new DUP instruction. Note that if an equivalent DUP instruction
444 // has already been created before, then use that one instead of creating
445 // a new one.
446 if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg2, LaneNumber, &DupDest)) {
447 DupDest = MRI.createVirtualRegister(RC);
448 BuildMI(MBB, MI, DL, *DupMCID, DupDest)
449 .addReg(SrcReg2, Src2IsKill)
450 .addImm(LaneNumber);
451 }
452 BuildMI(MBB, MI, DL, *MulMCID, MulDest)
453 .addReg(SrcReg0, Src0IsKill)
454 .addReg(SrcReg1, Src1IsKill)
455 .addReg(DupDest, Src2IsKill);
456 } else if (MI.getNumOperands() == 4) {
457 unsigned LaneNumber = MI.getOperand(3).getImm();
458 if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg1, LaneNumber, &DupDest)) {
459 DupDest = MRI.createVirtualRegister(RC);
460 BuildMI(MBB, MI, DL, *DupMCID, DupDest)
461 .addReg(SrcReg1, Src1IsKill)
462 .addImm(LaneNumber);
463 }
464 BuildMI(MBB, MI, DL, *MulMCID, MulDest)
465 .addReg(SrcReg0, Src0IsKill)
466 .addReg(DupDest, Src1IsKill);
467 } else {
468 return false;
469 }
470
471 ++NumModifiedInstr;
472 return true;
473 }
474
475 /// Load/Store Interleaving instructions are not always beneficial.
476 /// Replace them by ZIP instructions and classical load/store.
477 ///
478 /// For example:
479 /// st2 {v0.4s, v1.4s}, addr
480 ///
481 /// Is rewritten into:
482 /// zip1 v2.4s, v0.4s, v1.4s
483 /// zip2 v3.4s, v0.4s, v1.4s
484 /// stp q2, q3, addr
485 //
486 /// For example:
487 /// st4 {v0.4s, v1.4s, v2.4s, v3.4s}, addr
488 ///
489 /// Is rewritten into:
490 /// zip1 v4.4s, v0.4s, v2.4s
491 /// zip2 v5.4s, v0.4s, v2.4s
492 /// zip1 v6.4s, v1.4s, v3.4s
493 /// zip2 v7.4s, v1.4s, v3.4s
494 /// zip1 v8.4s, v4.4s, v6.4s
495 /// zip2 v9.4s, v4.4s, v6.4s
496 /// zip1 v10.4s, v5.4s, v7.4s
497 /// zip2 v11.4s, v5.4s, v7.4s
498 /// stp q8, q9, addr
499 /// stp q10, q11, addr+32
500 ///
501 /// Currently only instructions related to ST2 and ST4 are considered.
502 /// Other may be added later.
503 /// Return true if the SIMD instruction is modified.
optimizeLdStInterleave(MachineInstr & MI)504 bool AArch64SIMDInstrOpt::optimizeLdStInterleave(MachineInstr &MI) {
505
506 unsigned SeqReg, AddrReg;
507 unsigned StReg[4], StRegKill[4];
508 MachineInstr *DefiningMI;
509 const DebugLoc &DL = MI.getDebugLoc();
510 MachineBasicBlock &MBB = *MI.getParent();
511 SmallVector<unsigned, MaxNumRepl> ZipDest;
512 SmallVector<const MCInstrDesc*, MaxNumRepl> ReplInstrMCID;
513
514 // If current instruction matches any of the rewriting rules, then
515 // gather information about parameters of the new instructions.
516 bool Match = false;
517 for (auto &I : IRT) {
518 if (MI.getOpcode() == I.OrigOpc) {
519 SeqReg = MI.getOperand(0).getReg();
520 AddrReg = MI.getOperand(1).getReg();
521 DefiningMI = MRI->getUniqueVRegDef(SeqReg);
522 unsigned NumReg = determineSrcReg(MI);
523 if (!processSeqRegInst(DefiningMI, StReg, StRegKill, NumReg))
524 return false;
525
526 for (auto &Repl : I.ReplOpc) {
527 ReplInstrMCID.push_back(&TII->get(Repl));
528 // Generate destination registers but only for non-store instruction.
529 if (Repl != AArch64::STPQi && Repl != AArch64::STPDi)
530 ZipDest.push_back(MRI->createVirtualRegister(&I.RC));
531 }
532 Match = true;
533 break;
534 }
535 }
536
537 if (!Match)
538 return false;
539
540 // Determine if it is profitable to replace MI by the series of instructions
541 // represented in ReplInstrMCID.
542 if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()),
543 ReplInstrMCID))
544 return false;
545
546 // Generate the replacement instructions composed of ZIP1, ZIP2, and STP (at
547 // this point, the code generation is hardcoded and does not rely on the IRT
548 // table used above given that code generation for ST2 replacement is somewhat
549 // different than for ST4 replacement. We could have added more info into the
550 // table related to how we build new instructions but we may be adding more
551 // complexity with that).
552 switch (MI.getOpcode()) {
553 default:
554 return false;
555
556 case AArch64::ST2Twov16b:
557 case AArch64::ST2Twov8b:
558 case AArch64::ST2Twov8h:
559 case AArch64::ST2Twov4h:
560 case AArch64::ST2Twov4s:
561 case AArch64::ST2Twov2s:
562 case AArch64::ST2Twov2d:
563 // ZIP instructions
564 BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0])
565 .addReg(StReg[0])
566 .addReg(StReg[1]);
567 BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1])
568 .addReg(StReg[0], StRegKill[0])
569 .addReg(StReg[1], StRegKill[1]);
570 // STP instructions
571 BuildMI(MBB, MI, DL, *ReplInstrMCID[2])
572 .addReg(ZipDest[0])
573 .addReg(ZipDest[1])
574 .addReg(AddrReg)
575 .addImm(0);
576 break;
577
578 case AArch64::ST4Fourv16b:
579 case AArch64::ST4Fourv8b:
580 case AArch64::ST4Fourv8h:
581 case AArch64::ST4Fourv4h:
582 case AArch64::ST4Fourv4s:
583 case AArch64::ST4Fourv2s:
584 case AArch64::ST4Fourv2d:
585 // ZIP instructions
586 BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0])
587 .addReg(StReg[0])
588 .addReg(StReg[2]);
589 BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1])
590 .addReg(StReg[0], StRegKill[0])
591 .addReg(StReg[2], StRegKill[2]);
592 BuildMI(MBB, MI, DL, *ReplInstrMCID[2], ZipDest[2])
593 .addReg(StReg[1])
594 .addReg(StReg[3]);
595 BuildMI(MBB, MI, DL, *ReplInstrMCID[3], ZipDest[3])
596 .addReg(StReg[1], StRegKill[1])
597 .addReg(StReg[3], StRegKill[3]);
598 BuildMI(MBB, MI, DL, *ReplInstrMCID[4], ZipDest[4])
599 .addReg(ZipDest[0])
600 .addReg(ZipDest[2]);
601 BuildMI(MBB, MI, DL, *ReplInstrMCID[5], ZipDest[5])
602 .addReg(ZipDest[0])
603 .addReg(ZipDest[2]);
604 BuildMI(MBB, MI, DL, *ReplInstrMCID[6], ZipDest[6])
605 .addReg(ZipDest[1])
606 .addReg(ZipDest[3]);
607 BuildMI(MBB, MI, DL, *ReplInstrMCID[7], ZipDest[7])
608 .addReg(ZipDest[1])
609 .addReg(ZipDest[3]);
610 // stp instructions
611 BuildMI(MBB, MI, DL, *ReplInstrMCID[8])
612 .addReg(ZipDest[4])
613 .addReg(ZipDest[5])
614 .addReg(AddrReg)
615 .addImm(0);
616 BuildMI(MBB, MI, DL, *ReplInstrMCID[9])
617 .addReg(ZipDest[6])
618 .addReg(ZipDest[7])
619 .addReg(AddrReg)
620 .addImm(2);
621 break;
622 }
623
624 ++NumModifiedInstr;
625 return true;
626 }
627
628 /// Process The REG_SEQUENCE instruction, and extract the source
629 /// operands of the ST2/4 instruction from it.
630 /// Example of such instruction.
631 /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1;
632 /// Return true when the instruction is processed successfully.
processSeqRegInst(MachineInstr * DefiningMI,unsigned * StReg,unsigned * StRegKill,unsigned NumArg) const633 bool AArch64SIMDInstrOpt::processSeqRegInst(MachineInstr *DefiningMI,
634 unsigned* StReg, unsigned* StRegKill, unsigned NumArg) const {
635 assert(DefiningMI != nullptr);
636 if (DefiningMI->getOpcode() != AArch64::REG_SEQUENCE)
637 return false;
638
639 for (unsigned i=0; i<NumArg; i++) {
640 StReg[i] = DefiningMI->getOperand(2*i+1).getReg();
641 StRegKill[i] = getKillRegState(DefiningMI->getOperand(2*i+1).isKill());
642
643 // Validation check for the other arguments.
644 if (DefiningMI->getOperand(2*i+2).isImm()) {
645 switch (DefiningMI->getOperand(2*i+2).getImm()) {
646 default:
647 return false;
648
649 case AArch64::dsub0:
650 case AArch64::dsub1:
651 case AArch64::dsub2:
652 case AArch64::dsub3:
653 case AArch64::qsub0:
654 case AArch64::qsub1:
655 case AArch64::qsub2:
656 case AArch64::qsub3:
657 break;
658 }
659 }
660 else
661 return false;
662 }
663 return true;
664 }
665
666 /// Return the number of useful source registers for this instruction
667 /// (2 for ST2 and 4 for ST4).
determineSrcReg(MachineInstr & MI) const668 unsigned AArch64SIMDInstrOpt::determineSrcReg(MachineInstr &MI) const {
669 switch (MI.getOpcode()) {
670 default:
671 llvm_unreachable("Unsupported instruction for this pass");
672
673 case AArch64::ST2Twov16b:
674 case AArch64::ST2Twov8b:
675 case AArch64::ST2Twov8h:
676 case AArch64::ST2Twov4h:
677 case AArch64::ST2Twov4s:
678 case AArch64::ST2Twov2s:
679 case AArch64::ST2Twov2d:
680 return 2;
681
682 case AArch64::ST4Fourv16b:
683 case AArch64::ST4Fourv8b:
684 case AArch64::ST4Fourv8h:
685 case AArch64::ST4Fourv4h:
686 case AArch64::ST4Fourv4s:
687 case AArch64::ST4Fourv2s:
688 case AArch64::ST4Fourv2d:
689 return 4;
690 }
691 }
692
runOnMachineFunction(MachineFunction & MF)693 bool AArch64SIMDInstrOpt::runOnMachineFunction(MachineFunction &MF) {
694 if (skipFunction(MF.getFunction()))
695 return false;
696
697 TII = MF.getSubtarget().getInstrInfo();
698 MRI = &MF.getRegInfo();
699 const TargetSubtargetInfo &ST = MF.getSubtarget();
700 const AArch64InstrInfo *AAII =
701 static_cast<const AArch64InstrInfo *>(ST.getInstrInfo());
702 if (!AAII)
703 return false;
704 SchedModel.init(&ST);
705 if (!SchedModel.hasInstrSchedModel())
706 return false;
707
708 bool Changed = false;
709 for (auto OptimizationKind : {VectorElem, Interleave}) {
710 if (!shouldExitEarly(&MF, OptimizationKind)) {
711 SmallVector<MachineInstr *, 8> RemoveMIs;
712 for (MachineBasicBlock &MBB : MF) {
713 for (MachineInstr &MI : MBB) {
714 bool InstRewrite;
715 if (OptimizationKind == VectorElem)
716 InstRewrite = optimizeVectElement(MI) ;
717 else
718 InstRewrite = optimizeLdStInterleave(MI);
719 if (InstRewrite) {
720 // Add MI to the list of instructions to be removed given that it
721 // has been replaced.
722 RemoveMIs.push_back(&MI);
723 Changed = true;
724 }
725 }
726 }
727 for (MachineInstr *MI : RemoveMIs)
728 MI->eraseFromParent();
729 }
730 }
731
732 return Changed;
733 }
734
735 /// Returns an instance of the high cost ASIMD instruction replacement
736 /// optimization pass.
createAArch64SIMDInstrOptPass()737 FunctionPass *llvm::createAArch64SIMDInstrOptPass() {
738 return new AArch64SIMDInstrOpt();
739 }
740