xref: /freebsd/contrib/llvm-project/llvm/lib/Target/X86/X86FixupInstTuning.cpp (revision 06c3fb2749bda94cb5201f81ffdb8fa6c3161b2e) 
1 //===-- X86FixupInstTunings.cpp - replace instructions -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file does a tuning pass replacing slower machine instructions
10 // with faster ones. We do this here, as opposed to during normal ISel, as
11 // attempting to get the "right" instruction can break patterns. This pass
12 // is not meant search for special cases where an instruction can be transformed
13 // to another, it is only meant to do transformations where the old instruction
14 // is always replacable with the new instructions. For example:
15 //
16 //      `vpermq ymm` -> `vshufd ymm`
17 //          -- BAD, not always valid (lane cross/non-repeated mask)
18 //
19 //      `vpermilps ymm` -> `vshufd ymm`
20 //          -- GOOD, always replaceable
21 //
22 //===----------------------------------------------------------------------===//
23 
24 #include "X86.h"
25 #include "X86InstrInfo.h"
26 #include "X86Subtarget.h"
27 #include "llvm/ADT/Statistic.h"
28 #include "llvm/CodeGen/MachineFunctionPass.h"
29 #include "llvm/CodeGen/MachineInstrBuilder.h"
30 #include "llvm/CodeGen/MachineRegisterInfo.h"
31 
32 using namespace llvm;
33 
34 #define DEBUG_TYPE "x86-fixup-inst-tuning"
35 
36 STATISTIC(NumInstChanges, "Number of instructions changes");
37 
38 namespace {
39 class X86FixupInstTuningPass : public MachineFunctionPass {
40 public:
41   static char ID;
42 
X86FixupInstTuningPass()43   X86FixupInstTuningPass() : MachineFunctionPass(ID) {}
44 
getPassName() const45   StringRef getPassName() const override { return "X86 Fixup Inst Tuning"; }
46 
47   bool runOnMachineFunction(MachineFunction &MF) override;
48   bool processInstruction(MachineFunction &MF, MachineBasicBlock &MBB,
49                           MachineBasicBlock::iterator &I);
50 
51   // This pass runs after regalloc and doesn't support VReg operands.
getRequiredProperties() const52   MachineFunctionProperties getRequiredProperties() const override {
53     return MachineFunctionProperties().set(
54         MachineFunctionProperties::Property::NoVRegs);
55   }
56 
57 private:
58   const X86InstrInfo *TII = nullptr;
59   const X86Subtarget *ST = nullptr;
60   const MCSchedModel *SM = nullptr;
61 };
62 } // end anonymous namespace
63 
64 char X86FixupInstTuningPass::ID = 0;
65 
INITIALIZE_PASS(X86FixupInstTuningPass,DEBUG_TYPE,DEBUG_TYPE,false,false)66 INITIALIZE_PASS(X86FixupInstTuningPass, DEBUG_TYPE, DEBUG_TYPE, false, false)
67 
68 FunctionPass *llvm::createX86FixupInstTuning() {
69   return new X86FixupInstTuningPass();
70 }
71 
72 template <typename T>
CmpOptionals(T NewVal,T CurVal)73 static std::optional<bool> CmpOptionals(T NewVal, T CurVal) {
74   if (NewVal.has_value() && CurVal.has_value() && *NewVal != *CurVal)
75     return *NewVal < *CurVal;
76 
77   return std::nullopt;
78 }
79 
processInstruction(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator & I)80 bool X86FixupInstTuningPass::processInstruction(
81     MachineFunction &MF, MachineBasicBlock &MBB,
82     MachineBasicBlock::iterator &I) {
83   MachineInstr &MI = *I;
84   unsigned Opc = MI.getOpcode();
85   unsigned NumOperands = MI.getDesc().getNumOperands();
86 
87   auto GetInstTput = [&](unsigned Opcode) -> std::optional<double> {
88     // We already checked that SchedModel exists in `NewOpcPreferable`.
89     return MCSchedModel::getReciprocalThroughput(
90         *ST, *(SM->getSchedClassDesc(TII->get(Opcode).getSchedClass())));
91   };
92 
93   auto GetInstLat = [&](unsigned Opcode) -> std::optional<double> {
94     // We already checked that SchedModel exists in `NewOpcPreferable`.
95     return MCSchedModel::computeInstrLatency(
96         *ST, *(SM->getSchedClassDesc(TII->get(Opcode).getSchedClass())));
97   };
98 
99   auto GetInstSize = [&](unsigned Opcode) -> std::optional<unsigned> {
100     if (unsigned Size = TII->get(Opcode).getSize())
101       return Size;
102     // Zero size means we where unable to compute it.
103     return std::nullopt;
104   };
105 
106   auto NewOpcPreferable = [&](unsigned NewOpc,
107                               bool ReplaceInTie = true) -> bool {
108     std::optional<bool> Res;
109     if (SM->hasInstrSchedModel()) {
110       // Compare tput -> lat -> code size.
111       Res = CmpOptionals(GetInstTput(NewOpc), GetInstTput(Opc));
112       if (Res.has_value())
113         return *Res;
114 
115       Res = CmpOptionals(GetInstLat(NewOpc), GetInstLat(Opc));
116       if (Res.has_value())
117         return *Res;
118     }
119 
120     Res = CmpOptionals(GetInstSize(Opc), GetInstSize(NewOpc));
121     if (Res.has_value())
122       return *Res;
123 
124     // We either have either were unable to get tput/lat/codesize or all values
125     // were equal. Return specified option for a tie.
126     return ReplaceInTie;
127   };
128 
129   // `vpermilpd r, i` -> `vshufpd r, r, i`
130   // `vpermilpd r, i, k` -> `vshufpd r, r, i, k`
131   // `vshufpd` is always as fast or faster than `vpermilpd` and takes
132   // 1 less byte of code size for VEX and EVEX encoding.
133   auto ProcessVPERMILPDri = [&](unsigned NewOpc) -> bool {
134     if (!NewOpcPreferable(NewOpc))
135       return false;
136     unsigned MaskImm = MI.getOperand(NumOperands - 1).getImm();
137     MI.removeOperand(NumOperands - 1);
138     MI.addOperand(MI.getOperand(NumOperands - 2));
139     MI.setDesc(TII->get(NewOpc));
140     MI.addOperand(MachineOperand::CreateImm(MaskImm));
141     return true;
142   };
143 
144   // `vpermilps r, i` -> `vshufps r, r, i`
145   // `vpermilps r, i, k` -> `vshufps r, r, i, k`
146   // `vshufps` is always as fast or faster than `vpermilps` and takes
147   // 1 less byte of code size for VEX and EVEX encoding.
148   auto ProcessVPERMILPSri = [&](unsigned NewOpc) -> bool {
149     if (!NewOpcPreferable(NewOpc))
150       return false;
151     unsigned MaskImm = MI.getOperand(NumOperands - 1).getImm();
152     MI.removeOperand(NumOperands - 1);
153     MI.addOperand(MI.getOperand(NumOperands - 2));
154     MI.setDesc(TII->get(NewOpc));
155     MI.addOperand(MachineOperand::CreateImm(MaskImm));
156     return true;
157   };
158 
159   // `vpermilps m, i` -> `vpshufd m, i` iff no domain delay penalty on shuffles.
160   // `vpshufd` is always as fast or faster than `vpermilps` and takes 1 less
161   // byte of code size.
162   auto ProcessVPERMILPSmi = [&](unsigned NewOpc) -> bool {
163     // TODO: Might be work adding bypass delay if -Os/-Oz is enabled as
164     // `vpshufd` saves a byte of code size.
165     if (!ST->hasNoDomainDelayShuffle() ||
166         !NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
167       return false;
168     MI.setDesc(TII->get(NewOpc));
169     return true;
170   };
171 
172   // `vunpcklpd/vmovlhps r, r` -> `vunpcklqdq r, r`/`vshufpd r, r, 0x00`
173   // `vunpckhpd/vmovlhps r, r` -> `vunpckhqdq r, r`/`vshufpd r, r, 0xff`
174   // `vunpcklpd r, r, k` -> `vunpcklqdq r, r, k`/`vshufpd r, r, k, 0x00`
175   // `vunpckhpd r, r, k` -> `vunpckhqdq r, r, k`/`vshufpd r, r, k, 0xff`
176   // `vunpcklpd r, m` -> `vunpcklqdq r, m, k`
177   // `vunpckhpd r, m` -> `vunpckhqdq r, m, k`
178   // `vunpcklpd r, m, k` -> `vunpcklqdq r, m, k`
179   // `vunpckhpd r, m, k` -> `vunpckhqdq r, m, k`
180   // 1) If no bypass delay and `vunpck{l|h}qdq` faster than `vunpck{l|h}pd`
181   //        -> `vunpck{l|h}qdq`
182   // 2) If `vshufpd` faster than `vunpck{l|h}pd`
183   //        -> `vshufpd`
184   //
185   // `vunpcklps` -> `vunpckldq` (for all operand types if no bypass delay)
186   auto ProcessUNPCK = [&](unsigned NewOpc, unsigned MaskImm) -> bool {
187     if (!NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
188       return false;
189 
190     MI.setDesc(TII->get(NewOpc));
191     MI.addOperand(MachineOperand::CreateImm(MaskImm));
192     return true;
193   };
194 
195   auto ProcessUNPCKToIntDomain = [&](unsigned NewOpc) -> bool {
196     // TODO it may be worth it to set ReplaceInTie to `true` as there is no real
197     // downside to the integer unpck, but if someone doesn't specify exact
198     // target we won't find it faster.
199     if (!ST->hasNoDomainDelayShuffle() ||
200         !NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
201       return false;
202     MI.setDesc(TII->get(NewOpc));
203     return true;
204   };
205 
206   auto ProcessUNPCKLPDrr = [&](unsigned NewOpcIntDomain,
207                                unsigned NewOpc) -> bool {
208     if (ProcessUNPCKToIntDomain(NewOpcIntDomain))
209       return true;
210     return ProcessUNPCK(NewOpc, 0x00);
211   };
212   auto ProcessUNPCKHPDrr = [&](unsigned NewOpcIntDomain,
213                                unsigned NewOpc) -> bool {
214     if (ProcessUNPCKToIntDomain(NewOpcIntDomain))
215       return true;
216     return ProcessUNPCK(NewOpc, 0xff);
217   };
218 
219   auto ProcessUNPCKPDrm = [&](unsigned NewOpcIntDomain) -> bool {
220     return ProcessUNPCKToIntDomain(NewOpcIntDomain);
221   };
222 
223   auto ProcessUNPCKPS = [&](unsigned NewOpc) -> bool {
224     return ProcessUNPCKToIntDomain(NewOpc);
225   };
226 
227   switch (Opc) {
228   case X86::VPERMILPDri:
229     return ProcessVPERMILPDri(X86::VSHUFPDrri);
230   case X86::VPERMILPDYri:
231     return ProcessVPERMILPDri(X86::VSHUFPDYrri);
232   case X86::VPERMILPDZ128ri:
233     return ProcessVPERMILPDri(X86::VSHUFPDZ128rri);
234   case X86::VPERMILPDZ256ri:
235     return ProcessVPERMILPDri(X86::VSHUFPDZ256rri);
236   case X86::VPERMILPDZri:
237     return ProcessVPERMILPDri(X86::VSHUFPDZrri);
238   case X86::VPERMILPDZ128rikz:
239     return ProcessVPERMILPDri(X86::VSHUFPDZ128rrikz);
240   case X86::VPERMILPDZ256rikz:
241     return ProcessVPERMILPDri(X86::VSHUFPDZ256rrikz);
242   case X86::VPERMILPDZrikz:
243     return ProcessVPERMILPDri(X86::VSHUFPDZrrikz);
244   case X86::VPERMILPDZ128rik:
245     return ProcessVPERMILPDri(X86::VSHUFPDZ128rrik);
246   case X86::VPERMILPDZ256rik:
247     return ProcessVPERMILPDri(X86::VSHUFPDZ256rrik);
248   case X86::VPERMILPDZrik:
249     return ProcessVPERMILPDri(X86::VSHUFPDZrrik);
250 
251   case X86::VPERMILPSri:
252     return ProcessVPERMILPSri(X86::VSHUFPSrri);
253   case X86::VPERMILPSYri:
254     return ProcessVPERMILPSri(X86::VSHUFPSYrri);
255   case X86::VPERMILPSZ128ri:
256     return ProcessVPERMILPSri(X86::VSHUFPSZ128rri);
257   case X86::VPERMILPSZ256ri:
258     return ProcessVPERMILPSri(X86::VSHUFPSZ256rri);
259   case X86::VPERMILPSZri:
260     return ProcessVPERMILPSri(X86::VSHUFPSZrri);
261   case X86::VPERMILPSZ128rikz:
262     return ProcessVPERMILPSri(X86::VSHUFPSZ128rrikz);
263   case X86::VPERMILPSZ256rikz:
264     return ProcessVPERMILPSri(X86::VSHUFPSZ256rrikz);
265   case X86::VPERMILPSZrikz:
266     return ProcessVPERMILPSri(X86::VSHUFPSZrrikz);
267   case X86::VPERMILPSZ128rik:
268     return ProcessVPERMILPSri(X86::VSHUFPSZ128rrik);
269   case X86::VPERMILPSZ256rik:
270     return ProcessVPERMILPSri(X86::VSHUFPSZ256rrik);
271   case X86::VPERMILPSZrik:
272     return ProcessVPERMILPSri(X86::VSHUFPSZrrik);
273   case X86::VPERMILPSmi:
274     return ProcessVPERMILPSmi(X86::VPSHUFDmi);
275   case X86::VPERMILPSYmi:
276     // TODO: See if there is a more generic way we can test if the replacement
277     // instruction is supported.
278     return ST->hasAVX2() ? ProcessVPERMILPSmi(X86::VPSHUFDYmi) : false;
279   case X86::VPERMILPSZ128mi:
280     return ProcessVPERMILPSmi(X86::VPSHUFDZ128mi);
281   case X86::VPERMILPSZ256mi:
282     return ProcessVPERMILPSmi(X86::VPSHUFDZ256mi);
283   case X86::VPERMILPSZmi:
284     return ProcessVPERMILPSmi(X86::VPSHUFDZmi);
285   case X86::VPERMILPSZ128mikz:
286     return ProcessVPERMILPSmi(X86::VPSHUFDZ128mikz);
287   case X86::VPERMILPSZ256mikz:
288     return ProcessVPERMILPSmi(X86::VPSHUFDZ256mikz);
289   case X86::VPERMILPSZmikz:
290     return ProcessVPERMILPSmi(X86::VPSHUFDZmikz);
291   case X86::VPERMILPSZ128mik:
292     return ProcessVPERMILPSmi(X86::VPSHUFDZ128mik);
293   case X86::VPERMILPSZ256mik:
294     return ProcessVPERMILPSmi(X86::VPSHUFDZ256mik);
295   case X86::VPERMILPSZmik:
296     return ProcessVPERMILPSmi(X86::VPSHUFDZmik);
297 
298   case X86::MOVLHPSrr:
299   case X86::UNPCKLPDrr:
300     return ProcessUNPCKLPDrr(X86::PUNPCKLQDQrr, X86::SHUFPDrri);
301   case X86::VMOVLHPSrr:
302   case X86::VUNPCKLPDrr:
303     return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQrr, X86::VSHUFPDrri);
304   case X86::VUNPCKLPDYrr:
305     return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQYrr, X86::VSHUFPDYrri);
306     // VMOVLHPS is always 128 bits.
307   case X86::VMOVLHPSZrr:
308   case X86::VUNPCKLPDZ128rr:
309     return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ128rr, X86::VSHUFPDZ128rri);
310   case X86::VUNPCKLPDZ256rr:
311     return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ256rr, X86::VSHUFPDZ256rri);
312   case X86::VUNPCKLPDZrr:
313     return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZrr, X86::VSHUFPDZrri);
314   case X86::VUNPCKLPDZ128rrk:
315     return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ128rrk, X86::VSHUFPDZ128rrik);
316   case X86::VUNPCKLPDZ256rrk:
317     return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ256rrk, X86::VSHUFPDZ256rrik);
318   case X86::VUNPCKLPDZrrk:
319     return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZrrk, X86::VSHUFPDZrrik);
320   case X86::VUNPCKLPDZ128rrkz:
321     return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ128rrkz, X86::VSHUFPDZ128rrikz);
322   case X86::VUNPCKLPDZ256rrkz:
323     return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ256rrkz, X86::VSHUFPDZ256rrikz);
324   case X86::VUNPCKLPDZrrkz:
325     return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZrrkz, X86::VSHUFPDZrrikz);
326   case X86::UNPCKHPDrr:
327     return ProcessUNPCKHPDrr(X86::PUNPCKHQDQrr, X86::SHUFPDrri);
328   case X86::VUNPCKHPDrr:
329     return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQrr, X86::VSHUFPDrri);
330   case X86::VUNPCKHPDYrr:
331     return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQYrr, X86::VSHUFPDYrri);
332   case X86::VUNPCKHPDZ128rr:
333     return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ128rr, X86::VSHUFPDZ128rri);
334   case X86::VUNPCKHPDZ256rr:
335     return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ256rr, X86::VSHUFPDZ256rri);
336   case X86::VUNPCKHPDZrr:
337     return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZrr, X86::VSHUFPDZrri);
338   case X86::VUNPCKHPDZ128rrk:
339     return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ128rrk, X86::VSHUFPDZ128rrik);
340   case X86::VUNPCKHPDZ256rrk:
341     return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ256rrk, X86::VSHUFPDZ256rrik);
342   case X86::VUNPCKHPDZrrk:
343     return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZrrk, X86::VSHUFPDZrrik);
344   case X86::VUNPCKHPDZ128rrkz:
345     return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ128rrkz, X86::VSHUFPDZ128rrikz);
346   case X86::VUNPCKHPDZ256rrkz:
347     return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ256rrkz, X86::VSHUFPDZ256rrikz);
348   case X86::VUNPCKHPDZrrkz:
349     return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZrrkz, X86::VSHUFPDZrrikz);
350   case X86::UNPCKLPDrm:
351     return ProcessUNPCKPDrm(X86::PUNPCKLQDQrm);
352   case X86::VUNPCKLPDrm:
353     return ProcessUNPCKPDrm(X86::VPUNPCKLQDQrm);
354   case X86::VUNPCKLPDYrm:
355     return ProcessUNPCKPDrm(X86::VPUNPCKLQDQYrm);
356   case X86::VUNPCKLPDZ128rm:
357     return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ128rm);
358   case X86::VUNPCKLPDZ256rm:
359     return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ256rm);
360   case X86::VUNPCKLPDZrm:
361     return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZrm);
362   case X86::VUNPCKLPDZ128rmk:
363     return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ128rmk);
364   case X86::VUNPCKLPDZ256rmk:
365     return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ256rmk);
366   case X86::VUNPCKLPDZrmk:
367     return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZrmk);
368   case X86::VUNPCKLPDZ128rmkz:
369     return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ128rmkz);
370   case X86::VUNPCKLPDZ256rmkz:
371     return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ256rmkz);
372   case X86::VUNPCKLPDZrmkz:
373     return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZrmkz);
374   case X86::UNPCKHPDrm:
375     return ProcessUNPCKPDrm(X86::PUNPCKHQDQrm);
376   case X86::VUNPCKHPDrm:
377     return ProcessUNPCKPDrm(X86::VPUNPCKHQDQrm);
378   case X86::VUNPCKHPDYrm:
379     return ProcessUNPCKPDrm(X86::VPUNPCKHQDQYrm);
380   case X86::VUNPCKHPDZ128rm:
381     return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ128rm);
382   case X86::VUNPCKHPDZ256rm:
383     return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ256rm);
384   case X86::VUNPCKHPDZrm:
385     return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZrm);
386   case X86::VUNPCKHPDZ128rmk:
387     return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ128rmk);
388   case X86::VUNPCKHPDZ256rmk:
389     return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ256rmk);
390   case X86::VUNPCKHPDZrmk:
391     return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZrmk);
392   case X86::VUNPCKHPDZ128rmkz:
393     return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ128rmkz);
394   case X86::VUNPCKHPDZ256rmkz:
395     return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ256rmkz);
396   case X86::VUNPCKHPDZrmkz:
397     return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZrmkz);
398 
399   case X86::UNPCKLPSrr:
400     return ProcessUNPCKPS(X86::PUNPCKLDQrr);
401   case X86::VUNPCKLPSrr:
402     return ProcessUNPCKPS(X86::VPUNPCKLDQrr);
403   case X86::VUNPCKLPSYrr:
404     return ProcessUNPCKPS(X86::VPUNPCKLDQYrr);
405   case X86::VUNPCKLPSZ128rr:
406     return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rr);
407   case X86::VUNPCKLPSZ256rr:
408     return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rr);
409   case X86::VUNPCKLPSZrr:
410     return ProcessUNPCKPS(X86::VPUNPCKLDQZrr);
411   case X86::VUNPCKLPSZ128rrk:
412     return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rrk);
413   case X86::VUNPCKLPSZ256rrk:
414     return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rrk);
415   case X86::VUNPCKLPSZrrk:
416     return ProcessUNPCKPS(X86::VPUNPCKLDQZrrk);
417   case X86::VUNPCKLPSZ128rrkz:
418     return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rrkz);
419   case X86::VUNPCKLPSZ256rrkz:
420     return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rrkz);
421   case X86::VUNPCKLPSZrrkz:
422     return ProcessUNPCKPS(X86::VPUNPCKLDQZrrkz);
423   case X86::UNPCKHPSrr:
424     return ProcessUNPCKPS(X86::PUNPCKHDQrr);
425   case X86::VUNPCKHPSrr:
426     return ProcessUNPCKPS(X86::VPUNPCKHDQrr);
427   case X86::VUNPCKHPSYrr:
428     return ProcessUNPCKPS(X86::VPUNPCKHDQYrr);
429   case X86::VUNPCKHPSZ128rr:
430     return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rr);
431   case X86::VUNPCKHPSZ256rr:
432     return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rr);
433   case X86::VUNPCKHPSZrr:
434     return ProcessUNPCKPS(X86::VPUNPCKHDQZrr);
435   case X86::VUNPCKHPSZ128rrk:
436     return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rrk);
437   case X86::VUNPCKHPSZ256rrk:
438     return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rrk);
439   case X86::VUNPCKHPSZrrk:
440     return ProcessUNPCKPS(X86::VPUNPCKHDQZrrk);
441   case X86::VUNPCKHPSZ128rrkz:
442     return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rrkz);
443   case X86::VUNPCKHPSZ256rrkz:
444     return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rrkz);
445   case X86::VUNPCKHPSZrrkz:
446     return ProcessUNPCKPS(X86::VPUNPCKHDQZrrkz);
447   case X86::UNPCKLPSrm:
448     return ProcessUNPCKPS(X86::PUNPCKLDQrm);
449   case X86::VUNPCKLPSrm:
450     return ProcessUNPCKPS(X86::VPUNPCKLDQrm);
451   case X86::VUNPCKLPSYrm:
452     return ProcessUNPCKPS(X86::VPUNPCKLDQYrm);
453   case X86::VUNPCKLPSZ128rm:
454     return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rm);
455   case X86::VUNPCKLPSZ256rm:
456     return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rm);
457   case X86::VUNPCKLPSZrm:
458     return ProcessUNPCKPS(X86::VPUNPCKLDQZrm);
459   case X86::VUNPCKLPSZ128rmk:
460     return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rmk);
461   case X86::VUNPCKLPSZ256rmk:
462     return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rmk);
463   case X86::VUNPCKLPSZrmk:
464     return ProcessUNPCKPS(X86::VPUNPCKLDQZrmk);
465   case X86::VUNPCKLPSZ128rmkz:
466     return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rmkz);
467   case X86::VUNPCKLPSZ256rmkz:
468     return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rmkz);
469   case X86::VUNPCKLPSZrmkz:
470     return ProcessUNPCKPS(X86::VPUNPCKLDQZrmkz);
471   case X86::UNPCKHPSrm:
472     return ProcessUNPCKPS(X86::PUNPCKHDQrm);
473   case X86::VUNPCKHPSrm:
474     return ProcessUNPCKPS(X86::VPUNPCKHDQrm);
475   case X86::VUNPCKHPSYrm:
476     return ProcessUNPCKPS(X86::VPUNPCKHDQYrm);
477   case X86::VUNPCKHPSZ128rm:
478     return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rm);
479   case X86::VUNPCKHPSZ256rm:
480     return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rm);
481   case X86::VUNPCKHPSZrm:
482     return ProcessUNPCKPS(X86::VPUNPCKHDQZrm);
483   case X86::VUNPCKHPSZ128rmk:
484     return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rmk);
485   case X86::VUNPCKHPSZ256rmk:
486     return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rmk);
487   case X86::VUNPCKHPSZrmk:
488     return ProcessUNPCKPS(X86::VPUNPCKHDQZrmk);
489   case X86::VUNPCKHPSZ128rmkz:
490     return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rmkz);
491   case X86::VUNPCKHPSZ256rmkz:
492     return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rmkz);
493   case X86::VUNPCKHPSZrmkz:
494     return ProcessUNPCKPS(X86::VPUNPCKHDQZrmkz);
495   default:
496     return false;
497   }
498 }
499 
runOnMachineFunction(MachineFunction & MF)500 bool X86FixupInstTuningPass::runOnMachineFunction(MachineFunction &MF) {
501   LLVM_DEBUG(dbgs() << "Start X86FixupInstTuning\n";);
502   bool Changed = false;
503   ST = &MF.getSubtarget<X86Subtarget>();
504   TII = ST->getInstrInfo();
505   SM = &ST->getSchedModel();
506 
507   for (MachineBasicBlock &MBB : MF) {
508     for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) {
509       if (processInstruction(MF, MBB, I)) {
510         ++NumInstChanges;
511         Changed = true;
512       }
513     }
514   }
515   LLVM_DEBUG(dbgs() << "End X86FixupInstTuning\n";);
516   return Changed;
517 }
518