1 //===- X86CompressEVEX.cpp ------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass compresses instructions from EVEX space to legacy/VEX/EVEX space
10 // when possible in order to reduce code size or facilitate HW decoding.
11 //
12 // Possible compression:
13 // a. AVX512 instruction (EVEX) -> AVX instruction (VEX)
14 // b. Promoted instruction (EVEX) -> pre-promotion instruction (legacy/VEX)
15 // c. NDD (EVEX) -> non-NDD (legacy)
16 // d. NF_ND (EVEX) -> NF (EVEX)
17 // e. NonNF (EVEX) -> NF (EVEX)
18 //
19 // Compression a, b and c can always reduce code size, with some exceptions
20 // such as promoted 16-bit CRC32 which is as long as the legacy version.
21 //
22 // legacy:
23 // crc32w %si, %eax ## encoding: [0x66,0xf2,0x0f,0x38,0xf1,0xc6]
24 // promoted:
25 // crc32w %si, %eax ## encoding: [0x62,0xf4,0x7d,0x08,0xf1,0xc6]
26 //
27 // From performance perspective, these should be same (same uops and same EXE
28 // ports). From a FMV perspective, an older legacy encoding is preferred b/c it
29 // can execute in more places (broader HW install base). So we will still do
30 // the compression.
31 //
32 // Compression d can help hardware decode (HW may skip reading the NDD
33 // register) although the instruction length remains unchanged.
34 //
35 // Compression e can help hardware skip updating EFLAGS although the instruction
36 // length remains unchanged.
37 //===----------------------------------------------------------------------===//
38
39 #include "MCTargetDesc/X86BaseInfo.h"
40 #include "X86.h"
41 #include "X86InstrInfo.h"
42 #include "X86Subtarget.h"
43 #include "llvm/ADT/StringRef.h"
44 #include "llvm/CodeGen/MachineFunction.h"
45 #include "llvm/CodeGen/MachineFunctionPass.h"
46 #include "llvm/CodeGen/MachineInstr.h"
47 #include "llvm/CodeGen/MachineOperand.h"
48 #include "llvm/MC/MCInstrDesc.h"
49 #include "llvm/Pass.h"
50 #include <atomic>
51 #include <cassert>
52 #include <cstdint>
53
54 using namespace llvm;
55
56 #define COMP_EVEX_DESC "Compressing EVEX instrs when possible"
57 #define COMP_EVEX_NAME "x86-compress-evex"
58
59 #define DEBUG_TYPE COMP_EVEX_NAME
60
61 extern cl::opt<bool> X86EnableAPXForRelocation;
62
63 namespace {
64 // Including the generated EVEX compression tables.
65 #define GET_X86_COMPRESS_EVEX_TABLE
66 #include "X86GenInstrMapping.inc"
67
68 class CompressEVEXPass : public MachineFunctionPass {
69 public:
70 static char ID;
CompressEVEXPass()71 CompressEVEXPass() : MachineFunctionPass(ID) {}
getPassName() const72 StringRef getPassName() const override { return COMP_EVEX_DESC; }
73
74 bool runOnMachineFunction(MachineFunction &MF) override;
75
76 // This pass runs after regalloc and doesn't support VReg operands.
getRequiredProperties() const77 MachineFunctionProperties getRequiredProperties() const override {
78 return MachineFunctionProperties().setNoVRegs();
79 }
80 };
81
82 } // end anonymous namespace
83
84 char CompressEVEXPass::ID = 0;
85
usesExtendedRegister(const MachineInstr & MI)86 static bool usesExtendedRegister(const MachineInstr &MI) {
87 auto isHiRegIdx = [](MCRegister Reg) {
88 // Check for XMM register with indexes between 16 - 31.
89 if (Reg >= X86::XMM16 && Reg <= X86::XMM31)
90 return true;
91 // Check for YMM register with indexes between 16 - 31.
92 if (Reg >= X86::YMM16 && Reg <= X86::YMM31)
93 return true;
94 // Check for GPR with indexes between 16 - 31.
95 if (X86II::isApxExtendedReg(Reg))
96 return true;
97 return false;
98 };
99
100 // Check that operands are not ZMM regs or
101 // XMM/YMM regs with hi indexes between 16 - 31.
102 for (const MachineOperand &MO : MI.explicit_operands()) {
103 if (!MO.isReg())
104 continue;
105
106 MCRegister Reg = MO.getReg().asMCReg();
107 assert(!X86II::isZMMReg(Reg) &&
108 "ZMM instructions should not be in the EVEX->VEX tables");
109 if (isHiRegIdx(Reg))
110 return true;
111 }
112
113 return false;
114 }
115
116 // Do any custom cleanup needed to finalize the conversion.
performCustomAdjustments(MachineInstr & MI,unsigned NewOpc)117 static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) {
118 (void)NewOpc;
119 unsigned Opc = MI.getOpcode();
120 switch (Opc) {
121 case X86::VALIGNDZ128rri:
122 case X86::VALIGNDZ128rmi:
123 case X86::VALIGNQZ128rri:
124 case X86::VALIGNQZ128rmi: {
125 assert((NewOpc == X86::VPALIGNRrri || NewOpc == X86::VPALIGNRrmi) &&
126 "Unexpected new opcode!");
127 unsigned Scale =
128 (Opc == X86::VALIGNQZ128rri || Opc == X86::VALIGNQZ128rmi) ? 8 : 4;
129 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1);
130 Imm.setImm(Imm.getImm() * Scale);
131 break;
132 }
133 case X86::VSHUFF32X4Z256rmi:
134 case X86::VSHUFF32X4Z256rri:
135 case X86::VSHUFF64X2Z256rmi:
136 case X86::VSHUFF64X2Z256rri:
137 case X86::VSHUFI32X4Z256rmi:
138 case X86::VSHUFI32X4Z256rri:
139 case X86::VSHUFI64X2Z256rmi:
140 case X86::VSHUFI64X2Z256rri: {
141 assert((NewOpc == X86::VPERM2F128rri || NewOpc == X86::VPERM2I128rri ||
142 NewOpc == X86::VPERM2F128rmi || NewOpc == X86::VPERM2I128rmi) &&
143 "Unexpected new opcode!");
144 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1);
145 int64_t ImmVal = Imm.getImm();
146 // Set bit 5, move bit 1 to bit 4, copy bit 0.
147 Imm.setImm(0x20 | ((ImmVal & 2) << 3) | (ImmVal & 1));
148 break;
149 }
150 case X86::VRNDSCALEPDZ128rri:
151 case X86::VRNDSCALEPDZ128rmi:
152 case X86::VRNDSCALEPSZ128rri:
153 case X86::VRNDSCALEPSZ128rmi:
154 case X86::VRNDSCALEPDZ256rri:
155 case X86::VRNDSCALEPDZ256rmi:
156 case X86::VRNDSCALEPSZ256rri:
157 case X86::VRNDSCALEPSZ256rmi:
158 case X86::VRNDSCALESDZrri:
159 case X86::VRNDSCALESDZrmi:
160 case X86::VRNDSCALESSZrri:
161 case X86::VRNDSCALESSZrmi:
162 case X86::VRNDSCALESDZrri_Int:
163 case X86::VRNDSCALESDZrmi_Int:
164 case X86::VRNDSCALESSZrri_Int:
165 case X86::VRNDSCALESSZrmi_Int:
166 const MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1);
167 int64_t ImmVal = Imm.getImm();
168 // Ensure that only bits 3:0 of the immediate are used.
169 if ((ImmVal & 0xf) != ImmVal)
170 return false;
171 break;
172 }
173
174 return true;
175 }
176
CompressEVEXImpl(MachineInstr & MI,const X86Subtarget & ST)177 static bool CompressEVEXImpl(MachineInstr &MI, const X86Subtarget &ST) {
178 uint64_t TSFlags = MI.getDesc().TSFlags;
179
180 // Check for EVEX instructions only.
181 if ((TSFlags & X86II::EncodingMask) != X86II::EVEX)
182 return false;
183
184 // Instructions with mask or 512-bit vector can't be converted to VEX.
185 if (TSFlags & (X86II::EVEX_K | X86II::EVEX_L2))
186 return false;
187
188 auto IsRedundantNewDataDest = [&](unsigned &Opc) {
189 // $rbx = ADD64rr_ND $rbx, $rax / $rbx = ADD64rr_ND $rax, $rbx
190 // ->
191 // $rbx = ADD64rr $rbx, $rax
192 const MCInstrDesc &Desc = MI.getDesc();
193 Register Reg0 = MI.getOperand(0).getReg();
194 const MachineOperand &Op1 = MI.getOperand(1);
195 if (!Op1.isReg() || X86::getFirstAddrOperandIdx(MI) == 1 ||
196 X86::isCFCMOVCC(MI.getOpcode()))
197 return false;
198 Register Reg1 = Op1.getReg();
199 if (Reg1 == Reg0)
200 return true;
201
202 // Op1 and Op2 may be commutable for ND instructions.
203 if (!Desc.isCommutable() || Desc.getNumOperands() < 3 ||
204 !MI.getOperand(2).isReg() || MI.getOperand(2).getReg() != Reg0)
205 return false;
206 // Opcode may change after commute, e.g. SHRD -> SHLD
207 ST.getInstrInfo()->commuteInstruction(MI, false, 1, 2);
208 Opc = MI.getOpcode();
209 return true;
210 };
211
212 // EVEX_B has several meanings.
213 // AVX512:
214 // register form: rounding control or SAE
215 // memory form: broadcast
216 //
217 // APX:
218 // MAP4: NDD
219 //
220 // For AVX512 cases, EVEX prefix is needed in order to carry this information
221 // thus preventing the transformation to VEX encoding.
222 bool IsND = X86II::hasNewDataDest(TSFlags);
223 if (TSFlags & X86II::EVEX_B && !IsND)
224 return false;
225 unsigned Opc = MI.getOpcode();
226 // MOVBE*rr is special because it has semantic of NDD but not set EVEX_B.
227 bool IsNDLike = IsND || Opc == X86::MOVBE32rr || Opc == X86::MOVBE64rr;
228 bool IsRedundantNDD = IsNDLike ? IsRedundantNewDataDest(Opc) : false;
229
230 auto GetCompressedOpc = [&](unsigned Opc) -> unsigned {
231 ArrayRef<X86TableEntry> Table = ArrayRef(X86CompressEVEXTable);
232 const auto I = llvm::lower_bound(Table, Opc);
233 if (I == Table.end() || I->OldOpc != Opc)
234 return 0;
235
236 if (usesExtendedRegister(MI) || !checkPredicate(I->NewOpc, &ST) ||
237 !performCustomAdjustments(MI, I->NewOpc))
238 return 0;
239 return I->NewOpc;
240 };
241
242 // Redundant NDD ops cannot be safely compressed if either:
243 // - the legacy op would introduce a partial write that BreakFalseDeps
244 // identified as a potential stall, or
245 // - the op is writing to a subregister of a live register, i.e. the
246 // full (zeroed) result is used.
247 // Both cases are indicated by an implicit def of the superregister.
248 if (IsRedundantNDD) {
249 Register Dst = MI.getOperand(0).getReg();
250 if (Dst &&
251 (X86::GR16RegClass.contains(Dst) || X86::GR8RegClass.contains(Dst))) {
252 Register Super = getX86SubSuperRegister(Dst, 64);
253 if (MI.definesRegister(Super, /*TRI=*/nullptr))
254 IsRedundantNDD = false;
255 }
256
257 // ADDrm/mr instructions with NDD + relocation had been transformed to the
258 // instructions without NDD in X86SuppressAPXForRelocation pass. That is to
259 // keep backward compatibility with linkers without APX support.
260 if (!X86EnableAPXForRelocation)
261 assert(!isAddMemInstrWithRelocation(MI) &&
262 "Unexpected NDD instruction with relocation!");
263 }
264
265 // NonNF -> NF only if it's not a compressible NDD instruction and eflags is
266 // dead.
267 unsigned NewOpc = IsRedundantNDD
268 ? X86::getNonNDVariant(Opc)
269 : ((IsNDLike && ST.hasNF() &&
270 MI.registerDefIsDead(X86::EFLAGS, /*TRI=*/nullptr))
271 ? X86::getNFVariant(Opc)
272 : GetCompressedOpc(Opc));
273
274 if (!NewOpc)
275 return false;
276
277 const MCInstrDesc &NewDesc = ST.getInstrInfo()->get(NewOpc);
278 MI.setDesc(NewDesc);
279 unsigned AsmComment;
280 switch (NewDesc.TSFlags & X86II::EncodingMask) {
281 case X86II::LEGACY:
282 AsmComment = X86::AC_EVEX_2_LEGACY;
283 break;
284 case X86II::VEX:
285 AsmComment = X86::AC_EVEX_2_VEX;
286 break;
287 case X86II::EVEX:
288 AsmComment = X86::AC_EVEX_2_EVEX;
289 assert(IsND && (NewDesc.TSFlags & X86II::EVEX_NF) &&
290 "Unknown EVEX2EVEX compression");
291 break;
292 default:
293 llvm_unreachable("Unknown EVEX compression");
294 }
295 MI.setAsmPrinterFlag(AsmComment);
296 if (IsRedundantNDD)
297 MI.tieOperands(0, 1);
298
299 return true;
300 }
301
runOnMachineFunction(MachineFunction & MF)302 bool CompressEVEXPass::runOnMachineFunction(MachineFunction &MF) {
303 LLVM_DEBUG(dbgs() << "Start X86CompressEVEXPass\n";);
304 #ifndef NDEBUG
305 // Make sure the tables are sorted.
306 static std::atomic<bool> TableChecked(false);
307 if (!TableChecked.load(std::memory_order_relaxed)) {
308 assert(llvm::is_sorted(X86CompressEVEXTable) &&
309 "X86CompressEVEXTable is not sorted!");
310 TableChecked.store(true, std::memory_order_relaxed);
311 }
312 #endif
313 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
314 if (!ST.hasAVX512() && !ST.hasEGPR() && !ST.hasNDD())
315 return false;
316
317 bool Changed = false;
318
319 for (MachineBasicBlock &MBB : MF) {
320 // Traverse the basic block.
321 for (MachineInstr &MI : MBB)
322 Changed |= CompressEVEXImpl(MI, ST);
323 }
324 LLVM_DEBUG(dbgs() << "End X86CompressEVEXPass\n";);
325 return Changed;
326 }
327
INITIALIZE_PASS(CompressEVEXPass,COMP_EVEX_NAME,COMP_EVEX_DESC,false,false)328 INITIALIZE_PASS(CompressEVEXPass, COMP_EVEX_NAME, COMP_EVEX_DESC, false, false)
329
330 FunctionPass *llvm::createX86CompressEVEXPass() {
331 return new CompressEVEXPass();
332 }
333