1 //===- X86CompressEVEX.cpp ------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass compresses instructions from EVEX space to legacy/VEX/EVEX space 10 // when possible in order to reduce code size or facilitate HW decoding. 11 // 12 // Possible compression: 13 // a. AVX512 instruction (EVEX) -> AVX instruction (VEX) 14 // b. Promoted instruction (EVEX) -> pre-promotion instruction (legacy/VEX) 15 // c. NDD (EVEX) -> non-NDD (legacy) 16 // d. NF_ND (EVEX) -> NF (EVEX) 17 // e. NonNF (EVEX) -> NF (EVEX) 18 // 19 // Compression a, b and c can always reduce code size, with some exceptions 20 // such as promoted 16-bit CRC32 which is as long as the legacy version. 21 // 22 // legacy: 23 // crc32w %si, %eax ## encoding: [0x66,0xf2,0x0f,0x38,0xf1,0xc6] 24 // promoted: 25 // crc32w %si, %eax ## encoding: [0x62,0xf4,0x7d,0x08,0xf1,0xc6] 26 // 27 // From performance perspective, these should be same (same uops and same EXE 28 // ports). From a FMV perspective, an older legacy encoding is preferred b/c it 29 // can execute in more places (broader HW install base). So we will still do 30 // the compression. 31 // 32 // Compression d can help hardware decode (HW may skip reading the NDD 33 // register) although the instruction length remains unchanged. 34 // 35 // Compression e can help hardware skip updating EFLAGS although the instruction 36 // length remains unchanged. 37 //===----------------------------------------------------------------------===// 38 39 #include "MCTargetDesc/X86BaseInfo.h" 40 #include "MCTargetDesc/X86InstComments.h" 41 #include "X86.h" 42 #include "X86InstrInfo.h" 43 #include "X86Subtarget.h" 44 #include "llvm/ADT/StringRef.h" 45 #include "llvm/CodeGen/MachineFunction.h" 46 #include "llvm/CodeGen/MachineFunctionPass.h" 47 #include "llvm/CodeGen/MachineInstr.h" 48 #include "llvm/CodeGen/MachineOperand.h" 49 #include "llvm/MC/MCInstrDesc.h" 50 #include "llvm/Pass.h" 51 #include <atomic> 52 #include <cassert> 53 #include <cstdint> 54 55 using namespace llvm; 56 57 #define COMP_EVEX_DESC "Compressing EVEX instrs when possible" 58 #define COMP_EVEX_NAME "x86-compress-evex" 59 60 #define DEBUG_TYPE COMP_EVEX_NAME 61 62 namespace { 63 // Including the generated EVEX compression tables. 64 #define GET_X86_COMPRESS_EVEX_TABLE 65 #include "X86GenInstrMapping.inc" 66 67 class CompressEVEXPass : public MachineFunctionPass { 68 public: 69 static char ID; 70 CompressEVEXPass() : MachineFunctionPass(ID) {} 71 StringRef getPassName() const override { return COMP_EVEX_DESC; } 72 73 bool runOnMachineFunction(MachineFunction &MF) override; 74 75 // This pass runs after regalloc and doesn't support VReg operands. 76 MachineFunctionProperties getRequiredProperties() const override { 77 return MachineFunctionProperties().set( 78 MachineFunctionProperties::Property::NoVRegs); 79 } 80 }; 81 82 } // end anonymous namespace 83 84 char CompressEVEXPass::ID = 0; 85 86 static bool usesExtendedRegister(const MachineInstr &MI) { 87 auto isHiRegIdx = [](unsigned Reg) { 88 // Check for XMM register with indexes between 16 - 31. 89 if (Reg >= X86::XMM16 && Reg <= X86::XMM31) 90 return true; 91 // Check for YMM register with indexes between 16 - 31. 92 if (Reg >= X86::YMM16 && Reg <= X86::YMM31) 93 return true; 94 // Check for GPR with indexes between 16 - 31. 95 if (X86II::isApxExtendedReg(Reg)) 96 return true; 97 return false; 98 }; 99 100 // Check that operands are not ZMM regs or 101 // XMM/YMM regs with hi indexes between 16 - 31. 102 for (const MachineOperand &MO : MI.explicit_operands()) { 103 if (!MO.isReg()) 104 continue; 105 106 Register Reg = MO.getReg(); 107 assert(!X86II::isZMMReg(Reg) && 108 "ZMM instructions should not be in the EVEX->VEX tables"); 109 if (isHiRegIdx(Reg)) 110 return true; 111 } 112 113 return false; 114 } 115 116 // Do any custom cleanup needed to finalize the conversion. 117 static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) { 118 (void)NewOpc; 119 unsigned Opc = MI.getOpcode(); 120 switch (Opc) { 121 case X86::VALIGNDZ128rri: 122 case X86::VALIGNDZ128rmi: 123 case X86::VALIGNQZ128rri: 124 case X86::VALIGNQZ128rmi: { 125 assert((NewOpc == X86::VPALIGNRrri || NewOpc == X86::VPALIGNRrmi) && 126 "Unexpected new opcode!"); 127 unsigned Scale = 128 (Opc == X86::VALIGNQZ128rri || Opc == X86::VALIGNQZ128rmi) ? 8 : 4; 129 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1); 130 Imm.setImm(Imm.getImm() * Scale); 131 break; 132 } 133 case X86::VSHUFF32X4Z256rmi: 134 case X86::VSHUFF32X4Z256rri: 135 case X86::VSHUFF64X2Z256rmi: 136 case X86::VSHUFF64X2Z256rri: 137 case X86::VSHUFI32X4Z256rmi: 138 case X86::VSHUFI32X4Z256rri: 139 case X86::VSHUFI64X2Z256rmi: 140 case X86::VSHUFI64X2Z256rri: { 141 assert((NewOpc == X86::VPERM2F128rr || NewOpc == X86::VPERM2I128rr || 142 NewOpc == X86::VPERM2F128rm || NewOpc == X86::VPERM2I128rm) && 143 "Unexpected new opcode!"); 144 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1); 145 int64_t ImmVal = Imm.getImm(); 146 // Set bit 5, move bit 1 to bit 4, copy bit 0. 147 Imm.setImm(0x20 | ((ImmVal & 2) << 3) | (ImmVal & 1)); 148 break; 149 } 150 case X86::VRNDSCALEPDZ128rri: 151 case X86::VRNDSCALEPDZ128rmi: 152 case X86::VRNDSCALEPSZ128rri: 153 case X86::VRNDSCALEPSZ128rmi: 154 case X86::VRNDSCALEPDZ256rri: 155 case X86::VRNDSCALEPDZ256rmi: 156 case X86::VRNDSCALEPSZ256rri: 157 case X86::VRNDSCALEPSZ256rmi: 158 case X86::VRNDSCALESDZr: 159 case X86::VRNDSCALESDZm: 160 case X86::VRNDSCALESSZr: 161 case X86::VRNDSCALESSZm: 162 case X86::VRNDSCALESDZr_Int: 163 case X86::VRNDSCALESDZm_Int: 164 case X86::VRNDSCALESSZr_Int: 165 case X86::VRNDSCALESSZm_Int: 166 const MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1); 167 int64_t ImmVal = Imm.getImm(); 168 // Ensure that only bits 3:0 of the immediate are used. 169 if ((ImmVal & 0xf) != ImmVal) 170 return false; 171 break; 172 } 173 174 return true; 175 } 176 177 static bool CompressEVEXImpl(MachineInstr &MI, const X86Subtarget &ST) { 178 uint64_t TSFlags = MI.getDesc().TSFlags; 179 180 // Check for EVEX instructions only. 181 if ((TSFlags & X86II::EncodingMask) != X86II::EVEX) 182 return false; 183 184 // Instructions with mask or 512-bit vector can't be converted to VEX. 185 if (TSFlags & (X86II::EVEX_K | X86II::EVEX_L2)) 186 return false; 187 188 auto IsRedundantNewDataDest = [&](unsigned &Opc) { 189 // $rbx = ADD64rr_ND $rbx, $rax / $rbx = ADD64rr_ND $rax, $rbx 190 // -> 191 // $rbx = ADD64rr $rbx, $rax 192 const MCInstrDesc &Desc = MI.getDesc(); 193 Register Reg0 = MI.getOperand(0).getReg(); 194 const MachineOperand &Op1 = MI.getOperand(1); 195 if (!Op1.isReg() || X86::getFirstAddrOperandIdx(MI) == 1 || 196 X86::isCFCMOVCC(MI.getOpcode())) 197 return false; 198 Register Reg1 = Op1.getReg(); 199 if (Reg1 == Reg0) 200 return true; 201 202 // Op1 and Op2 may be commutable for ND instructions. 203 if (!Desc.isCommutable() || Desc.getNumOperands() < 3 || 204 !MI.getOperand(2).isReg() || MI.getOperand(2).getReg() != Reg0) 205 return false; 206 // Opcode may change after commute, e.g. SHRD -> SHLD 207 ST.getInstrInfo()->commuteInstruction(MI, false, 1, 2); 208 Opc = MI.getOpcode(); 209 return true; 210 }; 211 212 // EVEX_B has several meanings. 213 // AVX512: 214 // register form: rounding control or SAE 215 // memory form: broadcast 216 // 217 // APX: 218 // MAP4: NDD 219 // 220 // For AVX512 cases, EVEX prefix is needed in order to carry this information 221 // thus preventing the transformation to VEX encoding. 222 bool IsND = X86II::hasNewDataDest(TSFlags); 223 if (TSFlags & X86II::EVEX_B && !IsND) 224 return false; 225 unsigned Opc = MI.getOpcode(); 226 // MOVBE*rr is special because it has semantic of NDD but not set EVEX_B. 227 bool IsNDLike = IsND || Opc == X86::MOVBE32rr || Opc == X86::MOVBE64rr; 228 bool IsRedundantNDD = IsNDLike ? IsRedundantNewDataDest(Opc) : false; 229 230 auto GetCompressedOpc = [&](unsigned Opc) -> unsigned { 231 ArrayRef<X86TableEntry> Table = ArrayRef(X86CompressEVEXTable); 232 const auto I = llvm::lower_bound(Table, Opc); 233 if (I == Table.end() || I->OldOpc != Opc) 234 return 0; 235 236 if (usesExtendedRegister(MI) || !checkPredicate(I->NewOpc, &ST) || 237 !performCustomAdjustments(MI, I->NewOpc)) 238 return 0; 239 return I->NewOpc; 240 }; 241 // NonNF -> NF only if it's not a compressible NDD instruction and eflags is 242 // dead. 243 unsigned NewOpc = IsRedundantNDD 244 ? X86::getNonNDVariant(Opc) 245 : ((IsNDLike && ST.hasNF() && 246 MI.registerDefIsDead(X86::EFLAGS, /*TRI=*/nullptr)) 247 ? X86::getNFVariant(Opc) 248 : GetCompressedOpc(Opc)); 249 250 if (!NewOpc) 251 return false; 252 253 const MCInstrDesc &NewDesc = ST.getInstrInfo()->get(NewOpc); 254 MI.setDesc(NewDesc); 255 unsigned AsmComment; 256 switch (NewDesc.TSFlags & X86II::EncodingMask) { 257 case X86II::LEGACY: 258 AsmComment = X86::AC_EVEX_2_LEGACY; 259 break; 260 case X86II::VEX: 261 AsmComment = X86::AC_EVEX_2_VEX; 262 break; 263 case X86II::EVEX: 264 AsmComment = X86::AC_EVEX_2_EVEX; 265 assert(IsND && (NewDesc.TSFlags & X86II::EVEX_NF) && 266 "Unknown EVEX2EVEX compression"); 267 break; 268 default: 269 llvm_unreachable("Unknown EVEX compression"); 270 } 271 MI.setAsmPrinterFlag(AsmComment); 272 if (IsRedundantNDD) 273 MI.tieOperands(0, 1); 274 275 return true; 276 } 277 278 bool CompressEVEXPass::runOnMachineFunction(MachineFunction &MF) { 279 #ifndef NDEBUG 280 // Make sure the tables are sorted. 281 static std::atomic<bool> TableChecked(false); 282 if (!TableChecked.load(std::memory_order_relaxed)) { 283 assert(llvm::is_sorted(X86CompressEVEXTable) && 284 "X86CompressEVEXTable is not sorted!"); 285 TableChecked.store(true, std::memory_order_relaxed); 286 } 287 #endif 288 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>(); 289 if (!ST.hasAVX512() && !ST.hasEGPR() && !ST.hasNDD()) 290 return false; 291 292 bool Changed = false; 293 294 for (MachineBasicBlock &MBB : MF) { 295 // Traverse the basic block. 296 for (MachineInstr &MI : MBB) 297 Changed |= CompressEVEXImpl(MI, ST); 298 } 299 300 return Changed; 301 } 302 303 INITIALIZE_PASS(CompressEVEXPass, COMP_EVEX_NAME, COMP_EVEX_DESC, false, false) 304 305 FunctionPass *llvm::createX86CompressEVEXPass() { 306 return new CompressEVEXPass(); 307 } 308