1 //== llvm/CodeGen/GlobalISel/LegalizerHelper.h ---------------- -*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file A pass to convert the target-illegal operations created by IR -> MIR 10 /// translation into ones the target expects to be able to select. This may 11 /// occur in multiple phases, for example G_ADD <2 x i8> -> G_ADD <2 x i16> -> 12 /// G_ADD <4 x i16>. 13 /// 14 /// The LegalizerHelper class is where most of the work happens, and is 15 /// designed to be callable from other passes that find themselves with an 16 /// illegal instruction. 17 // 18 //===----------------------------------------------------------------------===// 19 20 #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H 21 #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H 22 23 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 24 #include "llvm/CodeGen/RuntimeLibcalls.h" 25 #include "llvm/CodeGen/TargetOpcodes.h" 26 27 namespace llvm { 28 // Forward declarations. 29 class APInt; 30 class GAnyLoad; 31 class GLoadStore; 32 class GStore; 33 class GenericMachineInstr; 34 class MachineFunction; 35 class MachineIRBuilder; 36 class MachineInstr; 37 class MachineInstrBuilder; 38 struct MachinePointerInfo; 39 template <typename T> class SmallVectorImpl; 40 class LegalizerInfo; 41 class MachineRegisterInfo; 42 class GISelChangeObserver; 43 class LostDebugLocObserver; 44 class TargetLowering; 45 46 class LegalizerHelper { 47 public: 48 /// Expose MIRBuilder so clients can set their own RecordInsertInstruction 49 /// functions 50 MachineIRBuilder &MIRBuilder; 51 52 /// To keep track of changes made by the LegalizerHelper. 53 GISelChangeObserver &Observer; 54 55 private: 56 MachineRegisterInfo &MRI; 57 const LegalizerInfo &LI; 58 const TargetLowering &TLI; 59 60 public: 61 enum LegalizeResult { 62 /// Instruction was already legal and no change was made to the 63 /// MachineFunction. 64 AlreadyLegal, 65 66 /// Instruction has been legalized and the MachineFunction changed. 67 Legalized, 68 69 /// Some kind of error has occurred and we could not legalize this 70 /// instruction. 71 UnableToLegalize, 72 }; 73 74 /// Expose LegalizerInfo so the clients can re-use. 75 const LegalizerInfo &getLegalizerInfo() const { return LI; } 76 const TargetLowering &getTargetLowering() const { return TLI; } 77 78 LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, 79 MachineIRBuilder &B); 80 LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, 81 GISelChangeObserver &Observer, MachineIRBuilder &B); 82 83 /// Replace \p MI by a sequence of legal instructions that can implement the 84 /// same operation. Note that this means \p MI may be deleted, so any iterator 85 /// steps should be performed before calling this function. \p Helper should 86 /// be initialized to the MachineFunction containing \p MI. 87 /// 88 /// Considered as an opaque blob, the legal code will use and define the same 89 /// registers as \p MI. 90 LegalizeResult legalizeInstrStep(MachineInstr &MI, 91 LostDebugLocObserver &LocObserver); 92 93 /// Legalize an instruction by emiting a runtime library call instead. 94 LegalizeResult libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver); 95 96 /// Legalize an instruction by reducing the width of the underlying scalar 97 /// type. 98 LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 99 100 /// Legalize an instruction by performing the operation on a wider scalar type 101 /// (for example a 16-bit addition can be safely performed at 32-bits 102 /// precision, ignoring the unused bits). 103 LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 104 105 /// Legalize an instruction by replacing the value type 106 LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 107 108 /// Legalize an instruction by splitting it into simpler parts, hopefully 109 /// understood by the target. 110 LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 111 112 /// Legalize a vector instruction by splitting into multiple components, each 113 /// acting on the same scalar type as the original but with fewer elements. 114 LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, 115 LLT NarrowTy); 116 117 /// Legalize a vector instruction by increasing the number of vector elements 118 /// involved and ignoring the added elements later. 119 LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, 120 LLT MoreTy); 121 122 /// Cast the given value to an LLT::scalar with an equivalent size. Returns 123 /// the register to use if an instruction was inserted. Returns the original 124 /// register if no coercion was necessary. 125 // 126 // This may also fail and return Register() if there is no legal way to cast. 127 Register coerceToScalar(Register Val); 128 129 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 130 /// Use by extending the operand's type to \p WideTy using the specified \p 131 /// ExtOpcode for the extension instruction, and replacing the vreg of the 132 /// operand in place. 133 void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, 134 unsigned ExtOpcode); 135 136 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 137 /// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and 138 /// replacing the vreg of the operand in place. 139 void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx); 140 141 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 142 /// Def by extending the operand's type to \p WideTy and truncating it back 143 /// with the \p TruncOpcode, and replacing the vreg of the operand in place. 144 void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0, 145 unsigned TruncOpcode = TargetOpcode::G_TRUNC); 146 147 // Legalize a single operand \p OpIdx of the machine instruction \p MI as a 148 // Def by truncating the operand's type to \p NarrowTy, replacing in place and 149 // extending back with \p ExtOpcode. 150 void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, 151 unsigned ExtOpcode); 152 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 153 /// Def by performing it with additional vector elements and extracting the 154 /// result elements, and replacing the vreg of the operand in place. 155 void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 156 157 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 158 /// Use by producing a vector with undefined high elements, extracting the 159 /// original vector type, and replacing the vreg of the operand in place. 160 void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 161 162 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 163 /// use by inserting a G_BITCAST to \p CastTy 164 void bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 165 166 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 167 /// def by inserting a G_BITCAST from \p CastTy 168 void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 169 170 private: 171 LegalizeResult 172 widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 173 LegalizeResult 174 widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 175 LegalizeResult 176 widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 177 LegalizeResult 178 widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 179 LegalizeResult widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx, 180 LLT WideTy); 181 LegalizeResult widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, 182 LLT WideTy); 183 LegalizeResult widenScalarMulo(MachineInstr &MI, unsigned TypeIdx, 184 LLT WideTy); 185 186 /// Helper function to split a wide generic register into bitwise blocks with 187 /// the given Type (which implies the number of blocks needed). The generic 188 /// registers created are appended to Ops, starting at bit 0 of Reg. 189 void extractParts(Register Reg, LLT Ty, int NumParts, 190 SmallVectorImpl<Register> &VRegs); 191 192 /// Version which handles irregular splits. 193 bool extractParts(Register Reg, LLT RegTy, LLT MainTy, 194 LLT &LeftoverTy, 195 SmallVectorImpl<Register> &VRegs, 196 SmallVectorImpl<Register> &LeftoverVRegs); 197 198 /// Version which handles irregular sub-vector splits. 199 void extractVectorParts(Register Reg, unsigned NumElst, 200 SmallVectorImpl<Register> &VRegs); 201 202 /// Helper function to build a wide generic register \p DstReg of type \p 203 /// RegTy from smaller parts. This will produce a G_MERGE_VALUES, 204 /// G_BUILD_VECTOR, G_CONCAT_VECTORS, or sequence of G_INSERT as appropriate 205 /// for the types. 206 /// 207 /// \p PartRegs must be registers of type \p PartTy. 208 /// 209 /// If \p ResultTy does not evenly break into \p PartTy sized pieces, the 210 /// remainder must be specified with \p LeftoverRegs of type \p LeftoverTy. 211 void insertParts(Register DstReg, LLT ResultTy, 212 LLT PartTy, ArrayRef<Register> PartRegs, 213 LLT LeftoverTy = LLT(), ArrayRef<Register> LeftoverRegs = {}); 214 215 /// Merge \p PartRegs with different types into \p DstReg. 216 void mergeMixedSubvectors(Register DstReg, ArrayRef<Register> PartRegs); 217 218 void appendVectorElts(SmallVectorImpl<Register> &Elts, Register Reg); 219 220 /// Unmerge \p SrcReg into smaller sized values, and append them to \p 221 /// Parts. The elements of \p Parts will be the greatest common divisor type 222 /// of \p DstTy, \p NarrowTy and the type of \p SrcReg. This will compute and 223 /// return the GCD type. 224 LLT extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy, 225 LLT NarrowTy, Register SrcReg); 226 227 /// Unmerge \p SrcReg into \p GCDTy typed registers. This will append all of 228 /// the unpacked registers to \p Parts. This version is if the common unmerge 229 /// type is already known. 230 void extractGCDType(SmallVectorImpl<Register> &Parts, LLT GCDTy, 231 Register SrcReg); 232 233 /// Produce a merge of values in \p VRegs to define \p DstReg. Perform a merge 234 /// from the least common multiple type, and convert as appropriate to \p 235 /// DstReg. 236 /// 237 /// \p VRegs should each have type \p GCDTy. This type should be greatest 238 /// common divisor type of \p DstReg, \p NarrowTy, and an undetermined source 239 /// type. 240 /// 241 /// \p NarrowTy is the desired result merge source type. If the source value 242 /// needs to be widened to evenly cover \p DstReg, inserts high bits 243 /// corresponding to the extension opcode \p PadStrategy. 244 /// 245 /// \p VRegs will be cleared, and the the result \p NarrowTy register pieces 246 /// will replace it. Returns The complete LCMTy that \p VRegs will cover when 247 /// merged. 248 LLT buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, 249 SmallVectorImpl<Register> &VRegs, 250 unsigned PadStrategy = TargetOpcode::G_ANYEXT); 251 252 /// Merge the values in \p RemergeRegs to an \p LCMTy typed value. Extract the 253 /// low bits into \p DstReg. This is intended to use the outputs from 254 /// buildLCMMergePieces after processing. 255 void buildWidenedRemergeToDst(Register DstReg, LLT LCMTy, 256 ArrayRef<Register> RemergeRegs); 257 258 /// Perform generic multiplication of values held in multiple registers. 259 /// Generated instructions use only types NarrowTy and i1. 260 /// Destination can be same or two times size of the source. 261 void multiplyRegisters(SmallVectorImpl<Register> &DstRegs, 262 ArrayRef<Register> Src1Regs, 263 ArrayRef<Register> Src2Regs, LLT NarrowTy); 264 265 void changeOpcode(MachineInstr &MI, unsigned NewOpcode); 266 267 LegalizeResult tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg, 268 LLT SrcTy, LLT NarrowTy, 269 unsigned ScalarOpc); 270 271 // Memcpy family legalization helpers. 272 LegalizeResult lowerMemset(MachineInstr &MI, Register Dst, Register Val, 273 uint64_t KnownLen, Align Alignment, 274 bool IsVolatile); 275 LegalizeResult lowerMemcpyInline(MachineInstr &MI, Register Dst, Register Src, 276 uint64_t KnownLen, Align DstAlign, 277 Align SrcAlign, bool IsVolatile); 278 LegalizeResult lowerMemcpy(MachineInstr &MI, Register Dst, Register Src, 279 uint64_t KnownLen, uint64_t Limit, Align DstAlign, 280 Align SrcAlign, bool IsVolatile); 281 LegalizeResult lowerMemmove(MachineInstr &MI, Register Dst, Register Src, 282 uint64_t KnownLen, Align DstAlign, Align SrcAlign, 283 bool IsVolatile); 284 285 public: 286 /// Return the alignment to use for a stack temporary object with the given 287 /// type. 288 Align getStackTemporaryAlignment(LLT Type, Align MinAlign = Align()) const; 289 290 /// Create a stack temporary based on the size in bytes and the alignment 291 MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, 292 MachinePointerInfo &PtrInfo); 293 294 /// Get a pointer to vector element \p Index located in memory for a vector of 295 /// type \p VecTy starting at a base address of \p VecPtr. If \p Index is out 296 /// of bounds the returned pointer is unspecified, but will be within the 297 /// vector bounds. 298 Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index); 299 300 /// Handles most opcodes. Split \p MI into same instruction on sub-vectors or 301 /// scalars with \p NumElts elements (1 for scalar). Supports uneven splits: 302 /// there can be leftover sub-vector with fewer then \p NumElts or a leftover 303 /// scalar. To avoid this use moreElements first and set MI number of elements 304 /// to multiple of \p NumElts. Non-vector operands that should be used on all 305 /// sub-instructions without split are listed in \p NonVecOpIndices. 306 LegalizeResult fewerElementsVectorMultiEltType( 307 GenericMachineInstr &MI, unsigned NumElts, 308 std::initializer_list<unsigned> NonVecOpIndices = {}); 309 310 LegalizeResult fewerElementsVectorPhi(GenericMachineInstr &MI, 311 unsigned NumElts); 312 313 LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 314 LLT MoreTy); 315 LegalizeResult moreElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, 316 LLT MoreTy); 317 318 LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI, 319 unsigned TypeIdx, 320 LLT NarrowTy); 321 LegalizeResult fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx, 322 LLT NarrowTy); 323 LegalizeResult fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI, 324 unsigned TypeIdx, 325 LLT NarrowTy); 326 327 LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx, 328 LLT NarrowTy); 329 330 LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, 331 LLT HalfTy, LLT ShiftAmtTy); 332 333 LegalizeResult fewerElementsVectorReductions(MachineInstr &MI, 334 unsigned TypeIdx, LLT NarrowTy); 335 336 LegalizeResult fewerElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, 337 LLT NarrowTy); 338 339 LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 340 LegalizeResult narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx, 341 LLT NarrowTy); 342 LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty); 343 LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 344 LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 345 LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 346 347 LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 348 LegalizeResult narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 349 LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 350 LegalizeResult narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 351 LegalizeResult narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 352 LegalizeResult narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 353 354 /// Perform Bitcast legalize action on G_EXTRACT_VECTOR_ELT. 355 LegalizeResult bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, 356 LLT CastTy); 357 358 /// Perform Bitcast legalize action on G_INSERT_VECTOR_ELT. 359 LegalizeResult bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, 360 LLT CastTy); 361 362 LegalizeResult lowerBitcast(MachineInstr &MI); 363 LegalizeResult lowerLoad(GAnyLoad &MI); 364 LegalizeResult lowerStore(GStore &MI); 365 LegalizeResult lowerBitCount(MachineInstr &MI); 366 LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI); 367 LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI); 368 LegalizeResult lowerFunnelShift(MachineInstr &MI); 369 LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI); 370 LegalizeResult lowerRotate(MachineInstr &MI); 371 372 LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI); 373 LegalizeResult lowerUITOFP(MachineInstr &MI); 374 LegalizeResult lowerSITOFP(MachineInstr &MI); 375 LegalizeResult lowerFPTOUI(MachineInstr &MI); 376 LegalizeResult lowerFPTOSI(MachineInstr &MI); 377 378 LegalizeResult lowerFPTRUNC_F64_TO_F16(MachineInstr &MI); 379 LegalizeResult lowerFPTRUNC(MachineInstr &MI); 380 LegalizeResult lowerFPOWI(MachineInstr &MI); 381 382 LegalizeResult lowerISFPCLASS(MachineInstr &MI); 383 384 LegalizeResult lowerMinMax(MachineInstr &MI); 385 LegalizeResult lowerFCopySign(MachineInstr &MI); 386 LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); 387 LegalizeResult lowerFMad(MachineInstr &MI); 388 LegalizeResult lowerIntrinsicRound(MachineInstr &MI); 389 LegalizeResult lowerFFloor(MachineInstr &MI); 390 LegalizeResult lowerMergeValues(MachineInstr &MI); 391 LegalizeResult lowerUnmergeValues(MachineInstr &MI); 392 LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI); 393 LegalizeResult lowerShuffleVector(MachineInstr &MI); 394 LegalizeResult lowerDynStackAlloc(MachineInstr &MI); 395 LegalizeResult lowerExtract(MachineInstr &MI); 396 LegalizeResult lowerInsert(MachineInstr &MI); 397 LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI); 398 LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI); 399 LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI); 400 LegalizeResult lowerShlSat(MachineInstr &MI); 401 LegalizeResult lowerBswap(MachineInstr &MI); 402 LegalizeResult lowerBitreverse(MachineInstr &MI); 403 LegalizeResult lowerReadWriteRegister(MachineInstr &MI); 404 LegalizeResult lowerSMULH_UMULH(MachineInstr &MI); 405 LegalizeResult lowerSelect(MachineInstr &MI); 406 LegalizeResult lowerDIVREM(MachineInstr &MI); 407 LegalizeResult lowerAbsToAddXor(MachineInstr &MI); 408 LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI); 409 LegalizeResult lowerVectorReduction(MachineInstr &MI); 410 LegalizeResult lowerMemcpyInline(MachineInstr &MI); 411 LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0); 412 }; 413 414 /// Helper function that creates a libcall to the given \p Name using the given 415 /// calling convention \p CC. 416 LegalizerHelper::LegalizeResult 417 createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, 418 const CallLowering::ArgInfo &Result, 419 ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC); 420 421 /// Helper function that creates the given libcall. 422 LegalizerHelper::LegalizeResult 423 createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, 424 const CallLowering::ArgInfo &Result, 425 ArrayRef<CallLowering::ArgInfo> Args); 426 427 /// Create a libcall to memcpy et al. 428 LegalizerHelper::LegalizeResult 429 createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 430 MachineInstr &MI, LostDebugLocObserver &LocObserver); 431 432 } // End namespace llvm. 433 434 #endif 435