1 //== llvm/CodeGen/GlobalISel/LegalizerHelper.h ---------------- -*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file A pass to convert the target-illegal operations created by IR -> MIR 10 /// translation into ones the target expects to be able to select. This may 11 /// occur in multiple phases, for example G_ADD <2 x i8> -> G_ADD <2 x i16> -> 12 /// G_ADD <4 x i16>. 13 /// 14 /// The LegalizerHelper class is where most of the work happens, and is 15 /// designed to be callable from other passes that find themselves with an 16 /// illegal instruction. 17 // 18 //===----------------------------------------------------------------------===// 19 20 #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H 21 #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H 22 23 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 24 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" 25 #include "llvm/CodeGen/RuntimeLibcalls.h" 26 #include "llvm/CodeGen/TargetOpcodes.h" 27 28 namespace llvm { 29 // Forward declarations. 30 class APInt; 31 class GAnyLoad; 32 class GLoadStore; 33 class GStore; 34 class GenericMachineInstr; 35 class MachineFunction; 36 class MachineIRBuilder; 37 class MachineInstr; 38 class MachineInstrBuilder; 39 struct MachinePointerInfo; 40 template <typename T> class SmallVectorImpl; 41 class LegalizerInfo; 42 class MachineRegisterInfo; 43 class GISelChangeObserver; 44 class LostDebugLocObserver; 45 class TargetLowering; 46 47 class LegalizerHelper { 48 public: 49 /// Expose MIRBuilder so clients can set their own RecordInsertInstruction 50 /// functions 51 MachineIRBuilder &MIRBuilder; 52 53 /// To keep track of changes made by the LegalizerHelper. 54 GISelChangeObserver &Observer; 55 56 private: 57 MachineRegisterInfo &MRI; 58 const LegalizerInfo &LI; 59 const TargetLowering &TLI; 60 GISelKnownBits *KB; 61 62 public: 63 enum LegalizeResult { 64 /// Instruction was already legal and no change was made to the 65 /// MachineFunction. 66 AlreadyLegal, 67 68 /// Instruction has been legalized and the MachineFunction changed. 69 Legalized, 70 71 /// Some kind of error has occurred and we could not legalize this 72 /// instruction. 73 UnableToLegalize, 74 }; 75 76 /// Expose LegalizerInfo so the clients can re-use. 77 const LegalizerInfo &getLegalizerInfo() const { return LI; } 78 const TargetLowering &getTargetLowering() const { return TLI; } 79 GISelKnownBits *getKnownBits() const { return KB; } 80 81 LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, 82 MachineIRBuilder &B); 83 LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, 84 GISelChangeObserver &Observer, MachineIRBuilder &B, 85 GISelKnownBits *KB = nullptr); 86 87 /// Replace \p MI by a sequence of legal instructions that can implement the 88 /// same operation. Note that this means \p MI may be deleted, so any iterator 89 /// steps should be performed before calling this function. \p Helper should 90 /// be initialized to the MachineFunction containing \p MI. 91 /// 92 /// Considered as an opaque blob, the legal code will use and define the same 93 /// registers as \p MI. 94 LegalizeResult legalizeInstrStep(MachineInstr &MI, 95 LostDebugLocObserver &LocObserver); 96 97 /// Legalize an instruction by emiting a runtime library call instead. 98 LegalizeResult libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver); 99 100 /// Legalize an instruction by reducing the width of the underlying scalar 101 /// type. 102 LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 103 104 /// Legalize an instruction by performing the operation on a wider scalar type 105 /// (for example a 16-bit addition can be safely performed at 32-bits 106 /// precision, ignoring the unused bits). 107 LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 108 109 /// Legalize an instruction by replacing the value type 110 LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 111 112 /// Legalize an instruction by splitting it into simpler parts, hopefully 113 /// understood by the target. 114 LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 115 116 /// Legalize a vector instruction by splitting into multiple components, each 117 /// acting on the same scalar type as the original but with fewer elements. 118 LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, 119 LLT NarrowTy); 120 121 /// Legalize a vector instruction by increasing the number of vector elements 122 /// involved and ignoring the added elements later. 123 LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, 124 LLT MoreTy); 125 126 /// Cast the given value to an LLT::scalar with an equivalent size. Returns 127 /// the register to use if an instruction was inserted. Returns the original 128 /// register if no coercion was necessary. 129 // 130 // This may also fail and return Register() if there is no legal way to cast. 131 Register coerceToScalar(Register Val); 132 133 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 134 /// Use by extending the operand's type to \p WideTy using the specified \p 135 /// ExtOpcode for the extension instruction, and replacing the vreg of the 136 /// operand in place. 137 void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, 138 unsigned ExtOpcode); 139 140 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 141 /// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and 142 /// replacing the vreg of the operand in place. 143 void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx); 144 145 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 146 /// Def by extending the operand's type to \p WideTy and truncating it back 147 /// with the \p TruncOpcode, and replacing the vreg of the operand in place. 148 void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0, 149 unsigned TruncOpcode = TargetOpcode::G_TRUNC); 150 151 // Legalize a single operand \p OpIdx of the machine instruction \p MI as a 152 // Def by truncating the operand's type to \p NarrowTy, replacing in place and 153 // extending back with \p ExtOpcode. 154 void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, 155 unsigned ExtOpcode); 156 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 157 /// Def by performing it with additional vector elements and extracting the 158 /// result elements, and replacing the vreg of the operand in place. 159 void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 160 161 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 162 /// Use by producing a vector with undefined high elements, extracting the 163 /// original vector type, and replacing the vreg of the operand in place. 164 void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 165 166 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 167 /// use by inserting a G_BITCAST to \p CastTy 168 void bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 169 170 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 171 /// def by inserting a G_BITCAST from \p CastTy 172 void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 173 174 private: 175 LegalizeResult 176 widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 177 LegalizeResult 178 widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 179 LegalizeResult 180 widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 181 LegalizeResult 182 widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 183 LegalizeResult widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx, 184 LLT WideTy); 185 LegalizeResult widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, 186 LLT WideTy); 187 LegalizeResult widenScalarMulo(MachineInstr &MI, unsigned TypeIdx, 188 LLT WideTy); 189 190 /// Helper function to build a wide generic register \p DstReg of type \p 191 /// RegTy from smaller parts. This will produce a G_MERGE_VALUES, 192 /// G_BUILD_VECTOR, G_CONCAT_VECTORS, or sequence of G_INSERT as appropriate 193 /// for the types. 194 /// 195 /// \p PartRegs must be registers of type \p PartTy. 196 /// 197 /// If \p ResultTy does not evenly break into \p PartTy sized pieces, the 198 /// remainder must be specified with \p LeftoverRegs of type \p LeftoverTy. 199 void insertParts(Register DstReg, LLT ResultTy, 200 LLT PartTy, ArrayRef<Register> PartRegs, 201 LLT LeftoverTy = LLT(), ArrayRef<Register> LeftoverRegs = {}); 202 203 /// Merge \p PartRegs with different types into \p DstReg. 204 void mergeMixedSubvectors(Register DstReg, ArrayRef<Register> PartRegs); 205 206 void appendVectorElts(SmallVectorImpl<Register> &Elts, Register Reg); 207 208 /// Unmerge \p SrcReg into smaller sized values, and append them to \p 209 /// Parts. The elements of \p Parts will be the greatest common divisor type 210 /// of \p DstTy, \p NarrowTy and the type of \p SrcReg. This will compute and 211 /// return the GCD type. 212 LLT extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy, 213 LLT NarrowTy, Register SrcReg); 214 215 /// Unmerge \p SrcReg into \p GCDTy typed registers. This will append all of 216 /// the unpacked registers to \p Parts. This version is if the common unmerge 217 /// type is already known. 218 void extractGCDType(SmallVectorImpl<Register> &Parts, LLT GCDTy, 219 Register SrcReg); 220 221 /// Produce a merge of values in \p VRegs to define \p DstReg. Perform a merge 222 /// from the least common multiple type, and convert as appropriate to \p 223 /// DstReg. 224 /// 225 /// \p VRegs should each have type \p GCDTy. This type should be greatest 226 /// common divisor type of \p DstReg, \p NarrowTy, and an undetermined source 227 /// type. 228 /// 229 /// \p NarrowTy is the desired result merge source type. If the source value 230 /// needs to be widened to evenly cover \p DstReg, inserts high bits 231 /// corresponding to the extension opcode \p PadStrategy. 232 /// 233 /// \p VRegs will be cleared, and the result \p NarrowTy register pieces 234 /// will replace it. Returns The complete LCMTy that \p VRegs will cover when 235 /// merged. 236 LLT buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, 237 SmallVectorImpl<Register> &VRegs, 238 unsigned PadStrategy = TargetOpcode::G_ANYEXT); 239 240 /// Merge the values in \p RemergeRegs to an \p LCMTy typed value. Extract the 241 /// low bits into \p DstReg. This is intended to use the outputs from 242 /// buildLCMMergePieces after processing. 243 void buildWidenedRemergeToDst(Register DstReg, LLT LCMTy, 244 ArrayRef<Register> RemergeRegs); 245 246 /// Perform generic multiplication of values held in multiple registers. 247 /// Generated instructions use only types NarrowTy and i1. 248 /// Destination can be same or two times size of the source. 249 void multiplyRegisters(SmallVectorImpl<Register> &DstRegs, 250 ArrayRef<Register> Src1Regs, 251 ArrayRef<Register> Src2Regs, LLT NarrowTy); 252 253 void changeOpcode(MachineInstr &MI, unsigned NewOpcode); 254 255 LegalizeResult tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg, 256 LLT SrcTy, LLT NarrowTy, 257 unsigned ScalarOpc); 258 259 // Memcpy family legalization helpers. 260 LegalizeResult lowerMemset(MachineInstr &MI, Register Dst, Register Val, 261 uint64_t KnownLen, Align Alignment, 262 bool IsVolatile); 263 LegalizeResult lowerMemcpyInline(MachineInstr &MI, Register Dst, Register Src, 264 uint64_t KnownLen, Align DstAlign, 265 Align SrcAlign, bool IsVolatile); 266 LegalizeResult lowerMemcpy(MachineInstr &MI, Register Dst, Register Src, 267 uint64_t KnownLen, uint64_t Limit, Align DstAlign, 268 Align SrcAlign, bool IsVolatile); 269 LegalizeResult lowerMemmove(MachineInstr &MI, Register Dst, Register Src, 270 uint64_t KnownLen, Align DstAlign, Align SrcAlign, 271 bool IsVolatile); 272 273 // Implements floating-point environment read/write via library function call. 274 LegalizeResult createGetStateLibcall(MachineIRBuilder &MIRBuilder, 275 MachineInstr &MI, 276 LostDebugLocObserver &LocObserver); 277 LegalizeResult createSetStateLibcall(MachineIRBuilder &MIRBuilder, 278 MachineInstr &MI, 279 LostDebugLocObserver &LocObserver); 280 LegalizeResult createResetStateLibcall(MachineIRBuilder &MIRBuilder, 281 MachineInstr &MI, 282 LostDebugLocObserver &LocObserver); 283 284 public: 285 /// Return the alignment to use for a stack temporary object with the given 286 /// type. 287 Align getStackTemporaryAlignment(LLT Type, Align MinAlign = Align()) const; 288 289 /// Create a stack temporary based on the size in bytes and the alignment 290 MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, 291 MachinePointerInfo &PtrInfo); 292 293 /// Get a pointer to vector element \p Index located in memory for a vector of 294 /// type \p VecTy starting at a base address of \p VecPtr. If \p Index is out 295 /// of bounds the returned pointer is unspecified, but will be within the 296 /// vector bounds. 297 Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index); 298 299 /// Handles most opcodes. Split \p MI into same instruction on sub-vectors or 300 /// scalars with \p NumElts elements (1 for scalar). Supports uneven splits: 301 /// there can be leftover sub-vector with fewer then \p NumElts or a leftover 302 /// scalar. To avoid this use moreElements first and set MI number of elements 303 /// to multiple of \p NumElts. Non-vector operands that should be used on all 304 /// sub-instructions without split are listed in \p NonVecOpIndices. 305 LegalizeResult fewerElementsVectorMultiEltType( 306 GenericMachineInstr &MI, unsigned NumElts, 307 std::initializer_list<unsigned> NonVecOpIndices = {}); 308 309 LegalizeResult fewerElementsVectorPhi(GenericMachineInstr &MI, 310 unsigned NumElts); 311 312 LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 313 LLT MoreTy); 314 LegalizeResult moreElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, 315 LLT MoreTy); 316 317 LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI, 318 unsigned TypeIdx, 319 LLT NarrowTy); 320 LegalizeResult fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx, 321 LLT NarrowTy); 322 LegalizeResult fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI, 323 unsigned TypeIdx, 324 LLT NarrowTy); 325 326 /// Equalize source and destination vector sizes of G_SHUFFLE_VECTOR. 327 LegalizeResult equalizeVectorShuffleLengths(MachineInstr &MI); 328 329 LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx, 330 LLT NarrowTy); 331 332 LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, 333 LLT HalfTy, LLT ShiftAmtTy); 334 335 LegalizeResult fewerElementsVectorReductions(MachineInstr &MI, 336 unsigned TypeIdx, LLT NarrowTy); 337 LegalizeResult fewerElementsVectorSeqReductions(MachineInstr &MI, 338 unsigned TypeIdx, 339 LLT NarrowTy); 340 341 LegalizeResult fewerElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, 342 LLT NarrowTy); 343 344 LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 345 LegalizeResult narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx, 346 LLT NarrowTy); 347 LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty); 348 LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 349 LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 350 LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 351 352 LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 353 LegalizeResult narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 354 LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 355 LegalizeResult narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 356 LegalizeResult narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 357 LegalizeResult narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 358 LegalizeResult narrowScalarFLDEXP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 359 360 /// Perform Bitcast legalize action on G_EXTRACT_VECTOR_ELT. 361 LegalizeResult bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, 362 LLT CastTy); 363 364 /// Perform Bitcast legalize action on G_INSERT_VECTOR_ELT. 365 LegalizeResult bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, 366 LLT CastTy); 367 368 LegalizeResult lowerFConstant(MachineInstr &MI); 369 LegalizeResult lowerBitcast(MachineInstr &MI); 370 LegalizeResult lowerLoad(GAnyLoad &MI); 371 LegalizeResult lowerStore(GStore &MI); 372 LegalizeResult lowerBitCount(MachineInstr &MI); 373 LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI); 374 LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI); 375 LegalizeResult lowerFunnelShift(MachineInstr &MI); 376 LegalizeResult lowerEXT(MachineInstr &MI); 377 LegalizeResult lowerTRUNC(MachineInstr &MI); 378 LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI); 379 LegalizeResult lowerRotate(MachineInstr &MI); 380 381 LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI); 382 LegalizeResult lowerUITOFP(MachineInstr &MI); 383 LegalizeResult lowerSITOFP(MachineInstr &MI); 384 LegalizeResult lowerFPTOUI(MachineInstr &MI); 385 LegalizeResult lowerFPTOSI(MachineInstr &MI); 386 387 LegalizeResult lowerFPTRUNC_F64_TO_F16(MachineInstr &MI); 388 LegalizeResult lowerFPTRUNC(MachineInstr &MI); 389 LegalizeResult lowerFPOWI(MachineInstr &MI); 390 391 LegalizeResult lowerISFPCLASS(MachineInstr &MI); 392 393 LegalizeResult lowerMinMax(MachineInstr &MI); 394 LegalizeResult lowerFCopySign(MachineInstr &MI); 395 LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); 396 LegalizeResult lowerFMad(MachineInstr &MI); 397 LegalizeResult lowerIntrinsicRound(MachineInstr &MI); 398 LegalizeResult lowerFFloor(MachineInstr &MI); 399 LegalizeResult lowerMergeValues(MachineInstr &MI); 400 LegalizeResult lowerUnmergeValues(MachineInstr &MI); 401 LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI); 402 LegalizeResult lowerShuffleVector(MachineInstr &MI); 403 Register getDynStackAllocTargetPtr(Register SPReg, Register AllocSize, 404 Align Alignment, LLT PtrTy); 405 LegalizeResult lowerDynStackAlloc(MachineInstr &MI); 406 LegalizeResult lowerStackSave(MachineInstr &MI); 407 LegalizeResult lowerStackRestore(MachineInstr &MI); 408 LegalizeResult lowerExtract(MachineInstr &MI); 409 LegalizeResult lowerInsert(MachineInstr &MI); 410 LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI); 411 LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI); 412 LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI); 413 LegalizeResult lowerShlSat(MachineInstr &MI); 414 LegalizeResult lowerBswap(MachineInstr &MI); 415 LegalizeResult lowerBitreverse(MachineInstr &MI); 416 LegalizeResult lowerReadWriteRegister(MachineInstr &MI); 417 LegalizeResult lowerSMULH_UMULH(MachineInstr &MI); 418 LegalizeResult lowerSelect(MachineInstr &MI); 419 LegalizeResult lowerDIVREM(MachineInstr &MI); 420 LegalizeResult lowerAbsToAddXor(MachineInstr &MI); 421 LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI); 422 LegalizeResult lowerVectorReduction(MachineInstr &MI); 423 LegalizeResult lowerMemcpyInline(MachineInstr &MI); 424 LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0); 425 LegalizeResult lowerVAArg(MachineInstr &MI); 426 }; 427 428 /// Helper function that creates a libcall to the given \p Name using the given 429 /// calling convention \p CC. 430 LegalizerHelper::LegalizeResult 431 createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, 432 const CallLowering::ArgInfo &Result, 433 ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC, 434 LostDebugLocObserver &LocObserver, MachineInstr *MI = nullptr); 435 436 /// Helper function that creates the given libcall. 437 LegalizerHelper::LegalizeResult 438 createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, 439 const CallLowering::ArgInfo &Result, 440 ArrayRef<CallLowering::ArgInfo> Args, 441 LostDebugLocObserver &LocObserver, MachineInstr *MI = nullptr); 442 443 /// Create a libcall to memcpy et al. 444 LegalizerHelper::LegalizeResult 445 createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 446 MachineInstr &MI, LostDebugLocObserver &LocObserver); 447 448 449 } // End namespace llvm. 450 451 #endif 452