1 //== llvm/CodeGen/GlobalISel/LegalizerHelper.h ---------------- -*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file A pass to convert the target-illegal operations created by IR -> MIR 10 /// translation into ones the target expects to be able to select. This may 11 /// occur in multiple phases, for example G_ADD <2 x i8> -> G_ADD <2 x i16> -> 12 /// G_ADD <4 x i16>. 13 /// 14 /// The LegalizerHelper class is where most of the work happens, and is 15 /// designed to be callable from other passes that find themselves with an 16 /// illegal instruction. 17 // 18 //===----------------------------------------------------------------------===// 19 20 #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H 21 #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H 22 23 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 24 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 25 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 26 #include "llvm/CodeGen/LowLevelType.h" 27 #include "llvm/CodeGen/MachineFunctionPass.h" 28 #include "llvm/CodeGen/RuntimeLibcalls.h" 29 30 namespace llvm { 31 // Forward declarations. 32 class LegalizerInfo; 33 class MachineRegisterInfo; 34 class GISelChangeObserver; 35 class LostDebugLocObserver; 36 class TargetLowering; 37 38 class LegalizerHelper { 39 public: 40 /// Expose MIRBuilder so clients can set their own RecordInsertInstruction 41 /// functions 42 MachineIRBuilder &MIRBuilder; 43 44 /// To keep track of changes made by the LegalizerHelper. 45 GISelChangeObserver &Observer; 46 47 private: 48 MachineRegisterInfo &MRI; 49 const LegalizerInfo &LI; 50 const TargetLowering &TLI; 51 52 public: 53 enum LegalizeResult { 54 /// Instruction was already legal and no change was made to the 55 /// MachineFunction. 56 AlreadyLegal, 57 58 /// Instruction has been legalized and the MachineFunction changed. 59 Legalized, 60 61 /// Some kind of error has occurred and we could not legalize this 62 /// instruction. 63 UnableToLegalize, 64 }; 65 66 /// Expose LegalizerInfo so the clients can re-use. 67 const LegalizerInfo &getLegalizerInfo() const { return LI; } 68 const TargetLowering &getTargetLowering() const { return TLI; } 69 70 LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, 71 MachineIRBuilder &B); 72 LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, 73 GISelChangeObserver &Observer, MachineIRBuilder &B); 74 75 /// Replace \p MI by a sequence of legal instructions that can implement the 76 /// same operation. Note that this means \p MI may be deleted, so any iterator 77 /// steps should be performed before calling this function. \p Helper should 78 /// be initialized to the MachineFunction containing \p MI. 79 /// 80 /// Considered as an opaque blob, the legal code will use and define the same 81 /// registers as \p MI. 82 LegalizeResult legalizeInstrStep(MachineInstr &MI, 83 LostDebugLocObserver &LocObserver); 84 85 /// Legalize an instruction by emiting a runtime library call instead. 86 LegalizeResult libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver); 87 88 /// Legalize an instruction by reducing the width of the underlying scalar 89 /// type. 90 LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 91 92 /// Legalize an instruction by performing the operation on a wider scalar type 93 /// (for example a 16-bit addition can be safely performed at 32-bits 94 /// precision, ignoring the unused bits). 95 LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 96 97 /// Legalize an instruction by replacing the value type 98 LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 99 100 /// Legalize an instruction by splitting it into simpler parts, hopefully 101 /// understood by the target. 102 LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 103 104 /// Legalize a vector instruction by splitting into multiple components, each 105 /// acting on the same scalar type as the original but with fewer elements. 106 LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, 107 LLT NarrowTy); 108 109 /// Legalize a vector instruction by increasing the number of vector elements 110 /// involved and ignoring the added elements later. 111 LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, 112 LLT MoreTy); 113 114 /// Cast the given value to an LLT::scalar with an equivalent size. Returns 115 /// the register to use if an instruction was inserted. Returns the original 116 /// register if no coercion was necessary. 117 // 118 // This may also fail and return Register() if there is no legal way to cast. 119 Register coerceToScalar(Register Val); 120 121 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 122 /// Use by extending the operand's type to \p WideTy using the specified \p 123 /// ExtOpcode for the extension instruction, and replacing the vreg of the 124 /// operand in place. 125 void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, 126 unsigned ExtOpcode); 127 128 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 129 /// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and 130 /// replacing the vreg of the operand in place. 131 void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx); 132 133 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 134 /// Def by extending the operand's type to \p WideTy and truncating it back 135 /// with the \p TruncOpcode, and replacing the vreg of the operand in place. 136 void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0, 137 unsigned TruncOpcode = TargetOpcode::G_TRUNC); 138 139 // Legalize a single operand \p OpIdx of the machine instruction \p MI as a 140 // Def by truncating the operand's type to \p NarrowTy, replacing in place and 141 // extending back with \p ExtOpcode. 142 void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, 143 unsigned ExtOpcode); 144 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 145 /// Def by performing it with additional vector elements and extracting the 146 /// result elements, and replacing the vreg of the operand in place. 147 void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 148 149 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 150 /// Use by producing a vector with undefined high elements, extracting the 151 /// original vector type, and replacing the vreg of the operand in place. 152 void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 153 154 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 155 /// use by inserting a G_BITCAST to \p CastTy 156 void bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 157 158 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 159 /// def by inserting a G_BITCAST from \p CastTy 160 void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 161 162 /// Widen \p OrigReg to \p WideTy by merging to a wider type, padding with 163 /// G_IMPLICIT_DEF, and producing dead results. 164 Register widenWithUnmerge(LLT WideTy, Register OrigReg); 165 166 private: 167 LegalizeResult 168 widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 169 LegalizeResult 170 widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 171 LegalizeResult 172 widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 173 LegalizeResult 174 widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 175 LegalizeResult widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx, 176 LLT WideTy); 177 LegalizeResult widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, 178 LLT WideTy); 179 LegalizeResult widenScalarMulo(MachineInstr &MI, unsigned TypeIdx, 180 LLT WideTy); 181 182 /// Helper function to split a wide generic register into bitwise blocks with 183 /// the given Type (which implies the number of blocks needed). The generic 184 /// registers created are appended to Ops, starting at bit 0 of Reg. 185 void extractParts(Register Reg, LLT Ty, int NumParts, 186 SmallVectorImpl<Register> &VRegs); 187 188 /// Version which handles irregular splits. 189 bool extractParts(Register Reg, LLT RegTy, LLT MainTy, 190 LLT &LeftoverTy, 191 SmallVectorImpl<Register> &VRegs, 192 SmallVectorImpl<Register> &LeftoverVRegs); 193 194 /// Version which handles irregular sub-vector splits. 195 void extractVectorParts(Register Reg, unsigned NumElst, 196 SmallVectorImpl<Register> &VRegs); 197 198 /// Helper function to build a wide generic register \p DstReg of type \p 199 /// RegTy from smaller parts. This will produce a G_MERGE_VALUES, 200 /// G_BUILD_VECTOR, G_CONCAT_VECTORS, or sequence of G_INSERT as appropriate 201 /// for the types. 202 /// 203 /// \p PartRegs must be registers of type \p PartTy. 204 /// 205 /// If \p ResultTy does not evenly break into \p PartTy sized pieces, the 206 /// remainder must be specified with \p LeftoverRegs of type \p LeftoverTy. 207 void insertParts(Register DstReg, LLT ResultTy, 208 LLT PartTy, ArrayRef<Register> PartRegs, 209 LLT LeftoverTy = LLT(), ArrayRef<Register> LeftoverRegs = {}); 210 211 /// Merge \p PartRegs with different types into \p DstReg. 212 void mergeMixedSubvectors(Register DstReg, ArrayRef<Register> PartRegs); 213 214 void appendVectorElts(SmallVectorImpl<Register> &Elts, Register Reg); 215 216 /// Unmerge \p SrcReg into smaller sized values, and append them to \p 217 /// Parts. The elements of \p Parts will be the greatest common divisor type 218 /// of \p DstTy, \p NarrowTy and the type of \p SrcReg. This will compute and 219 /// return the GCD type. 220 LLT extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy, 221 LLT NarrowTy, Register SrcReg); 222 223 /// Unmerge \p SrcReg into \p GCDTy typed registers. This will append all of 224 /// the unpacked registers to \p Parts. This version is if the common unmerge 225 /// type is already known. 226 void extractGCDType(SmallVectorImpl<Register> &Parts, LLT GCDTy, 227 Register SrcReg); 228 229 /// Produce a merge of values in \p VRegs to define \p DstReg. Perform a merge 230 /// from the least common multiple type, and convert as appropriate to \p 231 /// DstReg. 232 /// 233 /// \p VRegs should each have type \p GCDTy. This type should be greatest 234 /// common divisor type of \p DstReg, \p NarrowTy, and an undetermined source 235 /// type. 236 /// 237 /// \p NarrowTy is the desired result merge source type. If the source value 238 /// needs to be widened to evenly cover \p DstReg, inserts high bits 239 /// corresponding to the extension opcode \p PadStrategy. 240 /// 241 /// \p VRegs will be cleared, and the the result \p NarrowTy register pieces 242 /// will replace it. Returns The complete LCMTy that \p VRegs will cover when 243 /// merged. 244 LLT buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, 245 SmallVectorImpl<Register> &VRegs, 246 unsigned PadStrategy = TargetOpcode::G_ANYEXT); 247 248 /// Merge the values in \p RemergeRegs to an \p LCMTy typed value. Extract the 249 /// low bits into \p DstReg. This is intended to use the outputs from 250 /// buildLCMMergePieces after processing. 251 void buildWidenedRemergeToDst(Register DstReg, LLT LCMTy, 252 ArrayRef<Register> RemergeRegs); 253 254 /// Perform generic multiplication of values held in multiple registers. 255 /// Generated instructions use only types NarrowTy and i1. 256 /// Destination can be same or two times size of the source. 257 void multiplyRegisters(SmallVectorImpl<Register> &DstRegs, 258 ArrayRef<Register> Src1Regs, 259 ArrayRef<Register> Src2Regs, LLT NarrowTy); 260 261 void changeOpcode(MachineInstr &MI, unsigned NewOpcode); 262 263 LegalizeResult tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg, 264 LLT SrcTy, LLT NarrowTy, 265 unsigned ScalarOpc); 266 267 // Memcpy family legalization helpers. 268 LegalizeResult lowerMemset(MachineInstr &MI, Register Dst, Register Val, 269 uint64_t KnownLen, Align Alignment, 270 bool IsVolatile); 271 LegalizeResult lowerMemcpyInline(MachineInstr &MI, Register Dst, Register Src, 272 uint64_t KnownLen, Align DstAlign, 273 Align SrcAlign, bool IsVolatile); 274 LegalizeResult lowerMemcpy(MachineInstr &MI, Register Dst, Register Src, 275 uint64_t KnownLen, uint64_t Limit, Align DstAlign, 276 Align SrcAlign, bool IsVolatile); 277 LegalizeResult lowerMemmove(MachineInstr &MI, Register Dst, Register Src, 278 uint64_t KnownLen, Align DstAlign, Align SrcAlign, 279 bool IsVolatile); 280 281 public: 282 /// Return the alignment to use for a stack temporary object with the given 283 /// type. 284 Align getStackTemporaryAlignment(LLT Type, Align MinAlign = Align()) const; 285 286 /// Create a stack temporary based on the size in bytes and the alignment 287 MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, 288 MachinePointerInfo &PtrInfo); 289 290 /// Get a pointer to vector element \p Index located in memory for a vector of 291 /// type \p VecTy starting at a base address of \p VecPtr. If \p Index is out 292 /// of bounds the returned pointer is unspecified, but will be within the 293 /// vector bounds. 294 Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index); 295 296 /// Handles most opcodes. Split \p MI into same instruction on sub-vectors or 297 /// scalars with \p NumElts elements (1 for scalar). Supports uneven splits: 298 /// there can be leftover sub-vector with fewer then \p NumElts or a leftover 299 /// scalar. To avoid this use moreElements first and set MI number of elements 300 /// to multiple of \p NumElts. Non-vector operands that should be used on all 301 /// sub-instructions without split are listed in \p NonVecOpIndices. 302 LegalizeResult fewerElementsVectorMultiEltType( 303 GenericMachineInstr &MI, unsigned NumElts, 304 std::initializer_list<unsigned> NonVecOpIndices = {}); 305 306 LegalizeResult fewerElementsVectorPhi(GenericMachineInstr &MI, 307 unsigned NumElts); 308 309 LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 310 LLT MoreTy); 311 LegalizeResult moreElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, 312 LLT MoreTy); 313 314 LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI, 315 unsigned TypeIdx, 316 LLT NarrowTy); 317 LegalizeResult fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx, 318 LLT NarrowTy); 319 LegalizeResult fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI, 320 unsigned TypeIdx, 321 LLT NarrowTy); 322 323 LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx, 324 LLT NarrowTy); 325 326 LegalizeResult fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx, 327 LLT NarrowTy); 328 329 LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, 330 LLT HalfTy, LLT ShiftAmtTy); 331 332 LegalizeResult fewerElementsVectorReductions(MachineInstr &MI, 333 unsigned TypeIdx, LLT NarrowTy); 334 335 LegalizeResult fewerElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, 336 LLT NarrowTy); 337 338 LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 339 LegalizeResult narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx, 340 LLT NarrowTy); 341 LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty); 342 LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 343 LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 344 LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 345 346 LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 347 LegalizeResult narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 348 LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 349 LegalizeResult narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 350 LegalizeResult narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 351 LegalizeResult narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 352 353 /// Perform Bitcast legalize action on G_EXTRACT_VECTOR_ELT. 354 LegalizeResult bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, 355 LLT CastTy); 356 357 /// Perform Bitcast legalize action on G_INSERT_VECTOR_ELT. 358 LegalizeResult bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, 359 LLT CastTy); 360 361 LegalizeResult lowerBitcast(MachineInstr &MI); 362 LegalizeResult lowerLoad(GAnyLoad &MI); 363 LegalizeResult lowerStore(GStore &MI); 364 LegalizeResult lowerBitCount(MachineInstr &MI); 365 LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI); 366 LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI); 367 LegalizeResult lowerFunnelShift(MachineInstr &MI); 368 LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI); 369 LegalizeResult lowerRotate(MachineInstr &MI); 370 371 LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI); 372 LegalizeResult lowerUITOFP(MachineInstr &MI); 373 LegalizeResult lowerSITOFP(MachineInstr &MI); 374 LegalizeResult lowerFPTOUI(MachineInstr &MI); 375 LegalizeResult lowerFPTOSI(MachineInstr &MI); 376 377 LegalizeResult lowerFPTRUNC_F64_TO_F16(MachineInstr &MI); 378 LegalizeResult lowerFPTRUNC(MachineInstr &MI); 379 LegalizeResult lowerFPOWI(MachineInstr &MI); 380 381 LegalizeResult lowerMinMax(MachineInstr &MI); 382 LegalizeResult lowerFCopySign(MachineInstr &MI); 383 LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); 384 LegalizeResult lowerFMad(MachineInstr &MI); 385 LegalizeResult lowerIntrinsicRound(MachineInstr &MI); 386 LegalizeResult lowerFFloor(MachineInstr &MI); 387 LegalizeResult lowerMergeValues(MachineInstr &MI); 388 LegalizeResult lowerUnmergeValues(MachineInstr &MI); 389 LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI); 390 LegalizeResult lowerShuffleVector(MachineInstr &MI); 391 LegalizeResult lowerDynStackAlloc(MachineInstr &MI); 392 LegalizeResult lowerExtract(MachineInstr &MI); 393 LegalizeResult lowerInsert(MachineInstr &MI); 394 LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI); 395 LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI); 396 LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI); 397 LegalizeResult lowerShlSat(MachineInstr &MI); 398 LegalizeResult lowerBswap(MachineInstr &MI); 399 LegalizeResult lowerBitreverse(MachineInstr &MI); 400 LegalizeResult lowerReadWriteRegister(MachineInstr &MI); 401 LegalizeResult lowerSMULH_UMULH(MachineInstr &MI); 402 LegalizeResult lowerSelect(MachineInstr &MI); 403 LegalizeResult lowerDIVREM(MachineInstr &MI); 404 LegalizeResult lowerAbsToAddXor(MachineInstr &MI); 405 LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI); 406 LegalizeResult lowerVectorReduction(MachineInstr &MI); 407 LegalizeResult lowerMemcpyInline(MachineInstr &MI); 408 LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0); 409 }; 410 411 /// Helper function that creates a libcall to the given \p Name using the given 412 /// calling convention \p CC. 413 LegalizerHelper::LegalizeResult 414 createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, 415 const CallLowering::ArgInfo &Result, 416 ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC); 417 418 /// Helper function that creates the given libcall. 419 LegalizerHelper::LegalizeResult 420 createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, 421 const CallLowering::ArgInfo &Result, 422 ArrayRef<CallLowering::ArgInfo> Args); 423 424 /// Create a libcall to memcpy et al. 425 LegalizerHelper::LegalizeResult 426 createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 427 MachineInstr &MI, LostDebugLocObserver &LocObserver); 428 429 } // End namespace llvm. 430 431 #endif 432