1 //== llvm/CodeGen/GlobalISel/LegalizerHelper.h ---------------- -*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file A pass to convert the target-illegal operations created by IR -> MIR 10 /// translation into ones the target expects to be able to select. This may 11 /// occur in multiple phases, for example G_ADD <2 x i8> -> G_ADD <2 x i16> -> 12 /// G_ADD <4 x i16>. 13 /// 14 /// The LegalizerHelper class is where most of the work happens, and is 15 /// designed to be callable from other passes that find themselves with an 16 /// illegal instruction. 17 // 18 //===----------------------------------------------------------------------===// 19 20 #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H 21 #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H 22 23 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 24 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 25 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 26 #include "llvm/CodeGen/LowLevelType.h" 27 #include "llvm/CodeGen/MachineFunctionPass.h" 28 #include "llvm/CodeGen/RuntimeLibcalls.h" 29 30 namespace llvm { 31 // Forward declarations. 32 class LegalizerInfo; 33 class Legalizer; 34 class MachineRegisterInfo; 35 class GISelChangeObserver; 36 class LostDebugLocObserver; 37 class TargetLowering; 38 39 class LegalizerHelper { 40 public: 41 /// Expose MIRBuilder so clients can set their own RecordInsertInstruction 42 /// functions 43 MachineIRBuilder &MIRBuilder; 44 45 /// To keep track of changes made by the LegalizerHelper. 46 GISelChangeObserver &Observer; 47 48 private: 49 MachineRegisterInfo &MRI; 50 const LegalizerInfo &LI; 51 const TargetLowering &TLI; 52 53 public: 54 enum LegalizeResult { 55 /// Instruction was already legal and no change was made to the 56 /// MachineFunction. 57 AlreadyLegal, 58 59 /// Instruction has been legalized and the MachineFunction changed. 60 Legalized, 61 62 /// Some kind of error has occurred and we could not legalize this 63 /// instruction. 64 UnableToLegalize, 65 }; 66 67 /// Expose LegalizerInfo so the clients can re-use. 68 const LegalizerInfo &getLegalizerInfo() const { return LI; } 69 const TargetLowering &getTargetLowering() const { return TLI; } 70 71 LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, 72 MachineIRBuilder &B); 73 LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, 74 GISelChangeObserver &Observer, MachineIRBuilder &B); 75 76 /// Replace \p MI by a sequence of legal instructions that can implement the 77 /// same operation. Note that this means \p MI may be deleted, so any iterator 78 /// steps should be performed before calling this function. \p Helper should 79 /// be initialized to the MachineFunction containing \p MI. 80 /// 81 /// Considered as an opaque blob, the legal code will use and define the same 82 /// registers as \p MI. 83 LegalizeResult legalizeInstrStep(MachineInstr &MI, 84 LostDebugLocObserver &LocObserver); 85 86 /// Legalize an instruction by emiting a runtime library call instead. 87 LegalizeResult libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver); 88 89 /// Legalize an instruction by reducing the width of the underlying scalar 90 /// type. 91 LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 92 93 /// Legalize an instruction by performing the operation on a wider scalar type 94 /// (for example a 16-bit addition can be safely performed at 32-bits 95 /// precision, ignoring the unused bits). 96 LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 97 98 /// Legalize an instruction by replacing the value type 99 LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 100 101 /// Legalize an instruction by splitting it into simpler parts, hopefully 102 /// understood by the target. 103 LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 104 105 /// Legalize a vector instruction by splitting into multiple components, each 106 /// acting on the same scalar type as the original but with fewer elements. 107 LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, 108 LLT NarrowTy); 109 110 /// Legalize a vector instruction by increasing the number of vector elements 111 /// involved and ignoring the added elements later. 112 LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, 113 LLT MoreTy); 114 115 /// Cast the given value to an LLT::scalar with an equivalent size. Returns 116 /// the register to use if an instruction was inserted. Returns the original 117 /// register if no coercion was necessary. 118 // 119 // This may also fail and return Register() if there is no legal way to cast. 120 Register coerceToScalar(Register Val); 121 122 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 123 /// Use by extending the operand's type to \p WideTy using the specified \p 124 /// ExtOpcode for the extension instruction, and replacing the vreg of the 125 /// operand in place. 126 void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, 127 unsigned ExtOpcode); 128 129 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 130 /// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and 131 /// replacing the vreg of the operand in place. 132 void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx); 133 134 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 135 /// Def by extending the operand's type to \p WideTy and truncating it back 136 /// with the \p TruncOpcode, and replacing the vreg of the operand in place. 137 void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0, 138 unsigned TruncOpcode = TargetOpcode::G_TRUNC); 139 140 // Legalize a single operand \p OpIdx of the machine instruction \p MI as a 141 // Def by truncating the operand's type to \p NarrowTy, replacing in place and 142 // extending back with \p ExtOpcode. 143 void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, 144 unsigned ExtOpcode); 145 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 146 /// Def by performing it with additional vector elements and extracting the 147 /// result elements, and replacing the vreg of the operand in place. 148 void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 149 150 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 151 /// Use by producing a vector with undefined high elements, extracting the 152 /// original vector type, and replacing the vreg of the operand in place. 153 void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 154 155 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 156 /// use by inserting a G_BITCAST to \p CastTy 157 void bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 158 159 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 160 /// def by inserting a G_BITCAST from \p CastTy 161 void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 162 163 /// Widen \p OrigReg to \p WideTy by merging to a wider type, padding with 164 /// G_IMPLICIT_DEF, and producing dead results. 165 Register widenWithUnmerge(LLT WideTy, Register OrigReg); 166 167 private: 168 LegalizeResult 169 widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 170 LegalizeResult 171 widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 172 LegalizeResult 173 widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 174 LegalizeResult 175 widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 176 LegalizeResult widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx, 177 LLT WideTy); 178 LegalizeResult widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, 179 LLT WideTy); 180 LegalizeResult widenScalarMulo(MachineInstr &MI, unsigned TypeIdx, 181 LLT WideTy); 182 183 /// Helper function to split a wide generic register into bitwise blocks with 184 /// the given Type (which implies the number of blocks needed). The generic 185 /// registers created are appended to Ops, starting at bit 0 of Reg. 186 void extractParts(Register Reg, LLT Ty, int NumParts, 187 SmallVectorImpl<Register> &VRegs); 188 189 /// Version which handles irregular splits. 190 bool extractParts(Register Reg, LLT RegTy, LLT MainTy, 191 LLT &LeftoverTy, 192 SmallVectorImpl<Register> &VRegs, 193 SmallVectorImpl<Register> &LeftoverVRegs); 194 195 /// Helper function to build a wide generic register \p DstReg of type \p 196 /// RegTy from smaller parts. This will produce a G_MERGE_VALUES, 197 /// G_BUILD_VECTOR, G_CONCAT_VECTORS, or sequence of G_INSERT as appropriate 198 /// for the types. 199 /// 200 /// \p PartRegs must be registers of type \p PartTy. 201 /// 202 /// If \p ResultTy does not evenly break into \p PartTy sized pieces, the 203 /// remainder must be specified with \p LeftoverRegs of type \p LeftoverTy. 204 void insertParts(Register DstReg, LLT ResultTy, 205 LLT PartTy, ArrayRef<Register> PartRegs, 206 LLT LeftoverTy = LLT(), ArrayRef<Register> LeftoverRegs = {}); 207 208 /// Unmerge \p SrcReg into smaller sized values, and append them to \p 209 /// Parts. The elements of \p Parts will be the greatest common divisor type 210 /// of \p DstTy, \p NarrowTy and the type of \p SrcReg. This will compute and 211 /// return the GCD type. 212 LLT extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy, 213 LLT NarrowTy, Register SrcReg); 214 215 /// Unmerge \p SrcReg into \p GCDTy typed registers. This will append all of 216 /// the unpacked registers to \p Parts. This version is if the common unmerge 217 /// type is already known. 218 void extractGCDType(SmallVectorImpl<Register> &Parts, LLT GCDTy, 219 Register SrcReg); 220 221 /// Produce a merge of values in \p VRegs to define \p DstReg. Perform a merge 222 /// from the least common multiple type, and convert as appropriate to \p 223 /// DstReg. 224 /// 225 /// \p VRegs should each have type \p GCDTy. This type should be greatest 226 /// common divisor type of \p DstReg, \p NarrowTy, and an undetermined source 227 /// type. 228 /// 229 /// \p NarrowTy is the desired result merge source type. If the source value 230 /// needs to be widened to evenly cover \p DstReg, inserts high bits 231 /// corresponding to the extension opcode \p PadStrategy. 232 /// 233 /// \p VRegs will be cleared, and the the result \p NarrowTy register pieces 234 /// will replace it. Returns The complete LCMTy that \p VRegs will cover when 235 /// merged. 236 LLT buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, 237 SmallVectorImpl<Register> &VRegs, 238 unsigned PadStrategy = TargetOpcode::G_ANYEXT); 239 240 /// Merge the values in \p RemergeRegs to an \p LCMTy typed value. Extract the 241 /// low bits into \p DstReg. This is intended to use the outputs from 242 /// buildLCMMergePieces after processing. 243 void buildWidenedRemergeToDst(Register DstReg, LLT LCMTy, 244 ArrayRef<Register> RemergeRegs); 245 246 /// Perform generic multiplication of values held in multiple registers. 247 /// Generated instructions use only types NarrowTy and i1. 248 /// Destination can be same or two times size of the source. 249 void multiplyRegisters(SmallVectorImpl<Register> &DstRegs, 250 ArrayRef<Register> Src1Regs, 251 ArrayRef<Register> Src2Regs, LLT NarrowTy); 252 253 void changeOpcode(MachineInstr &MI, unsigned NewOpcode); 254 255 LegalizeResult tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg, 256 LLT SrcTy, LLT NarrowTy, 257 unsigned ScalarOpc); 258 259 public: 260 /// Return the alignment to use for a stack temporary object with the given 261 /// type. 262 Align getStackTemporaryAlignment(LLT Type, Align MinAlign = Align()) const; 263 264 /// Create a stack temporary based on the size in bytes and the alignment 265 MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, 266 MachinePointerInfo &PtrInfo); 267 268 /// Get a pointer to vector element \p Index located in memory for a vector of 269 /// type \p VecTy starting at a base address of \p VecPtr. If \p Index is out 270 /// of bounds the returned pointer is unspecified, but will be within the 271 /// vector bounds. 272 Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index); 273 274 LegalizeResult fewerElementsVectorImplicitDef(MachineInstr &MI, 275 unsigned TypeIdx, LLT NarrowTy); 276 277 /// Legalize a instruction with a vector type where each operand may have a 278 /// different element type. All type indexes must have the same number of 279 /// elements. 280 LegalizeResult fewerElementsVectorMultiEltType(MachineInstr &MI, 281 unsigned TypeIdx, LLT NarrowTy); 282 283 LegalizeResult fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx, 284 LLT NarrowTy); 285 286 LegalizeResult 287 fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 288 289 LegalizeResult 290 fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 291 292 LegalizeResult fewerElementsVectorPhi(MachineInstr &MI, 293 unsigned TypeIdx, LLT NarrowTy); 294 295 LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 296 LLT MoreTy); 297 LegalizeResult moreElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, 298 LLT MoreTy); 299 300 LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI, 301 unsigned TypeIdx, 302 LLT NarrowTy); 303 LegalizeResult fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx, 304 LLT NarrowTy); 305 LegalizeResult fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI, 306 unsigned TypeIdx, 307 LLT NarrowTy); 308 309 LegalizeResult fewerElementsVectorMulo(MachineInstr &MI, unsigned TypeIdx, 310 LLT NarrowTy); 311 312 LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx, 313 LLT NarrowTy); 314 315 /// Legalize an instruction by reducing the operation width, either by 316 /// narrowing the type of the operation or by reducing the number of elements 317 /// of a vector. 318 /// The used strategy (narrow vs. fewerElements) is decided by \p NarrowTy. 319 /// Narrow is used if the scalar type of \p NarrowTy and \p DstTy differ, 320 /// fewerElements is used when the scalar type is the same but the number of 321 /// elements between \p NarrowTy and \p DstTy differ. 322 LegalizeResult reduceOperationWidth(MachineInstr &MI, unsigned TypeIdx, 323 LLT NarrowTy); 324 325 LegalizeResult fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx, 326 LLT NarrowTy); 327 328 LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, 329 LLT HalfTy, LLT ShiftAmtTy); 330 331 LegalizeResult fewerElementsVectorReductions(MachineInstr &MI, 332 unsigned TypeIdx, LLT NarrowTy); 333 334 LegalizeResult fewerElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, 335 LLT NarrowTy); 336 337 LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 338 LegalizeResult narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx, 339 LLT NarrowTy); 340 LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty); 341 LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 342 LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 343 LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 344 345 LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 346 LegalizeResult narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 347 LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 348 LegalizeResult narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 349 LegalizeResult narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 350 LegalizeResult narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 351 352 /// Perform Bitcast legalize action on G_EXTRACT_VECTOR_ELT. 353 LegalizeResult bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, 354 LLT CastTy); 355 356 /// Perform Bitcast legalize action on G_INSERT_VECTOR_ELT. 357 LegalizeResult bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, 358 LLT CastTy); 359 360 LegalizeResult lowerBitcast(MachineInstr &MI); 361 LegalizeResult lowerLoad(GAnyLoad &MI); 362 LegalizeResult lowerStore(GStore &MI); 363 LegalizeResult lowerBitCount(MachineInstr &MI); 364 LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI); 365 LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI); 366 LegalizeResult lowerFunnelShift(MachineInstr &MI); 367 LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI); 368 LegalizeResult lowerRotate(MachineInstr &MI); 369 370 LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI); 371 LegalizeResult lowerUITOFP(MachineInstr &MI); 372 LegalizeResult lowerSITOFP(MachineInstr &MI); 373 LegalizeResult lowerFPTOUI(MachineInstr &MI); 374 LegalizeResult lowerFPTOSI(MachineInstr &MI); 375 376 LegalizeResult lowerFPTRUNC_F64_TO_F16(MachineInstr &MI); 377 LegalizeResult lowerFPTRUNC(MachineInstr &MI); 378 LegalizeResult lowerFPOWI(MachineInstr &MI); 379 380 LegalizeResult lowerMinMax(MachineInstr &MI); 381 LegalizeResult lowerFCopySign(MachineInstr &MI); 382 LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); 383 LegalizeResult lowerFMad(MachineInstr &MI); 384 LegalizeResult lowerIntrinsicRound(MachineInstr &MI); 385 LegalizeResult lowerFFloor(MachineInstr &MI); 386 LegalizeResult lowerMergeValues(MachineInstr &MI); 387 LegalizeResult lowerUnmergeValues(MachineInstr &MI); 388 LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI); 389 LegalizeResult lowerShuffleVector(MachineInstr &MI); 390 LegalizeResult lowerDynStackAlloc(MachineInstr &MI); 391 LegalizeResult lowerExtract(MachineInstr &MI); 392 LegalizeResult lowerInsert(MachineInstr &MI); 393 LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI); 394 LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI); 395 LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI); 396 LegalizeResult lowerShlSat(MachineInstr &MI); 397 LegalizeResult lowerBswap(MachineInstr &MI); 398 LegalizeResult lowerBitreverse(MachineInstr &MI); 399 LegalizeResult lowerReadWriteRegister(MachineInstr &MI); 400 LegalizeResult lowerSMULH_UMULH(MachineInstr &MI); 401 LegalizeResult lowerSelect(MachineInstr &MI); 402 LegalizeResult lowerDIVREM(MachineInstr &MI); 403 LegalizeResult lowerAbsToAddXor(MachineInstr &MI); 404 LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI); 405 }; 406 407 /// Helper function that creates a libcall to the given \p Name using the given 408 /// calling convention \p CC. 409 LegalizerHelper::LegalizeResult 410 createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, 411 const CallLowering::ArgInfo &Result, 412 ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC); 413 414 /// Helper function that creates the given libcall. 415 LegalizerHelper::LegalizeResult 416 createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, 417 const CallLowering::ArgInfo &Result, 418 ArrayRef<CallLowering::ArgInfo> Args); 419 420 /// Create a libcall to memcpy et al. 421 LegalizerHelper::LegalizeResult 422 createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 423 MachineInstr &MI, LostDebugLocObserver &LocObserver); 424 425 } // End namespace llvm. 426 427 #endif 428