1 //== llvm/CodeGen/GlobalISel/LegalizerHelper.h ---------------- -*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file A pass to convert the target-illegal operations created by IR -> MIR 10 /// translation into ones the target expects to be able to select. This may 11 /// occur in multiple phases, for example G_ADD <2 x i8> -> G_ADD <2 x i16> -> 12 /// G_ADD <4 x i16>. 13 /// 14 /// The LegalizerHelper class is where most of the work happens, and is 15 /// designed to be callable from other passes that find themselves with an 16 /// illegal instruction. 17 // 18 //===----------------------------------------------------------------------===// 19 20 #ifndef LLVM_CODEGEN_GLOBALISEL_MACHINELEGALIZEHELPER_H 21 #define LLVM_CODEGEN_GLOBALISEL_MACHINELEGALIZEHELPER_H 22 23 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 24 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 25 #include "llvm/CodeGen/LowLevelType.h" 26 #include "llvm/CodeGen/MachineFunctionPass.h" 27 #include "llvm/CodeGen/RuntimeLibcalls.h" 28 29 namespace llvm { 30 // Forward declarations. 31 class LegalizerInfo; 32 class Legalizer; 33 class MachineRegisterInfo; 34 class GISelChangeObserver; 35 class TargetLowering; 36 37 class LegalizerHelper { 38 public: 39 /// Expose MIRBuilder so clients can set their own RecordInsertInstruction 40 /// functions 41 MachineIRBuilder &MIRBuilder; 42 43 /// To keep track of changes made by the LegalizerHelper. 44 GISelChangeObserver &Observer; 45 46 private: 47 MachineRegisterInfo &MRI; 48 const LegalizerInfo &LI; 49 const TargetLowering &TLI; 50 51 public: 52 enum LegalizeResult { 53 /// Instruction was already legal and no change was made to the 54 /// MachineFunction. 55 AlreadyLegal, 56 57 /// Instruction has been legalized and the MachineFunction changed. 58 Legalized, 59 60 /// Some kind of error has occurred and we could not legalize this 61 /// instruction. 62 UnableToLegalize, 63 }; 64 65 /// Expose LegalizerInfo so the clients can re-use. 66 const LegalizerInfo &getLegalizerInfo() const { return LI; } 67 const TargetLowering &getTargetLowering() const { return TLI; } 68 69 LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, 70 MachineIRBuilder &B); 71 LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, 72 GISelChangeObserver &Observer, MachineIRBuilder &B); 73 74 /// Replace \p MI by a sequence of legal instructions that can implement the 75 /// same operation. Note that this means \p MI may be deleted, so any iterator 76 /// steps should be performed before calling this function. \p Helper should 77 /// be initialized to the MachineFunction containing \p MI. 78 /// 79 /// Considered as an opaque blob, the legal code will use and define the same 80 /// registers as \p MI. 81 LegalizeResult legalizeInstrStep(MachineInstr &MI); 82 83 /// Legalize an instruction by emiting a runtime library call instead. 84 LegalizeResult libcall(MachineInstr &MI); 85 86 /// Legalize an instruction by reducing the width of the underlying scalar 87 /// type. 88 LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 89 90 /// Legalize an instruction by performing the operation on a wider scalar type 91 /// (for example a 16-bit addition can be safely performed at 32-bits 92 /// precision, ignoring the unused bits). 93 LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 94 95 /// Legalize an instruction by replacing the value type 96 LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 97 98 /// Legalize an instruction by splitting it into simpler parts, hopefully 99 /// understood by the target. 100 LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 101 102 /// Legalize a vector instruction by splitting into multiple components, each 103 /// acting on the same scalar type as the original but with fewer elements. 104 LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, 105 LLT NarrowTy); 106 107 /// Legalize a vector instruction by increasing the number of vector elements 108 /// involved and ignoring the added elements later. 109 LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, 110 LLT MoreTy); 111 112 /// Cast the given value to an LLT::scalar with an equivalent size. Returns 113 /// the register to use if an instruction was inserted. Returns the original 114 /// register if no coercion was necessary. 115 // 116 // This may also fail and return Register() if there is no legal way to cast. 117 Register coerceToScalar(Register Val); 118 119 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 120 /// Use by extending the operand's type to \p WideTy using the specified \p 121 /// ExtOpcode for the extension instruction, and replacing the vreg of the 122 /// operand in place. 123 void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, 124 unsigned ExtOpcode); 125 126 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 127 /// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and 128 /// replacing the vreg of the operand in place. 129 void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx); 130 131 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 132 /// Def by extending the operand's type to \p WideTy and truncating it back 133 /// with the \p TruncOpcode, and replacing the vreg of the operand in place. 134 void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0, 135 unsigned TruncOpcode = TargetOpcode::G_TRUNC); 136 137 // Legalize a single operand \p OpIdx of the machine instruction \p MI as a 138 // Def by truncating the operand's type to \p NarrowTy, replacing in place and 139 // extending back with \p ExtOpcode. 140 void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, 141 unsigned ExtOpcode); 142 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 143 /// Def by performing it with additional vector elements and extracting the 144 /// result elements, and replacing the vreg of the operand in place. 145 void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 146 147 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 148 /// Use by producing a vector with undefined high elements, extracting the 149 /// original vector type, and replacing the vreg of the operand in place. 150 void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 151 152 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 153 /// use by inserting a G_BITCAST to \p CastTy 154 void bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 155 156 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 157 /// def by inserting a G_BITCAST from \p CastTy 158 void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 159 160 /// Widen \p OrigReg to \p WideTy by merging to a wider type, padding with 161 /// G_IMPLICIT_DEF, and producing dead results. 162 Register widenWithUnmerge(LLT WideTy, Register OrigReg); 163 164 private: 165 LegalizeResult 166 widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 167 LegalizeResult 168 widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 169 LegalizeResult 170 widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 171 LegalizeResult 172 widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 173 LegalizeResult widenScalarAddoSubo(MachineInstr &MI, unsigned TypeIdx, 174 LLT WideTy); 175 LegalizeResult widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, 176 LLT WideTy); 177 178 /// Helper function to split a wide generic register into bitwise blocks with 179 /// the given Type (which implies the number of blocks needed). The generic 180 /// registers created are appended to Ops, starting at bit 0 of Reg. 181 void extractParts(Register Reg, LLT Ty, int NumParts, 182 SmallVectorImpl<Register> &VRegs); 183 184 /// Version which handles irregular splits. 185 bool extractParts(Register Reg, LLT RegTy, LLT MainTy, 186 LLT &LeftoverTy, 187 SmallVectorImpl<Register> &VRegs, 188 SmallVectorImpl<Register> &LeftoverVRegs); 189 190 /// Helper function to build a wide generic register \p DstReg of type \p 191 /// RegTy from smaller parts. This will produce a G_MERGE_VALUES, 192 /// G_BUILD_VECTOR, G_CONCAT_VECTORS, or sequence of G_INSERT as appropriate 193 /// for the types. 194 /// 195 /// \p PartRegs must be registers of type \p PartTy. 196 /// 197 /// If \p ResultTy does not evenly break into \p PartTy sized pieces, the 198 /// remainder must be specified with \p LeftoverRegs of type \p LeftoverTy. 199 void insertParts(Register DstReg, LLT ResultTy, 200 LLT PartTy, ArrayRef<Register> PartRegs, 201 LLT LeftoverTy = LLT(), ArrayRef<Register> LeftoverRegs = {}); 202 203 /// Unmerge \p SrcReg into smaller sized values, and append them to \p 204 /// Parts. The elements of \p Parts will be the greatest common divisor type 205 /// of \p DstTy, \p NarrowTy and the type of \p SrcReg. This will compute and 206 /// return the GCD type. 207 LLT extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy, 208 LLT NarrowTy, Register SrcReg); 209 210 /// Unmerge \p SrcReg into \p GCDTy typed registers. This will append all of 211 /// the unpacked registers to \p Parts. This version is if the common unmerge 212 /// type is already known. 213 void extractGCDType(SmallVectorImpl<Register> &Parts, LLT GCDTy, 214 Register SrcReg); 215 216 /// Produce a merge of values in \p VRegs to define \p DstReg. Perform a merge 217 /// from the least common multiple type, and convert as appropriate to \p 218 /// DstReg. 219 /// 220 /// \p VRegs should each have type \p GCDTy. This type should be greatest 221 /// common divisor type of \p DstReg, \p NarrowTy, and an undetermined source 222 /// type. 223 /// 224 /// \p NarrowTy is the desired result merge source type. If the source value 225 /// needs to be widened to evenly cover \p DstReg, inserts high bits 226 /// corresponding to the extension opcode \p PadStrategy. 227 /// 228 /// \p VRegs will be cleared, and the the result \p NarrowTy register pieces 229 /// will replace it. Returns The complete LCMTy that \p VRegs will cover when 230 /// merged. 231 LLT buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, 232 SmallVectorImpl<Register> &VRegs, 233 unsigned PadStrategy = TargetOpcode::G_ANYEXT); 234 235 /// Merge the values in \p RemergeRegs to an \p LCMTy typed value. Extract the 236 /// low bits into \p DstReg. This is intended to use the outputs from 237 /// buildLCMMergePieces after processing. 238 void buildWidenedRemergeToDst(Register DstReg, LLT LCMTy, 239 ArrayRef<Register> RemergeRegs); 240 241 /// Perform generic multiplication of values held in multiple registers. 242 /// Generated instructions use only types NarrowTy and i1. 243 /// Destination can be same or two times size of the source. 244 void multiplyRegisters(SmallVectorImpl<Register> &DstRegs, 245 ArrayRef<Register> Src1Regs, 246 ArrayRef<Register> Src2Regs, LLT NarrowTy); 247 248 void changeOpcode(MachineInstr &MI, unsigned NewOpcode); 249 250 public: 251 /// Return the alignment to use for a stack temporary object with the given 252 /// type. 253 Align getStackTemporaryAlignment(LLT Type, Align MinAlign = Align()) const; 254 255 /// Create a stack temporary based on the size in bytes and the alignment 256 MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, 257 MachinePointerInfo &PtrInfo); 258 259 /// Get a pointer to vector element \p Index located in memory for a vector of 260 /// type \p VecTy starting at a base address of \p VecPtr. If \p Index is out 261 /// of bounds the returned pointer is unspecified, but will be within the 262 /// vector bounds. 263 Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index); 264 265 LegalizeResult fewerElementsVectorImplicitDef(MachineInstr &MI, 266 unsigned TypeIdx, LLT NarrowTy); 267 268 /// Legalize a instruction with a vector type where each operand may have a 269 /// different element type. All type indexes must have the same number of 270 /// elements. 271 LegalizeResult fewerElementsVectorMultiEltType(MachineInstr &MI, 272 unsigned TypeIdx, LLT NarrowTy); 273 274 LegalizeResult fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx, 275 LLT NarrowTy); 276 277 LegalizeResult 278 fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 279 280 LegalizeResult 281 fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 282 283 LegalizeResult fewerElementsVectorPhi(MachineInstr &MI, 284 unsigned TypeIdx, LLT NarrowTy); 285 286 LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 287 LLT MoreTy); 288 289 LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI, 290 unsigned TypeIdx, 291 LLT NarrowTy); 292 LegalizeResult fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx, 293 LLT NarrowTy); 294 LegalizeResult fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI, 295 unsigned TypeIdx, 296 LLT NarrowTy); 297 298 LegalizeResult 299 reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 300 301 /// Legalize an instruction by reducing the operation width, either by 302 /// narrowing the type of the operation or by reducing the number of elements 303 /// of a vector. 304 /// The used strategy (narrow vs. fewerElements) is decided by \p NarrowTy. 305 /// Narrow is used if the scalar type of \p NarrowTy and \p DstTy differ, 306 /// fewerElements is used when the scalar type is the same but the number of 307 /// elements between \p NarrowTy and \p DstTy differ. 308 LegalizeResult reduceOperationWidth(MachineInstr &MI, unsigned TypeIdx, 309 LLT NarrowTy); 310 311 LegalizeResult fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx, 312 LLT NarrowTy); 313 314 LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, 315 LLT HalfTy, LLT ShiftAmtTy); 316 317 LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 318 LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty); 319 LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 320 LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 321 LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 322 323 LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 324 LegalizeResult narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 325 LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 326 LegalizeResult narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 327 LegalizeResult narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 328 LegalizeResult narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 329 330 /// Perform Bitcast legalize action on G_EXTRACT_VECTOR_ELT. 331 LegalizeResult bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, 332 LLT CastTy); 333 334 /// Perform Bitcast legalize action on G_INSERT_VECTOR_ELT. 335 LegalizeResult bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, 336 LLT CastTy); 337 338 LegalizeResult lowerBitcast(MachineInstr &MI); 339 LegalizeResult lowerLoad(MachineInstr &MI); 340 LegalizeResult lowerStore(MachineInstr &MI); 341 LegalizeResult lowerBitCount(MachineInstr &MI); 342 343 LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI); 344 LegalizeResult lowerUITOFP(MachineInstr &MI); 345 LegalizeResult lowerSITOFP(MachineInstr &MI); 346 LegalizeResult lowerFPTOUI(MachineInstr &MI); 347 LegalizeResult lowerFPTOSI(MachineInstr &MI); 348 349 LegalizeResult lowerFPTRUNC_F64_TO_F16(MachineInstr &MI); 350 LegalizeResult lowerFPTRUNC(MachineInstr &MI); 351 LegalizeResult lowerFPOWI(MachineInstr &MI); 352 353 LegalizeResult lowerMinMax(MachineInstr &MI); 354 LegalizeResult lowerFCopySign(MachineInstr &MI); 355 LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); 356 LegalizeResult lowerFMad(MachineInstr &MI); 357 LegalizeResult lowerIntrinsicRound(MachineInstr &MI); 358 LegalizeResult lowerFFloor(MachineInstr &MI); 359 LegalizeResult lowerMergeValues(MachineInstr &MI); 360 LegalizeResult lowerUnmergeValues(MachineInstr &MI); 361 LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI); 362 LegalizeResult lowerShuffleVector(MachineInstr &MI); 363 LegalizeResult lowerDynStackAlloc(MachineInstr &MI); 364 LegalizeResult lowerExtract(MachineInstr &MI); 365 LegalizeResult lowerInsert(MachineInstr &MI); 366 LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI); 367 LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI); 368 LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI); 369 LegalizeResult lowerShlSat(MachineInstr &MI); 370 LegalizeResult lowerBswap(MachineInstr &MI); 371 LegalizeResult lowerBitreverse(MachineInstr &MI); 372 LegalizeResult lowerReadWriteRegister(MachineInstr &MI); 373 LegalizeResult lowerSMULH_UMULH(MachineInstr &MI); 374 LegalizeResult lowerSelect(MachineInstr &MI); 375 376 }; 377 378 /// Helper function that creates a libcall to the given \p Name using the given 379 /// calling convention \p CC. 380 LegalizerHelper::LegalizeResult 381 createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, 382 const CallLowering::ArgInfo &Result, 383 ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC); 384 385 /// Helper function that creates the given libcall. 386 LegalizerHelper::LegalizeResult 387 createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, 388 const CallLowering::ArgInfo &Result, 389 ArrayRef<CallLowering::ArgInfo> Args); 390 391 /// Create a libcall to memcpy et al. 392 LegalizerHelper::LegalizeResult createMemLibcall(MachineIRBuilder &MIRBuilder, 393 MachineRegisterInfo &MRI, 394 MachineInstr &MI); 395 396 } // End namespace llvm. 397 398 #endif 399