1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineRegisterInfo.h" 30 #include "llvm/IR/IntrinsicsSPIRV.h" 31 #include "llvm/Support/Debug.h" 32 33 #define DEBUG_TYPE "spirv-isel" 34 35 using namespace llvm; 36 namespace CL = SPIRV::OpenCLExtInst; 37 namespace GL = SPIRV::GLSLExtInst; 38 39 using ExtInstList = 40 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 41 42 namespace { 43 44 #define GET_GLOBALISEL_PREDICATE_BITSET 45 #include "SPIRVGenGlobalISel.inc" 46 #undef GET_GLOBALISEL_PREDICATE_BITSET 47 48 class SPIRVInstructionSelector : public InstructionSelector { 49 const SPIRVSubtarget &STI; 50 const SPIRVInstrInfo &TII; 51 const SPIRVRegisterInfo &TRI; 52 const RegisterBankInfo &RBI; 53 SPIRVGlobalRegistry &GR; 54 MachineRegisterInfo *MRI; 55 56 public: 57 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 58 const SPIRVSubtarget &ST, 59 const RegisterBankInfo &RBI); 60 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 61 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 62 BlockFrequencyInfo *BFI) override; 63 // Common selection code. Instruction-specific selection occurs in spvSelect. 64 bool select(MachineInstr &I) override; 65 static const char *getName() { return DEBUG_TYPE; } 66 67 #define GET_GLOBALISEL_PREDICATES_DECL 68 #include "SPIRVGenGlobalISel.inc" 69 #undef GET_GLOBALISEL_PREDICATES_DECL 70 71 #define GET_GLOBALISEL_TEMPORARIES_DECL 72 #include "SPIRVGenGlobalISel.inc" 73 #undef GET_GLOBALISEL_TEMPORARIES_DECL 74 75 private: 76 // tblgen-erated 'select' implementation, used as the initial selector for 77 // the patterns that don't require complex C++. 78 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 79 80 // All instruction-specific selection that didn't happen in "select()". 81 // Is basically a large Switch/Case delegating to all other select method. 82 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 83 MachineInstr &I) const; 84 85 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 86 const MachineInstr *Init = nullptr) const; 87 88 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 89 MachineInstr &I, Register SrcReg, 90 unsigned Opcode) const; 91 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 92 unsigned Opcode) const; 93 94 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 95 MachineInstr &I) const; 96 bool selectStore(MachineInstr &I) const; 97 98 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 99 100 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 101 MachineInstr &I, unsigned NewOpcode) const; 102 103 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 104 MachineInstr &I) const; 105 106 bool selectFence(MachineInstr &I) const; 107 108 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 109 MachineInstr &I) const; 110 111 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 112 MachineInstr &I) const; 113 114 bool selectConstVector(Register ResVReg, const SPIRVType *ResType, 115 MachineInstr &I) const; 116 117 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 118 unsigned comparisonOpcode, MachineInstr &I) const; 119 120 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 121 MachineInstr &I) const; 122 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 123 MachineInstr &I) const; 124 125 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 126 int OpIdx) const; 127 void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 128 int OpIdx) const; 129 130 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 131 MachineInstr &I) const; 132 133 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 134 bool IsSigned) const; 135 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 136 bool IsSigned, unsigned Opcode) const; 137 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 138 bool IsSigned) const; 139 140 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 141 MachineInstr &I) const; 142 143 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 144 const SPIRVType *intTy, const SPIRVType *boolTy) const; 145 146 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 147 MachineInstr &I) const; 148 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 149 MachineInstr &I) const; 150 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 151 MachineInstr &I) const; 152 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 153 MachineInstr &I) const; 154 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 155 MachineInstr &I) const; 156 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 157 MachineInstr &I) const; 158 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 159 MachineInstr &I) const; 160 161 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 162 MachineInstr &I) const; 163 164 bool selectBranch(MachineInstr &I) const; 165 bool selectBranchCond(MachineInstr &I) const; 166 167 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 168 MachineInstr &I) const; 169 170 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 171 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 172 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 173 MachineInstr &I, CL::OpenCLExtInst CLInst, 174 GL::GLSLExtInst GLInst) const; 175 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 176 MachineInstr &I, const ExtInstList &ExtInsts) const; 177 178 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 179 MachineInstr &I) const; 180 181 Register buildI32Constant(uint32_t Val, MachineInstr &I, 182 const SPIRVType *ResType = nullptr) const; 183 184 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 185 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 186 MachineInstr &I) const; 187 }; 188 189 } // end anonymous namespace 190 191 #define GET_GLOBALISEL_IMPL 192 #include "SPIRVGenGlobalISel.inc" 193 #undef GET_GLOBALISEL_IMPL 194 195 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 196 const SPIRVSubtarget &ST, 197 const RegisterBankInfo &RBI) 198 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 199 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 200 #define GET_GLOBALISEL_PREDICATES_INIT 201 #include "SPIRVGenGlobalISel.inc" 202 #undef GET_GLOBALISEL_PREDICATES_INIT 203 #define GET_GLOBALISEL_TEMPORARIES_INIT 204 #include "SPIRVGenGlobalISel.inc" 205 #undef GET_GLOBALISEL_TEMPORARIES_INIT 206 { 207 } 208 209 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 210 CodeGenCoverage *CoverageInfo, 211 ProfileSummaryInfo *PSI, 212 BlockFrequencyInfo *BFI) { 213 MRI = &MF.getRegInfo(); 214 GR.setCurrentFunc(MF); 215 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 216 } 217 218 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 219 220 // Defined in SPIRVLegalizerInfo.cpp. 221 extern bool isTypeFoldingSupported(unsigned Opcode); 222 223 bool SPIRVInstructionSelector::select(MachineInstr &I) { 224 assert(I.getParent() && "Instruction should be in a basic block!"); 225 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 226 227 Register Opcode = I.getOpcode(); 228 // If it's not a GMIR instruction, we've selected it already. 229 if (!isPreISelGenericOpcode(Opcode)) { 230 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 231 auto *Def = MRI->getVRegDef(I.getOperand(1).getReg()); 232 if (isTypeFoldingSupported(Def->getOpcode())) { 233 auto Res = selectImpl(I, *CoverageInfo); 234 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 235 if (Res) 236 return Res; 237 } 238 MRI->replaceRegWith(I.getOperand(1).getReg(), I.getOperand(0).getReg()); 239 I.removeFromParent(); 240 return true; 241 } else if (I.getNumDefs() == 1) { 242 // Make all vregs 32 bits (for SPIR-V IDs). 243 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32)); 244 } 245 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 246 } 247 248 if (I.getNumOperands() != I.getNumExplicitOperands()) { 249 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 250 return false; 251 } 252 253 // Common code for getting return reg+type, and removing selected instr 254 // from parent occurs here. Instr-specific selection happens in spvSelect(). 255 bool HasDefs = I.getNumDefs() > 0; 256 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 257 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 258 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 259 if (spvSelect(ResVReg, ResType, I)) { 260 if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs). 261 MRI->setType(ResVReg, LLT::scalar(32)); 262 I.removeFromParent(); 263 return true; 264 } 265 return false; 266 } 267 268 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 269 const SPIRVType *ResType, 270 MachineInstr &I) const { 271 assert(!isTypeFoldingSupported(I.getOpcode()) || 272 I.getOpcode() == TargetOpcode::G_CONSTANT); 273 const unsigned Opcode = I.getOpcode(); 274 switch (Opcode) { 275 case TargetOpcode::G_CONSTANT: 276 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 277 I); 278 case TargetOpcode::G_GLOBAL_VALUE: 279 return selectGlobalValue(ResVReg, I); 280 case TargetOpcode::G_IMPLICIT_DEF: 281 return selectOpUndef(ResVReg, ResType, I); 282 283 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 284 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 285 return selectIntrinsic(ResVReg, ResType, I); 286 case TargetOpcode::G_BITREVERSE: 287 return selectBitreverse(ResVReg, ResType, I); 288 289 case TargetOpcode::G_BUILD_VECTOR: 290 return selectConstVector(ResVReg, ResType, I); 291 292 case TargetOpcode::G_SHUFFLE_VECTOR: { 293 MachineBasicBlock &BB = *I.getParent(); 294 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 295 .addDef(ResVReg) 296 .addUse(GR.getSPIRVTypeID(ResType)) 297 .addUse(I.getOperand(1).getReg()) 298 .addUse(I.getOperand(2).getReg()); 299 for (auto V : I.getOperand(3).getShuffleMask()) 300 MIB.addImm(V); 301 return MIB.constrainAllUses(TII, TRI, RBI); 302 } 303 case TargetOpcode::G_MEMMOVE: 304 case TargetOpcode::G_MEMCPY: 305 case TargetOpcode::G_MEMSET: 306 return selectMemOperation(ResVReg, I); 307 308 case TargetOpcode::G_ICMP: 309 return selectICmp(ResVReg, ResType, I); 310 case TargetOpcode::G_FCMP: 311 return selectFCmp(ResVReg, ResType, I); 312 313 case TargetOpcode::G_FRAME_INDEX: 314 return selectFrameIndex(ResVReg, ResType, I); 315 316 case TargetOpcode::G_LOAD: 317 return selectLoad(ResVReg, ResType, I); 318 case TargetOpcode::G_STORE: 319 return selectStore(I); 320 321 case TargetOpcode::G_BR: 322 return selectBranch(I); 323 case TargetOpcode::G_BRCOND: 324 return selectBranchCond(I); 325 326 case TargetOpcode::G_PHI: 327 return selectPhi(ResVReg, ResType, I); 328 329 case TargetOpcode::G_FPTOSI: 330 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 331 case TargetOpcode::G_FPTOUI: 332 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 333 334 case TargetOpcode::G_SITOFP: 335 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 336 case TargetOpcode::G_UITOFP: 337 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 338 339 case TargetOpcode::G_CTPOP: 340 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 341 case TargetOpcode::G_SMIN: 342 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 343 case TargetOpcode::G_UMIN: 344 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 345 346 case TargetOpcode::G_SMAX: 347 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 348 case TargetOpcode::G_UMAX: 349 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 350 351 case TargetOpcode::G_FMA: 352 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 353 354 case TargetOpcode::G_FPOW: 355 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 356 case TargetOpcode::G_FPOWI: 357 return selectExtInst(ResVReg, ResType, I, CL::pown); 358 359 case TargetOpcode::G_FEXP: 360 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 361 case TargetOpcode::G_FEXP2: 362 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 363 364 case TargetOpcode::G_FLOG: 365 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 366 case TargetOpcode::G_FLOG2: 367 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 368 case TargetOpcode::G_FLOG10: 369 return selectLog10(ResVReg, ResType, I); 370 371 case TargetOpcode::G_FABS: 372 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 373 case TargetOpcode::G_ABS: 374 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 375 376 case TargetOpcode::G_FMINNUM: 377 case TargetOpcode::G_FMINIMUM: 378 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::FMin); 379 case TargetOpcode::G_FMAXNUM: 380 case TargetOpcode::G_FMAXIMUM: 381 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::FMax); 382 383 case TargetOpcode::G_FCOPYSIGN: 384 return selectExtInst(ResVReg, ResType, I, CL::copysign); 385 386 case TargetOpcode::G_FCEIL: 387 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 388 case TargetOpcode::G_FFLOOR: 389 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 390 391 case TargetOpcode::G_FCOS: 392 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 393 case TargetOpcode::G_FSIN: 394 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 395 396 case TargetOpcode::G_FSQRT: 397 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 398 399 case TargetOpcode::G_CTTZ: 400 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 401 return selectExtInst(ResVReg, ResType, I, CL::ctz); 402 case TargetOpcode::G_CTLZ: 403 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 404 return selectExtInst(ResVReg, ResType, I, CL::clz); 405 406 case TargetOpcode::G_INTRINSIC_ROUND: 407 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 408 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 409 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 410 case TargetOpcode::G_INTRINSIC_TRUNC: 411 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 412 case TargetOpcode::G_FRINT: 413 case TargetOpcode::G_FNEARBYINT: 414 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 415 416 case TargetOpcode::G_SMULH: 417 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 418 case TargetOpcode::G_UMULH: 419 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 420 421 case TargetOpcode::G_SEXT: 422 return selectExt(ResVReg, ResType, I, true); 423 case TargetOpcode::G_ANYEXT: 424 case TargetOpcode::G_ZEXT: 425 return selectExt(ResVReg, ResType, I, false); 426 case TargetOpcode::G_TRUNC: 427 return selectTrunc(ResVReg, ResType, I); 428 case TargetOpcode::G_FPTRUNC: 429 case TargetOpcode::G_FPEXT: 430 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 431 432 case TargetOpcode::G_PTRTOINT: 433 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 434 case TargetOpcode::G_INTTOPTR: 435 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 436 case TargetOpcode::G_BITCAST: 437 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 438 case TargetOpcode::G_ADDRSPACE_CAST: 439 return selectAddrSpaceCast(ResVReg, ResType, I); 440 case TargetOpcode::G_PTR_ADD: { 441 // Currently, we get G_PTR_ADD only as a result of translating 442 // global variables, initialized with constant expressions like GV + Const 443 // (see test opencl/basic/progvar_prog_scope_init.ll). 444 // TODO: extend the handler once we have other cases. 445 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 446 Register GV = I.getOperand(1).getReg(); 447 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 448 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 449 (*II).getOpcode() == TargetOpcode::COPY || 450 (*II).getOpcode() == SPIRV::OpVariable) && 451 isImm(I.getOperand(2), MRI)); 452 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 453 MachineBasicBlock &BB = *I.getParent(); 454 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 455 .addDef(ResVReg) 456 .addUse(GR.getSPIRVTypeID(ResType)) 457 .addImm(static_cast<uint32_t>( 458 SPIRV::Opcode::InBoundsPtrAccessChain)) 459 .addUse(GV) 460 .addUse(Idx) 461 .addUse(I.getOperand(2).getReg()); 462 return MIB.constrainAllUses(TII, TRI, RBI); 463 } 464 465 case TargetOpcode::G_ATOMICRMW_OR: 466 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 467 case TargetOpcode::G_ATOMICRMW_ADD: 468 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 469 case TargetOpcode::G_ATOMICRMW_AND: 470 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 471 case TargetOpcode::G_ATOMICRMW_MAX: 472 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 473 case TargetOpcode::G_ATOMICRMW_MIN: 474 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 475 case TargetOpcode::G_ATOMICRMW_SUB: 476 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 477 case TargetOpcode::G_ATOMICRMW_XOR: 478 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 479 case TargetOpcode::G_ATOMICRMW_UMAX: 480 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 481 case TargetOpcode::G_ATOMICRMW_UMIN: 482 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 483 case TargetOpcode::G_ATOMICRMW_XCHG: 484 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 485 case TargetOpcode::G_ATOMIC_CMPXCHG: 486 return selectAtomicCmpXchg(ResVReg, ResType, I); 487 488 case TargetOpcode::G_FENCE: 489 return selectFence(I); 490 491 default: 492 return false; 493 } 494 } 495 496 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 497 const SPIRVType *ResType, 498 MachineInstr &I, 499 CL::OpenCLExtInst CLInst) const { 500 return selectExtInst(ResVReg, ResType, I, 501 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 502 } 503 504 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 505 const SPIRVType *ResType, 506 MachineInstr &I, 507 CL::OpenCLExtInst CLInst, 508 GL::GLSLExtInst GLInst) const { 509 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 510 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 511 return selectExtInst(ResVReg, ResType, I, ExtInsts); 512 } 513 514 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 515 const SPIRVType *ResType, 516 MachineInstr &I, 517 const ExtInstList &Insts) const { 518 519 for (const auto &Ex : Insts) { 520 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 521 uint32_t Opcode = Ex.second; 522 if (STI.canUseExtInstSet(Set)) { 523 MachineBasicBlock &BB = *I.getParent(); 524 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 525 .addDef(ResVReg) 526 .addUse(GR.getSPIRVTypeID(ResType)) 527 .addImm(static_cast<uint32_t>(Set)) 528 .addImm(Opcode); 529 const unsigned NumOps = I.getNumOperands(); 530 for (unsigned i = 1; i < NumOps; ++i) 531 MIB.add(I.getOperand(i)); 532 return MIB.constrainAllUses(TII, TRI, RBI); 533 } 534 } 535 return false; 536 } 537 538 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 539 const SPIRVType *ResType, 540 MachineInstr &I, 541 Register SrcReg, 542 unsigned Opcode) const { 543 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 544 .addDef(ResVReg) 545 .addUse(GR.getSPIRVTypeID(ResType)) 546 .addUse(SrcReg) 547 .constrainAllUses(TII, TRI, RBI); 548 } 549 550 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 551 const SPIRVType *ResType, 552 MachineInstr &I, 553 unsigned Opcode) const { 554 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 555 Opcode); 556 } 557 558 static SPIRV::Scope::Scope getScope(SyncScope::ID Ord) { 559 switch (Ord) { 560 case SyncScope::SingleThread: 561 return SPIRV::Scope::Invocation; 562 case SyncScope::System: 563 return SPIRV::Scope::Device; 564 default: 565 llvm_unreachable("Unsupported synchronization Scope ID."); 566 } 567 } 568 569 static void addMemoryOperands(MachineMemOperand *MemOp, 570 MachineInstrBuilder &MIB) { 571 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 572 if (MemOp->isVolatile()) 573 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 574 if (MemOp->isNonTemporal()) 575 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 576 if (MemOp->getAlign().value()) 577 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 578 579 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 580 MIB.addImm(SpvMemOp); 581 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 582 MIB.addImm(MemOp->getAlign().value()); 583 } 584 } 585 586 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 587 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 588 if (Flags & MachineMemOperand::Flags::MOVolatile) 589 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 590 if (Flags & MachineMemOperand::Flags::MONonTemporal) 591 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 592 593 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 594 MIB.addImm(SpvMemOp); 595 } 596 597 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 598 const SPIRVType *ResType, 599 MachineInstr &I) const { 600 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 601 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 602 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 603 .addDef(ResVReg) 604 .addUse(GR.getSPIRVTypeID(ResType)) 605 .addUse(Ptr); 606 if (!I.getNumMemOperands()) { 607 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 608 I.getOpcode() == 609 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 610 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 611 } else { 612 addMemoryOperands(*I.memoperands_begin(), MIB); 613 } 614 return MIB.constrainAllUses(TII, TRI, RBI); 615 } 616 617 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 618 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 619 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 620 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 621 MachineBasicBlock &BB = *I.getParent(); 622 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 623 .addUse(Ptr) 624 .addUse(StoreVal); 625 if (!I.getNumMemOperands()) { 626 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 627 I.getOpcode() == 628 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 629 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 630 } else { 631 addMemoryOperands(*I.memoperands_begin(), MIB); 632 } 633 return MIB.constrainAllUses(TII, TRI, RBI); 634 } 635 636 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 637 MachineInstr &I) const { 638 MachineBasicBlock &BB = *I.getParent(); 639 Register SrcReg = I.getOperand(1).getReg(); 640 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 641 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 642 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 643 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 644 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 645 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 646 Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII); 647 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 648 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 649 // TODO: check if we have such GV, add init, use buildGlobalVariable. 650 Type *LLVMArrTy = ArrayType::get( 651 IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num); 652 GlobalVariable *GV = 653 new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage); 654 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 655 GR.add(GV, GR.CurMF, VarReg); 656 657 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 658 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 659 .addDef(VarReg) 660 .addUse(GR.getSPIRVTypeID(VarTy)) 661 .addImm(SPIRV::StorageClass::UniformConstant) 662 .addUse(Const) 663 .constrainAllUses(TII, TRI, RBI); 664 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 665 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 666 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 667 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 668 } 669 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 670 .addUse(I.getOperand(0).getReg()) 671 .addUse(SrcReg) 672 .addUse(I.getOperand(2).getReg()); 673 if (I.getNumMemOperands()) 674 addMemoryOperands(*I.memoperands_begin(), MIB); 675 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 676 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 677 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 678 .addUse(MIB->getOperand(0).getReg()); 679 return Result; 680 } 681 682 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 683 const SPIRVType *ResType, 684 MachineInstr &I, 685 unsigned NewOpcode) const { 686 assert(I.hasOneMemOperand()); 687 const MachineMemOperand *MemOp = *I.memoperands_begin(); 688 uint32_t Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID())); 689 Register ScopeReg = buildI32Constant(Scope, I); 690 691 Register Ptr = I.getOperand(1).getReg(); 692 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 693 // auto ScSem = 694 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 695 AtomicOrdering AO = MemOp->getSuccessOrdering(); 696 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 697 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); 698 699 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 700 .addDef(ResVReg) 701 .addUse(GR.getSPIRVTypeID(ResType)) 702 .addUse(Ptr) 703 .addUse(ScopeReg) 704 .addUse(MemSemReg) 705 .addUse(I.getOperand(2).getReg()) 706 .constrainAllUses(TII, TRI, RBI); 707 } 708 709 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 710 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 711 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 712 Register MemSemReg = buildI32Constant(MemSem, I); 713 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 714 uint32_t Scope = static_cast<uint32_t>(getScope(Ord)); 715 Register ScopeReg = buildI32Constant(Scope, I); 716 MachineBasicBlock &BB = *I.getParent(); 717 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 718 .addUse(ScopeReg) 719 .addUse(MemSemReg) 720 .constrainAllUses(TII, TRI, RBI); 721 } 722 723 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 724 const SPIRVType *ResType, 725 MachineInstr &I) const { 726 Register ScopeReg; 727 Register MemSemEqReg; 728 Register MemSemNeqReg; 729 Register Ptr = I.getOperand(2).getReg(); 730 if (!isa<GIntrinsic>(I)) { 731 assert(I.hasOneMemOperand()); 732 const MachineMemOperand *MemOp = *I.memoperands_begin(); 733 unsigned Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID())); 734 ScopeReg = buildI32Constant(Scope, I); 735 736 unsigned ScSem = static_cast<uint32_t>( 737 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 738 AtomicOrdering AO = MemOp->getSuccessOrdering(); 739 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 740 MemSemEqReg = buildI32Constant(MemSemEq, I); 741 AtomicOrdering FO = MemOp->getFailureOrdering(); 742 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 743 MemSemNeqReg = 744 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); 745 } else { 746 ScopeReg = I.getOperand(5).getReg(); 747 MemSemEqReg = I.getOperand(6).getReg(); 748 MemSemNeqReg = I.getOperand(7).getReg(); 749 } 750 751 Register Cmp = I.getOperand(3).getReg(); 752 Register Val = I.getOperand(4).getReg(); 753 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 754 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass); 755 const DebugLoc &DL = I.getDebugLoc(); 756 bool Result = 757 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 758 .addDef(ACmpRes) 759 .addUse(GR.getSPIRVTypeID(SpvValTy)) 760 .addUse(Ptr) 761 .addUse(ScopeReg) 762 .addUse(MemSemEqReg) 763 .addUse(MemSemNeqReg) 764 .addUse(Val) 765 .addUse(Cmp) 766 .constrainAllUses(TII, TRI, RBI); 767 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 768 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 769 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 770 .addDef(CmpSuccReg) 771 .addUse(GR.getSPIRVTypeID(BoolTy)) 772 .addUse(ACmpRes) 773 .addUse(Cmp) 774 .constrainAllUses(TII, TRI, RBI); 775 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 776 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 777 .addDef(TmpReg) 778 .addUse(GR.getSPIRVTypeID(ResType)) 779 .addUse(ACmpRes) 780 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 781 .addImm(0) 782 .constrainAllUses(TII, TRI, RBI); 783 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 784 .addDef(ResVReg) 785 .addUse(GR.getSPIRVTypeID(ResType)) 786 .addUse(CmpSuccReg) 787 .addUse(TmpReg) 788 .addImm(1) 789 .constrainAllUses(TII, TRI, RBI); 790 return Result; 791 } 792 793 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 794 switch (SC) { 795 case SPIRV::StorageClass::Workgroup: 796 case SPIRV::StorageClass::CrossWorkgroup: 797 case SPIRV::StorageClass::Function: 798 return true; 799 default: 800 return false; 801 } 802 } 803 804 // In SPIR-V address space casting can only happen to and from the Generic 805 // storage class. We can also only case Workgroup, CrossWorkgroup, or Function 806 // pointers to and from Generic pointers. As such, we can convert e.g. from 807 // Workgroup to Function by going via a Generic pointer as an intermediary. All 808 // other combinations can only be done by a bitcast, and are probably not safe. 809 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 810 const SPIRVType *ResType, 811 MachineInstr &I) const { 812 // If the AddrSpaceCast user is single and in OpConstantComposite or 813 // OpVariable, we should select OpSpecConstantOp. 814 auto UIs = MRI->use_instructions(ResVReg); 815 if (!UIs.empty() && ++UIs.begin() == UIs.end() && 816 (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite || 817 UIs.begin()->getOpcode() == SPIRV::OpVariable || 818 isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) { 819 Register NewReg = I.getOperand(1).getReg(); 820 MachineBasicBlock &BB = *I.getParent(); 821 SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 822 ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII, 823 SPIRV::StorageClass::Generic); 824 bool Result = 825 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 826 .addDef(ResVReg) 827 .addUse(GR.getSPIRVTypeID(ResType)) 828 .addImm(static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)) 829 .addUse(NewReg) 830 .constrainAllUses(TII, TRI, RBI); 831 return Result; 832 } 833 Register SrcPtr = I.getOperand(1).getReg(); 834 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 835 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr); 836 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg); 837 838 // Casting from an eligable pointer to Generic. 839 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 840 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 841 // Casting from Generic to an eligable pointer. 842 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 843 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 844 // Casting between 2 eligable pointers using Generic as an intermediary. 845 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 846 Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass); 847 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 848 SrcPtrTy, I, TII, SPIRV::StorageClass::Generic); 849 MachineBasicBlock &BB = *I.getParent(); 850 const DebugLoc &DL = I.getDebugLoc(); 851 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 852 .addDef(Tmp) 853 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 854 .addUse(SrcPtr) 855 .constrainAllUses(TII, TRI, RBI); 856 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 857 .addDef(ResVReg) 858 .addUse(GR.getSPIRVTypeID(ResType)) 859 .addUse(Tmp) 860 .constrainAllUses(TII, TRI, RBI); 861 } 862 // TODO Should this case just be disallowed completely? 863 // We're casting 2 other arbitrary address spaces, so have to bitcast. 864 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 865 } 866 867 static unsigned getFCmpOpcode(unsigned PredNum) { 868 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 869 switch (Pred) { 870 case CmpInst::FCMP_OEQ: 871 return SPIRV::OpFOrdEqual; 872 case CmpInst::FCMP_OGE: 873 return SPIRV::OpFOrdGreaterThanEqual; 874 case CmpInst::FCMP_OGT: 875 return SPIRV::OpFOrdGreaterThan; 876 case CmpInst::FCMP_OLE: 877 return SPIRV::OpFOrdLessThanEqual; 878 case CmpInst::FCMP_OLT: 879 return SPIRV::OpFOrdLessThan; 880 case CmpInst::FCMP_ONE: 881 return SPIRV::OpFOrdNotEqual; 882 case CmpInst::FCMP_ORD: 883 return SPIRV::OpOrdered; 884 case CmpInst::FCMP_UEQ: 885 return SPIRV::OpFUnordEqual; 886 case CmpInst::FCMP_UGE: 887 return SPIRV::OpFUnordGreaterThanEqual; 888 case CmpInst::FCMP_UGT: 889 return SPIRV::OpFUnordGreaterThan; 890 case CmpInst::FCMP_ULE: 891 return SPIRV::OpFUnordLessThanEqual; 892 case CmpInst::FCMP_ULT: 893 return SPIRV::OpFUnordLessThan; 894 case CmpInst::FCMP_UNE: 895 return SPIRV::OpFUnordNotEqual; 896 case CmpInst::FCMP_UNO: 897 return SPIRV::OpUnordered; 898 default: 899 llvm_unreachable("Unknown predicate type for FCmp"); 900 } 901 } 902 903 static unsigned getICmpOpcode(unsigned PredNum) { 904 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 905 switch (Pred) { 906 case CmpInst::ICMP_EQ: 907 return SPIRV::OpIEqual; 908 case CmpInst::ICMP_NE: 909 return SPIRV::OpINotEqual; 910 case CmpInst::ICMP_SGE: 911 return SPIRV::OpSGreaterThanEqual; 912 case CmpInst::ICMP_SGT: 913 return SPIRV::OpSGreaterThan; 914 case CmpInst::ICMP_SLE: 915 return SPIRV::OpSLessThanEqual; 916 case CmpInst::ICMP_SLT: 917 return SPIRV::OpSLessThan; 918 case CmpInst::ICMP_UGE: 919 return SPIRV::OpUGreaterThanEqual; 920 case CmpInst::ICMP_UGT: 921 return SPIRV::OpUGreaterThan; 922 case CmpInst::ICMP_ULE: 923 return SPIRV::OpULessThanEqual; 924 case CmpInst::ICMP_ULT: 925 return SPIRV::OpULessThan; 926 default: 927 llvm_unreachable("Unknown predicate type for ICmp"); 928 } 929 } 930 931 static unsigned getPtrCmpOpcode(unsigned Pred) { 932 switch (static_cast<CmpInst::Predicate>(Pred)) { 933 case CmpInst::ICMP_EQ: 934 return SPIRV::OpPtrEqual; 935 case CmpInst::ICMP_NE: 936 return SPIRV::OpPtrNotEqual; 937 default: 938 llvm_unreachable("Unknown predicate type for pointer comparison"); 939 } 940 } 941 942 // Return the logical operation, or abort if none exists. 943 static unsigned getBoolCmpOpcode(unsigned PredNum) { 944 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 945 switch (Pred) { 946 case CmpInst::ICMP_EQ: 947 return SPIRV::OpLogicalEqual; 948 case CmpInst::ICMP_NE: 949 return SPIRV::OpLogicalNotEqual; 950 default: 951 llvm_unreachable("Unknown predicate type for Bool comparison"); 952 } 953 } 954 955 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 956 const SPIRVType *ResType, 957 MachineInstr &I) const { 958 MachineBasicBlock &BB = *I.getParent(); 959 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 960 .addDef(ResVReg) 961 .addUse(GR.getSPIRVTypeID(ResType)) 962 .addUse(I.getOperand(1).getReg()) 963 .constrainAllUses(TII, TRI, RBI); 964 } 965 966 bool SPIRVInstructionSelector::selectConstVector(Register ResVReg, 967 const SPIRVType *ResType, 968 MachineInstr &I) const { 969 // TODO: only const case is supported for now. 970 assert(std::all_of( 971 I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) { 972 if (MO.isDef()) 973 return true; 974 if (!MO.isReg()) 975 return false; 976 SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg()); 977 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && 978 ConstTy->getOperand(1).isReg()); 979 Register ConstReg = ConstTy->getOperand(1).getReg(); 980 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); 981 assert(Const); 982 return (Const->getOpcode() == TargetOpcode::G_CONSTANT || 983 Const->getOpcode() == TargetOpcode::G_FCONSTANT); 984 })); 985 986 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 987 TII.get(SPIRV::OpConstantComposite)) 988 .addDef(ResVReg) 989 .addUse(GR.getSPIRVTypeID(ResType)); 990 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 991 MIB.addUse(I.getOperand(i).getReg()); 992 return MIB.constrainAllUses(TII, TRI, RBI); 993 } 994 995 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 996 const SPIRVType *ResType, 997 unsigned CmpOpc, 998 MachineInstr &I) const { 999 Register Cmp0 = I.getOperand(2).getReg(); 1000 Register Cmp1 = I.getOperand(3).getReg(); 1001 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 1002 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 1003 "CMP operands should have the same type"); 1004 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 1005 .addDef(ResVReg) 1006 .addUse(GR.getSPIRVTypeID(ResType)) 1007 .addUse(Cmp0) 1008 .addUse(Cmp1) 1009 .constrainAllUses(TII, TRI, RBI); 1010 } 1011 1012 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 1013 const SPIRVType *ResType, 1014 MachineInstr &I) const { 1015 auto Pred = I.getOperand(1).getPredicate(); 1016 unsigned CmpOpc; 1017 1018 Register CmpOperand = I.getOperand(2).getReg(); 1019 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 1020 CmpOpc = getPtrCmpOpcode(Pred); 1021 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 1022 CmpOpc = getBoolCmpOpcode(Pred); 1023 else 1024 CmpOpc = getICmpOpcode(Pred); 1025 return selectCmp(ResVReg, ResType, CmpOpc, I); 1026 } 1027 1028 void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB, 1029 const MachineInstr &I, 1030 int OpIdx) const { 1031 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 1032 "Expected G_FCONSTANT"); 1033 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 1034 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 1035 } 1036 1037 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 1038 const MachineInstr &I, 1039 int OpIdx) const { 1040 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 1041 "Expected G_CONSTANT"); 1042 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 1043 } 1044 1045 Register 1046 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 1047 const SPIRVType *ResType) const { 1048 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 1049 const SPIRVType *SpvI32Ty = 1050 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 1051 // Find a constant in DT or build a new one. 1052 auto ConstInt = ConstantInt::get(LLVMTy, Val); 1053 Register NewReg = GR.find(ConstInt, GR.CurMF); 1054 if (!NewReg.isValid()) { 1055 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 1056 GR.add(ConstInt, GR.CurMF, NewReg); 1057 MachineInstr *MI; 1058 MachineBasicBlock &BB = *I.getParent(); 1059 if (Val == 0) { 1060 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1061 .addDef(NewReg) 1062 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 1063 } else { 1064 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1065 .addDef(NewReg) 1066 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 1067 .addImm(APInt(32, Val).getZExtValue()); 1068 } 1069 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 1070 } 1071 return NewReg; 1072 } 1073 1074 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 1075 const SPIRVType *ResType, 1076 MachineInstr &I) const { 1077 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 1078 return selectCmp(ResVReg, ResType, CmpOp, I); 1079 } 1080 1081 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 1082 MachineInstr &I) const { 1083 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1084 return GR.getOrCreateConsIntVector(0, I, ResType, TII); 1085 return GR.getOrCreateConstInt(0, I, ResType, TII); 1086 } 1087 1088 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 1089 const SPIRVType *ResType, 1090 MachineInstr &I) const { 1091 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1092 APInt One = 1093 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 1094 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1095 return GR.getOrCreateConsIntVector(One.getZExtValue(), I, ResType, TII); 1096 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 1097 } 1098 1099 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 1100 const SPIRVType *ResType, 1101 MachineInstr &I, 1102 bool IsSigned) const { 1103 // To extend a bool, we need to use OpSelect between constants. 1104 Register ZeroReg = buildZerosVal(ResType, I); 1105 Register OneReg = buildOnesVal(IsSigned, ResType, I); 1106 bool IsScalarBool = 1107 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 1108 unsigned Opcode = 1109 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 1110 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1111 .addDef(ResVReg) 1112 .addUse(GR.getSPIRVTypeID(ResType)) 1113 .addUse(I.getOperand(1).getReg()) 1114 .addUse(OneReg) 1115 .addUse(ZeroReg) 1116 .constrainAllUses(TII, TRI, RBI); 1117 } 1118 1119 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 1120 const SPIRVType *ResType, 1121 MachineInstr &I, bool IsSigned, 1122 unsigned Opcode) const { 1123 Register SrcReg = I.getOperand(1).getReg(); 1124 // We can convert bool value directly to float type without OpConvert*ToF, 1125 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 1126 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 1127 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1128 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 1129 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 1130 const unsigned NumElts = ResType->getOperand(2).getImm(); 1131 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 1132 } 1133 SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1134 selectSelect(SrcReg, TmpType, I, false); 1135 } 1136 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 1137 } 1138 1139 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 1140 const SPIRVType *ResType, 1141 MachineInstr &I, bool IsSigned) const { 1142 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) 1143 return selectSelect(ResVReg, ResType, I, IsSigned); 1144 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1145 return selectUnOp(ResVReg, ResType, I, Opcode); 1146 } 1147 1148 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 1149 Register ResVReg, 1150 MachineInstr &I, 1151 const SPIRVType *IntTy, 1152 const SPIRVType *BoolTy) const { 1153 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 1154 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1155 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 1156 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 1157 Register Zero = buildZerosVal(IntTy, I); 1158 Register One = buildOnesVal(false, IntTy, I); 1159 MachineBasicBlock &BB = *I.getParent(); 1160 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1161 .addDef(BitIntReg) 1162 .addUse(GR.getSPIRVTypeID(IntTy)) 1163 .addUse(IntReg) 1164 .addUse(One) 1165 .constrainAllUses(TII, TRI, RBI); 1166 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1167 .addDef(ResVReg) 1168 .addUse(GR.getSPIRVTypeID(BoolTy)) 1169 .addUse(BitIntReg) 1170 .addUse(Zero) 1171 .constrainAllUses(TII, TRI, RBI); 1172 } 1173 1174 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 1175 const SPIRVType *ResType, 1176 MachineInstr &I) const { 1177 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) { 1178 Register IntReg = I.getOperand(1).getReg(); 1179 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 1180 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 1181 } 1182 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 1183 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1184 return selectUnOp(ResVReg, ResType, I, Opcode); 1185 } 1186 1187 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 1188 const SPIRVType *ResType, 1189 const APInt &Imm, 1190 MachineInstr &I) const { 1191 unsigned TyOpcode = ResType->getOpcode(); 1192 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 1193 MachineBasicBlock &BB = *I.getParent(); 1194 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 1195 Imm.isZero()) 1196 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1197 .addDef(ResVReg) 1198 .addUse(GR.getSPIRVTypeID(ResType)) 1199 .constrainAllUses(TII, TRI, RBI); 1200 if (TyOpcode == SPIRV::OpTypeInt) { 1201 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 1202 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 1203 if (Reg == ResVReg) 1204 return true; 1205 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 1206 .addDef(ResVReg) 1207 .addUse(Reg) 1208 .constrainAllUses(TII, TRI, RBI); 1209 } 1210 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1211 .addDef(ResVReg) 1212 .addUse(GR.getSPIRVTypeID(ResType)); 1213 // <=32-bit integers should be caught by the sdag pattern. 1214 assert(Imm.getBitWidth() > 32); 1215 addNumImm(Imm, MIB); 1216 return MIB.constrainAllUses(TII, TRI, RBI); 1217 } 1218 1219 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 1220 const SPIRVType *ResType, 1221 MachineInstr &I) const { 1222 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1223 .addDef(ResVReg) 1224 .addUse(GR.getSPIRVTypeID(ResType)) 1225 .constrainAllUses(TII, TRI, RBI); 1226 } 1227 1228 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1229 assert(MO.isReg()); 1230 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1231 if (TypeInst->getOpcode() != SPIRV::ASSIGN_TYPE) 1232 return false; 1233 assert(TypeInst->getOperand(1).isReg()); 1234 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1235 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 1236 } 1237 1238 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1239 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1240 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1241 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 1242 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 1243 } 1244 1245 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 1246 const SPIRVType *ResType, 1247 MachineInstr &I) const { 1248 MachineBasicBlock &BB = *I.getParent(); 1249 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 1250 .addDef(ResVReg) 1251 .addUse(GR.getSPIRVTypeID(ResType)) 1252 // object to insert 1253 .addUse(I.getOperand(3).getReg()) 1254 // composite to insert into 1255 .addUse(I.getOperand(2).getReg()); 1256 for (unsigned i = 4; i < I.getNumOperands(); i++) 1257 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1258 return MIB.constrainAllUses(TII, TRI, RBI); 1259 } 1260 1261 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 1262 const SPIRVType *ResType, 1263 MachineInstr &I) const { 1264 MachineBasicBlock &BB = *I.getParent(); 1265 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1266 .addDef(ResVReg) 1267 .addUse(GR.getSPIRVTypeID(ResType)) 1268 .addUse(I.getOperand(2).getReg()); 1269 for (unsigned i = 3; i < I.getNumOperands(); i++) 1270 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1271 return MIB.constrainAllUses(TII, TRI, RBI); 1272 } 1273 1274 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 1275 const SPIRVType *ResType, 1276 MachineInstr &I) const { 1277 if (isImm(I.getOperand(4), MRI)) 1278 return selectInsertVal(ResVReg, ResType, I); 1279 MachineBasicBlock &BB = *I.getParent(); 1280 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 1281 .addDef(ResVReg) 1282 .addUse(GR.getSPIRVTypeID(ResType)) 1283 .addUse(I.getOperand(2).getReg()) 1284 .addUse(I.getOperand(3).getReg()) 1285 .addUse(I.getOperand(4).getReg()) 1286 .constrainAllUses(TII, TRI, RBI); 1287 } 1288 1289 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 1290 const SPIRVType *ResType, 1291 MachineInstr &I) const { 1292 if (isImm(I.getOperand(3), MRI)) 1293 return selectExtractVal(ResVReg, ResType, I); 1294 MachineBasicBlock &BB = *I.getParent(); 1295 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 1296 .addDef(ResVReg) 1297 .addUse(GR.getSPIRVTypeID(ResType)) 1298 .addUse(I.getOperand(2).getReg()) 1299 .addUse(I.getOperand(3).getReg()) 1300 .constrainAllUses(TII, TRI, RBI); 1301 } 1302 1303 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 1304 const SPIRVType *ResType, 1305 MachineInstr &I) const { 1306 const bool IsGEPInBounds = I.getOperand(2).getImm(); 1307 1308 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 1309 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 1310 // we have to use Op[InBounds]AccessChain. 1311 const unsigned Opcode = STI.isVulkanEnv() 1312 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 1313 : SPIRV::OpAccessChain) 1314 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 1315 : SPIRV::OpPtrAccessChain); 1316 1317 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1318 .addDef(ResVReg) 1319 .addUse(GR.getSPIRVTypeID(ResType)) 1320 // Object to get a pointer to. 1321 .addUse(I.getOperand(3).getReg()); 1322 // Adding indices. 1323 const unsigned StartingIndex = 1324 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 1325 ? 5 1326 : 4; 1327 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 1328 Res.addUse(I.getOperand(i).getReg()); 1329 return Res.constrainAllUses(TII, TRI, RBI); 1330 } 1331 1332 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 1333 const SPIRVType *ResType, 1334 MachineInstr &I) const { 1335 MachineBasicBlock &BB = *I.getParent(); 1336 switch (cast<GIntrinsic>(I).getIntrinsicID()) { 1337 case Intrinsic::spv_load: 1338 return selectLoad(ResVReg, ResType, I); 1339 case Intrinsic::spv_store: 1340 return selectStore(I); 1341 case Intrinsic::spv_extractv: 1342 return selectExtractVal(ResVReg, ResType, I); 1343 case Intrinsic::spv_insertv: 1344 return selectInsertVal(ResVReg, ResType, I); 1345 case Intrinsic::spv_extractelt: 1346 return selectExtractElt(ResVReg, ResType, I); 1347 case Intrinsic::spv_insertelt: 1348 return selectInsertElt(ResVReg, ResType, I); 1349 case Intrinsic::spv_gep: 1350 return selectGEP(ResVReg, ResType, I); 1351 case Intrinsic::spv_unref_global: 1352 case Intrinsic::spv_init_global: { 1353 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 1354 MachineInstr *Init = I.getNumExplicitOperands() > 2 1355 ? MRI->getVRegDef(I.getOperand(2).getReg()) 1356 : nullptr; 1357 assert(MI); 1358 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 1359 } 1360 case Intrinsic::spv_undef: { 1361 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1362 .addDef(ResVReg) 1363 .addUse(GR.getSPIRVTypeID(ResType)); 1364 return MIB.constrainAllUses(TII, TRI, RBI); 1365 } 1366 case Intrinsic::spv_const_composite: { 1367 // If no values are attached, the composite is null constant. 1368 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 1369 unsigned Opcode = 1370 IsNull ? SPIRV::OpConstantNull : SPIRV::OpConstantComposite; 1371 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1372 .addDef(ResVReg) 1373 .addUse(GR.getSPIRVTypeID(ResType)); 1374 // skip type MD node we already used when generated assign.type for this 1375 if (!IsNull) { 1376 for (unsigned i = I.getNumExplicitDefs() + 1; 1377 i < I.getNumExplicitOperands(); ++i) { 1378 MIB.addUse(I.getOperand(i).getReg()); 1379 } 1380 } 1381 return MIB.constrainAllUses(TII, TRI, RBI); 1382 } 1383 case Intrinsic::spv_assign_name: { 1384 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 1385 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 1386 for (unsigned i = I.getNumExplicitDefs() + 2; 1387 i < I.getNumExplicitOperands(); ++i) { 1388 MIB.addImm(I.getOperand(i).getImm()); 1389 } 1390 return MIB.constrainAllUses(TII, TRI, RBI); 1391 } 1392 case Intrinsic::spv_switch: { 1393 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 1394 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 1395 if (I.getOperand(i).isReg()) 1396 MIB.addReg(I.getOperand(i).getReg()); 1397 else if (I.getOperand(i).isCImm()) 1398 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 1399 else if (I.getOperand(i).isMBB()) 1400 MIB.addMBB(I.getOperand(i).getMBB()); 1401 else 1402 llvm_unreachable("Unexpected OpSwitch operand"); 1403 } 1404 return MIB.constrainAllUses(TII, TRI, RBI); 1405 } 1406 case Intrinsic::spv_cmpxchg: 1407 return selectAtomicCmpXchg(ResVReg, ResType, I); 1408 case Intrinsic::spv_unreachable: 1409 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); 1410 break; 1411 case Intrinsic::spv_alloca: 1412 return selectFrameIndex(ResVReg, ResType, I); 1413 case Intrinsic::spv_assume: 1414 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 1415 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 1416 .addUse(I.getOperand(1).getReg()); 1417 break; 1418 case Intrinsic::spv_expect: 1419 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 1420 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 1421 .addDef(ResVReg) 1422 .addUse(GR.getSPIRVTypeID(ResType)) 1423 .addUse(I.getOperand(2).getReg()) 1424 .addUse(I.getOperand(3).getReg()); 1425 break; 1426 default: 1427 llvm_unreachable("Intrinsic selection not implemented"); 1428 } 1429 return true; 1430 } 1431 1432 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 1433 const SPIRVType *ResType, 1434 MachineInstr &I) const { 1435 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1436 .addDef(ResVReg) 1437 .addUse(GR.getSPIRVTypeID(ResType)) 1438 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 1439 .constrainAllUses(TII, TRI, RBI); 1440 } 1441 1442 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 1443 // InstructionSelector walks backwards through the instructions. We can use 1444 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 1445 // first, so can generate an OpBranchConditional here. If there is no 1446 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 1447 const MachineInstr *PrevI = I.getPrevNode(); 1448 MachineBasicBlock &MBB = *I.getParent(); 1449 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 1450 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 1451 .addUse(PrevI->getOperand(0).getReg()) 1452 .addMBB(PrevI->getOperand(1).getMBB()) 1453 .addMBB(I.getOperand(0).getMBB()) 1454 .constrainAllUses(TII, TRI, RBI); 1455 } 1456 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 1457 .addMBB(I.getOperand(0).getMBB()) 1458 .constrainAllUses(TII, TRI, RBI); 1459 } 1460 1461 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 1462 // InstructionSelector walks backwards through the instructions. For an 1463 // explicit conditional branch with no fallthrough, we use both a G_BR and a 1464 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 1465 // generate the OpBranchConditional in selectBranch above. 1466 // 1467 // If an OpBranchConditional has been generated, we simply return, as the work 1468 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 1469 // implicit fallthrough to the next basic block, so we need to create an 1470 // OpBranchConditional with an explicit "false" argument pointing to the next 1471 // basic block that LLVM would fall through to. 1472 const MachineInstr *NextI = I.getNextNode(); 1473 // Check if this has already been successfully selected. 1474 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 1475 return true; 1476 // Must be relying on implicit block fallthrough, so generate an 1477 // OpBranchConditional with the "next" basic block as the "false" target. 1478 MachineBasicBlock &MBB = *I.getParent(); 1479 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 1480 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 1481 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 1482 .addUse(I.getOperand(0).getReg()) 1483 .addMBB(I.getOperand(1).getMBB()) 1484 .addMBB(NextMBB) 1485 .constrainAllUses(TII, TRI, RBI); 1486 } 1487 1488 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 1489 const SPIRVType *ResType, 1490 MachineInstr &I) const { 1491 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 1492 .addDef(ResVReg) 1493 .addUse(GR.getSPIRVTypeID(ResType)); 1494 const unsigned NumOps = I.getNumOperands(); 1495 for (unsigned i = 1; i < NumOps; i += 2) { 1496 MIB.addUse(I.getOperand(i + 0).getReg()); 1497 MIB.addMBB(I.getOperand(i + 1).getMBB()); 1498 } 1499 return MIB.constrainAllUses(TII, TRI, RBI); 1500 } 1501 1502 bool SPIRVInstructionSelector::selectGlobalValue( 1503 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 1504 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 1505 MachineIRBuilder MIRBuilder(I); 1506 const GlobalValue *GV = I.getOperand(1).getGlobal(); 1507 Type *GVType = GV->getValueType(); 1508 SPIRVType *PointerBaseType; 1509 if (GVType->isArrayTy()) { 1510 SPIRVType *ArrayElementType = 1511 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 1512 SPIRV::AccessQualifier::ReadWrite, false); 1513 PointerBaseType = GR.getOrCreateSPIRVArrayType( 1514 ArrayElementType, GVType->getArrayNumElements(), I, TII); 1515 } else { 1516 PointerBaseType = GR.getOrCreateSPIRVType( 1517 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1518 } 1519 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 1520 PointerBaseType, I, TII, 1521 addressSpaceToStorageClass(GV->getAddressSpace())); 1522 std::string GlobalIdent = GV->getGlobalIdentifier(); 1523 // We have functions as operands in tests with blocks of instruction e.g. in 1524 // transcoding/global_block.ll. These operands are not used and should be 1525 // substituted by zero constants. Their type is expected to be always 1526 // OpTypePointer Function %uchar. 1527 if (isa<Function>(GV)) { 1528 const Constant *ConstVal = GV; 1529 MachineBasicBlock &BB = *I.getParent(); 1530 Register NewReg = GR.find(ConstVal, GR.CurMF); 1531 if (!NewReg.isValid()) { 1532 Register NewReg = ResVReg; 1533 GR.add(ConstVal, GR.CurMF, NewReg); 1534 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1535 .addDef(NewReg) 1536 .addUse(GR.getSPIRVTypeID(ResType)) 1537 .constrainAllUses(TII, TRI, RBI); 1538 } 1539 assert(NewReg != ResVReg); 1540 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 1541 .addDef(ResVReg) 1542 .addUse(NewReg) 1543 .constrainAllUses(TII, TRI, RBI); 1544 } 1545 auto GlobalVar = cast<GlobalVariable>(GV); 1546 assert(GlobalVar->getName() != "llvm.global.annotations"); 1547 1548 bool HasInit = GlobalVar->hasInitializer() && 1549 !isa<UndefValue>(GlobalVar->getInitializer()); 1550 // Skip empty declaration for GVs with initilaizers till we get the decl with 1551 // passed initializer. 1552 if (HasInit && !Init) 1553 return true; 1554 1555 unsigned AddrSpace = GV->getAddressSpace(); 1556 SPIRV::StorageClass::StorageClass Storage = 1557 addressSpaceToStorageClass(AddrSpace); 1558 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 1559 Storage != SPIRV::StorageClass::Function; 1560 SPIRV::LinkageType::LinkageType LnkType = 1561 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 1562 ? SPIRV::LinkageType::Import 1563 : SPIRV::LinkageType::Export; 1564 1565 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 1566 Storage, Init, GlobalVar->isConstant(), 1567 HasLnkTy, LnkType, MIRBuilder, true); 1568 return Reg.isValid(); 1569 } 1570 1571 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 1572 const SPIRVType *ResType, 1573 MachineInstr &I) const { 1574 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 1575 return selectExtInst(ResVReg, ResType, I, CL::log10); 1576 } 1577 1578 // There is no log10 instruction in the GLSL Extended Instruction set, so it 1579 // is implemented as: 1580 // log10(x) = log2(x) * (1 / log2(10)) 1581 // = log2(x) * 0.30103 1582 1583 MachineIRBuilder MIRBuilder(I); 1584 MachineBasicBlock &BB = *I.getParent(); 1585 1586 // Build log2(x). 1587 Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1588 bool Result = 1589 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1590 .addDef(VarReg) 1591 .addUse(GR.getSPIRVTypeID(ResType)) 1592 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1593 .addImm(GL::Log2) 1594 .add(I.getOperand(1)) 1595 .constrainAllUses(TII, TRI, RBI); 1596 1597 // Build 0.30103. 1598 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 1599 ResType->getOpcode() == SPIRV::OpTypeFloat); 1600 // TODO: Add matrix implementation once supported by the HLSL frontend. 1601 const SPIRVType *SpirvScalarType = 1602 ResType->getOpcode() == SPIRV::OpTypeVector 1603 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 1604 : ResType; 1605 Register ScaleReg = 1606 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 1607 1608 // Multiply log2(x) by 0.30103 to get log10(x) result. 1609 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 1610 ? SPIRV::OpVectorTimesScalar 1611 : SPIRV::OpFMulS; 1612 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1613 .addDef(ResVReg) 1614 .addUse(GR.getSPIRVTypeID(ResType)) 1615 .addUse(VarReg) 1616 .addUse(ScaleReg) 1617 .constrainAllUses(TII, TRI, RBI); 1618 1619 return Result; 1620 } 1621 1622 namespace llvm { 1623 InstructionSelector * 1624 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 1625 const SPIRVSubtarget &Subtarget, 1626 const RegisterBankInfo &RBI) { 1627 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 1628 } 1629 } // namespace llvm 1630