Lines Matching full:sl
1964 const SDLoc &SL, in lowerKernArgParameterPtr() argument
1982 return DAG.getConstant(Offset, SL, PtrVT); in lowerKernArgParameterPtr()
1985 SDValue BasePtr = DAG.getCopyFromReg(Chain, SL, in lowerKernArgParameterPtr()
1988 return DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Offset)); in lowerKernArgParameterPtr()
1992 const SDLoc &SL) const { in getImplicitArgPtr()
1995 return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset); in getImplicitArgPtr()
1999 const SDLoc &SL) const { in getLDSKernelId()
2005 return DAG.getConstant(*KnownSize, SL, MVT::i32); in getLDSKernelId()
2010 const SDLoc &SL, SDValue Val, in convertArgType() argument
2019 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val, in convertArgType()
2020 DAG.getConstant(0, SL, MVT::i32)); in convertArgType()
2027 Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT)); in convertArgType()
2031 Val = getFPExtOrFPRound(DAG, Val, SL, VT); in convertArgType()
2033 Val = DAG.getSExtOrTrunc(Val, SL, VT); in convertArgType()
2035 Val = DAG.getZExtOrTrunc(Val, SL, VT); in convertArgType()
2041 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain, in lowerKernargMemParameter() argument
2058 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset); in lowerKernargMemParameter()
2059 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4), in lowerKernargMemParameter()
2063 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32); in lowerKernargMemParameter()
2064 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt); in lowerKernargMemParameter()
2066 SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract); in lowerKernargMemParameter()
2067 ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal); in lowerKernargMemParameter()
2068 ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg); in lowerKernargMemParameter()
2071 return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL); in lowerKernargMemParameter()
2074 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset); in lowerKernargMemParameter()
2075 SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment, in lowerKernargMemParameter()
2079 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg); in lowerKernargMemParameter()
2080 return DAG.getMergeValues({ Val, Load.getValue(1) }, SL); in lowerKernargMemParameter()
2084 const SDLoc &SL, SDValue Chain, in lowerStackParameter() argument
2126 ExtType, SL, VA.getLocVT(), Chain, FIN, in lowerStackParameter()
3441 SDLoc SL; in passSpecialInputs() local
3460 Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y, in passSpecialInputs()
3461 DAG.getShiftAmountConstant(10, MVT::i32, SL)); in passSpecialInputs()
3463 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y; in passSpecialInputs()
3469 Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z, in passSpecialInputs()
3470 DAG.getShiftAmountConstant(20, MVT::i32, SL)); in passSpecialInputs()
3472 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z; in passSpecialInputs()
4024 SDLoc SL(Op); in LowerSTACKSAVE() local
4026 SDValue CopyFromSP = DAG.getCopyFromReg(Op->getOperand(0), SL, SP, MVT::i32); in LowerSTACKSAVE()
4032 DAG.getNode(AMDGPUISD::WAVE_ADDRESS, SL, MVT::i32, CopyFromSP); in LowerSTACKSAVE()
4033 return DAG.getMergeValues({VectorAddress, CopyFromSP.getValue(1)}, SL); in LowerSTACKSAVE()
4038 SDLoc SL(Op); in lowerGET_ROUNDING() local
4043 SDValue GetRoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32); in lowerGET_ROUNDING()
4046 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32); in lowerGET_ROUNDING()
4047 SDValue GetReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, Op->getVTList(), in lowerGET_ROUNDING()
4079 DAG.getConstant(AMDGPU::FltRoundConversionTable, SL, MVT::i64); in lowerGET_ROUNDING()
4081 SDValue Two = DAG.getConstant(2, SL, MVT::i32); in lowerGET_ROUNDING()
4083 DAG.getNode(ISD::SHL, SL, MVT::i32, GetReg, Two); in lowerGET_ROUNDING()
4088 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits); in lowerGET_ROUNDING()
4089 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue); in lowerGET_ROUNDING()
4091 SDValue EntryMask = DAG.getConstant(0xf, SL, MVT::i32); in lowerGET_ROUNDING()
4093 DAG.getNode(ISD::AND, SL, MVT::i32, TruncTable, EntryMask); in lowerGET_ROUNDING()
4097 SDValue Four = DAG.getConstant(4, SL, MVT::i32); in lowerGET_ROUNDING()
4099 DAG.getSetCC(SL, MVT::i1, TableEntry, Four, ISD::SETULT); in lowerGET_ROUNDING()
4100 SDValue EnumOffset = DAG.getNode(ISD::ADD, SL, MVT::i32, TableEntry, Four); in lowerGET_ROUNDING()
4101 SDValue Result = DAG.getNode(ISD::SELECT, SL, MVT::i32, IsStandardValue, in lowerGET_ROUNDING()
4104 return DAG.getMergeValues({Result, GetReg.getValue(1)}, SL); in lowerGET_ROUNDING()
4109 SDLoc SL(Op); in lowerSET_ROUNDING() local
4121 AMDGPU::decodeFltRoundToHWConversionTable(ClampedVal), SL, MVT::i32); in lowerSET_ROUNDING()
4133 AMDGPU::FltRoundToHWConversionTable & 0xffff, SL, MVT::i32); in lowerSET_ROUNDING()
4135 SDValue Two = DAG.getConstant(2, SL, MVT::i32); in lowerSET_ROUNDING()
4137 DAG.getNode(ISD::SHL, SL, MVT::i32, NewMode, Two); in lowerSET_ROUNDING()
4140 DAG.getNode(ISD::SRL, SL, MVT::i32, BitTable, RoundModeTimesNumBits); in lowerSET_ROUNDING()
4148 DAG.getConstant(AMDGPU::FltRoundToHWConversionTable, SL, MVT::i64); in lowerSET_ROUNDING()
4150 SDValue Four = DAG.getConstant(4, SL, MVT::i32); in lowerSET_ROUNDING()
4151 SDValue OffsetEnum = DAG.getNode(ISD::SUB, SL, MVT::i32, NewMode, Four); in lowerSET_ROUNDING()
4153 DAG.getNode(ISD::UMIN, SL, MVT::i32, NewMode, OffsetEnum); in lowerSET_ROUNDING()
4155 SDValue Two = DAG.getConstant(2, SL, MVT::i32); in lowerSET_ROUNDING()
4157 DAG.getNode(ISD::SHL, SL, MVT::i32, IndexVal, Two); in lowerSET_ROUNDING()
4160 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits); in lowerSET_ROUNDING()
4161 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue); in lowerSET_ROUNDING()
4172 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32); in lowerSET_ROUNDING()
4173 NewMode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, in lowerSET_ROUNDING()
4180 DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32); in lowerSET_ROUNDING()
4183 SDValue RoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32); in lowerSET_ROUNDING()
4186 DAG.getNode(ISD::INTRINSIC_VOID, SL, Op->getVTList(), Op.getOperand(0), in lowerSET_ROUNDING()
4218 SDLoc SL(Op); in lowerFP_EXTEND() local
4220 DAG.getNode(ISD::BITCAST, SL, SrcVT.changeTypeToInteger(), Src); in lowerFP_EXTEND()
4226 return DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast); in lowerFP_EXTEND()
4230 SDLoc SL(Op); in lowerGET_FPENV() local
4236 SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32); in lowerGET_FPENV()
4239 SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32); in lowerGET_FPENV()
4243 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32); in lowerGET_FPENV()
4244 SDValue GetModeReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList, in lowerGET_FPENV()
4246 SDValue GetTrapReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList, in lowerGET_FPENV()
4249 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, GetModeReg.getValue(1), in lowerGET_FPENV()
4253 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, GetModeReg, GetTrapReg); in lowerGET_FPENV()
4254 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr); in lowerGET_FPENV()
4256 return DAG.getMergeValues({Result, TokenReg}, SL); in lowerGET_FPENV()
4260 SDLoc SL(Op); in lowerSET_FPENV() local
4264 SDValue Input = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op.getOperand(1)); in lowerSET_FPENV()
4265 SDValue NewModeReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input, in lowerSET_FPENV()
4266 DAG.getConstant(0, SL, MVT::i32)); in lowerSET_FPENV()
4267 SDValue NewTrapReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input, in lowerSET_FPENV()
4268 DAG.getConstant(1, SL, MVT::i32)); in lowerSET_FPENV()
4271 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32); in lowerSET_FPENV()
4272 NewModeReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, in lowerSET_FPENV()
4274 NewTrapReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, in lowerSET_FPENV()
4279 SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32); in lowerSET_FPENV()
4282 SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32); in lowerSET_FPENV()
4285 DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32); in lowerSET_FPENV()
4287 DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0), in lowerSET_FPENV()
4290 DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0), in lowerSET_FPENV()
4292 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, SetTrapReg, SetModeReg); in lowerSET_FPENV()
5684 SDLoc SL(Op); in splitUnaryVectorOp() local
5685 SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo, in splitUnaryVectorOp()
5687 SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi, in splitUnaryVectorOp()
5709 SDLoc SL(Op); in splitBinaryVectorOp() local
5711 SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, in splitBinaryVectorOp()
5713 SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, in splitBinaryVectorOp()
5740 SDLoc SL(Op); in splitTernaryVectorOp() local
5743 SDValue OpLo = DAG.getNode(Opc, SL, ResVT.first, Lo0, Lo1, Lo2, in splitTernaryVectorOp()
5745 SDValue OpHi = DAG.getNode(Opc, SL, ResVT.second, Hi0, Hi1, Hi2, in splitTernaryVectorOp()
6058 SDLoc SL(N); in lowerFCMPIntrinsic() local
6061 Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in lowerFCMPIntrinsic()
6062 Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in lowerFCMPIntrinsic()
6069 SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0, in lowerFCMPIntrinsic()
6073 return DAG.getZExtOrTrunc(SetCC, SL, VT); in lowerFCMPIntrinsic()
6080 SDLoc SL(N); in lowerBALLOTIntrinsic() local
6084 return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Src.getOperand(0), in lowerBALLOTIntrinsic()
6090 return DAG.getConstant(0, SL, VT); in lowerBALLOTIntrinsic()
6102 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT); in lowerBALLOTIntrinsic()
6109 AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32), in lowerBALLOTIntrinsic()
6110 DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE)); in lowerBALLOTIntrinsic()
6120 SDLoc SL(N); in lowerLaneOp() local
6123 auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1, in lowerLaneOp()
6147 Operands.push_back(DAG.getTargetConstant(IID, SL, MVT::i32)); in lowerLaneOp()
6153 Operands.push_back(DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue, in lowerLaneOp()
6157 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, ValT, Operands); in lowerLaneOp()
6177 SL, MVT::i32); in lowerLaneOp()
6181 SL, MVT::i32); in lowerLaneOp()
6186 SL, MVT::i32); in lowerLaneOp()
6190 SDValue Trunc = DAG.getAnyExtOrTrunc(LaneOp, SL, IntVT); in lowerLaneOp()
6197 auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue { in lowerLaneOp()
6217 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, OperandEltVT, in lowerLaneOp()
6218 Operand, DAG.getVectorIdxConstant(i, SL)); in lowerLaneOp()
6227 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue, in lowerLaneOp()
6230 Scalars.push_back(DAG.getNode(N->getOpcode(), SL, EltVT, Operands)); in lowerLaneOp()
6234 return DAG.getBuildVector(VecVT, SL, Scalars); in lowerLaneOp()
6252 Src0SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src0, in lowerLaneOp()
6253 DAG.getConstant(EltIdx, SL, MVT::i32)); in lowerLaneOp()
6256 Src1SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src1, in lowerLaneOp()
6257 DAG.getConstant(EltIdx, SL, MVT::i32)); in lowerLaneOp()
6260 Src2SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src2, in lowerLaneOp()
6261 DAG.getConstant(EltIdx, SL, MVT::i32)); in lowerLaneOp()
6269 return DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, Pieces); in lowerLaneOp()
6314 SDLoc SL(N); in ReplaceNodeResults() local
6315 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, in ReplaceNodeResults()
6317 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt)); in ReplaceNodeResults()
6326 SDLoc SL(N); in ReplaceNodeResults() local
6340 Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1)); in ReplaceNodeResults()
6342 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1); in ReplaceNodeResults()
6343 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt)); in ReplaceNodeResults()
6416 SDLoc SL(N); in ReplaceNodeResults() local
6419 SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1)); in ReplaceNodeResults()
6420 SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2)); in ReplaceNodeResults()
6424 LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS); in ReplaceNodeResults()
6425 RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS); in ReplaceNodeResults()
6429 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, SelectVT, in ReplaceNodeResults()
6433 NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect); in ReplaceNodeResults()
6434 Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect)); in ReplaceNodeResults()
6441 SDLoc SL(N); in ReplaceNodeResults() local
6442 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
6444 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, in ReplaceNodeResults()
6446 DAG.getConstant(0x80008000, SL, MVT::i32)); in ReplaceNodeResults()
6447 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
6454 SDLoc SL(N); in ReplaceNodeResults() local
6455 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
6457 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, in ReplaceNodeResults()
6459 DAG.getConstant(0x7fff7fff, SL, MVT::i32)); in ReplaceNodeResults()
6460 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
6792 SDLoc SL(Op); in lowerMUL() local
6795 DAG.getMachineNode(AMDGPU::S_MUL_U64_U32_PSEUDO, SL, VT, Op0, Op1), 0); in lowerMUL()
6800 DAG.getMachineNode(AMDGPU::S_MUL_I64_I32_PSEUDO, SL, VT, Op0, Op1), 0); in lowerMUL()
6807 SDLoc SL(Op); in lowerXMULO() local
6818 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32); in lowerXMULO()
6819 SDValue Result = DAG.getNode(ISD::SHL, SL, VT, LHS, ShiftAmt); in lowerXMULO()
6820 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, in lowerXMULO()
6822 SL, VT, Result, ShiftAmt), in lowerXMULO()
6824 return DAG.getMergeValues({ Result, Overflow }, SL); in lowerXMULO()
6828 SDValue Result = DAG.getNode(ISD::MUL, SL, VT, LHS, RHS); in lowerXMULO()
6830 SL, VT, LHS, RHS); in lowerXMULO()
6833 ? DAG.getNode(ISD::SRA, SL, VT, Result, in lowerXMULO()
6834 DAG.getConstant(VT.getScalarSizeInBits() - 1, SL, MVT::i32)) in lowerXMULO()
6835 : DAG.getConstant(0, SL, VT); in lowerXMULO()
6836 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE); in lowerXMULO()
6838 return DAG.getMergeValues({ Result, Overflow }, SL); in lowerXMULO()
6867 SDLoc SL(Op); in lowerTrapEndpgm() local
6869 return DAG.getNode(AMDGPUISD::ENDPGM_TRAP, SL, MVT::Other, Chain); in lowerTrapEndpgm()
6885 SDLoc SL(Op); in lowerTrapHsaQueuePtr() local
6893 loadImplicitKernelArgument(DAG, MVT::i64, SL, Align(8), QUEUE_PTR); in lowerTrapHsaQueuePtr()
6903 QueuePtr = DAG.getConstant(0, SL, MVT::i64); in lowerTrapHsaQueuePtr()
6911 SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01, in lowerTrapHsaQueuePtr()
6917 DAG.getTargetConstant(TrapID, SL, MVT::i16), in lowerTrapHsaQueuePtr()
6921 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTrapHsaQueuePtr()
6926 SDLoc SL(Op); in lowerTrapHsa() local
6932 return DAG.getNode(AMDGPUISD::SIMULATED_TRAP, SL, MVT::Other, Chain); in lowerTrapHsa()
6937 DAG.getTargetConstant(TrapID, SL, MVT::i16) in lowerTrapHsa()
6939 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTrapHsa()
6943 SDLoc SL(Op); in lowerDEBUGTRAP() local
6961 DAG.getTargetConstant(TrapID, SL, MVT::i16) in lowerDEBUGTRAP()
6963 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerDEBUGTRAP()
7053 SDLoc SL(Op); in lowerADDRSPACECAST() local
7075 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64); in lowerADDRSPACECAST()
7081 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
7087 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
7088 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
7090 return DAG.getNode(ISD::SELECT, SL, MVT::i32, NonNull, Ptr, in lowerADDRSPACECAST()
7100 SDValue Aperture = getSegmentAperture(SrcAS, SL, DAG); in lowerADDRSPACECAST()
7102 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture); in lowerADDRSPACECAST()
7103 CvtPtr = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr); in lowerADDRSPACECAST()
7109 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
7112 = DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
7114 return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, CvtPtr, in lowerADDRSPACECAST()
7123 SDValue Hi = DAG.getConstant(Info->get32BitAddressHighBits(), SL, MVT::i32); in lowerADDRSPACECAST()
7124 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Hi); in lowerADDRSPACECAST()
7125 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in lowerADDRSPACECAST()
7130 return DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
7136 MF.getFunction(), "invalid addrspacecast", SL.getDebugLoc()); in lowerADDRSPACECAST()
7157 SDLoc SL(Op); in lowerINSERT_SUBVECTOR() local
7170 Vec = DAG.getNode(ISD::BITCAST, SL, NewVecVT, Vec); in lowerINSERT_SUBVECTOR()
7171 Ins = DAG.getNode(ISD::BITCAST, SL, NewInsVT, Ins); in lowerINSERT_SUBVECTOR()
7178 Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Ins, in lowerINSERT_SUBVECTOR()
7179 DAG.getConstant(I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
7181 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NewVecVT, Vec, Elt, in lowerINSERT_SUBVECTOR()
7182 DAG.getConstant(IdxVal / 2 + I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
7185 return DAG.getNode(ISD::BITCAST, SL, VecVT, Vec); in lowerINSERT_SUBVECTOR()
7189 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins, in lowerINSERT_SUBVECTOR()
7190 DAG.getConstant(I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
7191 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, VecVT, Vec, Elt, in lowerINSERT_SUBVECTOR()
7192 DAG.getConstant(IdxVal + I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
7206 SDLoc SL(Op); in lowerINSERT_VECTOR_ELT() local
7212 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec); in lowerINSERT_VECTOR_ELT()
7214 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
7215 DAG.getConstant(0, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
7216 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
7217 DAG.getConstant(1, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
7219 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf); in lowerINSERT_VECTOR_ELT()
7220 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf); in lowerINSERT_VECTOR_ELT()
7224 SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16, in lowerINSERT_VECTOR_ELT()
7226 DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal), in lowerINSERT_VECTOR_ELT()
7227 DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
7229 InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf); in lowerINSERT_VECTOR_ELT()
7232 DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) : in lowerINSERT_VECTOR_ELT()
7233 DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf }); in lowerINSERT_VECTOR_ELT()
7235 return DAG.getNode(ISD::BITCAST, SL, VecVT, Concat); in lowerINSERT_VECTOR_ELT()
7253 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerINSERT_VECTOR_ELT()
7254 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerINSERT_VECTOR_ELT()
7255 SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT, in lowerINSERT_VECTOR_ELT()
7256 DAG.getConstant(EltMask, SL, IntVT), ScaledIdx); in lowerINSERT_VECTOR_ELT()
7259 SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT, in lowerINSERT_VECTOR_ELT()
7260 DAG.getSplatBuildVector(VecVT, SL, InsVal)); in lowerINSERT_VECTOR_ELT()
7263 SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal); in lowerINSERT_VECTOR_ELT()
7266 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec); in lowerINSERT_VECTOR_ELT()
7267 SDValue RHS = DAG.getNode(ISD::AND, SL, IntVT, in lowerINSERT_VECTOR_ELT()
7268 DAG.getNOT(SL, BFM, IntVT), BCVec); in lowerINSERT_VECTOR_ELT()
7271 SDValue BFI = DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS); in lowerINSERT_VECTOR_ELT()
7273 return DAG.getNode(ISD::BITCAST, SL, VecVT, BFI); in lowerINSERT_VECTOR_ELT()
7278 SDLoc SL(Op); in lowerEXTRACT_VECTOR_ELT() local
7304 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
7305 DAG.getConstant(0, SL, MVT::i32))); in lowerEXTRACT_VECTOR_ELT()
7307 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
7308 DAG.getConstant(1, SL, MVT::i32))); in lowerEXTRACT_VECTOR_ELT()
7313 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
7314 DAG.getConstant(P, SL, MVT::i32)); in lowerEXTRACT_VECTOR_ELT()
7317 Lo = DAG.getBitcast(LoVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64, in lowerEXTRACT_VECTOR_ELT()
7319 Hi = DAG.getBitcast(HiVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64, in lowerEXTRACT_VECTOR_ELT()
7327 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
7328 DAG.getConstant(P, SL, MVT::i32)); in lowerEXTRACT_VECTOR_ELT()
7332 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64, in lowerEXTRACT_VECTOR_ELT()
7335 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64, in lowerEXTRACT_VECTOR_ELT()
7342 SDValue IdxMask = DAG.getConstant(NElem / 2 - 1, SL, IdxVT); in lowerEXTRACT_VECTOR_ELT()
7343 SDValue NewIdx = DAG.getNode(ISD::AND, SL, IdxVT, Idx, IdxMask); in lowerEXTRACT_VECTOR_ELT()
7344 SDValue Half = DAG.getSelectCC(SL, Idx, IdxMask, Hi, Lo, ISD::SETUGT); in lowerEXTRACT_VECTOR_ELT()
7345 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Half, NewIdx); in lowerEXTRACT_VECTOR_ELT()
7357 Vec = DAG.getAnyExtOrTrunc(Src, SL, IntVT); in lowerEXTRACT_VECTOR_ELT()
7363 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerEXTRACT_VECTOR_ELT()
7366 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerEXTRACT_VECTOR_ELT()
7368 SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec); in lowerEXTRACT_VECTOR_ELT()
7369 SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx); in lowerEXTRACT_VECTOR_ELT()
7372 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt); in lowerEXTRACT_VECTOR_ELT()
7373 return DAG.getNode(ISD::BITCAST, SL, ResultVT, Result); in lowerEXTRACT_VECTOR_ELT()
7376 return DAG.getAnyExtOrTrunc(Elt, SL, ResultVT); in lowerEXTRACT_VECTOR_ELT()
7386 SDLoc SL(Op); in lowerVECTOR_SHUFFLE() local
7410 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, in lowerVECTOR_SHUFFLE()
7412 DAG.getConstant(EltIdx, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
7423 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, in lowerVECTOR_SHUFFLE()
7424 Vec0, DAG.getConstant(EltIdx0, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
7427 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, in lowerVECTOR_SHUFFLE()
7428 Vec1, DAG.getConstant(EltIdx1, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
7429 Pieces.push_back(DAG.getBuildVector(PackVT, SL, { Elt0, Elt1 })); in lowerVECTOR_SHUFFLE()
7433 return DAG.getNode(ISD::CONCAT_VECTORS, SL, ResultVT, Pieces); in lowerVECTOR_SHUFFLE()
7442 SDLoc SL(Op); in lowerSCALAR_TO_VECTOR() local
7449 return DAG.getBuildVector(ResultVT, SL, VElts); in lowerSCALAR_TO_VECTOR()
7454 SDLoc SL(Op); in lowerBUILD_VECTOR() local
7470 SDValue Lo = DAG.getBuildVector(HalfVT, SL, LoOps); in lowerBUILD_VECTOR()
7471 SDValue Hi = DAG.getBuildVector(HalfVT, SL, HiOps); in lowerBUILD_VECTOR()
7473 SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, HalfIntVT, Lo); in lowerBUILD_VECTOR()
7474 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, HalfIntVT, Hi); in lowerBUILD_VECTOR()
7476 SDValue Blend = DAG.getBuildVector(MVT::getVectorVT(HalfIntVT, 2), SL, in lowerBUILD_VECTOR()
7478 return DAG.getNode(ISD::BITCAST, SL, VT, Blend); in lowerBUILD_VECTOR()
7493 SDValue Vec = DAG.getBuildVector(QuarterVT, SL, Parts[P]); in lowerBUILD_VECTOR()
7494 Casts[P] = DAG.getNode(ISD::BITCAST, SL, QuarterIntVT, Vec); in lowerBUILD_VECTOR()
7498 DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 4), SL, Casts); in lowerBUILD_VECTOR()
7499 return DAG.getNode(ISD::BITCAST, SL, VT, Blend); in lowerBUILD_VECTOR()
7514 SDValue Vec = DAG.getBuildVector(QuarterVT, SL, Parts[P]); in lowerBUILD_VECTOR()
7515 Casts[P] = DAG.getNode(ISD::BITCAST, SL, QuarterIntVT, Vec); in lowerBUILD_VECTOR()
7519 DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 8), SL, Casts); in lowerBUILD_VECTOR()
7520 return DAG.getNode(ISD::BITCAST, SL, VT, Blend); in lowerBUILD_VECTOR()
7531 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
7532 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
7533 return DAG.getNode(ISD::BITCAST, SL, VT, ExtLo); in lowerBUILD_VECTOR()
7536 Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi); in lowerBUILD_VECTOR()
7537 Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi); in lowerBUILD_VECTOR()
7539 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi, in lowerBUILD_VECTOR()
7540 DAG.getConstant(16, SL, MVT::i32)); in lowerBUILD_VECTOR()
7542 return DAG.getNode(ISD::BITCAST, SL, VT, ShlHi); in lowerBUILD_VECTOR()
7544 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
7545 Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
7547 SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi); in lowerBUILD_VECTOR()
7548 return DAG.getNode(ISD::BITCAST, SL, VT, Or); in lowerBUILD_VECTOR()
7701 SDLoc SL(Op); in lowerImplicitZextParam() local
7703 DAG, MVT::i32, MVT::i32, SL, DAG.getEntryNode(), Offset, Align(4), false); in lowerImplicitZextParam()
7705 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param, in lowerImplicitZextParam()
8364 SDLoc SL(Op); in lowerWaveID() local
8366 SDValue TTMP8 = DAG.getCopyFromReg(DAG.getEntryNode(), SL, AMDGPU::TTMP8, VT); in lowerWaveID()
8367 return DAG.getNode(AMDGPUISD::BFE_U32, SL, VT, TTMP8, in lowerWaveID()
8368 DAG.getConstant(25, SL, VT), DAG.getConstant(5, SL, VT)); in lowerWaveID()
8374 SDLoc SL(Op); in lowerWorkitemID() local
8378 return DAG.getConstant(0, SL, MVT::i32); in lowerWorkitemID()
8392 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Val, in lowerWorkitemID()
8701 SDLoc SL(Op); in LowerINTRINSIC_WO_CHAIN() local
8704 SDValue Aperture = getSegmentAperture(AS, SL, DAG); in LowerINTRINSIC_WO_CHAIN()
8708 SDValue SrcHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, SrcVec, in LowerINTRINSIC_WO_CHAIN()
8709 DAG.getConstant(1, SL, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
8710 return DAG.getSetCC(SL, MVT::i1, SrcHi, Aperture, ISD::SETEQ); in LowerINTRINSIC_WO_CHAIN()
8736 SDLoc SL(Op); in LowerINTRINSIC_WO_CHAIN() local
8737 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(4), SL, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8738 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
8748 SDLoc SL(Op); in LowerINTRINSIC_WO_CHAIN() local
8749 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(6), SL, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8750 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
10154 const SDLoc &SL, EVT VT) { in getLoadExtOrTrunc() argument
10156 return DAG.getNode(ISD::TRUNCATE, SL, VT, Op); in getLoadExtOrTrunc()
10160 return DAG.getNode(ISD::SIGN_EXTEND, SL, VT, Op); in getLoadExtOrTrunc()
10162 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, Op); in getLoadExtOrTrunc()
10164 return DAG.getNode(ISD::ANY_EXTEND, SL, VT, Op); in getLoadExtOrTrunc()
10194 SDLoc SL(Ld); in widenLoad() local
10202 ISD::UNINDEXED, ISD::NON_EXTLOAD, MVT::i32, SL, Ld->getChain(), Ptr, in widenLoad()
10216 Cvt = DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, NewLoad, in widenLoad()
10220 Cvt = DAG.getZeroExtendInReg(NewLoad, SL, TruncVT); in widenLoad()
10232 Cvt = getLoadExtOrTrunc(DAG, Ld->getExtensionType(), Cvt, SL, IntVT); in widenLoad()
10236 Cvt = DAG.getNode(ISD::BITCAST, SL, VT, Cvt); in widenLoad()
10238 return DAG.getMergeValues({ Cvt, NewLoad.getValue(1) }, SL); in widenLoad()
10446 SDLoc SL(Op); in lowerFastUnsafeFDIV() local
10475 return DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS); in lowerFastUnsafeFDIV()
10481 SDValue FNegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in lowerFastUnsafeFDIV()
10482 return DAG.getNode(AMDGPUISD::RCP, SL, VT, FNegRHS); in lowerFastUnsafeFDIV()
10493 SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS); in lowerFastUnsafeFDIV()
10494 return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip, Flags); in lowerFastUnsafeFDIV()
10499 SDLoc SL(Op); in lowerFastUnsafeFDIV64() local
10510 SDValue NegY = DAG.getNode(ISD::FNEG, SL, VT, Y); in lowerFastUnsafeFDIV64()
10511 SDValue One = DAG.getConstantFP(1.0, SL, VT); in lowerFastUnsafeFDIV64()
10513 SDValue R = DAG.getNode(AMDGPUISD::RCP, SL, VT, Y); in lowerFastUnsafeFDIV64()
10514 SDValue Tmp0 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One); in lowerFastUnsafeFDIV64()
10516 R = DAG.getNode(ISD::FMA, SL, VT, Tmp0, R, R); in lowerFastUnsafeFDIV64()
10517 SDValue Tmp1 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One); in lowerFastUnsafeFDIV64()
10518 R = DAG.getNode(ISD::FMA, SL, VT, Tmp1, R, R); in lowerFastUnsafeFDIV64()
10519 SDValue Ret = DAG.getNode(ISD::FMUL, SL, VT, X, R); in lowerFastUnsafeFDIV64()
10520 SDValue Tmp2 = DAG.getNode(ISD::FMA, SL, VT, NegY, Ret, X); in lowerFastUnsafeFDIV64()
10521 return DAG.getNode(ISD::FMA, SL, VT, Tmp2, R, Ret); in lowerFastUnsafeFDIV64()
10524 static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, in getFPBinOp() argument
10528 return DAG.getNode(Opcode, SL, VT, A, B, Flags); in getFPBinOp()
10541 return DAG.getNode(Opcode, SL, VTList, in getFPBinOp()
10546 static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, in getFPTernOp() argument
10550 return DAG.getNode(Opcode, SL, VT, {A, B, C}, Flags); in getFPTernOp()
10563 return DAG.getNode(Opcode, SL, VTList, in getFPTernOp()
10572 SDLoc SL(Op); in LowerFDIV16() local
10576 SDValue CvtSrc0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in LowerFDIV16()
10577 SDValue CvtSrc1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in LowerFDIV16()
10579 SDValue RcpSrc1 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, CvtSrc1); in LowerFDIV16()
10580 SDValue Quot = DAG.getNode(ISD::FMUL, SL, MVT::f32, CvtSrc0, RcpSrc1); in LowerFDIV16()
10582 SDValue FPRoundFlag = DAG.getTargetConstant(0, SL, MVT::i32); in LowerFDIV16()
10583 SDValue BestQuot = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot, FPRoundFlag); in LowerFDIV16()
10585 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, BestQuot, Src1, Src0); in LowerFDIV16()
10591 SDLoc SL(Op); in lowerFDIV_FAST() local
10595 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS, Flags); in lowerFDIV_FAST()
10598 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32); in lowerFDIV_FAST()
10601 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32); in lowerFDIV_FAST()
10603 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in lowerFDIV_FAST()
10608 SDValue r2 = DAG.getSetCC(SL, SetCCVT, r1, K0, ISD::SETOGT); in lowerFDIV_FAST()
10610 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One, Flags); in lowerFDIV_FAST()
10612 r1 = DAG.getNode(ISD::FMUL, SL, MVT::f32, RHS, r3, Flags); in lowerFDIV_FAST()
10615 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1, Flags); in lowerFDIV_FAST()
10617 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0, Flags); in lowerFDIV_FAST()
10619 return DAG.getNode(ISD::FMUL, SL, MVT::f32, r3, Mul, Flags); in lowerFDIV_FAST()
10644 SDLoc SL(Op); in LowerFDIV32() local
10648 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in LowerFDIV32()
10652 SDValue DenominatorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, in LowerFDIV32()
10654 SDValue NumeratorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, in LowerFDIV32()
10658 SDValue ApproxRcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, in LowerFDIV32()
10660 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f32, in LowerFDIV32()
10665 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i32); in LowerFDIV32()
10687 SDNode *GetReg = DAG.getMachineNode(AMDGPU::S_GETREG_B32, SL, in LowerFDIV32()
10693 {DAG.getEntryNode(), SDValue(GetReg, 0), SDValue(GetReg, 1)}, SL); in LowerFDIV32()
10701 EnableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, BindParamVTs, Glue, in LowerFDIV32()
10706 SL, MVT::i32); in LowerFDIV32()
10707 EnableDenorm = DAG.getMachineNode(AMDGPU::S_SETREG_B32, SL, BindParamVTs, in LowerFDIV32()
10717 NegDivScale0 = DAG.getMergeValues(Ops, SL); in LowerFDIV32()
10720 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, in LowerFDIV32()
10723 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp, in LowerFDIV32()
10726 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled, in LowerFDIV32()
10729 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul, in LowerFDIV32()
10732 SDValue Fma3 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, in LowerFDIV32()
10735 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3, in LowerFDIV32()
10744 DisableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, MVT::Other, in LowerFDIV32()
10752 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, SL, MVT::i32); in LowerFDIV32()
10755 AMDGPU::S_SETREG_B32, SL, MVT::Other, in LowerFDIV32()
10759 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in LowerFDIV32()
10765 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32, in LowerFDIV32()
10768 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS, Flags); in LowerFDIV32()
10775 SDLoc SL(Op); in LowerFDIV64() local
10779 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFDIV64()
10783 SDValue DivScale0 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, Y, Y, X); in LowerFDIV64()
10785 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0); in LowerFDIV64()
10787 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0); in LowerFDIV64()
10789 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One); in LowerFDIV64()
10791 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp); in LowerFDIV64()
10793 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One); in LowerFDIV64()
10795 SDValue DivScale1 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, X, Y, X); in LowerFDIV64()
10797 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1); in LowerFDIV64()
10798 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3); in LowerFDIV64()
10800 SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64, in LowerFDIV64()
10809 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32); in LowerFDIV64()
10812 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); in LowerFDIV64()
10813 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y); in LowerFDIV64()
10814 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0); in LowerFDIV64()
10815 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1); in LowerFDIV64()
10817 SDValue NumHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi); in LowerFDIV64()
10818 SDValue DenHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi); in LowerFDIV64()
10821 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale0BC, Hi); in LowerFDIV64()
10823 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale1BC, Hi); in LowerFDIV64()
10825 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ); in LowerFDIV64()
10826 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ); in LowerFDIV64()
10827 Scale = DAG.getNode(ISD::XOR, SL, MVT::i1, CmpNum, CmpDen); in LowerFDIV64()
10832 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64, in LowerFDIV64()
10835 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas, Y, X); in LowerFDIV64()
10964 SDLoc SL(Op); in lowerFSQRTF16() local
10968 DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op.getOperand(0), Flags); in lowerFSQRTF16()
10970 SDValue SqrtID = DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, SL, MVT::i32); in lowerFSQRTF16()
10972 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32, SqrtID, Ext, Flags); in lowerFSQRTF16()
10974 return DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Sqrt, in lowerFSQRTF16()
10975 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in lowerFSQRTF16()
11340 SDLoc SL(N); in performSHLPtrCombine() local
11343 SDValue ShlX = DAG.getNode(ISD::SHL, SL, VT, N0.getOperand(0), N1); in performSHLPtrCombine()
11344 SDValue COffset = DAG.getConstant(Offset, SL, VT); in performSHLPtrCombine()
11351 return DAG.getNode(ISD::ADD, SL, VT, ShlX, COffset, Flags); in performSHLPtrCombine()
11371 SDLoc SL(N); in performMemSDNodeCombine() local
11404 const SDLoc &SL, in splitBinaryBitConstantOp() argument
11418 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi); in splitBinaryBitConstantOp()
11536 SDLoc SL(N); in performAndCombine() local
11537 SDValue BFE = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in performAndCombine()
11539 DAG.getConstant(Offset, SL, MVT::i32), in performAndCombine()
11540 DAG.getConstant(Bits, SL, MVT::i32)); in performAndCombine()
11542 SDValue Ext = DAG.getNode(ISD::AssertZext, SL, VT, BFE, in performAndCombine()
12128 static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src, in getDWordFromOffset() argument
12137 return DAG.getBitcastedAnyExtOrTrunc(Src, SL, MVT::i32); in getDWordFromOffset()
12143 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Src, in getDWordFromOffset()
12144 DAG.getConstant(DWordOffset, SL, MVT::i32)); in getDWordFromOffset()
12148 ISD::EXTRACT_VECTOR_ELT, SL, ScalarTy, Src, in getDWordFromOffset()
12149 DAG.getConstant(DWordOffset / (ScalarTySize / 32), SL, MVT::i32)); in getDWordFromOffset()
12152 Ret = DAG.getNode(ISD::SRL, SL, Ret.getValueType(), Ret, in getDWordFromOffset()
12153 DAG.getConstant(ShiftVal, SL, MVT::i32)); in getDWordFromOffset()
12154 return DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32); in getDWordFromOffset()
12171 MVT::getVectorVT(MVT::getIntegerVT(ScalarTySize), NumAvailElements), SL, in getDWordFromOffset()
12173 return Ret = DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32); in getDWordFromOffset()
12178 Ret = DAG.getNode(ISD::SRL, SL, Src.getValueType(), Src, in getDWordFromOffset()
12179 DAG.getConstant(ShiftVal, SL, MVT::i32)); in getDWordFromOffset()
12180 return DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32); in getDWordFromOffset()
12408 SDLoc SL(N); in performOrCombine() local
12411 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc); in performOrCombine()
12416 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in performOrCombine()
12418 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performOrCombine()
12915 SelectionDAG &DAG, const SDLoc &SL, EVT VT, const APFloat &C) const { in getCanonicalConstantFP() argument
12922 APFloat::getZero(C.getSemantics(), C.isNegative()), SL, VT); in getCanonicalConstantFP()
12934 return DAG.getConstantFP(CanonicalQNaN, SL, VT); in getCanonicalConstantFP()
12942 return DAG.getConstantFP(CanonicalQNaN, SL, VT); in getCanonicalConstantFP()
12946 return DAG.getConstantFP(C, SL, VT); in getCanonicalConstantFP()
12980 SDLoc SL(N); in performFCanonicalizeCombine() local
12990 NewElts[I] = getCanonicalConstantFP(DAG, SL, EltVT, in performFCanonicalizeCombine()
12996 NewElts[I] = DAG.getNode(ISD::FCANONICALIZE, SL, EltVT, Op); in performFCanonicalizeCombine()
13006 NewElts[1]: DAG.getConstantFP(0.0f, SL, EltVT); in performFCanonicalizeCombine()
13011 NewElts[0] : DAG.getConstantFP(0.0f, SL, EltVT); in performFCanonicalizeCombine()
13014 return DAG.getBuildVector(VT, SL, NewElts); in performFCanonicalizeCombine()
13047 const SDLoc &SL, SDValue Src, in performIntMed3ImmCombine() argument
13075 return DAG.getNode(Med3Opc, SL, VT, Src, MaxVal, MinVal); in performIntMed3ImmCombine()
13097 const SDLoc &SL, in performFPMed3ImmCombine() argument
13122 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Op0.getOperand(0)); in performFPMed3ImmCombine()
13139 return DAG.getNode(AMDGPUISD::FMED3, SL, K0->getValueType(0), in performFPMed3ImmCombine()
13273 SDLoc SL(N); in performFMed3Combine() local
13283 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src2); in performFMed3Combine()
13304 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src0); in performFMed3Combine()
13386 SDLoc SL(N); in performExtractVectorEltCombine() local
13389 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec.getOperand(0), Idx); in performExtractVectorEltCombine()
13390 return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt); in performExtractVectorEltCombine()
13399 SDLoc SL(N); in performExtractVectorEltCombine() local
13421 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, in performExtractVectorEltCombine()
13423 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, in performExtractVectorEltCombine()
13428 return DAG.getNode(Opc, SL, ResVT, Elt0, Elt1, Vec->getFlags()); in performExtractVectorEltCombine()
13435 SDLoc SL(N); in performExtractVectorEltCombine() local
13439 SDValue IC = DAG.getVectorIdxConstant(I, SL); in performExtractVectorEltCombine()
13440 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec, IC); in performExtractVectorEltCombine()
13444 V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ); in performExtractVectorEltCombine()
13463 SDLoc SL(N); in performExtractVectorEltCombine() local
13465 SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec); in performExtractVectorEltCombine()
13468 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast, in performExtractVectorEltCombine()
13469 DAG.getConstant(EltIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
13471 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt, in performExtractVectorEltCombine()
13472 DAG.getConstant(LeftoverBitIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
13476 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VecEltAsIntVT, Srl); in performExtractVectorEltCombine()
13480 return DAG.getNode(ISD::BITCAST, SL, VecEltVT, Trunc); in performExtractVectorEltCombine()
13484 return DAG.getAnyExtOrTrunc(Trunc, SL, ResVT); in performExtractVectorEltCombine()
13504 SDLoc SL(N); in performInsertVectorEltCombine() local
13510 SDValue IC = DAG.getConstant(I, SL, IdxVT); in performInsertVectorEltCombine()
13511 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC); in performInsertVectorEltCombine()
13512 SDValue V = DAG.getSelectCC(SL, Idx, IC, Ins, Elt, ISD::SETEQ); in performInsertVectorEltCombine()
13516 return DAG.getBuildVector(VecVT, SL, Ops); in performInsertVectorEltCombine()
13553 SDLoc SL(N); in performFPRoundCombine() local
13577 SDValue A1 = DAG.getNode(ISD::FMINNUM_IEEE, SL, VT, A, B); in performFPRoundCombine()
13578 SDValue B1 = DAG.getNode(ISD::FMAXNUM_IEEE, SL, VT, A, B); in performFPRoundCombine()
13579 SDValue C1 = DAG.getNode(ISD::FMAXNUM_IEEE, SL, VT, A1, C); in performFPRoundCombine()
13580 return DAG.getNode(ISD::FMINNUM_IEEE, SL, VT, B1, C1); in performFPRoundCombine()
13640 SDLoc SL(N); in reassociateScalarOps() local
13641 SDValue Add1 = DAG.getNode(Opc, SL, VT, Op0, Op1); in reassociateScalarOps()
13642 return DAG.getNode(Opc, SL, VT, Add1, Op2); in reassociateScalarOps()
13645 static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, in getMad64_32() argument
13651 SDValue Mad = DAG.getNode(MadOpc, SL, VTs, N0, N1, N2); in getMad64_32()
13652 return DAG.getNode(ISD::TRUNCATE, SL, VT, Mad); in getMad64_32()
13669 SDLoc SL(N); in tryFoldToMad64_32() local
13730 MulLHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulLHS); in tryFoldToMad64_32()
13731 MulRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulRHS); in tryFoldToMad64_32()
13732 AddRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, AddRHS); in tryFoldToMad64_32()
13747 SDValue One = DAG.getConstant(1, SL, MVT::i32); in tryFoldToMad64_32()
13749 auto MulLHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulLHS); in tryFoldToMad64_32()
13750 auto MulRHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulRHS); in tryFoldToMad64_32()
13752 getMad64_32(DAG, SL, MVT::i64, MulLHSLo, MulRHSLo, AddRHS, MulSignedLo); in tryFoldToMad64_32()
13756 std::tie(AccumLo, AccumHi) = DAG.SplitScalar(Accum, SL, MVT::i32, MVT::i32); in tryFoldToMad64_32()
13760 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulLHS, One); in tryFoldToMad64_32()
13761 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSHi, MulRHSLo); in tryFoldToMad64_32()
13762 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi); in tryFoldToMad64_32()
13767 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulRHS, One); in tryFoldToMad64_32()
13768 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSLo, MulRHSHi); in tryFoldToMad64_32()
13769 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi); in tryFoldToMad64_32()
13772 Accum = DAG.getBuildVector(MVT::v2i32, SL, {AccumLo, AccumHi}); in tryFoldToMad64_32()
13777 Accum = DAG.getNode(ISD::TRUNCATE, SL, VT, Accum); in tryFoldToMad64_32()
13894 static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL, in resolveSources() argument
13901 auto EltOp = getDWordFromOffset(DAG, SL, Elt->SrcOp, Elt->DWordOffset); in resolveSources()
13907 return DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp, in resolveSources()
13908 DAG.getConstant(Elt->PermMask, SL, MVT::i32)); in resolveSources()
13930 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset); in resolveSources()
13932 getDWordFromOffset(DAG, SL, SecondElt->SrcOp, SecondElt->DWordOffset); in resolveSources()
13934 Perms.push_back(DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, FirstVal, in resolveSources()
13936 DAG.getConstant(PermMask, SL, MVT::i32))); in resolveSources()
13947 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset); in resolveSources()
13950 DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp, in resolveSources()
13951 DAG.getConstant(FirstElt->PermMask, SL, MVT::i32))); in resolveSources()
13958 ? DAG.getNode(ISD::OR, SL, MVT::i32, Perms[0], Perms[1]) in resolveSources()
14044 SDLoc SL(N); in performAddCombine() local
14114 Src2s.push_back(DAG.getConstant(0, SL, MVT::i32)); in performAddCombine()
14168 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset); in performAddCombine()
14171 auto SecondEltOp = getDWordFromOffset(DAG, SL, SecondElt->SrcOp, in performAddCombine()
14174 Src0 = DAG.getBitcastedAnyExtOrTrunc(FirstEltOp, SL, in performAddCombine()
14176 Src1 = DAG.getBitcastedAnyExtOrTrunc(SecondEltOp, SL, in performAddCombine()
14182 Src0 = resolveSources(DAG, SL, Src0s, false, true); in performAddCombine()
14183 Src1 = resolveSources(DAG, SL, Src1s, false, true); in performAddCombine()
14188 DAG.getExtOrTrunc(*IsSigned, Src2s[ChainLength - 1], SL, MVT::i32); in performAddCombine()
14192 SL, MVT::i64); in performAddCombine()
14195 auto Dot = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, IID, Src0, in performAddCombine()
14196 Src1, Src2, DAG.getTargetConstant(0, SL, MVT::i1)); in performAddCombine()
14198 return DAG.getExtOrTrunc(*IsSigned, Dot, SL, VT); in performAddCombine()
14223 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performAddCombine()
14225 return DAG.getNode(Opc, SL, VTList, Args); in performAddCombine()
14246 SDLoc SL(N); in performSubCombine() local
14264 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performSubCombine()
14266 return DAG.getNode(Opc, SL, VTList, Args); in performSubCombine()
14312 SDLoc SL(N); in performFAddCombine() local
14325 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); in performFAddCombine()
14326 return DAG.getNode(FusedOp, SL, VT, A, Two, RHS); in performFAddCombine()
14337 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); in performFAddCombine()
14338 return DAG.getNode(FusedOp, SL, VT, A, Two, LHS); in performFAddCombine()
14352 SDLoc SL(N); in performFSubCombine() local
14369 const SDValue Two = DAG.getConstantFP(2.0, SL, VT); in performFSubCombine()
14370 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFSubCombine()
14372 return DAG.getNode(FusedOp, SL, VT, A, Two, NegRHS); in performFSubCombine()
14384 const SDValue NegTwo = DAG.getConstantFP(-2.0, SL, VT); in performFSubCombine()
14385 return DAG.getNode(FusedOp, SL, VT, A, NegTwo, LHS); in performFSubCombine()
14396 SDLoc SL(N); in performFDivCombine() local
14419 DAG.getNode(AMDGPUISD::RSQ, SL, VT, RHS.getOperand(0), Flags); in performFDivCombine()
14420 return IsNegative ? DAG.getNode(ISD::FNEG, SL, VT, Rsq, Flags) : Rsq; in performFDivCombine()
14432 SDLoc SL(N); in performFMACombine() local
14496 return DAG.getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc, in performFMACombine()
14497 DAG.getTargetConstant(0, SL, MVT::i1)); in performFMACombine()
14506 SDLoc SL(N); in performSetCCCombine() local
14533 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14534 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
14559 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14560 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
14591 return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14592 DAG.getConstant(Mask, SL, MVT::i32)); in performSetCCCombine()
14602 SDLoc SL(N); in performCvtF32UByteNCombine() local
14629 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, SL, in performCvtF32UByteNCombine()
14648 return DAG.getNode(N->getOpcode(), SL, MVT::f32, DemandedSrc); in performCvtF32UByteNCombine()
14768 SDLoc SL(N); in PerformDAGCombine() local
14772 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src); in PerformDAGCombine()
14774 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src); in PerformDAGCombine()
14775 return DAG.getNode(ISD::BITCAST, SL, VT, Ext); in PerformDAGCombine()
15005 SDLoc SL(Node); in legalizeTargetIndependentNode() local
15012 = DAG.getCopyToReg(Node->getOperand(0), SL, VReg, SrcVal, in legalizeTargetIndependentNode()
15015 = DAG.getCopyToReg(ToVReg, SL, SDValue(DestReg, 0), in legalizeTargetIndependentNode()