Lines Matching refs:DAG
51 unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) { in numBitsUnsigned() argument
52 return DAG.computeKnownBits(Op).countMaxActiveBits(); in numBitsUnsigned()
55 unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) { in numBitsSigned() argument
58 return DAG.ComputeMaxSignificantBits(Op); in numBitsSigned()
857 const SelectionDAG &DAG, in isLoadBitCastBeneficial() argument
872 return allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(), in isLoadBitCastBeneficial()
909 SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize, in getNegatedExpression() argument
925 SDValue NegSrc = getNegatedExpression(Src, DAG, LegalOperations, in getNegatedExpression()
928 return DAG.getNode(AMDGPUISD::RCP, SL, VT, NegSrc, Op->getFlags()); in getNegatedExpression()
935 return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations, in getNegatedExpression()
1269 const SDLoc &DL, SelectionDAG &DAG) const { in LowerReturn()
1273 return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain); in LowerReturn()
1292 SelectionDAG &DAG, in addTokenForArgument() argument
1305 for (SDNode *U : DAG.getEntryNode().getNode()->uses()) { in addTokenForArgument()
1322 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
1329 SelectionDAG &DAG = CLI.DAG; in lowerUnhandledCall() local
1331 const Function &Fn = DAG.getMachineFunction().getFunction(); in lowerUnhandledCall()
1342 DAG.getContext()->diagnose(NoCalls); in lowerUnhandledCall()
1346 InVals.push_back(DAG.getUNDEF(Arg.VT)); in lowerUnhandledCall()
1349 return DAG.getEntryNode(); in lowerUnhandledCall()
1358 SelectionDAG &DAG) const { in LowerDYNAMIC_STACKALLOC()
1359 const Function &Fn = DAG.getMachineFunction().getFunction(); in LowerDYNAMIC_STACKALLOC()
1363 DAG.getContext()->diagnose(NoDynamicAlloca); in LowerDYNAMIC_STACKALLOC()
1364 auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()), Op.getOperand(0)}; in LowerDYNAMIC_STACKALLOC()
1365 return DAG.getMergeValues(Ops, SDLoc()); in LowerDYNAMIC_STACKALLOC()
1369 SelectionDAG &DAG) const { in LowerOperation()
1372 Op->print(errs(), &DAG); in LowerOperation()
1376 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG); in LowerOperation()
1377 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); in LowerOperation()
1378 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); in LowerOperation()
1379 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG); in LowerOperation()
1380 case ISD::SDIVREM: return LowerSDIVREM(Op, DAG); in LowerOperation()
1381 case ISD::FREM: return LowerFREM(Op, DAG); in LowerOperation()
1382 case ISD::FCEIL: return LowerFCEIL(Op, DAG); in LowerOperation()
1383 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG); in LowerOperation()
1384 case ISD::FRINT: return LowerFRINT(Op, DAG); in LowerOperation()
1385 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG); in LowerOperation()
1387 return LowerFROUNDEVEN(Op, DAG); in LowerOperation()
1388 case ISD::FROUND: return LowerFROUND(Op, DAG); in LowerOperation()
1389 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG); in LowerOperation()
1391 return LowerFLOG2(Op, DAG); in LowerOperation()
1394 return LowerFLOGCommon(Op, DAG); in LowerOperation()
1397 return lowerFEXP(Op, DAG); in LowerOperation()
1399 return lowerFEXP2(Op, DAG); in LowerOperation()
1400 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); in LowerOperation()
1401 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); in LowerOperation()
1402 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG); in LowerOperation()
1405 return LowerFP_TO_INT(Op, DAG); in LowerOperation()
1410 return LowerCTLZ_CTTZ(Op, DAG); in LowerOperation()
1411 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); in LowerOperation()
1418 SelectionDAG &DAG) const { in ReplaceNodeResults()
1429 if (SDValue Lowered = LowerFLOG2(SDValue(N, 0), DAG)) in ReplaceNodeResults()
1434 if (SDValue Lowered = LowerFLOGCommon(SDValue(N, 0), DAG)) in ReplaceNodeResults()
1438 if (SDValue Lowered = lowerFEXP2(SDValue(N, 0), DAG)) in ReplaceNodeResults()
1443 if (SDValue Lowered = lowerFEXP(SDValue(N, 0), DAG)) in ReplaceNodeResults()
1448 if (auto Lowered = lowerCTLZResults(SDValue(N, 0u), DAG)) in ReplaceNodeResults()
1458 SelectionDAG &DAG) const { in LowerGlobalAddress()
1460 const DataLayout &DL = DAG.getDataLayout(); in LowerGlobalAddress()
1467 return DAG.getConstant(*Address, SDLoc(Op), Op.getValueType()); in LowerGlobalAddress()
1476 const Function &Fn = DAG.getMachineFunction().getFunction(); in LowerGlobalAddress()
1480 DAG.getContext()->diagnose(BadLDSDecl); in LowerGlobalAddress()
1487 SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode()); in LowerGlobalAddress()
1488 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in LowerGlobalAddress()
1489 Trap, DAG.getRoot()); in LowerGlobalAddress()
1490 DAG.setRoot(OutputChain); in LowerGlobalAddress()
1491 return DAG.getUNDEF(Op.getValueType()); in LowerGlobalAddress()
1502 return DAG.getConstant(Offset, SDLoc(Op), Op.getValueType()); in LowerGlobalAddress()
1508 SelectionDAG &DAG) const { in LowerCONCAT_VECTORS()
1518 : EVT::getVectorVT(*DAG.getContext(), in LowerCONCAT_VECTORS()
1522 SDValue NewIn = DAG.getNode(ISD::BITCAST, SL, NewEltVT, In); in LowerCONCAT_VECTORS()
1524 DAG.ExtractVectorElements(NewIn, Args); in LowerCONCAT_VECTORS()
1529 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in LowerCONCAT_VECTORS()
1531 SDValue BV = DAG.getBuildVector(NewVT, SL, Args); in LowerCONCAT_VECTORS()
1532 return DAG.getNode(ISD::BITCAST, SL, VT, BV); in LowerCONCAT_VECTORS()
1537 DAG.ExtractVectorElements(U.get(), Args); in LowerCONCAT_VECTORS()
1539 return DAG.getBuildVector(Op.getValueType(), SL, Args); in LowerCONCAT_VECTORS()
1543 SelectionDAG &DAG) const { in LowerEXTRACT_SUBVECTOR()
1556 EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumSrcElt / 2); in LowerEXTRACT_SUBVECTOR()
1559 : EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElt / 2); in LowerEXTRACT_SUBVECTOR()
1560 SDValue Tmp = DAG.getNode(ISD::BITCAST, SL, NewSrcVT, Op.getOperand(0)); in LowerEXTRACT_SUBVECTOR()
1562 DAG.ExtractVectorElements(Tmp, Args, Start / 2, NumElt / 2); in LowerEXTRACT_SUBVECTOR()
1566 Tmp = DAG.getBuildVector(NewVT, SL, Args); in LowerEXTRACT_SUBVECTOR()
1568 return DAG.getNode(ISD::BITCAST, SL, VT, Tmp); in LowerEXTRACT_SUBVECTOR()
1571 DAG.ExtractVectorElements(Op.getOperand(0), Args, Start, in LowerEXTRACT_SUBVECTOR()
1574 return DAG.getBuildVector(Op.getValueType(), SL, Args); in LowerEXTRACT_SUBVECTOR()
1598 SelectionDAG &DAG = DCI.DAG; in combineFMinMaxLegacyImpl() local
1617 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacyImpl()
1618 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacyImpl()
1636 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacyImpl()
1637 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacyImpl()
1642 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacyImpl()
1643 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacyImpl()
1654 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS); in combineFMinMaxLegacyImpl()
1655 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS); in combineFMinMaxLegacyImpl()
1672 SelectionDAG &DAG = DCI.DAG; in combineFMinMaxLegacy() local
1694 return DAG.getNode(ISD::FNEG, DL, VT, Combined); in combineFMinMaxLegacy()
1703 AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const { in split64BitValue()
1706 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in split64BitValue()
1708 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in split64BitValue()
1709 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in split64BitValue()
1711 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in split64BitValue()
1712 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in split64BitValue()
1717 SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const { in getLoHalf64()
1720 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getLoHalf64()
1721 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in getLoHalf64()
1722 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in getLoHalf64()
1725 SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const { in getHiHalf64()
1728 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getHiHalf64()
1729 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in getHiHalf64()
1730 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in getHiHalf64()
1737 AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const { in getSplitDestVTs()
1742 LoVT = EVT::getVectorVT(*DAG.getContext(), EltVT, LoNumElts); in getSplitDestVTs()
1745 : EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts - LoNumElts); in getSplitDestVTs()
1754 SelectionDAG &DAG) const { in splitVector()
1759 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, in splitVector()
1760 DAG.getVectorIdxConstant(0, DL)); in splitVector()
1761 SDValue Hi = DAG.getNode( in splitVector()
1763 HiVT, N, DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), DL)); in splitVector()
1768 SelectionDAG &DAG) const { in SplitVectorLoad()
1778 std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(Load, DAG); in SplitVectorLoad()
1779 return DAG.getMergeValues(Ops, SL); in SplitVectorLoad()
1791 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG); in SplitVectorLoad()
1792 std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG); in SplitVectorLoad()
1793 std::tie(Lo, Hi) = splitVector(Op, SL, LoVT, HiVT, DAG); in SplitVectorLoad()
1799 SDValue LoLoad = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT, in SplitVectorLoad()
1802 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Size)); in SplitVectorLoad()
1804 DAG.getExtLoad(Load->getExtensionType(), SL, HiVT, Load->getChain(), in SplitVectorLoad()
1811 Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad); in SplitVectorLoad()
1813 Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad, in SplitVectorLoad()
1814 DAG.getVectorIdxConstant(0, SL)); in SplitVectorLoad()
1815 Join = DAG.getNode( in SplitVectorLoad()
1818 DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), SL)); in SplitVectorLoad()
1821 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in SplitVectorLoad()
1824 return DAG.getMergeValues(Ops, SL); in SplitVectorLoad()
1828 SelectionDAG &DAG) const { in WidenOrSplitVectorLoad()
1842 !SrcValue.isDereferenceable(16, *DAG.getContext(), DAG.getDataLayout()))) in WidenOrSplitVectorLoad()
1843 return SplitVectorLoad(Op, DAG); in WidenOrSplitVectorLoad()
1848 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4); in WidenOrSplitVectorLoad()
1850 EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(), 4); in WidenOrSplitVectorLoad()
1851 SDValue WideLoad = DAG.getExtLoad( in WidenOrSplitVectorLoad()
1854 return DAG.getMergeValues( in WidenOrSplitVectorLoad()
1855 {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad, in WidenOrSplitVectorLoad()
1856 DAG.getVectorIdxConstant(0, SL)), in WidenOrSplitVectorLoad()
1862 SelectionDAG &DAG) const { in SplitVectorStore()
1870 return scalarizeVectorStore(Store, DAG); in SplitVectorStore()
1881 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG); in SplitVectorStore()
1882 std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG); in SplitVectorStore()
1883 std::tie(Lo, Hi) = splitVector(Val, SL, LoVT, HiVT, DAG); in SplitVectorStore()
1885 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, LoMemVT.getStoreSize()); in SplitVectorStore()
1893 DAG.getTruncStore(Chain, SL, Lo, BasePtr, SrcValue, LoMemVT, BaseAlign, in SplitVectorStore()
1896 DAG.getTruncStore(Chain, SL, Hi, HiPtr, SrcValue.getWithOffset(Size), in SplitVectorStore()
1899 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore); in SplitVectorStore()
1905 SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG, in LowerDIVREM24() argument
1914 unsigned LHSSignBits = DAG.ComputeNumSignBits(LHS); in LowerDIVREM24()
1918 unsigned RHSSignBits = DAG.ComputeNumSignBits(RHS); in LowerDIVREM24()
1931 SDValue jq = DAG.getConstant(1, DL, IntVT); in LowerDIVREM24()
1935 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS); in LowerDIVREM24()
1938 jq = DAG.getNode(ISD::SRA, DL, VT, jq, in LowerDIVREM24()
1939 DAG.getConstant(BitSize - 2, DL, VT)); in LowerDIVREM24()
1942 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT)); in LowerDIVREM24()
1952 SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia); in LowerDIVREM24()
1955 SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib); in LowerDIVREM24()
1957 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT, in LowerDIVREM24()
1958 fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb)); in LowerDIVREM24()
1961 fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq); in LowerDIVREM24()
1964 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq); in LowerDIVREM24()
1966 MachineFunction &MF = DAG.getMachineFunction(); in LowerDIVREM24()
1979 SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa); in LowerDIVREM24()
1982 SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq); in LowerDIVREM24()
1985 fr = DAG.getNode(ISD::FABS, DL, FltVT, fr); in LowerDIVREM24()
1988 fb = DAG.getNode(ISD::FABS, DL, FltVT, fb); in LowerDIVREM24()
1990 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerDIVREM24()
1993 SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE); in LowerDIVREM24()
1996 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT)); in LowerDIVREM24()
1999 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq); in LowerDIVREM24()
2002 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS); in LowerDIVREM24()
2003 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem); in LowerDIVREM24()
2008 = DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), DivBits)); in LowerDIVREM24()
2009 Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize); in LowerDIVREM24()
2010 Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize); in LowerDIVREM24()
2012 SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT); in LowerDIVREM24()
2013 Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask); in LowerDIVREM24()
2014 Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask); in LowerDIVREM24()
2017 return DAG.getMergeValues({ Div, Rem }, DL); in LowerDIVREM24()
2021 SelectionDAG &DAG, in LowerUDIVREM64() argument
2028 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext()); in LowerUDIVREM64()
2030 SDValue One = DAG.getConstant(1, DL, HalfVT); in LowerUDIVREM64()
2031 SDValue Zero = DAG.getConstant(0, DL, HalfVT); in LowerUDIVREM64()
2036 std::tie(LHS_Lo, LHS_Hi) = DAG.SplitScalar(LHS, DL, HalfVT, HalfVT); in LowerUDIVREM64()
2040 std::tie(RHS_Lo, RHS_Hi) = DAG.SplitScalar(RHS, DL, HalfVT, HalfVT); in LowerUDIVREM64()
2042 if (DAG.MaskedValueIsZero(RHS, APInt::getHighBitsSet(64, 32)) && in LowerUDIVREM64()
2043 DAG.MaskedValueIsZero(LHS, APInt::getHighBitsSet(64, 32))) { in LowerUDIVREM64()
2045 SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT), in LowerUDIVREM64()
2048 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero}); in LowerUDIVREM64()
2049 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero}); in LowerUDIVREM64()
2051 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV)); in LowerUDIVREM64()
2052 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM)); in LowerUDIVREM64()
2060 MachineFunction &MF = DAG.getMachineFunction(); in LowerUDIVREM64()
2070 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo); in LowerUDIVREM64()
2071 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi); in LowerUDIVREM64()
2072 SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi, in LowerUDIVREM64()
2073 DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32), in LowerUDIVREM64()
2075 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1); in LowerUDIVREM64()
2076 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp, in LowerUDIVREM64()
2077 DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32)); in LowerUDIVREM64()
2078 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1, in LowerUDIVREM64()
2079 DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32)); in LowerUDIVREM64()
2080 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2); in LowerUDIVREM64()
2081 SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc, in LowerUDIVREM64()
2082 DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32), in LowerUDIVREM64()
2084 SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2); in LowerUDIVREM64()
2085 SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc); in LowerUDIVREM64()
2086 SDValue Rcp64 = DAG.getBitcast(VT, in LowerUDIVREM64()
2087 DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi})); in LowerUDIVREM64()
2089 SDValue Zero64 = DAG.getConstant(0, DL, VT); in LowerUDIVREM64()
2090 SDValue One64 = DAG.getConstant(1, DL, VT); in LowerUDIVREM64()
2091 SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1); in LowerUDIVREM64()
2092 SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1); in LowerUDIVREM64()
2095 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS); in LowerUDIVREM64()
2096 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64); in LowerUDIVREM64()
2097 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1); in LowerUDIVREM64()
2100 DAG.SplitScalar(Mulhi1, DL, HalfVT, HalfVT); in LowerUDIVREM64()
2101 SDValue Add1_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Lo, in LowerUDIVREM64()
2103 SDValue Add1_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Hi, in LowerUDIVREM64()
2105 SDValue Add1 = DAG.getBitcast(VT, in LowerUDIVREM64()
2106 DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi})); in LowerUDIVREM64()
2109 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1); in LowerUDIVREM64()
2110 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2); in LowerUDIVREM64()
2113 DAG.SplitScalar(Mulhi2, DL, HalfVT, HalfVT); in LowerUDIVREM64()
2114 SDValue Add2_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Lo, in LowerUDIVREM64()
2116 SDValue Add2_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Hi, in LowerUDIVREM64()
2118 SDValue Add2 = DAG.getBitcast(VT, in LowerUDIVREM64()
2119 DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi})); in LowerUDIVREM64()
2121 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2); in LowerUDIVREM64()
2123 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3); in LowerUDIVREM64()
2126 std::tie(Mul3_Lo, Mul3_Hi) = DAG.SplitScalar(Mul3, DL, HalfVT, HalfVT); in LowerUDIVREM64()
2127 SDValue Sub1_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Lo, in LowerUDIVREM64()
2129 SDValue Sub1_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Hi, in LowerUDIVREM64()
2131 SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi); in LowerUDIVREM64()
2132 SDValue Sub1 = DAG.getBitcast(VT, in LowerUDIVREM64()
2133 DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi})); in LowerUDIVREM64()
2135 SDValue MinusOne = DAG.getConstant(0xffffffffu, DL, HalfVT); in LowerUDIVREM64()
2136 SDValue C1 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero, in LowerUDIVREM64()
2138 SDValue C2 = DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero, in LowerUDIVREM64()
2140 SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ); in LowerUDIVREM64()
2147 SDValue Sub2_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Lo, in LowerUDIVREM64()
2149 SDValue Sub2_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Mi, in LowerUDIVREM64()
2151 SDValue Sub2_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi, in LowerUDIVREM64()
2153 SDValue Sub2 = DAG.getBitcast(VT, in LowerUDIVREM64()
2154 DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi})); in LowerUDIVREM64()
2156 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64); in LowerUDIVREM64()
2158 SDValue C4 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero, in LowerUDIVREM64()
2160 SDValue C5 = DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero, in LowerUDIVREM64()
2162 SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ); in LowerUDIVREM64()
2165 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64); in LowerUDIVREM64()
2167 SDValue Sub3_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Lo, in LowerUDIVREM64()
2169 SDValue Sub3_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi, in LowerUDIVREM64()
2171 SDValue Sub3_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub3_Mi, in LowerUDIVREM64()
2173 SDValue Sub3 = DAG.getBitcast(VT, in LowerUDIVREM64()
2174 DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi})); in LowerUDIVREM64()
2179 SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE); in LowerUDIVREM64()
2180 SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE); in LowerUDIVREM64()
2182 SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE); in LowerUDIVREM64()
2183 SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE); in LowerUDIVREM64()
2193 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo); in LowerUDIVREM64()
2194 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo); in LowerUDIVREM64()
2196 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ); in LowerUDIVREM64()
2197 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero}); in LowerUDIVREM64()
2198 REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM); in LowerUDIVREM64()
2200 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ); in LowerUDIVREM64()
2207 SDValue POS = DAG.getConstant(bitPos, DL, HalfVT); in LowerUDIVREM64()
2209 SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS); in LowerUDIVREM64()
2210 HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, One); in LowerUDIVREM64()
2211 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit); in LowerUDIVREM64()
2214 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT)); in LowerUDIVREM64()
2216 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit); in LowerUDIVREM64()
2218 SDValue BIT = DAG.getConstant(1ULL << bitPos, DL, HalfVT); in LowerUDIVREM64()
2219 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE); in LowerUDIVREM64()
2221 DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT); in LowerUDIVREM64()
2224 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS); in LowerUDIVREM64()
2225 REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE); in LowerUDIVREM64()
2228 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi}); in LowerUDIVREM64()
2229 DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV); in LowerUDIVREM64()
2235 SelectionDAG &DAG) const { in LowerUDIVREM()
2241 LowerUDIVREM64(Op, DAG, Results); in LowerUDIVREM()
2242 return DAG.getMergeValues(Results, DL); in LowerUDIVREM()
2246 if (SDValue Res = LowerDIVREM24(Op, DAG, false)) in LowerUDIVREM()
2257 SDValue Z = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Y); in LowerUDIVREM()
2260 SDValue NegY = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Y); in LowerUDIVREM()
2261 SDValue NegYZ = DAG.getNode(ISD::MUL, DL, VT, NegY, Z); in LowerUDIVREM()
2262 Z = DAG.getNode(ISD::ADD, DL, VT, Z, in LowerUDIVREM()
2263 DAG.getNode(ISD::MULHU, DL, VT, Z, NegYZ)); in LowerUDIVREM()
2266 SDValue Q = DAG.getNode(ISD::MULHU, DL, VT, X, Z); in LowerUDIVREM()
2268 DAG.getNode(ISD::SUB, DL, VT, X, DAG.getNode(ISD::MUL, DL, VT, Q, Y)); in LowerUDIVREM()
2271 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerUDIVREM()
2272 SDValue One = DAG.getConstant(1, DL, VT); in LowerUDIVREM()
2273 SDValue Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE); in LowerUDIVREM()
2274 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2275 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q); in LowerUDIVREM()
2276 R = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2277 DAG.getNode(ISD::SUB, DL, VT, R, Y), R); in LowerUDIVREM()
2280 Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE); in LowerUDIVREM()
2281 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2282 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q); in LowerUDIVREM()
2283 R = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
2284 DAG.getNode(ISD::SUB, DL, VT, R, Y), R); in LowerUDIVREM()
2286 return DAG.getMergeValues({Q, R}, DL); in LowerUDIVREM()
2290 SelectionDAG &DAG) const { in LowerSDIVREM()
2297 SDValue Zero = DAG.getConstant(0, DL, VT); in LowerSDIVREM()
2298 SDValue NegOne = DAG.getConstant(-1, DL, VT); in LowerSDIVREM()
2301 if (SDValue Res = LowerDIVREM24(Op, DAG, true)) in LowerSDIVREM()
2306 DAG.ComputeNumSignBits(LHS) > 32 && in LowerSDIVREM()
2307 DAG.ComputeNumSignBits(RHS) > 32) { in LowerSDIVREM()
2308 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext()); in LowerSDIVREM()
2311 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero); in LowerSDIVREM()
2312 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero); in LowerSDIVREM()
2313 SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT), in LowerSDIVREM()
2316 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)), in LowerSDIVREM()
2317 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1)) in LowerSDIVREM()
2319 return DAG.getMergeValues(Res, DL); in LowerSDIVREM()
2322 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT); in LowerSDIVREM()
2323 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT); in LowerSDIVREM()
2324 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign); in LowerSDIVREM()
2327 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign); in LowerSDIVREM()
2328 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign); in LowerSDIVREM()
2330 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign); in LowerSDIVREM()
2331 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign); in LowerSDIVREM()
2333 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS); in LowerSDIVREM()
2336 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign); in LowerSDIVREM()
2337 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign); in LowerSDIVREM()
2339 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign); in LowerSDIVREM()
2340 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign); in LowerSDIVREM()
2346 return DAG.getMergeValues(Res, DL); in LowerSDIVREM()
2350 SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const { in LowerFREM()
2357 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags); in LowerFREM()
2358 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags); in LowerFREM()
2359 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags); in LowerFREM()
2361 return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags); in LowerFREM()
2364 SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const { in LowerFCEIL()
2372 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFCEIL()
2374 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFCEIL()
2375 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFCEIL()
2378 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFCEIL()
2380 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT); in LowerFCEIL()
2381 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); in LowerFCEIL()
2382 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); in LowerFCEIL()
2384 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero); in LowerFCEIL()
2386 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFCEIL()
2390 SelectionDAG &DAG) { in extractF64Exponent() argument
2394 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in extractF64Exponent()
2396 DAG.getConstant(FractBits - 32, SL, MVT::i32), in extractF64Exponent()
2397 DAG.getConstant(ExpBits, SL, MVT::i32)); in extractF64Exponent()
2398 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart, in extractF64Exponent()
2399 DAG.getConstant(1023, SL, MVT::i32)); in extractF64Exponent()
2404 SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const { in LowerFTRUNC()
2410 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in LowerFTRUNC()
2414 SDValue Hi = getHiHalf64(Src, DAG); in LowerFTRUNC()
2416 SDValue Exp = extractF64Exponent(Hi, SL, DAG); in LowerFTRUNC()
2421 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32); in LowerFTRUNC()
2422 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask); in LowerFTRUNC()
2425 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit}); in LowerFTRUNC()
2426 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64); in LowerFTRUNC()
2428 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src); in LowerFTRUNC()
2430 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64); in LowerFTRUNC()
2432 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp); in LowerFTRUNC()
2433 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64); in LowerFTRUNC()
2434 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not); in LowerFTRUNC()
2437 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32); in LowerFTRUNC()
2439 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32); in LowerFTRUNC()
2441 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT); in LowerFTRUNC()
2442 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT); in LowerFTRUNC()
2444 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0); in LowerFTRUNC()
2445 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1); in LowerFTRUNC()
2447 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2); in LowerFTRUNC()
2451 SelectionDAG &DAG) const { in LowerFROUNDEVEN()
2458 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64); in LowerFROUNDEVEN()
2459 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); in LowerFROUNDEVEN()
2463 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign); in LowerFROUNDEVEN()
2464 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign); in LowerFROUNDEVEN()
2466 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); in LowerFROUNDEVEN()
2469 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64); in LowerFROUNDEVEN()
2472 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFROUNDEVEN()
2473 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT); in LowerFROUNDEVEN()
2475 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2); in LowerFROUNDEVEN()
2479 SelectionDAG &DAG) const { in LowerFNEARBYINT()
2483 return DAG.getNode(ISD::FROUNDEVEN, SDLoc(Op), Op.getValueType(), in LowerFNEARBYINT()
2487 SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const { in LowerFRINT()
2490 return DAG.getNode(ISD::FROUNDEVEN, SDLoc(Op), VT, Arg); in LowerFRINT()
2498 SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const { in LowerFROUND()
2503 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X); in LowerFROUND()
2507 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T); in LowerFROUND()
2509 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff); in LowerFROUND()
2511 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT); in LowerFROUND()
2512 const SDValue One = DAG.getConstantFP(1.0, SL, VT); in LowerFROUND()
2515 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerFROUND()
2517 const SDValue Half = DAG.getConstantFP(0.5, SL, VT); in LowerFROUND()
2518 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE); in LowerFROUND()
2519 SDValue OneOrZeroFP = DAG.getNode(ISD::SELECT, SL, VT, Cmp, One, Zero); in LowerFROUND()
2521 SDValue SignedOffset = DAG.getNode(ISD::FCOPYSIGN, SL, VT, OneOrZeroFP, X); in LowerFROUND()
2522 return DAG.getNode(ISD::FADD, SL, VT, T, SignedOffset); in LowerFROUND()
2525 SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const { in LowerFFLOOR()
2533 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFFLOOR()
2535 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFFLOOR()
2536 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64); in LowerFFLOOR()
2539 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFFLOOR()
2541 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT); in LowerFFLOOR()
2542 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); in LowerFFLOOR()
2543 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); in LowerFFLOOR()
2545 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero); in LowerFFLOOR()
2547 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFFLOOR()
2574 bool AMDGPUTargetLowering::allowApproxFunc(const SelectionDAG &DAG, in allowApproxFunc() argument
2578 auto &Options = DAG.getTarget().Options; in allowApproxFunc()
2582 bool AMDGPUTargetLowering::needsDenormHandlingF32(const SelectionDAG &DAG, in needsDenormHandlingF32() argument
2586 DAG.getMachineFunction() in needsDenormHandlingF32()
2591 SDValue AMDGPUTargetLowering::getIsLtSmallestNormal(SelectionDAG &DAG, in getIsLtSmallestNormal() argument
2598 DAG.getConstantFP(APFloat::getSmallestNormalized(Semantics), SL, VT); in getIsLtSmallestNormal()
2602 SDValue IsLtSmallestNormal = DAG.getSetCC( in getIsLtSmallestNormal()
2603 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Src, in getIsLtSmallestNormal()
2609 SDValue AMDGPUTargetLowering::getIsFinite(SelectionDAG &DAG, SDValue Src, in getIsFinite() argument
2614 SDValue Inf = DAG.getConstantFP(APFloat::getInf(Semantics), SL, VT); in getIsFinite()
2616 SDValue Fabs = DAG.getNode(ISD::FABS, SL, VT, Src, Flags); in getIsFinite()
2617 SDValue IsFinite = DAG.getSetCC( in getIsFinite()
2618 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Fabs, in getIsFinite()
2626 AMDGPUTargetLowering::getScaledLogInput(SelectionDAG &DAG, const SDLoc SL, in getScaledLogInput() argument
2628 if (!needsDenormHandlingF32(DAG, Src, Flags)) in getScaledLogInput()
2634 DAG.getConstantFP(APFloat::getSmallestNormalized(Semantics), SL, VT); in getScaledLogInput()
2636 SDValue IsLtSmallestNormal = DAG.getSetCC( in getScaledLogInput()
2637 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Src, in getScaledLogInput()
2640 SDValue Scale32 = DAG.getConstantFP(0x1.0p+32, SL, VT); in getScaledLogInput()
2641 SDValue One = DAG.getConstantFP(1.0, SL, VT); in getScaledLogInput()
2643 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, Scale32, One, Flags); in getScaledLogInput()
2645 SDValue ScaledInput = DAG.getNode(ISD::FMUL, SL, VT, Src, ScaleFactor, Flags); in getScaledLogInput()
2649 SDValue AMDGPUTargetLowering::LowerFLOG2(SDValue Op, SelectionDAG &DAG) const { in LowerFLOG2()
2664 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags); in LowerFLOG2()
2665 SDValue Log = DAG.getNode(AMDGPUISD::LOG, SL, MVT::f32, Ext, Flags); in LowerFLOG2()
2666 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log, in LowerFLOG2()
2667 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in LowerFLOG2()
2671 getScaledLogInput(DAG, SL, Src, Flags); in LowerFLOG2()
2673 return DAG.getNode(AMDGPUISD::LOG, SL, VT, Src, Flags); in LowerFLOG2()
2675 SDValue Log2 = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags); in LowerFLOG2()
2677 SDValue ThirtyTwo = DAG.getConstantFP(32.0, SL, VT); in LowerFLOG2()
2678 SDValue Zero = DAG.getConstantFP(0.0, SL, VT); in LowerFLOG2()
2680 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, ThirtyTwo, Zero); in LowerFLOG2()
2681 return DAG.getNode(ISD::FSUB, SL, VT, Log2, ResultOffset, Flags); in LowerFLOG2()
2684 static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X, in getMad() argument
2686 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Y, Flags); in getMad()
2687 return DAG.getNode(ISD::FADD, SL, VT, Mul, C, Flags); in getMad()
2691 SelectionDAG &DAG) const { in LowerFLOGCommon()
2706 X = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, X, Flags); in LowerFLOGCommon()
2709 SDValue Lowered = LowerFLOGUnsafe(X, DL, DAG, IsLog10, Flags); in LowerFLOGCommon()
2711 return DAG.getNode(ISD::FP_ROUND, DL, VT, Lowered, in LowerFLOGCommon()
2712 DAG.getTargetConstant(0, DL, MVT::i32), Flags); in LowerFLOGCommon()
2718 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, DL, X, Flags); in LowerFLOGCommon()
2722 SDValue Y = DAG.getNode(AMDGPUISD::LOG, DL, VT, X, Flags); in LowerFLOGCommon()
2734 SDValue C = DAG.getConstantFP(IsLog10 ? c_log10 : c_log, DL, VT); in LowerFLOGCommon()
2735 SDValue CC = DAG.getConstantFP(IsLog10 ? cc_log10 : cc_log, DL, VT); in LowerFLOGCommon()
2737 R = DAG.getNode(ISD::FMUL, DL, VT, Y, C, Flags); in LowerFLOGCommon()
2738 SDValue NegR = DAG.getNode(ISD::FNEG, DL, VT, R, Flags); in LowerFLOGCommon()
2739 SDValue FMA0 = DAG.getNode(ISD::FMA, DL, VT, Y, C, NegR, Flags); in LowerFLOGCommon()
2740 SDValue FMA1 = DAG.getNode(ISD::FMA, DL, VT, Y, CC, FMA0, Flags); in LowerFLOGCommon()
2741 R = DAG.getNode(ISD::FADD, DL, VT, R, FMA1, Flags); in LowerFLOGCommon()
2751 SDValue CH = DAG.getConstantFP(IsLog10 ? ch_log10 : ch_log, DL, VT); in LowerFLOGCommon()
2752 SDValue CT = DAG.getConstantFP(IsLog10 ? ct_log10 : ct_log, DL, VT); in LowerFLOGCommon()
2754 SDValue YAsInt = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Y); in LowerFLOGCommon()
2755 SDValue MaskConst = DAG.getConstant(0xfffff000, DL, MVT::i32); in LowerFLOGCommon()
2756 SDValue YHInt = DAG.getNode(ISD::AND, DL, MVT::i32, YAsInt, MaskConst); in LowerFLOGCommon()
2757 SDValue YH = DAG.getNode(ISD::BITCAST, DL, MVT::f32, YHInt); in LowerFLOGCommon()
2758 SDValue YT = DAG.getNode(ISD::FSUB, DL, VT, Y, YH, Flags); in LowerFLOGCommon()
2760 SDValue YTCT = DAG.getNode(ISD::FMUL, DL, VT, YT, CT, Flags); in LowerFLOGCommon()
2761 SDValue Mad0 = getMad(DAG, DL, VT, YH, CT, YTCT, Flags); in LowerFLOGCommon()
2762 SDValue Mad1 = getMad(DAG, DL, VT, YT, CH, Mad0, Flags); in LowerFLOGCommon()
2763 R = getMad(DAG, DL, VT, YH, CH, Mad1); in LowerFLOGCommon()
2771 SDValue IsFinite = getIsFinite(DAG, Y, Flags); in LowerFLOGCommon()
2772 R = DAG.getNode(ISD::SELECT, DL, VT, IsFinite, R, Y, Flags); in LowerFLOGCommon()
2776 SDValue Zero = DAG.getConstantFP(0.0f, DL, VT); in LowerFLOGCommon()
2778 DAG.getConstantFP(IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f, DL, VT); in LowerFLOGCommon()
2780 DAG.getNode(ISD::SELECT, DL, VT, IsScaled, ShiftK, Zero, Flags); in LowerFLOGCommon()
2781 R = DAG.getNode(ISD::FSUB, DL, VT, R, Shift, Flags); in LowerFLOGCommon()
2787 SDValue AMDGPUTargetLowering::LowerFLOG10(SDValue Op, SelectionDAG &DAG) const { in LowerFLOG10()
2788 return LowerFLOGCommon(Op, DAG); in LowerFLOG10()
2794 SelectionDAG &DAG, bool IsLog10, in LowerFLOGUnsafe() argument
2804 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, SL, Src, Flags); in LowerFLOGUnsafe()
2806 SDValue LogSrc = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags); in LowerFLOGUnsafe()
2808 DAG.getConstantFP(-32.0 * Log2BaseInverted, SL, VT); in LowerFLOGUnsafe()
2810 SDValue Zero = DAG.getConstantFP(0.0f, SL, VT); in LowerFLOGUnsafe()
2812 SDValue ResultOffset = DAG.getNode(ISD::SELECT, SL, VT, IsScaled, in LowerFLOGUnsafe()
2815 SDValue Log2Inv = DAG.getConstantFP(Log2BaseInverted, SL, VT); in LowerFLOGUnsafe()
2818 return DAG.getNode(ISD::FMA, SL, VT, LogSrc, Log2Inv, ResultOffset, in LowerFLOGUnsafe()
2820 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, LogSrc, Log2Inv, Flags); in LowerFLOGUnsafe()
2821 return DAG.getNode(ISD::FADD, SL, VT, Mul, ResultOffset); in LowerFLOGUnsafe()
2825 SDValue Log2Operand = DAG.getNode(LogOp, SL, VT, Src, Flags); in LowerFLOGUnsafe()
2826 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT); in LowerFLOGUnsafe()
2828 return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand, in LowerFLOGUnsafe()
2832 SDValue AMDGPUTargetLowering::lowerFEXP2(SDValue Op, SelectionDAG &DAG) const { in lowerFEXP2()
2844 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags); in lowerFEXP2()
2845 SDValue Log = DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Ext, Flags); in lowerFEXP2()
2846 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log, in lowerFEXP2()
2847 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in lowerFEXP2()
2852 if (!needsDenormHandlingF32(DAG, Src, Flags)) in lowerFEXP2()
2853 return DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Src, Flags); in lowerFEXP2()
2859 SDValue RangeCheckConst = DAG.getConstantFP(-0x1.f80000p+6f, SL, VT); in lowerFEXP2()
2861 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in lowerFEXP2()
2864 DAG.getSetCC(SL, SetCCVT, Src, RangeCheckConst, ISD::SETOLT); in lowerFEXP2()
2866 SDValue SixtyFour = DAG.getConstantFP(0x1.0p+6f, SL, VT); in lowerFEXP2()
2867 SDValue Zero = DAG.getConstantFP(0.0, SL, VT); in lowerFEXP2()
2870 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, SixtyFour, Zero); in lowerFEXP2()
2872 SDValue AddInput = DAG.getNode(ISD::FADD, SL, VT, Src, AddOffset, Flags); in lowerFEXP2()
2873 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, AddInput, Flags); in lowerFEXP2()
2875 SDValue TwoExpNeg64 = DAG.getConstantFP(0x1.0p-64f, SL, VT); in lowerFEXP2()
2876 SDValue One = DAG.getConstantFP(1.0, SL, VT); in lowerFEXP2()
2878 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, TwoExpNeg64, One); in lowerFEXP2()
2880 return DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScale, Flags); in lowerFEXP2()
2884 SelectionDAG &DAG, in lowerFEXPUnsafe() argument
2887 const SDValue Log2E = DAG.getConstantFP(numbers::log2e, SL, VT); in lowerFEXPUnsafe()
2889 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) { in lowerFEXPUnsafe()
2891 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Log2E, Flags); in lowerFEXPUnsafe()
2892 return DAG.getNode(VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP in lowerFEXPUnsafe()
2897 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in lowerFEXPUnsafe()
2899 SDValue Threshold = DAG.getConstantFP(-0x1.5d58a0p+6f, SL, VT); in lowerFEXPUnsafe()
2900 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT); in lowerFEXPUnsafe()
2902 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+6f, SL, VT); in lowerFEXPUnsafe()
2904 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags); in lowerFEXPUnsafe()
2907 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X); in lowerFEXPUnsafe()
2909 SDValue ExpInput = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, Log2E, Flags); in lowerFEXPUnsafe()
2911 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, ExpInput, Flags); in lowerFEXPUnsafe()
2913 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.969d48p-93f, SL, VT); in lowerFEXPUnsafe()
2915 DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScaleFactor, Flags); in lowerFEXPUnsafe()
2917 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, Exp2, in lowerFEXPUnsafe()
2924 SelectionDAG &DAG, in lowerFEXP10Unsafe() argument
2929 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) { in lowerFEXP10Unsafe()
2931 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT); in lowerFEXP10Unsafe()
2932 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT); in lowerFEXP10Unsafe()
2934 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, X, K0, Flags); in lowerFEXP10Unsafe()
2935 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags); in lowerFEXP10Unsafe()
2936 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, X, K1, Flags); in lowerFEXP10Unsafe()
2937 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags); in lowerFEXP10Unsafe()
2938 return DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1); in lowerFEXP10Unsafe()
2947 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in lowerFEXP10Unsafe()
2949 SDValue Threshold = DAG.getConstantFP(-0x1.2f7030p+5f, SL, VT); in lowerFEXP10Unsafe()
2950 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT); in lowerFEXP10Unsafe()
2952 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+5f, SL, VT); in lowerFEXP10Unsafe()
2953 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags); in lowerFEXP10Unsafe()
2955 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X); in lowerFEXP10Unsafe()
2957 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT); in lowerFEXP10Unsafe()
2958 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT); in lowerFEXP10Unsafe()
2960 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K0, Flags); in lowerFEXP10Unsafe()
2961 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags); in lowerFEXP10Unsafe()
2962 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K1, Flags); in lowerFEXP10Unsafe()
2963 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags); in lowerFEXP10Unsafe()
2965 SDValue MulExps = DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1, Flags); in lowerFEXP10Unsafe()
2967 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.9f623ep-107f, SL, VT); in lowerFEXP10Unsafe()
2969 DAG.getNode(ISD::FMUL, SL, VT, MulExps, ResultScaleFactor, Flags); in lowerFEXP10Unsafe()
2971 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, MulExps, in lowerFEXP10Unsafe()
2975 SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const { in lowerFEXP()
2984 if (allowApproxFunc(DAG, Flags)) // TODO: Does this really require fast? in lowerFEXP()
2985 return lowerFEXPUnsafe(X, SL, DAG, Flags); in lowerFEXP()
2994 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, X, Flags); in lowerFEXP()
2995 SDValue Lowered = lowerFEXPUnsafe(Ext, SL, DAG, Flags); in lowerFEXP()
2996 return DAG.getNode(ISD::FP_ROUND, SL, VT, Lowered, in lowerFEXP()
2997 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in lowerFEXP()
3004 if (allowApproxFunc(DAG, Flags)) { in lowerFEXP()
3005 return IsExp10 ? lowerFEXP10Unsafe(X, SL, DAG, Flags) in lowerFEXP()
3006 : lowerFEXPUnsafe(X, SL, DAG, Flags); in lowerFEXP()
3043 SDValue C = DAG.getConstantFP(IsExp10 ? c_exp10 : c_exp, SL, VT); in lowerFEXP()
3044 SDValue CC = DAG.getConstantFP(IsExp10 ? cc_exp10 : cc_exp, SL, VT); in lowerFEXP()
3046 PH = DAG.getNode(ISD::FMUL, SL, VT, X, C, Flags); in lowerFEXP()
3047 SDValue NegPH = DAG.getNode(ISD::FNEG, SL, VT, PH, Flags); in lowerFEXP()
3048 SDValue FMA0 = DAG.getNode(ISD::FMA, SL, VT, X, C, NegPH, Flags); in lowerFEXP()
3049 PL = DAG.getNode(ISD::FMA, SL, VT, X, CC, FMA0, Flags); in lowerFEXP()
3057 SDValue CH = DAG.getConstantFP(IsExp10 ? ch_exp10 : ch_exp, SL, VT); in lowerFEXP()
3058 SDValue CL = DAG.getConstantFP(IsExp10 ? cl_exp10 : cl_exp, SL, VT); in lowerFEXP()
3060 SDValue XAsInt = DAG.getNode(ISD::BITCAST, SL, MVT::i32, X); in lowerFEXP()
3061 SDValue MaskConst = DAG.getConstant(0xfffff000, SL, MVT::i32); in lowerFEXP()
3062 SDValue XHAsInt = DAG.getNode(ISD::AND, SL, MVT::i32, XAsInt, MaskConst); in lowerFEXP()
3063 SDValue XH = DAG.getNode(ISD::BITCAST, SL, VT, XHAsInt); in lowerFEXP()
3064 SDValue XL = DAG.getNode(ISD::FSUB, SL, VT, X, XH, Flags); in lowerFEXP()
3066 PH = DAG.getNode(ISD::FMUL, SL, VT, XH, CH, Flags); in lowerFEXP()
3068 SDValue XLCL = DAG.getNode(ISD::FMUL, SL, VT, XL, CL, Flags); in lowerFEXP()
3069 SDValue Mad0 = getMad(DAG, SL, VT, XL, CH, XLCL, Flags); in lowerFEXP()
3070 PL = getMad(DAG, SL, VT, XH, CL, Mad0, Flags); in lowerFEXP()
3073 SDValue E = DAG.getNode(ISD::FROUNDEVEN, SL, VT, PH, Flags); in lowerFEXP()
3076 SDValue PHSubE = DAG.getNode(ISD::FSUB, SL, VT, PH, E, FlagsNoContract); in lowerFEXP()
3078 SDValue A = DAG.getNode(ISD::FADD, SL, VT, PHSubE, PL, Flags); in lowerFEXP()
3079 SDValue IntE = DAG.getNode(ISD::FP_TO_SINT, SL, MVT::i32, E); in lowerFEXP()
3080 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, A, Flags); in lowerFEXP()
3082 SDValue R = DAG.getNode(ISD::FLDEXP, SL, VT, Exp2, IntE, Flags); in lowerFEXP()
3085 DAG.getConstantFP(IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, SL, VT); in lowerFEXP()
3087 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in lowerFEXP()
3088 SDValue Zero = DAG.getConstantFP(0.0, SL, VT); in lowerFEXP()
3090 DAG.getSetCC(SL, SetCCVT, X, UnderflowCheckConst, ISD::SETOLT); in lowerFEXP()
3092 R = DAG.getNode(ISD::SELECT, SL, VT, Underflow, Zero, R); in lowerFEXP()
3097 DAG.getConstantFP(IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, SL, VT); in lowerFEXP()
3099 DAG.getSetCC(SL, SetCCVT, X, OverflowCheckConst, ISD::SETOGT); in lowerFEXP()
3101 DAG.getConstantFP(APFloat::getInf(APFloat::IEEEsingle()), SL, VT); in lowerFEXP()
3102 R = DAG.getNode(ISD::SELECT, SL, VT, Overflow, Inf, R); in lowerFEXP()
3117 SelectionDAG &DAG) const { in lowerCTLZResults()
3130 SDValue NumExtBits = DAG.getConstant(32u - NumBits, SL, MVT::i32); in lowerCTLZResults()
3134 NewOp = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Arg); in lowerCTLZResults()
3135 NewOp = DAG.getNode(ISD::SHL, SL, MVT::i32, NewOp, NumExtBits); in lowerCTLZResults()
3136 NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp); in lowerCTLZResults()
3138 NewOp = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Arg); in lowerCTLZResults()
3139 NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp); in lowerCTLZResults()
3140 NewOp = DAG.getNode(ISD::SUB, SL, MVT::i32, NewOp, NumExtBits); in lowerCTLZResults()
3143 return DAG.getNode(ISD::TRUNCATE, SL, ResultVT, NewOp); in lowerCTLZResults()
3146 SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const { in LowerCTLZ_CTTZ()
3169 SDValue NewOpr = DAG.getNode(NewOpc, SL, MVT::i32, Src); in LowerCTLZ_CTTZ()
3171 const SDValue ConstVal = DAG.getConstant( in LowerCTLZ_CTTZ()
3173 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, ConstVal); in LowerCTLZ_CTTZ()
3175 return DAG.getNode(ISD::ZERO_EXTEND, SL, Src.getValueType(), NewOpr); in LowerCTLZ_CTTZ()
3179 std::tie(Lo, Hi) = split64BitValue(Src, DAG); in LowerCTLZ_CTTZ()
3181 SDValue OprLo = DAG.getNode(NewOpc, SL, MVT::i32, Lo); in LowerCTLZ_CTTZ()
3182 SDValue OprHi = DAG.getNode(NewOpc, SL, MVT::i32, Hi); in LowerCTLZ_CTTZ()
3190 const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32); in LowerCTLZ_CTTZ()
3192 OprLo = DAG.getNode(AddOpc, SL, MVT::i32, OprLo, Const32); in LowerCTLZ_CTTZ()
3194 OprHi = DAG.getNode(AddOpc, SL, MVT::i32, OprHi, Const32); in LowerCTLZ_CTTZ()
3197 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, OprLo, OprHi); in LowerCTLZ_CTTZ()
3199 const SDValue Const64 = DAG.getConstant(64, SL, MVT::i32); in LowerCTLZ_CTTZ()
3200 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const64); in LowerCTLZ_CTTZ()
3203 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr); in LowerCTLZ_CTTZ()
3206 SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG, in LowerINT_TO_FP32() argument
3238 std::tie(Lo, Hi) = split64BitValue(Src, DAG); in LowerINT_TO_FP32()
3263 SDValue OppositeSign = DAG.getNode( in LowerINT_TO_FP32()
3264 ISD::SRA, SL, MVT::i32, DAG.getNode(ISD::XOR, SL, MVT::i32, Lo, Hi), in LowerINT_TO_FP32()
3265 DAG.getConstant(31, SL, MVT::i32)); in LowerINT_TO_FP32()
3267 DAG.getNode(ISD::ADD, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
3270 ShAmt = DAG.getNode(AMDGPUISD::FFBH_I32, SL, MVT::i32, Hi); in LowerINT_TO_FP32()
3273 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
3274 DAG.getConstant(1, SL, MVT::i32)); in LowerINT_TO_FP32()
3275 ShAmt = DAG.getNode(ISD::UMIN, SL, MVT::i32, ShAmt, MaxShAmt); in LowerINT_TO_FP32()
3280 Sign = DAG.getNode(ISD::SRA, SL, MVT::i64, Src, in LowerINT_TO_FP32()
3281 DAG.getConstant(63, SL, MVT::i64)); in LowerINT_TO_FP32()
3283 DAG.getNode(ISD::XOR, SL, MVT::i64, in LowerINT_TO_FP32()
3284 DAG.getNode(ISD::ADD, SL, MVT::i64, Src, Sign), Sign); in LowerINT_TO_FP32()
3285 std::tie(Lo, Hi) = split64BitValue(Abs, DAG); in LowerINT_TO_FP32()
3288 ShAmt = DAG.getNode(ISD::CTLZ, SL, MVT::i32, Hi); in LowerINT_TO_FP32()
3292 SDValue Norm = DAG.getNode(ISD::SHL, SL, MVT::i64, Src, ShAmt); in LowerINT_TO_FP32()
3294 std::tie(Lo, Hi) = split64BitValue(Norm, DAG); in LowerINT_TO_FP32()
3297 SDValue Adjust = DAG.getNode(ISD::UMIN, SL, MVT::i32, in LowerINT_TO_FP32()
3298 DAG.getConstant(1, SL, MVT::i32), Lo); in LowerINT_TO_FP32()
3300 Norm = DAG.getNode(ISD::OR, SL, MVT::i32, Hi, Adjust); in LowerINT_TO_FP32()
3304 SDValue FVal = DAG.getNode(Opc, SL, MVT::f32, Norm); in LowerINT_TO_FP32()
3308 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
3312 return DAG.getNode(ISD::FLDEXP, SL, MVT::f32, FVal, ShAmt); in LowerINT_TO_FP32()
3317 SDValue Exp = DAG.getNode(ISD::SHL, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
3318 DAG.getConstant(23, SL, MVT::i32)); in LowerINT_TO_FP32()
3320 DAG.getNode(ISD::ADD, SL, MVT::i32, in LowerINT_TO_FP32()
3321 DAG.getNode(ISD::BITCAST, SL, MVT::i32, FVal), Exp); in LowerINT_TO_FP32()
3324 Sign = DAG.getNode(ISD::SHL, SL, MVT::i32, in LowerINT_TO_FP32()
3325 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Sign), in LowerINT_TO_FP32()
3326 DAG.getConstant(31, SL, MVT::i32)); in LowerINT_TO_FP32()
3327 IVal = DAG.getNode(ISD::OR, SL, MVT::i32, IVal, Sign); in LowerINT_TO_FP32()
3329 return DAG.getNode(ISD::BITCAST, SL, MVT::f32, IVal); in LowerINT_TO_FP32()
3332 SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG, in LowerINT_TO_FP64() argument
3338 std::tie(Lo, Hi) = split64BitValue(Src, DAG); in LowerINT_TO_FP64()
3340 SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, in LowerINT_TO_FP64()
3343 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo); in LowerINT_TO_FP64()
3345 SDValue LdExp = DAG.getNode(ISD::FLDEXP, SL, MVT::f64, CvtHi, in LowerINT_TO_FP64()
3346 DAG.getConstant(32, SL, MVT::i32)); in LowerINT_TO_FP64()
3348 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo); in LowerINT_TO_FP64()
3352 SelectionDAG &DAG) const { in LowerUINT_TO_FP()
3364 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Src); in LowerUINT_TO_FP()
3365 return DAG.getNode(ISD::UINT_TO_FP, DL, DestVT, Ext); in LowerUINT_TO_FP()
3370 SDValue ToF32 = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f32, Src); in LowerUINT_TO_FP()
3371 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SL, /*isTarget=*/true); in LowerUINT_TO_FP()
3372 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag); in LowerUINT_TO_FP()
3381 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); in LowerUINT_TO_FP()
3383 DAG.getIntPtrConstant(0, SDLoc(Op), /*isTarget=*/true); in LowerUINT_TO_FP()
3385 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerUINT_TO_FP()
3391 return LowerINT_TO_FP32(Op, DAG, false); in LowerUINT_TO_FP()
3394 return LowerINT_TO_FP64(Op, DAG, false); in LowerUINT_TO_FP()
3398 SelectionDAG &DAG) const { in LowerSINT_TO_FP()
3410 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, Src); in LowerSINT_TO_FP()
3411 return DAG.getNode(ISD::SINT_TO_FP, DL, DestVT, Ext); in LowerSINT_TO_FP()
3416 SDValue ToF32 = DAG.getNode(ISD::SINT_TO_FP, SL, MVT::f32, Src); in LowerSINT_TO_FP()
3417 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SL, /*isTarget=*/true); in LowerSINT_TO_FP()
3418 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag); in LowerSINT_TO_FP()
3430 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); in LowerSINT_TO_FP()
3432 DAG.getIntPtrConstant(0, SDLoc(Op), /*isTarget=*/true); in LowerSINT_TO_FP()
3434 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerSINT_TO_FP()
3440 return LowerINT_TO_FP32(Op, DAG, true); in LowerSINT_TO_FP()
3443 return LowerINT_TO_FP64(Op, DAG, true); in LowerSINT_TO_FP()
3446 SDValue AMDGPUTargetLowering::LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG, in LowerFP_TO_INT64() argument
3464 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, SrcVT, Src); in LowerFP_TO_INT64()
3472 Sign = DAG.getNode(ISD::SRA, SL, MVT::i32, in LowerFP_TO_INT64()
3473 DAG.getNode(ISD::BITCAST, SL, MVT::i32, Trunc), in LowerFP_TO_INT64()
3474 DAG.getConstant(31, SL, MVT::i32)); in LowerFP_TO_INT64()
3475 Trunc = DAG.getNode(ISD::FABS, SL, SrcVT, Trunc); in LowerFP_TO_INT64()
3480 K0 = DAG.getConstantFP( in LowerFP_TO_INT64()
3483 K1 = DAG.getConstantFP( in LowerFP_TO_INT64()
3487 K0 = DAG.getConstantFP( in LowerFP_TO_INT64()
3489 K1 = DAG.getConstantFP( in LowerFP_TO_INT64()
3493 SDValue Mul = DAG.getNode(ISD::FMUL, SL, SrcVT, Trunc, K0); in LowerFP_TO_INT64()
3495 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, SrcVT, Mul); in LowerFP_TO_INT64()
3497 SDValue Fma = DAG.getNode(ISD::FMA, SL, SrcVT, FloorMul, K1, Trunc); in LowerFP_TO_INT64()
3499 SDValue Hi = DAG.getNode((Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT in LowerFP_TO_INT64()
3502 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma); in LowerFP_TO_INT64()
3504 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
3505 DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi})); in LowerFP_TO_INT64()
3510 Sign = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
3511 DAG.getBuildVector(MVT::v2i32, SL, {Sign, Sign})); in LowerFP_TO_INT64()
3514 DAG.getNode(ISD::SUB, SL, MVT::i64, in LowerFP_TO_INT64()
3515 DAG.getNode(ISD::XOR, SL, MVT::i64, Result, Sign), Sign); in LowerFP_TO_INT64()
3521 SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const { in LowerFP_TO_FP16()
3527 return DAG.getNode(AMDGPUISD::FP_TO_FP16, DL, Op.getValueType(), N0); in LowerFP_TO_FP16()
3540 SDValue Zero = DAG.getConstant(0, DL, MVT::i32); in LowerFP_TO_FP16()
3541 SDValue One = DAG.getConstant(1, DL, MVT::i32); in LowerFP_TO_FP16()
3542 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0); in LowerFP_TO_FP16()
3543 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U, in LowerFP_TO_FP16()
3544 DAG.getConstant(32, DL, MVT::i64)); in LowerFP_TO_FP16()
3545 UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32); in LowerFP_TO_FP16()
3546 U = DAG.getZExtOrTrunc(U, DL, MVT::i32); in LowerFP_TO_FP16()
3547 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3548 DAG.getConstant(20, DL, MVT::i64)); in LowerFP_TO_FP16()
3549 E = DAG.getNode(ISD::AND, DL, MVT::i32, E, in LowerFP_TO_FP16()
3550 DAG.getConstant(ExpMask, DL, MVT::i32)); in LowerFP_TO_FP16()
3553 E = DAG.getNode(ISD::ADD, DL, MVT::i32, E, in LowerFP_TO_FP16()
3554 DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32)); in LowerFP_TO_FP16()
3556 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3557 DAG.getConstant(8, DL, MVT::i32)); in LowerFP_TO_FP16()
3558 M = DAG.getNode(ISD::AND, DL, MVT::i32, M, in LowerFP_TO_FP16()
3559 DAG.getConstant(0xffe, DL, MVT::i32)); in LowerFP_TO_FP16()
3561 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3562 DAG.getConstant(0x1ff, DL, MVT::i32)); in LowerFP_TO_FP16()
3563 MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U); in LowerFP_TO_FP16()
3565 SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ); in LowerFP_TO_FP16()
3566 M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set); in LowerFP_TO_FP16()
3569 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32, in LowerFP_TO_FP16()
3570 DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32), in LowerFP_TO_FP16()
3571 Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32)); in LowerFP_TO_FP16()
3574 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
3575 DAG.getNode(ISD::SHL, DL, MVT::i32, E, in LowerFP_TO_FP16()
3576 DAG.getConstant(12, DL, MVT::i32))); in LowerFP_TO_FP16()
3579 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32, in LowerFP_TO_FP16()
3581 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero); in LowerFP_TO_FP16()
3582 B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B, in LowerFP_TO_FP16()
3583 DAG.getConstant(13, DL, MVT::i32)); in LowerFP_TO_FP16()
3585 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
3586 DAG.getConstant(0x1000, DL, MVT::i32)); in LowerFP_TO_FP16()
3588 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B); in LowerFP_TO_FP16()
3589 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B); in LowerFP_TO_FP16()
3590 SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE); in LowerFP_TO_FP16()
3591 D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1); in LowerFP_TO_FP16()
3593 SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT); in LowerFP_TO_FP16()
3594 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V, in LowerFP_TO_FP16()
3595 DAG.getConstant(0x7, DL, MVT::i32)); in LowerFP_TO_FP16()
3596 V = DAG.getNode(ISD::SRL, DL, MVT::i32, V, in LowerFP_TO_FP16()
3597 DAG.getConstant(2, DL, MVT::i32)); in LowerFP_TO_FP16()
3598 SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32), in LowerFP_TO_FP16()
3600 SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32), in LowerFP_TO_FP16()
3602 V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1); in LowerFP_TO_FP16()
3603 V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1); in LowerFP_TO_FP16()
3605 V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32), in LowerFP_TO_FP16()
3606 DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT); in LowerFP_TO_FP16()
3607 V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32), in LowerFP_TO_FP16()
3611 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3612 DAG.getConstant(16, DL, MVT::i32)); in LowerFP_TO_FP16()
3613 Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign, in LowerFP_TO_FP16()
3614 DAG.getConstant(0x8000, DL, MVT::i32)); in LowerFP_TO_FP16()
3616 V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V); in LowerFP_TO_FP16()
3617 return DAG.getZExtOrTrunc(V, DL, Op.getValueType()); in LowerFP_TO_FP16()
3621 SelectionDAG &DAG) const { in LowerFP_TO_INT()
3633 SDValue PromotedSrc = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src); in LowerFP_TO_INT()
3634 return DAG.getNode(Op.getOpcode(), DL, DestVT, PromotedSrc); in LowerFP_TO_INT()
3641 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src); in LowerFP_TO_INT()
3642 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToInt32); in LowerFP_TO_INT()
3652 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src); in LowerFP_TO_INT()
3655 return DAG.getNode(Ext, DL, MVT::i64, FpToInt32); in LowerFP_TO_INT()
3659 return LowerFP_TO_INT64(Op, DAG, OpOpcode == ISD::FP_TO_SINT); in LowerFP_TO_INT()
3665 SelectionDAG &DAG) const { in LowerSIGN_EXTEND_INREG()
3678 DAG.ExtractVectorElements(Src, Args, 0, NElts); in LowerSIGN_EXTEND_INREG()
3680 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType()); in LowerSIGN_EXTEND_INREG()
3682 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp); in LowerSIGN_EXTEND_INREG()
3684 return DAG.getBuildVector(VT, DL, Args); in LowerSIGN_EXTEND_INREG()
3691 static bool isU24(SDValue Op, SelectionDAG &DAG) { in isU24() argument
3692 return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24; in isU24()
3695 static bool isI24(SDValue Op, SelectionDAG &DAG) { in isI24() argument
3699 AMDGPUTargetLowering::numBitsSigned(Op, DAG) <= 24; in isI24()
3704 SelectionDAG &DAG = DCI.DAG; in simplifyMul24() local
3705 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in simplifyMul24()
3736 SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(LHS, Demanded, DAG); in simplifyMul24()
3737 SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(RHS, Demanded, DAG); in simplifyMul24()
3739 return DAG.getNode(NewOpcode, SDLoc(Node24), Node24->getVTList(), in simplifyMul24()
3754 static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset, in constantFoldBFE() argument
3759 return DAG.getConstant(Result, DL, MVT::i32); in constantFoldBFE()
3762 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32); in constantFoldBFE()
3807 SelectionDAG &DAG = DCI.DAG; in performLoadCombine() local
3822 return SplitVectorLoad(SDValue(LN, 0), DAG); in performLoadCombine()
3825 std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG); in performLoadCombine()
3827 return DAG.getMergeValues(Ops, SDLoc(N)); in performLoadCombine()
3837 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT); in performLoadCombine()
3840 = DAG.getLoad(NewVT, SL, LN->getChain(), in performLoadCombine()
3843 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad); in performLoadCombine()
3863 SelectionDAG &DAG = DCI.DAG; in performStoreCombine() local
3876 return SplitVectorStore(SDValue(SN, 0), DAG); in performStoreCombine()
3878 return expandUnalignedStore(SN, DAG); in performStoreCombine()
3888 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT); in performStoreCombine()
3894 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val); in performStoreCombine()
3896 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal); in performStoreCombine()
3897 DAG.ReplaceAllUsesOfValueWith(Val, CastBack); in performStoreCombine()
3900 return DAG.getStore(SN->getChain(), SL, CastVal, in performStoreCombine()
3909 SelectionDAG &DAG = DCI.DAG; in performAssertSZExtCombine() local
3922 SDValue NewInReg = DAG.getNode(N->getOpcode(), SL, SrcVT, Src, N1); in performAssertSZExtCombine()
3923 return DAG.getNode(ISD::TRUNCATE, SL, N->getValueType(0), NewInReg); in performAssertSZExtCombine()
3956 return SDValue(DCI.DAG.UpdateNodeOperands(N, N->getOperand(0), PeekSign), in performIntrinsicWOChainCombine()
3970 SelectionDAG &DAG = DCI.DAG; in splitBinaryBitConstantOpImpl() local
3972 std::tie(Lo, Hi) = split64BitValue(LHS, DAG); in splitBinaryBitConstantOpImpl()
3974 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32); in splitBinaryBitConstantOpImpl()
3975 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32); in splitBinaryBitConstantOpImpl()
3977 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS); in splitBinaryBitConstantOpImpl()
3978 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS); in splitBinaryBitConstantOpImpl()
3985 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd}); in splitBinaryBitConstantOpImpl()
3986 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in splitBinaryBitConstantOpImpl()
4003 SelectionDAG &DAG = DCI.DAG; in performShlCombine() local
4017 SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL, in performShlCombine()
4018 { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) }); in performShlCombine()
4019 return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec); in performShlCombine()
4025 KnownBits Known = DAG.computeKnownBits(X); in performShlCombine()
4030 SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0)); in performShlCombine()
4031 return DAG.getZExtOrTrunc(Shl, SL, VT); in performShlCombine()
4046 SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32); in performShlCombine()
4048 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS); in performShlCombine()
4049 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt); in performShlCombine()
4051 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in performShlCombine()
4053 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift}); in performShlCombine()
4054 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performShlCombine()
4066 SelectionDAG &DAG = DCI.DAG; in performSraCombine() local
4072 SDValue Hi = getHiHalf64(N->getOperand(0), DAG); in performSraCombine()
4073 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
4074 DAG.getConstant(31, SL, MVT::i32)); in performSraCombine()
4076 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift}); in performSraCombine()
4077 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
4082 SDValue Hi = getHiHalf64(N->getOperand(0), DAG); in performSraCombine()
4083 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
4084 DAG.getConstant(31, SL, MVT::i32)); in performSraCombine()
4085 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift}); in performSraCombine()
4086 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
4101 SelectionDAG &DAG = DCI.DAG; in performSrlCombine() local
4111 return DAG.getNode( in performSrlCombine()
4113 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)), in performSrlCombine()
4114 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1))); in performSrlCombine()
4128 SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in performSrlCombine()
4130 SDValue Hi = getHiHalf64(LHS, DAG); in performSrlCombine()
4132 SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32); in performSrlCombine()
4133 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst); in performSrlCombine()
4135 SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero}); in performSrlCombine()
4137 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair); in performSrlCombine()
4143 SelectionDAG &DAG = DCI.DAG; in performTruncateCombine() local
4155 Elt0 = DAG.getNode(ISD::BITCAST, SL, in performTruncateCombine()
4159 return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0); in performTruncateCombine()
4176 SrcElt = DAG.getNode(ISD::BITCAST, SL, in performTruncateCombine()
4180 return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt); in performTruncateCombine()
4197 KnownBits Known = DAG.computeKnownBits(Amt); in performTruncateCombine()
4207 EVT::getVectorVT(*DAG.getContext(), MVT::i32, in performTruncateCombine()
4210 EVT NewShiftVT = getShiftAmountTy(MidVT, DAG.getDataLayout()); in performTruncateCombine()
4211 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT, in performTruncateCombine()
4216 Amt = DAG.getZExtOrTrunc(Amt, SL, NewShiftVT); in performTruncateCombine()
4220 SDValue ShrunkShift = DAG.getNode(Src.getOpcode(), SL, MidVT, in performTruncateCombine()
4222 return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift); in performTruncateCombine()
4234 static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL, in getMul24() argument
4238 return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1); in getMul24()
4244 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1); in getMul24()
4245 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1); in getMul24()
4247 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi); in getMul24()
4275 SelectionDAG &DAG = DCI.DAG; in performMulCombine() local
4301 SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N1, MulOper); in performMulCombine()
4302 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N1); in performMulCombine()
4306 SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N0, MulOper); in performMulCombine()
4307 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N0); in performMulCombine()
4326 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) { in performMulCombine()
4327 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulCombine()
4328 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulCombine()
4329 Mul = getMul24(DAG, DL, N0, N1, Size, false); in performMulCombine()
4330 } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) { in performMulCombine()
4331 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulCombine()
4332 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulCombine()
4333 Mul = getMul24(DAG, DL, N0, N1, Size, true); in performMulCombine()
4340 return DAG.getSExtOrTrunc(Mul, DL, VT); in performMulCombine()
4349 SelectionDAG &DAG = DCI.DAG; in performMulLoHiCombine() local
4370 if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) { in performMulLoHiCombine()
4371 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulLoHiCombine()
4372 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulLoHiCombine()
4377 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) { in performMulLoHiCombine()
4378 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulLoHiCombine()
4379 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulLoHiCombine()
4387 SDValue Lo = DAG.getNode(LoOpcode, DL, MVT::i32, N0, N1); in performMulLoHiCombine()
4388 SDValue Hi = DAG.getNode(HiOpcode, DL, MVT::i32, N0, N1); in performMulLoHiCombine()
4409 SelectionDAG &DAG = DCI.DAG; in performMulhsCombine() local
4415 if (!isI24(N0, DAG) || !isI24(N1, DAG)) in performMulhsCombine()
4418 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulhsCombine()
4419 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulhsCombine()
4421 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1); in performMulhsCombine()
4423 return DAG.getSExtOrTrunc(Mulhi, DL, VT); in performMulhsCombine()
4442 SelectionDAG &DAG = DCI.DAG; in performMulhuCombine() local
4448 if (!isU24(N0, DAG) || !isU24(N1, DAG)) in performMulhuCombine()
4451 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulhuCombine()
4452 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulhuCombine()
4454 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1); in performMulhuCombine()
4456 return DAG.getZExtOrTrunc(Mulhi, DL, VT); in performMulhuCombine()
4459 SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG, in getFFBX_U32() argument
4464 EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT); in getFFBX_U32()
4470 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op); in getFFBX_U32()
4472 SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op); in getFFBX_U32()
4474 FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX); in getFFBX_U32()
4492 SelectionDAG &DAG = DCI.DAG; in performCtlz_CttzCombine() local
4503 return getFFBX_U32(DAG, CmpLHS, SL, Opc); in performCtlz_CttzCombine()
4514 return getFFBX_U32(DAG, CmpLHS, SL, Opc); in performCtlz_CttzCombine()
4526 SelectionDAG &DAG = DCI.DAG; in distributeOpThroughSelect() local
4529 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond, in distributeOpThroughSelect()
4532 return DAG.getNode(Op, SL, VT, NewSelect); in distributeOpThroughSelect()
4545 SelectionDAG &DAG = DCI.DAG; in foldFreeOpFromSelect() local
4605 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in foldFreeOpFromSelect()
4610 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, in foldFreeOpFromSelect()
4613 return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect); in foldFreeOpFromSelect()
4638 SelectionDAG &DAG = DCI.DAG; in performSelectCombine() local
4639 if (DAG.isConstantValueOfAnyType(True) && in performSelectCombine()
4640 !DAG.isConstantValueOfAnyType(False)) { in performSelectCombine()
4649 SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC); in performSelectCombine()
4650 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True); in performSelectCombine()
4747 SelectionDAG &DAG = DCI.DAG; in performFNegCombine() local
4767 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
4772 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4776 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags()); in performFNegCombine()
4780 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4795 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4797 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags()); in performFNegCombine()
4801 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4820 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS); in performFNegCombine()
4823 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4827 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS); in performFNegCombine()
4831 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4855 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
4856 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
4859 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags()); in performFNegCombine()
4863 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
4869 Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags()); in performFNegCombine()
4871 SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags()); in performFNegCombine()
4876 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Res); in performFNegCombine()
4877 DAG.ReplaceAllUsesWith(N0, Neg); in performFNegCombine()
4900 return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0)); in performFNegCombine()
4908 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc); in performFNegCombine()
4909 return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags()); in performFNegCombine()
4916 return DAG.getNode(ISD::FP_ROUND, SL, VT, in performFNegCombine()
4924 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc); in performFNegCombine()
4925 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1)); in performFNegCombine()
4937 SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src, in performFNegCombine()
4938 DAG.getConstant(0x8000, SL, SrcVT)); in performFNegCombine()
4939 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg); in performFNegCombine()
4964 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::f32, HighBits); in performFNegCombine()
4965 SDValue NegHi = DAG.getNode(ISD::FNEG, SL, MVT::f32, CastHi); in performFNegCombine()
4967 DAG.getNode(ISD::BITCAST, SL, HighBits.getValueType(), NegHi); in performFNegCombine()
4973 DAG.getNode(ISD::BUILD_VECTOR, SL, BCSrc.getValueType(), Ops); in performFNegCombine()
4974 SDValue Result = DAG.getNode(ISD::BITCAST, SL, VT, Build); in performFNegCombine()
4977 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Result)); in performFNegCombine()
4989 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(1)); in performFNegCombine()
4991 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(2)); in performFNegCombine()
4993 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, LHS); in performFNegCombine()
4994 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, RHS); in performFNegCombine()
4996 return DAG.getNode(ISD::SELECT, SL, MVT::f32, BCSrc.getOperand(0), NegLHS, in performFNegCombine()
5009 SelectionDAG &DAG = DCI.DAG; in performFAbsCombine() local
5023 SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src, in performFAbsCombine()
5024 DAG.getConstant(0x7fff, SL, SrcVT)); in performFAbsCombine()
5025 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs); in performFAbsCombine()
5041 return DCI.DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0)); in performRcpCombine()
5046 SelectionDAG &DAG = DCI.DAG; in PerformDAGCombine() local
5075 CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt)); in PerformDAGCombine()
5078 return DAG.getBuildVector(DestVT, SL, CastedElts); in PerformDAGCombine()
5094 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
5095 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), in PerformDAGCombine()
5096 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); in PerformDAGCombine()
5097 return DAG.getNode(ISD::BITCAST, SL, DestVT, BV); in PerformDAGCombine()
5104 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
5105 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), in PerformDAGCombine()
5106 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); in PerformDAGCombine()
5108 return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec); in PerformDAGCombine()
5167 return DAG.getConstant(0, DL, MVT::i32); in PerformDAGCombine()
5182 unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom); in PerformDAGCombine()
5186 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal); in PerformDAGCombine()
5194 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom, in PerformDAGCombine()
5195 DAG.getValueType(SmallVT)); in PerformDAGCombine()
5198 return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT); in PerformDAGCombine()
5203 return constantFoldBFE<int32_t>(DAG, in PerformDAGCombine()
5210 return constantFoldBFE<uint32_t>(DAG, in PerformDAGCombine()
5219 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32); in PerformDAGCombine()
5220 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32, in PerformDAGCombine()
5230 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), in PerformDAGCombine()
5232 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in PerformDAGCombine()
5279 return DAG.getConstantFP(FTZ(V0), DL, VT); in PerformDAGCombine()
5291 SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG, in CreateLiveInRegister() argument
5296 MachineFunction &MF = DAG.getMachineFunction(); in CreateLiveInRegister()
5308 return DAG.getRegister(VReg, VT); in CreateLiveInRegister()
5310 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT); in CreateLiveInRegister()
5327 SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG, in loadStackInputValue() argument
5331 MachineFunction &MF = DAG.getMachineFunction(); in loadStackInputValue()
5336 SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32); in loadStackInputValue()
5338 return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, Align(4), in loadStackInputValue()
5343 SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG, in storeStackInputValue() argument
5348 MachineFunction &MF = DAG.getMachineFunction(); in storeStackInputValue()
5352 SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32); in storeStackInputValue()
5355 DAG.getCopyFromReg(Chain, SL, Info->getStackPtrOffsetReg(), MVT::i32); in storeStackInputValue()
5356 Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, SP, Ptr); in storeStackInputValue()
5357 SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, Align(4), in storeStackInputValue()
5362 SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG, in loadInputValue() argument
5369 CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) : in loadInputValue()
5370 loadStackInputValue(DAG, VT, SL, Arg.getStackOffset()); in loadInputValue()
5377 V = DAG.getNode(ISD::SRL, SL, VT, V, in loadInputValue()
5378 DAG.getShiftAmountConstant(Shift, VT, SL)); in loadInputValue()
5379 return DAG.getNode(ISD::AND, SL, VT, V, in loadInputValue()
5380 DAG.getConstant(Mask >> Shift, SL, VT)); in loadInputValue()
5578 SelectionDAG &DAG, int Enabled, in getSqrtEstimate() argument
5586 return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand); in getSqrtEstimate()
5596 SelectionDAG &DAG, int Enabled, in getRecipEstimate() argument
5607 return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand); in getRecipEstimate()
5631 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const { in computeKnownBitsForTargetNode() argument
5668 KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1); in computeKnownBitsForTargetNode()
5669 KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1); in computeKnownBitsForTargetNode()
5714 KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1); in computeKnownBitsForTargetNode()
5715 KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1); in computeKnownBitsForTargetNode()
5747 Align Alignment = GA->getGlobal()->getPointerAlignment(DAG.getDataLayout()); in computeKnownBitsForTargetNode()
5759 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(2), Depth + 1); in computeKnownBitsForTargetNode()
5763 KnownBits Known1 = DAG.computeKnownBits(Op.getOperand(1), Depth + 1); in computeKnownBitsForTargetNode()
5767 KnownBits Known0 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1); in computeKnownBitsForTargetNode()
5783 DAG.getMachineFunction().getFunction(), workitemIntrinsicDim(IID)); in computeKnownBitsForTargetNode()
5795 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, in ComputeNumSignBitsForTargetNode() argument
5808 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1); in ComputeNumSignBitsForTargetNode()
5836 unsigned Tmp2 = DAG.ComputeNumSignBits(Op.getOperand(2), Depth + 1); in ComputeNumSignBitsForTargetNode()
5840 unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth + 1); in ComputeNumSignBitsForTargetNode()
5844 unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1); in ComputeNumSignBitsForTargetNode()
5893 const SelectionDAG &DAG, in isKnownNeverNaNForTargetNode() argument
5911 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) && in isKnownNeverNaNForTargetNode()
5912 DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); in isKnownNeverNaNForTargetNode()
5922 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) && in isKnownNeverNaNForTargetNode()
5923 DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && in isKnownNeverNaNForTargetNode()
5924 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1); in isKnownNeverNaNForTargetNode()
5946 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); in isKnownNeverNaNForTargetNode()
5968 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); in isKnownNeverNaNForTargetNode()
5973 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && in isKnownNeverNaNForTargetNode()
5974 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1); in isKnownNeverNaNForTargetNode()
5994 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && in isKnownNeverNaNForTargetNode()
5995 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1) && in isKnownNeverNaNForTargetNode()
5996 DAG.isKnownNeverNaN(Op.getOperand(3), SNaN, Depth + 1); in isKnownNeverNaNForTargetNode()