Lines Matching refs:DAG
157 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
169 getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, in getCopyFromParts() argument
175 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getCopyFromParts()
176 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts, in getCopyFromParts()
181 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V, in getCopyFromParts()
197 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits); in getCopyFromParts()
200 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); in getCopyFromParts()
203 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V, in getCopyFromParts()
205 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2, in getCopyFromParts()
208 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]); in getCopyFromParts()
209 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]); in getCopyFromParts()
212 if (DAG.getDataLayout().isBigEndian()) in getCopyFromParts()
215 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi); in getCopyFromParts()
220 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits); in getCopyFromParts()
221 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT, in getCopyFromParts()
226 if (DAG.getDataLayout().isBigEndian()) in getCopyFromParts()
228 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); in getCopyFromParts()
229 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi); in getCopyFromParts()
230 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi, in getCopyFromParts()
231 DAG.getConstant(Lo.getValueSizeInBits(), DL, in getCopyFromParts()
233 TotalVT, DAG.getDataLayout()))); in getCopyFromParts()
234 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo); in getCopyFromParts()
235 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi); in getCopyFromParts()
242 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]); in getCopyFromParts()
243 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]); in getCopyFromParts()
244 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) in getCopyFromParts()
246 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi); in getCopyFromParts()
251 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); in getCopyFromParts()
252 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, in getCopyFromParts()
269 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); in getCopyFromParts()
270 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val); in getCopyFromParts()
275 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); in getCopyFromParts()
284 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val, in getCopyFromParts()
285 DAG.getValueType(ValueVT)); in getCopyFromParts()
286 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); in getCopyFromParts()
288 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); in getCopyFromParts()
296 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())); in getCopyFromParts()
298 if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr( in getCopyFromParts()
300 return DAG.getNode(ISD::STRICT_FP_ROUND, DL, in getCopyFromParts()
301 DAG.getVTList(ValueVT, MVT::Other), InChain, Val, in getCopyFromParts()
305 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange); in getCopyFromParts()
308 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val); in getCopyFromParts()
315 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val); in getCopyFromParts()
316 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); in getCopyFromParts()
341 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, in getCopyFromPartsVector() argument
350 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getCopyFromPartsVector()
362 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, in getCopyFromPartsVector()
366 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, in getCopyFromPartsVector()
383 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT, in getCopyFromPartsVector()
392 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT, in getCopyFromPartsVector()
401 *DAG.getContext(), IntermediateVT.getScalarType(), in getCopyFromPartsVector()
403 : EVT::getVectorVT(*DAG.getContext(), in getCopyFromPartsVector()
406 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS in getCopyFromPartsVector()
420 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); in getCopyFromPartsVector()
432 EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(), in getCopyFromPartsVector()
434 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val, in getCopyFromPartsVector()
435 DAG.getVectorIdxConstant(0, DL)); in getCopyFromPartsVector()
439 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); in getCopyFromPartsVector()
443 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); in getCopyFromPartsVector()
447 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT); in getCopyFromPartsVector()
454 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); in getCopyFromPartsVector()
460 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); in getCopyFromPartsVector()
463 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); in getCopyFromPartsVector()
465 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val); in getCopyFromPartsVector()
466 return DAG.getBitcast(ValueVT, Val); in getCopyFromPartsVector()
470 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion"); in getCopyFromPartsVector()
471 return DAG.getUNDEF(ValueVT); in getCopyFromPartsVector()
479 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val); in getCopyFromPartsVector()
485 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); in getCopyFromPartsVector()
486 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val); in getCopyFromPartsVector()
487 Val = DAG.getBitcast(ValueSVT, Val); in getCopyFromPartsVector()
490 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT) in getCopyFromPartsVector()
491 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT); in getCopyFromPartsVector()
495 return DAG.getBuildVector(ValueVT, DL, Val); in getCopyFromPartsVector()
498 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
507 getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, in getCopyToParts() argument
512 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getCopyToParts()
513 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT, in getCopyToParts()
520 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V, in getCopyToParts()
524 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && in getCopyToParts()
543 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val); in getCopyToParts()
548 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); in getCopyToParts()
549 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); in getCopyToParts()
554 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); in getCopyToParts()
555 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val); in getCopyToParts()
557 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); in getCopyToParts()
562 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); in getCopyToParts()
568 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); in getCopyToParts()
569 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); in getCopyToParts()
571 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); in getCopyToParts()
581 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V, in getCopyToParts()
583 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); in getCopyToParts()
598 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val, in getCopyToParts()
599 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL)); in getCopyToParts()
601 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V, in getCopyToParts()
604 if (DAG.getDataLayout().isBigEndian()) in getCopyToParts()
609 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); in getCopyToParts()
610 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); in getCopyToParts()
615 Parts[0] = DAG.getNode(ISD::BITCAST, DL, in getCopyToParts()
616 EVT::getIntegerVT(*DAG.getContext(), in getCopyToParts()
623 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits); in getCopyToParts()
627 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, in getCopyToParts()
628 ThisVT, Part0, DAG.getIntPtrConstant(1, DL)); in getCopyToParts()
629 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, in getCopyToParts()
630 ThisVT, Part0, DAG.getIntPtrConstant(0, DL)); in getCopyToParts()
633 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0); in getCopyToParts()
634 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1); in getCopyToParts()
639 if (DAG.getDataLayout().isBigEndian()) in getCopyToParts()
643 static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, in widenVectorToPartType() argument
663 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && in widenVectorToPartType()
665 Val = DAG.getNode(ISD::BITCAST, DL, in widenVectorToPartType()
674 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT), in widenVectorToPartType()
675 Val, DAG.getVectorIdxConstant(0, DL)); in widenVectorToPartType()
680 DAG.ExtractVectorElements(Val, Ops); in widenVectorToPartType()
681 SDValue EltUndef = DAG.getUNDEF(PartEVT); in widenVectorToPartType()
685 return DAG.getBuildVector(PartVT, DL, Ops); in widenVectorToPartType()
690 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL, in getCopyToPartsVector() argument
696 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getCopyToPartsVector()
705 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); in getCopyToPartsVector()
706 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) { in getCopyToPartsVector()
715 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); in getCopyToPartsVector()
719 TLI.getTypeAction(*DAG.getContext(), ValueVT) == in getCopyToPartsVector()
723 EVT::getVectorVT(*DAG.getContext(), ValueVT.getVectorElementType(), in getCopyToPartsVector()
725 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT); in getCopyToPartsVector()
726 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT); in getCopyToPartsVector()
739 Val = DAG.getBitcast(ValueVT.getScalarType(), Val); in getCopyToPartsVector()
740 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val); in getCopyToPartsVector()
742 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val, in getCopyToPartsVector()
743 DAG.getVectorIdxConstant(0, DL)); in getCopyToPartsVector()
748 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); in getCopyToPartsVector()
749 Val = DAG.getBitcast(IntermediateType, Val); in getCopyToPartsVector()
750 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); in getCopyToPartsVector()
766 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates, in getCopyToPartsVector()
770 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, in getCopyToPartsVector()
789 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt); in getCopyToPartsVector()
795 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val); in getCopyToPartsVector()
800 ValueVT = EVT::getVectorVT(*DAG.getContext(), in getCopyToPartsVector()
803 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); in getCopyToPartsVector()
806 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) { in getCopyToPartsVector()
821 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val, in getCopyToPartsVector()
822 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL)); in getCopyToPartsVector()
824 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val, in getCopyToPartsVector()
825 DAG.getVectorIdxConstant(i, DL)); in getCopyToPartsVector()
834 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv); in getCopyToPartsVector()
843 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V, in getCopyToPartsVector()
877 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, in getCopyFromRegs() argument
885 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getCopyFromRegs()
896 *DAG.getContext(), *CallConv, RegVTs[Value]) in getCopyFromRegs()
903 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); in getCopyFromRegs()
905 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue); in getCopyFromRegs()
931 Parts[i] = DAG.getConstant(0, dl, RegisterVT); in getCopyFromRegs()
940 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits); in getCopyFromRegs()
944 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1); in getCopyFromRegs()
951 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl, in getCopyFromRegs()
952 RegisterVT, P, DAG.getValueType(FromVT)); in getCopyFromRegs()
955 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs, in getCopyFromRegs()
961 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values); in getCopyFromRegs()
964 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, in getCopyToRegs() argument
968 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getCopyToRegs()
979 *DAG.getContext(), *CallConv, RegVTs[Value]) in getCopyToRegs()
985 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part], in getCopyToRegs()
995 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]); in getCopyToRegs()
997 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue); in getCopyToRegs()
1017 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in getCopyToRegs()
1022 SelectionDAG &DAG, in AddInlineAsmOperands() argument
1024 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in AddInlineAsmOperands()
1035 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); in AddInlineAsmOperands()
1040 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32); in AddInlineAsmOperands()
1052 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I])); in AddInlineAsmOperands()
1055 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && in AddInlineAsmOperands()
1063 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value], in AddInlineAsmOperands()
1068 Ops.push_back(DAG.getRegister(TheReg, RegisterVT)); in AddInlineAsmOperands()
1094 Context = DAG.getContext(); in init()
1096 SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout()); in init()
1098 *DAG.getMachineFunction().getFunction().getParent()); in init()
1120 SDValue Root = DAG.getRoot(); in updateRoot()
1142 Root = DAG.getTokenFactor(getCurSDLoc(), Pending); in updateRoot()
1144 DAG.setRoot(Root); in updateRoot()
1207 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(), in handleDebugDeclare()
1216 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(), in handleDebugDeclare()
1219 DAG.AddDbgValue(SDV, IsParameter); in handleDebugDeclare()
1235 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) { in visitDbgInfo()
1267 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs(); in visitDbgInfo()
1274 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder); in visitDbgInfo()
1275 DAG.AddDbgLabel(SDV); in visitDbgInfo()
1343 DAG, [&](SDNode *) { NodeInserted = true; }); in visit()
1357 DAG.addPCSections(It->second.getNode(), PCSectionsMD); in visit()
1359 DAG.addMMRAMetadata(It->second.getNode(), MMRA); in visit()
1389 static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, in handleDanglingVariadicDebugInfo() argument
1401 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {}, in handleDanglingVariadicDebugInfo()
1404 DAG.AddDbgValue(SDV, /*isParameter=*/false); in handleDanglingVariadicDebugInfo()
1414 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr); in addDanglingDebugInfo()
1490 DAG.AddDbgValue(SDV, false); in resolveDanglingDebugInfo()
1500 DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder); in resolveDanglingDebugInfo()
1501 DAG.AddDbgValue(SDV, false); in resolveDanglingDebugInfo()
1567 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder); in salvageUnresolvedDbgValue()
1568 DAG.AddDbgValue(SDV, false); in salvageUnresolvedDbgValue()
1653 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in handleDebugValue()
1671 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, in handleDebugValue()
1696 SDDbgValue *SDV = DAG.getVRegDbgValue( in handleDebugValue()
1698 DAG.AddDbgValue(SDV, false); in handleDebugValue()
1714 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies, in handleDebugValue()
1716 DAG.AddDbgValue(SDV, /*isParameter=*/false); in handleDebugValue()
1737 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), in getCopyFromRegs()
1738 DAG.getDataLayout(), InReg, Ty, in getCopyFromRegs()
1740 SDValue Chain = DAG.getEntryNode(); in getCopyFromRegs()
1741 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, in getCopyFromRegs()
1795 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getValueImpl()
1798 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true); in getValueImpl()
1801 return DAG.getConstant(*CI, getCurSDLoc(), VT); in getValueImpl()
1804 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT); in getValueImpl()
1807 return DAG.getNode(ISD::PtrAuthGlobalAddress, getCurSDLoc(), VT, in getValueImpl()
1815 return DAG.getConstant(0, getCurSDLoc(), in getValueImpl()
1816 TLI.getPointerTy(DAG.getDataLayout(), AS)); in getValueImpl()
1820 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1)); in getValueImpl()
1823 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT); in getValueImpl()
1826 return DAG.getUNDEF(VT); in getValueImpl()
1847 return DAG.getMergeValues(Constants, getCurSDLoc()); in getValueImpl()
1862 return DAG.getMergeValues(Ops, getCurSDLoc()); in getValueImpl()
1863 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); in getValueImpl()
1871 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs); in getValueImpl()
1879 Constants[i] = DAG.getUNDEF(EltVT); in getValueImpl()
1881 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT); in getValueImpl()
1883 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT); in getValueImpl()
1886 return DAG.getMergeValues(Constants, getCurSDLoc()); in getValueImpl()
1890 return DAG.getBlockAddress(BA, VT); in getValueImpl()
1900 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, in getValueImpl()
1901 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1)); in getValueImpl()
1914 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); in getValueImpl()
1919 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType()); in getValueImpl()
1923 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT); in getValueImpl()
1925 Op = DAG.getConstant(0, getCurSDLoc(), EltVT); in getValueImpl()
1927 return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op); in getValueImpl()
1939 return DAG.getFrameIndex( in getValueImpl()
1940 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType())); in getValueImpl()
1947 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg, in getValueImpl()
1949 SDValue Chain = DAG.getEntryNode(); in getValueImpl()
1950 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); in getValueImpl()
1954 return DAG.getMDNode(cast<MDNode>(MD->getMetadata())); in getValueImpl()
1957 return DAG.getBasicBlock(FuncInfo.MBBMap[BB]); in getValueImpl()
1980 DAG.getMachineFunction().setHasEHCatchret(true); in visitCatchRet()
1989 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, in visitCatchRet()
1990 getControlRoot(), DAG.getBasicBlock(TargetMBB))); in visitCatchRet()
2009 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other, in visitCatchRet()
2010 getControlRoot(), DAG.getBasicBlock(TargetMBB), in visitCatchRet()
2011 DAG.getBasicBlock(SuccessorColorMBB)); in visitCatchRet()
2012 DAG.setRoot(Ret); in visitCatchRet()
2160 SDValue Ret = DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, in visitCleanupRet()
2161 getControlRoot(), DAG.getBasicBlock(CleanupPadMBB)); in visitCleanupRet()
2162 DAG.setRoot(Ret); in visitCleanupRet()
2170 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitRet()
2171 auto &DL = DAG.getDataLayout(); in visitRet()
2198 DAG.getDataLayout().getAllocaAddrSpace()), in visitRet()
2202 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]); in visitRet()
2216 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, in visitRet()
2221 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]); in visitRet()
2222 Chains[i] = DAG.getStore( in visitRet()
2225 Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), in visitRet()
2229 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), in visitRet()
2264 getCopyToParts(DAG, getCurSDLoc(), in visitRet()
2315 DAG.getRegister(SwiftError.getOrCreateVRegUseAt( in visitRet()
2320 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg(); in visitRet()
2322 DAG.getMachineFunction().getFunction().getCallingConv(); in visitRet()
2323 Chain = DAG.getTargetLoweringInfo().LowerReturn( in visitRet()
2324 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG); in visitRet()
2331 DAG.setRoot(Chain); in visitRet()
2471 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()), in EmitBranchForMergedCondition()
2570 const auto &TLI = DAG.getTargetLoweringInfo(); in shouldKeepJumpConditionsTogether()
2676 MachineFunction &MF = DAG.getMachineFunction(); in FindMergedConditions()
2794 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, in visitBr()
2795 getControlRoot(), DAG.getBasicBlock(Succ0MBB)); in visitBr()
2797 DAG.setRoot(Br); in visitBr()
2826 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp && in visitBr()
2841 DAG.getTargetLoweringInfo().getJumpConditionMergingParams( in visitBr()
2875 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()), in visitBr()
2896 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(), in visitSwitchCase()
2897 DAG.getBasicBlock(CB.TrueBB))); in visitSwitchCase()
2902 auto &TLI = DAG.getTargetLoweringInfo(); in visitSwitchCase()
2903 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType()); in visitSwitchCase()
2909 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) && in visitSwitchCase()
2912 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) && in visitSwitchCase()
2914 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType()); in visitSwitchCase()
2915 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True); in visitSwitchCase()
2923 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT); in visitSwitchCase()
2924 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT); in visitSwitchCase()
2926 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC); in visitSwitchCase()
2938 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT), in visitSwitchCase()
2941 SDValue SUB = DAG.getNode(ISD::SUB, dl, in visitSwitchCase()
2942 VT, CmpOp, DAG.getConstant(Low, dl, VT)); in visitSwitchCase()
2943 Cond = DAG.getSetCC(dl, MVT::i1, SUB, in visitSwitchCase()
2944 DAG.getConstant(High-Low, dl, VT), ISD::SETULE); in visitSwitchCase()
2960 SDValue True = DAG.getConstant(1, dl, Cond.getValueType()); in visitSwitchCase()
2961 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True); in visitSwitchCase()
2964 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, in visitSwitchCase()
2966 DAG.getBasicBlock(CB.TrueBB)); in visitSwitchCase()
2973 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, in visitSwitchCase()
2974 DAG.getBasicBlock(CB.FalseBB)); in visitSwitchCase()
2976 DAG.setRoot(BrCond); in visitSwitchCase()
2984 EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); in visitJumpTable()
2985 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy); in visitJumpTable()
2986 SDValue Table = DAG.getJumpTable(JT.JTI, PTy); in visitJumpTable()
2987 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other, in visitJumpTable()
2989 DAG.setRoot(BrJumpTable); in visitJumpTable()
3003 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp, in visitJumpTableHeader()
3004 DAG.getConstant(JTH.First, dl, VT)); in visitJumpTableHeader()
3011 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitJumpTableHeader()
3012 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout())); in visitJumpTableHeader()
3015 FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout())); in visitJumpTableHeader()
3016 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, in visitJumpTableHeader()
3024 SDValue CMP = DAG.getSetCC( in visitJumpTableHeader()
3025 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), in visitJumpTableHeader()
3027 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT); in visitJumpTableHeader()
3029 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, in visitJumpTableHeader()
3031 DAG.getBasicBlock(JT.Default)); in visitJumpTableHeader()
3035 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, in visitJumpTableHeader()
3036 DAG.getBasicBlock(JT.MBB)); in visitJumpTableHeader()
3038 DAG.setRoot(BrCond); in visitJumpTableHeader()
3042 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo, in visitJumpTableHeader()
3043 DAG.getBasicBlock(JT.MBB))); in visitJumpTableHeader()
3045 DAG.setRoot(CopyTo); in visitJumpTableHeader()
3051 static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, in getLoadStackGuard() argument
3053 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getLoadStackGuard()
3054 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); in getLoadStackGuard()
3055 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); in getLoadStackGuard()
3056 MachineFunction &MF = DAG.getMachineFunction(); in getLoadStackGuard()
3059 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain); in getLoadStackGuard()
3066 DAG.getEVTAlign(PtrTy)); in getLoadStackGuard()
3067 DAG.setNodeMemRefs(Node, {MemRef}); in getLoadStackGuard()
3070 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy); in getLoadStackGuard()
3084 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitSPDescriptorParent()
3085 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); in visitSPDescriptorParent()
3086 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); in visitSPDescriptorParent()
3093 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy); in visitSPDescriptorParent()
3096 DAG.getDataLayout().getPrefTypeAlign(PointerType::get(M.getContext(), 0)); in visitSPDescriptorParent()
3099 SDValue GuardVal = DAG.getLoad( in visitSPDescriptorParent()
3100 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr, in visitSPDescriptorParent()
3101 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align, in visitSPDescriptorParent()
3105 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl); in visitSPDescriptorParent()
3123 TargetLowering::CallLoweringInfo CLI(DAG); in visitSPDescriptorParent()
3125 .setChain(DAG.getEntryNode()) in visitSPDescriptorParent()
3130 DAG.setRoot(Result.second); in visitSPDescriptorParent()
3136 SDValue Chain = DAG.getEntryNode(); in visitSPDescriptorParent()
3138 Guard = getLoadStackGuard(DAG, dl, Chain); in visitSPDescriptorParent()
3143 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr, in visitSPDescriptorParent()
3149 SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(), in visitSPDescriptorParent()
3150 *DAG.getContext(), in visitSPDescriptorParent()
3155 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, in visitSPDescriptorParent()
3157 Cmp, DAG.getBasicBlock(SPD.getFailureMBB())); in visitSPDescriptorParent()
3159 SDValue Br = DAG.getNode(ISD::BR, dl, in visitSPDescriptorParent()
3161 DAG.getBasicBlock(SPD.getSuccessMBB())); in visitSPDescriptorParent()
3163 DAG.setRoot(Br); in visitSPDescriptorParent()
3176 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitSPDescriptorFailure()
3180 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid, in visitSPDescriptorFailure()
3187 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain); in visitSPDescriptorFailure()
3192 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain); in visitSPDescriptorFailure()
3194 DAG.setRoot(Chain); in visitSPDescriptorFailure()
3207 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT)); in visitBitTestHeader()
3210 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitBitTestHeader()
3225 VT = TLI.getPointerTy(DAG.getDataLayout()); in visitBitTestHeader()
3226 Sub = DAG.getZExtOrTrunc(Sub, dl, VT); in visitBitTestHeader()
3231 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub); in visitBitTestHeader()
3243 SDValue RangeCmp = DAG.getSetCC(dl, in visitBitTestHeader()
3244 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), in visitBitTestHeader()
3246 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()), in visitBitTestHeader()
3249 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp, in visitBitTestHeader()
3250 DAG.getBasicBlock(B.Default)); in visitBitTestHeader()
3255 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB)); in visitBitTestHeader()
3257 DAG.setRoot(Root); in visitBitTestHeader()
3269 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT); in visitBitTestCase()
3272 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitBitTestCase()
3276 Cmp = DAG.getSetCC( in visitBitTestCase()
3277 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), in visitBitTestCase()
3278 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT), in visitBitTestCase()
3282 Cmp = DAG.getSetCC( in visitBitTestCase()
3283 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), in visitBitTestCase()
3284 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE); in visitBitTestCase()
3287 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT, in visitBitTestCase()
3288 DAG.getConstant(1, dl, VT), ShiftOp); in visitBitTestCase()
3291 SDValue AndOp = DAG.getNode(ISD::AND, dl, in visitBitTestCase()
3292 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT)); in visitBitTestCase()
3293 Cmp = DAG.getSetCC( in visitBitTestCase()
3294 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), in visitBitTestCase()
3295 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE); in visitBitTestCase()
3307 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl, in visitBitTestCase()
3309 Cmp, DAG.getBasicBlock(B.TargetBB)); in visitBitTestCase()
3313 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd, in visitBitTestCase()
3314 DAG.getBasicBlock(NextMBB)); in visitBitTestCase()
3316 DAG.setRoot(BrAnd); in visitBitTestCase()
3369 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitInvoke()
3371 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(), in visitInvoke()
3372 TLI.getPointerTy(DAG.getDataLayout()))); in visitInvoke()
3373 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain in visitInvoke()
3374 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops)); in visitInvoke()
3414 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), in visitInvoke()
3415 DAG.getBasicBlock(Return))); in visitInvoke()
3451 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), in visitCallBr()
3453 DAG.getBasicBlock(Return))); in visitCallBr()
3466 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitLandingPad()
3481 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs); in visitLandingPad()
3488 Ops[0] = DAG.getZExtOrTrunc( in visitLandingPad()
3489 DAG.getCopyFromReg(DAG.getEntryNode(), dl, in visitLandingPad()
3491 TLI.getPointerTy(DAG.getDataLayout())), in visitLandingPad()
3494 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())); in visitLandingPad()
3496 Ops[1] = DAG.getZExtOrTrunc( in visitLandingPad()
3497 DAG.getCopyFromReg(DAG.getEntryNode(), dl, in visitLandingPad()
3499 TLI.getPointerTy(DAG.getDataLayout())), in visitLandingPad()
3503 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl, in visitLandingPad()
3504 DAG.getVTList(ValueVTs), Ops); in visitLandingPad()
3537 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), in visitIndirectBr()
3543 if (!DAG.getTarget().Options.TrapUnreachable) in visitUnreachable()
3549 if (DAG.getTarget().Options.NoTrapAfterNoreturn) in visitUnreachable()
3556 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot())); in visitUnreachable()
3565 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(), in visitUnary()
3585 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), in visitBinary()
3594 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy( in visitShift()
3595 Op1.getValueType(), DAG.getDataLayout()); in visitShift()
3602 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy); in visitShift()
3624 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, in visitShift()
3636 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1, in visitSDiv()
3646 auto &TLI = DAG.getTargetLoweringInfo(); in visitICmp()
3648 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); in visitICmp()
3654 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT); in visitICmp()
3655 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT); in visitICmp()
3658 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitICmp()
3660 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode)); in visitICmp()
3675 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); in visitFCmp()
3677 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitFCmp()
3679 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition)); in visitFCmp()
3692 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), in visitSelect()
3718 LLVMContext &Ctx = *DAG.getContext(); in visitSelect()
3719 auto &TLI = DAG.getTargetLoweringInfo(); in visitSelect()
3805 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i)); in visitSelect()
3807 Values[i] = DAG.getNegative(Values[i], dl, VT); in visitSelect()
3814 Values[i] = DAG.getNode( in visitSelect()
3820 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), in visitSelect()
3821 DAG.getVTList(ValueVTs), Values)); in visitSelect()
3827 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitTrunc()
3829 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N)); in visitTrunc()
3836 auto &TLI = DAG.getTargetLoweringInfo(); in visitZExt()
3837 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitZExt()
3848 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N)); in visitZExt()
3852 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags)); in visitZExt()
3859 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitSExt()
3861 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N)); in visitSExt()
3868 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitFPTrunc()
3869 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitFPTrunc()
3870 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N, in visitFPTrunc()
3871 DAG.getTargetConstant( in visitFPTrunc()
3872 0, dl, TLI.getPointerTy(DAG.getDataLayout())))); in visitFPTrunc()
3878 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitFPExt()
3880 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N)); in visitFPExt()
3886 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitFPToUI()
3888 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N)); in visitFPToUI()
3894 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitFPToSI()
3896 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N)); in visitFPToSI()
3902 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitUIToFP()
3908 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N, Flags)); in visitUIToFP()
3914 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitSIToFP()
3916 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N)); in visitSIToFP()
3923 auto &TLI = DAG.getTargetLoweringInfo(); in visitPtrToInt()
3924 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitPtrToInt()
3927 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); in visitPtrToInt()
3928 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT); in visitPtrToInt()
3929 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT); in visitPtrToInt()
3937 auto &TLI = DAG.getTargetLoweringInfo(); in visitIntToPtr()
3938 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntToPtr()
3939 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); in visitIntToPtr()
3940 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT); in visitIntToPtr()
3941 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT); in visitIntToPtr()
3948 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitBitCast()
3954 setValue(&I, DAG.getNode(ISD::BITCAST, dl, in visitBitCast()
3961 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false, in visitBitCast()
3968 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitAddrSpaceCast()
3971 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitAddrSpaceCast()
3977 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS); in visitAddrSpaceCast()
3983 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitInsertElement()
3986 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(), in visitInsertElement()
3987 TLI.getVectorIdxTy(DAG.getDataLayout())); in visitInsertElement()
3988 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(), in visitInsertElement()
3989 TLI.getValueType(DAG.getDataLayout(), I.getType()), in visitInsertElement()
3994 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitExtractElement()
3996 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(), in visitExtractElement()
3997 TLI.getVectorIdxTy(DAG.getDataLayout())); in visitExtractElement()
3998 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(), in visitExtractElement()
3999 TLI.getValueType(DAG.getDataLayout(), I.getType()), in visitExtractElement()
4012 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitShuffleVector()
4013 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitShuffleVector()
4020 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1, in visitShuffleVector()
4021 DAG.getVectorIdxConstant(0, DL)); in visitShuffleVector()
4022 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt)); in visitShuffleVector()
4035 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask)); in visitShuffleVector()
4073 ConcatOps.push_back(DAG.getUNDEF(SrcVT)); in visitShuffleVector()
4079 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps)); in visitShuffleVector()
4086 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), in visitShuffleVector()
4090 SDValue UndefVal = DAG.getUNDEF(SrcVT); in visitShuffleVector()
4097 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1); in visitShuffleVector()
4098 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2); in visitShuffleVector()
4109 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps); in visitShuffleVector()
4114 Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result, in visitShuffleVector()
4115 DAG.getVectorIdxConstant(0, DL)); in visitShuffleVector()
4149 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used. in visitShuffleVector()
4157 Src = DAG.getUNDEF(VT); in visitShuffleVector()
4159 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src, in visitShuffleVector()
4160 DAG.getVectorIdxConstant(StartIdx[Input], DL)); in visitShuffleVector()
4173 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps)); in visitShuffleVector()
4187 Res = DAG.getUNDEF(EltVT); in visitShuffleVector()
4192 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src, in visitShuffleVector()
4193 DAG.getVectorIdxConstant(Idx, DL)); in visitShuffleVector()
4199 setValue(&I, DAG.getBuildVector(VT, DL, Ops)); in visitShuffleVector()
4213 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitInsertValue()
4215 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs); in visitInsertValue()
4217 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs); in visitInsertValue()
4225 setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); in visitInsertValue()
4233 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : in visitInsertValue()
4239 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) : in visitInsertValue()
4244 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : in visitInsertValue()
4247 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), in visitInsertValue()
4248 DAG.getVTList(AggValueVTs), Values)); in visitInsertValue()
4260 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitExtractValue()
4262 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs); in visitExtractValue()
4268 setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); in visitExtractValue()
4279 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) : in visitExtractValue()
4282 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), in visitExtractValue()
4283 DAG.getVTList(ValValueVTs), Values)); in visitExtractValue()
4293 auto &TLI = DAG.getTargetLoweringInfo(); in visitGetElementPtr()
4303 LLVMContext &Context = *DAG.getContext(); in visitGetElementPtr()
4305 N = DAG.getSplat(VT, dl, N); in visitGetElementPtr()
4316 DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field); in visitGetElementPtr()
4324 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, in visitGetElementPtr()
4325 DAG.getConstant(Offset, dl, N.getValueType()), Flags); in visitGetElementPtr()
4331 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS); in visitGetElementPtr()
4334 GTI.getSequentialElementStride(DAG.getDataLayout()); in visitGetElementPtr()
4351 LLVMContext &Context = *DAG.getContext(); in visitGetElementPtr()
4354 OffsVal = DAG.getConstant( in visitGetElementPtr()
4357 OffsVal = DAG.getConstant(Offs, dl, IdxTy); in visitGetElementPtr()
4365 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType()); in visitGetElementPtr()
4367 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags); in visitGetElementPtr()
4377 IdxN = DAG.getSplat(VT, dl, IdxN); in visitGetElementPtr()
4382 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType()); in visitGetElementPtr()
4386 SDValue VScale = DAG.getNode( in visitGetElementPtr()
4388 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy)); in visitGetElementPtr()
4390 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale); in visitGetElementPtr()
4391 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale); in visitGetElementPtr()
4398 IdxN = DAG.getNode(ISD::SHL, dl, in visitGetElementPtr()
4400 DAG.getConstant(Amt, dl, IdxN.getValueType())); in visitGetElementPtr()
4402 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl, in visitGetElementPtr()
4404 IdxN = DAG.getNode(ISD::MUL, dl, in visitGetElementPtr()
4410 N = DAG.getNode(ISD::ADD, dl, in visitGetElementPtr()
4415 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS); in visitGetElementPtr()
4416 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS); in visitGetElementPtr()
4423 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy); in visitGetElementPtr()
4436 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitAlloca()
4437 auto &DL = DAG.getDataLayout(); in visitAlloca()
4445 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr); in visitAlloca()
4448 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize, in visitAlloca()
4449 DAG.getVScale(dl, IntPtr, in visitAlloca()
4454 DAG.getConstant(TySize.getFixedValue(), dl, MVT::getIntegerVT(64)); in visitAlloca()
4455 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize, in visitAlloca()
4456 DAG.getZExtOrTrunc(TySizeValue, dl, IntPtr)); in visitAlloca()
4462 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign(); in visitAlloca()
4472 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize, in visitAlloca()
4473 DAG.getConstant(StackAlignMask, dl, IntPtr), Flags); in visitAlloca()
4476 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize, in visitAlloca()
4477 DAG.getConstant(~StackAlignMask, dl, IntPtr)); in visitAlloca()
4481 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)}; in visitAlloca()
4482 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other); in visitAlloca()
4483 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops); in visitAlloca()
4485 DAG.setRoot(DSA.getValue(1)); in visitAlloca()
4517 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitLoad()
4538 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets); in visitLoad()
4548 TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo); in visitLoad()
4560 LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), in visitLoad()
4563 Root = DAG.getEntryNode(); in visitLoad()
4568 Root = DAG.getRoot(); in visitLoad()
4574 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG); in visitLoad()
4589 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in visitLoad()
4601 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]); in visitLoad()
4602 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment, in visitLoad()
4607 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]); in visitLoad()
4613 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in visitLoad()
4616 DAG.setRoot(Chain); in visitLoad()
4621 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl, in visitLoad()
4622 DAG.getVTList(ValueVTs), Values)); in visitLoad()
4626 assert(DAG.getTargetLoweringInfo().supportSwiftError() && in visitStoreToSwiftError()
4632 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), in visitStoreToSwiftError()
4643 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg, in visitStoreToSwiftError()
4645 DAG.setRoot(CopyNode); in visitStoreToSwiftError()
4649 assert(DAG.getTargetLoweringInfo().supportSwiftError() && in visitLoadFromSwiftError()
4662 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), in visitLoadFromSwiftError()
4668 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty, in visitLoadFromSwiftError()
4674 SDValue L = DAG.getCopyFromReg( in visitLoadFromSwiftError()
4688 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitStore()
4705 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), in visitStore()
4723 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); in visitStore()
4729 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in visitStore()
4741 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]); in visitStore()
4744 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]); in visitStore()
4746 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags, AAInfo); in visitStore()
4750 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in visitStore()
4753 DAG.setRoot(StoreNode); in visitStore()
4787 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); in visitMaskedStore()
4795 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitMaskedStore()
4799 const auto &TLI = DAG.getTargetLoweringInfo(); in visitMaskedStore()
4805 ? TLI.visitMaskedStore(DAG, sdl, getMemoryRoot(), MMO, Ptr, Src0, in visitMaskedStore()
4807 : DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, in visitMaskedStore()
4810 DAG.setRoot(StoreNode); in visitMaskedStore()
4833 SelectionDAG& DAG = SDB->DAG; in getUniformBase() local
4834 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getUniformBase()
4835 const DataLayout &DL = DAG.getDataLayout(); in getUniformBase()
4848 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts); in getUniformBase()
4849 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT); in getUniformBase()
4851 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL)); in getUniformBase()
4883 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL)); in getUniformBase()
4897 .value_or(DAG.getEVTAlign(VT.getScalarType())); in visitMaskedScatter()
4898 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitMaskedScatter()
4908 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitMaskedScatter()
4912 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); in visitMaskedScatter()
4915 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); in visitMaskedScatter()
4922 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); in visitMaskedScatter()
4926 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl, in visitMaskedScatter()
4928 DAG.setRoot(Scatter); in visitMaskedScatter()
4962 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); in visitMaskedLoad()
4972 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); in visitMaskedLoad()
4978 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitMaskedLoad()
4982 const auto &TLI = DAG.getTargetLoweringInfo(); in visitMaskedLoad()
4991 Res = TLI.visitMaskedLoad(DAG, sdl, InChain, MMO, Load, Ptr, Src0, Mask); in visitMaskedLoad()
4994 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO, in visitMaskedLoad()
5009 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitMaskedGather()
5010 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitMaskedGather()
5013 .value_or(DAG.getEVTAlign(VT.getScalarType())); in visitMaskedGather()
5017 SDValue Root = DAG.getRoot(); in visitMaskedGather()
5025 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitMaskedGather()
5031 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); in visitMaskedGather()
5034 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); in visitMaskedGather()
5041 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); in visitMaskedGather()
5045 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, in visitMaskedGather()
5061 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other); in visitAtomicCmpXchg()
5063 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitAtomicCmpXchg()
5064 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); in visitAtomicCmpXchg()
5066 MachineFunction &MF = DAG.getMachineFunction(); in visitAtomicCmpXchg()
5069 LocationSize::precise(MemVT.getStoreSize()), DAG.getEVTAlign(MemVT), in visitAtomicCmpXchg()
5072 SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, in visitAtomicCmpXchg()
5081 DAG.setRoot(OutChain); in visitAtomicCmpXchg()
5117 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitAtomicRMW()
5118 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); in visitAtomicRMW()
5120 MachineFunction &MF = DAG.getMachineFunction(); in visitAtomicRMW()
5123 LocationSize::precise(MemVT.getStoreSize()), DAG.getEVTAlign(MemVT), in visitAtomicRMW()
5127 DAG.getAtomic(NT, dl, MemVT, InChain, in visitAtomicRMW()
5134 DAG.setRoot(OutChain); in visitAtomicRMW()
5139 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitFence()
5142 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl, in visitFence()
5143 TLI.getFenceOperandTy(DAG.getDataLayout())); in visitFence()
5144 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl, in visitFence()
5145 TLI.getFenceOperandTy(DAG.getDataLayout())); in visitFence()
5146 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops); in visitFence()
5148 DAG.setRoot(N); in visitFence()
5158 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitAtomicLoad()
5159 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitAtomicLoad()
5160 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); in visitAtomicLoad()
5166 auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo); in visitAtomicLoad()
5168 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitAtomicLoad()
5173 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG); in visitAtomicLoad()
5176 SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain, in visitAtomicLoad()
5181 L = DAG.getPtrExtOrTrunc(L, dl, VT); in visitAtomicLoad()
5184 DAG.setRoot(OutChain); in visitAtomicLoad()
5195 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitAtomicStore()
5197 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType()); in visitAtomicStore()
5203 auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); in visitAtomicStore()
5205 MachineFunction &MF = DAG.getMachineFunction(); in visitAtomicStore()
5213 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT); in visitAtomicStore()
5217 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO); in visitAtomicStore()
5220 DAG.setRoot(OutChain); in visitAtomicStore()
5239 Ops.push_back(DAG.getRoot()); in visitTargetIntrinsic()
5247 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitTargetIntrinsic()
5249 DAG.getMachineFunction(), in visitTargetIntrinsic()
5255 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(), in visitTargetIntrinsic()
5256 TLI.getPointerTy(DAG.getDataLayout()))); in visitTargetIntrinsic()
5267 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true); in visitTargetIntrinsic()
5271 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT)); in visitTargetIntrinsic()
5274 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT)); in visitTargetIntrinsic()
5279 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs); in visitTargetIntrinsic()
5284 SDVTList VTs = DAG.getVTList(ValueVTs); in visitTargetIntrinsic()
5290 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); in visitTargetIntrinsic()
5301 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken); in visitTargetIntrinsic()
5306 TLI.CollectTargetIntrinsicOperands(I, Ops, DAG); in visitTargetIntrinsic()
5317 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, in visitTargetIntrinsic()
5321 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops); in visitTargetIntrinsic()
5323 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops); in visitTargetIntrinsic()
5325 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops); in visitTargetIntrinsic()
5333 DAG.setRoot(Chain); in visitTargetIntrinsic()
5338 Result = lowerRangeToAssertZExt(DAG, I, Result); in visitTargetIntrinsic()
5345 DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne()); in visitTargetIntrinsic()
5358 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) { in GetSignificand() argument
5359 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, in GetSignificand()
5360 DAG.getConstant(0x007fffff, dl, MVT::i32)); in GetSignificand()
5361 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1, in GetSignificand()
5362 DAG.getConstant(0x3f800000, dl, MVT::i32)); in GetSignificand()
5363 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2); in GetSignificand()
5371 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, in GetExponent() argument
5373 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, in GetExponent()
5374 DAG.getConstant(0x7f800000, dl, MVT::i32)); in GetExponent()
5375 SDValue t1 = DAG.getNode( in GetExponent()
5377 DAG.getConstant(23, dl, in GetExponent()
5378 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout()))); in GetExponent()
5379 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1, in GetExponent()
5380 DAG.getConstant(127, dl, MVT::i32)); in GetExponent()
5381 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); in GetExponent()
5385 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, in getF32Constant() argument
5387 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl, in getF32Constant()
5392 SelectionDAG &DAG) { in getLimitedPrecisionExp2() argument
5396 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); in getLimitedPrecisionExp2()
5399 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); in getLimitedPrecisionExp2()
5400 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); in getLimitedPrecisionExp2()
5404 DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, in getLimitedPrecisionExp2()
5405 DAG.getConstant(23, dl, in getLimitedPrecisionExp2()
5406 DAG.getTargetLoweringInfo().getShiftAmountTy( in getLimitedPrecisionExp2()
5407 MVT::i32, DAG.getDataLayout()))); in getLimitedPrecisionExp2()
5418 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in getLimitedPrecisionExp2()
5419 getF32Constant(DAG, 0x3e814304, dl)); in getLimitedPrecisionExp2()
5420 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, in getLimitedPrecisionExp2()
5421 getF32Constant(DAG, 0x3f3c50c8, dl)); in getLimitedPrecisionExp2()
5422 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); in getLimitedPrecisionExp2()
5423 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, in getLimitedPrecisionExp2()
5424 getF32Constant(DAG, 0x3f7f5e7e, dl)); in getLimitedPrecisionExp2()
5434 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in getLimitedPrecisionExp2()
5435 getF32Constant(DAG, 0x3da235e3, dl)); in getLimitedPrecisionExp2()
5436 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, in getLimitedPrecisionExp2()
5437 getF32Constant(DAG, 0x3e65b8f3, dl)); in getLimitedPrecisionExp2()
5438 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); in getLimitedPrecisionExp2()
5439 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, in getLimitedPrecisionExp2()
5440 getF32Constant(DAG, 0x3f324b07, dl)); in getLimitedPrecisionExp2()
5441 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); in getLimitedPrecisionExp2()
5442 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, in getLimitedPrecisionExp2()
5443 getF32Constant(DAG, 0x3f7ff8fd, dl)); in getLimitedPrecisionExp2()
5455 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in getLimitedPrecisionExp2()
5456 getF32Constant(DAG, 0x3924b03e, dl)); in getLimitedPrecisionExp2()
5457 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, in getLimitedPrecisionExp2()
5458 getF32Constant(DAG, 0x3ab24b87, dl)); in getLimitedPrecisionExp2()
5459 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); in getLimitedPrecisionExp2()
5460 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, in getLimitedPrecisionExp2()
5461 getF32Constant(DAG, 0x3c1d8c17, dl)); in getLimitedPrecisionExp2()
5462 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); in getLimitedPrecisionExp2()
5463 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, in getLimitedPrecisionExp2()
5464 getF32Constant(DAG, 0x3d634a1d, dl)); in getLimitedPrecisionExp2()
5465 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); in getLimitedPrecisionExp2()
5466 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, in getLimitedPrecisionExp2()
5467 getF32Constant(DAG, 0x3e75fe14, dl)); in getLimitedPrecisionExp2()
5468 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); in getLimitedPrecisionExp2()
5469 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, in getLimitedPrecisionExp2()
5470 getF32Constant(DAG, 0x3f317234, dl)); in getLimitedPrecisionExp2()
5471 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); in getLimitedPrecisionExp2()
5472 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, in getLimitedPrecisionExp2()
5473 getF32Constant(DAG, 0x3f800000, dl)); in getLimitedPrecisionExp2()
5477 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX); in getLimitedPrecisionExp2()
5478 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, in getLimitedPrecisionExp2()
5479 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX)); in getLimitedPrecisionExp2()
5484 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, in expandExp() argument
5495 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op, in expandExp()
5496 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32)); in expandExp()
5497 return getLimitedPrecisionExp2(t0, dl, DAG); in expandExp()
5501 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags); in expandExp()
5506 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, in expandLog() argument
5512 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); in expandLog()
5515 SDValue Exp = GetExponent(DAG, Op1, TLI, dl); in expandLog()
5517 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, in expandLog()
5518 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32)); in expandLog()
5522 SDValue X = GetSignificand(DAG, Op1, dl); in expandLog()
5533 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in expandLog()
5534 getF32Constant(DAG, 0xbe74c456, dl)); in expandLog()
5535 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, in expandLog()
5536 getF32Constant(DAG, 0x3fb3a2b1, dl)); in expandLog()
5537 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); in expandLog()
5538 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, in expandLog()
5539 getF32Constant(DAG, 0x3f949a29, dl)); in expandLog()
5550 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in expandLog()
5551 getF32Constant(DAG, 0xbd67b6d6, dl)); in expandLog()
5552 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, in expandLog()
5553 getF32Constant(DAG, 0x3ee4f4b8, dl)); in expandLog()
5554 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); in expandLog()
5555 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, in expandLog()
5556 getF32Constant(DAG, 0x3fbc278b, dl)); in expandLog()
5557 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); in expandLog()
5558 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, in expandLog()
5559 getF32Constant(DAG, 0x40348e95, dl)); in expandLog()
5560 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); in expandLog()
5561 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, in expandLog()
5562 getF32Constant(DAG, 0x3fdef31a, dl)); in expandLog()
5575 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in expandLog()
5576 getF32Constant(DAG, 0xbc91e5ac, dl)); in expandLog()
5577 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, in expandLog()
5578 getF32Constant(DAG, 0x3e4350aa, dl)); in expandLog()
5579 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); in expandLog()
5580 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, in expandLog()
5581 getF32Constant(DAG, 0x3f60d3e3, dl)); in expandLog()
5582 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); in expandLog()
5583 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, in expandLog()
5584 getF32Constant(DAG, 0x4011cdf0, dl)); in expandLog()
5585 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); in expandLog()
5586 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, in expandLog()
5587 getF32Constant(DAG, 0x406cfd1c, dl)); in expandLog()
5588 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); in expandLog()
5589 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, in expandLog()
5590 getF32Constant(DAG, 0x408797cb, dl)); in expandLog()
5591 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); in expandLog()
5592 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, in expandLog()
5593 getF32Constant(DAG, 0x4006dcab, dl)); in expandLog()
5596 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); in expandLog()
5600 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags); in expandLog()
5605 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, in expandLog2() argument
5611 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); in expandLog2()
5614 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl); in expandLog2()
5618 SDValue X = GetSignificand(DAG, Op1, dl); in expandLog2()
5629 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in expandLog2()
5630 getF32Constant(DAG, 0xbeb08fe0, dl)); in expandLog2()
5631 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, in expandLog2()
5632 getF32Constant(DAG, 0x40019463, dl)); in expandLog2()
5633 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); in expandLog2()
5634 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, in expandLog2()
5635 getF32Constant(DAG, 0x3fd6633d, dl)); in expandLog2()
5646 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in expandLog2()
5647 getF32Constant(DAG, 0xbda7262e, dl)); in expandLog2()
5648 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, in expandLog2()
5649 getF32Constant(DAG, 0x3f25280b, dl)); in expandLog2()
5650 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); in expandLog2()
5651 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, in expandLog2()
5652 getF32Constant(DAG, 0x4007b923, dl)); in expandLog2()
5653 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); in expandLog2()
5654 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, in expandLog2()
5655 getF32Constant(DAG, 0x40823e2f, dl)); in expandLog2()
5656 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); in expandLog2()
5657 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, in expandLog2()
5658 getF32Constant(DAG, 0x4020d29c, dl)); in expandLog2()
5672 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in expandLog2()
5673 getF32Constant(DAG, 0xbcd2769e, dl)); in expandLog2()
5674 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, in expandLog2()
5675 getF32Constant(DAG, 0x3e8ce0b9, dl)); in expandLog2()
5676 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); in expandLog2()
5677 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, in expandLog2()
5678 getF32Constant(DAG, 0x3fa22ae7, dl)); in expandLog2()
5679 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); in expandLog2()
5680 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, in expandLog2()
5681 getF32Constant(DAG, 0x40525723, dl)); in expandLog2()
5682 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); in expandLog2()
5683 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, in expandLog2()
5684 getF32Constant(DAG, 0x40aaf200, dl)); in expandLog2()
5685 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); in expandLog2()
5686 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, in expandLog2()
5687 getF32Constant(DAG, 0x40c39dad, dl)); in expandLog2()
5688 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); in expandLog2()
5689 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, in expandLog2()
5690 getF32Constant(DAG, 0x4042902c, dl)); in expandLog2()
5693 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); in expandLog2()
5697 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags); in expandLog2()
5702 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, in expandLog10() argument
5708 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); in expandLog10()
5711 SDValue Exp = GetExponent(DAG, Op1, TLI, dl); in expandLog10()
5712 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, in expandLog10()
5713 getF32Constant(DAG, 0x3e9a209a, dl)); in expandLog10()
5717 SDValue X = GetSignificand(DAG, Op1, dl); in expandLog10()
5728 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in expandLog10()
5729 getF32Constant(DAG, 0xbdd49a13, dl)); in expandLog10()
5730 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, in expandLog10()
5731 getF32Constant(DAG, 0x3f1c0789, dl)); in expandLog10()
5732 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); in expandLog10()
5733 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, in expandLog10()
5734 getF32Constant(DAG, 0x3f011300, dl)); in expandLog10()
5744 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in expandLog10()
5745 getF32Constant(DAG, 0x3d431f31, dl)); in expandLog10()
5746 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, in expandLog10()
5747 getF32Constant(DAG, 0x3ea21fb2, dl)); in expandLog10()
5748 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); in expandLog10()
5749 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, in expandLog10()
5750 getF32Constant(DAG, 0x3f6ae232, dl)); in expandLog10()
5751 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); in expandLog10()
5752 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, in expandLog10()
5753 getF32Constant(DAG, 0x3f25f7c3, dl)); in expandLog10()
5765 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, in expandLog10()
5766 getF32Constant(DAG, 0x3c5d51ce, dl)); in expandLog10()
5767 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, in expandLog10()
5768 getF32Constant(DAG, 0x3e00685a, dl)); in expandLog10()
5769 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); in expandLog10()
5770 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, in expandLog10()
5771 getF32Constant(DAG, 0x3efb6798, dl)); in expandLog10()
5772 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); in expandLog10()
5773 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, in expandLog10()
5774 getF32Constant(DAG, 0x3f88d192, dl)); in expandLog10()
5775 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); in expandLog10()
5776 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, in expandLog10()
5777 getF32Constant(DAG, 0x3fc4316c, dl)); in expandLog10()
5778 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); in expandLog10()
5779 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8, in expandLog10()
5780 getF32Constant(DAG, 0x3f57ce70, dl)); in expandLog10()
5783 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); in expandLog10()
5787 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags); in expandLog10()
5792 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, in expandExp2() argument
5796 return getLimitedPrecisionExp2(Op, dl, DAG); in expandExp2()
5799 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags); in expandExp2()
5805 SelectionDAG &DAG, const TargetLowering &TLI, in expandPow() argument
5823 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS, in expandPow()
5824 getF32Constant(DAG, 0x40549a78, dl)); in expandPow()
5825 return getLimitedPrecisionExp2(t0, dl, DAG); in expandPow()
5829 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags); in expandPow()
5834 SelectionDAG &DAG) { in ExpandPowI() argument
5843 return DAG.getConstantFP(1.0, DL, LHS.getValueType()); in ExpandPowI()
5845 if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI( in ExpandPowI()
5846 Val, DAG.shouldOptForSize())) { in ExpandPowI()
5862 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare); in ExpandPowI()
5867 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(), in ExpandPowI()
5874 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(), in ExpandPowI()
5875 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res); in ExpandPowI()
5881 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); in ExpandPowI()
5886 SelectionDAG &DAG, const TargetLowering &TLI) { in expandDivFix() argument
5890 LLVMContext &Ctx = *DAG.getContext(); in expandDivFix()
5925 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT); in expandDivFix()
5926 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT); in expandDivFix()
5927 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout()); in expandDivFix()
5931 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS, in expandDivFix()
5932 DAG.getConstant(1, DL, ShiftTy)); in expandDivFix()
5933 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale); in expandDivFix()
5935 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res, in expandDivFix()
5936 DAG.getConstant(1, DL, ShiftTy)); in expandDivFix()
5937 return DAG.getZExtOrTrunc(Res, DL, VT); in expandDivFix()
5941 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale); in expandDivFix()
5985 MachineFunction &MF = DAG.getMachineFunction(); in EmitFuncArgumentDbgValue()
5986 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); in EmitFuncArgumentDbgValue()
6149 SDDbgValue *SDV = DAG.getConstantDbgValue( in EmitFuncArgumentDbgValue()
6151 DAG.AddDbgValue(SDV, false); in EmitFuncArgumentDbgValue()
6165 const auto &TLI = DAG.getTargetLoweringInfo(); in EmitFuncArgumentDbgValue()
6166 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second, in EmitFuncArgumentDbgValue()
6218 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(), in getDbgValue()
6221 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), in getDbgValue()
6251 SDValue Callee = DAG.getExternalSymbol( in lowerCallToExternalSymbol()
6253 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())); in lowerCallToExternalSymbol()
6298 SDDbgValue *SDV = DAG.getVRegDbgValue( in visitEntryValueDbgValue()
6300 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/); in visitEntryValueDbgValue()
6314 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped)); in visitConvergenceControl()
6317 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped)); in visitConvergenceControl()
6322 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped, in visitConvergenceControl()
6340 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitVectorHistogram()
6341 DataLayout TargetDL = DAG.getDataLayout(); in visitVectorHistogram()
6343 Align Alignment = DAG.getEVTAlign(VT); in visitVectorHistogram()
6347 SDValue Root = DAG.getRoot(); in visitVectorHistogram()
6357 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitVectorHistogram()
6363 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); in visitVectorHistogram()
6367 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); in visitVectorHistogram()
6374 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); in visitVectorHistogram()
6377 SDValue ID = DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32); in visitVectorHistogram()
6380 SDValue Histogram = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), VT, sdl, in visitVectorHistogram()
6384 DAG.setRoot(Histogram); in visitVectorHistogram()
6390 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitIntrinsicCall()
6405 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
6406 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1))); in visitIntrinsicCall()
6413 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, in visitIntrinsicCall()
6414 TLI.getValueType(DAG.getDataLayout(), I.getType()), in visitIntrinsicCall()
6419 DAG.getNode(ISD::ADDROFRETURNADDR, sdl, in visitIntrinsicCall()
6420 TLI.getValueType(DAG.getDataLayout(), I.getType()))); in visitIntrinsicCall()
6424 DAG.getNode(ISD::SPONENTRY, sdl, in visitIntrinsicCall()
6425 TLI.getValueType(DAG.getDataLayout(), I.getType()))); in visitIntrinsicCall()
6428 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, in visitIntrinsicCall()
6429 TLI.getFrameIndexTy(DAG.getDataLayout()), in visitIntrinsicCall()
6437 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata())); in visitIntrinsicCall()
6438 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
6439 Res = DAG.getNode(ISD::READ_REGISTER, sdl, in visitIntrinsicCall()
6440 DAG.getVTList(VT, MVT::Other), Chain, RegName); in visitIntrinsicCall()
6442 DAG.setRoot(Res.getValue(1)); in visitIntrinsicCall()
6450 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata())); in visitIntrinsicCall()
6451 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain, in visitIntrinsicCall()
6468 SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol, in visitIntrinsicCall()
6489 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol, in visitIntrinsicCall()
6506 SDValue MS = DAG.getMemset( in visitIntrinsicCall()
6522 SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol, in visitIntrinsicCall()
6542 SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, &I, in visitIntrinsicCall()
6558 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); in visitIntrinsicCall()
6560 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz, in visitIntrinsicCall()
6574 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); in visitIntrinsicCall()
6576 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz, in visitIntrinsicCall()
6590 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); in visitIntrinsicCall()
6592 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz, in visitIntrinsicCall()
6599 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall); in visitIntrinsicCall()
6600 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other, in visitIntrinsicCall()
6603 DAG.setRoot(Res); in visitIntrinsicCall()
6608 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall); in visitIntrinsicCall()
6612 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl, in visitIntrinsicCall()
6614 SDValue Res = DAG.getNode( in visitIntrinsicCall()
6616 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops); in visitIntrinsicCall()
6618 DAG.setRoot(Res.getValue(1)); in visitIntrinsicCall()
6645 SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder); in visitIntrinsicCall()
6646 DAG.AddDbgLabel(SDV); in visitIntrinsicCall()
6688 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV); in visitIntrinsicCall()
6689 Res = DAG.getConstant(TypeID, sdl, MVT::i32); in visitIntrinsicCall()
6696 DAG.getMachineFunction().setCallsEHReturn(true); in visitIntrinsicCall()
6697 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl, in visitIntrinsicCall()
6704 DAG.getMachineFunction().setCallsUnwindInit(true); in visitIntrinsicCall()
6707 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl, in visitIntrinsicCall()
6708 TLI.getPointerTy(DAG.getDataLayout()), in visitIntrinsicCall()
6712 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); in visitIntrinsicCall()
6721 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); in visitIntrinsicCall()
6732 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl, in visitIntrinsicCall()
6733 DAG.getVTList(MVT::i32, MVT::Other), Ops); in visitIntrinsicCall()
6735 DAG.setRoot(Op.getValue(1)); in visitIntrinsicCall()
6739 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, in visitIntrinsicCall()
6743 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other, in visitIntrinsicCall()
6766 getValue(I.getArgOperand(1)), DAG)); in visitIntrinsicCall()
6769 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); in visitIntrinsicCall()
6773 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); in visitIntrinsicCall()
6777 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); in visitIntrinsicCall()
6780 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); in visitIntrinsicCall()
6784 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); in visitIntrinsicCall()
6788 getValue(I.getArgOperand(1)), DAG, TLI, Flags)); in visitIntrinsicCall()
6837 setValue(&I, DAG.getNode(Opcode, sdl, in visitIntrinsicCall()
6857 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
6858 setValue(&I, DAG.getNode(Opcode, sdl, RetVT, in visitIntrinsicCall()
6863 setValue(&I, DAG.getNode(ISD::FMINNUM, sdl, in visitIntrinsicCall()
6869 setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl, in visitIntrinsicCall()
6875 setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl, in visitIntrinsicCall()
6881 setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl, in visitIntrinsicCall()
6887 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl, in visitIntrinsicCall()
6893 setValue(&I, DAG.getNode(ISD::FLDEXP, sdl, in visitIntrinsicCall()
6900 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs); in visitIntrinsicCall()
6901 SDVTList VTs = DAG.getVTList(ValueVTs); in visitIntrinsicCall()
6903 DAG.getNode(ISD::FFREXP, sdl, VTs, getValue(I.getArgOperand(0)))); in visitIntrinsicCall()
6907 setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl, in visitIntrinsicCall()
6913 setValue(&I, DAG.getNode( in visitIntrinsicCall()
6934 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
6939 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); in visitIntrinsicCall()
6942 Result = DAG.getNode( in visitIntrinsicCall()
6944 DAG.getTargetConstant((int)*RoundMode, sdl, in visitIntrinsicCall()
6945 TLI.getPointerTy(DAG.getDataLayout()))); in visitIntrinsicCall()
6951 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
6953 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) { in visitIntrinsicCall()
6954 setValue(&I, DAG.getNode(ISD::FMA, sdl, in visitIntrinsicCall()
6961 SDValue Mul = DAG.getNode( in visitIntrinsicCall()
6964 SDValue Add = DAG.getNode(ISD::FADD, sdl, in visitIntrinsicCall()
6972 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16, in visitIntrinsicCall()
6973 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16, in visitIntrinsicCall()
6975 DAG.getTargetConstant(0, sdl, in visitIntrinsicCall()
6979 setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl, in visitIntrinsicCall()
6980 TLI.getValueType(DAG.getDataLayout(), I.getType()), in visitIntrinsicCall()
6981 DAG.getNode(ISD::BITCAST, sdl, MVT::f16, in visitIntrinsicCall()
6985 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
6986 setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT, in visitIntrinsicCall()
6988 DAG.getValueType(VT.getScalarType()))); in visitIntrinsicCall()
6992 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
6993 setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT, in visitIntrinsicCall()
6995 DAG.getValueType(VT.getScalarType()))); in visitIntrinsicCall()
6999 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other, in visitIntrinsicCall()
7002 DAG.setRoot(Res.getValue(0)); in visitIntrinsicCall()
7005 const DataLayout DLayout = DAG.getDataLayout(); in visitIntrinsicCall()
7010 MachineFunction &MF = DAG.getMachineFunction(); in visitIntrinsicCall()
7020 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG); in visitIntrinsicCall()
7025 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32); in visitIntrinsicCall()
7026 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags); in visitIntrinsicCall()
7031 const DataLayout DLayout = DAG.getDataLayout(); in visitIntrinsicCall()
7033 Align TempAlign = DAG.getEVTAlign(EnvVT); in visitIntrinsicCall()
7038 Res = DAG.getNode( in visitIntrinsicCall()
7040 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()), in visitIntrinsicCall()
7044 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value()); in visitIntrinsicCall()
7047 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); in visitIntrinsicCall()
7048 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitIntrinsicCall()
7051 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO); in visitIntrinsicCall()
7052 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI); in visitIntrinsicCall()
7055 DAG.setRoot(Res.getValue(1)); in visitIntrinsicCall()
7059 const DataLayout DLayout = DAG.getDataLayout(); in visitIntrinsicCall()
7062 Align TempAlign = DAG.getEVTAlign(EnvVT); in visitIntrinsicCall()
7067 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env); in visitIntrinsicCall()
7071 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value()); in visitIntrinsicCall()
7074 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); in visitIntrinsicCall()
7075 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign, in visitIntrinsicCall()
7077 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitIntrinsicCall()
7080 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO); in visitIntrinsicCall()
7082 DAG.setRoot(Chain); in visitIntrinsicCall()
7086 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot())); in visitIntrinsicCall()
7089 Res = DAG.getNode( in visitIntrinsicCall()
7091 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()), in visitIntrinsicCall()
7093 DAG.getRoot()); in visitIntrinsicCall()
7095 DAG.setRoot(Res.getValue(1)); in visitIntrinsicCall()
7098 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()}, in visitIntrinsicCall()
7100 DAG.setRoot(Res); in visitIntrinsicCall()
7103 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot()); in visitIntrinsicCall()
7104 DAG.setRoot(Res); in visitIntrinsicCall()
7109 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp)); in visitIntrinsicCall()
7114 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl, in visitIntrinsicCall()
7115 DAG.getVTList(MVT::i64, MVT::Other), Op); in visitIntrinsicCall()
7117 DAG.setRoot(Res.getValue(1)); in visitIntrinsicCall()
7122 Res = DAG.getNode(ISD::READSTEADYCOUNTER, sdl, in visitIntrinsicCall()
7123 DAG.getVTList(MVT::i64, MVT::Other), Op); in visitIntrinsicCall()
7125 DAG.setRoot(Res.getValue(1)); in visitIntrinsicCall()
7129 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl, in visitIntrinsicCall()
7134 setValue(&I, DAG.getNode(ISD::BSWAP, sdl, in visitIntrinsicCall()
7142 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF, in visitIntrinsicCall()
7150 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF, in visitIntrinsicCall()
7157 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg)); in visitIntrinsicCall()
7170 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z)); in visitIntrinsicCall()
7173 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z)); in visitIntrinsicCall()
7180 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2)); in visitIntrinsicCall()
7186 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2)); in visitIntrinsicCall()
7192 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2)); in visitIntrinsicCall()
7198 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2)); in visitIntrinsicCall()
7204 setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2)); in visitIntrinsicCall()
7210 setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2)); in visitIntrinsicCall()
7220 setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl, in visitIntrinsicCall()
7232 Op1, Op2, Op3, DAG, TLI)); in visitIntrinsicCall()
7238 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2)); in visitIntrinsicCall()
7244 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2)); in visitIntrinsicCall()
7250 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2)); in visitIntrinsicCall()
7256 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2)); in visitIntrinsicCall()
7262 setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1)); in visitIntrinsicCall()
7268 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
7269 setValue(&I, DAG.getNode(ISD::SCMP, sdl, DestVT, Op1, Op2)); in visitIntrinsicCall()
7275 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
7276 setValue(&I, DAG.getNode(ISD::UCMP, sdl, DestVT, Op1, Op2)); in visitIntrinsicCall()
7281 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
7282 Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op); in visitIntrinsicCall()
7284 DAG.setRoot(Res.getValue(1)); in visitIntrinsicCall()
7289 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res)); in visitIntrinsicCall()
7293 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout()); in visitIntrinsicCall()
7294 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
7300 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy), in visitIntrinsicCall()
7302 DAG.setRoot(Op); in visitIntrinsicCall()
7307 MachineFunction &MF = DAG.getMachineFunction(); in visitIntrinsicCall()
7309 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
7312 Res = getLoadStackGuard(DAG, sdl, Chain); in visitIntrinsicCall()
7313 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy); in visitIntrinsicCall()
7316 Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType()); in visitIntrinsicCall()
7317 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global), in visitIntrinsicCall()
7322 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl); in visitIntrinsicCall()
7323 DAG.setRoot(Chain); in visitIntrinsicCall()
7329 MachineFunction &MF = DAG.getMachineFunction(); in visitIntrinsicCall()
7334 Src = getLoadStackGuard(DAG, sdl, Chain); in visitIntrinsicCall()
7342 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout()); in visitIntrinsicCall()
7344 SDValue FIN = DAG.getFrameIndex(FI, PtrTy); in visitIntrinsicCall()
7347 Res = DAG.getStore( in visitIntrinsicCall()
7349 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), in visitIntrinsicCall()
7352 DAG.setRoot(Res); in visitIntrinsicCall()
7379 MachineFunction &MF = DAG.getMachineFunction(); in visitIntrinsicCall()
7383 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label); in visitIntrinsicCall()
7384 DAG.setRoot(Res); in visitIntrinsicCall()
7396 Ops[4] = DAG.getSrcValue(I.getArgOperand(0)); in visitIntrinsicCall()
7397 Ops[5] = DAG.getSrcValue(F); in visitIntrinsicCall()
7399 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops); in visitIntrinsicCall()
7401 DAG.setRoot(Res); in visitIntrinsicCall()
7405 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl, in visitIntrinsicCall()
7406 TLI.getPointerTy(DAG.getDataLayout()), in visitIntrinsicCall()
7410 assert(DAG.getMachineFunction().getFunction().hasGC() && in visitIntrinsicCall()
7424 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot()); in visitIntrinsicCall()
7426 DAG.setRoot(Res.getValue(1)); in visitIntrinsicCall()
7442 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot())); in visitIntrinsicCall()
7445 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot())); in visitIntrinsicCall()
7448 DAG.setRoot(DAG.getNode( in visitIntrinsicCall()
7450 DAG.getTargetConstant( in visitIntrinsicCall()
7466 TargetLowering::CallLoweringInfo CLI(DAG); in visitIntrinsicCall()
7469 DAG.getExternalSymbol(TrapFuncName.data(), in visitIntrinsicCall()
7470 TLI.getPointerTy(DAG.getDataLayout())), in visitIntrinsicCall()
7474 DAG.setRoot(Result.second); in visitIntrinsicCall()
7508 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT); in visitIntrinsicCall()
7509 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2)); in visitIntrinsicCall()
7516 Ops[0] = DAG.getRoot(); in visitIntrinsicCall()
7518 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl, in visitIntrinsicCall()
7520 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl, in visitIntrinsicCall()
7522 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl, in visitIntrinsicCall()
7524 SDValue Result = DAG.getMemIntrinsicNode( in visitIntrinsicCall()
7525 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops, in visitIntrinsicCall()
7533 DAG.setRoot(Result); in visitIntrinsicCall()
7565 ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject) in visitIntrinsicCall()
7567 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize, in visitIntrinsicCall()
7569 DAG.setRoot(Res); in visitIntrinsicCall()
7577 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr); in visitIntrinsicCall()
7578 DAG.setRoot(Res); in visitIntrinsicCall()
7584 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType()))); in visitIntrinsicCall()
7590 SDValue InputChain = DAG.getRoot(); in visitIntrinsicCall()
7593 Res = DAG.getNode(ISD::CLEAR_CACHE, sdl, DAG.getVTList(MVT::Other), in visitIntrinsicCall()
7596 DAG.setRoot(Res); in visitIntrinsicCall()
7635 MachineFunction &MF = DAG.getMachineFunction(); in visitIntrinsicCall()
7636 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); in visitIntrinsicCall()
7661 MachineFunction &MF = DAG.getMachineFunction(); in visitIntrinsicCall()
7677 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT); in visitIntrinsicCall()
7679 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym); in visitIntrinsicCall()
7682 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl); in visitIntrinsicCall()
7692 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); in visitIntrinsicCall()
7695 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT); in visitIntrinsicCall()
7697 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32); in visitIntrinsicCall()
7704 const auto &Triple = DAG.getTarget().getTargetTriple(); in visitIntrinsicCall()
7713 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in visitIntrinsicCall()
7723 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL, in visitIntrinsicCall()
7726 DAG.setRoot(patchableNode); in visitIntrinsicCall()
7733 const auto &Triple = DAG.getTarget().getTargetTriple(); in visitIntrinsicCall()
7745 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in visitIntrinsicCall()
7756 MachineSDNode *MN = DAG.getMachineNode( in visitIntrinsicCall()
7759 DAG.setRoot(patchableNode); in visitIntrinsicCall()
7793 I.getArgOperand(1), Offset, DAG.getDataLayout())); in visitIntrinsicCall()
7797 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0)); in visitIntrinsicCall()
7807 I.getArgOperand(Op), Offset, DAG.getDataLayout())); in visitIntrinsicCall()
7817 Targets.push_back({Offset, DAG.getTargetGlobalAddress( in visitIntrinsicCall()
7827 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32)); in visitIntrinsicCall()
7831 Ops.push_back(DAG.getRoot()); // Chain in visitIntrinsicCall()
7832 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl, in visitIntrinsicCall()
7835 DAG.setRoot(N); in visitIntrinsicCall()
7849 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); in visitIntrinsicCall()
7852 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)), in visitIntrinsicCall()
7855 DAG.setRoot(Val); in visitIntrinsicCall()
7890 TargetLowering::CallLoweringInfo CLI(DAG); in visitIntrinsicCall()
7916 TLI.getValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); in visitIntrinsicCall()
7918 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); in visitIntrinsicCall()
7922 Mask = DAG.getPtrExtOrTrunc(Mask, sdl, PtrVT); in visitIntrinsicCall()
7924 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask)); in visitIntrinsicCall()
7932 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
7942 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT, in visitIntrinsicCall()
7945 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index); in visitIntrinsicCall()
7946 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount); in visitIntrinsicCall()
7947 SDValue VectorStep = DAG.getStepVector(sdl, VecTy); in visitIntrinsicCall()
7948 SDValue VectorInduction = DAG.getNode( in visitIntrinsicCall()
7950 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction, in visitIntrinsicCall()
7971 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
7975 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count); in visitIntrinsicCall()
7979 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT, in visitIntrinsicCall()
7982 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL); in visitIntrinsicCall()
7984 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin); in visitIntrinsicCall()
8001 auto SourceIndex = DAG.getVectorIdxConstant(i * Stride, sdl); in visitIntrinsicCall()
8002 Subvectors.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ReducedTy, in visitIntrinsicCall()
8008 Subvectors.push_back(DAG.getNode(ISD::ADD, sdl, ReducedTy, in visitIntrinsicCall()
8032 SDValue AllZero = DAG.getConstant(0, DL, OpVT); in visitIntrinsicCall()
8033 OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in visitIntrinsicCall()
8035 Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE); in visitIntrinsicCall()
8051 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltTy, in visitIntrinsicCall()
8055 DAG.getElementCount(DL, NewEltTy, OpVT.getVectorElementCount()); in visitIntrinsicCall()
8057 SDValue StepVec = DAG.getStepVector(DL, NewVT); in visitIntrinsicCall()
8058 SDValue SplatVL = DAG.getSplat(NewVT, DL, VL); in visitIntrinsicCall()
8059 SDValue StepVL = DAG.getNode(ISD::SUB, DL, NewVT, SplatVL, StepVec); in visitIntrinsicCall()
8060 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, Op); in visitIntrinsicCall()
8061 SDValue And = DAG.getNode(ISD::AND, DL, NewVT, StepVL, Ext); in visitIntrinsicCall()
8062 SDValue Max = DAG.getNode(ISD::VECREDUCE_UMAX, DL, NewEltTy, And); in visitIntrinsicCall()
8063 SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltTy, VL, Max); in visitIntrinsicCall()
8065 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
8066 SDValue Ret = DAG.getZExtOrTrunc(Sub, DL, RetTy); in visitIntrinsicCall()
8078 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); in visitIntrinsicCall()
8080 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl); in visitIntrinsicCall()
8082 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
8083 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec, in visitIntrinsicCall()
8090 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitIntrinsicCall()
8094 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); in visitIntrinsicCall()
8096 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl); in visitIntrinsicCall()
8099 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index)); in visitIntrinsicCall()
8118 setValue(&I, DAG.getNode(ISD::VECTOR_COMPRESS, sdl, in visitIntrinsicCall()
8143 SDValue Chain = DAG.getRoot(); in visitConstrainedFPIntrinsic()
8175 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitConstrainedFPIntrinsic()
8176 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType()); in visitConstrainedFPIntrinsic()
8177 SDVTList VTs = DAG.getVTList(VT, MVT::Other); in visitConstrainedFPIntrinsic()
8199 !TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) { in visitConstrainedFPIntrinsic()
8201 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags); in visitConstrainedFPIntrinsic()
8219 DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()))); in visitConstrainedFPIntrinsic()
8227 Opers.push_back(DAG.getCondCode(Condition)); in visitConstrainedFPIntrinsic()
8232 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags); in visitConstrainedFPIntrinsic()
8290 Alignment = DAG.getEVTAlign(VT); in visitVPLoad()
8293 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); in visitVPLoad()
8294 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitVPLoad()
8297 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2], in visitVPLoad()
8308 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitVPGather()
8315 Alignment = DAG.getEVTAlign(VT.getScalarType()); in visitVPGather()
8318 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitVPGather()
8327 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout())); in visitVPGather()
8330 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())); in visitVPGather()
8336 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index); in visitVPGather()
8338 LD = DAG.getGatherVP( in visitVPGather()
8339 DAG.getVTList(VT, MVT::Other), VT, DL, in visitVPGather()
8340 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO, in visitVPGather()
8355 Alignment = DAG.getEVTAlign(VT); in visitVPStore()
8357 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); in visitVPStore()
8358 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitVPStore()
8361 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset, in visitVPStore()
8364 DAG.setRoot(ST); in visitVPStore()
8371 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitVPScatter()
8378 Alignment = DAG.getEVTAlign(VT.getScalarType()); in visitVPScatter()
8381 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitVPScatter()
8390 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout())); in visitVPScatter()
8394 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())); in visitVPScatter()
8400 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index); in visitVPScatter()
8402 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL, in visitVPScatter()
8406 DAG.setRoot(ST); in visitVPScatter()
8417 Alignment = DAG.getEVTAlign(VT.getScalarType()); in visitVPStridedLoad()
8422 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); in visitVPStridedLoad()
8424 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitVPStridedLoad()
8428 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], in visitVPStridedLoad()
8444 Alignment = DAG.getEVTAlign(VT.getScalarType()); in visitVPStridedStore()
8447 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( in visitVPStridedStore()
8451 SDValue ST = DAG.getStridedStoreVP( in visitVPStridedStore()
8453 DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3], in visitVPStridedStore()
8457 DAG.setRoot(ST); in visitVPStridedStore()
8462 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitVPCmp()
8487 EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL); in visitVPCmp()
8489 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitVPCmp()
8492 DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL)); in visitVPCmp()
8506 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitVectorPredicationIntrinsic()
8507 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs); in visitVectorPredicationIntrinsic()
8508 SDVTList VTs = DAG.getVTList(ValueVTs); in visitVectorPredicationIntrinsic()
8521 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op); in visitVectorPredicationIntrinsic()
8530 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags); in visitVectorPredicationIntrinsic()
8558 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), ValueVTs[0])) { in visitVectorPredicationIntrinsic()
8559 setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags)); in visitVectorPredicationIntrinsic()
8561 SDValue Mul = DAG.getNode( in visitVectorPredicationIntrinsic()
8565 DAG.getNode(ISD::VP_FADD, DL, VTs, in visitVectorPredicationIntrinsic()
8572 const DataLayout DLayout = DAG.getDataLayout(); in visitVectorPredicationIntrinsic()
8575 SDValue Check = DAG.getTargetConstant(Constant, DL, MVT::i32); in visitVectorPredicationIntrinsic()
8576 SDValue V = DAG.getNode(ISD::VP_IS_FPCLASS, DL, DestVT, in visitVectorPredicationIntrinsic()
8583 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType()); in visitVectorPredicationIntrinsic()
8584 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType()); in visitVectorPredicationIntrinsic()
8585 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1], in visitVectorPredicationIntrinsic()
8587 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1], in visitVectorPredicationIntrinsic()
8594 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitVectorPredicationIntrinsic()
8596 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), in visitVectorPredicationIntrinsic()
8598 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1], in visitVectorPredicationIntrinsic()
8600 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1], in visitVectorPredicationIntrinsic()
8613 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]}); in visitVectorPredicationIntrinsic()
8623 MachineFunction &MF = DAG.getMachineFunction(); in lowerStartEH()
8641 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel); in lowerStartEH()
8649 MachineFunction &MF = DAG.getMachineFunction(); in lowerEndEH()
8654 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel); in lowerEndEH()
8681 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel)); in lowerInvokable()
8685 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in lowerInvokable()
8702 DAG.setRoot(Result.second); in lowerInvokable()
8706 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB, in lowerInvokable()
8718 auto &DL = DAG.getDataLayout(); in LowerCallTo()
8726 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerCallTo()
8763 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V), in LowerCallTo()
8789 if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget())) in LowerCallTo()
8814 TargetLowering::CallLoweringInfo CLI(DAG); in LowerCallTo()
8836 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first); in LowerCallTo()
8848 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src); in LowerCallTo()
8849 DAG.setRoot(CopyNode); in LowerCallTo()
8869 LoadTy, Builder.DAG.getDataLayout())) in getMemCmpLoad()
8880 Root = Builder.DAG.getEntryNode(); in getMemCmpLoad()
8884 Root = Builder.DAG.getRoot(); in getMemCmpLoad()
8889 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr, in getMemCmpLoad()
8902 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in processIntegerCallValue()
8904 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT); in processIntegerCallValue()
8918 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), in visitMemCmpBCmpCall()
8920 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT)); in visitMemCmpBCmpCall()
8924 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); in visitMemCmpBCmpCall()
8926 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS), in visitMemCmpBCmpCall()
8944 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitMemCmpBCmpCall()
8991 LoadL = DAG.getBitcast(CmpVT, LoadL); in visitMemCmpBCmpCall()
8992 LoadR = DAG.getBitcast(CmpVT, LoadR); in visitMemCmpBCmpCall()
8995 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE); in visitMemCmpBCmpCall()
9010 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); in visitMemChrCall()
9012 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(), in visitMemChrCall()
9034 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne(); in visitMemPCpyCall()
9035 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne(); in visitMemPCpyCall()
9045 SDValue MC = DAG.getMemcpy( in visitMemPCpyCall()
9051 DAG.setRoot(MC); in visitMemPCpyCall()
9054 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType()); in visitMemPCpyCall()
9057 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(), in visitMemPCpyCall()
9071 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); in visitStrCpyCall()
9073 TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(), in visitStrCpyCall()
9079 DAG.setRoot(Res.second); in visitStrCpyCall()
9094 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); in visitStrCmpCall()
9096 TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(), in visitStrCmpCall()
9117 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); in visitStrLenCall()
9119 TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(), in visitStrLenCall()
9138 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); in visitStrNLenCall()
9140 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(), in visitStrNLenCall()
9168 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags)); in visitUnaryFloatCall()
9189 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags)); in visitBinaryFloatCall()
9237 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(), in visitCall()
9463 DAG.getDataLayout())) in LowerCallSiteWithPtrAuthBundle()
9519 SelectionDAG &DAG) { in patchMatchingInput() argument
9523 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo(); in patchMatchingInput()
9524 const auto &TLI = DAG.getTargetLoweringInfo(); in patchMatchingInput()
9548 SelectionDAG &DAG) { in getAddressForMemoryInput() argument
9549 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getAddressForMemoryInput()
9563 OpInfo.CallOperand = DAG.getConstantPool( in getAddressForMemoryInput()
9564 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout())); in getAddressForMemoryInput()
9570 auto &DL = DAG.getDataLayout(); in getAddressForMemoryInput()
9572 MachineFunction &MF = DAG.getMachineFunction(); in getAddressForMemoryInput()
9580 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL)); in getAddressForMemoryInput()
9581 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot, in getAddressForMemoryInput()
9598 getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL, in getRegistersForValue() argument
9601 LLVMContext &Context = *DAG.getContext(); in getRegistersForValue()
9602 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getRegistersForValue()
9604 MachineFunction &MF = DAG.getMachineFunction(); in getRegistersForValue()
9648 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand); in getRegistersForValue()
9657 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand); in getRegistersForValue()
9787 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitInlineAsm()
9789 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call); in visitInlineAsm()
9825 SDValue Glue, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot(); in visitInlineAsm()
9860 patchMatchingInput(OpInfo, Input, DAG); in visitInlineAsm()
9864 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); in visitInlineAsm()
9906 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG); in visitInlineAsm()
9920 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol( in visitInlineAsm()
9921 IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout()))); in visitInlineAsm()
9927 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc)); in visitInlineAsm()
9931 AsmNodeOperands.push_back(DAG.getTargetConstant( in visitInlineAsm()
9932 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); in visitInlineAsm()
9943 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo); in visitInlineAsm()
9945 const MachineFunction &MF = DAG.getMachineFunction(); in visitInlineAsm()
9956 const MachineFunction &MF = DAG.getMachineFunction(); in visitInlineAsm()
9985 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(), in visitInlineAsm()
10007 false, 0, getCurSDLoc(), DAG, AsmNodeOperands); in visitInlineAsm()
10031 MachineFunction &MF = DAG.getMachineFunction(); in visitInlineAsm()
10048 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, &Call); in visitInlineAsm()
10050 OpInfo.getMatchedOperand(), dl, DAG, in visitInlineAsm()
10062 AsmNodeOperands.push_back(DAG.getTargetConstant( in visitInlineAsm()
10063 Flag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); in visitInlineAsm()
10077 Ops, DAG); in visitInlineAsm()
10094 AsmNodeOperands.push_back(DAG.getTargetConstant( in visitInlineAsm()
10095 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); in visitInlineAsm()
10105 TLI.getPointerTy(DAG.getDataLayout()) && in visitInlineAsm()
10116 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, in visitInlineAsm()
10135 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(), in visitInlineAsm()
10144 DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32)); in visitInlineAsm()
10179 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, in visitInlineAsm()
10183 0, dl, DAG, AsmNodeOperands); in visitInlineAsm()
10191 false, 0, getCurSDLoc(), DAG, in visitInlineAsm()
10202 Chain = DAG.getNode(ISDOpc, getCurSDLoc(), in visitInlineAsm()
10203 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); in visitInlineAsm()
10223 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType); in visitInlineAsm()
10237 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V); in visitInlineAsm()
10243 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V); in visitInlineAsm()
10261 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), in visitInlineAsm()
10267 OpInfo, DAG); in visitInlineAsm()
10281 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr), in visitInlineAsm()
10303 SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), in visitInlineAsm()
10304 DAG.getVTList(ResultVTs), ResultValues); in visitInlineAsm()
10310 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains); in visitInlineAsm()
10319 DAG.setRoot(Chain); in visitInlineAsm()
10324 LLVMContext &Ctx = *DAG.getContext(); in emitInlineAsmError()
10328 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in emitInlineAsmError()
10330 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs); in emitInlineAsmError()
10337 Ops.push_back(DAG.getUNDEF(VT)); in emitInlineAsmError()
10339 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc())); in emitInlineAsmError()
10343 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), in visitVAStart()
10346 DAG.getSrcValue(I.getArgOperand(0)))); in visitVAStart()
10350 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitVAArg()
10351 const DataLayout &DL = DAG.getDataLayout(); in visitVAArg()
10352 SDValue V = DAG.getVAArg( in visitVAArg()
10353 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(), in visitVAArg()
10354 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)), in visitVAArg()
10356 DAG.setRoot(V.getValue(1)); in visitVAArg()
10359 V = DAG.getPtrExtOrTrunc( in visitVAArg()
10360 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType())); in visitVAArg()
10365 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), in visitVAEnd()
10368 DAG.getSrcValue(I.getArgOperand(0)))); in visitVAEnd()
10372 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), in visitVACopy()
10376 DAG.getSrcValue(I.getArgOperand(0)), in visitVACopy()
10377 DAG.getSrcValue(I.getArgOperand(1)))); in visitVACopy()
10380 SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG, in lowerRangeToAssertZExt() argument
10396 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits); in lowerRangeToAssertZExt()
10400 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op, in lowerRangeToAssertZExt()
10401 DAG.getValueType(SmallVT)); in lowerRangeToAssertZExt()
10412 return DAG.getMergeValues(Ops, SL); in lowerRangeToAssertZExt()
10473 SelectionDAG &DAG = Builder.DAG; in addStackMapLiveVars() local
10480 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType())); in addStackMapLiveVars()
10511 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL); in visitStackmap()
10525 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType()); in visitStackmap()
10531 DAG.getTargetConstant(Shad->getAsZExtVal(), DL, Shad.getValueType()); in visitStackmap()
10538 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in visitStackmap()
10539 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops); in visitStackmap()
10542 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL); in visitStackmap()
10547 DAG.setRoot(Chain); in visitStackmap()
10571 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl, in visitPatchpoint()
10574 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(), in visitPatchpoint()
10591 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType(); in visitPatchpoint()
10593 TargetLowering::CallLoweringInfo CLI(DAG); in visitPatchpoint()
10629 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64)); in visitPatchpoint()
10631 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32)); in visitPatchpoint()
10641 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32)); in visitPatchpoint()
10644 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32)); in visitPatchpoint()
10662 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitPatchpoint()
10664 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs); in visitPatchpoint()
10670 NodeTys = DAG.getVTList(ValueVTs); in visitPatchpoint()
10672 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in visitPatchpoint()
10675 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops); in visitPatchpoint()
10692 DAG.ReplaceAllUsesOfValuesWith(From, To, 2); in visitPatchpoint()
10694 DAG.ReplaceAllUsesWith(Call, PPV.getNode()); in visitPatchpoint()
10695 DAG.DeleteNode(Call); in visitPatchpoint()
10703 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitVectorReduce()
10709 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitVectorReduce()
10718 Res = DAG.getNode(ISD::FADD, dl, VT, Op1, in visitVectorReduce()
10719 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags), in visitVectorReduce()
10722 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags); in visitVectorReduce()
10726 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1, in visitVectorReduce()
10727 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags), in visitVectorReduce()
10730 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags); in visitVectorReduce()
10733 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1); in visitVectorReduce()
10736 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1); in visitVectorReduce()
10739 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1); in visitVectorReduce()
10742 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1); in visitVectorReduce()
10745 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1); in visitVectorReduce()
10748 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1); in visitVectorReduce()
10751 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1); in visitVectorReduce()
10754 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1); in visitVectorReduce()
10757 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1); in visitVectorReduce()
10760 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags); in visitVectorReduce()
10763 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags); in visitVectorReduce()
10766 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags); in visitVectorReduce()
10769 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags); in visitVectorReduce()
10803 auto &DL = CLI.DAG.getDataLayout(); in LowerCallTo()
10829 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(), in LowerCallTo()
10840 MachineFunction &MF = CLI.DAG.getMachineFunction(); in LowerCallTo()
10846 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL)); in LowerCallTo()
11060 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB, in LowerCallTo()
11107 CLI.DAG.setRoot(CLI.Chain); in LowerCallTo()
11140 MachineFunction &MF = CLI.DAG.getMachineFunction(); in LowerCallTo()
11143 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot, in LowerCallTo()
11144 CLI.DAG.getConstant(Offsets[i], CLI.DL, in LowerCallTo()
11146 SDValue L = CLI.DAG.getLoad( in LowerCallTo()
11148 MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(), in LowerCallTo()
11155 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains); in LowerCallTo()
11172 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr, in LowerCallTo()
11184 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL, in LowerCallTo()
11185 CLI.DAG.getVTList(RetTys), ReturnValues); in LowerCallTo()
11195 SelectionDAG &DAG) const { in LowerOperationWrapper()
11196 SDValue Res = LowerOperation(SDValue(N, 0), DAG); in LowerOperationWrapper()
11218 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { in LowerOperation()
11231 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in CopyValueToVirtualRegister()
11235 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(), in CopyValueToVirtualRegister()
11237 SDValue Chain = DAG.getEntryNode(); in CopyValueToVirtualRegister()
11244 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType); in CopyValueToVirtualRegister()
11440 SelectionDAG &DAG = SDB->DAG; in LowerArguments() local
11442 const DataLayout &DL = DAG.getDataLayout(); in LowerArguments()
11452 ComputeValueVTs(*TLI, DAG.getDataLayout(), in LowerArguments()
11454 DAG.getDataLayout().getAllocaAddrSpace()), in LowerArguments()
11461 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]); in LowerArguments()
11478 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs); in LowerArguments()
11489 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); in LowerArguments()
11621 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals); in LowerArguments()
11638 DAG.setRoot(NewRoot); in LowerArguments()
11646 ComputeValueVTs(*TLI, DAG.getDataLayout(), in LowerArguments()
11648 DAG.getDataLayout().getAllocaAddrSpace()), in LowerArguments()
11654 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot, in LowerArguments()
11657 MachineFunction& MF = SDB->DAG.getMachineFunction(); in LowerArguments()
11663 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue); in LowerArguments()
11664 DAG.setRoot(NewRoot); in LowerArguments()
11675 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs); in LowerArguments()
11726 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts, in LowerArguments()
11743 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues), in LowerArguments()
11755 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0; in LowerArguments()
11795 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in LowerArguments()
11798 DAG.setRoot(NewRoot); in LowerArguments()
11825 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in HandlePHINodesInSuccessorBlocks()
11887 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs); in HandlePHINodesInSuccessorBlocks()
11889 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT); in HandlePHINodesInSuccessorBlocks()
11916 DAG.setRoot(MaybeTC); in updateDAGForMaybeTailCall()
11957 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS, in lowerWorkItem()
11958 DAG.getConstant(CommonBit, DL, VT)); in lowerWorkItem()
11959 SDValue Cond = DAG.getSetCC( in lowerWorkItem()
11960 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT), in lowerWorkItem()
11977 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond, in lowerWorkItem()
11978 DAG.getBasicBlock(Small.MBB)); in lowerWorkItem()
11980 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond, in lowerWorkItem()
11981 DAG.getBasicBlock(DefaultMBB)); in lowerWorkItem()
11983 DAG.setRoot(BrCond); in lowerWorkItem()
12339 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, in visitSwitch()
12340 getControlRoot(), DAG.getBasicBlock(DefaultMBB))); in visitSwitch()
12345 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(), in visitSwitch()
12346 DAG.getBFI()); in visitSwitch()
12396 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitStepVector()
12398 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitStepVector()
12399 setValue(&I, DAG.getStepVector(DL, ResultVT)); in visitStepVector()
12403 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitVectorReverse()
12404 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitVectorReverse()
12411 setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V)); in visitVectorReverse()
12422 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask)); in visitVectorReverse()
12429 InVec.getValueType().getHalfNumVectorElementsVT(*DAG.getContext()); in visitVectorDeinterleave()
12434 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec, in visitVectorDeinterleave()
12435 DAG.getVectorIdxConstant(0, DL)); in visitVectorDeinterleave()
12436 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec, in visitVectorDeinterleave()
12437 DAG.getVectorIdxConstant(OutNumElts, DL)); in visitVectorDeinterleave()
12442 SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi, in visitVectorDeinterleave()
12444 SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi, in visitVectorDeinterleave()
12446 SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc()); in visitVectorDeinterleave()
12451 SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, in visitVectorDeinterleave()
12452 DAG.getVTList(OutVT, OutVT), Lo, Hi); in visitVectorDeinterleave()
12461 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitVectorInterleave()
12462 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitVectorInterleave()
12468 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1); in visitVectorInterleave()
12469 setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT), in visitVectorInterleave()
12474 SDValue Res = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, in visitVectorInterleave()
12475 DAG.getVTList(InVT, InVT), InVec0, InVec1); in visitVectorInterleave()
12476 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0), in visitVectorInterleave()
12483 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), in visitFreeze()
12492 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i], in visitFreeze()
12495 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), in visitFreeze()
12496 DAG.getVTList(ValueVTs), Values)); in visitFreeze()
12500 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitVectorSplice()
12501 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); in visitVectorSplice()
12510 setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2, in visitVectorSplice()
12511 DAG.getVectorIdxConstant(Imm, DL))); in visitVectorSplice()
12523 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask)); in visitVectorSplice()
12577 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in visitCallBrLandingPad()
12578 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo(); in visitCallBrLandingPad()
12579 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); in visitCallBrLandingPad()
12582 SDValue Chain = DAG.getRoot(); in visitCallBrLandingPad()
12586 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR); in visitCallBrLandingPad()
12594 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); in visitCallBrLandingPad()
12600 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo); in visitCallBrLandingPad()
12613 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR); in visitCallBrLandingPad()
12621 OpInfo, DAG); in visitCallBrLandingPad()
12631 SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), in visitCallBrLandingPad()
12632 DAG.getVTList(ResultVTs), ResultValues); in visitCallBrLandingPad()