Lines Matching refs:DAG
90 SelectionDAG &DAG; member in __anon2068f58e0111::SelectionDAGLegalize
100 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in getSetCCResultType()
106 SelectionDAGLegalize(SelectionDAG &DAG, in SelectionDAGLegalize() argument
109 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG), in SelectionDAGLegalize()
214 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); in ReplaceNode()
215 dbgs() << " with: "; New->dump(&DAG)); in ReplaceNode()
220 DAG.ReplaceAllUsesWith(Old, New); in ReplaceNode()
227 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); in ReplaceNode()
228 dbgs() << " with: "; New->dump(&DAG)); in ReplaceNode()
230 DAG.ReplaceAllUsesWith(Old, New); in ReplaceNode()
237 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG)); in ReplaceNode()
239 DAG.ReplaceAllUsesWith(Old, New); in ReplaceNode()
242 New[i]->dump(&DAG)); in ReplaceNode()
250 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); in ReplaceNodeWithValue()
251 dbgs() << " with: "; New->dump(&DAG)); in ReplaceNodeWithValue()
253 DAG.ReplaceAllUsesOfValueWith(Old, New); in ReplaceNodeWithValue()
291 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask); in ShuffleWithNarrowerEltType()
305 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask); in ShuffleWithNarrowerEltType()
325 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl, in ExpandConstantFP()
343 Type *SType = SVT.getTypeForEVT(*DAG.getContext()); in ExpandConstantFP()
345 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout())); in ExpandConstantFP()
353 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout())); in ExpandConstantFP()
356 SDValue Result = DAG.getExtLoad( in ExpandConstantFP()
357 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx, in ExpandConstantFP()
358 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT, in ExpandConstantFP()
362 SDValue Result = DAG.getLoad( in ExpandConstantFP()
363 OrigVT, dl, DAG.getEntryNode(), CPIdx, in ExpandConstantFP()
364 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); in ExpandConstantFP()
372 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(), in ExpandConstant()
373 TLI.getPointerTy(DAG.getDataLayout())); in ExpandConstant()
375 SDValue Result = DAG.getLoad( in ExpandConstant()
376 VT, dl, DAG.getEntryNode(), CPIdx, in ExpandConstant()
377 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); in ExpandConstant()
394 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in ExpandINSERT_VECTOR_ELT()
405 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps); in ExpandINSERT_VECTOR_ELT()
436 SDValue Con = DAG.getConstant(CFP->getValueAPF(). in OptimizeFloatStore()
439 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), in OptimizeFloatStore()
447 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). in OptimizeFloatStore()
449 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), in OptimizeFloatStore()
458 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32); in OptimizeFloatStore()
459 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32); in OptimizeFloatStore()
460 if (DAG.getDataLayout().isBigEndian()) in OptimizeFloatStore()
463 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), in OptimizeFloatStore()
465 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(4), dl); in OptimizeFloatStore()
466 Hi = DAG.getStore(Chain, dl, Hi, Ptr, in OptimizeFloatStore()
470 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); in OptimizeFloatStore()
501 const DataLayout &DL = DAG.getDataLayout(); in LegalizeStoreOps()
502 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, in LegalizeStoreOps()
505 SDValue Result = TLI.expandUnalignedStore(ST, DAG); in LegalizeStoreOps()
513 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LegalizeStoreOps()
522 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); in LegalizeStoreOps()
523 SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), in LegalizeStoreOps()
537 auto &DL = DAG.getDataLayout(); in LegalizeStoreOps()
543 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedValue()); in LegalizeStoreOps()
544 Value = DAG.getZeroExtendInReg(Value, dl, StVT); in LegalizeStoreOps()
546 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT, in LegalizeStoreOps()
561 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); in LegalizeStoreOps()
562 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); in LegalizeStoreOps()
569 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), in LegalizeStoreOps()
575 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl); in LegalizeStoreOps()
576 Hi = DAG.getNode( in LegalizeStoreOps()
578 DAG.getConstant(RoundWidth, dl, in LegalizeStoreOps()
580 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, in LegalizeStoreOps()
587 Hi = DAG.getNode( in LegalizeStoreOps()
589 DAG.getConstant(ExtraWidth, dl, in LegalizeStoreOps()
591 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT, in LegalizeStoreOps()
596 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in LegalizeStoreOps()
597 DAG.getConstant(IncrementSize, dl, in LegalizeStoreOps()
599 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, in LegalizeStoreOps()
605 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); in LegalizeStoreOps()
614 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, in LegalizeStoreOps()
616 SDValue Result = TLI.expandUnalignedStore(ST, DAG); in LegalizeStoreOps()
622 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LegalizeStoreOps()
635 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); in LegalizeStoreOps()
636 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), in LegalizeStoreOps()
641 Value = DAG.getNode(ISD::TRUNCATE, dl, in LegalizeStoreOps()
642 TLI.getTypeToTransformTo(*DAG.getContext(), StVT), in LegalizeStoreOps()
645 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT, in LegalizeStoreOps()
673 const DataLayout &DL = DAG.getDataLayout(); in LegalizeLoadOps()
676 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT, in LegalizeLoadOps()
678 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG); in LegalizeLoadOps()
683 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) { in LegalizeLoadOps()
694 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); in LegalizeLoadOps()
695 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); in LegalizeLoadOps()
702 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); in LegalizeLoadOps()
703 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); in LegalizeLoadOps()
733 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); in LegalizeLoadOps()
742 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), in LegalizeLoadOps()
750 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, in LegalizeLoadOps()
752 Result, DAG.getValueType(SrcVT)); in LegalizeLoadOps()
755 Result = DAG.getNode(ISD::AssertZext, dl, in LegalizeLoadOps()
757 DAG.getValueType(SrcVT)); in LegalizeLoadOps()
773 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); in LegalizeLoadOps()
774 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); in LegalizeLoadOps()
777 auto &DL = DAG.getDataLayout(); in LegalizeLoadOps()
782 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
789 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl); in LegalizeLoadOps()
790 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
796 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in LegalizeLoadOps()
800 Hi = DAG.getNode( in LegalizeLoadOps()
802 DAG.getConstant(RoundWidth, dl, in LegalizeLoadOps()
806 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
811 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
818 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl); in LegalizeLoadOps()
819 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
825 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in LegalizeLoadOps()
829 Hi = DAG.getNode( in LegalizeLoadOps()
831 DAG.getConstant(ExtraWidth, dl, in LegalizeLoadOps()
835 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
852 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { in LegalizeLoadOps()
860 const DataLayout &DL = DAG.getDataLayout(); in LegalizeLoadOps()
861 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, in LegalizeLoadOps()
863 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG); in LegalizeLoadOps()
882 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr, in LegalizeLoadOps()
886 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); in LegalizeLoadOps()
901 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain, in LegalizeLoadOps()
904 DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP, in LegalizeLoadOps()
922 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, in LegalizeLoadOps()
928 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, in LegalizeLoadOps()
930 Result, DAG.getValueType(SrcVT)); in LegalizeLoadOps()
932 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT); in LegalizeLoadOps()
944 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); in LegalizeLoadOps()
945 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); in LegalizeLoadOps()
956 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); in LegalizeOp()
965 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == in LegalizeOp()
970 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == in LegalizeOp()
1131 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1143 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1267 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1); in LegalizeOp()
1274 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); in LegalizeOp()
1289 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2); in LegalizeOp()
1294 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); in LegalizeOp()
1312 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { in LegalizeOp()
1360 Node->dump( &DAG); in LegalizeOp()
1401 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())) in ExpandExtractFromVectorThroughStack()
1423 StackPtr = DAG.CreateStackTemporary(VecVT); in ExpandExtractFromVectorThroughStack()
1425 StackPtr, DAG.getMachineFunction(), VecVT.isScalableVector()); in ExpandExtractFromVectorThroughStack()
1426 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO); in ExpandExtractFromVectorThroughStack()
1432 DAG.getDataLayout().getPrefTypeAlign( in ExpandExtractFromVectorThroughStack()
1433 Op.getValueType().getTypeForEVT(*DAG.getContext()))); in ExpandExtractFromVectorThroughStack()
1436 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, in ExpandExtractFromVectorThroughStack()
1438 NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, in ExpandExtractFromVectorThroughStack()
1441 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); in ExpandExtractFromVectorThroughStack()
1442 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, in ExpandExtractFromVectorThroughStack()
1448 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1)); in ExpandExtractFromVectorThroughStack()
1456 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0); in ExpandExtractFromVectorThroughStack()
1471 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); in ExpandInsertToVectorThroughStack()
1474 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); in ExpandInsertToVectorThroughStack()
1477 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); in ExpandInsertToVectorThroughStack()
1480 Idx = DAG.getFreeze(Idx); in ExpandInsertToVectorThroughStack()
1485 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, PartVT, Idx); in ExpandInsertToVectorThroughStack()
1488 Ch = DAG.getStore( in ExpandInsertToVectorThroughStack()
1490 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction())); in ExpandInsertToVectorThroughStack()
1493 TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); in ExpandInsertToVectorThroughStack()
1496 Ch = DAG.getTruncStore( in ExpandInsertToVectorThroughStack()
1498 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), in ExpandInsertToVectorThroughStack()
1503 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo); in ExpandInsertToVectorThroughStack()
1519 SDValue FIPtr = DAG.CreateStackTemporary(VT); in ExpandVectorBuildThroughStack()
1522 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); in ExpandVectorBuildThroughStack()
1542 DAG.getMemBasePlusOffset(FIPtr, TypeSize::getFixed(Offset), dl); in ExpandVectorBuildThroughStack()
1545 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, in ExpandVectorBuildThroughStack()
1549 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), in ExpandVectorBuildThroughStack()
1555 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in ExpandVectorBuildThroughStack()
1557 StoreChain = DAG.getEntryNode(); in ExpandVectorBuildThroughStack()
1560 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo); in ExpandVectorBuildThroughStack()
1572 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits); in getSignAsIntValue()
1575 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value); in getSignAsIntValue()
1581 auto &DataLayout = DAG.getDataLayout(); in getSignAsIntValue()
1585 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); in getSignAsIntValue()
1589 MachineFunction &MF = DAG.getMachineFunction(); in getSignAsIntValue()
1591 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr, in getSignAsIntValue()
1604 DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(ByteOffset), DL); in getSignAsIntValue()
1610 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, in getSignAsIntValue()
1622 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue); in modifySignAsInt()
1625 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr, in modifySignAsInt()
1627 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr, in modifySignAsInt()
1641 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); in ExpandFCOPYSIGN()
1642 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, in ExpandFCOPYSIGN()
1649 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag); in ExpandFCOPYSIGN()
1650 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue); in ExpandFCOPYSIGN()
1651 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit, in ExpandFCOPYSIGN()
1652 DAG.getConstant(0, DL, IntVT), ISD::SETNE); in ExpandFCOPYSIGN()
1653 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue); in ExpandFCOPYSIGN()
1660 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT); in ExpandFCOPYSIGN()
1661 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue, in ExpandFCOPYSIGN()
1669 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit); in ExpandFCOPYSIGN()
1673 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT); in ExpandFCOPYSIGN()
1674 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst); in ExpandFCOPYSIGN()
1676 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT); in ExpandFCOPYSIGN()
1677 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst); in ExpandFCOPYSIGN()
1681 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit); in ExpandFCOPYSIGN()
1689 DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit, Flags); in ExpandFCOPYSIGN()
1702 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); in ExpandFNEG()
1704 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask); in ExpandFNEG()
1717 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT); in ExpandFABS()
1718 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero); in ExpandFABS()
1725 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT); in ExpandFABS()
1726 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue, in ExpandFABS()
1745 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl); in ExpandDYNAMIC_STACKALLOC()
1748 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); in ExpandDYNAMIC_STACKALLOC()
1751 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering(); in ExpandDYNAMIC_STACKALLOC()
1757 Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value in ExpandDYNAMIC_STACKALLOC()
1759 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, in ExpandDYNAMIC_STACKALLOC()
1760 DAG.getConstant(-Alignment.value(), dl, VT)); in ExpandDYNAMIC_STACKALLOC()
1761 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain in ExpandDYNAMIC_STACKALLOC()
1763 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl); in ExpandDYNAMIC_STACKALLOC()
1775 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode()); in EmitStackConvert()
1782 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); in EmitStackConvert()
1783 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType); in EmitStackConvert()
1793 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign( in EmitStackConvert()
1794 SrcOp.getValueType().getTypeForEVT(*DAG.getContext())); in EmitStackConvert()
1795 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign); in EmitStackConvert()
1800 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); in EmitStackConvert()
1807 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo, in EmitStackConvert()
1811 Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign); in EmitStackConvert()
1816 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign); in EmitStackConvert()
1819 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, in EmitStackConvert()
1827 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); in ExpandSCALAR_TO_VECTOR()
1832 SDValue Ch = DAG.getTruncStore( in ExpandSCALAR_TO_VECTOR()
1833 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, in ExpandSCALAR_TO_VECTOR()
1834 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI), in ExpandSCALAR_TO_VECTOR()
1836 return DAG.getLoad( in ExpandSCALAR_TO_VECTOR()
1838 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); in ExpandSCALAR_TO_VECTOR()
1842 ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, in ExpandBVWithShuffles() argument
1865 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); in ExpandBVWithShuffles()
1894 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first, in ExpandBVWithShuffles()
1918 Vec2 = DAG.getUNDEF(VT); in ExpandBVWithShuffles()
1927 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); in ExpandBVWithShuffles()
1970 return DAG.getUNDEF(VT); in ExpandBUILD_VECTOR()
1973 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); in ExpandBUILD_VECTOR()
1991 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), in ExpandBUILD_VECTOR()
1996 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); in ExpandBUILD_VECTOR()
2002 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout())); in ExpandBUILD_VECTOR()
2004 return DAG.getLoad( in ExpandBUILD_VECTOR()
2005 VT, dl, DAG.getEntryNode(), CPIdx, in ExpandBUILD_VECTOR()
2006 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), in ExpandBUILD_VECTOR()
2028 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); in ExpandBUILD_VECTOR()
2031 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); in ExpandBUILD_VECTOR()
2033 Vec2 = DAG.getUNDEF(VT); in ExpandBUILD_VECTOR()
2036 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); in ExpandBUILD_VECTOR()
2040 if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) in ExpandBUILD_VECTOR()
2054 return DAG.getSplatBuildVector(VT, DL, SplatVal); in ExpandSPLAT_VECTOR()
2065 EVT CodePtrTy = TLI.getPointerTy(DAG.getDataLayout()); in ExpandLibCall()
2068 Callee = DAG.getExternalSymbol(LibcallName, CodePtrTy); in ExpandLibCall()
2070 Callee = DAG.getUNDEF(CodePtrTy); in ExpandLibCall()
2071 DAG.getContext()->emitError(Twine("no libcall available for ") + in ExpandLibCall()
2072 Node->getOperationName(&DAG)); in ExpandLibCall()
2076 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); in ExpandLibCall()
2082 SDValue InChain = DAG.getEntryNode(); in ExpandLibCall()
2087 const Function &F = DAG.getMachineFunction().getFunction(); in ExpandLibCall()
2089 TLI.isInTailCallPosition(DAG, Node, TCChain) && in ExpandLibCall()
2094 TargetLowering::CallLoweringInfo CLI(DAG); in ExpandLibCall()
2108 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG)); in ExpandLibCall()
2110 return {DAG.getRoot(), DAG.getRoot()}; in ExpandLibCall()
2113 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG)); in ExpandLibCall()
2123 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); in ExpandLibCall()
2143 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); in ExpandFrexpLibCall()
2149 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT); in ExpandFrexpLibCall()
2152 PtrArgEntry.Ty = PointerType::get(*DAG.getContext(), in ExpandFrexpLibCall()
2153 DAG.getDataLayout().getAllocaAddrSpace()); in ExpandFrexpLibCall()
2164 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); in ExpandFrexpLibCall()
2166 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo); in ExpandFrexpLibCall()
2167 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in ExpandFrexpLibCall()
2168 LoadExp.getValue(1), DAG.getRoot()); in ExpandFrexpLibCall()
2169 DAG.setRoot(OutputChain); in ExpandFrexpLibCall()
2186 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, in ExpandFPLibCall()
2267 SDValue InChain = DAG.getEntryNode(); in ExpandDivRemLibCall()
2270 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); in ExpandDivRemLibCall()
2276 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); in ExpandDivRemLibCall()
2285 SDValue FIPtr = DAG.CreateStackTemporary(RetVT); in ExpandDivRemLibCall()
2292 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), in ExpandDivRemLibCall()
2293 TLI.getPointerTy(DAG.getDataLayout())); in ExpandDivRemLibCall()
2296 TargetLowering::CallLoweringInfo CLI(DAG); in ExpandDivRemLibCall()
2308 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo()); in ExpandDivRemLibCall()
2360 SDValue InChain = DAG.getEntryNode(); in ExpandSinCosLibCall()
2363 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); in ExpandSinCosLibCall()
2376 SDValue SinPtr = DAG.CreateStackTemporary(RetVT); in ExpandSinCosLibCall()
2384 SDValue CosPtr = DAG.CreateStackTemporary(RetVT); in ExpandSinCosLibCall()
2391 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), in ExpandSinCosLibCall()
2392 TLI.getPointerTy(DAG.getDataLayout())); in ExpandSinCosLibCall()
2395 TargetLowering::CallLoweringInfo CLI(DAG); in ExpandSinCosLibCall()
2397 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee, in ExpandSinCosLibCall()
2403 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo())); in ExpandSinCosLibCall()
2405 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo())); in ExpandSinCosLibCall()
2428 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT); in expandLdexp()
2435 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT); in expandLdexp()
2436 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT); in expandLdexp()
2438 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT); in expandLdexp()
2451 SDValue NGtMaxExp = DAG.getSetCC(dl, SetCCVT, N, MaxExp, ISD::SETGT); in expandLdexp()
2453 SDValue DecN0 = DAG.getNode(ISD::SUB, dl, ExpVT, N, MaxExp, NSW); in expandLdexp()
2454 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT); in expandLdexp()
2455 SDValue ClampN_Big = DAG.getNode(ISD::SMIN, dl, ExpVT, N, ClampMaxVal); in expandLdexp()
2457 DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW); in expandLdexp()
2460 DAG.getSetCC(dl, SetCCVT, N, DoubleMaxExp, ISD::SETUGT); in expandLdexp()
2462 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT); in expandLdexp()
2463 SDValue ScaleUp0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleUpVal); in expandLdexp()
2464 SDValue ScaleUp1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleUp0, ScaleUpVal); in expandLdexp()
2467 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleUpTwice, DecN1, DecN0); in expandLdexp()
2469 DAG.getNode(ISD::SELECT, dl, VT, ScaleUpTwice, ScaleUp1, ScaleUp0); in expandLdexp()
2472 SDValue NLtMinExp = DAG.getSetCC(dl, SetCCVT, N, MinExp, ISD::SETLT); in expandLdexp()
2474 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT); in expandLdexp()
2475 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT); in expandLdexp()
2477 SDValue IncN0 = DAG.getNode(ISD::ADD, dl, ExpVT, N, Increment0, NUW_NSW); in expandLdexp()
2480 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT); in expandLdexp()
2481 SDValue ClampN_Small = DAG.getNode(ISD::SMAX, dl, ExpVT, N, ClampMinVal); in expandLdexp()
2483 DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW); in expandLdexp()
2485 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT); in expandLdexp()
2486 SDValue ScaleDown0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleDownVal); in expandLdexp()
2487 SDValue ScaleDown1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleDown0, ScaleDownVal); in expandLdexp()
2489 SDValue ScaleDownTwice = DAG.getSetCC( in expandLdexp()
2490 dl, SetCCVT, N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT), in expandLdexp()
2494 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleDownTwice, IncN1, IncN0); in expandLdexp()
2496 DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0); in expandLdexp()
2500 SDValue NewX = DAG.getNode( in expandLdexp()
2502 DAG.getNode(ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small, X)); in expandLdexp()
2504 SDValue NewN = DAG.getNode( in expandLdexp()
2506 DAG.getNode(ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small, N)); in expandLdexp()
2508 SDValue BiasedN = DAG.getNode(ISD::ADD, dl, ExpVT, NewN, MaxExp, NSW); in expandLdexp()
2511 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl); in expandLdexp()
2512 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT); in expandLdexp()
2514 SDValue AsInt = DAG.getNode(ISD::SHL, dl, AsIntVT, CastExpToValTy, in expandLdexp()
2516 SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt); in expandLdexp()
2517 return DAG.getNode(ISD::FMUL, dl, VT, NewX, AsFP); in expandLdexp()
2548 SDValue NegSmallestNormalizedInt = DAG.getConstant( in expandFrexp()
2552 SDValue SmallestNormalizedInt = DAG.getConstant( in expandFrexp()
2558 DAG.getConstant(APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT); in expandFrexp()
2567 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT); in expandFrexp()
2569 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT); in expandFrexp()
2577 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT); in expandFrexp()
2578 SDValue ScaleUp = DAG.getNode(ISD::FMUL, dl, VT, Val, ScaleUpK); in expandFrexp()
2581 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandFrexp()
2583 SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val); in expandFrexp()
2585 SDValue Abs = DAG.getNode(ISD::AND, dl, AsIntVT, AsInt, SignMask); in expandFrexp()
2588 DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt); in expandFrexp()
2589 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal, in expandFrexp()
2593 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT); in expandFrexp()
2595 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT); in expandFrexp()
2596 SDValue Zero = DAG.getConstant(0, dl, ExpVT); in expandFrexp()
2598 SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp); in expandFrexp()
2600 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt); in expandFrexp()
2603 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask); in expandFrexp()
2606 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs); in expandFrexp()
2610 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl); in expandFrexp()
2612 DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt); in expandFrexp()
2613 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT); in expandFrexp()
2615 SDValue NormalBiasedExp = DAG.getNode(ISD::ADD, dl, ExpVT, Exp, MinExp); in expandFrexp()
2616 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT); in expandFrexp()
2618 DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero); in expandFrexp()
2621 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask); in expandFrexp()
2623 SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT); in expandFrexp()
2624 SDValue Or = DAG.getNode(ISD::OR, dl, AsIntVT, MaskedFractAsInt, FPHalf); in expandFrexp()
2625 SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or); in expandFrexp()
2628 DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias); in expandFrexp()
2631 DAG.getNode(ISD::SELECT, dl, VT, DenormOrZero, Val, MaskedFract); in expandFrexp()
2634 DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp); in expandFrexp()
2636 return DAG.getMergeValues({Result0, Result1}, dl); in expandFrexp()
2664 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); in ExpandLegalINT_TO_FP()
2670 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo, in ExpandLegalINT_TO_FP()
2671 DAG.getConstant(0x80000000u, dl, MVT::i32)); in ExpandLegalINT_TO_FP()
2674 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32); in ExpandLegalINT_TO_FP()
2677 if (DAG.getDataLayout().isBigEndian()) in ExpandLegalINT_TO_FP()
2680 SDValue MemChain = DAG.getEntryNode(); in ExpandLegalINT_TO_FP()
2683 SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot, in ExpandLegalINT_TO_FP()
2687 DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl); in ExpandLegalINT_TO_FP()
2689 DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo()); in ExpandLegalINT_TO_FP()
2690 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); in ExpandLegalINT_TO_FP()
2694 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo()); in ExpandLegalINT_TO_FP()
2696 SDValue Bias = DAG.getConstantFP( in ExpandLegalINT_TO_FP()
2704 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other}, in ExpandLegalINT_TO_FP()
2710 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT); in ExpandLegalINT_TO_FP()
2717 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); in ExpandLegalINT_TO_FP()
2718 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT); in ExpandLegalINT_TO_FP()
2746 SDValue SignBitTest = DAG.getSetCC( in ExpandLegalINT_TO_FP()
2747 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT); in ExpandLegalINT_TO_FP()
2749 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout()); in ExpandLegalINT_TO_FP()
2750 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT); in ExpandLegalINT_TO_FP()
2751 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst); in ExpandLegalINT_TO_FP()
2752 SDValue AndConst = DAG.getConstant(1, dl, SrcVT); in ExpandLegalINT_TO_FP()
2753 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst); in ExpandLegalINT_TO_FP()
2754 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr); in ExpandLegalINT_TO_FP()
2760 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0); in ExpandLegalINT_TO_FP()
2761 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2763 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2775 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or); in ExpandLegalINT_TO_FP()
2776 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt); in ExpandLegalINT_TO_FP()
2777 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); in ExpandLegalINT_TO_FP()
2780 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast); in ExpandLegalINT_TO_FP()
2791 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >= in ExpandLegalINT_TO_FP()
2797 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2800 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); in ExpandLegalINT_TO_FP()
2802 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0, in ExpandLegalINT_TO_FP()
2803 DAG.getConstant(0, dl, SrcVT), ISD::SETLT); in ExpandLegalINT_TO_FP()
2804 SDValue Zero = DAG.getIntPtrConstant(0, dl), in ExpandLegalINT_TO_FP()
2805 Four = DAG.getIntPtrConstant(4, dl); in ExpandLegalINT_TO_FP()
2806 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(), in ExpandLegalINT_TO_FP()
2821 if (DAG.getDataLayout().isLittleEndian()) in ExpandLegalINT_TO_FP()
2824 Type::getInt64Ty(*DAG.getContext()), FF); in ExpandLegalINT_TO_FP()
2827 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout())); in ExpandLegalINT_TO_FP()
2829 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); in ExpandLegalINT_TO_FP()
2833 FudgeInReg = DAG.getLoad( in ExpandLegalINT_TO_FP()
2834 MVT::f32, dl, DAG.getEntryNode(), CPIdx, in ExpandLegalINT_TO_FP()
2835 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), in ExpandLegalINT_TO_FP()
2838 SDValue Load = DAG.getExtLoad( in ExpandLegalINT_TO_FP()
2839 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, in ExpandLegalINT_TO_FP()
2840 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, in ExpandLegalINT_TO_FP()
2848 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2854 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); in ExpandLegalINT_TO_FP()
2903 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other}, in PromoteLegalINT_TO_FP()
2905 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, in PromoteLegalINT_TO_FP()
2913 DAG.getNode(OpToUse, dl, DestVT, in PromoteLegalINT_TO_FP()
2914 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, in PromoteLegalINT_TO_FP()
2957 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other); in PromoteLegalFP_TO_INT()
2958 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp); in PromoteLegalFP_TO_INT()
2960 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); in PromoteLegalFP_TO_INT()
2964 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); in PromoteLegalFP_TO_INT()
2989 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0), in PromoteLegalFP_TO_INT_SAT()
2991 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); in PromoteLegalFP_TO_INT_SAT()
2997 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); in ExpandPARITY()
3003 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op); in ExpandPARITY()
3007 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result, in ExpandPARITY()
3008 DAG.getConstant(1ULL << (--i), dl, ShVT)); in ExpandPARITY()
3009 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift); in ExpandPARITY()
3013 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT)); in ExpandPARITY()
3030 DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(0)); in PromoteReduction()
3041 DAG.getNode(ISD::FP_EXTEND, DL, NewVecVT, Node->getOperand(j)); in PromoteReduction()
3046 SDValue Res = DAG.getNode(Node->getOpcode(), DL, NewScalarVT, Operands, in PromoteReduction()
3050 return DAG.getNode(ISD::FP_ROUND, DL, ScalarVT, Res, in PromoteReduction()
3051 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)); in PromoteReduction()
3062 if ((Tmp1 = TLI.expandABS(Node, DAG))) in ExpandNode()
3067 if ((Tmp1 = TLI.expandABD(Node, DAG))) in ExpandNode()
3074 if ((Tmp1 = TLI.expandAVG(Node, DAG))) in ExpandNode()
3078 if ((Tmp1 = TLI.expandCTPOP(Node, DAG))) in ExpandNode()
3083 if ((Tmp1 = TLI.expandCTLZ(Node, DAG))) in ExpandNode()
3088 if ((Tmp1 = TLI.expandCTTZ(Node, DAG))) in ExpandNode()
3092 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG))) in ExpandNode()
3096 if ((Tmp1 = TLI.expandBSWAP(Node, DAG))) in ExpandNode()
3105 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3108 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, in ExpandNode()
3109 TLI.getPointerTy(DAG.getDataLayout())); in ExpandNode()
3110 SDValue Offset = DAG.getNode(ISD::ADD, dl, in ExpandNode()
3112 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, in ExpandNode()
3115 SDValue FA = DAG.getNode( in ExpandNode()
3116 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()), in ExpandNode()
3117 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()))); in ExpandNode()
3118 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(), in ExpandNode()
3123 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); in ExpandNode()
3140 DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3146 Results.push_back(DAG.getConstant(0, dl, MVT::i32)); in ExpandNode()
3151 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); in ExpandNode()
3152 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
3153 SDValue Swap = DAG.getAtomicCmpSwap( in ExpandNode()
3163 SDValue Swap = DAG.getAtomic( in ExpandNode()
3174 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
3175 SDValue Res = DAG.getAtomicCmpSwap( in ExpandNode()
3188 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res, in ExpandNode()
3189 DAG.getValueType(AtomicType)); in ExpandNode()
3190 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType, in ExpandNode()
3191 Node->getOperand(2), DAG.getValueType(AtomicType)); in ExpandNode()
3195 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res, in ExpandNode()
3196 DAG.getValueType(AtomicType)); in ExpandNode()
3197 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
3201 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType); in ExpandNode()
3202 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
3209 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); in ExpandNode()
3225 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS); in ExpandNode()
3226 SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(), in ExpandNode()
3243 Results.push_back(DAG.getConstant(0, dl, VT)); in ExpandNode()
3246 Results.push_back(DAG.getConstantFP(0, dl, VT)); in ExpandNode()
3272 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) { in ExpandNode()
3310 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op)); in ExpandNode()
3325 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, in ExpandNode()
3326 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op)); in ExpandNode()
3328 Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32); in ExpandNode()
3330 Op = DAG.getNode( in ExpandNode()
3332 DAG.getConstant(16, dl, in ExpandNode()
3333 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout()))); in ExpandNode()
3334 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op); in ExpandNode()
3337 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op); in ExpandNode()
3344 Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, in ExpandNode()
3345 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)); in ExpandNode()
3347 if (!DAG.isKnownNeverSNaN(Op)) { in ExpandNode()
3348 Op = DAG.getNode(ISD::FCANONICALIZE, dl, MVT::f32, Op, Node->getFlags()); in ExpandNode()
3350 Op = DAG.getNode( in ExpandNode()
3351 ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op), in ExpandNode()
3352 DAG.getConstant(16, dl, in ExpandNode()
3353 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout()))); in ExpandNode()
3357 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16, in ExpandNode()
3358 DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op)); in ExpandNode()
3360 Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0)); in ExpandNode()
3377 SDValue One = DAG.getConstant(1, dl, VT); in ExpandNode()
3378 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); in ExpandNode()
3379 SDValue Zero = DAG.getConstant(0, dl, VT); in ExpandNode()
3380 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); in ExpandNode()
3387 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); in ExpandNode()
3390 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy); in ExpandNode()
3391 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), in ExpandNode()
3393 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); in ExpandNode()
3399 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) { in ExpandNode()
3415 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) in ExpandNode()
3419 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) { in ExpandNode()
3426 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) in ExpandNode()
3430 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) { in ExpandNode()
3432 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2); in ExpandNode()
3441 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG)); in ExpandNode()
3444 Results.push_back(DAG.expandVAArg(Node)); in ExpandNode()
3448 Results.push_back(DAG.expandVACopy(Node)); in ExpandNode()
3453 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), in ExpandNode()
3483 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); in ExpandNode()
3495 EVT::getVectorVT(*DAG.getContext(), NewEltVT, in ExpandNode()
3500 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); in ExpandNode()
3501 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); in ExpandNode()
3529 Ops.push_back(DAG.getUNDEF(EltVT)); in ExpandNode()
3534 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, in ExpandNode()
3535 DAG.getVectorIdxConstant(Idx, dl))); in ExpandNode()
3538 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1, in ExpandNode()
3539 DAG.getVectorIdxConstant(Idx - NumElems, dl))); in ExpandNode()
3542 Tmp1 = DAG.getBuildVector(VT, dl, Ops); in ExpandNode()
3544 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3549 Results.push_back(TLI.expandVectorSplice(Node, DAG)); in ExpandNode()
3556 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), in ExpandNode()
3557 DAG.getConstant(OpTy.getSizeInBits() / 2, dl, in ExpandNode()
3560 DAG.getDataLayout()))); in ExpandNode()
3561 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3564 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), in ExpandNode()
3574 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, in ExpandNode()
3578 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ExpandNode()
3586 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, in ExpandNode()
3593 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3609 Test, Node->getFlags(), SDLoc(Node), DAG)) in ExpandNode()
3628 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred); in ExpandNode()
3634 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) in ExpandNode()
3640 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(Node, DAG)) in ExpandNode()
3652 SDVTList VTs = DAG.getVTList(VT, VT); in ExpandNode()
3653 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); in ExpandNode()
3699 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); in ExpandNode()
3701 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); in ExpandNode()
3710 SDValue Res = DAG.getNode(Node->getOpcode(), dl, {MVT::f32, MVT::Other}, in ExpandNode()
3712 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, in ExpandNode()
3729 DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, in ExpandNode()
3730 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)); in ExpandNode()
3732 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); in ExpandNode()
3741 DAG.shouldOptForSize())) in ExpandNode()
3755 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); in ExpandNode()
3756 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); in ExpandNode()
3766 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT); in ExpandNode()
3767 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT)); in ExpandNode()
3768 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); in ExpandNode()
3773 if (TLI.expandREM(Node, Tmp1, DAG)) in ExpandNode()
3782 SDVTList VTs = DAG.getVTList(VT, VT); in ExpandNode()
3783 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), in ExpandNode()
3794 SDVTList VTs = DAG.getVTList(VT, VT); in ExpandNode()
3796 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), in ExpandNode()
3810 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); in ExpandNode()
3811 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS)); in ExpandNode()
3816 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext()); in ExpandNode()
3819 HalfType, DAG, in ExpandNode()
3822 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); in ExpandNode()
3823 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); in ExpandNode()
3824 SDValue Shift = DAG.getConstant( in ExpandNode()
3826 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); in ExpandNode()
3827 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in ExpandNode()
3828 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); in ExpandNode()
3836 SDVTList VTs = DAG.getVTList(VT, VT); in ExpandNode()
3857 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), in ExpandNode()
3863 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext()); in ExpandNode()
3868 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, in ExpandNode()
3870 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in ExpandNode()
3871 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); in ExpandNode()
3873 DAG.getConstant(HalfType.getSizeInBits(), dl, in ExpandNode()
3874 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); in ExpandNode()
3875 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in ExpandNode()
3876 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); in ExpandNode()
3882 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) in ExpandNode()
3887 if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG)) in ExpandNode()
3894 Results.push_back(TLI.expandAddSubSat(Node, DAG)); in ExpandNode()
3898 Results.push_back(TLI.expandCMP(Node, DAG)); in ExpandNode()
3902 Results.push_back(TLI.expandShlSat(Node, DAG)); in ExpandNode()
3908 Results.push_back(TLI.expandFixedPointMul(Node, DAG)); in ExpandNode()
3918 DAG)) { in ExpandNode()
3941 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS); in ExpandNode()
3947 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); in ExpandNode()
3950 SDValue One = DAG.getConstant(1, dl, VT); in ExpandNode()
3952 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One); in ExpandNode()
3953 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt); in ExpandNode()
3959 SDValue Zero = DAG.getConstant(0, dl, VT); in ExpandNode()
3961 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ) in ExpandNode()
3962 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ); in ExpandNode()
3963 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2, in ExpandNode()
3964 DAG.getZExtOrTrunc(Carry, dl, SetCCType)); in ExpandNode()
3967 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2); in ExpandNode()
3970 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT)); in ExpandNode()
3976 TLI.expandSADDSUBO(Node, Result, Overflow, DAG); in ExpandNode()
3984 TLI.expandUADDSUBO(Node, Result, Overflow, DAG); in ExpandNode()
3992 if (TLI.expandMULO(Node, Result, Overflow, DAG)) { in ExpandNode()
4000 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); in ExpandNode()
4001 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); in ExpandNode()
4002 Tmp2 = DAG.getNode( in ExpandNode()
4004 DAG.getConstant(PairTy.getSizeInBits() / 2, dl, in ExpandNode()
4005 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout()))); in ExpandNode()
4006 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); in ExpandNode()
4014 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), in ExpandNode()
4018 Tmp1 = DAG.getSelectCC(dl, Tmp1, in ExpandNode()
4019 DAG.getConstant(0, dl, Tmp1.getValueType()), in ExpandNode()
4031 const DataLayout &TD = DAG.getDataLayout(); in ExpandNode()
4035 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); in ExpandNode()
4042 Index = DAG.getNode( in ExpandNode()
4044 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType())); in ExpandNode()
4046 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index, in ExpandNode()
4047 DAG.getConstant(EntrySize, dl, Index.getValueType())); in ExpandNode()
4048 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), in ExpandNode()
4051 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); in ExpandNode()
4052 SDValue LD = DAG.getExtLoad( in ExpandNode()
4054 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT); in ExpandNode()
4060 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, in ExpandNode()
4061 TLI.getPICJumpTableRelocBase(Table, DAG)); in ExpandNode()
4064 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, JTI, DAG); in ExpandNode()
4076 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2), in ExpandNode()
4085 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, in ExpandNode()
4086 DAG.getConstant(1, dl, Tmp2.getValueType())); in ExpandNode()
4087 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, in ExpandNode()
4088 DAG.getCondCode(ISD::SETNE), Tmp3, in ExpandNode()
4089 DAG.getConstant(0, dl, Tmp3.getValueType()), in ExpandNode()
4113 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl, in ExpandNode()
4121 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(), in ExpandNode()
4125 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), in ExpandNode()
4128 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1, in ExpandNode()
4137 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0)); in ExpandNode()
4140 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0)); in ExpandNode()
4159 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, in ExpandNode()
4160 DAG.getBoolConstant(true, dl, VT, Tmp1VT), in ExpandNode()
4161 DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3); in ExpandNode()
4185 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); in ExpandNode()
4187 DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4, Node->getFlags())); in ExpandNode()
4200 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); in ExpandNode()
4210 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); in ExpandNode()
4217 DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, in ExpandNode()
4230 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), in ExpandNode()
4233 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType()); in ExpandNode()
4234 CC = DAG.getCondCode(ISD::SETNE); in ExpandNode()
4235 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4252 DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, in ExpandNode()
4262 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4265 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType()); in ExpandNode()
4266 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE); in ExpandNode()
4267 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, in ExpandNode()
4291 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), in ExpandNode()
4292 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
4294 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), in ExpandNode()
4295 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
4296 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, in ExpandNode()
4300 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); in ExpandNode()
4319 Results.push_back(TLI.expandVecReduce(Node, DAG)); in ExpandNode()
4323 Results.push_back(TLI.expandVPCTTZElements(Node, DAG)); in ExpandNode()
4368 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); in ExpandNode()
4369 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), in ExpandNode()
4416 TargetLowering::CallLoweringInfo CLI(DAG); in ConvertNodeToLibcall()
4420 CallingConv::C, Type::getVoidTy(*DAG.getContext()), in ConvertNodeToLibcall()
4421 DAG.getExternalSymbol("__sync_synchronize", in ConvertNodeToLibcall()
4422 TLI.getPointerTy(DAG.getDataLayout())), in ConvertNodeToLibcall()
4464 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, in ConvertNodeToLibcall()
4475 TargetLowering::CallLoweringInfo CLI(DAG); in ConvertNodeToLibcall()
4478 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), in ConvertNodeToLibcall()
4479 DAG.getExternalSymbol( in ConvertNodeToLibcall()
4480 "abort", TLI.getPointerTy(DAG.getDataLayout())), in ConvertNodeToLibcall()
4493 DAG, RTLIB::CLEAR_CACHE, MVT::isVoid, {StartVal, EndVal}, CallOptions, in ConvertNodeToLibcall()
4669 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node), in ConvertNodeToLibcall()
4673 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4680 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0), in ConvertNodeToLibcall()
4682 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4690 DAG.getLibInfo().getIntSize() == in ConvertNodeToLibcall()
4695 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)"); in ConvertNodeToLibcall()
4696 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ConvertNodeToLibcall()
4774 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32, Node->getOperand(1), in ConvertNodeToLibcall()
4784 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions, in ConvertNodeToLibcall()
4835 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, in ConvertNodeToLibcall()
4840 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain); in ConvertNodeToLibcall()
4879 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain); in ConvertNodeToLibcall()
4882 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first)); in ConvertNodeToLibcall()
4907 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain); in ConvertNodeToLibcall()
4936 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1), in ConvertNodeToLibcall()
5001 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl); in ConvertNodeToLibcall()
5004 DAG.makeStateFunctionCall(RTLIB::FESETENV, Ptr, Chain, dl)); in ConvertNodeToLibcall()
5011 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl)); in ConvertNodeToLibcall()
5018 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl)); in ConvertNodeToLibcall()
5025 SDValue StackPtr = DAG.CreateStackTemporary(ModeVT); in ConvertNodeToLibcall()
5027 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr, in ConvertNodeToLibcall()
5029 SDValue LdInst = DAG.getLoad( in ConvertNodeToLibcall()
5031 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); in ConvertNodeToLibcall()
5041 SDValue StackPtr = DAG.CreateStackTemporary(ModeVT); in ConvertNodeToLibcall()
5043 SDValue StInst = DAG.getStore( in ConvertNodeToLibcall()
5045 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); in ConvertNodeToLibcall()
5047 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl)); in ConvertNodeToLibcall()
5054 const DataLayout &DL = DAG.getDataLayout(); in ConvertNodeToLibcall()
5056 SDValue Mode = DAG.getConstant(-1LL, dl, PtrTy); in ConvertNodeToLibcall()
5057 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode, in ConvertNodeToLibcall()
5121 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5123 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5132 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1, in PromoteNode()
5133 DAG.getConstant(TopBit, dl, NVT)); in PromoteNode()
5138 Tmp1 = DAG.getNode(NewOpc, dl, NVT, Tmp1); in PromoteNode()
5141 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, in PromoteNode()
5142 DAG.getConstant(NVT.getSizeInBits() - in PromoteNode()
5145 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
5154 DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5157 auto ShiftConstant = DAG.getShiftAmountConstant( in PromoteNode()
5160 DAG.getNode(ISD::SHL, dl, NVT, AnyExtendedNode, ShiftConstant); in PromoteNode()
5163 auto CTLZResult = DAG.getNode(Node->getOpcode(), dl, NVT, LeftShiftResult); in PromoteNode()
5164 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, CTLZResult)); in PromoteNode()
5170 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5171 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5172 Tmp1 = DAG.getNode( in PromoteNode()
5174 DAG.getConstant(DiffBits, dl, in PromoteNode()
5175 TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); in PromoteNode()
5177 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
5210 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), in PromoteNode()
5214 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); in PromoteNode()
5218 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); in PromoteNode()
5219 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); in PromoteNode()
5271 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5272 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5274 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
5275 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); in PromoteNode()
5283 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5284 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5285 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); in PromoteNode()
5287 auto &DL = DAG.getDataLayout(); in PromoteNode()
5289 Tmp2 = DAG.getNode( in PromoteNode()
5291 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT))); in PromoteNode()
5292 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
5293 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); in PromoteNode()
5311 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5312 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5314 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); in PromoteNode()
5317 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); in PromoteNode()
5319 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, in PromoteNode()
5320 DAG.getIntPtrConstant(0, dl)); in PromoteNode()
5328 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); in PromoteNode()
5329 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); in PromoteNode()
5333 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); in PromoteNode()
5338 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5339 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5340 Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2, in PromoteNode()
5342 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3)); in PromoteNode()
5362 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5363 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5366 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5367 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
5369 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond}, in PromoteNode()
5374 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1); in PromoteNode()
5376 Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1, in PromoteNode()
5377 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)); in PromoteNode()
5397 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT); in PromoteNode()
5399 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT); in PromoteNode()
5401 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains); in PromoteNode()
5402 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in PromoteNode()
5403 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs, in PromoteNode()
5409 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5410 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5411 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, in PromoteNode()
5422 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5423 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
5424 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), in PromoteNode()
5439 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5440 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5441 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, in PromoteNode()
5444 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3, in PromoteNode()
5445 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true))); in PromoteNode()
5455 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5457 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5459 Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), in PromoteNode()
5461 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5463 Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5464 {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)}); in PromoteNode()
5469 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5470 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5471 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); in PromoteNode()
5473 DAG.getNode(ISD::FP_ROUND, dl, OVT, in PromoteNode()
5474 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), in PromoteNode()
5475 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true))); in PromoteNode()
5478 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5480 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5482 Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5484 Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), in PromoteNode()
5486 Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5488 Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5489 {Tmp4.getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)}); in PromoteNode()
5496 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5498 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
5507 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3, in PromoteNode()
5508 DAG.getIntPtrConstant(isTrunc, dl, /*isTarget=*/true))); in PromoteNode()
5512 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5514 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5516 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5517 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)}); in PromoteNode()
5522 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5523 Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1); in PromoteNode()
5526 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2, in PromoteNode()
5527 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true))); in PromoteNode()
5558 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5559 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5561 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2, in PromoteNode()
5562 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true))); in PromoteNode()
5586 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5588 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5590 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5591 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)}); in PromoteNode()
5613 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op)); in PromoteNode()
5617 DAG.getNode(MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS, in PromoteNode()
5619 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); in PromoteNode()
5648 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT); in PromoteNode()
5649 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); in PromoteNode()
5651 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5655 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); in PromoteNode()
5656 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); in PromoteNode()
5658 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, in PromoteNode()
5663 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps); in PromoteNode()
5664 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec)); in PromoteNode()
5696 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT); in PromoteNode()
5697 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); in PromoteNode()
5699 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5700 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); in PromoteNode()
5704 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); in PromoteNode()
5705 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); in PromoteNode()
5707 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, in PromoteNode()
5710 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT, in PromoteNode()
5714 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec)); in PromoteNode()
5732 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); in PromoteNode()
5733 SDValue Undef = DAG.getUNDEF(MidVT); in PromoteNode()
5740 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts); in PromoteNode()
5741 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); in PromoteNode()
5749 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); in PromoteNode()
5763 SDValue NewAtomic = DAG.getAtomic(AM->getOpcode(), SL, NVT, AM->getChain(), in PromoteNode()
5767 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); in PromoteNode()
5782 DAG.getAtomic(ISD::ATOMIC_LOAD, SL, NVT, DAG.getVTList(NVT, MVT::Other), in PromoteNode()
5784 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); in PromoteNode()
5793 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NewScalarType, Scalar); in PromoteNode()
5794 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5795 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); in PromoteNode()
5798 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NewScalarType, Scalar); in PromoteNode()
5799 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5801 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2, in PromoteNode()
5802 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true))); in PromoteNode()