Lines Matching refs:ISD
145 bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { in isConstantSplatVector()
146 if (N->getOpcode() == ISD::SPLAT_VECTOR) { in isConstantSplatVector()
180 bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { in isConstantSplatVectorAllOnes()
182 while (N->getOpcode() == ISD::BITCAST) in isConstantSplatVectorAllOnes()
185 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { in isConstantSplatVectorAllOnes()
190 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; in isConstantSplatVectorAllOnes()
229 bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { in isConstantSplatVectorAllZeros()
231 while (N->getOpcode() == ISD::BITCAST) in isConstantSplatVectorAllZeros()
234 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { in isConstantSplatVectorAllZeros()
239 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; in isConstantSplatVectorAllZeros()
271 bool ISD::isBuildVectorAllOnes(const SDNode *N) { in isBuildVectorAllOnes()
275 bool ISD::isBuildVectorAllZeros(const SDNode *N) { in isBuildVectorAllZeros()
279 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { in isBuildVectorOfConstantSDNodes()
280 if (N->getOpcode() != ISD::BUILD_VECTOR) in isBuildVectorOfConstantSDNodes()
292 bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) { in isBuildVectorOfConstantFPSDNodes()
293 if (N->getOpcode() != ISD::BUILD_VECTOR) in isBuildVectorOfConstantFPSDNodes()
305 bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize, in isVectorShrinkable()
313 if (N->getOpcode() == ISD::ZERO_EXTEND) { in isVectorShrinkable()
318 if (N->getOpcode() == ISD::SIGN_EXTEND) { in isVectorShrinkable()
323 if (N->getOpcode() != ISD::BUILD_VECTOR) in isVectorShrinkable()
342 bool ISD::allOperandsUndef(const SDNode *N) { in allOperandsUndef()
351 bool ISD::isFreezeUndef(const SDNode *N) { in isFreezeUndef()
352 return N->getOpcode() == ISD::FREEZE && N->getOperand(0).isUndef(); in isFreezeUndef()
356 bool ISD::matchUnaryPredicateImpl(SDValue Op, in matchUnaryPredicateImpl()
364 if (ISD::BUILD_VECTOR != Op.getOpcode() && in matchUnaryPredicateImpl()
365 ISD::SPLAT_VECTOR != Op.getOpcode()) in matchUnaryPredicateImpl()
383 template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
385 template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
388 bool ISD::matchBinaryPredicate( in matchBinaryPredicate()
402 (LHS.getOpcode() != ISD::BUILD_VECTOR && in matchBinaryPredicate()
403 LHS.getOpcode() != ISD::SPLAT_VECTOR)) in matchBinaryPredicate()
425 ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) { in getVecReduceBaseOpcode()
429 case ISD::VECREDUCE_FADD: in getVecReduceBaseOpcode()
430 case ISD::VECREDUCE_SEQ_FADD: in getVecReduceBaseOpcode()
431 case ISD::VP_REDUCE_FADD: in getVecReduceBaseOpcode()
432 case ISD::VP_REDUCE_SEQ_FADD: in getVecReduceBaseOpcode()
433 return ISD::FADD; in getVecReduceBaseOpcode()
434 case ISD::VECREDUCE_FMUL: in getVecReduceBaseOpcode()
435 case ISD::VECREDUCE_SEQ_FMUL: in getVecReduceBaseOpcode()
436 case ISD::VP_REDUCE_FMUL: in getVecReduceBaseOpcode()
437 case ISD::VP_REDUCE_SEQ_FMUL: in getVecReduceBaseOpcode()
438 return ISD::FMUL; in getVecReduceBaseOpcode()
439 case ISD::VECREDUCE_ADD: in getVecReduceBaseOpcode()
440 case ISD::VP_REDUCE_ADD: in getVecReduceBaseOpcode()
441 return ISD::ADD; in getVecReduceBaseOpcode()
442 case ISD::VECREDUCE_MUL: in getVecReduceBaseOpcode()
443 case ISD::VP_REDUCE_MUL: in getVecReduceBaseOpcode()
444 return ISD::MUL; in getVecReduceBaseOpcode()
445 case ISD::VECREDUCE_AND: in getVecReduceBaseOpcode()
446 case ISD::VP_REDUCE_AND: in getVecReduceBaseOpcode()
447 return ISD::AND; in getVecReduceBaseOpcode()
448 case ISD::VECREDUCE_OR: in getVecReduceBaseOpcode()
449 case ISD::VP_REDUCE_OR: in getVecReduceBaseOpcode()
450 return ISD::OR; in getVecReduceBaseOpcode()
451 case ISD::VECREDUCE_XOR: in getVecReduceBaseOpcode()
452 case ISD::VP_REDUCE_XOR: in getVecReduceBaseOpcode()
453 return ISD::XOR; in getVecReduceBaseOpcode()
454 case ISD::VECREDUCE_SMAX: in getVecReduceBaseOpcode()
455 case ISD::VP_REDUCE_SMAX: in getVecReduceBaseOpcode()
456 return ISD::SMAX; in getVecReduceBaseOpcode()
457 case ISD::VECREDUCE_SMIN: in getVecReduceBaseOpcode()
458 case ISD::VP_REDUCE_SMIN: in getVecReduceBaseOpcode()
459 return ISD::SMIN; in getVecReduceBaseOpcode()
460 case ISD::VECREDUCE_UMAX: in getVecReduceBaseOpcode()
461 case ISD::VP_REDUCE_UMAX: in getVecReduceBaseOpcode()
462 return ISD::UMAX; in getVecReduceBaseOpcode()
463 case ISD::VECREDUCE_UMIN: in getVecReduceBaseOpcode()
464 case ISD::VP_REDUCE_UMIN: in getVecReduceBaseOpcode()
465 return ISD::UMIN; in getVecReduceBaseOpcode()
466 case ISD::VECREDUCE_FMAX: in getVecReduceBaseOpcode()
467 case ISD::VP_REDUCE_FMAX: in getVecReduceBaseOpcode()
468 return ISD::FMAXNUM; in getVecReduceBaseOpcode()
469 case ISD::VECREDUCE_FMIN: in getVecReduceBaseOpcode()
470 case ISD::VP_REDUCE_FMIN: in getVecReduceBaseOpcode()
471 return ISD::FMINNUM; in getVecReduceBaseOpcode()
472 case ISD::VECREDUCE_FMAXIMUM: in getVecReduceBaseOpcode()
473 case ISD::VP_REDUCE_FMAXIMUM: in getVecReduceBaseOpcode()
474 return ISD::FMAXIMUM; in getVecReduceBaseOpcode()
475 case ISD::VECREDUCE_FMINIMUM: in getVecReduceBaseOpcode()
476 case ISD::VP_REDUCE_FMINIMUM: in getVecReduceBaseOpcode()
477 return ISD::FMINIMUM; in getVecReduceBaseOpcode()
481 bool ISD::isVPOpcode(unsigned Opcode) { in isVPOpcode()
486 case ISD::VPSD: \ in isVPOpcode()
492 bool ISD::isVPBinaryOp(unsigned Opcode) { in isVPBinaryOp()
496 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD: in isVPBinaryOp()
504 bool ISD::isVPReduction(unsigned Opcode) { in isVPReduction()
508 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD: in isVPReduction()
517 std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) { in getVPMaskIdx()
522 case ISD::VPSD: \ in getVPMaskIdx()
529 std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) { in getVPExplicitVectorLengthIdx()
534 case ISD::VPSD: \ in getVPExplicitVectorLengthIdx()
540 std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode, in getBaseOpcodeForVP()
546 #define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC: in getBaseOpcodeForVP()
547 #define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC; in getBaseOpcodeForVP()
554 unsigned ISD::getVPForBaseOpcode(unsigned Opcode) { in getVPForBaseOpcode()
559 #define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC: in getVPForBaseOpcode()
560 #define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC; in getVPForBaseOpcode()
565 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { in getExtForLoadExtType()
567 case ISD::EXTLOAD: in getExtForLoadExtType()
568 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND; in getExtForLoadExtType()
569 case ISD::SEXTLOAD: in getExtForLoadExtType()
570 return ISD::SIGN_EXTEND; in getExtForLoadExtType()
571 case ISD::ZEXTLOAD: in getExtForLoadExtType()
572 return ISD::ZERO_EXTEND; in getExtForLoadExtType()
580 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { in getSetCCSwappedOperands()
585 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits in getSetCCSwappedOperands()
590 static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) { in getSetCCInverseImpl()
597 if (Operation > ISD::SETTRUE2) in getSetCCInverseImpl()
600 return ISD::CondCode(Operation); in getSetCCInverseImpl()
603 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) { in getSetCCInverse()
607 ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op, in getSetCCInverse()
615 static int isSignedOp(ISD::CondCode Opcode) { in isSignedOp()
618 case ISD::SETEQ: in isSignedOp()
619 case ISD::SETNE: return 0; in isSignedOp()
620 case ISD::SETLT: in isSignedOp()
621 case ISD::SETLE: in isSignedOp()
622 case ISD::SETGT: in isSignedOp()
623 case ISD::SETGE: return 1; in isSignedOp()
624 case ISD::SETULT: in isSignedOp()
625 case ISD::SETULE: in isSignedOp()
626 case ISD::SETUGT: in isSignedOp()
627 case ISD::SETUGE: return 2; in isSignedOp()
631 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, in getSetCCOrOperation()
636 return ISD::SETCC_INVALID; in getSetCCOrOperation()
642 if (Op > ISD::SETTRUE2) in getSetCCOrOperation()
646 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT in getSetCCOrOperation()
647 Op = ISD::SETNE; in getSetCCOrOperation()
649 return ISD::CondCode(Op); in getSetCCOrOperation()
652 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, in getSetCCAndOperation()
657 return ISD::SETCC_INVALID; in getSetCCAndOperation()
660 ISD::CondCode Result = ISD::CondCode(Op1 & Op2); in getSetCCAndOperation()
666 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT in getSetCCAndOperation()
667 case ISD::SETOEQ: // SETEQ & SETU[LG]E in getSetCCAndOperation()
668 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE in getSetCCAndOperation()
669 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE in getSetCCAndOperation()
670 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE in getSetCCAndOperation()
720 case ISD::TargetExternalSymbol: in AddNodeIDCustom()
721 case ISD::ExternalSymbol: in AddNodeIDCustom()
722 case ISD::MCSymbol: in AddNodeIDCustom()
725 case ISD::TargetConstant: in AddNodeIDCustom()
726 case ISD::Constant: { in AddNodeIDCustom()
732 case ISD::TargetConstantFP: in AddNodeIDCustom()
733 case ISD::ConstantFP: in AddNodeIDCustom()
736 case ISD::TargetGlobalAddress: in AddNodeIDCustom()
737 case ISD::GlobalAddress: in AddNodeIDCustom()
738 case ISD::TargetGlobalTLSAddress: in AddNodeIDCustom()
739 case ISD::GlobalTLSAddress: { in AddNodeIDCustom()
746 case ISD::BasicBlock: in AddNodeIDCustom()
749 case ISD::Register: in AddNodeIDCustom()
752 case ISD::RegisterMask: in AddNodeIDCustom()
755 case ISD::SRCVALUE: in AddNodeIDCustom()
758 case ISD::FrameIndex: in AddNodeIDCustom()
759 case ISD::TargetFrameIndex: in AddNodeIDCustom()
762 case ISD::LIFETIME_START: in AddNodeIDCustom()
763 case ISD::LIFETIME_END: in AddNodeIDCustom()
769 case ISD::PSEUDO_PROBE: in AddNodeIDCustom()
774 case ISD::JumpTable: in AddNodeIDCustom()
775 case ISD::TargetJumpTable: in AddNodeIDCustom()
779 case ISD::ConstantPool: in AddNodeIDCustom()
780 case ISD::TargetConstantPool: { in AddNodeIDCustom()
791 case ISD::TargetIndex: { in AddNodeIDCustom()
798 case ISD::LOAD: { in AddNodeIDCustom()
806 case ISD::STORE: { in AddNodeIDCustom()
814 case ISD::VP_LOAD: { in AddNodeIDCustom()
822 case ISD::VP_STORE: { in AddNodeIDCustom()
830 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: { in AddNodeIDCustom()
837 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: { in AddNodeIDCustom()
844 case ISD::VP_GATHER: { in AddNodeIDCustom()
852 case ISD::VP_SCATTER: { in AddNodeIDCustom()
860 case ISD::MLOAD: { in AddNodeIDCustom()
868 case ISD::MSTORE: { in AddNodeIDCustom()
876 case ISD::MGATHER: { in AddNodeIDCustom()
884 case ISD::MSCATTER: { in AddNodeIDCustom()
892 case ISD::ATOMIC_CMP_SWAP: in AddNodeIDCustom()
893 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: in AddNodeIDCustom()
894 case ISD::ATOMIC_SWAP: in AddNodeIDCustom()
895 case ISD::ATOMIC_LOAD_ADD: in AddNodeIDCustom()
896 case ISD::ATOMIC_LOAD_SUB: in AddNodeIDCustom()
897 case ISD::ATOMIC_LOAD_AND: in AddNodeIDCustom()
898 case ISD::ATOMIC_LOAD_CLR: in AddNodeIDCustom()
899 case ISD::ATOMIC_LOAD_OR: in AddNodeIDCustom()
900 case ISD::ATOMIC_LOAD_XOR: in AddNodeIDCustom()
901 case ISD::ATOMIC_LOAD_NAND: in AddNodeIDCustom()
902 case ISD::ATOMIC_LOAD_MIN: in AddNodeIDCustom()
903 case ISD::ATOMIC_LOAD_MAX: in AddNodeIDCustom()
904 case ISD::ATOMIC_LOAD_UMIN: in AddNodeIDCustom()
905 case ISD::ATOMIC_LOAD_UMAX: in AddNodeIDCustom()
906 case ISD::ATOMIC_LOAD: in AddNodeIDCustom()
907 case ISD::ATOMIC_STORE: { in AddNodeIDCustom()
915 case ISD::VECTOR_SHUFFLE: { in AddNodeIDCustom()
921 case ISD::TargetBlockAddress: in AddNodeIDCustom()
922 case ISD::BlockAddress: { in AddNodeIDCustom()
929 case ISD::AssertAlign: in AddNodeIDCustom()
932 case ISD::PREFETCH: in AddNodeIDCustom()
933 case ISD::INTRINSIC_VOID: in AddNodeIDCustom()
934 case ISD::INTRINSIC_W_CHAIN: in AddNodeIDCustom()
973 case ISD::HANDLENODE: in doNotCSE()
974 case ISD::EH_LABEL: in doNotCSE()
1017 if (N->getOpcode() == ISD::DELETED_NODE) in RemoveDeadNodes()
1104 N->NodeType = ISD::DELETED_NODE; in DeallocateNode()
1119 if (N->getOpcode() > ISD::BUILTIN_OP_END) in VerifySDNode()
1122 case ISD::BUILD_PAIR: { in VerifySDNode()
1136 case ISD::BUILD_VECTOR: { in VerifySDNode()
1177 case ISD::HANDLENODE: return false; // noop. in RemoveNodeFromCSEMaps()
1178 case ISD::CONDCODE: in RemoveNodeFromCSEMaps()
1184 case ISD::ExternalSymbol: in RemoveNodeFromCSEMaps()
1187 case ISD::TargetExternalSymbol: { in RemoveNodeFromCSEMaps()
1193 case ISD::MCSymbol: { in RemoveNodeFromCSEMaps()
1198 case ISD::VALUETYPE: { in RemoveNodeFromCSEMaps()
1210 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); in RemoveNodeFromCSEMaps()
1211 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); in RemoveNodeFromCSEMaps()
1325 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(), in SelectionDAG()
1379 case ISD::Constant: in FindNodeOrInsertPos()
1380 case ISD::ConstantFP: in FindNodeOrInsertPos()
1393 case ISD::Constant: in FindNodeOrInsertPos()
1394 case ISD::ConstantFP: in FindNodeOrInsertPos()
1435 ? getNode(ISD::FP_EXTEND, DL, VT, Op) in getFPExtendOrRound()
1436 : getNode(ISD::FP_ROUND, DL, VT, Op, in getFPExtendOrRound()
1447 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op}) in getStrictFPExtendOrRound()
1448 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other}, in getStrictFPExtendOrRound()
1456 getNode(ISD::ANY_EXTEND, DL, VT, Op) : in getAnyExtOrTrunc()
1457 getNode(ISD::TRUNCATE, DL, VT, Op); in getAnyExtOrTrunc()
1462 getNode(ISD::SIGN_EXTEND, DL, VT, Op) : in getSExtOrTrunc()
1463 getNode(ISD::TRUNCATE, DL, VT, Op); in getSExtOrTrunc()
1468 getNode(ISD::ZERO_EXTEND, DL, VT, Op) : in getZExtOrTrunc()
1469 getNode(ISD::TRUNCATE, DL, VT, Op); in getZExtOrTrunc()
1520 return getNode(ISD::TRUNCATE, SL, VT, Op); in getBoolExtOrTrunc()
1541 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT)); in getZeroExtendInReg()
1559 return getNode(ISD::VP_AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT), Mask, in getVPZeroExtendInReg()
1576 return getNode(ISD::SUB, DL, VT, getConstant(0, DL, VT), Val); in getNegative()
1581 return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT)); in getNOT()
1586 return getNode(ISD::XOR, DL, VT, Val, TrueValue); in getLogicalNOT()
1592 return getNode(ISD::VP_XOR, DL, VT, Val, TrueValue, Mask, EVL); in getVPLogicalNOT()
1603 return getNode(ISD::VP_ZERO_EXTEND, DL, VT, Op, Mask, EVL); in getVPZExtOrTrunc()
1605 return getNode(ISD::VP_TRUNCATE, DL, VT, Op, Mask, EVL); in getVPZExtOrTrunc()
1674 TLI->isOperationLegal(ISD::SPLAT_VECTOR, VT)) { in getConstant()
1685 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts); in getConstant()
1719 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); in getConstant()
1725 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; in getConstant()
1787 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; in getConstantFP()
1842 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; in getGlobalAddress()
1844 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; in getGlobalAddress()
1864 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; in getFrameIndex()
1883 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; in getJumpTable()
1902 return getNode(ISD::JUMP_TABLE_DEBUG_INFO, DL, MVT::Glue, Chain, in getJumpTableDebugInfo()
1915 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; in getConstantPool()
1943 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; in getConstantPool()
1964 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), std::nullopt); in getBasicBlock()
2017 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { in getCondCode()
2046 return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT)); in getVScale()
2068 ISD::STEP_VECTOR, DL, ResVT, in getStepVector()
2190 while (V.getOpcode() == ISD::BITCAST) in getVectorShuffle()
2222 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV); in getVectorShuffle()
2231 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, VTs, Ops); in getVectorShuffle()
2269 AddNodeIDNode(ID, ISD::Register, VTs, std::nullopt); in getRegister()
2284 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), std::nullopt); in getRegisterMask()
2298 return getLabelNode(ISD::EH_LABEL, dl, Root, Label); in getEHLabel()
2323 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; in getBlockAddress()
2343 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), std::nullopt); in getSrcValue()
2358 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), std::nullopt); in getMDNode()
2375 return getNode(ISD::BITCAST, SDLoc(V), VT, V); in getBitcast()
2383 AddNodeIDNode(ID, ISD::ADDRSPACECAST, VTs, Ops); in getAddrSpaceCast()
2401 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V); in getFreeze()
2428 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, in expandVAArg()
2432 getNode(ISD::AND, dl, VAList.getValueType(), VAList, in expandVAArg()
2437 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, in expandVAArg()
2528 ISD::CondCode Cond, const SDLoc &dl) { in FoldSetCC()
2544 case ISD::SETFALSE: in FoldSetCC()
2545 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT); in FoldSetCC()
2546 case ISD::SETTRUE: in FoldSetCC()
2547 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT); in FoldSetCC()
2549 case ISD::SETOEQ: in FoldSetCC()
2550 case ISD::SETOGT: in FoldSetCC()
2551 case ISD::SETOGE: in FoldSetCC()
2552 case ISD::SETOLT: in FoldSetCC()
2553 case ISD::SETOLE: in FoldSetCC()
2554 case ISD::SETONE: in FoldSetCC()
2555 case ISD::SETO: in FoldSetCC()
2556 case ISD::SETUO: in FoldSetCC()
2557 case ISD::SETUEQ: in FoldSetCC()
2558 case ISD::SETUNE: in FoldSetCC()
2569 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) in FoldSetCC()
2580 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT); in FoldSetCC()
2600 case ISD::SETEQ: if (R==APFloat::cmpUnordered) in FoldSetCC()
2603 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT, in FoldSetCC()
2605 case ISD::SETNE: if (R==APFloat::cmpUnordered) in FoldSetCC()
2608 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan || in FoldSetCC()
2611 case ISD::SETLT: if (R==APFloat::cmpUnordered) in FoldSetCC()
2614 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT, in FoldSetCC()
2616 case ISD::SETGT: if (R==APFloat::cmpUnordered) in FoldSetCC()
2619 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl, in FoldSetCC()
2621 case ISD::SETLE: if (R==APFloat::cmpUnordered) in FoldSetCC()
2624 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan || in FoldSetCC()
2627 case ISD::SETGE: if (R==APFloat::cmpUnordered) in FoldSetCC()
2630 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan || in FoldSetCC()
2632 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT, in FoldSetCC()
2634 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT, in FoldSetCC()
2636 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered || in FoldSetCC()
2639 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT, in FoldSetCC()
2641 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered || in FoldSetCC()
2644 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan || in FoldSetCC()
2647 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl, in FoldSetCC()
2649 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT, in FoldSetCC()
2654 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); in FoldSetCC()
2665 switch (ISD::getUnorderedFlavor(Cond)) { in FoldSetCC()
2760 case ISD::SPLAT_VECTOR: in isSplatValue()
2765 case ISD::ADD: in isSplatValue()
2766 case ISD::SUB: in isSplatValue()
2767 case ISD::AND: in isSplatValue()
2768 case ISD::XOR: in isSplatValue()
2769 case ISD::OR: { in isSplatValue()
2780 case ISD::ABS: in isSplatValue()
2781 case ISD::TRUNCATE: in isSplatValue()
2782 case ISD::SIGN_EXTEND: in isSplatValue()
2783 case ISD::ZERO_EXTEND: in isSplatValue()
2786 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || in isSplatValue()
2787 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) in isSplatValue()
2803 case ISD::BUILD_VECTOR: { in isSplatValue()
2819 case ISD::VECTOR_SHUFFLE: { in isSplatValue()
2857 case ISD::EXTRACT_SUBVECTOR: { in isSplatValue()
2873 case ISD::ANY_EXTEND_VECTOR_INREG: in isSplatValue()
2874 case ISD::SIGN_EXTEND_VECTOR_INREG: in isSplatValue()
2875 case ISD::ZERO_EXTEND_VECTOR_INREG: { in isSplatValue()
2890 case ISD::BITCAST: { in isSplatValue()
2974 case ISD::SPLAT_VECTOR: in getSplatSourceVector()
2977 case ISD::VECTOR_SHUFFLE: { in getSplatSourceVector()
3007 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector, in getSplatValue()
3016 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || in getValidShiftAmountRange()
3017 V.getOpcode() == ISD::SRA) && in getValidShiftAmountRange()
3065 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || in getValidShiftAmount()
3066 V.getOpcode() == ISD::SRA) && in getValidShiftAmount()
3087 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || in getValidMinimumShiftAmount()
3088 V.getOpcode() == ISD::SRA) && in getValidMinimumShiftAmount()
3108 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || in getValidMaximumShiftAmount()
3109 V.getOpcode() == ISD::SRA) && in getValidMaximumShiftAmount()
3173 case ISD::MERGE_VALUES: in computeKnownBits()
3176 case ISD::SPLAT_VECTOR: { in computeKnownBits()
3185 case ISD::SPLAT_VECTOR_PARTS: { in computeKnownBits()
3194 case ISD::STEP_VECTOR: { in computeKnownBits()
3221 case ISD::BUILD_VECTOR: in computeKnownBits()
3247 case ISD::VECTOR_SHUFFLE: { in computeKnownBits()
3275 case ISD::VSCALE: { in computeKnownBits()
3281 case ISD::CONCAT_VECTORS: { in computeKnownBits()
3303 case ISD::INSERT_SUBVECTOR: { in computeKnownBits()
3329 case ISD::EXTRACT_SUBVECTOR: { in computeKnownBits()
3341 case ISD::SCALAR_TO_VECTOR: { in computeKnownBits()
3356 case ISD::BITCAST: { in computeKnownBits()
3423 case ISD::AND: in computeKnownBits()
3429 case ISD::OR: in computeKnownBits()
3435 case ISD::XOR: in computeKnownBits()
3441 case ISD::MUL: { in computeKnownBits()
3460 case ISD::MULHU: { in computeKnownBits()
3466 case ISD::MULHS: { in computeKnownBits()
3472 case ISD::ABDU: { in computeKnownBits()
3478 case ISD::ABDS: { in computeKnownBits()
3491 case ISD::UMUL_LOHI: { in computeKnownBits()
3502 case ISD::SMUL_LOHI: { in computeKnownBits()
3513 case ISD::AVGFLOORU: { in computeKnownBits()
3519 case ISD::AVGCEILU: { in computeKnownBits()
3525 case ISD::AVGFLOORS: { in computeKnownBits()
3531 case ISD::AVGCEILS: { in computeKnownBits()
3537 case ISD::SELECT: in computeKnownBits()
3538 case ISD::VSELECT: in computeKnownBits()
3548 case ISD::SELECT_CC: in computeKnownBits()
3558 case ISD::SMULO: in computeKnownBits()
3559 case ISD::UMULO: in computeKnownBits()
3571 case ISD::SETCC: in computeKnownBits()
3572 case ISD::SETCCCARRY: in computeKnownBits()
3573 case ISD::STRICT_FSETCC: in computeKnownBits()
3574 case ISD::STRICT_FSETCCS: { in computeKnownBits()
3583 case ISD::SHL: { in computeKnownBits()
3600 case ISD::SRL: in computeKnownBits()
3611 case ISD::SRA: in computeKnownBits()
3617 case ISD::FSHL: in computeKnownBits()
3618 case ISD::FSHR: in computeKnownBits()
3625 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1), in computeKnownBits()
3634 if (Opcode == ISD::FSHL) { in computeKnownBits()
3648 case ISD::SHL_PARTS: in computeKnownBits()
3649 case ISD::SRA_PARTS: in computeKnownBits()
3650 case ISD::SRL_PARTS: { in computeKnownBits()
3663 if (Opcode == ISD::SHL_PARTS) in computeKnownBits()
3665 else if (Opcode == ISD::SRA_PARTS) in computeKnownBits()
3678 case ISD::SIGN_EXTEND_INREG: { in computeKnownBits()
3684 case ISD::CTTZ: in computeKnownBits()
3685 case ISD::CTTZ_ZERO_UNDEF: { in computeKnownBits()
3693 case ISD::CTLZ: in computeKnownBits()
3694 case ISD::CTLZ_ZERO_UNDEF: { in computeKnownBits()
3702 case ISD::CTPOP: { in computeKnownBits()
3709 case ISD::PARITY: { in computeKnownBits()
3714 case ISD::LOAD: { in computeKnownBits()
3717 if (ISD::isNON_EXTLoad(LD) && Cst) { in computeKnownBits()
3791 if (ISD::isZEXTLoad(Op.getNode())) in computeKnownBits()
3793 else if (ISD::isSEXTLoad(Op.getNode())) in computeKnownBits()
3795 else if (ISD::isEXTLoad(Op.getNode())) in computeKnownBits()
3804 case ISD::ZERO_EXTEND_VECTOR_INREG: { in computeKnownBits()
3813 case ISD::ZERO_EXTEND: { in computeKnownBits()
3818 case ISD::SIGN_EXTEND_VECTOR_INREG: { in computeKnownBits()
3829 case ISD::SIGN_EXTEND: { in computeKnownBits()
3836 case ISD::ANY_EXTEND_VECTOR_INREG: { in computeKnownBits()
3845 case ISD::ANY_EXTEND: { in computeKnownBits()
3850 case ISD::TRUNCATE: { in computeKnownBits()
3855 case ISD::AssertZext: { in computeKnownBits()
3863 case ISD::AssertAlign: { in computeKnownBits()
3874 case ISD::FGETSIGN: in computeKnownBits()
3878 case ISD::ADD: in computeKnownBits()
3879 case ISD::SUB: { in computeKnownBits()
3884 Op.getOpcode() == ISD::ADD, Flags.hasNoSignedWrap(), in computeKnownBits()
3888 case ISD::USUBO: in computeKnownBits()
3889 case ISD::SSUBO: in computeKnownBits()
3890 case ISD::USUBO_CARRY: in computeKnownBits()
3891 case ISD::SSUBO_CARRY: in computeKnownBits()
3901 case ISD::SUBC: { in computeKnownBits()
3907 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) { in computeKnownBits()
3920 case ISD::UADDO: in computeKnownBits()
3921 case ISD::SADDO: in computeKnownBits()
3922 case ISD::UADDO_CARRY: in computeKnownBits()
3923 case ISD::SADDO_CARRY: in computeKnownBits()
3933 case ISD::ADDC: in computeKnownBits()
3934 case ISD::ADDE: { in computeKnownBits()
3939 if (Opcode == ISD::ADDE) in computeKnownBits()
3942 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) { in computeKnownBits()
3955 case ISD::UDIV: { in computeKnownBits()
3961 case ISD::SDIV: { in computeKnownBits()
3967 case ISD::SREM: { in computeKnownBits()
3973 case ISD::UREM: { in computeKnownBits()
3979 case ISD::EXTRACT_ELEMENT: { in computeKnownBits()
3992 case ISD::EXTRACT_VECTOR_ELT: { in computeKnownBits()
4020 case ISD::INSERT_VECTOR_ELT: { in computeKnownBits()
4050 case ISD::BITREVERSE: { in computeKnownBits()
4055 case ISD::BSWAP: { in computeKnownBits()
4060 case ISD::ABS: { in computeKnownBits()
4067 case ISD::USUBSAT: { in computeKnownBits()
4073 case ISD::UMIN: { in computeKnownBits()
4079 case ISD::UMAX: { in computeKnownBits()
4085 case ISD::SMIN: in computeKnownBits()
4086 case ISD::SMAX: { in computeKnownBits()
4089 bool IsMax = (Opcode == ISD::SMAX); in computeKnownBits()
4092 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) in computeKnownBits()
4137 case ISD::UINT_TO_FP: { in computeKnownBits()
4141 case ISD::SINT_TO_FP: { in computeKnownBits()
4149 case ISD::FP_TO_UINT_SAT: { in computeKnownBits()
4155 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: in computeKnownBits()
4168 case ISD::ATOMIC_CMP_SWAP: in computeKnownBits()
4169 case ISD::ATOMIC_SWAP: in computeKnownBits()
4170 case ISD::ATOMIC_LOAD_ADD: in computeKnownBits()
4171 case ISD::ATOMIC_LOAD_SUB: in computeKnownBits()
4172 case ISD::ATOMIC_LOAD_AND: in computeKnownBits()
4173 case ISD::ATOMIC_LOAD_CLR: in computeKnownBits()
4174 case ISD::ATOMIC_LOAD_OR: in computeKnownBits()
4175 case ISD::ATOMIC_LOAD_XOR: in computeKnownBits()
4176 case ISD::ATOMIC_LOAD_NAND: in computeKnownBits()
4177 case ISD::ATOMIC_LOAD_MIN: in computeKnownBits()
4178 case ISD::ATOMIC_LOAD_MAX: in computeKnownBits()
4179 case ISD::ATOMIC_LOAD_UMIN: in computeKnownBits()
4180 case ISD::ATOMIC_LOAD_UMAX: in computeKnownBits()
4181 case ISD::ATOMIC_LOAD: { in computeKnownBits()
4186 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) in computeKnownBits()
4188 else if (Op->getOpcode() == ISD::ATOMIC_LOAD && in computeKnownBits()
4189 cast<AtomicSDNode>(Op)->getExtensionType() == ISD::ZEXTLOAD) in computeKnownBits()
4194 case ISD::FrameIndex: in computeKnownBits()
4195 case ISD::TargetFrameIndex: in computeKnownBits()
4201 if (Opcode < ISD::BUILTIN_OP_END) in computeKnownBits()
4204 case ISD::INTRINSIC_WO_CHAIN: in computeKnownBits()
4205 case ISD::INTRINSIC_W_CHAIN: in computeKnownBits()
4206 case ISD::INTRINSIC_VOID: in computeKnownBits()
4257 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 && in computeOverflowForUnsignedAdd()
4262 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 && in computeOverflowForUnsignedAdd()
4354 if (ISD::matchUnaryPredicate(Val, [BitWidth](ConstantSDNode *C) { in isKnownToBeAPowerOfTwo()
4361 if (Val.getOpcode() == ISD::SHL) { in isKnownToBeAPowerOfTwo()
4371 if (Val.getOpcode() == ISD::SRL) { in isKnownToBeAPowerOfTwo()
4379 if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR) in isKnownToBeAPowerOfTwo()
4383 if (Val.getOpcode() == ISD::BUILD_VECTOR) in isKnownToBeAPowerOfTwo()
4392 if (Val.getOpcode() == ISD::SPLAT_VECTOR) in isKnownToBeAPowerOfTwo()
4398 if (Val.getOpcode() == ISD::VSCALE && in isKnownToBeAPowerOfTwo()
4403 if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX || in isKnownToBeAPowerOfTwo()
4404 Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX) in isKnownToBeAPowerOfTwo()
4408 if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT) in isKnownToBeAPowerOfTwo()
4422 if (Val.getOpcode() == ISD::ZERO_EXTEND) in isKnownToBeAPowerOfTwo()
4434 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP) in isKnownToBeAPowerOfTwoFP()
4475 case ISD::AssertSext: in ComputeNumSignBits()
4478 case ISD::AssertZext: in ComputeNumSignBits()
4481 case ISD::MERGE_VALUES: in ComputeNumSignBits()
4484 case ISD::SPLAT_VECTOR: { in ComputeNumSignBits()
4492 case ISD::BUILD_VECTOR: in ComputeNumSignBits()
4519 case ISD::VECTOR_SHUFFLE: { in ComputeNumSignBits()
4543 case ISD::BITCAST: { in ComputeNumSignBits()
4589 case ISD::FP_TO_SINT_SAT: in ComputeNumSignBits()
4593 case ISD::SIGN_EXTEND: in ComputeNumSignBits()
4596 case ISD::SIGN_EXTEND_INREG: in ComputeNumSignBits()
4602 case ISD::SIGN_EXTEND_VECTOR_INREG: { in ComputeNumSignBits()
4611 case ISD::SRA: in ComputeNumSignBits()
4618 case ISD::SHL: in ComputeNumSignBits()
4628 if (ISD::isExtOpcode(Op.getOperand(0).getOpcode())) { in ComputeNumSignBits()
4648 case ISD::AND: in ComputeNumSignBits()
4649 case ISD::OR: in ComputeNumSignBits()
4650 case ISD::XOR: // NOT is handled here. in ComputeNumSignBits()
4662 case ISD::SELECT: in ComputeNumSignBits()
4663 case ISD::VSELECT: in ComputeNumSignBits()
4668 case ISD::SELECT_CC: in ComputeNumSignBits()
4674 case ISD::SMIN: in ComputeNumSignBits()
4675 case ISD::SMAX: { in ComputeNumSignBits()
4678 bool IsMax = (Opcode == ISD::SMAX); in ComputeNumSignBits()
4681 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) in ComputeNumSignBits()
4701 case ISD::UMIN: in ComputeNumSignBits()
4702 case ISD::UMAX: in ComputeNumSignBits()
4708 case ISD::SADDO: in ComputeNumSignBits()
4709 case ISD::UADDO: in ComputeNumSignBits()
4710 case ISD::SADDO_CARRY: in ComputeNumSignBits()
4711 case ISD::UADDO_CARRY: in ComputeNumSignBits()
4712 case ISD::SSUBO: in ComputeNumSignBits()
4713 case ISD::USUBO: in ComputeNumSignBits()
4714 case ISD::SSUBO_CARRY: in ComputeNumSignBits()
4715 case ISD::USUBO_CARRY: in ComputeNumSignBits()
4716 case ISD::SMULO: in ComputeNumSignBits()
4717 case ISD::UMULO: in ComputeNumSignBits()
4728 case ISD::SETCC: in ComputeNumSignBits()
4729 case ISD::SETCCCARRY: in ComputeNumSignBits()
4730 case ISD::STRICT_FSETCC: in ComputeNumSignBits()
4731 case ISD::STRICT_FSETCCS: { in ComputeNumSignBits()
4739 case ISD::ROTL: in ComputeNumSignBits()
4740 case ISD::ROTR: in ComputeNumSignBits()
4752 if (Opcode == ISD::ROTR) in ComputeNumSignBits()
4760 case ISD::ADD: in ComputeNumSignBits()
4761 case ISD::ADDC: in ComputeNumSignBits()
4788 case ISD::SUB: in ComputeNumSignBits()
4816 case ISD::MUL: { in ComputeNumSignBits()
4828 case ISD::AVGCEILS: in ComputeNumSignBits()
4829 case ISD::AVGFLOORS: in ComputeNumSignBits()
4835 case ISD::SREM: in ComputeNumSignBits()
4841 case ISD::TRUNCATE: { in ComputeNumSignBits()
4849 case ISD::EXTRACT_ELEMENT: { in ComputeNumSignBits()
4864 case ISD::INSERT_VECTOR_ELT: { in ComputeNumSignBits()
4896 case ISD::EXTRACT_VECTOR_ELT: { in ComputeNumSignBits()
4924 case ISD::EXTRACT_SUBVECTOR: { in ComputeNumSignBits()
4935 case ISD::CONCAT_VECTORS: { in ComputeNumSignBits()
4955 case ISD::INSERT_SUBVECTOR: { in ComputeNumSignBits()
4981 case ISD::LOAD: { in ComputeNumSignBits()
4990 case ISD::SEXTLOAD: in ComputeNumSignBits()
4993 case ISD::ZEXTLOAD: in ComputeNumSignBits()
5009 case ISD::ATOMIC_CMP_SWAP: in ComputeNumSignBits()
5010 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: in ComputeNumSignBits()
5011 case ISD::ATOMIC_SWAP: in ComputeNumSignBits()
5012 case ISD::ATOMIC_LOAD_ADD: in ComputeNumSignBits()
5013 case ISD::ATOMIC_LOAD_SUB: in ComputeNumSignBits()
5014 case ISD::ATOMIC_LOAD_AND: in ComputeNumSignBits()
5015 case ISD::ATOMIC_LOAD_CLR: in ComputeNumSignBits()
5016 case ISD::ATOMIC_LOAD_OR: in ComputeNumSignBits()
5017 case ISD::ATOMIC_LOAD_XOR: in ComputeNumSignBits()
5018 case ISD::ATOMIC_LOAD_NAND: in ComputeNumSignBits()
5019 case ISD::ATOMIC_LOAD_MIN: in ComputeNumSignBits()
5020 case ISD::ATOMIC_LOAD_MAX: in ComputeNumSignBits()
5021 case ISD::ATOMIC_LOAD_UMIN: in ComputeNumSignBits()
5022 case ISD::ATOMIC_LOAD_UMAX: in ComputeNumSignBits()
5023 case ISD::ATOMIC_LOAD: { in ComputeNumSignBits()
5029 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND) in ComputeNumSignBits()
5031 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) in ComputeNumSignBits()
5033 if (Op->getOpcode() == ISD::ATOMIC_LOAD) { in ComputeNumSignBits()
5034 ISD::LoadExtType ETy = cast<AtomicSDNode>(Op)->getExtensionType(); in ComputeNumSignBits()
5035 if (ETy == ISD::SEXTLOAD) in ComputeNumSignBits()
5037 if (ETy == ISD::ZEXTLOAD) in ComputeNumSignBits()
5052 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known. in ComputeNumSignBits()
5055 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known. in ComputeNumSignBits()
5058 case ISD::NON_EXTLOAD: in ComputeNumSignBits()
5094 if (Opcode >= ISD::BUILTIN_OP_END || in ComputeNumSignBits()
5095 Opcode == ISD::INTRINSIC_WO_CHAIN || in ComputeNumSignBits()
5096 Opcode == ISD::INTRINSIC_W_CHAIN || in ComputeNumSignBits()
5097 Opcode == ISD::INTRINSIC_VOID) { in ComputeNumSignBits()
5130 if (Op.getOpcode() == ISD::FREEZE) in isGuaranteedNotToBeUndefOrPoison()
5151 if (Opcode == ISD::FREEZE) in isGuaranteedNotToBeUndefOrPoison()
5161 case ISD::CONDCODE: in isGuaranteedNotToBeUndefOrPoison()
5162 case ISD::VALUETYPE: in isGuaranteedNotToBeUndefOrPoison()
5163 case ISD::FrameIndex: in isGuaranteedNotToBeUndefOrPoison()
5164 case ISD::TargetFrameIndex: in isGuaranteedNotToBeUndefOrPoison()
5165 case ISD::CopyFromReg: in isGuaranteedNotToBeUndefOrPoison()
5168 case ISD::UNDEF: in isGuaranteedNotToBeUndefOrPoison()
5171 case ISD::BUILD_VECTOR: in isGuaranteedNotToBeUndefOrPoison()
5183 case ISD::VECTOR_SHUFFLE: { in isGuaranteedNotToBeUndefOrPoison()
5207 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || in isGuaranteedNotToBeUndefOrPoison()
5208 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) in isGuaranteedNotToBeUndefOrPoison()
5254 case ISD::FREEZE: in canCreateUndefOrPoison()
5255 case ISD::CONCAT_VECTORS: in canCreateUndefOrPoison()
5256 case ISD::INSERT_SUBVECTOR: in canCreateUndefOrPoison()
5257 case ISD::SADDSAT: in canCreateUndefOrPoison()
5258 case ISD::UADDSAT: in canCreateUndefOrPoison()
5259 case ISD::SSUBSAT: in canCreateUndefOrPoison()
5260 case ISD::USUBSAT: in canCreateUndefOrPoison()
5261 case ISD::MULHU: in canCreateUndefOrPoison()
5262 case ISD::MULHS: in canCreateUndefOrPoison()
5263 case ISD::SMIN: in canCreateUndefOrPoison()
5264 case ISD::SMAX: in canCreateUndefOrPoison()
5265 case ISD::UMIN: in canCreateUndefOrPoison()
5266 case ISD::UMAX: in canCreateUndefOrPoison()
5267 case ISD::AND: in canCreateUndefOrPoison()
5268 case ISD::XOR: in canCreateUndefOrPoison()
5269 case ISD::ROTL: in canCreateUndefOrPoison()
5270 case ISD::ROTR: in canCreateUndefOrPoison()
5271 case ISD::FSHL: in canCreateUndefOrPoison()
5272 case ISD::FSHR: in canCreateUndefOrPoison()
5273 case ISD::BSWAP: in canCreateUndefOrPoison()
5274 case ISD::CTPOP: in canCreateUndefOrPoison()
5275 case ISD::BITREVERSE: in canCreateUndefOrPoison()
5276 case ISD::PARITY: in canCreateUndefOrPoison()
5277 case ISD::SIGN_EXTEND: in canCreateUndefOrPoison()
5278 case ISD::TRUNCATE: in canCreateUndefOrPoison()
5279 case ISD::SIGN_EXTEND_INREG: in canCreateUndefOrPoison()
5280 case ISD::SIGN_EXTEND_VECTOR_INREG: in canCreateUndefOrPoison()
5281 case ISD::ZERO_EXTEND_VECTOR_INREG: in canCreateUndefOrPoison()
5282 case ISD::BITCAST: in canCreateUndefOrPoison()
5283 case ISD::BUILD_VECTOR: in canCreateUndefOrPoison()
5284 case ISD::BUILD_PAIR: in canCreateUndefOrPoison()
5287 case ISD::SELECT_CC: in canCreateUndefOrPoison()
5288 case ISD::SETCC: { in canCreateUndefOrPoison()
5297 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4; in canCreateUndefOrPoison()
5298 ISD::CondCode CCCode = cast<CondCodeSDNode>(Op.getOperand(CCOp))->get(); in canCreateUndefOrPoison()
5306 case ISD::OR: in canCreateUndefOrPoison()
5307 case ISD::ZERO_EXTEND: in canCreateUndefOrPoison()
5308 case ISD::ADD: in canCreateUndefOrPoison()
5309 case ISD::SUB: in canCreateUndefOrPoison()
5310 case ISD::MUL: in canCreateUndefOrPoison()
5314 case ISD::SHL: in canCreateUndefOrPoison()
5315 case ISD::SRL: in canCreateUndefOrPoison()
5316 case ISD::SRA: in canCreateUndefOrPoison()
5323 case ISD::SCALAR_TO_VECTOR: in canCreateUndefOrPoison()
5327 case ISD::INSERT_VECTOR_ELT: in canCreateUndefOrPoison()
5328 case ISD::EXTRACT_VECTOR_ELT: { in canCreateUndefOrPoison()
5331 SDValue Idx = Op.getOperand(Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1); in canCreateUndefOrPoison()
5340 case ISD::VECTOR_SHUFFLE: { in canCreateUndefOrPoison()
5351 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || in canCreateUndefOrPoison()
5352 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) in canCreateUndefOrPoison()
5364 if (Opcode == ISD::OR) in isADDLike()
5367 if (Opcode == ISD::XOR) in isADDLike()
5374 (Op.getOpcode() == ISD::ADD || isADDLike(Op)); in isBaseWithConstantOffset()
5393 case ISD::FADD: in isKnownNeverNaN()
5394 case ISD::FSUB: in isKnownNeverNaN()
5395 case ISD::FMUL: in isKnownNeverNaN()
5396 case ISD::FDIV: in isKnownNeverNaN()
5397 case ISD::FREM: in isKnownNeverNaN()
5398 case ISD::FSIN: in isKnownNeverNaN()
5399 case ISD::FCOS: in isKnownNeverNaN()
5400 case ISD::FTAN: in isKnownNeverNaN()
5401 case ISD::FASIN: in isKnownNeverNaN()
5402 case ISD::FACOS: in isKnownNeverNaN()
5403 case ISD::FATAN: in isKnownNeverNaN()
5404 case ISD::FSINH: in isKnownNeverNaN()
5405 case ISD::FCOSH: in isKnownNeverNaN()
5406 case ISD::FTANH: in isKnownNeverNaN()
5407 case ISD::FMA: in isKnownNeverNaN()
5408 case ISD::FMAD: { in isKnownNeverNaN()
5414 case ISD::FCANONICALIZE: in isKnownNeverNaN()
5415 case ISD::FEXP: in isKnownNeverNaN()
5416 case ISD::FEXP2: in isKnownNeverNaN()
5417 case ISD::FEXP10: in isKnownNeverNaN()
5418 case ISD::FTRUNC: in isKnownNeverNaN()
5419 case ISD::FFLOOR: in isKnownNeverNaN()
5420 case ISD::FCEIL: in isKnownNeverNaN()
5421 case ISD::FROUND: in isKnownNeverNaN()
5422 case ISD::FROUNDEVEN: in isKnownNeverNaN()
5423 case ISD::FRINT: in isKnownNeverNaN()
5424 case ISD::LRINT: in isKnownNeverNaN()
5425 case ISD::LLRINT: in isKnownNeverNaN()
5426 case ISD::FNEARBYINT: in isKnownNeverNaN()
5427 case ISD::FLDEXP: { in isKnownNeverNaN()
5432 case ISD::FABS: in isKnownNeverNaN()
5433 case ISD::FNEG: in isKnownNeverNaN()
5434 case ISD::FCOPYSIGN: { in isKnownNeverNaN()
5437 case ISD::SELECT: in isKnownNeverNaN()
5440 case ISD::FP_EXTEND: in isKnownNeverNaN()
5441 case ISD::FP_ROUND: { in isKnownNeverNaN()
5446 case ISD::SINT_TO_FP: in isKnownNeverNaN()
5447 case ISD::UINT_TO_FP: in isKnownNeverNaN()
5449 case ISD::FSQRT: // Need is known positive in isKnownNeverNaN()
5450 case ISD::FLOG: in isKnownNeverNaN()
5451 case ISD::FLOG2: in isKnownNeverNaN()
5452 case ISD::FLOG10: in isKnownNeverNaN()
5453 case ISD::FPOWI: in isKnownNeverNaN()
5454 case ISD::FPOW: { in isKnownNeverNaN()
5460 case ISD::FMINNUM: in isKnownNeverNaN()
5461 case ISD::FMAXNUM: { in isKnownNeverNaN()
5467 case ISD::FMINNUM_IEEE: in isKnownNeverNaN()
5468 case ISD::FMAXNUM_IEEE: { in isKnownNeverNaN()
5478 case ISD::FMINIMUM: in isKnownNeverNaN()
5479 case ISD::FMAXIMUM: { in isKnownNeverNaN()
5484 case ISD::EXTRACT_VECTOR_ELT: { in isKnownNeverNaN()
5487 case ISD::BUILD_VECTOR: { in isKnownNeverNaN()
5494 if (Opcode >= ISD::BUILTIN_OP_END || in isKnownNeverNaN()
5495 Opcode == ISD::INTRINSIC_WO_CHAIN || in isKnownNeverNaN()
5496 Opcode == ISD::INTRINSIC_W_CHAIN || in isKnownNeverNaN()
5497 Opcode == ISD::INTRINSIC_VOID) { in isKnownNeverNaN()
5510 return ISD::matchUnaryFpPredicate( in isKnownNeverZeroFloat()
5522 if (ISD::matchUnaryPredicate(Op, in isKnownNeverZero()
5532 case ISD::OR: in isKnownNeverZero()
5536 case ISD::VSELECT: in isKnownNeverZero()
5537 case ISD::SELECT: in isKnownNeverZero()
5541 case ISD::SHL: { in isKnownNeverZero()
5555 case ISD::UADDSAT: in isKnownNeverZero()
5556 case ISD::UMAX: in isKnownNeverZero()
5562 case ISD::SMAX: { in isKnownNeverZero()
5577 case ISD::SMIN: { in isKnownNeverZero()
5592 case ISD::UMIN: in isKnownNeverZero()
5596 case ISD::ROTL: in isKnownNeverZero()
5597 case ISD::ROTR: in isKnownNeverZero()
5598 case ISD::BITREVERSE: in isKnownNeverZero()
5599 case ISD::BSWAP: in isKnownNeverZero()
5600 case ISD::CTPOP: in isKnownNeverZero()
5601 case ISD::ABS: in isKnownNeverZero()
5604 case ISD::SRA: in isKnownNeverZero()
5605 case ISD::SRL: { in isKnownNeverZero()
5618 case ISD::UDIV: in isKnownNeverZero()
5619 case ISD::SDIV: in isKnownNeverZero()
5626 case ISD::ADD: in isKnownNeverZero()
5634 case ISD::SUB: { in isKnownNeverZero()
5644 case ISD::MUL: in isKnownNeverZero()
5651 case ISD::ZERO_EXTEND: in isKnownNeverZero()
5652 case ISD::SIGN_EXTEND: in isKnownNeverZero()
5654 case ISD::VSCALE: { in isKnownNeverZero()
5672 return Op.getOpcode() == ISD::FABS; in cannotBeOrderedNegativeFP()
5696 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND) in getBitwiseNotOperand()
5702 ExtArg.getOperand(0).getOpcode() == ISD::TRUNCATE && in getBitwiseNotOperand()
5715 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND || in haveNoCommonBitsSetCommutative()
5716 NotOperand->getOpcode() == ISD::TRUNCATE) in haveNoCommonBitsSetCommutative()
5721 if (Other->getOpcode() == ISD::AND) in haveNoCommonBitsSetCommutative()
5728 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE) in haveNoCommonBitsSetCommutative()
5731 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE) in haveNoCommonBitsSetCommutative()
5734 if (A->getOpcode() == ISD::AND) in haveNoCommonBitsSetCommutative()
5777 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT || in FoldBUILD_VECTOR()
5823 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR || in foldCONCAT_VECTORS()
5853 else if (Op.getOpcode() == ISD::BUILD_VECTOR) in foldCONCAT_VECTORS()
5908 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!"); in getNode()
5916 case ISD::FNEG: in getNode()
5917 case ISD::FABS: in getNode()
5918 case ISD::FCEIL: in getNode()
5919 case ISD::FTRUNC: in getNode()
5920 case ISD::FFLOOR: in getNode()
5921 case ISD::FP_EXTEND: in getNode()
5922 case ISD::FP_TO_SINT: in getNode()
5923 case ISD::FP_TO_UINT: in getNode()
5924 case ISD::FP_TO_FP16: in getNode()
5925 case ISD::FP_TO_BF16: in getNode()
5926 case ISD::TRUNCATE: in getNode()
5927 case ISD::ANY_EXTEND: in getNode()
5928 case ISD::ZERO_EXTEND: in getNode()
5929 case ISD::SIGN_EXTEND: in getNode()
5930 case ISD::UINT_TO_FP: in getNode()
5931 case ISD::SINT_TO_FP: in getNode()
5932 case ISD::FP16_TO_FP: in getNode()
5933 case ISD::BF16_TO_FP: in getNode()
5934 case ISD::BITCAST: in getNode()
5935 case ISD::ABS: in getNode()
5936 case ISD::BITREVERSE: in getNode()
5937 case ISD::BSWAP: in getNode()
5938 case ISD::CTLZ: in getNode()
5939 case ISD::CTLZ_ZERO_UNDEF: in getNode()
5940 case ISD::CTTZ: in getNode()
5941 case ISD::CTTZ_ZERO_UNDEF: in getNode()
5942 case ISD::CTPOP: in getNode()
5943 case ISD::STEP_VECTOR: { in getNode()
5952 case ISD::STEP_VECTOR: in getNode()
5955 assert(OpOpcode == ISD::TargetConstant && in getNode()
5959 case ISD::FREEZE: in getNode()
5965 case ISD::TokenFactor: in getNode()
5966 case ISD::MERGE_VALUES: in getNode()
5967 case ISD::CONCAT_VECTORS: in getNode()
5969 case ISD::BUILD_VECTOR: { in getNode()
5976 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); in getNode()
5977 case ISD::FP_EXTEND: in getNode()
5988 case ISD::FP_TO_SINT: in getNode()
5989 case ISD::FP_TO_UINT: in getNode()
5993 case ISD::SINT_TO_FP: in getNode()
5994 case ISD::UINT_TO_FP: in getNode()
5999 case ISD::SIGN_EXTEND: in getNode()
6010 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) { in getNode()
6012 if (OpOpcode == ISD::ZERO_EXTEND) in getNode()
6016 if (OpOpcode == ISD::UNDEF) in getNode()
6020 case ISD::ZERO_EXTEND: in getNode()
6031 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x) in getNode()
6034 return getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0), Flags); in getNode()
6036 if (OpOpcode == ISD::UNDEF) in getNode()
6044 if (OpOpcode == ISD::TRUNCATE) { in getNode()
6047 if (OpOp.getOpcode() != ISD::AND) { in getNode()
6058 case ISD::ANY_EXTEND: in getNode()
6070 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || in getNode()
6071 OpOpcode == ISD::ANY_EXTEND) { in getNode()
6073 if (OpOpcode == ISD::ZERO_EXTEND) in getNode()
6078 if (OpOpcode == ISD::UNDEF) in getNode()
6082 if (OpOpcode == ISD::TRUNCATE) { in getNode()
6090 case ISD::TRUNCATE: in getNode()
6101 if (OpOpcode == ISD::TRUNCATE) in getNode()
6102 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0)); in getNode()
6103 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || in getNode()
6104 OpOpcode == ISD::ANY_EXTEND) { in getNode()
6110 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0)); in getNode()
6113 if (OpOpcode == ISD::UNDEF) in getNode()
6115 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes) in getNode()
6119 case ISD::ANY_EXTEND_VECTOR_INREG: in getNode()
6120 case ISD::ZERO_EXTEND_VECTOR_INREG: in getNode()
6121 case ISD::SIGN_EXTEND_VECTOR_INREG: in getNode()
6129 case ISD::ABS: in getNode()
6131 if (OpOpcode == ISD::UNDEF) in getNode()
6134 case ISD::BSWAP: in getNode()
6138 if (OpOpcode == ISD::UNDEF) in getNode()
6141 if (OpOpcode == ISD::BSWAP) in getNode()
6144 case ISD::BITREVERSE: in getNode()
6146 if (OpOpcode == ISD::UNDEF) in getNode()
6149 case ISD::BITCAST: in getNode()
6153 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) in getNode()
6154 return getNode(ISD::BITCAST, DL, VT, N1.getOperand(0)); in getNode()
6155 if (OpOpcode == ISD::UNDEF) in getNode()
6158 case ISD::SCALAR_TO_VECTOR: in getNode()
6165 if (OpOpcode == ISD::UNDEF) in getNode()
6168 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && in getNode()
6174 case ISD::FNEG: in getNode()
6176 if (OpOpcode == ISD::UNDEF) in getNode()
6179 if (OpOpcode == ISD::FNEG) // --X -> X in getNode()
6182 case ISD::FABS: in getNode()
6183 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) in getNode()
6184 return getNode(ISD::FABS, DL, VT, N1.getOperand(0)); in getNode()
6186 case ISD::VSCALE: in getNode()
6189 case ISD::CTPOP: in getNode()
6193 case ISD::CTLZ: in getNode()
6194 case ISD::CTTZ: in getNode()
6198 case ISD::VECREDUCE_ADD: in getNode()
6200 return getNode(ISD::VECREDUCE_XOR, DL, VT, N1); in getNode()
6202 case ISD::VECREDUCE_SMIN: in getNode()
6203 case ISD::VECREDUCE_UMAX: in getNode()
6205 return getNode(ISD::VECREDUCE_OR, DL, VT, N1); in getNode()
6207 case ISD::VECREDUCE_SMAX: in getNode()
6208 case ISD::VECREDUCE_UMIN: in getNode()
6210 return getNode(ISD::VECREDUCE_AND, DL, VT, N1); in getNode()
6212 case ISD::SPLAT_VECTOR: in getNode()
6255 case ISD::ADD: return C1 + C2; in FoldValue()
6256 case ISD::SUB: return C1 - C2; in FoldValue()
6257 case ISD::MUL: return C1 * C2; in FoldValue()
6258 case ISD::AND: return C1 & C2; in FoldValue()
6259 case ISD::OR: return C1 | C2; in FoldValue()
6260 case ISD::XOR: return C1 ^ C2; in FoldValue()
6261 case ISD::SHL: return C1 << C2; in FoldValue()
6262 case ISD::SRL: return C1.lshr(C2); in FoldValue()
6263 case ISD::SRA: return C1.ashr(C2); in FoldValue()
6264 case ISD::ROTL: return C1.rotl(C2); in FoldValue()
6265 case ISD::ROTR: return C1.rotr(C2); in FoldValue()
6266 case ISD::SMIN: return C1.sle(C2) ? C1 : C2; in FoldValue()
6267 case ISD::SMAX: return C1.sge(C2) ? C1 : C2; in FoldValue()
6268 case ISD::UMIN: return C1.ule(C2) ? C1 : C2; in FoldValue()
6269 case ISD::UMAX: return C1.uge(C2) ? C1 : C2; in FoldValue()
6270 case ISD::SADDSAT: return C1.sadd_sat(C2); in FoldValue()
6271 case ISD::UADDSAT: return C1.uadd_sat(C2); in FoldValue()
6272 case ISD::SSUBSAT: return C1.ssub_sat(C2); in FoldValue()
6273 case ISD::USUBSAT: return C1.usub_sat(C2); in FoldValue()
6274 case ISD::SSHLSAT: return C1.sshl_sat(C2); in FoldValue()
6275 case ISD::USHLSAT: return C1.ushl_sat(C2); in FoldValue()
6276 case ISD::UDIV: in FoldValue()
6280 case ISD::UREM: in FoldValue()
6284 case ISD::SDIV: in FoldValue()
6288 case ISD::SREM: in FoldValue()
6292 case ISD::AVGFLOORS: in FoldValue()
6294 case ISD::AVGFLOORU: in FoldValue()
6296 case ISD::AVGCEILS: in FoldValue()
6298 case ISD::AVGCEILU: in FoldValue()
6300 case ISD::ABDS: in FoldValue()
6302 case ISD::ABDU: in FoldValue()
6304 case ISD::MULHS: in FoldValue()
6306 case ISD::MULHU: in FoldValue()
6321 if (Opcode == ISD::AND || Opcode == ISD::MUL) in FoldValueWithUndef()
6330 if (GA->getOpcode() != ISD::GlobalAddress) in FoldSymbolOffset()
6339 case ISD::ADD: break; in FoldSymbolOffset()
6340 case ISD::SUB: Offset = -uint64_t(Offset); break; in FoldSymbolOffset()
6349 case ISD::SDIV: in isUndef()
6350 case ISD::UDIV: in isUndef()
6351 case ISD::SREM: in isUndef()
6352 case ISD::UREM: { in isUndef()
6360 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) && in isUndef()
6381 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS) in FoldConstantArithmetic()
6403 case ISD::SIGN_EXTEND: in FoldConstantArithmetic()
6406 case ISD::TRUNCATE: in FoldConstantArithmetic()
6410 case ISD::ZERO_EXTEND: in FoldConstantArithmetic()
6413 case ISD::ANY_EXTEND: in FoldConstantArithmetic()
6420 case ISD::ABS: in FoldConstantArithmetic()
6423 case ISD::BITREVERSE: in FoldConstantArithmetic()
6426 case ISD::BSWAP: in FoldConstantArithmetic()
6429 case ISD::CTPOP: in FoldConstantArithmetic()
6432 case ISD::CTLZ: in FoldConstantArithmetic()
6433 case ISD::CTLZ_ZERO_UNDEF: in FoldConstantArithmetic()
6436 case ISD::CTTZ: in FoldConstantArithmetic()
6437 case ISD::CTTZ_ZERO_UNDEF: in FoldConstantArithmetic()
6440 case ISD::UINT_TO_FP: in FoldConstantArithmetic()
6441 case ISD::SINT_TO_FP: { in FoldConstantArithmetic()
6444 (void)apf.convertFromAPInt(Val, Opcode == ISD::SINT_TO_FP, in FoldConstantArithmetic()
6448 case ISD::FP16_TO_FP: in FoldConstantArithmetic()
6449 case ISD::BF16_TO_FP: { in FoldConstantArithmetic()
6451 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf() in FoldConstantArithmetic()
6461 case ISD::STEP_VECTOR: in FoldConstantArithmetic()
6465 case ISD::BITCAST: in FoldConstantArithmetic()
6482 case ISD::FNEG: in FoldConstantArithmetic()
6485 case ISD::FABS: in FoldConstantArithmetic()
6488 case ISD::FCEIL: { in FoldConstantArithmetic()
6494 case ISD::FTRUNC: { in FoldConstantArithmetic()
6500 case ISD::FFLOOR: { in FoldConstantArithmetic()
6506 case ISD::FP_EXTEND: { in FoldConstantArithmetic()
6514 case ISD::FP_TO_SINT: in FoldConstantArithmetic()
6515 case ISD::FP_TO_UINT: { in FoldConstantArithmetic()
6517 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT); in FoldConstantArithmetic()
6525 case ISD::FP_TO_FP16: in FoldConstantArithmetic()
6526 case ISD::FP_TO_BF16: { in FoldConstantArithmetic()
6530 (void)V.convert(Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf() in FoldConstantArithmetic()
6535 case ISD::BITCAST: in FoldConstantArithmetic()
6552 if (Opcode == ISD::BITCAST) in FoldConstantArithmetic()
6595 (Ops[0].getOpcode() == ISD::BITCAST || in FoldConstantArithmetic()
6596 Ops[1].getOpcode() == ISD::BITCAST)) { in FoldConstantArithmetic()
6648 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) && in FoldConstantArithmetic()
6649 Ops[0].getOpcode() == ISD::STEP_VECTOR) { in FoldConstantArithmetic()
6651 if (ISD::isConstantSplatVector(Ops[1].getNode(), RHSVal)) { in FoldConstantArithmetic()
6652 APInt NewStep = Opcode == ISD::MUL in FoldConstantArithmetic()
6665 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE || in FoldConstantArithmetic()
6666 Op.getOpcode() == ISD::BUILD_VECTOR || in FoldConstantArithmetic()
6667 Op.getOpcode() == ISD::SPLAT_VECTOR; in FoldConstantArithmetic()
6680 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType()); in FoldConstantArithmetic()
6681 ISD::NodeType ExtendCode = in FoldConstantArithmetic()
6682 (Opcode == ISD::SETCC && SVT != VT.getScalarType()) in FoldConstantArithmetic()
6684 : ISD::SIGN_EXTEND; in FoldConstantArithmetic()
6706 if (Op.getOpcode() != ISD::BUILD_VECTOR && in FoldConstantArithmetic()
6707 Op.getOpcode() != ISD::SPLAT_VECTOR) { in FoldConstantArithmetic()
6716 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I); in FoldConstantArithmetic()
6731 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp); in FoldConstantArithmetic()
6745 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant && in FoldConstantArithmetic()
6746 ScalarResult.getOpcode() != ISD::ConstantFP) in FoldConstantArithmetic()
6775 case ISD::FADD: in foldConstantFPMath()
6778 case ISD::FSUB: in foldConstantFPMath()
6781 case ISD::FMUL: in foldConstantFPMath()
6784 case ISD::FDIV: in foldConstantFPMath()
6787 case ISD::FREM: in foldConstantFPMath()
6790 case ISD::FCOPYSIGN: in foldConstantFPMath()
6793 case ISD::FMINNUM: in foldConstantFPMath()
6795 case ISD::FMAXNUM: in foldConstantFPMath()
6797 case ISD::FMINIMUM: in foldConstantFPMath()
6799 case ISD::FMAXIMUM: in foldConstantFPMath()
6804 if (N1CFP && Opcode == ISD::FP_ROUND) { in foldConstantFPMath()
6815 case ISD::FSUB: in foldConstantFPMath()
6822 case ISD::FADD: in foldConstantFPMath()
6823 case ISD::FMUL: in foldConstantFPMath()
6824 case ISD::FDIV: in foldConstantFPMath()
6825 case ISD::FREM: in foldConstantFPMath()
6846 AddNodeIDNode(ID, ISD::AssertAlign, VTs, {Val}); in getAssertAlign()
6889 else if (N1.getOpcode() == ISD::SPLAT_VECTOR && in canonicalizeCommutativeBinop()
6890 N2.getOpcode() == ISD::STEP_VECTOR) in canonicalizeCommutativeBinop()
6896 assert(N1.getOpcode() != ISD::DELETED_NODE && in getNode()
6897 N2.getOpcode() != ISD::DELETED_NODE && in getNode()
6912 case ISD::TokenFactor: in getNode()
6916 if (N1.getOpcode() == ISD::EntryToken) return N2; in getNode()
6917 if (N2.getOpcode() == ISD::EntryToken) return N1; in getNode()
6920 case ISD::BUILD_VECTOR: { in getNode()
6927 case ISD::CONCAT_VECTORS: { in getNode()
6933 case ISD::AND: in getNode()
6944 case ISD::OR: in getNode()
6945 case ISD::XOR: in getNode()
6946 case ISD::ADD: in getNode()
6947 case ISD::SUB: in getNode()
6955 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() && in getNode()
6957 return getNode(ISD::XOR, DL, VT, N1, N2); in getNode()
6959 case ISD::MUL: in getNode()
6964 return getNode(ISD::AND, DL, VT, N1, N2); in getNode()
6965 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { in getNode()
6971 case ISD::UDIV: in getNode()
6972 case ISD::UREM: in getNode()
6973 case ISD::MULHU: in getNode()
6974 case ISD::MULHS: in getNode()
6975 case ISD::SDIV: in getNode()
6976 case ISD::SREM: in getNode()
6977 case ISD::SADDSAT: in getNode()
6978 case ISD::SSUBSAT: in getNode()
6979 case ISD::UADDSAT: in getNode()
6980 case ISD::USUBSAT: in getNode()
6986 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT) in getNode()
6987 return getNode(ISD::OR, DL, VT, N1, N2); in getNode()
6989 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT) in getNode()
6990 return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT)); in getNode()
6993 case ISD::SCMP: in getNode()
6994 case ISD::UCMP: in getNode()
7004 case ISD::AVGFLOORS: in getNode()
7005 case ISD::AVGFLOORU: in getNode()
7006 case ISD::AVGCEILS: in getNode()
7007 case ISD::AVGCEILU: in getNode()
7012 case ISD::ABDS: in getNode()
7013 case ISD::ABDU: in getNode()
7018 case ISD::SMIN: in getNode()
7019 case ISD::UMAX: in getNode()
7024 return getNode(ISD::OR, DL, VT, N1, N2); in getNode()
7026 case ISD::SMAX: in getNode()
7027 case ISD::UMIN: in getNode()
7032 return getNode(ISD::AND, DL, VT, N1, N2); in getNode()
7034 case ISD::FADD: in getNode()
7035 case ISD::FSUB: in getNode()
7036 case ISD::FMUL: in getNode()
7037 case ISD::FDIV: in getNode()
7038 case ISD::FREM: in getNode()
7045 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. in getNode()
7051 case ISD::SHL: in getNode()
7052 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { in getNode()
7058 case ISD::SRA: in getNode()
7059 case ISD::SRL: in getNode()
7063 case ISD::ROTL: in getNode()
7064 case ISD::ROTR: in getNode()
7087 case ISD::FP_ROUND: in getNode()
7095 case ISD::AssertSext: in getNode()
7096 case ISD::AssertZext: { in getNode()
7108 case ISD::SIGN_EXTEND_INREG: { in getNode()
7134 if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) { in getNode()
7150 if (N1.getOpcode() == ISD::SPLAT_VECTOR && in getNode()
7153 ISD::SPLAT_VECTOR, DL, VT, in getNode()
7158 case ISD::FP_TO_SINT_SAT: in getNode()
7159 case ISD::FP_TO_UINT_SAT: { in getNode()
7174 case ISD::EXTRACT_VECTOR_ELT: in getNode()
7194 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS && in getNode()
7198 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in getNode()
7205 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR || in getNode()
7206 N1.getOpcode() == ISD::SPLAT_VECTOR)) { in getNode()
7207 assert((N1.getOpcode() != ISD::BUILD_VECTOR || in getNode()
7211 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0; in getNode()
7225 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { in getNode()
7242 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); in getNode()
7254 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && in getNode()
7257 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), in getNode()
7261 case ISD::EXTRACT_ELEMENT: in getNode()
7271 if (N1.getOpcode() == ISD::BUILD_PAIR) in getNode()
7282 case ISD::EXTRACT_SUBVECTOR: { in getNode()
7312 if (N1.getOpcode() == ISD::CONCAT_VECTORS && in getNode()
7320 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) && in getNode()
7337 case ISD::SUB: in getNode()
7339 case ISD::SIGN_EXTEND_INREG: in getNode()
7340 case ISD::UDIV: in getNode()
7341 case ISD::SDIV: in getNode()
7342 case ISD::UREM: in getNode()
7343 case ISD::SREM: in getNode()
7344 case ISD::SSUBSAT: in getNode()
7345 case ISD::USUBSAT: in getNode()
7354 case ISD::XOR: in getNode()
7360 case ISD::ADD: in getNode()
7361 case ISD::SUB: in getNode()
7362 case ISD::UDIV: in getNode()
7363 case ISD::SDIV: in getNode()
7364 case ISD::UREM: in getNode()
7365 case ISD::SREM: in getNode()
7367 case ISD::MUL: in getNode()
7368 case ISD::AND: in getNode()
7369 case ISD::SSUBSAT: in getNode()
7370 case ISD::USUBSAT: in getNode()
7372 case ISD::OR: in getNode()
7373 case ISD::SADDSAT: in getNode()
7374 case ISD::UADDSAT: in getNode()
7418 assert(N1.getOpcode() != ISD::DELETED_NODE && in getNode()
7419 N2.getOpcode() != ISD::DELETED_NODE && in getNode()
7420 N3.getOpcode() != ISD::DELETED_NODE && in getNode()
7424 case ISD::FMA: in getNode()
7425 case ISD::FMAD: { in getNode()
7436 if (Opcode == ISD::FMAD) { in getNode()
7445 case ISD::BUILD_VECTOR: { in getNode()
7452 case ISD::CONCAT_VECTORS: { in getNode()
7458 case ISD::SETCC: { in getNode()
7478 case ISD::SELECT: in getNode()
7479 case ISD::VSELECT: in getNode()
7483 case ISD::VECTOR_SHUFFLE: in getNode()
7485 case ISD::VECTOR_SPLICE: { in getNode()
7490 case ISD::INSERT_VECTOR_ELT: { in getNode()
7509 case ISD::INSERT_SUBVECTOR: { in getNode()
7542 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR && in getNode()
7547 case ISD::BITCAST: in getNode()
7552 case ISD::VP_TRUNCATE: in getNode()
7553 case ISD::VP_SIGN_EXTEND: in getNode()
7554 case ISD::VP_ZERO_EXTEND: in getNode()
7559 case ISD::VECTOR_COMPRESS: { in getNode()
7636 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in getStackArgumentTokenFactor()
7663 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value); in getMemsetValue()
7668 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value, in getMemsetValue()
7695 return DAG.getNode(ISD::BITCAST, dl, VT, in getMemsetStringVal()
7746 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags); in getMemBasePlusOffset()
7753 if (Src.getOpcode() == ISD::GlobalAddress) in isMemSrcFromConstant()
7755 else if (Src.getOpcode() == ISD::ADD && in isMemSrcFromConstant()
7756 Src.getOperand(0).getOpcode() == ISD::GlobalAddress && in isMemSrcFromConstant()
7757 Src.getOperand(1).getOpcode() == ISD::Constant) { in isMemSrcFromConstant()
7790 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in chainLoadsAndStoresForMemcpy()
7942 ISD::EXTLOAD, dl, NVT, Chain, in getMemcpyLoadsAndStores()
8002 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in getMemcpyLoadsAndStores()
8093 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); in getMemmoveLoadsAndStores()
8108 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in getMemmoveLoadsAndStores()
8216 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); in getMemsetStores()
8225 SDValue TailValue = DAG.getNode(ISD::BITCAST, dl, SVT, MemSetValue); in getMemsetStores()
8226 Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, TailValue, in getMemsetStores()
8243 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in getMemsetStores()
8662 assert(Opcode == ISD::ATOMIC_CMP_SWAP || in getAtomicCmpSwap()
8663 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); in getAtomicCmpSwap()
8673 assert((Opcode == ISD::ATOMIC_LOAD_ADD || in getAtomic()
8674 Opcode == ISD::ATOMIC_LOAD_SUB || in getAtomic()
8675 Opcode == ISD::ATOMIC_LOAD_AND || in getAtomic()
8676 Opcode == ISD::ATOMIC_LOAD_CLR || in getAtomic()
8677 Opcode == ISD::ATOMIC_LOAD_OR || in getAtomic()
8678 Opcode == ISD::ATOMIC_LOAD_XOR || in getAtomic()
8679 Opcode == ISD::ATOMIC_LOAD_NAND || in getAtomic()
8680 Opcode == ISD::ATOMIC_LOAD_MIN || in getAtomic()
8681 Opcode == ISD::ATOMIC_LOAD_MAX || in getAtomic()
8682 Opcode == ISD::ATOMIC_LOAD_UMIN || in getAtomic()
8683 Opcode == ISD::ATOMIC_LOAD_UMAX || in getAtomic()
8684 Opcode == ISD::ATOMIC_LOAD_FADD || in getAtomic()
8685 Opcode == ISD::ATOMIC_LOAD_FSUB || in getAtomic()
8686 Opcode == ISD::ATOMIC_LOAD_FMAX || in getAtomic()
8687 Opcode == ISD::ATOMIC_LOAD_FMIN || in getAtomic()
8688 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP || in getAtomic()
8689 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP || in getAtomic()
8690 Opcode == ISD::ATOMIC_SWAP || in getAtomic()
8691 Opcode == ISD::ATOMIC_STORE) && in getAtomic()
8696 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : in getAtomic()
8705 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); in getAtomic()
8721 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops); in getMergeValues()
8743 assert((Opcode == ISD::INTRINSIC_VOID || in getMemIntrinsicNode()
8744 Opcode == ISD::INTRINSIC_W_CHAIN || in getMemIntrinsicNode()
8745 Opcode == ISD::PREFETCH || in getMemIntrinsicNode()
8747 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && in getMemIntrinsicNode()
8785 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END; in getLifetimeNode()
8815 const unsigned Opcode = ISD::PSEUDO_PROBE; in getPseudoProbeNode()
8849 if (Ptr.getOpcode() != ISD::ADD || in InferPointerInfo()
8875 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, in getLoad()
8899 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, in getLoad()
8904 ExtType = ISD::NON_EXTLOAD; in getLoad()
8905 } else if (ExtType == ISD::NON_EXTLOAD) { in getLoad()
8920 bool Indexed = AM != ISD::UNINDEXED; in getLoad()
8927 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops); in getLoad()
8955 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, in getLoad()
8962 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, in getLoad()
8966 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, in getExtLoad()
8973 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo, in getExtLoad()
8977 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, in getExtLoad()
8981 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, in getExtLoad()
8987 ISD::MemIndexedMode AM) { in getIndexedLoad()
9028 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); in getStore()
9031 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO)); in getStore()
9040 ISD::UNINDEXED, false, VT, MMO); in getStore()
9095 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); in getTruncStore()
9098 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO)); in getTruncStore()
9107 ISD::UNINDEXED, true, SVT, MMO); in getTruncStore()
9119 ISD::MemIndexedMode AM) { in getIndexedStore()
9125 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); in getIndexedStore()
9147 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, in getLoadVP()
9169 SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM, in getLoadVP()
9170 ISD::LoadExtType ExtType, EVT VT, in getLoadVP()
9175 bool Indexed = AM != ISD::UNINDEXED; in getLoadVP()
9182 AddNodeIDNode(ID, ISD::VP_LOAD, VTs, Ops); in getLoadVP()
9212 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, in getLoadVP()
9221 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, in getLoadVP()
9225 SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, in getExtLoadVP()
9233 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask, in getExtLoadVP()
9238 SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, in getExtLoadVP()
9243 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask, in getExtLoadVP()
9249 ISD::MemIndexedMode AM) { in getIndexedLoadVP()
9266 ISD::MemIndexedMode AM, bool IsTruncating, in getStoreVP()
9269 bool Indexed = AM != ISD::UNINDEXED; in getStoreVP()
9275 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); in getStoreVP()
9330 EVL, VT, MMO, ISD::UNINDEXED, in getTruncStoreVP()
9346 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); in getTruncStoreVP()
9349 dl.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO)); in getTruncStoreVP()
9359 ISD::UNINDEXED, true, IsCompressing, SVT, MMO); in getTruncStoreVP()
9371 ISD::MemIndexedMode AM) { in getIndexedStoreVP()
9378 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); in getIndexedStoreVP()
9400 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, in getStridedLoadVP()
9403 bool Indexed = AM != ISD::UNINDEXED; in getStridedLoadVP()
9410 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops); in getStridedLoadVP()
9439 return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr, in getStridedLoadVP()
9444 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, in getExtStridedLoadVP()
9448 return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef, in getExtStridedLoadVP()
9457 ISD::MemIndexedMode AM, in getStridedStoreVP()
9460 bool Indexed = AM != ISD::UNINDEXED; in getStridedStoreVP()
9466 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); in getStridedStoreVP()
9499 Stride, Mask, EVL, VT, MMO, ISD::UNINDEXED, in getTruncStridedStoreVP()
9515 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); in getTruncStridedStoreVP()
9518 DL.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO)); in getTruncStridedStoreVP()
9526 VTs, ISD::UNINDEXED, true, in getTruncStridedStoreVP()
9539 ISD::MemIndexType IndexType) { in getGatherVP()
9543 AddNodeIDNode(ID, ISD::VP_GATHER, VTs, Ops); in getGatherVP()
9583 ISD::MemIndexType IndexType) { in getScatterVP()
9587 AddNodeIDNode(ID, ISD::VP_SCATTER, VTs, Ops); in getScatterVP()
9628 ISD::MemIndexedMode AM, in getMaskedLoad()
9629 ISD::LoadExtType ExtTy, bool isExpanding) { in getMaskedLoad()
9630 bool Indexed = AM != ISD::UNINDEXED; in getMaskedLoad()
9637 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); in getMaskedLoad()
9661 ISD::MemIndexedMode AM) { in getIndexedMaskedLoad()
9674 ISD::MemIndexedMode AM, bool IsTruncating, in getMaskedStore()
9678 bool Indexed = AM != ISD::UNINDEXED; in getMaskedStore()
9685 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); in getMaskedStore()
9710 ISD::MemIndexedMode AM) { in getIndexedMaskedStore()
9722 ISD::MemIndexType IndexType, in getMaskedGather()
9723 ISD::LoadExtType ExtTy) { in getMaskedGather()
9727 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); in getMaskedGather()
9769 ISD::MemIndexType IndexType, in getMaskedScatter()
9774 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); in getMaskedScatter()
9815 ISD::MemIndexType IndexType) { in getMaskedHistogram()
9819 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTs, Ops); in getMaskedHistogram()
9856 AddNodeIDNode(ID, ISD::GET_FPENV_MEM, VTs, Ops); in getGetFPEnv()
9859 ISD::GET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO)); in getGetFPEnv()
9866 auto *N = newSDNode<FPStateAccessSDNode>(ISD::GET_FPENV_MEM, dl.getIROrder(), in getGetFPEnv()
9883 AddNodeIDNode(ID, ISD::SET_FPENV_MEM, VTs, Ops); in getSetFPEnv()
9886 ISD::SET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO)); in getSetFPEnv()
9893 auto *N = newSDNode<FPStateAccessSDNode>(ISD::SET_FPENV_MEM, dl.getIROrder(), in getSetFPEnv()
9952 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true)) in simplifyShift()
9984 if (Opcode == ISD::FADD) in simplifyFPBinop()
9989 if (Opcode == ISD::FSUB) in simplifyFPBinop()
9995 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV) in simplifyFPBinop()
10000 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros()) in simplifyFPBinop()
10010 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops); in getVAArg()
10050 assert(Op.getOpcode() != ISD::DELETED_NODE && in getNode()
10056 case ISD::BUILD_VECTOR: in getNode()
10061 case ISD::CONCAT_VECTORS: in getNode()
10065 case ISD::SELECT_CC: in getNode()
10079 case ISD::BR_CC: in getNode()
10084 case ISD::VP_ADD: in getNode()
10085 case ISD::VP_SUB: in getNode()
10088 Opcode = ISD::VP_XOR; in getNode()
10090 case ISD::VP_MUL: in getNode()
10093 Opcode = ISD::VP_AND; in getNode()
10095 case ISD::VP_REDUCE_MUL: in getNode()
10098 Opcode = ISD::VP_REDUCE_AND; in getNode()
10100 case ISD::VP_REDUCE_ADD: in getNode()
10103 Opcode = ISD::VP_REDUCE_XOR; in getNode()
10105 case ISD::VP_REDUCE_SMAX: in getNode()
10106 case ISD::VP_REDUCE_UMIN: in getNode()
10110 Opcode = ISD::VP_REDUCE_AND; in getNode()
10112 case ISD::VP_REDUCE_SMIN: in getNode()
10113 case ISD::VP_REDUCE_UMAX: in getNode()
10117 Opcode = ISD::VP_REDUCE_OR; in getNode()
10169 assert(Op.getOpcode() != ISD::DELETED_NODE && in getNode()
10174 case ISD::SADDO: in getNode()
10175 case ISD::UADDO: in getNode()
10176 case ISD::SSUBO: in getNode()
10177 case ISD::USUBO: { in getNode()
10192 return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags); in getNode()
10201 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO) in getNode()
10202 return getNode(ISD::MERGE_VALUES, DL, VTList, in getNode()
10203 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2), in getNode()
10204 getNode(ISD::AND, DL, VTList.VTs[1], F1, F2)}, in getNode()
10207 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) { in getNode()
10209 return getNode(ISD::MERGE_VALUES, DL, VTList, in getNode()
10210 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2), in getNode()
10211 getNode(ISD::AND, DL, VTList.VTs[1], NotF1, F2)}, in getNode()
10217 case ISD::SADDO_CARRY: in getNode()
10218 case ISD::UADDO_CARRY: in getNode()
10219 case ISD::SSUBO_CARRY: in getNode()
10220 case ISD::USUBO_CARRY: in getNode()
10229 case ISD::SMUL_LOHI: in getNode()
10230 case ISD::UMUL_LOHI: { in getNode()
10244 if (Opcode == ISD::SMUL_LOHI) { in getNode()
10256 return getNode(ISD::MERGE_VALUES, DL, VTList, {Lo, Hi}, Flags); in getNode()
10260 case ISD::FFREXP: { in getNode()
10272 return getNode(ISD::MERGE_VALUES, DL, VTList, {Result0, Result1}, Flags); in getNode()
10277 case ISD::STRICT_FP_EXTEND: in getNode()
10292 case ISD::STRICT_FP_ROUND: in getNode()
10312 case ISD::SRA_PARTS: in getNode()
10313 case ISD::SRL_PARTS: in getNode()
10314 case ISD::SHL_PARTS: in getNode()
10315 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && in getNode()
10318 else if (N3.getOpcode() == ISD::AND) in getNode()
10787 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break; in mutateStrictFPToFP()
10789 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break; in mutateStrictFPToFP()
11172 case ISD::ADD: { in salvageDebugInfo()
11241 case ISD::TRUNCATE: { in salvageDebugInfo()
11778 assert(AllNodes.front().getOpcode() == ISD::EntryToken && in AssignTopologicalOrder()
11817 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other, in makeEquivalentMemoryOrdering()
11896 case ISD::ADD: in isNeutralConstant()
11897 case ISD::OR: in isNeutralConstant()
11898 case ISD::XOR: in isNeutralConstant()
11899 case ISD::UMAX: in isNeutralConstant()
11901 case ISD::MUL: in isNeutralConstant()
11903 case ISD::AND: in isNeutralConstant()
11904 case ISD::UMIN: in isNeutralConstant()
11906 case ISD::SMAX: in isNeutralConstant()
11908 case ISD::SMIN: in isNeutralConstant()
11910 case ISD::SUB: in isNeutralConstant()
11911 case ISD::SHL: in isNeutralConstant()
11912 case ISD::SRA: in isNeutralConstant()
11913 case ISD::SRL: in isNeutralConstant()
11915 case ISD::UDIV: in isNeutralConstant()
11916 case ISD::SDIV: in isNeutralConstant()
11921 case ISD::FADD: in isNeutralConstant()
11924 case ISD::FSUB: in isNeutralConstant()
11927 case ISD::FMUL: in isNeutralConstant()
11929 case ISD::FDIV: in isNeutralConstant()
11931 case ISD::FMINNUM: in isNeutralConstant()
11932 case ISD::FMAXNUM: { in isNeutralConstant()
11941 if (Opcode == ISD::FMAXNUM) in isNeutralConstant()
11952 while (V.getOpcode() == ISD::BITCAST) in peekThroughBitcasts()
11958 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse()) in peekThroughOneUseBitcasts()
11964 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR) in peekThroughExtractSubvectors()
11970 while (V.getOpcode() == ISD::TRUNCATE) in peekThroughTruncates()
11976 if (V.getOpcode() != ISD::XOR) in isBitwiseNot()
12002 if (N->getOpcode() == ISD::SPLAT_VECTOR) { in isConstOrConstSplat()
12054 if (N.getOpcode() == ISD::SPLAT_VECTOR) in isConstOrConstSplatFP()
12223 if (getOpcode() == ISD::TokenFactor) { in reachesChainWithoutSideEffects()
12264 SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, in matchBinOpReduction()
12265 ArrayRef<ISD::NodeType> CandidateBinOps, in matchBinOpReduction()
12268 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT || in matchBinOpReduction()
12274 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) { in matchBinOpReduction()
12285 case ISD::FADD: in matchBinOpReduction()
12304 BinOp = (ISD::NodeType)CandidateBinOp; in matchBinOpReduction()
12305 return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op, in matchBinOpReduction()
12360 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR || in matchBinOpReduction()
12361 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR || in matchBinOpReduction()
12376 BinOp = (ISD::NodeType)CandidateBinOp; in matchBinOpReduction()
12407 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, in UnrollVectorOp()
12435 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, in UnrollVectorOp()
12449 case ISD::VSELECT: in UnrollVectorOp()
12450 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands)); in UnrollVectorOp()
12452 case ISD::SHL: in UnrollVectorOp()
12453 case ISD::SRA: in UnrollVectorOp()
12454 case ISD::SRL: in UnrollVectorOp()
12455 case ISD::ROTL: in UnrollVectorOp()
12456 case ISD::ROTR: in UnrollVectorOp()
12461 case ISD::SIGN_EXTEND_INREG: { in UnrollVectorOp()
12480 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO || in UnrollVectorOverflowOp()
12481 Opcode == ISD::USUBO || Opcode == ISD::SSUBO || in UnrollVectorOverflowOp()
12482 Opcode == ISD::UMULO || Opcode == ISD::SMULO) && in UnrollVectorOverflowOp()
12598 getNode(ISD::EXTRACT_ELEMENT, DL, LoVT, N, getIntPtrConstant(0, DL)); in SplitScalar()
12600 getNode(ISD::EXTRACT_ELEMENT, DL, HiVT, N, getIntPtrConstant(1, DL)); in SplitScalar()
12662 getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL)); in SplitVector()
12667 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, in SplitVector()
12684 SDValue Lo = getNode(ISD::UMIN, DL, VT, N, HalfNumElts); in SplitEVL()
12685 SDValue Hi = getNode(ISD::USUBSAT, DL, VT, N, HalfNumElts); in SplitEVL()
12694 return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N, in WidenVector()
12709 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op, in ExtractVectorElements()
13028 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP) in isConstant()
13087 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) in isConstantIntBuildVectorOrConstantInt()
13092 if (GA->getOpcode() == ISD::GlobalAddress && in isConstantIntBuildVectorOrConstantInt()
13095 if ((N.getOpcode() == ISD::SPLAT_VECTOR) && in isConstantIntBuildVectorOrConstantInt()
13107 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode())) in isConstantFPBuildVectorOrConstantFP()
13110 if ((N.getOpcode() == ISD::SPLAT_VECTOR) && in isConstantFPBuildVectorOrConstantFP()
13146 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs); in getTokenFactor()
13150 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals); in getTokenFactor()
13158 case ISD::ADD: in getNeutralElement()
13159 case ISD::OR: in getNeutralElement()
13160 case ISD::XOR: in getNeutralElement()
13161 case ISD::UMAX: in getNeutralElement()
13163 case ISD::MUL: in getNeutralElement()
13165 case ISD::AND: in getNeutralElement()
13166 case ISD::UMIN: in getNeutralElement()
13168 case ISD::SMAX: in getNeutralElement()
13170 case ISD::SMIN: in getNeutralElement()
13172 case ISD::FADD: in getNeutralElement()
13174 case ISD::FMUL: in getNeutralElement()
13176 case ISD::FMINNUM: in getNeutralElement()
13177 case ISD::FMAXNUM: { in getNeutralElement()
13183 if (Opcode == ISD::FMAXNUM) in getNeutralElement()
13188 case ISD::FMINIMUM: in getNeutralElement()
13189 case ISD::FMAXIMUM: { in getNeutralElement()
13194 if (Opcode == ISD::FMAXIMUM) in getNeutralElement()