Lines Matching refs:ISD
99 if (Value->getOpcode() == ISD::AssertZext) in parametersInCSRMatch()
101 if (Value->getOpcode() != ISD::CopyFromReg) in parametersInCSRMatch()
243 if (isOperationLegalOrCustom(ISD::STORE, NewVT) && in findOptimalMemOpLowering()
247 isOperationLegalOrCustom(ISD::STORE, MVT::f64) && in findOptimalMemOpLowering()
293 ISD::CondCode &CCCode, in softenSetCCOperands()
303 ISD::CondCode &CCCode, in softenSetCCOperands()
319 case ISD::SETEQ: in softenSetCCOperands()
320 case ISD::SETOEQ: in softenSetCCOperands()
325 case ISD::SETNE: in softenSetCCOperands()
326 case ISD::SETUNE: in softenSetCCOperands()
331 case ISD::SETGE: in softenSetCCOperands()
332 case ISD::SETOGE: in softenSetCCOperands()
337 case ISD::SETLT: in softenSetCCOperands()
338 case ISD::SETOLT: in softenSetCCOperands()
343 case ISD::SETLE: in softenSetCCOperands()
344 case ISD::SETOLE: in softenSetCCOperands()
349 case ISD::SETGT: in softenSetCCOperands()
350 case ISD::SETOGT: in softenSetCCOperands()
355 case ISD::SETO: in softenSetCCOperands()
358 case ISD::SETUO: in softenSetCCOperands()
363 case ISD::SETONE: in softenSetCCOperands()
367 case ISD::SETUEQ: in softenSetCCOperands()
379 case ISD::SETULT: in softenSetCCOperands()
384 case ISD::SETULE: in softenSetCCOperands()
389 case ISD::SETUGT: in softenSetCCOperands()
394 case ISD::SETUGE: in softenSetCCOperands()
433 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second, in softenSetCCOperands()
435 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl, in softenSetCCOperands()
485 return DAG.getNode(ISD::BRIND, dl, MVT::Other, Chain, Addr); in expandIndirectJTBranch()
533 case ISD::XOR: in ShrinkDemandedConstant()
534 case ISD::AND: in ShrinkDemandedConstant()
535 case ISD::OR: { in ShrinkDemandedConstant()
542 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C)) in ShrinkDemandedConstant()
610 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)), in ShrinkDemandedOp()
611 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1))); in ShrinkDemandedOp()
613 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, VT, X); in ShrinkDemandedOp()
692 case ISD::BITCAST: { in SimplifyMultipleUseDemandedBits()
751 case ISD::FREEZE: { in SimplifyMultipleUseDemandedBits()
758 case ISD::AND: { in SimplifyMultipleUseDemandedBits()
771 case ISD::OR: { in SimplifyMultipleUseDemandedBits()
784 case ISD::XOR: { in SimplifyMultipleUseDemandedBits()
796 case ISD::SHL: { in SimplifyMultipleUseDemandedBits()
811 case ISD::SETCC: { in SimplifyMultipleUseDemandedBits()
814 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in SimplifyMultipleUseDemandedBits()
826 if (CC == ISD::SETLT && Op1.getValueType().isInteger() && in SimplifyMultipleUseDemandedBits()
827 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode()))) in SimplifyMultipleUseDemandedBits()
832 case ISD::SIGN_EXTEND_INREG: { in SimplifyMultipleUseDemandedBits()
846 case ISD::ANY_EXTEND_VECTOR_INREG: in SimplifyMultipleUseDemandedBits()
847 case ISD::SIGN_EXTEND_VECTOR_INREG: in SimplifyMultipleUseDemandedBits()
848 case ISD::ZERO_EXTEND_VECTOR_INREG: { in SimplifyMultipleUseDemandedBits()
864 case ISD::INSERT_VECTOR_ELT: { in SimplifyMultipleUseDemandedBits()
877 case ISD::INSERT_SUBVECTOR: { in SimplifyMultipleUseDemandedBits()
891 case ISD::VECTOR_SHUFFLE: { in SimplifyMultipleUseDemandedBits()
921 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) in SimplifyMultipleUseDemandedBits()
959 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) && in combineShiftToAVG()
973 if (Add.getOpcode() != ISD::ADD) in combineShiftToAVG()
998 (ExtOpA.getOpcode() == ISD::ADD && in combineShiftToAVG()
1000 (ExtOpB.getOpcode() == ISD::ADD && in combineShiftToAVG()
1025 case ISD::SRA: { in combineShiftToAVG()
1038 case ISD::SRL: { in combineShiftToAVG()
1053 unsigned AVGOpc = IsCeil ? (IsSigned ? ISD::AVGCEILS : ISD::AVGCEILU) in combineShiftToAVG()
1054 : (IsSigned ? ISD::AVGFLOORS : ISD::AVGFLOORU); in combineShiftToAVG()
1127 if (Op.getOpcode() == ISD::TargetConstant) in SimplifyDemandedBits()
1130 if (Op.getOpcode() == ISD::Constant) { in SimplifyDemandedBits()
1136 if (Op.getOpcode() == ISD::ConstantFP) { in SimplifyDemandedBits()
1164 case ISD::SCALAR_TO_VECTOR: { in SimplifyDemandedBits()
1183 case ISD::BUILD_VECTOR: in SimplifyDemandedBits()
1188 case ISD::SPLAT_VECTOR: { in SimplifyDemandedBits()
1200 case ISD::LOAD: { in SimplifyDemandedBits()
1206 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { in SimplifyDemandedBits()
1215 case ISD::INSERT_VECTOR_ELT: { in SimplifyDemandedBits()
1253 case ISD::INSERT_SUBVECTOR: { in SimplifyDemandedBits()
1298 case ISD::EXTRACT_SUBVECTOR: { in SimplifyDemandedBits()
1325 case ISD::CONCAT_VECTORS: { in SimplifyDemandedBits()
1345 case ISD::VECTOR_SHUFFLE: { in SimplifyDemandedBits()
1388 case ISD::AND: { in SimplifyDemandedBits()
1417 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1424 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() && in SimplifyDemandedBits()
1426 ISD::isBuildVectorOfConstantSDNodes(Op0.getOperand(0).getNode())) && in SimplifyDemandedBits()
1437 TLO.DAG.getNode(ISD::AND, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1439 TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, NewAnd, in SimplifyDemandedBits()
1486 case ISD::OR: { in SimplifyDemandedBits()
1537 if (Op0.getOpcode() == ISD::AND && Op1.getOpcode() == ISD::AND && in SimplifyDemandedBits()
1545 if (Alt.getOpcode() == ISD::OR) { in SimplifyDemandedBits()
1549 if (SDValue C12 = TLO.DAG.FoldConstantArithmetic(ISD::OR, dl, VT, in SimplifyDemandedBits()
1551 SDValue MaskX = TLO.DAG.getNode(ISD::AND, dl, VT, X, C12); in SimplifyDemandedBits()
1552 SDValue MaskY = TLO.DAG.getNode(ISD::AND, dl, VT, Y, C2); in SimplifyDemandedBits()
1554 Op, TLO.DAG.getNode(ISD::OR, dl, VT, MaskX, MaskY)); in SimplifyDemandedBits()
1565 case ISD::XOR: { in SimplifyDemandedBits()
1590 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
1602 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC)); in SimplifyDemandedBits()
1615 if ((Op0Opcode == ISD::SRL || Op0Opcode == ISD::SHL) && Op0.hasOneUse()) { in SimplifyDemandedBits()
1623 Ones = Op0Opcode == ISD::SHL ? Ones.shl(ShiftAmt) in SimplifyDemandedBits()
1663 case ISD::SELECT: in SimplifyDemandedBits()
1678 case ISD::VSELECT: in SimplifyDemandedBits()
1689 case ISD::SELECT_CC: in SimplifyDemandedBits()
1704 case ISD::SETCC: { in SimplifyDemandedBits()
1707 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in SimplifyDemandedBits()
1719 if (CC == ISD::SETLT && Op1.getValueType().isInteger() && in SimplifyDemandedBits()
1720 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode()))) in SimplifyDemandedBits()
1732 case ISD::SHL: { in SimplifyDemandedBits()
1747 if (Op0.getOpcode() == ISD::SRL) { in SimplifyDemandedBits()
1752 unsigned Opc = ISD::SHL; in SimplifyDemandedBits()
1756 Opc = ISD::SRL; in SimplifyDemandedBits()
1768 if (Op0.getOpcode() == ISD::ANY_EXTEND) { in SimplifyDemandedBits()
1773 isTypeDesirableForOp(ISD::SHL, InnerVT)) { in SimplifyDemandedBits()
1775 ISD::SHL, dl, InnerVT, InnerOp, in SimplifyDemandedBits()
1778 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl)); in SimplifyDemandedBits()
1787 if (InnerOp.getOpcode() == ISD::SRL && Op0.hasOneUse() && in SimplifyDemandedBits()
1798 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, in SimplifyDemandedBits()
1801 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA)); in SimplifyDemandedBits()
1830 SDValue NewOp = TLO.DAG.getNode(ISD::SHL, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
1847 isTypeDesirableForOp(ISD::SHL, SmallVT) && in SimplifyDemandedBits()
1849 (!TLO.LegalOperations() || isOperationLegal(ISD::SHL, SmallVT))) { in SimplifyDemandedBits()
1854 ISD::SHL, dl, SmallVT, in SimplifyDemandedBits()
1855 TLO.DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)), in SimplifyDemandedBits()
1858 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl)); in SimplifyDemandedBits()
1871 isTypeDesirableForOp(ISD::SHL, HalfVT) && in SimplifyDemandedBits()
1873 (!TLO.LegalOperations() || isOperationLegal(ISD::SHL, HalfVT))) { in SimplifyDemandedBits()
1882 SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0); in SimplifyDemandedBits()
1885 SDValue NewShift = TLO.DAG.getNode(ISD::SHL, dl, HalfVT, NewOp, in SimplifyDemandedBits()
1888 TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift); in SimplifyDemandedBits()
1928 case ISD::SRL: { in SimplifyDemandedBits()
1943 if (Op0.getOpcode() == ISD::SHL) { in SimplifyDemandedBits()
1948 unsigned Opc = ISD::SRL; in SimplifyDemandedBits()
1952 Opc = ISD::SHL; in SimplifyDemandedBits()
1974 isTypeDesirableForOp(ISD::SRL, HalfVT) && in SimplifyDemandedBits()
1976 (!TLO.LegalOperations() || isOperationLegal(ISD::SRL, HalfVT)) && in SimplifyDemandedBits()
1979 SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0); in SimplifyDemandedBits()
1983 TLO.DAG.getNode(ISD::SRL, dl, HalfVT, NewOp, NewShiftAmt); in SimplifyDemandedBits()
1985 Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift)); in SimplifyDemandedBits()
2003 SDValue NewOp = TLO.DAG.getNode(ISD::SRL, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
2020 case ISD::SRA: { in SimplifyDemandedBits()
2037 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2047 if (Op0.getOpcode() == ISD::SHL) { in SimplifyDemandedBits()
2058 getOperationAction(ISD::SIGN_EXTEND_INREG, ExtVT) == Legal) in SimplifyDemandedBits()
2060 Op, TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, in SimplifyDemandedBits()
2099 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags)); in SimplifyDemandedBits()
2106 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA)); in SimplifyDemandedBits()
2118 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
2131 case ISD::FSHL: in SimplifyDemandedBits()
2132 case ISD::FSHR: { in SimplifyDemandedBits()
2136 bool IsFSHL = (Op.getOpcode() == ISD::FSHL); in SimplifyDemandedBits()
2193 case ISD::ROTL: in SimplifyDemandedBits()
2194 case ISD::ROTR: { in SimplifyDemandedBits()
2197 bool IsROTL = (Op.getOpcode() == ISD::ROTL); in SimplifyDemandedBits()
2219 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) && in SimplifyDemandedBits()
2222 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2224 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) && in SimplifyDemandedBits()
2227 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2240 case ISD::SMIN: in SimplifyDemandedBits()
2241 case ISD::SMAX: in SimplifyDemandedBits()
2242 case ISD::UMIN: in SimplifyDemandedBits()
2243 case ISD::UMAX: { in SimplifyDemandedBits()
2250 (Opc == ISD::SMIN || Opc == ISD::UMAX) ? ISD::OR : ISD::AND; in SimplifyDemandedBits()
2262 case ISD::SMIN: in SimplifyDemandedBits()
2269 case ISD::SMAX: in SimplifyDemandedBits()
2276 case ISD::UMIN: in SimplifyDemandedBits()
2283 case ISD::UMAX: in SimplifyDemandedBits()
2293 case ISD::BITREVERSE: { in SimplifyDemandedBits()
2303 case ISD::BSWAP: { in SimplifyDemandedBits()
2320 unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL; in SimplifyDemandedBits()
2337 case ISD::CTPOP: { in SimplifyDemandedBits()
2342 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, in SimplifyDemandedBits()
2348 case ISD::SIGN_EXTEND_INREG: { in SimplifyDemandedBits()
2366 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt)); in SimplifyDemandedBits()
2401 case ISD::BUILD_PAIR: { in SimplifyDemandedBits()
2419 case ISD::ZERO_EXTEND_VECTOR_INREG: in SimplifyDemandedBits()
2423 case ISD::ZERO_EXTEND: { in SimplifyDemandedBits()
2428 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG; in SimplifyDemandedBits()
2439 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND; in SimplifyDemandedBits()
2464 case ISD::SIGN_EXTEND_VECTOR_INREG: in SimplifyDemandedBits()
2468 case ISD::SIGN_EXTEND: { in SimplifyDemandedBits()
2473 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG; in SimplifyDemandedBits()
2495 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND; in SimplifyDemandedBits()
2512 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND; in SimplifyDemandedBits()
2527 case ISD::ANY_EXTEND_VECTOR_INREG: in SimplifyDemandedBits()
2531 case ISD::ANY_EXTEND: { in SimplifyDemandedBits()
2536 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG; in SimplifyDemandedBits()
2558 case ISD::TRUNCATE: { in SimplifyDemandedBits()
2573 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc)); in SimplifyDemandedBits()
2580 case ISD::SRL: in SimplifyDemandedBits()
2583 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT)) in SimplifyDemandedBits()
2615 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0)); in SimplifyDemandedBits()
2617 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt)); in SimplifyDemandedBits()
2625 case ISD::AssertZext: { in SimplifyDemandedBits()
2638 case ISD::EXTRACT_VECTOR_ELT: { in SimplifyDemandedBits()
2679 case ISD::BITCAST: { in SimplifyDemandedBits()
2691 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT); in SimplifyDemandedBits()
2692 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32); in SimplifyDemandedBits()
2699 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src); in SimplifyDemandedBits()
2702 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign); in SimplifyDemandedBits()
2706 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt)); in SimplifyDemandedBits()
2781 case ISD::MUL: in SimplifyDemandedBits()
2790 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, Op.getOperand(0), AmtC); in SimplifyDemandedBits()
2799 SDValue And1 = TLO.DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), One); in SimplifyDemandedBits()
2803 case ISD::ADD: in SimplifyDemandedBits()
2804 case ISD::SUB: { in SimplifyDemandedBits()
2814 if (Op.getOpcode() == ISD::MUL) in SimplifyDemandedBits()
2835 if (Op.getOpcode() == ISD::SUB && DemandedBits.isOne() && in SimplifyDemandedBits()
2878 if (Mul.getOpcode() != ISD::MUL || !Mul.hasOneUse()) in SimplifyDemandedBits()
2893 auto foldMul = [&](ISD::NodeType NT, SDValue X, SDValue Y, in SimplifyDemandedBits()
2896 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, X, ShlAmtC); in SimplifyDemandedBits()
2901 if (isOperationLegalOrCustom(ISD::SHL, VT)) { in SimplifyDemandedBits()
2902 if (Op.getOpcode() == ISD::ADD) { in SimplifyDemandedBits()
2905 return foldMul(ISD::SUB, Op0.getOperand(0), Op1, ShAmt); in SimplifyDemandedBits()
2908 return foldMul(ISD::SUB, Op1.getOperand(0), Op0, ShAmt); in SimplifyDemandedBits()
2910 if (Op.getOpcode() == ISD::SUB) { in SimplifyDemandedBits()
2913 return foldMul(ISD::ADD, Op1.getOperand(0), Op0, ShAmt); in SimplifyDemandedBits()
2917 if (Op.getOpcode() == ISD::MUL) { in SimplifyDemandedBits()
2921 Op.getOpcode() == ISD::ADD, Flags.hasNoSignedWrap(), in SimplifyDemandedBits()
2928 if (Op.getOpcode() >= ISD::BUILTIN_OP_END || in SimplifyDemandedBits()
2929 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN) { in SimplifyDemandedBits()
3105 case ISD::SCALAR_TO_VECTOR: { in SimplifyDemandedVectorElts()
3111 if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in SimplifyDemandedVectorElts()
3134 case ISD::BITCAST: { in SimplifyDemandedVectorElts()
3228 case ISD::FREEZE: { in SimplifyDemandedVectorElts()
3236 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && DemandedElts == 1) in SimplifyDemandedVectorElts()
3238 Op, TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, in SimplifyDemandedVectorElts()
3242 case ISD::BUILD_VECTOR: { in SimplifyDemandedVectorElts()
3272 case ISD::CONCAT_VECTORS: { in SimplifyDemandedVectorElts()
3307 case ISD::INSERT_SUBVECTOR: { in SimplifyDemandedVectorElts()
3325 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in SimplifyDemandedVectorElts()
3351 case ISD::EXTRACT_SUBVECTOR: { in SimplifyDemandedVectorElts()
3379 case ISD::INSERT_VECTOR_ELT: { in SimplifyDemandedVectorElts()
3410 case ISD::VSELECT: { in SimplifyDemandedVectorElts()
3447 case ISD::VECTOR_SHUFFLE: { in SimplifyDemandedVectorElts()
3521 case ISD::ANY_EXTEND_VECTOR_INREG: in SimplifyDemandedVectorElts()
3522 case ISD::SIGN_EXTEND_VECTOR_INREG: in SimplifyDemandedVectorElts()
3523 case ISD::ZERO_EXTEND_VECTOR_INREG: { in SimplifyDemandedVectorElts()
3534 if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG && in SimplifyDemandedVectorElts()
3541 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) { in SimplifyDemandedVectorElts()
3549 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND && in SimplifyDemandedVectorElts()
3560 ISD::AND, DL, SrcVT, {Src.getOperand(1), Mask})) { in SimplifyDemandedVectorElts()
3561 Fold = TLO.DAG.getNode(ISD::AND, DL, SrcVT, Src.getOperand(0), Fold); in SimplifyDemandedVectorElts()
3571 case ISD::ADD: { in SimplifyDemandedVectorElts()
3582 case ISD::AVGCEILS: in SimplifyDemandedVectorElts()
3583 case ISD::AVGCEILU: in SimplifyDemandedVectorElts()
3584 case ISD::AVGFLOORS: in SimplifyDemandedVectorElts()
3585 case ISD::AVGFLOORU: in SimplifyDemandedVectorElts()
3586 case ISD::OR: in SimplifyDemandedVectorElts()
3587 case ISD::XOR: in SimplifyDemandedVectorElts()
3588 case ISD::SUB: in SimplifyDemandedVectorElts()
3589 case ISD::FADD: in SimplifyDemandedVectorElts()
3590 case ISD::FSUB: in SimplifyDemandedVectorElts()
3591 case ISD::FMUL: in SimplifyDemandedVectorElts()
3592 case ISD::FDIV: in SimplifyDemandedVectorElts()
3593 case ISD::FREM: { in SimplifyDemandedVectorElts()
3616 case ISD::SHL: in SimplifyDemandedVectorElts()
3617 case ISD::SRL: in SimplifyDemandedVectorElts()
3618 case ISD::SRA: in SimplifyDemandedVectorElts()
3619 case ISD::ROTL: in SimplifyDemandedVectorElts()
3620 case ISD::ROTR: { in SimplifyDemandedVectorElts()
3643 case ISD::MUL: in SimplifyDemandedVectorElts()
3644 case ISD::MULHU: in SimplifyDemandedVectorElts()
3645 case ISD::MULHS: in SimplifyDemandedVectorElts()
3646 case ISD::AND: { in SimplifyDemandedVectorElts()
3684 case ISD::TRUNCATE: in SimplifyDemandedVectorElts()
3685 case ISD::SIGN_EXTEND: in SimplifyDemandedVectorElts()
3686 case ISD::ZERO_EXTEND: in SimplifyDemandedVectorElts()
3691 if (Op.getOpcode() == ISD::ZERO_EXTEND) { in SimplifyDemandedVectorElts()
3699 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) { in SimplifyDemandedVectorElts()
3730 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in computeKnownBitsForTargetNode()
3731 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in computeKnownBitsForTargetNode()
3732 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in computeKnownBitsForTargetNode()
3733 Op.getOpcode() == ISD::INTRINSIC_VOID) && in computeKnownBitsForTargetNode()
3764 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in ComputeNumSignBitsForTargetNode()
3765 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in ComputeNumSignBitsForTargetNode()
3766 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in ComputeNumSignBitsForTargetNode()
3767 Op.getOpcode() == ISD::INTRINSIC_VOID) && in ComputeNumSignBitsForTargetNode()
3782 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in SimplifyDemandedVectorEltsForTargetNode()
3783 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in SimplifyDemandedVectorEltsForTargetNode()
3784 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in SimplifyDemandedVectorEltsForTargetNode()
3785 Op.getOpcode() == ISD::INTRINSIC_VOID) && in SimplifyDemandedVectorEltsForTargetNode()
3794 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in SimplifyDemandedBitsForTargetNode()
3795 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in SimplifyDemandedBitsForTargetNode()
3796 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in SimplifyDemandedBitsForTargetNode()
3797 Op.getOpcode() == ISD::INTRINSIC_VOID) && in SimplifyDemandedBitsForTargetNode()
3808 (Op.getOpcode() >= ISD::BUILTIN_OP_END || in SimplifyMultipleUseDemandedBitsForTargetNode()
3809 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in SimplifyMultipleUseDemandedBitsForTargetNode()
3810 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in SimplifyMultipleUseDemandedBitsForTargetNode()
3811 Op.getOpcode() == ISD::INTRINSIC_VOID) && in SimplifyMultipleUseDemandedBitsForTargetNode()
3842 (Op.getOpcode() >= ISD::BUILTIN_OP_END || in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3843 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3844 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3845 Op.getOpcode() == ISD::INTRINSIC_VOID) && in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3862 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in canCreateUndefOrPoisonForTargetNode()
3863 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in canCreateUndefOrPoisonForTargetNode()
3864 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in canCreateUndefOrPoisonForTargetNode()
3865 Op.getOpcode() == ISD::INTRINSIC_VOID) && in canCreateUndefOrPoisonForTargetNode()
3876 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in isKnownNeverNaNForTargetNode()
3877 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in isKnownNeverNaNForTargetNode()
3878 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in isKnownNeverNaNForTargetNode()
3879 Op.getOpcode() == ISD::INTRINSIC_VOID) && in isKnownNeverNaNForTargetNode()
3890 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in isSplatValueForTargetNode()
3891 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in isSplatValueForTargetNode()
3892 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in isSplatValueForTargetNode()
3893 Op.getOpcode() == ISD::INTRINSIC_VOID) && in isSplatValueForTargetNode()
3977 ISD::CondCode Cond, const SDLoc &DL, in foldSetCCWithAnd()
3979 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND) in foldSetCCWithAnd()
3984 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() || in foldSetCCWithAnd()
3985 (Cond != ISD::SETEQ && Cond != ISD::SETNE)) in foldSetCCWithAnd()
3990 if (Cond == ISD::SETNE && isNullConstant(N1) && in foldSetCCWithAnd()
4015 Cond == ISD::SETEQ ? ISD::SETGE : ISD::SETLT); in foldSetCCWithAnd()
4044 Cond = ISD::getSetCCInverse(Cond, OpVT); in foldSetCCWithAnd()
4062 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y); in foldSetCCWithAnd()
4080 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI, in optimizeSetCCOfSignedTruncationCheck()
4088 if (N0->getOpcode() != ISD::ADD) in optimizeSetCCOfSignedTruncationCheck()
4103 ISD::CondCode NewCond; in optimizeSetCCOfSignedTruncationCheck()
4104 if (Cond == ISD::CondCode::SETULT) { in optimizeSetCCOfSignedTruncationCheck()
4105 NewCond = ISD::CondCode::SETEQ; in optimizeSetCCOfSignedTruncationCheck()
4106 } else if (Cond == ISD::CondCode::SETULE) { in optimizeSetCCOfSignedTruncationCheck()
4107 NewCond = ISD::CondCode::SETEQ; in optimizeSetCCOfSignedTruncationCheck()
4110 } else if (Cond == ISD::CondCode::SETUGT) { in optimizeSetCCOfSignedTruncationCheck()
4111 NewCond = ISD::CondCode::SETNE; in optimizeSetCCOfSignedTruncationCheck()
4114 } else if (Cond == ISD::CondCode::SETUGE) { in optimizeSetCCOfSignedTruncationCheck()
4115 NewCond = ISD::CondCode::SETNE; in optimizeSetCCOfSignedTruncationCheck()
4157 ISD::SIGN_EXTEND_INREG, DL, XVT, X, in optimizeSetCCOfSignedTruncationCheck()
4164 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond, in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4168 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4184 case ISD::SHL: in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4185 NewShiftOpcode = ISD::SRL; in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4187 case ISD::SRL: in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4188 NewShiftOpcode = ISD::SHL; in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4209 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse()) in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4227 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C); in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4236 ISD::CondCode Cond, const SDLoc &DL, in foldSetCCWithBinOp()
4239 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && in foldSetCCWithBinOp()
4241 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode"); in foldSetCCWithBinOp()
4258 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR) in foldSetCCWithBinOp()
4267 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One); in foldSetCCWithBinOp()
4275 ISD::CondCode Cond, const SDLoc &dl, in simplifySetCCWithCTPOP()
4280 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() && in simplifySetCCWithCTPOP()
4284 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse()) in simplifySetCCWithCTPOP()
4293 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) { in simplifySetCCWithCTPOP()
4299 if (C1.ugt(CostLimit + (Cond == ISD::SETULT))) in simplifySetCCWithCTPOP()
4301 if (C1 == 0 && (Cond == ISD::SETULT)) in simplifySetCCWithCTPOP()
4304 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT); in simplifySetCCWithCTPOP()
4309 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne); in simplifySetCCWithCTPOP()
4310 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add); in simplifySetCCWithCTPOP()
4312 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE; in simplifySetCCWithCTPOP()
4317 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) { in simplifySetCCWithCTPOP()
4325 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne); in simplifySetCCWithCTPOP()
4332 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add); in simplifySetCCWithCTPOP()
4339 SDValue Xor = DAG.getNode(ISD::XOR, dl, CTVT, CTOp, Add); in simplifySetCCWithCTPOP()
4340 ISD::CondCode CmpCond = Cond == ISD::SETEQ ? ISD::SETUGT : ISD::SETULE; in simplifySetCCWithCTPOP()
4348 ISD::CondCode Cond, const SDLoc &dl, in foldSetCCWithRotate()
4350 if (Cond != ISD::SETEQ && Cond != ISD::SETNE) in foldSetCCWithRotate()
4358 if (X.getOpcode() == ISD::ROTL || X.getOpcode() == ISD::ROTR) in foldSetCCWithRotate()
4376 if (N0.hasOneUse() && N0.getOpcode() == ISD::OR && C1->isZero()) { in foldSetCCWithRotate()
4378 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(1)); in foldSetCCWithRotate()
4382 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(0)); in foldSetCCWithRotate()
4391 ISD::CondCode Cond, const SDLoc &dl, in foldSetCCWithFunnelShift()
4395 if (Cond != ISD::SETEQ && Cond != ISD::SETNE) in foldSetCCWithFunnelShift()
4403 (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR)) in foldSetCCWithFunnelShift()
4413 if (N0.getOpcode() == ISD::FSHR) in foldSetCCWithFunnelShift()
4419 if (Or.getOpcode() != ISD::OR || !Or.hasOneUse()) in foldSetCCWithFunnelShift()
4441 SDValue Shift = DAG.getNode(ISD::SHL, dl, OpVT, Y, NewShAmt); in foldSetCCWithFunnelShift()
4442 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X); in foldSetCCWithFunnelShift()
4448 SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, Y, NewShAmt); in foldSetCCWithFunnelShift()
4449 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X); in foldSetCCWithFunnelShift()
4459 ISD::CondCode Cond, bool foldBooleans, in SimplifySetCC()
4480 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond); in SimplifySetCC()
4493 DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) && in SimplifySetCC()
4494 !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1})) in SimplifySetCC()
4516 if (C1.isZero() && (Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4517 N0.getOpcode() == ISD::MUL && N0.hasOneUse() && in SimplifySetCC()
4523 unsigned LogicOp = Cond == ISD::SETEQ ? ISD::OR : ISD::AND; in SimplifySetCC()
4530 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) && in SimplifySetCC()
4531 N0.getOperand(0).getOpcode() == ISD::CTLZ && in SimplifySetCC()
4534 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4536 if ((C1 == 0) == (Cond == ISD::SETEQ)) { in SimplifySetCC()
4539 Cond = ISD::SETNE; in SimplifySetCC()
4543 Cond = ISD::SETEQ; in SimplifySetCC()
4559 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4564 if (N0->getOpcode() == ISD::ZERO_EXTEND) { in SimplifySetCC()
4568 } else if (N0->getOpcode() == ISD::AND) { in SimplifySetCC()
4575 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) { in SimplifySetCC()
4582 if (LN0->getExtensionType() == ISD::ZEXTLOAD) { in SimplifySetCC()
4585 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) { in SimplifySetCC()
4600 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) { in SimplifySetCC()
4602 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt); in SimplifySetCC()
4606 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4621 bool SExt = (N0Opc == ISD::SIGN_EXTEND); in SimplifySetCC()
4623 TopSetCC.getOpcode() == ISD::SETCC && in SimplifySetCC()
4624 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) && in SimplifySetCC()
4628 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) || in SimplifySetCC()
4629 (!N1C->isZero() && Cond == ISD::SETNE); in SimplifySetCC()
4634 ISD::CondCode InvCond = ISD::getSetCCInverse( in SimplifySetCC()
4648 !ISD::isSignedIntSetCC(Cond) && in SimplifySetCC()
4649 N0.getOpcode() == ISD::AND && C1 == 0 && in SimplifySetCC()
4665 if (Lod->getExtensionType() != ISD::NON_EXTLOAD) in SimplifySetCC()
4672 if (!shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) in SimplifySetCC()
4710 DAG.getNode(ISD::AND, dl, newVT, NewLoad, in SimplifySetCC()
4717 if (N0.getOpcode() == ISD::ZERO_EXTEND) { in SimplifySetCC()
4725 case ISD::SETUGT: in SimplifySetCC()
4726 case ISD::SETUGE: in SimplifySetCC()
4727 case ISD::SETEQ: in SimplifySetCC()
4729 case ISD::SETULT: in SimplifySetCC()
4730 case ISD::SETULE: in SimplifySetCC()
4731 case ISD::SETNE: in SimplifySetCC()
4733 case ISD::SETGT: in SimplifySetCC()
4734 case ISD::SETGE: in SimplifySetCC()
4737 case ISD::SETLT: in SimplifySetCC()
4738 case ISD::SETLE: in SimplifySetCC()
4748 case ISD::SETEQ: in SimplifySetCC()
4749 case ISD::SETNE: in SimplifySetCC()
4750 case ISD::SETUGT: in SimplifySetCC()
4751 case ISD::SETUGE: in SimplifySetCC()
4752 case ISD::SETULT: in SimplifySetCC()
4753 case ISD::SETULE: { in SimplifySetCC()
4756 (isOperationLegal(ISD::SETCC, newVT) && in SimplifySetCC()
4770 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in SimplifySetCC()
4771 (Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4782 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT); in SimplifySetCC()
4787 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0), in SimplifySetCC()
4795 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { in SimplifySetCC()
4798 if ((N0.getOpcode() == ISD::SETCC || VT.getScalarType() != MVT::i1) && in SimplifySetCC()
4804 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne()); in SimplifySetCC()
4806 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in SimplifySetCC()
4808 if (N0.getOpcode() == ISD::SETCC) { in SimplifySetCC()
4809 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in SimplifySetCC()
4810 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType()); in SimplifySetCC()
4817 if ((N0.getOpcode() == ISD::XOR || in SimplifySetCC()
4818 (N0.getOpcode() == ISD::AND && in SimplifySetCC()
4819 N0.getOperand(0).getOpcode() == ISD::XOR && in SimplifySetCC()
4830 if (N0.getOpcode() == ISD::XOR) { in SimplifySetCC()
4833 assert(N0.getOpcode() == ISD::AND && in SimplifySetCC()
4834 N0.getOperand(0).getOpcode() == ISD::XOR); in SimplifySetCC()
4836 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(), in SimplifySetCC()
4842 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4846 if (Op0.getOpcode() == ISD::TRUNCATE) in SimplifySetCC()
4849 if ((Op0.getOpcode() == ISD::XOR) && in SimplifySetCC()
4850 Op0.getOperand(0).getOpcode() == ISD::SETCC && in SimplifySetCC()
4851 Op0.getOperand(1).getOpcode() == ISD::SETCC) { in SimplifySetCC()
4861 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ; in SimplifySetCC()
4865 if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) { in SimplifySetCC()
4868 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4869 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4872 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4873 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4878 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4880 if (Op0.getOpcode() == ISD::AssertZext && in SimplifySetCC()
4884 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4892 if (N0.getOpcode() == ISD::UREM && N1C->isZero() && in SimplifySetCC()
4893 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { in SimplifySetCC()
4902 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4903 N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) && in SimplifySetCC()
4908 Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE); in SimplifySetCC()
4923 if (ISD::isSignedIntSetCC(Cond)) { in SimplifySetCC()
4932 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) { in SimplifySetCC()
4940 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT; in SimplifySetCC()
4952 if (Cond == ISD::SETLE || Cond == ISD::SETULE) { in SimplifySetCC()
4960 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT; in SimplifySetCC()
4972 if (Cond == ISD::SETLT || Cond == ISD::SETULT) { in SimplifySetCC()
4980 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
4986 ISD::SETEQ); in SimplifySetCC()
4990 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) { in SimplifySetCC()
4998 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
5004 ISD::SETEQ); in SimplifySetCC()
5008 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) { in SimplifySetCC()
5025 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0) in SimplifySetCC()
5031 if (RHS.getOpcode() == ISD::SHL && in SimplifySetCC()
5039 if (LHS.getOpcode() == ISD::SHL && in SimplifySetCC()
5055 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits); in SimplifySetCC()
5057 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask); in SimplifySetCC()
5066 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) { in SimplifySetCC()
5083 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) || in SimplifySetCC()
5084 (Cond == ISD::SETUGE && C1.isMinSignedValue())) in SimplifySetCC()
5087 ISD::SETLT); in SimplifySetCC()
5091 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) || in SimplifySetCC()
5092 (Cond == ISD::SETULE && C1.isMaxSignedValue())) in SimplifySetCC()
5095 ISD::SETGT); in SimplifySetCC()
5112 (Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
5114 N0.getOpcode() == ISD::AND) { in SimplifySetCC()
5116 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3 in SimplifySetCC()
5122 ISD::TRUNCATE, dl, VT, in SimplifySetCC()
5123 DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
5126 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) { in SimplifySetCC()
5133 ISD::TRUNCATE, dl, VT, in SimplifySetCC()
5134 DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
5144 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
5145 N0.getOpcode() == ISD::AND && N0.hasOneUse()) { in SimplifySetCC()
5152 ISD::SRL, dl, ShValTy, N0.getOperand(0), in SimplifySetCC()
5159 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE || in SimplifySetCC()
5160 Cond == ISD::SETULE || Cond == ISD::SETUGT) { in SimplifySetCC()
5161 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT); in SimplifySetCC()
5168 ISD::CondCode NewCond = Cond; in SimplifySetCC()
5172 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE; in SimplifySetCC()
5181 DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
5198 if (Cond == ISD::SETO || Cond == ISD::SETUO) in SimplifySetCC()
5202 if (N0.getOpcode() == ISD::FNEG) { in SimplifySetCC()
5203 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond); in SimplifySetCC()
5206 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1); in SimplifySetCC()
5212 if (isOperationLegalOrCustom(ISD::IS_FPCLASS, N0.getValueType()) && in SimplifySetCC()
5214 bool IsFabs = N0.getOpcode() == ISD::FABS; in SimplifySetCC()
5216 if ((Cond == ISD::SETOEQ || Cond == ISD::SETUEQ) && CFP->isInfinity()) { in SimplifySetCC()
5219 if (Cond == ISD::SETUEQ) in SimplifySetCC()
5221 return DAG.getNode(ISD::IS_FPCLASS, dl, VT, Op, in SimplifySetCC()
5234 ISD::CondCode NewCond = ISD::SETCC_INVALID; in SimplifySetCC()
5236 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break; in SimplifySetCC()
5237 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break; in SimplifySetCC()
5238 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break; in SimplifySetCC()
5239 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break; in SimplifySetCC()
5242 if (NewCond != ISD::SETCC_INVALID && in SimplifySetCC()
5255 bool EqTrue = ISD::isTrueWhenEqual(Cond); in SimplifySetCC()
5256 unsigned UOF = ISD::getUnorderedFlavor(Cond); in SimplifySetCC()
5263 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO; in SimplifySetCC()
5288 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
5290 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB || in SimplifySetCC()
5291 N0.getOpcode() == ISD::XOR) { in SimplifySetCC()
5316 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) in SimplifySetCC()
5324 if (N0.getOpcode() == ISD::XOR && N0.getNode()->hasOneUse()) in SimplifySetCC()
5334 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) in SimplifySetCC()
5355 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB || in SimplifySetCC()
5356 N1.getOpcode() == ISD::XOR) in SimplifySetCC()
5365 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) && in SimplifySetCC()
5366 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { in SimplifySetCC()
5370 if (N0.getOpcode() == ISD::UREM) { in SimplifySetCC()
5373 } else if (N0.getOpcode() == ISD::SREM) { in SimplifySetCC()
5385 case ISD::SETEQ: // X == Y -> ~(X^Y) in SimplifySetCC()
5386 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1); in SimplifySetCC()
5391 case ISD::SETNE: // X != Y --> (X^Y) in SimplifySetCC()
5392 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1); in SimplifySetCC()
5394 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y in SimplifySetCC()
5395 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y in SimplifySetCC()
5397 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp); in SimplifySetCC()
5401 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X in SimplifySetCC()
5402 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X in SimplifySetCC()
5404 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp); in SimplifySetCC()
5408 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y in SimplifySetCC()
5409 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y in SimplifySetCC()
5411 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp); in SimplifySetCC()
5415 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X in SimplifySetCC()
5416 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X in SimplifySetCC()
5418 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp); in SimplifySetCC()
5425 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT)); in SimplifySetCC()
5448 if (N->getOpcode() == ISD::ADD) { in isGAPlusOffset()
5571 ISD::NodeType ExtOpc = in LowerAsmOperandForConstraint()
5572 IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND; in LowerAsmOperandForConstraint()
5574 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue(); in LowerAsmOperandForConstraint()
5598 if (OpCode == ISD::ADD || OpCode == ISD::SUB) { in LowerAsmOperandForConstraint()
5602 else if (OpCode == ISD::ADD && in LowerAsmOperandForConstraint()
5607 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue(); in LowerAsmOperandForConstraint()
6125 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern)) in BuildExactSDIV()
6129 if (Op1.getOpcode() == ISD::BUILD_VECTOR) { in BuildExactSDIV()
6132 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) { in BuildExactSDIV()
6148 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags); in BuildExactSDIV()
6152 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor); in BuildExactSDIV()
6188 if (!ISD::matchUnaryPredicate(Op1, BuildUDIVPattern)) in BuildExactUDIV()
6192 if (Op1.getOpcode() == ISD::BUILD_VECTOR) { in BuildExactUDIV()
6195 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) { in BuildExactUDIV()
6211 Res = DAG.getNode(ISD::SRL, dl, VT, Res, Shift, Flags); in BuildExactUDIV()
6215 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor); in BuildExactUDIV()
6261 SDValue Cmp = DAG.getSetCC(DL, CCVT, N0, Zero, ISD::SETLT); in buildSDIVPow2WithCMov()
6262 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in buildSDIVPow2WithCMov()
6263 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0); in buildSDIVPow2WithCMov()
6271 DAG.getNode(ISD::SRA, DL, VT, CMov, DAG.getConstant(Lg2, DL, VT)); in buildSDIVPow2WithCMov()
6279 return DAG.getNode(ISD::SUB, DL, VT, Zero, SRA); in buildSDIVPow2WithCMov()
6311 !isOperationLegal(ISD::MUL, MulVT)) in BuildSDIV()
6355 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern)) in BuildSDIV()
6359 if (N1.getOpcode() == ISD::BUILD_VECTOR) { in BuildSDIV()
6364 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) { in BuildSDIV()
6387 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X); in BuildSDIV()
6388 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y); in BuildSDIV()
6389 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y); in BuildSDIV()
6390 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y, in BuildSDIV()
6392 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
6395 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization)) in BuildSDIV()
6396 return DAG.getNode(ISD::MULHS, dl, VT, X, Y); in BuildSDIV()
6397 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) { in BuildSDIV()
6399 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildSDIV()
6408 if (isOperationLegalOrCustom(ISD::MUL, WideVT)) { in BuildSDIV()
6409 X = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, X); in BuildSDIV()
6410 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, Y); in BuildSDIV()
6411 Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y); in BuildSDIV()
6412 Y = DAG.getNode(ISD::SRL, dl, WideVT, Y, in BuildSDIV()
6414 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
6426 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor); in BuildSDIV()
6428 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor); in BuildSDIV()
6432 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift); in BuildSDIV()
6437 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift); in BuildSDIV()
6439 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask); in BuildSDIV()
6441 return DAG.getNode(ISD::ADD, dl, VT, Q, T); in BuildSDIV()
6473 !isOperationLegal(ISD::MUL, MulVT)) in BuildUDIV()
6535 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern)) in BuildUDIV()
6539 if (N1.getOpcode() == ISD::BUILD_VECTOR) { in BuildUDIV()
6544 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) { in BuildUDIV()
6561 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift); in BuildUDIV()
6570 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X); in BuildUDIV()
6571 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y); in BuildUDIV()
6572 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y); in BuildUDIV()
6573 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y, in BuildUDIV()
6575 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
6578 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization)) in BuildUDIV()
6579 return DAG.getNode(ISD::MULHU, dl, VT, X, Y); in BuildUDIV()
6580 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) { in BuildUDIV()
6582 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildUDIV()
6591 if (isOperationLegalOrCustom(ISD::MUL, WideVT)) { in BuildUDIV()
6592 X = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, X); in BuildUDIV()
6593 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, Y); in BuildUDIV()
6594 Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y); in BuildUDIV()
6595 Y = DAG.getNode(ISD::SRL, dl, WideVT, Y, in BuildUDIV()
6597 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
6610 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q); in BuildUDIV()
6618 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT)); in BuildUDIV()
6622 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q); in BuildUDIV()
6627 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift); in BuildUDIV()
6634 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ); in BuildUDIV()
6673 ISD::CondCode Cond, in buildUREMEqFold()
6689 SDValue CompTargetNode, ISD::CondCode Cond, in prepareUREMEqFold()
6697 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in prepareUREMEqFold()
6708 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareUREMEqFold()
6801 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern)) in prepareUREMEqFold()
6814 if (D.getOpcode() == ISD::BUILD_VECTOR) { in prepareUREMEqFold()
6828 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) { in prepareUREMEqFold()
6842 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT)) in prepareUREMEqFold()
6846 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode); in prepareUREMEqFold()
6850 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareUREMEqFold()
6857 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareUREMEqFold()
6860 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareUREMEqFold()
6867 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT)); in prepareUREMEqFold()
6882 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE); in prepareUREMEqFold()
6887 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) { in prepareUREMEqFold()
6890 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true, in prepareUREMEqFold()
6892 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels, in prepareUREMEqFold()
6899 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT)) in prepareUREMEqFold()
6900 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC, in prepareUREMEqFold()
6913 ISD::CondCode Cond, in buildSREMEqFold()
6930 SDValue CompTargetNode, ISD::CondCode Cond, in prepareSREMEqFold()
6955 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in prepareSREMEqFold()
6967 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareSREMEqFold()
7072 if (!ISD::matchUnaryPredicate(D, BuildSREMPattern)) in prepareSREMEqFold()
7085 if (D.getOpcode() == ISD::BUILD_VECTOR) { in prepareSREMEqFold()
7104 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) { in prepareSREMEqFold()
7122 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareSREMEqFold()
7127 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT)) in prepareSREMEqFold()
7131 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal); in prepareSREMEqFold()
7139 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareSREMEqFold()
7142 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareSREMEqFold()
7149 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT)); in prepareSREMEqFold()
7163 if (!isOperationLegalOrCustom(ISD::SETCC, SETCCVT) || in prepareSREMEqFold()
7164 !isOperationLegalOrCustom(ISD::AND, VT) || in prepareSREMEqFold()
7166 !isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) in prepareSREMEqFold()
7179 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ); in prepareSREMEqFold()
7183 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax); in prepareSREMEqFold()
7192 SDValue Blended = DAG.getNode(ISD::VSELECT, DL, SETCCVT, DivisorIsIntMin, in prepareSREMEqFold()
7221 return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ); in getSqrtInputTest()
7230 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op); in getSqrtInputTest()
7231 return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT); in getSqrtInputTest()
7239 if (Op.getOpcode() == ISD::FNEG || Op.getOpcode() == ISD::VP_FNEG) { in getNegatedExpression()
7256 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) { in getNegatedExpression()
7257 bool IsFreeExtend = Opcode == ISD::FP_EXTEND && in getNegatedExpression()
7276 case ISD::ConstantFP: { in getNegatedExpression()
7280 isOperationLegal(ISD::ConstantFP, VT) || in getNegatedExpression()
7298 case ISD::BUILD_VECTOR: { in getNegatedExpression()
7306 (isOperationLegal(ISD::ConstantFP, VT) && in getNegatedExpression()
7307 isOperationLegal(ISD::BUILD_VECTOR, VT)) || in getNegatedExpression()
7330 case ISD::FADD: { in getNegatedExpression()
7335 if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT)) in getNegatedExpression()
7358 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags); in getNegatedExpression()
7367 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags); in getNegatedExpression()
7374 case ISD::FSUB: { in getNegatedExpression()
7389 return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags); in getNegatedExpression()
7391 case ISD::FMUL: in getNegatedExpression()
7392 case ISD::FDIV: { in getNegatedExpression()
7422 if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL) in getNegatedExpression()
7435 case ISD::FMA: in getNegatedExpression()
7436 case ISD::FMAD: { in getNegatedExpression()
7487 case ISD::FP_EXTEND: in getNegatedExpression()
7488 case ISD::FSIN: in getNegatedExpression()
7493 case ISD::FP_ROUND: in getNegatedExpression()
7496 return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1)); in getNegatedExpression()
7498 case ISD::SELECT: in getNegatedExpression()
7499 case ISD::VSELECT: { in getNegatedExpression()
7548 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || in expandMUL_LOHI()
7549 Opcode == ISD::SMUL_LOHI); in expandMUL_LOHI()
7552 isOperationLegalOrCustom(ISD::MULHS, HiLoVT); in expandMUL_LOHI()
7554 isOperationLegalOrCustom(ISD::MULHU, HiLoVT); in expandMUL_LOHI()
7556 isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT); in expandMUL_LOHI()
7558 isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT); in expandMUL_LOHI()
7574 Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R); in expandMUL_LOHI()
7579 Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R); in expandMUL_LOHI()
7580 Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R); in expandMUL_LOHI()
7589 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) { in expandMUL_LOHI()
7590 LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS); in expandMUL_LOHI()
7591 RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS); in expandMUL_LOHI()
7604 if (Opcode != ISD::MUL) { in expandMUL_LOHI()
7613 if (!VT.isVector() && Opcode == ISD::MUL && in expandMUL_LOHI()
7629 isOperationLegalOrCustom(ISD::SRL, VT) && in expandMUL_LOHI()
7630 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) { in expandMUL_LOHI()
7631 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift); in expandMUL_LOHI()
7632 LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH); in expandMUL_LOHI()
7633 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift); in expandMUL_LOHI()
7634 RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH); in expandMUL_LOHI()
7645 if (Opcode == ISD::MUL) { in expandMUL_LOHI()
7646 RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH); in expandMUL_LOHI()
7647 LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL); in expandMUL_LOHI()
7648 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH); in expandMUL_LOHI()
7649 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH); in expandMUL_LOHI()
7656 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in expandMUL_LOHI()
7657 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7658 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in expandMUL_LOHI()
7659 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi); in expandMUL_LOHI()
7662 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7668 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7676 bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) && in expandMUL_LOHI()
7677 isOperationLegalOrCustom(ISD::ADDE, VT)); in expandMUL_LOHI()
7679 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next, in expandMUL_LOHI()
7682 Next = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(VT, BoolType), Next, in expandMUL_LOHI()
7686 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7687 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7689 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI)) in expandMUL_LOHI()
7693 Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero, in expandMUL_LOHI()
7696 Hi = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi, in expandMUL_LOHI()
7699 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7701 if (Opcode == ISD::SMUL_LOHI) { in expandMUL_LOHI()
7702 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7703 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL)); in expandMUL_LOHI()
7704 Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT); in expandMUL_LOHI()
7706 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7707 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL)); in expandMUL_LOHI()
7708 Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT); in expandMUL_LOHI()
7711 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7712 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7713 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7762 if (Opcode == ISD::SREM || Opcode == ISD::SDIV || Opcode == ISD::SDIVREM) in expandDIVREMByConstant()
7765 (Opcode == ISD::UREM || Opcode == ISD::UDIV || Opcode == ISD::UDIVREM) && in expandDIVREMByConstant()
7785 if (!isOperationLegalOrCustom(ISD::MULHU, HiLoVT) && in expandDIVREMByConstant()
7786 !isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT)) in expandDIVREMByConstant()
7821 if (Opcode != ISD::UDIV) { in expandDIVREMByConstant()
7823 PartialRem = DAG.getNode(ISD::AND, dl, HiLoVT, LL, in expandDIVREMByConstant()
7828 ISD::OR, dl, HiLoVT, in expandDIVREMByConstant()
7829 DAG.getNode(ISD::SRL, dl, HiLoVT, LL, in expandDIVREMByConstant()
7831 DAG.getNode(ISD::SHL, dl, HiLoVT, LH, in expandDIVREMByConstant()
7834 LH = DAG.getNode(ISD::SRL, dl, HiLoVT, LH, in expandDIVREMByConstant()
7841 if (isOperationLegalOrCustom(ISD::UADDO_CARRY, HiLoVT)) { in expandDIVREMByConstant()
7843 Sum = DAG.getNode(ISD::UADDO, dl, VTList, LL, LH); in expandDIVREMByConstant()
7844 Sum = DAG.getNode(ISD::UADDO_CARRY, dl, VTList, Sum, in expandDIVREMByConstant()
7847 Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, LL, LH); in expandDIVREMByConstant()
7848 SDValue Carry = DAG.getSetCC(dl, SetCCType, Sum, LL, ISD::SETULT); in expandDIVREMByConstant()
7857 Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, Sum, Carry); in expandDIVREMByConstant()
7867 DAG.getNode(ISD::UREM, dl, HiLoVT, Sum, in expandDIVREMByConstant()
7871 if (Opcode != ISD::UREM) { in expandDIVREMByConstant()
7873 SDValue Dividend = DAG.getNode(ISD::BUILD_PAIR, dl, VT, LL, LH); in expandDIVREMByConstant()
7874 SDValue Rem = DAG.getNode(ISD::BUILD_PAIR, dl, VT, RemL, RemH); in expandDIVREMByConstant()
7876 Dividend = DAG.getNode(ISD::SUB, dl, VT, Dividend, Rem); in expandDIVREMByConstant()
7882 SDValue Quotient = DAG.getNode(ISD::MUL, dl, VT, Dividend, in expandDIVREMByConstant()
7892 if (Opcode != ISD::UDIV) { in expandDIVREMByConstant()
7897 RemL = DAG.getNode(ISD::SHL, dl, HiLoVT, RemL, in expandDIVREMByConstant()
7899 RemL = DAG.getNode(ISD::ADD, dl, HiLoVT, RemL, PartialRem); in expandDIVREMByConstant()
7910 return ISD::matchUnaryPredicate( in isNonZeroModBitWidthOrUndef()
7927 bool IsFSHL = Node->getOpcode() == ISD::VP_FSHL; in expandVPFunnelShift()
7936 ShAmt = DAG.getNode(ISD::VP_UREM, DL, ShVT, Z, BitWidthC, Mask, VL); in expandVPFunnelShift()
7937 InvShAmt = DAG.getNode(ISD::VP_SUB, DL, ShVT, BitWidthC, ShAmt, Mask, VL); in expandVPFunnelShift()
7938 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt, Mask, in expandVPFunnelShift()
7940 ShY = DAG.getNode(ISD::VP_SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt, Mask, in expandVPFunnelShift()
7948 ShAmt = DAG.getNode(ISD::VP_AND, DL, ShVT, Z, BitMask, Mask, VL); in expandVPFunnelShift()
7950 SDValue NotZ = DAG.getNode(ISD::VP_XOR, DL, ShVT, Z, in expandVPFunnelShift()
7952 InvShAmt = DAG.getNode(ISD::VP_AND, DL, ShVT, NotZ, BitMask, Mask, VL); in expandVPFunnelShift()
7955 ShAmt = DAG.getNode(ISD::VP_UREM, DL, ShVT, Z, BitWidthC, Mask, VL); in expandVPFunnelShift()
7956 InvShAmt = DAG.getNode(ISD::VP_SUB, DL, ShVT, BitMask, ShAmt, Mask, VL); in expandVPFunnelShift()
7961 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, ShAmt, Mask, VL); in expandVPFunnelShift()
7962 SDValue ShY1 = DAG.getNode(ISD::VP_SRL, DL, VT, Y, One, Mask, VL); in expandVPFunnelShift()
7963 ShY = DAG.getNode(ISD::VP_SRL, DL, VT, ShY1, InvShAmt, Mask, VL); in expandVPFunnelShift()
7965 SDValue ShX1 = DAG.getNode(ISD::VP_SHL, DL, VT, X, One, Mask, VL); in expandVPFunnelShift()
7966 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, ShX1, InvShAmt, Mask, VL); in expandVPFunnelShift()
7967 ShY = DAG.getNode(ISD::VP_SRL, DL, VT, Y, ShAmt, Mask, VL); in expandVPFunnelShift()
7970 return DAG.getNode(ISD::VP_OR, DL, VT, ShX, ShY, Mask, VL); in expandVPFunnelShift()
7980 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandFunnelShift()
7981 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandFunnelShift()
7982 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandFunnelShift()
7983 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandFunnelShift()
7991 bool IsFSHL = Node->getOpcode() == ISD::FSHL; in expandFunnelShift()
7997 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL; in expandFunnelShift()
8004 Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z); in expandFunnelShift()
8011 X = DAG.getNode(ISD::SRL, DL, VT, X, One); in expandFunnelShift()
8014 Y = DAG.getNode(ISD::SHL, DL, VT, Y, One); in expandFunnelShift()
8028 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC); in expandFunnelShift()
8029 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt); in expandFunnelShift()
8030 ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt); in expandFunnelShift()
8031 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt); in expandFunnelShift()
8038 ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask); in expandFunnelShift()
8040 InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask); in expandFunnelShift()
8043 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC); in expandFunnelShift()
8044 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt); in expandFunnelShift()
8049 ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt); in expandFunnelShift()
8050 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One); in expandFunnelShift()
8051 ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt); in expandFunnelShift()
8053 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One); in expandFunnelShift()
8054 ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt); in expandFunnelShift()
8055 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt); in expandFunnelShift()
8058 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY); in expandFunnelShift()
8066 bool IsLeft = Node->getOpcode() == ISD::ROTL; in expandROT()
8075 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL; in expandROT()
8078 SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1); in expandROT()
8083 (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandROT()
8084 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandROT()
8085 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandROT()
8086 !isOperationLegalOrCustomOrPromote(ISD::OR, VT) || in expandROT()
8087 !isOperationLegalOrCustomOrPromote(ISD::AND, VT))) in expandROT()
8090 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL; in expandROT()
8091 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL; in expandROT()
8098 SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1); in expandROT()
8099 SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC); in expandROT()
8101 SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC); in expandROT()
8107 SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC); in expandROT()
8109 SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt); in expandROT()
8114 return DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal); in expandROT()
8124 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS; in expandShiftParts()
8125 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS; in expandShiftParts()
8137 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, in expandShiftParts()
8139 SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, in expandShiftParts()
8145 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
8146 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt); in expandShiftParts()
8148 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
8149 Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt); in expandShiftParts()
8155 SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, in expandShiftParts()
8158 DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE); in expandShiftParts()
8161 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
8162 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
8164 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
8165 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
8202 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src); in expandFP_TO_SINT()
8205 ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask), in expandFP_TO_SINT()
8207 SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias); in expandFP_TO_SINT()
8209 SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT, in expandFP_TO_SINT()
8210 DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask), in expandFP_TO_SINT()
8214 SDValue R = DAG.getNode(ISD::OR, dl, IntVT, in expandFP_TO_SINT()
8215 DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask), in expandFP_TO_SINT()
8222 DAG.getNode(ISD::SHL, dl, DstVT, R, in expandFP_TO_SINT()
8224 DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit), in expandFP_TO_SINT()
8226 DAG.getNode(ISD::SRL, dl, DstVT, R, in expandFP_TO_SINT()
8228 DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent), in expandFP_TO_SINT()
8230 ISD::SETGT); in expandFP_TO_SINT()
8232 SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT, in expandFP_TO_SINT()
8233 DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign); in expandFP_TO_SINT()
8236 DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT); in expandFP_TO_SINT()
8255 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT : in expandFP_TO_UINT()
8256 ISD::FP_TO_SINT; in expandFP_TO_UINT()
8258 !isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT))) in expandFP_TO_UINT()
8270 Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other }, in expandFP_TO_UINT()
8274 Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src); in expandFP_TO_UINT()
8280 Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT)) in expandFP_TO_UINT()
8287 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, in expandFP_TO_UINT()
8291 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT); in expandFP_TO_UINT()
8314 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other }, in expandFP_TO_UINT()
8316 SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other }, in expandFP_TO_UINT()
8320 SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs); in expandFP_TO_UINT()
8321 SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val); in expandFP_TO_UINT()
8323 Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs); in expandFP_TO_UINT()
8330 SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src); in expandFP_TO_UINT()
8332 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, in expandFP_TO_UINT()
8333 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst)); in expandFP_TO_UINT()
8334 False = DAG.getNode(ISD::XOR, dl, DstVT, False, in expandFP_TO_UINT()
8359 if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) || in expandUINT_TO_FP()
8360 !isOperationLegalOrCustom(ISD::FADD, DstVT) || in expandUINT_TO_FP()
8361 !isOperationLegalOrCustom(ISD::FSUB, DstVT) || in expandUINT_TO_FP()
8362 !isOperationLegalOrCustomOrPromote(ISD::OR, SrcVT) || in expandUINT_TO_FP()
8363 !isOperationLegalOrCustomOrPromote(ISD::AND, SrcVT))) in expandUINT_TO_FP()
8381 SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask); in expandUINT_TO_FP()
8382 SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift); in expandUINT_TO_FP()
8383 SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52); in expandUINT_TO_FP()
8384 SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84); in expandUINT_TO_FP()
8388 DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52); in expandUINT_TO_FP()
8389 Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub); in expandUINT_TO_FP()
8397 assert((Opcode == ISD::FMINNUM || Opcode == ISD::FMAXNUM || in createSelectForFMINNUM_FMAXNUM()
8398 Opcode == ISD::STRICT_FMINNUM || Opcode == ISD::STRICT_FMAXNUM) && in createSelectForFMINNUM_FMAXNUM()
8402 ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT; in createSelectForFMINNUM_FMAXNUM()
8420 unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ? in expandFMINNUM_FMAXNUM()
8421 ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE; in expandFMINNUM_FMAXNUM()
8436 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0, in expandFMINNUM_FMAXNUM()
8440 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1, in expandFMINNUM_FMAXNUM()
8458 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM; in expandFMINNUM_FMAXNUM()
8478 bool IsMax = Opc == ISD::FMAXIMUM; in expandFMINIMUM_FMAXIMUM()
8484 unsigned CompOpcIeee = IsMax ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE; in expandFMINIMUM_FMAXIMUM()
8485 unsigned CompOpc = IsMax ? ISD::FMAXNUM : ISD::FMINNUM; in expandFMINIMUM_FMAXIMUM()
8497 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandFMINIMUM_FMAXIMUM()
8502 DAG.getSetCC(DL, CCVT, LHS, RHS, IsMax ? ISD::SETGT : ISD::SETLT); in expandFMINIMUM_FMAXIMUM()
8511 MinMax = DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, LHS, RHS, ISD::SETUO), in expandFMINIMUM_FMAXIMUM()
8519 DAG.getConstantFP(0.0, DL, VT), ISD::SETEQ); in expandFMINIMUM_FMAXIMUM()
8523 DL, VT, DAG.getNode(ISD::IS_FPCLASS, DL, CCVT, LHS, TestZero), LHS, in expandFMINIMUM_FMAXIMUM()
8526 DL, VT, DAG.getNode(ISD::IS_FPCLASS, DL, CCVT, RHS, TestZero), RHS, in expandFMINIMUM_FMAXIMUM()
8574 Op = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::f64, Op, in expandIS_FPCLASS()
8596 isOperationLegalOrCustom(ISD::SETCC, OperandVT.getScalarType())) { in expandIS_FPCLASS()
8597 ISD::CondCode OrderedCmpOpcode = IsInverted ? ISD::SETUNE : ISD::SETOEQ; in expandIS_FPCLASS()
8598 ISD::CondCode UnorderedCmpOpcode = IsInverted ? ISD::SETONE : ISD::SETUEQ; in expandIS_FPCLASS()
8614 isCondCodeLegalOrCustom(IsInverted ? ISD::SETO : ISD::SETUO, in expandIS_FPCLASS()
8617 IsInverted ? ISD::SETO : ISD::SETUO); in expandIS_FPCLASS()
8621 isCondCodeLegalOrCustom(IsInverted ? ISD::SETUNE : ISD::SETOEQ, in expandIS_FPCLASS()
8623 isOperationLegalOrCustom(ISD::FABS, OperandVT.getScalarType())) { in expandIS_FPCLASS()
8625 SDValue Abs = DAG.getNode(ISD::FABS, DL, OperandVT, Op); in expandIS_FPCLASS()
8629 IsInverted ? ISD::SETUNE : ISD::SETOEQ); in expandIS_FPCLASS()
8665 Res = DAG.getNode(ISD::OR, DL, ResultVT, Res, PartialRes); in expandIS_FPCLASS()
8677 SDValue IntBitV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, IntBitMaskV); in expandIS_FPCLASS()
8678 IntBitIsSetV = DAG.getSetCC(DL, ResultVT, IntBitV, ZeroV, ISD::SETNE); in expandIS_FPCLASS()
8684 SDValue AbsV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ValueMaskV); in expandIS_FPCLASS()
8686 DAG.getConstant(0.0, DL, IntVT), ISD::SETLT); in expandIS_FPCLASS()
8696 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ExpMaskV, ISD::SETLT); in expandIS_FPCLASS()
8700 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ExpMaskV, ISD::SETULT); in expandIS_FPCLASS()
8704 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ExpMaskV, ISD::SETLT); in expandIS_FPCLASS()
8705 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV); in expandIS_FPCLASS()
8714 SDValue ExpBits = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ExpMaskV); in expandIS_FPCLASS()
8716 DAG.getSetCC(DL, ResultVT, ExpBits, ZeroV, ISD::SETEQ); in expandIS_FPCLASS()
8726 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ZeroV, ISD::SETEQ); in expandIS_FPCLASS()
8728 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ZeroV, ISD::SETEQ); in expandIS_FPCLASS()
8730 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, SignBitV, ISD::SETEQ); in expandIS_FPCLASS()
8740 DAG.getNode(ISD::SUB, DL, IntVT, V, DAG.getConstant(1, DL, IntVT)); in expandIS_FPCLASS()
8741 PartialRes = DAG.getSetCC(DL, ResultVT, VMinusOneV, MantissaV, ISD::SETULT); in expandIS_FPCLASS()
8743 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV); in expandIS_FPCLASS()
8749 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, InfV, ISD::SETEQ); in expandIS_FPCLASS()
8751 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETEQ); in expandIS_FPCLASS()
8755 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, NegInfV, ISD::SETEQ); in expandIS_FPCLASS()
8765 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT); in expandIS_FPCLASS()
8769 SDValue ExpBits = DAG.getNode(ISD::AND, DL, IntVT, AbsV, ExpMaskV); in expandIS_FPCLASS()
8771 DAG.getSetCC(DL, ResultVT, ExpBits, ZeroV, ISD::SETEQ); in expandIS_FPCLASS()
8773 DAG.getSetCC(DL, ResultVT, getIntBitIsSet(), ExpIsZero, ISD::SETEQ); in expandIS_FPCLASS()
8774 PartialRes = DAG.getNode(ISD::OR, DL, ResultVT, PartialRes, IsPseudo); in expandIS_FPCLASS()
8779 DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETGE); in expandIS_FPCLASS()
8783 SDValue IsNan = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT); in expandIS_FPCLASS()
8785 DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETLT); in expandIS_FPCLASS()
8786 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, IsNan, IsNotQnan); in expandIS_FPCLASS()
8795 SDValue ExpMinus1 = DAG.getNode(ISD::SUB, DL, IntVT, AbsV, ExpLSBV); in expandIS_FPCLASS()
8798 PartialRes = DAG.getSetCC(DL, ResultVT, ExpMinus1, ExpLimitV, ISD::SETULT); in expandIS_FPCLASS()
8800 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV); in expandIS_FPCLASS()
8803 DAG.getNode(ISD::XOR, DL, ResultVT, SignV, ResultInvertionMask); in expandIS_FPCLASS()
8804 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, PosSignV); in expandIS_FPCLASS()
8808 DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, getIntBitIsSet()); in expandIS_FPCLASS()
8815 Res = DAG.getNode(ISD::XOR, DL, ResultVT, Res, ResultInvertionMask); in expandIS_FPCLASS()
8823 return TLI.isOperationLegalOrCustom(ISD::ADD, VT) && in canExpandVectorCTPOP()
8824 TLI.isOperationLegalOrCustom(ISD::SUB, VT) && in canExpandVectorCTPOP()
8825 TLI.isOperationLegalOrCustom(ISD::SRL, VT) && in canExpandVectorCTPOP()
8826 (Len == 8 || TLI.isOperationLegalOrCustom(ISD::MUL, VT)) && in canExpandVectorCTPOP()
8827 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT); in canExpandVectorCTPOP()
8856 Op = DAG.getNode(ISD::SUB, dl, VT, Op, in expandCTPOP()
8857 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8858 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8862 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33), in expandCTPOP()
8863 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8864 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8868 Op = DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8869 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
8870 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8882 return DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8883 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
8884 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8892 ISD::MUL, getTypeToTransformTo(*DAG.getContext(), VT))) { in expandCTPOP()
8895 V = DAG.getNode(ISD::MUL, dl, VT, Op, Mask01); in expandCTPOP()
8900 V = DAG.getNode(ISD::ADD, dl, VT, V, in expandCTPOP()
8901 DAG.getNode(ISD::SHL, dl, VT, V, ShiftC)); in expandCTPOP()
8904 return DAG.getNode(ISD::SRL, dl, VT, V, DAG.getConstant(Len - 8, dl, ShVT)); in expandCTPOP()
8933 Tmp1 = DAG.getNode(ISD::VP_AND, dl, VT, in expandVPCTPOP()
8934 DAG.getNode(ISD::VP_SRL, dl, VT, Op, in expandVPCTPOP()
8937 Op = DAG.getNode(ISD::VP_SUB, dl, VT, Op, Tmp1, Mask, VL); in expandVPCTPOP()
8940 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Op, Mask33, Mask, VL); in expandVPCTPOP()
8941 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, in expandVPCTPOP()
8942 DAG.getNode(ISD::VP_SRL, dl, VT, Op, in expandVPCTPOP()
8945 Op = DAG.getNode(ISD::VP_ADD, dl, VT, Tmp2, Tmp3, Mask, VL); in expandVPCTPOP()
8948 Tmp4 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(4, dl, ShVT), in expandVPCTPOP()
8950 Tmp5 = DAG.getNode(ISD::VP_ADD, dl, VT, Op, Tmp4, Mask, VL); in expandVPCTPOP()
8951 Op = DAG.getNode(ISD::VP_AND, dl, VT, Tmp5, Mask0F, Mask, VL); in expandVPCTPOP()
8959 ISD::VP_MUL, getTypeToTransformTo(*DAG.getContext(), VT))) { in expandVPCTPOP()
8962 V = DAG.getNode(ISD::VP_MUL, dl, VT, Op, Mask01, Mask, VL); in expandVPCTPOP()
8967 V = DAG.getNode(ISD::VP_ADD, dl, VT, V, in expandVPCTPOP()
8968 DAG.getNode(ISD::VP_SHL, dl, VT, V, ShiftC, Mask, VL), in expandVPCTPOP()
8972 return DAG.getNode(ISD::VP_SRL, dl, VT, V, DAG.getConstant(Len - 8, dl, ShVT), in expandVPCTPOP()
8984 if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF && in expandCTLZ()
8985 isOperationLegalOrCustom(ISD::CTLZ, VT)) in expandCTLZ()
8986 return DAG.getNode(ISD::CTLZ, dl, VT, Op); in expandCTLZ()
8989 if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) { in expandCTLZ()
8992 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op); in expandCTLZ()
8994 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ); in expandCTLZ()
9002 (!isOperationLegalOrCustom(ISD::CTPOP, VT) && in expandCTLZ()
9004 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandCTLZ()
9005 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandCTLZ()
9019 Op = DAG.getNode(ISD::OR, dl, VT, Op, in expandCTLZ()
9020 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp)); in expandCTLZ()
9023 return DAG.getNode(ISD::CTPOP, dl, VT, Op); in expandCTLZ()
9044 Op = DAG.getNode(ISD::VP_OR, dl, VT, Op, in expandVPCTLZ()
9045 DAG.getNode(ISD::VP_SRL, dl, VT, Op, Tmp, Mask, VL), Mask, in expandVPCTLZ()
9048 Op = DAG.getNode(ISD::VP_XOR, dl, VT, Op, DAG.getConstant(-1, dl, VT), Mask, in expandVPCTLZ()
9050 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Op, Mask, VL); in expandVPCTLZ()
9064 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); in CTTZTableLookup()
9066 ISD::SRL, DL, VT, in CTTZTableLookup()
9067 DAG.getNode(ISD::MUL, DL, VT, DAG.getNode(ISD::AND, DL, VT, Op, Neg), in CTTZTableLookup()
9083 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, DL, VT, DAG.getEntryNode(), in CTTZTableLookup()
9086 if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) in CTTZTableLookup()
9092 SDValue SrcIsZero = DAG.getSetCC(DL, SetCCVT, Op, Zero, ISD::SETEQ); in CTTZTableLookup()
9104 if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF && in expandCTTZ()
9105 isOperationLegalOrCustom(ISD::CTTZ, VT)) in expandCTTZ()
9106 return DAG.getNode(ISD::CTTZ, dl, VT, Op); in expandCTTZ()
9109 if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) { in expandCTTZ()
9112 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op); in expandCTTZ()
9114 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ); in expandCTTZ()
9122 (!isOperationLegalOrCustom(ISD::CTPOP, VT) && in expandCTTZ()
9123 !isOperationLegalOrCustom(ISD::CTLZ, VT) && in expandCTTZ()
9125 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandCTTZ()
9126 !isOperationLegalOrCustomOrPromote(ISD::AND, VT) || in expandCTTZ()
9127 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandCTTZ()
9131 if (!VT.isVector() && isOperationExpand(ISD::CTPOP, VT) && in expandCTTZ()
9132 !isOperationLegal(ISD::CTLZ, VT)) in expandCTTZ()
9141 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT), in expandCTTZ()
9142 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT))); in expandCTTZ()
9145 if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) { in expandCTTZ()
9146 return DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT), in expandCTTZ()
9147 DAG.getNode(ISD::CTLZ, dl, VT, Tmp)); in expandCTTZ()
9150 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp); in expandCTTZ()
9161 SDValue Not = DAG.getNode(ISD::VP_XOR, dl, VT, Op, in expandVPCTTZ()
9163 SDValue MinusOne = DAG.getNode(ISD::VP_SUB, dl, VT, Op, in expandVPCTTZ()
9165 SDValue Tmp = DAG.getNode(ISD::VP_AND, dl, VT, Not, MinusOne, Mask, VL); in expandVPCTTZ()
9166 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Tmp, Mask, VL); in expandVPCTTZ()
9190 Source = DAG.getNode(ISD::VP_SETCC, DL, SrcVT, Source, AllZero, in expandVPCTTZElements()
9191 DAG.getCondCode(ISD::SETNE), Mask, EVL); in expandVPCTTZElements()
9198 DAG.getNode(ISD::VP_SELECT, DL, ResVecVT, Source, StepVec, Splat, EVL); in expandVPCTTZElements()
9199 return DAG.getNode(ISD::VP_REDUCE_UMIN, DL, ResVT, ExtEVL, Select, Mask, EVL); in expandVPCTTZElements()
9209 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
9210 isOperationLegal(ISD::SMAX, VT)) { in expandABS()
9213 return DAG.getNode(ISD::SMAX, dl, VT, Op, in expandABS()
9214 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
9218 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
9219 isOperationLegal(ISD::UMIN, VT)) { in expandABS()
9222 return DAG.getNode(ISD::UMIN, dl, VT, Op, in expandABS()
9223 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
9227 if (IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
9228 isOperationLegal(ISD::SMIN, VT)) { in expandABS()
9231 return DAG.getNode(ISD::SMIN, dl, VT, Op, in expandABS()
9232 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
9237 (!isOperationLegalOrCustom(ISD::SRA, VT) || in expandABS()
9238 (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) || in expandABS()
9239 (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) || in expandABS()
9240 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandABS()
9245 ISD::SRA, dl, VT, Op, in expandABS()
9247 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift); in expandABS()
9251 return DAG.getNode(ISD::SUB, dl, VT, Xor, Shift); in expandABS()
9254 return DAG.getNode(ISD::SUB, dl, VT, Shift, Xor); in expandABS()
9262 bool IsSigned = N->getOpcode() == ISD::ABDS; in expandABD()
9266 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX; in expandABD()
9267 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN; in expandABD()
9271 return DAG.getNode(ISD::SUB, dl, VT, Max, Min); in expandABD()
9275 if (!IsSigned && isOperationLegal(ISD::USUBSAT, VT)) in expandABD()
9276 return DAG.getNode(ISD::OR, dl, VT, in expandABD()
9277 DAG.getNode(ISD::USUBSAT, dl, VT, LHS, RHS), in expandABD()
9278 DAG.getNode(ISD::USUBSAT, dl, VT, RHS, LHS)); in expandABD()
9281 ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT; in expandABD()
9288 SDValue Diff = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); in expandABD()
9289 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Diff, Cmp); in expandABD()
9290 return DAG.getNode(ISD::SUB, dl, VT, Cmp, Xor); in expandABD()
9295 return DAG.getSelect(dl, VT, Cmp, DAG.getNode(ISD::SUB, dl, VT, LHS, RHS), in expandABD()
9296 DAG.getNode(ISD::SUB, dl, VT, RHS, LHS)); in expandABD()
9306 bool IsFloor = Opc == ISD::AVGFLOORS || Opc == ISD::AVGFLOORU; in expandAVG()
9307 bool IsSigned = Opc == ISD::AVGCEILS || Opc == ISD::AVGFLOORS; in expandAVG()
9308 unsigned SumOpc = IsFloor ? ISD::ADD : ISD::SUB; in expandAVG()
9309 unsigned SignOpc = IsFloor ? ISD::AND : ISD::OR; in expandAVG()
9310 unsigned ShiftOpc = IsSigned ? ISD::SRA : ISD::SRL; in expandAVG()
9311 unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in expandAVG()
9312 assert((Opc == ISD::AVGFLOORS || Opc == ISD::AVGCEILS || in expandAVG()
9313 Opc == ISD::AVGFLOORU || Opc == ISD::AVGCEILU) && in expandAVG()
9323 SDValue Sum = DAG.getNode(ISD::ADD, dl, VT, LHS, RHS); in expandAVG()
9325 Sum = DAG.getNode(ISD::ADD, dl, VT, Sum, DAG.getConstant(1, dl, VT)); in expandAVG()
9337 SDValue Avg = DAG.getNode(ISD::ADD, dl, ExtVT, LHS, RHS); in expandAVG()
9339 Avg = DAG.getNode(ISD::ADD, dl, ExtVT, Avg, in expandAVG()
9342 Avg = DAG.getNode(ISD::SRL, dl, ExtVT, Avg, in expandAVG()
9344 return DAG.getNode(ISD::TRUNCATE, dl, VT, Avg); in expandAVG()
9355 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); in expandAVG()
9376 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9378 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9379 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9381 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9382 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9383 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); in expandBSWAP()
9384 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9385 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
9386 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
9387 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
9389 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
9390 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9392 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
9393 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9395 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9396 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9398 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9399 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9400 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, in expandBSWAP()
9402 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9403 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, in expandBSWAP()
9405 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
9406 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, in expandBSWAP()
9408 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
9409 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); in expandBSWAP()
9410 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); in expandBSWAP()
9411 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
9412 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
9413 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); in expandBSWAP()
9414 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
9415 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); in expandBSWAP()
9435 Tmp1 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9437 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9439 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp1, Tmp2, Mask, EVL); in expandVPBSWAP()
9441 Tmp4 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9443 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(0xFF00, dl, VT), in expandVPBSWAP()
9445 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9447 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9449 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBSWAP()
9451 Tmp1 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9453 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL); in expandVPBSWAP()
9454 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL); in expandVPBSWAP()
9455 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL); in expandVPBSWAP()
9457 Tmp8 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT), in expandVPBSWAP()
9459 Tmp7 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9461 Tmp7 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT), in expandVPBSWAP()
9463 Tmp6 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9465 Tmp6 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9467 Tmp5 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9469 Tmp5 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9471 Tmp4 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9473 Tmp4 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp4, in expandVPBSWAP()
9475 Tmp3 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9477 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp3, in expandVPBSWAP()
9479 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT), in expandVPBSWAP()
9481 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBSWAP()
9483 Tmp1 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT), in expandVPBSWAP()
9485 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp7, Mask, EVL); in expandVPBSWAP()
9486 Tmp6 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp6, Tmp5, Mask, EVL); in expandVPBSWAP()
9487 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL); in expandVPBSWAP()
9488 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL); in expandVPBSWAP()
9489 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp6, Mask, EVL); in expandVPBSWAP()
9490 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL); in expandVPBSWAP()
9491 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp4, Mask, EVL); in expandVPBSWAP()
9514 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); in expandBITREVERSE()
9517 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
9518 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
9519 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
9520 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
9521 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9524 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
9525 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
9526 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
9527 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
9528 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9531 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
9532 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
9533 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
9534 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
9535 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9543 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); in expandBITREVERSE()
9546 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); in expandBITREVERSE()
9549 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); in expandBITREVERSE()
9550 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); in expandBITREVERSE()
9557 assert(N->getOpcode() == ISD::VP_BITREVERSE); in expandVPBITREVERSE()
9579 Tmp = (Sz > 8 ? DAG.getNode(ISD::VP_BSWAP, dl, VT, Op, Mask, EVL) : Op); in expandVPBITREVERSE()
9582 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT), in expandVPBITREVERSE()
9584 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9586 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT), in expandVPBITREVERSE()
9588 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT), in expandVPBITREVERSE()
9590 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9593 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT), in expandVPBITREVERSE()
9595 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9597 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT), in expandVPBITREVERSE()
9599 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT), in expandVPBITREVERSE()
9601 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9604 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT), in expandVPBITREVERSE()
9606 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9608 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT), in expandVPBITREVERSE()
9610 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT), in expandVPBITREVERSE()
9612 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9626 ISD::LoadExtType ExtType = LD->getExtensionType(); in scalarizeVectorLoad()
9656 DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR, in scalarizeVectorLoad()
9666 SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount); in scalarizeVectorLoad()
9668 DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask); in scalarizeVectorLoad()
9669 SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt); in scalarizeVectorLoad()
9671 if (ExtType != ISD::NON_EXTLOAD) { in scalarizeVectorLoad()
9672 unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType); in scalarizeVectorLoad()
9702 SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains); in scalarizeVectorLoad()
9742 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value, in scalarizeVectorStore()
9744 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt); in scalarizeVectorStore()
9745 SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc); in scalarizeVectorStore()
9751 DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount); in scalarizeVectorStore()
9752 CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt); in scalarizeVectorStore()
9767 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value, in scalarizeVectorStore()
9782 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores); in scalarizeVectorStore()
9787 assert(LD->getAddressingMode() == ISD::UNINDEXED && in expandUnalignedLoad()
9799 if (!isOperationLegalOrCustom(ISD::LOAD, intVT) && in expandUnalignedLoad()
9809 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); in expandUnalignedLoad()
9811 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : in expandUnalignedLoad()
9812 ISD::ANY_EXTEND, dl, VT, Result); in expandUnalignedLoad()
9859 DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, in expandUnalignedLoad()
9871 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in expandUnalignedLoad()
9894 ISD::LoadExtType HiExtType = LD->getExtensionType(); in expandUnalignedLoad()
9897 if (HiExtType == ISD::NON_EXTLOAD) in expandUnalignedLoad()
9898 HiExtType = ISD::ZEXTLOAD; in expandUnalignedLoad()
9903 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), in expandUnalignedLoad()
9918 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, in expandUnalignedLoad()
9926 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); in expandUnalignedLoad()
9927 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); in expandUnalignedLoad()
9929 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in expandUnalignedLoad()
9937 assert(ST->getAddressingMode() == ISD::UNINDEXED && in expandUnalignedStore()
9951 if (!isOperationLegalOrCustom(ISD::STORE, intVT) && in expandUnalignedStore()
9960 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val); in expandUnalignedStore()
10016 ISD::EXTLOAD, dl, RegVT, Store, StackPtr, in expandUnalignedStore()
10025 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in expandUnalignedStore()
10045 ISD::AND, dl, VT, Lo, in expandUnalignedStore()
10048 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); in expandUnalignedStore()
10064 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); in expandUnalignedStore()
10086 MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg); in IncrementMemoryAddress()
10091 Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg); in IncrementMemoryAddress()
10096 Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale); in IncrementMemoryAddress()
10104 return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment); in IncrementMemoryAddress()
10126 unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT; in clampDynamicVectorIndex()
10129 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub); in clampDynamicVectorIndex()
10133 return DAG.getNode(ISD::AND, dl, IdxVT, Idx, in clampDynamicVectorIndex()
10137 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, in clampDynamicVectorIndex()
10172 DAG.getNode(ISD::MUL, dl, IdxVT, Index, in getVectorSubVecPointer()
10175 Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index, in getVectorSubVecPointer()
10223 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node."); in lowerCmpEqZeroToCtlzSrl()
10226 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in lowerCmpEqZeroToCtlzSrl()
10228 if (isNullConstant(Op.getOperand(1)) && CC == ISD::SETEQ) { in lowerCmpEqZeroToCtlzSrl()
10233 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0)); in lowerCmpEqZeroToCtlzSrl()
10236 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext); in lowerCmpEqZeroToCtlzSrl()
10237 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz, in lowerCmpEqZeroToCtlzSrl()
10239 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc); in lowerCmpEqZeroToCtlzSrl()
10253 if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(Op1, true) && BoolVT == VT && in expandIntMINMAX()
10257 return DAG.getNode(ISD::SUB, DL, VT, Op0, in expandIntMINMAX()
10258 DAG.getSetCC(DL, VT, Op0, Zero, ISD::SETEQ)); in expandIntMINMAX()
10263 if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) && in expandIntMINMAX()
10264 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
10265 return DAG.getNode(ISD::SUB, DL, VT, Op0, in expandIntMINMAX()
10266 DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1)); in expandIntMINMAX()
10271 if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) && in expandIntMINMAX()
10272 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
10273 return DAG.getNode(ISD::ADD, DL, VT, Op0, in expandIntMINMAX()
10274 DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0)); in expandIntMINMAX()
10279 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandIntMINMAX()
10285 auto buildMinMax = [&](ISD::CondCode PrefCC, ISD::CondCode AltCC, in expandIntMINMAX()
10286 ISD::CondCode PrefCommuteCC, in expandIntMINMAX()
10287 ISD::CondCode AltCommuteCC) { in expandIntMINMAX()
10289 for (ISD::CondCode CC : {PrefCC, AltCC}) { in expandIntMINMAX()
10290 if (DAG.doesNodeExist(ISD::SETCC, BoolVTList, in expandIntMINMAX()
10296 for (ISD::CondCode CC : {PrefCommuteCC, AltCommuteCC}) { in expandIntMINMAX()
10297 if (DAG.doesNodeExist(ISD::SETCC, BoolVTList, in expandIntMINMAX()
10312 case ISD::SMAX: in expandIntMINMAX()
10313 return buildMinMax(ISD::SETGT, ISD::SETGE, ISD::SETLT, ISD::SETLE); in expandIntMINMAX()
10314 case ISD::SMIN: in expandIntMINMAX()
10315 return buildMinMax(ISD::SETLT, ISD::SETLE, ISD::SETGT, ISD::SETGE); in expandIntMINMAX()
10316 case ISD::UMAX: in expandIntMINMAX()
10317 return buildMinMax(ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE); in expandIntMINMAX()
10318 case ISD::UMIN: in expandIntMINMAX()
10319 return buildMinMax(ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE); in expandIntMINMAX()
10336 if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) { in expandAddSubSat()
10337 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS); in expandAddSubSat()
10338 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS); in expandAddSubSat()
10342 if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) { in expandAddSubSat()
10344 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS); in expandAddSubSat()
10345 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS); in expandAddSubSat()
10350 case ISD::SADDSAT: in expandAddSubSat()
10351 OverflowOp = ISD::SADDO; in expandAddSubSat()
10353 case ISD::UADDSAT: in expandAddSubSat()
10354 OverflowOp = ISD::UADDO; in expandAddSubSat()
10356 case ISD::SSUBSAT: in expandAddSubSat()
10357 OverflowOp = ISD::SSUBO; in expandAddSubSat()
10359 case ISD::USUBSAT: in expandAddSubSat()
10360 OverflowOp = ISD::USUBO; in expandAddSubSat()
10369 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandAddSubSat()
10380 if (Opcode == ISD::UADDSAT) { in expandAddSubSat()
10384 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask); in expandAddSubSat()
10390 if (Opcode == ISD::USUBSAT) { in expandAddSubSat()
10395 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not); in expandAddSubSat()
10401 if (Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) { in expandAddSubSat()
10416 bool RHSIsNonNegative = Opcode == ISD::SADDSAT ? KnownRHS.isNonNegative() in expandAddSubSat()
10424 bool RHSIsNegative = Opcode == ISD::SADDSAT ? KnownRHS.isNegative() in expandAddSubSat()
10435 SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff, in expandAddSubSat()
10437 Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin); in expandAddSubSat()
10450 auto LTPredicate = (Opcode == ISD::UCMP ? ISD::SETULT : ISD::SETLT); in expandCMP()
10451 auto GTPredicate = (Opcode == ISD::UCMP ? ISD::SETUGT : ISD::SETGT); in expandCMP()
10472 return DAG.getSExtOrTrunc(DAG.getNode(ISD::SUB, dl, BoolVT, IsGT, IsLT), dl, in expandCMP()
10478 bool IsSigned = Opcode == ISD::SSHLSAT; in expandShlSat()
10484 assert((Node->getOpcode() == ISD::SSHLSAT || in expandShlSat()
10485 Node->getOpcode() == ISD::USHLSAT) && in expandShlSat()
10490 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandShlSat()
10497 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS); in expandShlSat()
10499 DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS); in expandShlSat()
10506 DAG.getSetCC(dl, BoolVT, LHS, DAG.getConstant(0, dl, VT), ISD::SETLT); in expandShlSat()
10511 SDValue Cond = DAG.getSetCC(dl, BoolVT, LHS, Orig, ISD::SETNE); in expandShlSat()
10544 SDValue LLL = DAG.getNode(ISD::AND, dl, VT, LL, Mask); in forceExpandWideMUL()
10545 SDValue RLL = DAG.getNode(ISD::AND, dl, VT, RL, Mask); in forceExpandWideMUL()
10547 SDValue T = DAG.getNode(ISD::MUL, dl, VT, LLL, RLL); in forceExpandWideMUL()
10548 SDValue TL = DAG.getNode(ISD::AND, dl, VT, T, Mask); in forceExpandWideMUL()
10551 SDValue TH = DAG.getNode(ISD::SRL, dl, VT, T, Shift); in forceExpandWideMUL()
10552 SDValue LLH = DAG.getNode(ISD::SRL, dl, VT, LL, Shift); in forceExpandWideMUL()
10553 SDValue RLH = DAG.getNode(ISD::SRL, dl, VT, RL, Shift); in forceExpandWideMUL()
10555 SDValue U = DAG.getNode(ISD::ADD, dl, VT, in forceExpandWideMUL()
10556 DAG.getNode(ISD::MUL, dl, VT, LLH, RLL), TH); in forceExpandWideMUL()
10557 SDValue UL = DAG.getNode(ISD::AND, dl, VT, U, Mask); in forceExpandWideMUL()
10558 SDValue UH = DAG.getNode(ISD::SRL, dl, VT, U, Shift); in forceExpandWideMUL()
10560 SDValue V = DAG.getNode(ISD::ADD, dl, VT, in forceExpandWideMUL()
10561 DAG.getNode(ISD::MUL, dl, VT, LLL, RLH), UL); in forceExpandWideMUL()
10562 SDValue VH = DAG.getNode(ISD::SRL, dl, VT, V, Shift); in forceExpandWideMUL()
10565 DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::MUL, dl, VT, LLH, RLH), in forceExpandWideMUL()
10566 DAG.getNode(ISD::ADD, dl, VT, UH, VH)); in forceExpandWideMUL()
10567 Lo = DAG.getNode(ISD::ADD, dl, VT, TL, in forceExpandWideMUL()
10568 DAG.getNode(ISD::SHL, dl, VT, V, Shift)); in forceExpandWideMUL()
10570 Hi = DAG.getNode(ISD::ADD, dl, VT, W, in forceExpandWideMUL()
10571 DAG.getNode(ISD::ADD, dl, VT, in forceExpandWideMUL()
10572 DAG.getNode(ISD::MUL, dl, VT, RH, LL), in forceExpandWideMUL()
10573 DAG.getNode(ISD::MUL, dl, VT, RL, LH))); in forceExpandWideMUL()
10591 assert(Ret.getOpcode() == ISD::MERGE_VALUES && in forceExpandWideMUL()
10618 ISD::SRA, dl, VT, LHS, in forceExpandWideMUL()
10621 ISD::SRA, dl, VT, RHS, in forceExpandWideMUL()
10633 assert((Node->getOpcode() == ISD::SMULFIX || in expandFixedPointMul()
10634 Node->getOpcode() == ISD::UMULFIX || in expandFixedPointMul()
10635 Node->getOpcode() == ISD::SMULFIXSAT || in expandFixedPointMul()
10636 Node->getOpcode() == ISD::UMULFIXSAT) && in expandFixedPointMul()
10644 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT || in expandFixedPointMul()
10645 Node->getOpcode() == ISD::UMULFIXSAT); in expandFixedPointMul()
10646 bool Signed = (Node->getOpcode() == ISD::SMULFIX || in expandFixedPointMul()
10647 Node->getOpcode() == ISD::SMULFIXSAT); in expandFixedPointMul()
10654 if (isOperationLegalOrCustom(ISD::MUL, VT)) in expandFixedPointMul()
10655 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
10656 } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) { in expandFixedPointMul()
10658 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
10669 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); in expandFixedPointMul()
10670 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); in expandFixedPointMul()
10673 } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) { in expandFixedPointMul()
10675 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
10693 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; in expandFixedPointMul()
10694 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU; in expandFixedPointMul()
10701 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
10703 } else if (isOperationLegalOrCustom(ISD::MUL, WideVT)) { in expandFixedPointMul()
10705 unsigned Ext = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in expandFixedPointMul()
10708 SDValue Res = DAG.getNode(ISD::MUL, dl, WideVT, LHSExt, RHSExt); in expandFixedPointMul()
10709 Lo = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in expandFixedPointMul()
10711 DAG.getNode(ISD::SRA, dl, WideVT, Res, in expandFixedPointMul()
10713 Hi = DAG.getNode(ISD::TRUNCATE, dl, VT, Shifted); in expandFixedPointMul()
10729 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo, in expandFixedPointMul()
10745 ISD::SETUGT); in expandFixedPointMul()
10757 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo, in expandFixedPointMul()
10759 SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE); in expandFixedPointMul()
10764 ISD::SETLT); in expandFixedPointMul()
10775 Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT); in expandFixedPointMul()
10781 Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT); in expandFixedPointMul()
10789 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || in expandFixedPointDiv()
10790 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && in expandFixedPointDiv()
10794 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT; in expandFixedPointDiv()
10795 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT; in expandFixedPointDiv()
10826 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS, in expandFixedPointDiv()
10829 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS, in expandFixedPointDiv()
10842 isOperationLegalOrCustom(ISD::SDIVREM, VT)) { in expandFixedPointDiv()
10843 Quot = DAG.getNode(ISD::SDIVREM, dl, in expandFixedPointDiv()
10849 Quot = DAG.getNode(ISD::SDIV, dl, VT, in expandFixedPointDiv()
10851 Rem = DAG.getNode(ISD::SREM, dl, VT, in expandFixedPointDiv()
10855 SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE); in expandFixedPointDiv()
10856 SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT); in expandFixedPointDiv()
10857 SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT); in expandFixedPointDiv()
10858 SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg); in expandFixedPointDiv()
10859 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot, in expandFixedPointDiv()
10862 DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg), in expandFixedPointDiv()
10865 Quot = DAG.getNode(ISD::UDIV, dl, VT, in expandFixedPointDiv()
10876 bool IsAdd = Node->getOpcode() == ISD::UADDO; in expandUADDSUBO()
10879 unsigned OpcCarry = IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY; in expandUADDSUBO()
10889 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, in expandUADDSUBO()
10904 DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETEQ); in expandUADDSUBO()
10909 DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETNE); in expandUADDSUBO()
10911 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; in expandUADDSUBO()
10922 bool IsAdd = Node->getOpcode() == ISD::SADDO; in expandSADDSUBO()
10924 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, in expandSADDSUBO()
10932 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT; in expandSADDSUBO()
10935 SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE); in expandSADDSUBO()
10948 SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT); in expandSADDSUBO()
10950 DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT); in expandSADDSUBO()
10953 DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl, in expandSADDSUBO()
10964 bool isSigned = Node->getOpcode() == ISD::SMULO; in expandMULO()
10974 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt); in expandMULO()
10976 DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL, in expandMULO()
10978 LHS, ISD::SETNE); in expandMULO()
10991 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, in expandMULO()
10992 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; in expandMULO()
10994 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandMULO()
11003 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); in expandMULO()
11004 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in expandMULO()
11007 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, in expandMULO()
11008 DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt)); in expandMULO()
11020 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt); in expandMULO()
11021 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE); in expandMULO()
11024 DAG.getConstant(0, dl, VT), ISD::SETNE); in expandMULO()
11030 Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow); in expandMULO()
11039 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode()); in expandVecReduce()
11073 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res); in expandVecReduce()
11095 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode()); in expandVecReduceSeq()
11108 bool isSigned = Node->getOpcode() == ISD::SREM; in expandREM()
11109 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; in expandREM()
11110 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in expandREM()
11121 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor); in expandREM()
11122 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul); in expandREM()
11130 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT; in expandFP_TO_INT_SAT()
11158 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src); in expandFP_TO_INT_SAT()
11178 bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) && in expandFP_TO_INT_SAT()
11179 isOperationLegal(ISD::FMAXNUM, SrcVT); in expandFP_TO_INT_SAT()
11184 Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode); in expandFP_TO_INT_SAT()
11186 Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode); in expandFP_TO_INT_SAT()
11188 SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, in expandFP_TO_INT_SAT()
11200 SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO); in expandFP_TO_INT_SAT()
11211 DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src); in expandFP_TO_INT_SAT()
11220 SDValue ULT = DAG.getSetCC(dl, SetCCVT, Src, MinFloatNode, ISD::SETULT); in expandFP_TO_INT_SAT()
11223 SDValue OGT = DAG.getSetCC(dl, SetCCVT, Src, MaxFloatNode, ISD::SETOGT); in expandFP_TO_INT_SAT()
11233 SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO); in expandFP_TO_INT_SAT()
11253 DAG.getNode(ISD::AND, dl, WideIntVT, OpAsInt, in expandRoundInexactToOdd()
11256 if (isOperationLegalOrCustom(ISD::FABS, OperandVT)) { in expandRoundInexactToOdd()
11257 AbsWide = DAG.getNode(ISD::FABS, dl, OperandVT, Op); in expandRoundInexactToOdd()
11260 ISD::AND, dl, WideIntVT, OpAsInt, in expandRoundInexactToOdd()
11270 SDValue NarrowBits = DAG.getNode(ISD::BITCAST, dl, ResultIntVT, AbsNarrow); in expandRoundInexactToOdd()
11273 SDValue And = DAG.getNode(ISD::AND, dl, ResultIntVT, NarrowBits, One); in expandRoundInexactToOdd()
11278 SDValue AlreadyOdd = DAG.getSetCC(dl, ResultIntVTCCVT, And, Zero, ISD::SETNE); in expandRoundInexactToOdd()
11284 DAG.getSetCC(dl, WideSetCCVT, AbsWide, AbsNarrowAsWide, ISD::SETUEQ); in expandRoundInexactToOdd()
11285 KeepNarrow = DAG.getNode(ISD::OR, dl, WideSetCCVT, KeepNarrow, AlreadyOdd); in expandRoundInexactToOdd()
11289 DAG.getSetCC(dl, WideSetCCVT, AbsWide, AbsNarrowAsWide, ISD::SETOGT); in expandRoundInexactToOdd()
11295 SDValue Adjusted = DAG.getNode(ISD::ADD, dl, ResultIntVT, NarrowBits, Adjust); in expandRoundInexactToOdd()
11299 SignBit = DAG.getNode(ISD::SRL, dl, WideIntVT, SignBit, ShiftCnst); in expandRoundInexactToOdd()
11300 SignBit = DAG.getNode(ISD::TRUNCATE, dl, ResultIntVT, SignBit); in expandRoundInexactToOdd()
11301 Op = DAG.getNode(ISD::OR, dl, ResultIntVT, Op, SignBit); in expandRoundInexactToOdd()
11302 return DAG.getNode(ISD::BITCAST, dl, ResultVT, Op); in expandRoundInexactToOdd()
11306 assert(Node->getOpcode() == ISD::FP_ROUND && "Unexpected opcode!"); in expandFP_ROUND()
11312 return DAG.getNode(ISD::FP_TO_BF16, dl, VT, Node->getOperand(0)); in expandFP_ROUND()
11318 Op, Op, ISD::SETUO); in expandFP_ROUND()
11328 Op = DAG.getNode(ISD::BITCAST, dl, I32, Op); in expandFP_ROUND()
11333 DAG.getNode(ISD::OR, dl, I32, Op, DAG.getConstant(0x400000, dl, I32)); in expandFP_ROUND()
11337 SDValue Lsb = DAG.getNode(ISD::SRL, dl, I32, Op, in expandFP_ROUND()
11339 Lsb = DAG.getNode(ISD::AND, dl, I32, Lsb, One); in expandFP_ROUND()
11341 DAG.getNode(ISD::ADD, dl, I32, DAG.getConstant(0x7fff, dl, I32), Lsb); in expandFP_ROUND()
11342 SDValue Add = DAG.getNode(ISD::ADD, dl, I32, Op, RoundingBias); in expandFP_ROUND()
11349 Op = DAG.getNode(ISD::SRL, dl, I32, Op, in expandFP_ROUND()
11351 Op = DAG.getNode(ISD::BITCAST, dl, I32, Op); in expandFP_ROUND()
11353 Op = DAG.getNode(ISD::TRUNCATE, dl, I16, Op); in expandFP_ROUND()
11354 return DAG.getNode(ISD::BITCAST, dl, VT, Op); in expandFP_ROUND()
11361 assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!"); in expandVectorSplice()
11398 SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, OffsetToV2); in expandVectorSplice()
11422 TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VLBytes); in expandVectorSplice()
11426 StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes); in expandVectorSplice()
11470 ISD::isConstantSplatVector(Passthru.getNode(), PassthruSplatVal); in expandVECTOR_COMPRESS()
11482 ISD::TRUNCATE, DL, MaskVT.changeVectorElementType(MVT::i1), Mask); in expandVECTOR_COMPRESS()
11483 Popcount = DAG.getNode(ISD::ZERO_EXTEND, DL, in expandVECTOR_COMPRESS()
11485 Popcount = DAG.getNode(ISD::VECREDUCE_ADD, DL, ScalarVT, Popcount); in expandVECTOR_COMPRESS()
11498 SDValue ValI = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Vec, Idx); in expandVECTOR_COMPRESS()
11508 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MaskScalarVT, Mask, Idx)); in expandVECTOR_COMPRESS()
11510 MaskI = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, MaskI); in expandVECTOR_COMPRESS()
11511 MaskI = DAG.getNode(ISD::ZERO_EXTEND, DL, PositionVT, MaskI); in expandVECTOR_COMPRESS()
11512 OutPos = DAG.getNode(ISD::ADD, DL, PositionVT, OutPos, MaskI); in expandVECTOR_COMPRESS()
11518 DAG.getSetCC(DL, MVT::i1, OutPos, EndOfVector, ISD::CondCode::SETUGT); in expandVECTOR_COMPRESS()
11519 OutPos = DAG.getNode(ISD::UMIN, DL, PositionVT, OutPos, EndOfVector); in expandVECTOR_COMPRESS()
11543 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); in LegalizeSetCCCondCode()
11554 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); in LegalizeSetCCCondCode()
11566 InvCC = ISD::getSetCCSwappedOperands(InvCC); in LegalizeSetCCCondCode()
11577 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; in LegalizeSetCCCondCode()
11582 case ISD::SETUO: in LegalizeSetCCCondCode()
11583 if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) { in LegalizeSetCCCondCode()
11584 CC1 = ISD::SETUNE; in LegalizeSetCCCondCode()
11585 CC2 = ISD::SETUNE; in LegalizeSetCCCondCode()
11586 Opc = ISD::OR; in LegalizeSetCCCondCode()
11589 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && in LegalizeSetCCCondCode()
11593 case ISD::SETO: in LegalizeSetCCCondCode()
11594 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && in LegalizeSetCCCondCode()
11596 CC1 = ISD::SETOEQ; in LegalizeSetCCCondCode()
11597 CC2 = ISD::SETOEQ; in LegalizeSetCCCondCode()
11598 Opc = ISD::AND; in LegalizeSetCCCondCode()
11600 case ISD::SETONE: in LegalizeSetCCCondCode()
11601 case ISD::SETUEQ: in LegalizeSetCCCondCode()
11606 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; in LegalizeSetCCCondCode()
11608 (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) || in LegalizeSetCCCondCode()
11609 TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) { in LegalizeSetCCCondCode()
11610 CC1 = ISD::SETOGT; in LegalizeSetCCCondCode()
11611 CC2 = ISD::SETOLT; in LegalizeSetCCCondCode()
11612 Opc = ISD::OR; in LegalizeSetCCCondCode()
11617 case ISD::SETOEQ: in LegalizeSetCCCondCode()
11618 case ISD::SETOGT: in LegalizeSetCCCondCode()
11619 case ISD::SETOGE: in LegalizeSetCCCondCode()
11620 case ISD::SETOLT: in LegalizeSetCCCondCode()
11621 case ISD::SETOLE: in LegalizeSetCCCondCode()
11622 case ISD::SETUNE: in LegalizeSetCCCondCode()
11623 case ISD::SETUGT: in LegalizeSetCCCondCode()
11624 case ISD::SETUGE: in LegalizeSetCCCondCode()
11625 case ISD::SETULT: in LegalizeSetCCCondCode()
11626 case ISD::SETULE: in LegalizeSetCCCondCode()
11631 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; in LegalizeSetCCCondCode()
11632 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; in LegalizeSetCCCondCode()
11633 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); in LegalizeSetCCCondCode()
11638 case ISD::SETLE: in LegalizeSetCCCondCode()
11639 case ISD::SETGT: in LegalizeSetCCCondCode()
11640 case ISD::SETGE: in LegalizeSetCCCondCode()
11641 case ISD::SETLT: in LegalizeSetCCCondCode()
11642 case ISD::SETNE: in LegalizeSetCCCondCode()
11643 case ISD::SETEQ: in LegalizeSetCCCondCode()
11650 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { in LegalizeSetCCCondCode()
11671 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1), in LegalizeSetCCCondCode()
11677 assert((Opc == ISD::OR || Opc == ISD::AND) && "Unexpected opcode"); in LegalizeSetCCCondCode()
11678 Opc = Opc == ISD::OR ? ISD::VP_OR : ISD::VP_AND; in LegalizeSetCCCondCode()