Lines Matching refs:ISD
272 assert(N->getOpcode() != ISD::DELETED_NODE && in AddToWorklist()
277 if (N->getOpcode() == ISD::HANDLENODE) in AddToWorklist()
406 ISD::CondCode CC);
592 SDValue N2, SDValue N3, ISD::CondCode CC,
596 ISD::CondCode CC);
599 SDValue N2, SDValue N3, ISD::CondCode CC);
608 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
720 case ISD::Constant: in getStoreSource()
721 case ISD::ConstantFP: in getStoreSource()
723 case ISD::BUILD_VECTOR: in getStoreSource()
724 if (ISD::isBuildVectorOfConstantSDNodes(StoreVal.getNode()) || in getStoreSource()
725 ISD::isBuildVectorOfConstantFPSDNodes(StoreVal.getNode())) in getStoreSource()
728 case ISD::EXTRACT_VECTOR_ELT: in getStoreSource()
729 case ISD::EXTRACT_SUBVECTOR: in getStoreSource()
731 case ISD::LOAD: in getStoreSource()
753 bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
870 ISD::NodeType ExtType);
969 if (N.getOpcode() == ISD::SETCC) { in isSetCCEquivalent()
977 (N.getOpcode() == ISD::STRICT_FSETCC || in isSetCCEquivalent()
978 N.getOpcode() == ISD::STRICT_FSETCCS)) { in isSetCCEquivalent()
985 if (N.getOpcode() != ISD::SELECT_CC || !TLI.isConstTrueVal(N.getOperand(2)) || in isSetCCEquivalent()
1030 if (ISD::isConstantSplatVector(N, Val)) in isConstantSplatVectorMaskForType()
1042 if (N.getOpcode() != ISD::BUILD_VECTOR && N.getOpcode() != ISD::SPLAT_VECTOR) in isConstantOrConstantVector()
1059 if (V.getOpcode() != ISD::BUILD_VECTOR) in isAnyConstantBuildVector()
1062 ISD::isBuildVectorOfConstantFPSDNodes(V.getNode()); in isAnyConstantBuildVector()
1068 (LD->getOperand(2).getOpcode() != ISD::TargetConstant || in canSplitIdx()
1086 if (N0.getOpcode() != ISD::ADD) in reassociationCanBreakAddressingModePattern()
1093 if ((N1.getOpcode() == ISD::VSCALE || in reassociationCanBreakAddressingModePattern()
1094 ((N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::MUL) && in reassociationCanBreakAddressingModePattern()
1095 N1.getOperand(0).getOpcode() == ISD::VSCALE && in reassociationCanBreakAddressingModePattern()
1098 int64_t ScalableOffset = N1.getOpcode() == ISD::VSCALE in reassociationCanBreakAddressingModePattern()
1101 (N1.getOpcode() == ISD::SHL in reassociationCanBreakAddressingModePattern()
1104 if (Opc == ISD::SUB) in reassociationCanBreakAddressingModePattern()
1123 if (Opc != ISD::ADD) in reassociationCanBreakAddressingModePattern()
1166 if (GA->getOpcode() == ISD::GlobalAddress && TLI.isOffsetFoldingLegal(GA)) in reassociationCanBreakAddressingModePattern()
1206 if (N0.getOpcode() == ISD::ADD && N0->getFlags().hasNoUnsignedWrap() && in reassociateOpsCommutative()
1225 if (Opc == ISD::AND || Opc == ISD::OR) { in reassociateOpsCommutative()
1233 if (Opc == ISD::XOR) { in reassociateOpsCommutative()
1270 if (Opc == ISD::AND || Opc == ISD::OR) { in reassociateOpsCommutative()
1271 if (N1->getOpcode() == ISD::SETCC && N00->getOpcode() == ISD::SETCC && in reassociateOpsCommutative()
1272 N01->getOpcode() == ISD::SETCC) { in reassociateOpsCommutative()
1273 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1.getOperand(2))->get(); in reassociateOpsCommutative()
1274 ISD::CondCode CC00 = cast<CondCodeSDNode>(N00.getOperand(2))->get(); in reassociateOpsCommutative()
1275 ISD::CondCode CC01 = cast<CondCodeSDNode>(N01.getOperand(2))->get(); in reassociateOpsCommutative()
1416 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0)); in ReplaceLoadWithPromotedLoad()
1431 if (ISD::isUNINDEXEDLoad(Op.getNode())) { in PromoteOperand()
1434 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD in PromoteOperand()
1445 case ISD::AssertSext: in PromoteOperand()
1447 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1)); in PromoteOperand()
1449 case ISD::AssertZext: in PromoteOperand()
1451 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1)); in PromoteOperand()
1453 case ISD::Constant: { in PromoteOperand()
1455 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteOperand()
1460 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT)) in PromoteOperand()
1462 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op); in PromoteOperand()
1466 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT)) in SExtPromoteOperand()
1478 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp, in SExtPromoteOperand()
1531 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1)); in PromoteIntBinOp()
1591 if (Opc == ISD::SRA) in PromoteIntShiftOp()
1593 else if (Opc == ISD::SRL) in PromoteIntShiftOp()
1604 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1)); in PromoteIntShiftOp()
1610 if (Op && Op.getOpcode() != ISD::DELETED_NODE) in PromoteIntShiftOp()
1648 if (!ISD::isUNINDEXEDLoad(Op.getNode())) in PromoteLoad()
1671 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD in PromoteLoad()
1676 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD); in PromoteLoad()
1799 assert(N->getOpcode() != ISD::DELETED_NODE && in Run()
1800 RV.getOpcode() != ISD::DELETED_NODE && in Run()
1818 if (RV.getOpcode() != ISD::EntryToken) in Run()
1837 case ISD::TokenFactor: return visitTokenFactor(N); in visit()
1838 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N); in visit()
1839 case ISD::ADD: return visitADD(N); in visit()
1840 case ISD::SUB: return visitSUB(N); in visit()
1841 case ISD::SADDSAT: in visit()
1842 case ISD::UADDSAT: return visitADDSAT(N); in visit()
1843 case ISD::SSUBSAT: in visit()
1844 case ISD::USUBSAT: return visitSUBSAT(N); in visit()
1845 case ISD::ADDC: return visitADDC(N); in visit()
1846 case ISD::SADDO: in visit()
1847 case ISD::UADDO: return visitADDO(N); in visit()
1848 case ISD::SUBC: return visitSUBC(N); in visit()
1849 case ISD::SSUBO: in visit()
1850 case ISD::USUBO: return visitSUBO(N); in visit()
1851 case ISD::ADDE: return visitADDE(N); in visit()
1852 case ISD::UADDO_CARRY: return visitUADDO_CARRY(N); in visit()
1853 case ISD::SADDO_CARRY: return visitSADDO_CARRY(N); in visit()
1854 case ISD::SUBE: return visitSUBE(N); in visit()
1855 case ISD::USUBO_CARRY: return visitUSUBO_CARRY(N); in visit()
1856 case ISD::SSUBO_CARRY: return visitSSUBO_CARRY(N); in visit()
1857 case ISD::SMULFIX: in visit()
1858 case ISD::SMULFIXSAT: in visit()
1859 case ISD::UMULFIX: in visit()
1860 case ISD::UMULFIXSAT: return visitMULFIX(N); in visit()
1861 case ISD::MUL: return visitMUL<EmptyMatchContext>(N); in visit()
1862 case ISD::SDIV: return visitSDIV(N); in visit()
1863 case ISD::UDIV: return visitUDIV(N); in visit()
1864 case ISD::SREM: in visit()
1865 case ISD::UREM: return visitREM(N); in visit()
1866 case ISD::MULHU: return visitMULHU(N); in visit()
1867 case ISD::MULHS: return visitMULHS(N); in visit()
1868 case ISD::AVGFLOORS: in visit()
1869 case ISD::AVGFLOORU: in visit()
1870 case ISD::AVGCEILS: in visit()
1871 case ISD::AVGCEILU: return visitAVG(N); in visit()
1872 case ISD::ABDS: in visit()
1873 case ISD::ABDU: return visitABD(N); in visit()
1874 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N); in visit()
1875 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N); in visit()
1876 case ISD::SMULO: in visit()
1877 case ISD::UMULO: return visitMULO(N); in visit()
1878 case ISD::SMIN: in visit()
1879 case ISD::SMAX: in visit()
1880 case ISD::UMIN: in visit()
1881 case ISD::UMAX: return visitIMINMAX(N); in visit()
1882 case ISD::AND: return visitAND(N); in visit()
1883 case ISD::OR: return visitOR(N); in visit()
1884 case ISD::XOR: return visitXOR(N); in visit()
1885 case ISD::SHL: return visitSHL(N); in visit()
1886 case ISD::SRA: return visitSRA(N); in visit()
1887 case ISD::SRL: return visitSRL(N); in visit()
1888 case ISD::ROTR: in visit()
1889 case ISD::ROTL: return visitRotate(N); in visit()
1890 case ISD::FSHL: in visit()
1891 case ISD::FSHR: return visitFunnelShift(N); in visit()
1892 case ISD::SSHLSAT: in visit()
1893 case ISD::USHLSAT: return visitSHLSAT(N); in visit()
1894 case ISD::ABS: return visitABS(N); in visit()
1895 case ISD::BSWAP: return visitBSWAP(N); in visit()
1896 case ISD::BITREVERSE: return visitBITREVERSE(N); in visit()
1897 case ISD::CTLZ: return visitCTLZ(N); in visit()
1898 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N); in visit()
1899 case ISD::CTTZ: return visitCTTZ(N); in visit()
1900 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N); in visit()
1901 case ISD::CTPOP: return visitCTPOP(N); in visit()
1902 case ISD::SELECT: return visitSELECT(N); in visit()
1903 case ISD::VSELECT: return visitVSELECT(N); in visit()
1904 case ISD::SELECT_CC: return visitSELECT_CC(N); in visit()
1905 case ISD::SETCC: return visitSETCC(N); in visit()
1906 case ISD::SETCCCARRY: return visitSETCCCARRY(N); in visit()
1907 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N); in visit()
1908 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N); in visit()
1909 case ISD::ANY_EXTEND: return visitANY_EXTEND(N); in visit()
1910 case ISD::AssertSext: in visit()
1911 case ISD::AssertZext: return visitAssertExt(N); in visit()
1912 case ISD::AssertAlign: return visitAssertAlign(N); in visit()
1913 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N); in visit()
1914 case ISD::SIGN_EXTEND_VECTOR_INREG: in visit()
1915 case ISD::ZERO_EXTEND_VECTOR_INREG: in visit()
1916 case ISD::ANY_EXTEND_VECTOR_INREG: return visitEXTEND_VECTOR_INREG(N); in visit()
1917 case ISD::TRUNCATE: return visitTRUNCATE(N); in visit()
1918 case ISD::BITCAST: return visitBITCAST(N); in visit()
1919 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N); in visit()
1920 case ISD::FADD: return visitFADD(N); in visit()
1921 case ISD::STRICT_FADD: return visitSTRICT_FADD(N); in visit()
1922 case ISD::FSUB: return visitFSUB(N); in visit()
1923 case ISD::FMUL: return visitFMUL(N); in visit()
1924 case ISD::FMA: return visitFMA<EmptyMatchContext>(N); in visit()
1925 case ISD::FMAD: return visitFMAD(N); in visit()
1926 case ISD::FDIV: return visitFDIV(N); in visit()
1927 case ISD::FREM: return visitFREM(N); in visit()
1928 case ISD::FSQRT: return visitFSQRT(N); in visit()
1929 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N); in visit()
1930 case ISD::FPOW: return visitFPOW(N); in visit()
1931 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N); in visit()
1932 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N); in visit()
1933 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N); in visit()
1934 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N); in visit()
1935 case ISD::LRINT: in visit()
1936 case ISD::LLRINT: return visitXRINT(N); in visit()
1937 case ISD::FP_ROUND: return visitFP_ROUND(N); in visit()
1938 case ISD::FP_EXTEND: return visitFP_EXTEND(N); in visit()
1939 case ISD::FNEG: return visitFNEG(N); in visit()
1940 case ISD::FABS: return visitFABS(N); in visit()
1941 case ISD::FFLOOR: return visitFFLOOR(N); in visit()
1942 case ISD::FMINNUM: in visit()
1943 case ISD::FMAXNUM: in visit()
1944 case ISD::FMINIMUM: in visit()
1945 case ISD::FMAXIMUM: return visitFMinMax(N); in visit()
1946 case ISD::FCEIL: return visitFCEIL(N); in visit()
1947 case ISD::FTRUNC: return visitFTRUNC(N); in visit()
1948 case ISD::FFREXP: return visitFFREXP(N); in visit()
1949 case ISD::BRCOND: return visitBRCOND(N); in visit()
1950 case ISD::BR_CC: return visitBR_CC(N); in visit()
1951 case ISD::LOAD: return visitLOAD(N); in visit()
1952 case ISD::STORE: return visitSTORE(N); in visit()
1953 case ISD::ATOMIC_STORE: return visitATOMIC_STORE(N); in visit()
1954 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N); in visit()
1955 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N); in visit()
1956 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N); in visit()
1957 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N); in visit()
1958 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N); in visit()
1959 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N); in visit()
1960 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N); in visit()
1961 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N); in visit()
1962 case ISD::MGATHER: return visitMGATHER(N); in visit()
1963 case ISD::MLOAD: return visitMLOAD(N); in visit()
1964 case ISD::MSCATTER: return visitMSCATTER(N); in visit()
1965 case ISD::MSTORE: return visitMSTORE(N); in visit()
1966 case ISD::VECTOR_COMPRESS: return visitVECTOR_COMPRESS(N); in visit()
1967 case ISD::LIFETIME_END: return visitLIFETIME_END(N); in visit()
1968 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N); in visit()
1969 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N); in visit()
1970 case ISD::FP_TO_BF16: return visitFP_TO_BF16(N); in visit()
1971 case ISD::BF16_TO_FP: return visitBF16_TO_FP(N); in visit()
1972 case ISD::FREEZE: return visitFREEZE(N); in visit()
1973 case ISD::GET_FPENV_MEM: return visitGET_FPENV_MEM(N); in visit()
1974 case ISD::SET_FPENV_MEM: return visitSET_FPENV_MEM(N); in visit()
1975 case ISD::VECREDUCE_FADD: in visit()
1976 case ISD::VECREDUCE_FMUL: in visit()
1977 case ISD::VECREDUCE_ADD: in visit()
1978 case ISD::VECREDUCE_MUL: in visit()
1979 case ISD::VECREDUCE_AND: in visit()
1980 case ISD::VECREDUCE_OR: in visit()
1981 case ISD::VECREDUCE_XOR: in visit()
1982 case ISD::VECREDUCE_SMAX: in visit()
1983 case ISD::VECREDUCE_SMIN: in visit()
1984 case ISD::VECREDUCE_UMAX: in visit()
1985 case ISD::VECREDUCE_UMIN: in visit()
1986 case ISD::VECREDUCE_FMAX: in visit()
1987 case ISD::VECREDUCE_FMIN: in visit()
1988 case ISD::VECREDUCE_FMAXIMUM: in visit()
1989 case ISD::VECREDUCE_FMINIMUM: return visitVECREDUCE(N); in visit()
1990 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC: in visit()
2008 assert(N->getOpcode() != ISD::DELETED_NODE && in combine()
2011 if (N->getOpcode() >= ISD::BUILTIN_OP_END || in combine()
2012 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) { in combine()
2026 case ISD::ADD: in combine()
2027 case ISD::SUB: in combine()
2028 case ISD::MUL: in combine()
2029 case ISD::AND: in combine()
2030 case ISD::OR: in combine()
2031 case ISD::XOR: in combine()
2034 case ISD::SHL: in combine()
2035 case ISD::SRA: in combine()
2036 case ISD::SRL: in combine()
2039 case ISD::SIGN_EXTEND: in combine()
2040 case ISD::ZERO_EXTEND: in combine()
2041 case ISD::ANY_EXTEND: in combine()
2044 case ISD::LOAD: in combine()
2106 if (N->hasOneUse() && N->use_begin()->getOpcode() == ISD::TokenFactor) in visitTokenFactor()
2136 case ISD::EntryToken: in visitTokenFactor()
2142 case ISD::TokenFactor: in visitTokenFactor()
2222 case ISD::EntryToken: in visitTokenFactor()
2229 case ISD::TokenFactor: in visitTokenFactor()
2233 case ISD::LIFETIME_START: in visitTokenFactor()
2234 case ISD::LIFETIME_END: in visitTokenFactor()
2235 case ISD::CopyFromReg: in visitTokenFactor()
2236 case ISD::CopyToReg: in visitTokenFactor()
2306 if (N->getOpcode() == ISD::TRUNCATE) { in isTruncateOf()
2314 N, m_c_SetCC(m_Value(Op), m_Zero(), m_SpecificCondCode(ISD::SETNE)))) in isTruncateOf()
2353 if (N->getOpcode() == ISD::ADD) { in canFoldInAddressingMode()
2362 } else if (N->getOpcode() == ISD::SUB) { in canFoldInAddressingMode()
2393 if (N1.getOpcode() != ISD::VSELECT || !N1.hasOneUse()) in foldSelectWithIdentityConstant()
2446 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) { in foldBinOpIntoSelect()
2451 if ((BinOpcode == ISD::SHL || BinOpcode == ISD::SRA || in foldBinOpIntoSelect()
2452 BinOpcode == ISD::SRL) && Sel.hasOneUse()) { in foldBinOpIntoSelect()
2462 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) in foldBinOpIntoSelect()
2481 (BinOpcode == ISD::AND || BinOpcode == ISD::OR) && in foldBinOpIntoSelect()
2496 if ((BinOpcode == ISD::AND && isNullOrNullSplat(CT)) || in foldBinOpIntoSelect()
2497 (BinOpcode == ISD::OR && isAllOnesOrAllOnesSplat(CT))) in foldBinOpIntoSelect()
2502 if ((BinOpcode == ISD::AND && isNullOrNullSplat(CF)) || in foldBinOpIntoSelect()
2503 (BinOpcode == ISD::OR && isAllOnesOrAllOnesSplat(CF))) in foldBinOpIntoSelect()
2530 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && in foldAddSubBoolOfMaskedVal()
2536 bool IsAdd = N->getOpcode() == ISD::ADD; in foldAddSubBoolOfMaskedVal()
2540 if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND) in foldAddSubBoolOfMaskedVal()
2549 m_SpecificCondCode(ISD::SETEQ)))) in foldAddSubBoolOfMaskedVal()
2560 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit); in foldAddSubBoolOfMaskedVal()
2569 if ((!LegalOperations || hasOperation(ISD::AVGCEILU, VT)) && in foldSubToAvg()
2573 return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B); in foldSubToAvg()
2575 if ((!LegalOperations || hasOperation(ISD::AVGCEILS, VT)) && in foldSubToAvg()
2579 return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B); in foldSubToAvg()
2588 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && in foldAddSubOfSignBit()
2593 bool IsAdd = N->getOpcode() == ISD::ADD; in foldAddSubOfSignBit()
2597 ShiftOp.getOpcode() != ISD::SRL) in foldAddSubOfSignBit()
2616 IsAdd ? ISD::ADD : ISD::SUB, DL, VT, in foldAddSubOfSignBit()
2618 SDValue NewShift = DAG.getNode(IsAdd ? ISD::SRA : ISD::SRL, DL, VT, in foldAddSubOfSignBit()
2620 return DAG.getNode(ISD::ADD, DL, VT, NewShift, NewC); in foldAddSubOfSignBit()
2648 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1})) in visitADDLike()
2654 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitADDLike()
2665 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitADDLike()
2673 if (N0.getOpcode() == ISD::SUB) { in visitADDLike()
2678 if (SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N1, N01})) in visitADDLike()
2679 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub); in visitADDLike()
2682 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N00})) in visitADDLike()
2683 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLike()
2690 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() && in visitADDLike()
2694 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) && in visitADDLike()
2695 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) && in visitADDLike()
2698 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not); in visitADDLike()
2708 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N01})) in visitADDLike()
2709 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add); in visitADDLike()
2716 if (!reassociationCanBreakAddressingModePattern(ISD::ADD, DL, N, N0, N1)) { in visitADDLike()
2717 if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags())) in visitADDLike()
2737 ISD::ADD, DL, VT, in visitADDLike()
2738 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)), in visitADDLike()
2750 reassociateReduction(ISD::VECREDUCE_ADD, ISD::ADD, DL, VT, N0, N1)) in visitADDLike()
2758 return DAG.getNode(ISD::SUB, DL, VT, N1, A); in visitADDLike()
2762 return DAG.getNode(ISD::SUB, DL, VT, N0, B); in visitADDLike()
2775 return DAG.getNode(ISD::SUB, DL, VT, C, B); in visitADDLike()
2780 return DAG.getNode(ISD::SUB, DL, VT, A, C); in visitADDLike()
2785 return DAG.getNode(ISD::SUB, DL, VT, B, C); in visitADDLike()
2797 return DAG.getNode(ISD::SUB, DL, VT, in visitADDLike()
2798 DAG.getNode(ISD::ADD, SDLoc(N0), VT, A, C), in visitADDLike()
2799 DAG.getNode(ISD::ADD, SDLoc(N1), VT, B, D)); in visitADDLike()
2802 if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) { in visitADDLike()
2807 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchUSUBSAT, in visitADDLike()
2809 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), in visitADDLike()
2819 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in visitADDLike()
2823 if (N0.getOpcode() == ISD::ADD) { in visitADDLike()
2835 return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0)); in visitADDLike()
2842 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLike()
2848 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not); in visitADDLike()
2853 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitADDLike()
2856 return DAG.getNode(ISD::ADD, DL, VT, Not, N0.getOperand(0)); in visitADDLike()
2880 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N1), VT, A, in visitADDLike()
2883 ISD::ADD, DL, VT, Mul, in visitADDLike()
2909 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N1), VT, A, in visitADDLike()
2911 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N1), VT, Mul, B, Flags); in visitADDLike()
2913 ISD::ADD, DL, VT, Add, in visitADDLike()
2934 if ((!LegalOperations || hasOperation(ISD::AVGFLOORU, VT)) && in foldAddToAvg()
2938 return DAG.getNode(ISD::AVGFLOORU, DL, VT, A, B); in foldAddToAvg()
2940 if ((!LegalOperations || hasOperation(ISD::AVGFLOORS, VT)) && in foldAddToAvg()
2944 return DAG.getNode(ISD::AVGFLOORS, DL, VT, A, B); in foldAddToAvg()
2970 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) && in visitADD()
2974 return DAG.getNode(ISD::OR, DL, VT, N0, N1, Flags); in visitADD()
2978 if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) { in visitADD()
2985 if (N0.getOpcode() == ISD::ADD && in visitADD()
2986 N0.getOperand(1).getOpcode() == ISD::VSCALE && in visitADD()
2987 N1.getOpcode() == ISD::VSCALE) { in visitADD()
2991 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS); in visitADD()
2995 if (N0.getOpcode() == ISD::STEP_VECTOR && in visitADD()
2996 N1.getOpcode() == ISD::STEP_VECTOR) { in visitADD()
3004 if (N0.getOpcode() == ISD::ADD && in visitADD()
3005 N0.getOperand(1).getOpcode() == ISD::STEP_VECTOR && in visitADD()
3006 N1.getOpcode() == ISD::STEP_VECTOR) { in visitADD()
3011 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), SV); in visitADD()
3022 bool IsSigned = Opcode == ISD::SADDSAT; in visitADDSAT()
3044 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitADDSAT()
3054 return DAG.getNode(ISD::ADD, DL, VT, N0, N1); in visitADDSAT()
3065 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) { in getAsCarry()
3070 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) { in getAsCarry()
3089 if (V.getOpcode() != ISD::UADDO_CARRY && V.getOpcode() != ISD::USUBO_CARRY && in getAsCarry()
3090 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO) in getAsCarry()
3113 if (N1.getOpcode() == ISD::ZERO_EXTEND) in foldAddSubMasked1()
3116 if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(N1->getOperand(1))) in foldAddSubMasked1()
3121 if (N10.getValueType() != VT && N10.getOpcode() == ISD::TRUNCATE) in foldAddSubMasked1()
3132 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N10); in foldAddSubMasked1()
3144 return DAG.getNode(ISD::SUB, DL, VT, N0, in visitADDLikeCommutative()
3145 DAG.getNode(ISD::SHL, DL, VT, Y, N)); in visitADDLikeCommutative()
3154 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLikeCommutative()
3160 return DAG.getNode(ISD::SUB, DL, VT, N1, Not); in visitADDLikeCommutative()
3163 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse()) { in visitADDLikeCommutative()
3168 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1); in visitADDLikeCommutative()
3169 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLikeCommutative()
3174 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLikeCommutative()
3175 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0)); in visitADDLikeCommutative()
3180 if (N0.getOpcode() == ISD::MUL && N0.getOperand(0) == N1 && in visitADDLikeCommutative()
3183 SDValue NewC = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), in visitADDLikeCommutative()
3185 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), NewC); in visitADDLikeCommutative()
3191 if (N0.getOpcode() == ISD::SIGN_EXTEND && in visitADDLikeCommutative()
3194 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitADDLikeCommutative()
3195 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt); in visitADDLikeCommutative()
3199 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitADDLikeCommutative()
3202 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitADDLikeCommutative()
3204 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt); in visitADDLikeCommutative()
3209 if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(N1.getOperand(1)) && in visitADDLikeCommutative()
3211 return DAG.getNode(ISD::UADDO_CARRY, DL, N1->getVTList(), in visitADDLikeCommutative()
3215 if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT)) in visitADDLikeCommutative()
3217 return DAG.getNode(ISD::UADDO_CARRY, DL, in visitADDLikeCommutative()
3232 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
3233 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitADDC()
3239 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0); in visitADDC()
3243 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, in visitADDC()
3248 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
3249 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitADDC()
3267 if (V.getOpcode() != ISD::XOR) in extractBooleanFlip()
3300 bool IsSigned = (ISD::SADDO == N->getOpcode()); in visitADDO()
3307 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
3321 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
3327 return DAG.getNode(ISD::SSUBO, DL, N->getVTList(), in visitADDO()
3332 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(), in visitADDO()
3355 if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(N1.getOperand(1))) { in visitUADDOLike()
3359 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0, Y, in visitUADDOLike()
3364 if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT)) in visitUADDOLike()
3366 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0, in visitUADDOLike()
3381 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(), in visitADDE()
3385 if (CarryIn.getOpcode() == ISD::CARRY_FALSE) in visitADDE()
3386 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1); in visitADDE()
3401 return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn); in visitUADDO_CARRY()
3406 TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0))) in visitUADDO_CARRY()
3407 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1); in visitUADDO_CARRY()
3416 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt, in visitUADDO_CARRY()
3434 DAG.getNodeIfExists(ISD::UADDO_CARRY, N->getVTList(), Ops, N->getFlags()); in visitUADDO_CARRY()
3469 if (Carry1.getOpcode() != ISD::UADDO) in combineUADDO_CARRYDiamond()
3478 if (Carry0.getOpcode() == ISD::UADDO_CARRY && in combineUADDO_CARRYDiamond()
3481 } else if (Carry0.getOpcode() == ISD::UADDO && in combineUADDO_CARRYDiamond()
3494 DAG.getNode(ISD::UADDO_CARRY, DL, Carry0->getVTList(), A, B, Z); in combineUADDO_CARRYDiamond()
3496 return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), X, in combineUADDO_CARRYDiamond()
3568 if (Opcode != ISD::UADDO && Opcode != ISD::USUBO) in combineCarryDiamond()
3589 if (Opcode == ISD::USUBO && CarryInOperandNum != 1) in combineCarryDiamond()
3593 unsigned NewOp = Opcode == ISD::UADDO ? ISD::UADDO_CARRY : ISD::USUBO_CARRY; in combineCarryDiamond()
3623 if (N->getOpcode() == ISD::AND) in combineCarryDiamond()
3635 SDValue Sub = DAG.getNode(ISD::USUBO_CARRY, DL, N->getVTList(), N1, in visitUADDO_CARRYLike()
3645 if ((N0.getOpcode() == ISD::ADD || in visitUADDO_CARRYLike()
3646 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 && in visitUADDO_CARRYLike()
3649 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), in visitUADDO_CARRYLike()
3673 return DAG.getNode(ISD::SSUBO_CARRY, SDLoc(N), N->getVTList(), N1, in visitSADDO_CARRYLike()
3690 return DAG.getNode(ISD::SADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn); in visitSADDO_CARRY()
3695 TLI.isOperationLegalOrCustom(ISD::SADDO, N->getValueType(0))) in visitSADDO_CARRY()
3696 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, N1); in visitSADDO_CARRY()
3717 return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS); in getTruncatedUSUBSAT()
3730 RHS = DAG.getNode(ISD::UMIN, DL, SrcVT, RHS, SatLimit); in getTruncatedUSUBSAT()
3731 RHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, RHS); in getTruncatedUSUBSAT()
3732 LHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, LHS); in getTruncatedUSUBSAT()
3733 return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS); in getTruncatedUSUBSAT()
3739 if (N->getOpcode() != ISD::SUB || in foldSubToUSubSat()
3740 !(!LegalOperations || hasOperation(ISD::USUBSAT, DstVT))) in foldSubToUSubSat()
3749 if (Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) { in foldSubToUSubSat()
3758 if (Op1.getOpcode() == ISD::UMIN && Op1.hasOneUse()) { in foldSubToUSubSat()
3768 if (Op1.getOpcode() == ISD::TRUNCATE && in foldSubToUSubSat()
3769 Op1.getOperand(0).getOpcode() == ISD::UMIN && in foldSubToUSubSat()
3773 if (MinLHS.getOpcode() == ISD::ZERO_EXTEND && MinLHS.getOperand(0) == Op0) in foldSubToUSubSat()
3776 if (MinRHS.getOpcode() == ISD::ZERO_EXTEND && MinRHS.getOperand(0) == Op0) in foldSubToUSubSat()
3790 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) in tryFoldToZero()
3803 if (N->getOpcode() == ISD::FREEZE && N.hasOneUse()) in visitSUB()
3814 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1})) in visitSUB()
3823 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitSUB()
3832 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3840 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) { in visitSUB()
3843 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA; in visitSUB()
3864 if (N1.getOpcode() == ISD::ABS && N1.hasOneUse() && in visitSUB()
3865 !TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitSUB()
3872 if (N1S && N1S.getOpcode() == ISD::SUB && in visitSUB()
3880 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
3883 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0))) in visitSUB()
3884 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1)); in visitSUB()
3887 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0)) in visitSUB()
3891 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1) in visitSUB()
3895 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1) in visitSUB()
3899 if (N0.getOpcode() == ISD::ADD) { in visitSUB()
3901 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N01, N1})) in visitSUB()
3902 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3906 if (N1.getOpcode() == ISD::ADD) { in visitSUB()
3908 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11})) in visitSUB()
3909 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0)); in visitSUB()
3913 if (N0.getOpcode() == ISD::SUB) { in visitSUB()
3915 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N01, N1})) in visitSUB()
3916 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3920 if (N0.getOpcode() == ISD::SUB) { in visitSUB()
3922 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N00, N1})) in visitSUB()
3923 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1)); in visitSUB()
3930 return DAG.getNode(ISD::ADD, DL, VT, A, C); in visitSUB()
3934 return DAG.getNode(ISD::SUB, DL, VT, A, C); in visitSUB()
3938 return DAG.getNode(ISD::SUB, DL, VT, A, B); in visitSUB()
3942 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3943 DAG.getNode(ISD::SUB, DL, VT, C, B)); in visitSUB()
3948 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getNOT(DL, B, VT)); in visitSUB()
3952 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3953 DAG.getNode(ISD::MUL, DL, VT, B, C)); in visitSUB()
3979 return DAG.getNode(ISD::ADD, DL, VT, A, DAG.getNOT(DL, B, VT)); in visitSUB()
3986 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0)); in visitSUB()
3987 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT)); in visitSUB()
3992 if (!reassociationCanBreakAddressingModePattern(ISD::SUB, DL, N, N0, N1) && in visitSUB()
3993 N0.getOpcode() == ISD::ADD && N0.hasOneUse() && in visitSUB()
3995 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
3996 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
3999 if (N1.getOpcode() == ISD::ADD && N1.hasOneUse() && in visitSUB()
4001 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0)); in visitSUB()
4002 return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1)); in visitSUB()
4006 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitSUB()
4008 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
4009 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
4012 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitSUB()
4014 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1); in visitSUB()
4015 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add); in visitSUB()
4021 if (N1.getOpcode() == ISD::ZERO_EXTEND && in visitSUB()
4025 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0)); in visitSUB()
4026 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt); in visitSUB()
4030 if ((!LegalOperations || hasOperation(ISD::ABS, VT)) && in visitSUB()
4033 return DAG.getNode(ISD::ABS, DL, VT, A); in visitSUB()
4046 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitSUB()
4049 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitSUB()
4051 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt); in visitSUB()
4056 if (N1.getOpcode() == ISD::VSCALE && N1.hasOneUse()) { in visitSUB()
4058 return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal)); in visitSUB()
4062 if (N1.getOpcode() == ISD::STEP_VECTOR && N1.hasOneUse()) { in visitSUB()
4064 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
4070 if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) { in visitSUB()
4074 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt); in visitSUB()
4075 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA); in visitSUB()
4082 if (N1.getOpcode() == ISD::SHL) { in visitSUB()
4085 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitSUB()
4089 if (N0.getOpcode() == ISD::USUBO_CARRY && isNullConstant(N0.getOperand(1)) && in visitSUB()
4091 return DAG.getNode(ISD::USUBO_CARRY, DL, N0->getVTList(), in visitSUB()
4094 if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT)) { in visitSUB()
4099 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X); in visitSUB()
4100 return DAG.getNode(ISD::UADDO_CARRY, DL, in visitSUB()
4113 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
4118 if (hasOperation(ISD::ABDS, VT) && in visitSUB()
4121 return DAG.getNode(ISD::ABDS, DL, VT, A, B); in visitSUB()
4124 if (hasOperation(ISD::ABDU, VT) && in visitSUB()
4127 return DAG.getNode(ISD::ABDU, DL, VT, A, B); in visitSUB()
4137 bool IsSigned = Opcode == ISD::SSUBSAT; in visitSUBSAT()
4158 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitSUBSAT()
4168 return DAG.getNode(ISD::SUB, DL, VT, N0, N1); in visitSUBSAT()
4181 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBC()
4182 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
4187 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
4191 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
4195 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBC()
4196 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
4205 bool IsSigned = (ISD::SSUBO == N->getOpcode()); in visitSUBO()
4212 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBO()
4223 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, in visitSUBO()
4232 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBO()
4237 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBO()
4249 if (CarryIn.getOpcode() == ISD::CARRY_FALSE) in visitSUBE()
4250 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1); in visitSUBE()
4263 TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0))) in visitUSUBO_CARRY()
4264 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1); in visitUSUBO_CARRY()
4278 TLI.isOperationLegalOrCustom(ISD::SSUBO, N->getValueType(0))) in visitSSUBO_CARRY()
4279 return DAG.getNode(ISD::SSUBO, SDLoc(N), N->getVTList(), N0, N1); in visitSSUBO_CARRY()
4323 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, DL, VT, {N0, N1})) in visitMUL()
4329 return Matcher.getNode(ISD::MUL, DL, VT, N1, N0); in visitMUL()
4342 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1); in visitMUL()
4367 return Matcher.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0); in visitMUL()
4375 return Matcher.getNode(ISD::SHL, DL, VT, N0, Trunc); in visitMUL()
4386 ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in visitMUL()
4387 Matcher.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4394 for (unsigned LoHiOpc : {ISD::UMUL_LOHI, ISD::SMUL_LOHI}) { in visitMUL()
4428 unsigned MathOp = ISD::DELETED_NODE; in visitMUL()
4434 MathOp = ISD::ADD; in visitMUL()
4436 MathOp = ISD::SUB; in visitMUL()
4438 if (MathOp != ISD::DELETED_NODE) { in visitMUL()
4440 MathOp == ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2(); in visitMUL()
4445 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT)); in visitMUL()
4448 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4458 if (sd_context_match(N0, Matcher, m_Opc(ISD::SHL))) { in visitMUL()
4460 if (SDValue C3 = DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N1, N01})) in visitMUL()
4461 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), C3); in visitMUL()
4470 if (sd_context_match(N0, Matcher, m_OneUse(m_Opc(ISD::SHL))) && in visitMUL()
4473 } else if (sd_context_match(N1, Matcher, m_OneUse(m_Opc(ISD::SHL))) && in visitMUL()
4479 SDValue Mul = Matcher.getNode(ISD::MUL, DL, VT, Sh.getOperand(0), Y); in visitMUL()
4480 return Matcher.getNode(ISD::SHL, DL, VT, Mul, Sh.getOperand(1)); in visitMUL()
4485 if (sd_context_match(N0, Matcher, m_Opc(ISD::ADD)) && in visitMUL()
4490 ISD::ADD, DL, VT, in visitMUL()
4491 Matcher.getNode(ISD::MUL, SDLoc(N0), VT, N0.getOperand(0), N1), in visitMUL()
4492 Matcher.getNode(ISD::MUL, SDLoc(N1), VT, N0.getOperand(1), N1)); in visitMUL()
4496 if (!UseVP && N0.getOpcode() == ISD::VSCALE && NC1) { in visitMUL()
4504 if (!UseVP && N0.getOpcode() == ISD::STEP_VECTOR && in visitMUL()
4505 ISD::isConstantSplatVector(N1.getNode(), MulVal)) { in visitMUL()
4513 if (!UseVP && (!LegalOperations || hasOperation(ISD::ABS, VT)) && in visitMUL()
4518 return Matcher.getNode(ISD::ABS, DL, VT, X); in visitMUL()
4537 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::AND, VT)) && in visitMUL()
4538 ISD::matchUnaryPredicate(N1, IsClearMask, /*AllowUndefs*/ true)) { in visitMUL()
4539 assert(N1.getOpcode() == ISD::BUILD_VECTOR && "Unknown constant vector"); in visitMUL()
4547 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getBuildVector(VT, DL, Mask)); in visitMUL()
4554 if (SDValue RMUL = reassociateOps(ISD::MUL, DL, N0, N1, N->getFlags())) in visitMUL()
4561 reassociateReduction(ISD::VECREDUCE_MUL, ISD::MUL, DL, VT, N0, N1)) in visitMUL()
4596 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM); in useDivRem()
4597 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in useDivRem()
4615 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) { in useDivRem()
4616 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM; in useDivRem()
4620 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV; in useDivRem()
4629 if (User == Node || User->getOpcode() == ISD::DELETED_NODE || in useDivRem()
4650 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV) in useDivRem()
4652 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM) in useDivRem()
4666 bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc); in simplifyDivRem()
4713 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1})) in visitSDIV()
4728 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitSDIV()
4741 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1); in visitSDIV()
4746 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(), in visitSDIV()
4748 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitSDIV()
4749 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitSDIV()
4781 return ISD::matchUnaryPredicate(Divisor, IsPowerOfTwo); in isDivisorPowerOfTwo()
4802 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1); in visitSDIVLike()
4804 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1); in visitSDIVLike()
4809 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0, in visitSDIVLike()
4814 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact); in visitSDIVLike()
4816 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl); in visitSDIVLike()
4818 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1); in visitSDIVLike()
4825 SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ); in visitSDIVLike()
4826 SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ); in visitSDIVLike()
4827 SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes); in visitSDIVLike()
4833 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra); in visitSDIVLike()
4836 SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT); in visitSDIVLike()
4861 if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1})) in visitUDIV()
4872 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitUDIV()
4886 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(), in visitUDIV()
4888 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitUDIV()
4889 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitUDIV()
4920 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitUDIVLike()
4925 if (N1.getOpcode() == ISD::SHL) { in visitUDIVLike()
4934 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc); in visitUDIVLike()
4936 return DAG.getNode(ISD::SRL, DL, VT, N0, Add); in visitUDIVLike()
4953 !DAG.doesNodeExist(ISD::SDIV, N->getVTList(), {N0, N1})) { in buildOptimizedSREM()
4969 bool isSigned = (Opcode == ISD::SREM); in visitREM()
4981 SDValue EqualsNeg1 = DAG.getSetCC(DL, CCVT, F0, N1, ISD::SETEQ); in visitREM()
4995 return DAG.getNode(ISD::UREM, DL, VT, N0, N1); in visitREM()
5000 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
5002 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
5008 if ((N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) && in visitREM()
5011 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
5013 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
5037 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV; in visitREM()
5041 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1); in visitREM()
5042 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitREM()
5063 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1})) in visitMULHS()
5069 return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0); in visitMULHS()
5077 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitMULHS()
5088 ISD::SRA, DL, VT, N0, in visitMULHS()
5097 if (!TLI.isOperationLegalOrCustom(ISD::MULHS, VT) && VT.isSimple() && in visitMULHS()
5102 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitMULHS()
5103 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0); in visitMULHS()
5104 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1); in visitMULHS()
5105 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); in visitMULHS()
5106 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1, in visitMULHS()
5108 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHS()
5122 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1})) in visitMULHU()
5128 return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0); in visitMULHU()
5136 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitMULHU()
5154 hasOperation(ISD::SRL, VT)) { in visitMULHU()
5158 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2); in visitMULHU()
5161 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitMULHU()
5167 if (!TLI.isOperationLegalOrCustom(ISD::MULHU, VT) && VT.isSimple() && in visitMULHU()
5172 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitMULHU()
5173 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0); in visitMULHU()
5174 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1); in visitMULHU()
5175 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); in visitMULHU()
5176 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1, in visitMULHU()
5178 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHU()
5197 bool IsSigned = Opcode == ISD::AVGCEILS || Opcode == ISD::AVGFLOORS; in visitAVG()
5224 if (sd_match(N, m_c_BinOp(ISD::AVGFLOORS, m_Value(X), m_Zero()))) in visitAVG()
5225 return DAG.getNode(ISD::SRA, DL, VT, X, in visitAVG()
5227 if (sd_match(N, m_c_BinOp(ISD::AVGFLOORU, m_Value(X), m_Zero()))) in visitAVG()
5228 return DAG.getNode(ISD::SRL, DL, VT, X, in visitAVG()
5238 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, AvgU); in visitAVG()
5245 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, AvgS); in visitAVG()
5251 if (Opcode == ISD::AVGFLOORU && !hasOperation(ISD::AVGFLOORU, VT) && in visitAVG()
5252 (!LegalOperations || hasOperation(ISD::AVGCEILU, VT))) { in visitAVG()
5255 ISD::AVGCEILU, DL, VT, N0, in visitAVG()
5256 DAG.getNode(ISD::ADD, DL, VT, N1, DAG.getAllOnesConstant(DL, VT))); in visitAVG()
5259 ISD::AVGCEILU, DL, VT, N1, in visitAVG()
5260 DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getAllOnesConstant(DL, VT))); in visitAVG()
5293 if (sd_match(N, m_c_BinOp(ISD::ABDS, m_Value(X), m_Zero())) && in visitABD()
5294 (!LegalOperations || hasOperation(ISD::ABS, VT))) in visitABD()
5295 return DAG.getNode(ISD::ABS, DL, VT, X); in visitABD()
5298 if (sd_match(N, m_c_BinOp(ISD::ABDU, m_Value(X), m_Zero()))) in visitABD()
5302 if (Opcode == ISD::ABDS && hasOperation(ISD::ABDU, VT) && in visitABD()
5304 return DAG.getNode(ISD::ABDU, DL, VT, N1, N0); in visitABD()
5359 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS)) in visitSMUL_LOHI()
5369 return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N0, N1); in visitSMUL_LOHI()
5374 return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N1, N0); in visitSMUL_LOHI()
5382 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitSMUL_LOHI()
5383 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0); in visitSMUL_LOHI()
5384 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1); in visitSMUL_LOHI()
5385 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi); in visitSMUL_LOHI()
5387 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo, in visitSMUL_LOHI()
5389 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitSMUL_LOHI()
5391 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitSMUL_LOHI()
5400 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU)) in visitUMUL_LOHI()
5410 return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N0, N1); in visitUMUL_LOHI()
5415 return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N1, N0); in visitUMUL_LOHI()
5435 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitUMUL_LOHI()
5436 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0); in visitUMUL_LOHI()
5437 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1); in visitUMUL_LOHI()
5438 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi); in visitUMUL_LOHI()
5440 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo, in visitUMUL_LOHI()
5442 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitUMUL_LOHI()
5444 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitUMUL_LOHI()
5456 bool IsSigned = (ISD::SMULO == N->getOpcode()); in visitMULO()
5490 return DAG.getNode(IsSigned ? ISD::SADDO : ISD::UADDO, DL, in visitMULO()
5495 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, N1); in visitMULO()
5497 DAG.getConstant(0, DL, VT), ISD::SETNE); in visitMULO()
5503 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1), in visitMULO()
5515 SDValue N3, ISD::CondCode CC, unsigned &BW, in isSaturatingMinMax()
5518 ISD::CondCode CC) { in isSaturatingMinMax()
5521 if (N0 != N2 && (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0))) in isSaturatingMinMax()
5532 return CC == ISD::SETLT ? ISD::SMIN : (CC == ISD::SETGT ? ISD::SMAX : 0); in isSaturatingMinMax()
5542 if (N0.getOpcode() == ISD::FP_TO_SINT && Opcode0 == ISD::SMAX) { in isSaturatingMinMax()
5561 ISD::CondCode N0CC; in isSaturatingMinMax()
5563 case ISD::SMIN: in isSaturatingMinMax()
5564 case ISD::SMAX: in isSaturatingMinMax()
5567 N0CC = N0.getOpcode() == ISD::SMIN ? ISD::SETLT : ISD::SETGT; in isSaturatingMinMax()
5569 case ISD::SELECT_CC: in isSaturatingMinMax()
5576 case ISD::SELECT: in isSaturatingMinMax()
5577 case ISD::VSELECT: in isSaturatingMinMax()
5578 if (N0.getOperand(0).getOpcode() != ISD::SETCC) in isSaturatingMinMax()
5594 ConstantSDNode *MinCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N1 : N01); in isSaturatingMinMax()
5595 ConstantSDNode *MaxCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N01 : N1); in isSaturatingMinMax()
5618 SDValue N3, ISD::CondCode CC, in PerformMinMaxFpToSatCombine()
5623 if (!Fp || Fp.getOpcode() != ISD::FP_TO_SINT) in PerformMinMaxFpToSatCombine()
5630 unsigned NewOpc = Unsigned ? ISD::FP_TO_UINT_SAT : ISD::FP_TO_SINT_SAT; in PerformMinMaxFpToSatCombine()
5640 SDValue N3, ISD::CondCode CC, in PerformUMinFpToSatCombine()
5646 (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0))) || in PerformUMinFpToSatCombine()
5647 N0.getOpcode() != ISD::FP_TO_UINT || CC != ISD::SETULT) in PerformUMinFpToSatCombine()
5665 if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(ISD::FP_TO_UINT_SAT, in PerformUMinFpToSatCombine()
5670 DAG.getNode(ISD::FP_TO_UINT_SAT, SDLoc(N0), NewVT, N0.getOperand(0), in PerformUMinFpToSatCombine()
5709 bool IsSatBroken = Opcode == ISD::UMIN && N0.getOpcode() == ISD::SMAX; in visitIMINMAX()
5714 case ISD::SMIN: AltOpcode = ISD::UMIN; break; in visitIMINMAX()
5715 case ISD::SMAX: AltOpcode = ISD::UMAX; break; in visitIMINMAX()
5716 case ISD::UMIN: AltOpcode = ISD::SMIN; break; in visitIMINMAX()
5717 case ISD::UMAX: AltOpcode = ISD::SMAX; break; in visitIMINMAX()
5724 if (Opcode == ISD::SMIN || Opcode == ISD::SMAX) in visitIMINMAX()
5726 N0, N1, N0, N1, Opcode == ISD::SMIN ? ISD::SETLT : ISD::SETGT, DAG)) in visitIMINMAX()
5728 if (Opcode == ISD::UMIN) in visitIMINMAX()
5729 if (SDValue S = PerformUMinFpToSatCombine(N0, N1, N0, N1, ISD::SETULT, DAG)) in visitIMINMAX()
5735 case ISD::SMIN: in visitIMINMAX()
5736 return ISD::VECREDUCE_SMIN; in visitIMINMAX()
5737 case ISD::SMAX: in visitIMINMAX()
5738 return ISD::VECREDUCE_SMAX; in visitIMINMAX()
5739 case ISD::UMIN: in visitIMINMAX()
5740 return ISD::VECREDUCE_UMIN; in visitIMINMAX()
5741 case ISD::UMAX: in visitIMINMAX()
5742 return ISD::VECREDUCE_UMAX; in visitIMINMAX()
5765 assert(ISD::isBitwiseLogicOp(LogicOpcode) && "Expected logic opcode"); in hoistLogicOpWithSameOpcodeHands()
5780 if (ISD::isExtOpcode(HandOpcode) || ISD::isExtVecInRegOpcode(HandOpcode) || in hoistLogicOpWithSameOpcodeHands()
5781 (HandOpcode == ISD::SIGN_EXTEND_INREG && in hoistLogicOpWithSameOpcodeHands()
5797 if ((HandOpcode == ISD::ANY_EXTEND || in hoistLogicOpWithSameOpcodeHands()
5798 HandOpcode == ISD::ANY_EXTEND_VECTOR_INREG) && in hoistLogicOpWithSameOpcodeHands()
5803 if (HandOpcode == ISD::SIGN_EXTEND_INREG) in hoistLogicOpWithSameOpcodeHands()
5809 if (HandOpcode == ISD::TRUNCATE) { in hoistLogicOpWithSameOpcodeHands()
5832 if ((HandOpcode == ISD::SHL || HandOpcode == ISD::SRL || in hoistLogicOpWithSameOpcodeHands()
5833 HandOpcode == ISD::SRA || HandOpcode == ISD::AND) && in hoistLogicOpWithSameOpcodeHands()
5843 if (HandOpcode == ISD::BSWAP) { in hoistLogicOpWithSameOpcodeHands()
5854 if ((HandOpcode == ISD::FSHL || HandOpcode == ISD::FSHR) && in hoistLogicOpWithSameOpcodeHands()
5873 if ((HandOpcode == ISD::BITCAST || HandOpcode == ISD::SCALAR_TO_VECTOR) && in hoistLogicOpWithSameOpcodeHands()
5896 if (HandOpcode == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) { in hoistLogicOpWithSameOpcodeHands()
5913 if (LogicOpcode == ISD::XOR && !ShOp.isUndef()) in hoistLogicOpWithSameOpcodeHands()
5926 if (LogicOpcode == ISD::XOR && !ShOp.isUndef()) in hoistLogicOpWithSameOpcodeHands()
5965 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get(); in foldLogicOfSetCCs()
5966 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get(); in foldLogicOfSetCCs()
5973 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero; in foldLogicOfSetCCs()
5975 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1; in foldLogicOfSetCCs()
5977 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero; in foldLogicOfSetCCs()
5979 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero; in foldLogicOfSetCCs()
5986 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL); in foldLogicOfSetCCs()
5992 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1; in foldLogicOfSetCCs()
5994 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero; in foldLogicOfSetCCs()
5996 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1; in foldLogicOfSetCCs()
5998 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1; in foldLogicOfSetCCs()
6005 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL); in foldLogicOfSetCCs()
6014 IsInteger && CC0 == ISD::SETNE && in foldLogicOfSetCCs()
6019 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One); in foldLogicOfSetCCs()
6021 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE); in foldLogicOfSetCCs()
6030 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) { in foldLogicOfSetCCs()
6031 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR); in foldLogicOfSetCCs()
6032 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR); in foldLogicOfSetCCs()
6033 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR); in foldLogicOfSetCCs()
6039 if ((IsAnd && CC1 == ISD::SETNE) || (!IsAnd && CC1 == ISD::SETEQ)) { in foldLogicOfSetCCs()
6049 if (LL == RL && ISD::matchBinaryPredicate(LR, RR, MatchDiffPow2)) { in foldLogicOfSetCCs()
6052 SDValue Max = DAG.getNode(ISD::UMAX, DL, OpVT, LR, RR); in foldLogicOfSetCCs()
6053 SDValue Min = DAG.getNode(ISD::UMIN, DL, OpVT, LR, RR); in foldLogicOfSetCCs()
6054 SDValue Offset = DAG.getNode(ISD::SUB, DL, OpVT, LL, Min); in foldLogicOfSetCCs()
6055 SDValue Diff = DAG.getNode(ISD::SUB, DL, OpVT, Max, Min); in foldLogicOfSetCCs()
6057 SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Offset, Mask); in foldLogicOfSetCCs()
6066 CC1 = ISD::getSetCCSwappedOperands(CC1); in foldLogicOfSetCCs()
6073 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, OpVT) in foldLogicOfSetCCs()
6074 : ISD::getSetCCOrOperation(CC0, CC1, OpVT); in foldLogicOfSetCCs()
6075 if (NewCC != ISD::SETCC_INVALID && in foldLogicOfSetCCs()
6078 TLI.isOperationLegal(ISD::SETCC, OpVT)))) in foldLogicOfSetCCs()
6096 ISD::CondCode CC, unsigned OrAndOpcode, in getMinMaxOpcodeForFP()
6107 if (((CC == ISD::SETLT || CC == ISD::SETLE) && (OrAndOpcode == ISD::OR)) || in getMinMaxOpcodeForFP()
6108 ((CC == ISD::SETGT || CC == ISD::SETGE) && (OrAndOpcode == ISD::AND))) in getMinMaxOpcodeForFP()
6111 ? ISD::FMINNUM_IEEE in getMinMaxOpcodeForFP()
6112 : ISD::DELETED_NODE; in getMinMaxOpcodeForFP()
6113 else if (((CC == ISD::SETGT || CC == ISD::SETGE) && in getMinMaxOpcodeForFP()
6114 (OrAndOpcode == ISD::OR)) || in getMinMaxOpcodeForFP()
6115 ((CC == ISD::SETLT || CC == ISD::SETLE) && in getMinMaxOpcodeForFP()
6116 (OrAndOpcode == ISD::AND))) in getMinMaxOpcodeForFP()
6119 ? ISD::FMAXNUM_IEEE in getMinMaxOpcodeForFP()
6120 : ISD::DELETED_NODE; in getMinMaxOpcodeForFP()
6130 else if (((CC == ISD::SETOLT || CC == ISD::SETOLE) && in getMinMaxOpcodeForFP()
6131 (OrAndOpcode == ISD::OR)) || in getMinMaxOpcodeForFP()
6132 ((CC == ISD::SETUGT || CC == ISD::SETUGE) && in getMinMaxOpcodeForFP()
6133 (OrAndOpcode == ISD::AND))) in getMinMaxOpcodeForFP()
6134 return isFMAXNUMFMINNUM ? ISD::FMINNUM in getMinMaxOpcodeForFP()
6137 ? ISD::FMINNUM_IEEE in getMinMaxOpcodeForFP()
6138 : ISD::DELETED_NODE; in getMinMaxOpcodeForFP()
6139 else if (((CC == ISD::SETOGT || CC == ISD::SETOGE) && in getMinMaxOpcodeForFP()
6140 (OrAndOpcode == ISD::OR)) || in getMinMaxOpcodeForFP()
6141 ((CC == ISD::SETULT || CC == ISD::SETULE) && in getMinMaxOpcodeForFP()
6142 (OrAndOpcode == ISD::AND))) in getMinMaxOpcodeForFP()
6143 return isFMAXNUMFMINNUM ? ISD::FMAXNUM in getMinMaxOpcodeForFP()
6146 ? ISD::FMAXNUM_IEEE in getMinMaxOpcodeForFP()
6147 : ISD::DELETED_NODE; in getMinMaxOpcodeForFP()
6148 return ISD::DELETED_NODE; in getMinMaxOpcodeForFP()
6154 (LogicOp->getOpcode() == ISD::AND || LogicOp->getOpcode() == ISD::OR) && in foldAndOrOfSETCC()
6160 if (LHS->getOpcode() != ISD::SETCC || RHS->getOpcode() != ISD::SETCC || in foldAndOrOfSETCC()
6176 ISD::CondCode CCL = cast<CondCodeSDNode>(LHS.getOperand(2))->get(); in foldAndOrOfSETCC()
6177 ISD::CondCode CCR = cast<CondCodeSDNode>(RHS.getOperand(2))->get(); in foldAndOrOfSETCC()
6192 bool isFMAXNUMFMINNUM_IEEE = TLI.isOperationLegal(ISD::FMAXNUM_IEEE, OpVT) && in foldAndOrOfSETCC()
6193 TLI.isOperationLegal(ISD::FMINNUM_IEEE, OpVT); in foldAndOrOfSETCC()
6194 bool isFMAXNUMFMINNUM = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, OpVT) && in foldAndOrOfSETCC()
6195 TLI.isOperationLegalOrCustom(ISD::FMINNUM, OpVT); in foldAndOrOfSETCC()
6196 if (((OpVT.isInteger() && TLI.isOperationLegal(ISD::UMAX, OpVT) && in foldAndOrOfSETCC()
6197 TLI.isOperationLegal(ISD::SMAX, OpVT) && in foldAndOrOfSETCC()
6198 TLI.isOperationLegal(ISD::UMIN, OpVT) && in foldAndOrOfSETCC()
6199 TLI.isOperationLegal(ISD::SMIN, OpVT)) || in foldAndOrOfSETCC()
6202 !ISD::isIntEqualitySetCC(CCL) && !ISD::isFPEqualitySetCC(CCL) && in foldAndOrOfSETCC()
6203 CCL != ISD::SETFALSE && CCL != ISD::SETO && CCL != ISD::SETUO && in foldAndOrOfSETCC()
6204 CCL != ISD::SETTRUE && in foldAndOrOfSETCC()
6205 (CCL == CCR || CCL == ISD::getSetCCSwappedOperands(CCR))) { in foldAndOrOfSETCC()
6208 ISD::CondCode CC = ISD::SETCC_INVALID; in foldAndOrOfSETCC()
6214 CC = ISD::getSetCCSwappedOperands(CCL); in foldAndOrOfSETCC()
6222 assert(CCL == ISD::getSetCCSwappedOperands(CCR) && "Unexpected CC"); in foldAndOrOfSETCC()
6238 if (CC == ISD::SETLT && isNullOrNullSplat(CommonValue)) in foldAndOrOfSETCC()
6239 CC = ISD::SETCC_INVALID; in foldAndOrOfSETCC()
6240 else if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(CommonValue)) in foldAndOrOfSETCC()
6241 CC = ISD::SETCC_INVALID; in foldAndOrOfSETCC()
6243 if (CC != ISD::SETCC_INVALID) { in foldAndOrOfSETCC()
6244 unsigned NewOpcode = ISD::DELETED_NODE; in foldAndOrOfSETCC()
6247 bool IsLess = (CC == ISD::SETLE || CC == ISD::SETULE || in foldAndOrOfSETCC()
6248 CC == ISD::SETLT || CC == ISD::SETULT); in foldAndOrOfSETCC()
6249 bool IsOr = (LogicOp->getOpcode() == ISD::OR); in foldAndOrOfSETCC()
6251 NewOpcode = IsSigned ? ISD::SMIN : ISD::UMIN; in foldAndOrOfSETCC()
6253 NewOpcode = IsSigned ? ISD::SMAX : ISD::UMAX; in foldAndOrOfSETCC()
6259 if (NewOpcode != ISD::DELETED_NODE) { in foldAndOrOfSETCC()
6271 CCL == (LogicOp->getOpcode() == ISD::AND ? ISD::SETNE : ISD::SETEQ) && in foldAndOrOfSETCC()
6280 DAG.doesNodeExist(ISD::ABS, DAG.getVTList(OpVT), {LHS0}))) { in foldAndOrOfSETCC()
6286 SDValue AbsOp = DAG.getNode(ISD::ABS, DL, OpVT, LHS0); in foldAndOrOfSETCC()
6287 return DAG.getNode(ISD::SETCC, DL, VT, AbsOp, in foldAndOrOfSETCC()
6315 SDValue AndOp = DAG.getNode(ISD::AND, DL, OpVT, NotOp, in foldAndOrOfSETCC()
6317 return DAG.getNode(ISD::SETCC, DL, VT, AndOp, in foldAndOrOfSETCC()
6321 SDValue AddOp = DAG.getNode(ISD::ADD, DL, OpVT, LHS0, in foldAndOrOfSETCC()
6323 SDValue AndOp = DAG.getNode(ISD::AND, DL, OpVT, AddOp, in foldAndOrOfSETCC()
6325 return DAG.getNode(ISD::SETCC, DL, VT, AndOp, in foldAndOrOfSETCC()
6349 if (T.getOpcode() != ISD::AND) in combineSelectAsExtAnd()
6359 return DAG.getNode(ISD::AND, DL, OpVT, CondMask, T.getOperand(0)); in combineSelectAsExtAnd()
6379 if (N1.getOpcode() == ISD::ADD) in visitANDLike()
6383 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL && in visitANDLike()
6402 DAG.getNode(ISD::ADD, DL0, VT, in visitANDLike()
6429 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) { in isAndLoadExtLoad()
6445 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT)) in isAndLoadExtLoad()
6448 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT)) in isAndLoadExtLoad()
6455 ISD::LoadExtType ExtType, EVT &MemVT, in isLegalNarrowLdSt()
6522 if (Load->getExtensionType() != ISD::NON_EXTLOAD && in isLegalNarrowLdSt()
6555 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) && in SearchForAndLoads()
6565 case ISD::LOAD: { in SearchForAndLoads()
6569 isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) { in SearchForAndLoads()
6572 if (Load->getExtensionType() == ISD::ZEXTLOAD && in SearchForAndLoads()
6584 case ISD::ZERO_EXTEND: in SearchForAndLoads()
6585 case ISD::AssertZext: { in SearchForAndLoads()
6588 EVT VT = Op.getOpcode() == ISD::AssertZext ? in SearchForAndLoads()
6598 case ISD::OR: in SearchForAndLoads()
6599 case ISD::XOR: in SearchForAndLoads()
6600 case ISD::AND: in SearchForAndLoads()
6657 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode), in BackwardsPropagateMask()
6661 if (And.getOpcode() == ISD ::AND) in BackwardsPropagateMask()
6672 DAG.getNode(ISD::AND, SDLoc(Op0), Op0.getValueType(), Op0, MaskOp); in BackwardsPropagateMask()
6676 DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(), Op1, MaskOp); in BackwardsPropagateMask()
6687 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0), in BackwardsPropagateMask()
6690 if (And.getOpcode() == ISD ::AND) in BackwardsPropagateMask()
6710 assert(N->getOpcode() == ISD::AND); in unfoldExtremeBitClearingToShifts()
6727 if (OuterShift == ISD::SHL) in unfoldExtremeBitClearingToShifts()
6728 InnerShift = ISD::SRL; in unfoldExtremeBitClearingToShifts()
6729 else if (OuterShift == ISD::SRL) in unfoldExtremeBitClearingToShifts()
6730 InnerShift = ISD::SHL; in unfoldExtremeBitClearingToShifts()
6762 assert(And->getOpcode() == ISD::AND && "Expected an 'and' op"); in combineShiftAnd1ToBitTest()
6766 if (And0.getOpcode() == ISD::ANY_EXTEND && And0.hasOneUse()) in combineShiftAnd1ToBitTest()
6783 if (Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse()) in combineShiftAnd1ToBitTest()
6788 if (Src.getOpcode() != ISD::SRL || !Src.hasOneUse()) in combineShiftAnd1ToBitTest()
6827 SDValue NewAnd = DAG.getNode(ISD::AND, DL, SrcVT, X, Mask); in combineShiftAnd1ToBitTest()
6829 SDValue Setcc = DAG.getSetCC(DL, CCVT, NewAnd, Zero, ISD::SETEQ); in combineShiftAnd1ToBitTest()
6852 return DAG.getNode(ISD::USUBSAT, DL, VT, X, in foldAndToUsubsat()
6863 assert(ISD::isBitwiseLogicOp(LogicOpcode) && in foldLogicOfShifts()
6872 !(ShiftOpcode == ISD::SHL || ShiftOpcode == ISD::SRL || in foldLogicOfShifts()
6873 ShiftOpcode == ISD::SRA)) in foldLogicOfShifts()
6911 assert(ISD::isBitwiseLogicOp(LogicOpcode) && in foldLogicTreeOfShifts()
6950 if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, DL, VT, {N0, N1})) in visitAND()
6956 return DAG.getNode(ISD::AND, DL, VT, N1, N0); in visitAND()
6967 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitAND()
6973 if (ISD::isConstantSplatVectorAllOnes(N1.getNode())) in visitAND()
6979 if (MLoad && MLoad->getExtensionType() == ISD::EXTLOAD && Splat && in visitAND()
6983 if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT, LoadVT)) { in visitAND()
6994 ISD::ZEXTLOAD, MLoad->isExpandingLoad()); in visitAND()
7022 if (SDValue RAND = reassociateOps(ISD::AND, DL, N0, N1, N->getFlags())) in visitAND()
7027 reassociateReduction(ISD::VECREDUCE_AND, ISD::AND, DL, VT, N0, N1)) in visitAND()
7034 if (N0.getOpcode() == ISD::OR && in visitAND()
7035 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset)) in visitAND()
7038 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { in visitAND()
7047 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0Op0); in visitAND()
7052 TLI.isTypeDesirableForOp(ISD::AND, SrcVT) && in visitAND()
7054 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, in visitAND()
7055 DAG.getNode(ISD::AND, DL, SrcVT, N0Op0, in visitAND()
7060 if (ISD::isExtOpcode(N0.getOpcode())) { in visitAND()
7063 if (N0Op0.getOpcode() == ISD::AND && in visitAND()
7064 (ExtOpc != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0Op0, VT)) && in visitAND()
7069 DAG.getNode(ISD::AND, DL, VT, N1, in visitAND()
7071 return DAG.getNode(ISD::AND, DL, VT, in visitAND()
7083 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitAND()
7085 N0.getOperand(0).getOpcode() == ISD::LOAD && in visitAND()
7087 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) { in visitAND()
7089 cast<LoadSDNode>((N0.getOpcode() == ISD::LOAD) ? N0 : N0.getOperand(0)); in visitAND()
7133 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD, in visitAND()
7145 case ISD::EXTLOAD: B = CanZextLoadProfitably; break; in visitAND()
7146 case ISD::ZEXTLOAD: in visitAND()
7147 case ISD::NON_EXTLOAD: B = true; break; in visitAND()
7158 if (Load->getExtensionType() == ISD::EXTLOAD) { in visitAND()
7159 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD, in visitAND()
7187 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() && N1C && in visitAND()
7188 ISD::isExtOpcode(N0.getOperand(0).getOpcode())) { in visitAND()
7195 (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, ExtVT))) { in visitAND()
7199 DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVT, Extendee); in visitAND()
7201 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ZeroExtExtendee, in visitAND()
7219 GN0->getIndexType(), ISD::ZEXTLOAD); in visitAND()
7230 if (N1C && N0.getOpcode() == ISD::LOAD && !VT.isVector()) in visitAND()
7263 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) { in visitAND()
7266 if (SubRHS.getOpcode() == ISD::ZERO_EXTEND && in visitAND()
7269 if (SubRHS.getOpcode() == ISD::SIGN_EXTEND && in visitAND()
7271 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, SubRHS.getOperand(0)); in visitAND()
7282 if (ISD::isUNINDEXEDLoad(N0.getNode()) && in visitAND()
7283 (ISD::isEXTLoad(N0.getNode()) || in visitAND()
7284 (ISD::isSEXTLoad(N0.getNode()) && N0.hasOneUse()))) { in visitAND()
7294 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { in visitAND()
7296 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(), in visitAND()
7305 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) { in visitAND()
7324 if (LHS->getOpcode() != ISD::SIGN_EXTEND) in visitAND()
7340 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitAND()
7342 if (hasOperation(ISD::USUBSAT, VT)) in visitAND()
7364 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWordLow()
7370 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL) in MatchBSwapHWordLow()
7372 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL) in MatchBSwapHWordLow()
7374 if (N0.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
7387 if (N1.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
7397 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL) in MatchBSwapHWordLow()
7399 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL) in MatchBSwapHWordLow()
7413 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
7424 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
7462 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00); in MatchBSwapHWordLow()
7465 Res = DAG.getNode(ISD::SRL, DL, VT, Res, in MatchBSwapHWordLow()
7482 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL) in isBSwapHWordElement()
7487 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL) in isBSwapHWordElement()
7492 if (Opc == ISD::AND) in isBSwapHWordElement()
7494 else if (Opc0 == ISD::AND) in isBSwapHWordElement()
7508 if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) { in isBSwapHWordElement()
7518 if (Opc == ISD::AND) { in isBSwapHWordElement()
7522 if (Opc0 != ISD::SRL) in isBSwapHWordElement()
7530 if (Opc0 != ISD::SHL) in isBSwapHWordElement()
7536 } else if (Opc == ISD::SHL) { in isBSwapHWordElement()
7563 if (N.getOpcode() == ISD::OR) in isBSwapHWordPair()
7567 if (N.getOpcode() == ISD::SRL && N.getOperand(0).getOpcode() == ISD::BSWAP) { in isBSwapHWordPair()
7585 assert(N->getOpcode() == ISD::OR && VT == MVT::i32 && in matchBSwapHWordOrAndAnd()
7587 if (!TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in matchBSwapHWordOrAndAnd()
7589 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND) in matchBSwapHWordOrAndAnd()
7603 if (Shift0.getOpcode() != ISD::SHL || Shift1.getOpcode() != ISD::SRL) in matchBSwapHWordOrAndAnd()
7615 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Shift0.getOperand(0)); in matchBSwapHWordOrAndAnd()
7617 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in matchBSwapHWordOrAndAnd()
7633 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWord()
7654 } else if (N0.getOpcode() == ISD::OR) { in MatchBSwapHWord()
7672 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, in MatchBSwapHWord()
7678 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) in MatchBSwapHWord()
7679 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
7680 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in MatchBSwapHWord()
7681 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
7682 return DAG.getNode(ISD::OR, DL, VT, in MatchBSwapHWord()
7683 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt), in MatchBSwapHWord()
7684 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt)); in MatchBSwapHWord()
7700 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND && in visitORLike()
7716 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
7718 return DAG.getNode(ISD::AND, DL, VT, X, in visitORLike()
7726 if (N0.getOpcode() == ISD::AND && in visitORLike()
7727 N1.getOpcode() == ISD::AND && in visitORLike()
7731 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
7733 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X); in visitORLike()
7747 if (V->getOpcode() == ISD::ZERO_EXTEND || V->getOpcode() == ISD::TRUNCATE) in visitORCommutative()
7753 if (N0Resized.getOpcode() == ISD::AND) { in visitORCommutative()
7767 return DAG.getNode(ISD::OR, DL, VT, DAG.getZExtOrTrunc(N00, DL, VT), in visitORCommutative()
7775 return DAG.getNode(ISD::OR, DL, VT, DAG.getZExtOrTrunc(N01, DL, VT), in visitORCommutative()
7784 return DAG.getNode(ISD::OR, DL, VT, X, N1); in visitORCommutative()
7790 return DAG.getNode(ISD::OR, DL, VT, X, Y); in visitORCommutative()
7796 if (V->getOpcode() == ISD::ZERO_EXTEND) in visitORCommutative()
7802 if (N0.getOpcode() == ISD::FSHL && N1.getOpcode() == ISD::SHL && in visitORCommutative()
7808 if (N0.getOpcode() == ISD::FSHR && N1.getOpcode() == ISD::SRL && in visitORCommutative()
7825 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotLo); in visitORCommutative()
7826 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, VT, NotHi); in visitORCommutative()
7827 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, in visitORCommutative()
7829 return DAG.getNOT(DL, DAG.getNode(ISD::OR, DL, VT, Lo, Hi), VT); in visitORCommutative()
7847 if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, DL, VT, {N0, N1})) in visitOR()
7853 return DAG.getNode(ISD::OR, DL, VT, N1, N0); in visitOR()
7861 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitOR()
7865 if (ISD::isConstantSplatVectorAllOnes(N1.getNode())) in visitOR()
7874 bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode()); in visitOR()
7875 bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode()); in visitOR()
7876 bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode()); in visitOR()
7877 bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode()); in visitOR()
7958 if (SDValue ROR = reassociateOps(ISD::OR, DL, N0, N1, N->getFlags())) in visitOR()
7963 reassociateReduction(ISD::VECREDUCE_OR, ISD::OR, DL, VT, N0, N1)) in visitOR()
7971 if (N0.getOpcode() == ISD::AND && N0->hasOneUse() && in visitOR()
7972 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect, true)) { in visitOR()
7973 if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT, in visitOR()
7975 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1); in visitOR()
7977 return DAG.getNode(ISD::AND, DL, VT, COR, IOR); in visitOR()
8003 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && in visitOR()
8019 if (Op.getOpcode() == ISD::AND && in stripConstantMask()
8031 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) { in matchRotateHalf()
8066 if (OppShift.getOpcode() != ISD::SHL && OppShift.getOpcode() != ISD::SRL) in extractShiftForRotate()
8080 if (OppShift.getOpcode() == ISD::SRL && OppShiftCst && in extractShiftForRotate()
8081 ExtractFrom.getOpcode() == ISD::ADD && in extractShiftForRotate()
8085 return DAG.getNode(ISD::SHL, DL, ShiftedVT, OppShiftLHS, in extractShiftForRotate()
8092 unsigned Opcode = ISD::DELETED_NODE; in extractShiftForRotate()
8105 if ((OppShift.getOpcode() != ISD::SRL || !SelectOpcode(ISD::SHL, ISD::MUL)) && in extractShiftForRotate()
8106 (OppShift.getOpcode() != ISD::SHL || !SelectOpcode(ISD::SRL, ISD::UDIV))) in extractShiftForRotate()
8234 if (Neg.getOpcode() != ISD::SUB) in matchRotateSub()
8269 (NegOp1.getOpcode() == ISD::TRUNCATE && Pos == NegOp1.getOperand(0))) in matchRotateSub()
8281 else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) { in matchRotateSub()
8352 if (PosOpcode == ISD::FSHL && isPowerOf2_32(EltBits)) { in MatchFunnelPosNeg()
8362 if (IsBinOpImm(N1, ISD::SRL, 1) && in MatchFunnelPosNeg()
8363 IsBinOpImm(InnerNeg, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
8365 TLI.isOperationLegalOrCustom(ISD::FSHL, VT)) { in MatchFunnelPosNeg()
8366 return DAG.getNode(ISD::FSHL, DL, VT, N0, N1.getOperand(0), Pos); in MatchFunnelPosNeg()
8371 if (IsBinOpImm(N0, ISD::SHL, 1) && in MatchFunnelPosNeg()
8372 IsBinOpImm(InnerPos, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
8374 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
8375 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
8381 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N0.getOperand(1) && in MatchFunnelPosNeg()
8382 IsBinOpImm(InnerPos, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
8384 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
8385 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
8402 bool HasROTL = hasOperation(ISD::ROTL, VT); in MatchRotate()
8403 bool HasROTR = hasOperation(ISD::ROTR, VT); in MatchRotate()
8404 bool HasFSHL = hasOperation(ISD::FSHL, VT); in MatchRotate()
8405 bool HasFSHR = hasOperation(ISD::FSHR, VT); in MatchRotate()
8412 HasROTL |= TLI.getOperationAction(ISD::ROTL, VT) == TargetLowering::Custom; in MatchRotate()
8413 HasROTR |= TLI.getOperationAction(ISD::ROTR, VT) == TargetLowering::Custom; in MatchRotate()
8420 if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE && in MatchRotate()
8424 return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(), Rot); in MatchRotate()
8470 if (RHSShift.getOpcode() == ISD::SHL) { in MatchRotate()
8477 if (LHSShift.getOpcode() != ISD::SHL || RHSShift.getOpcode() != ISD::SRL) in MatchRotate()
8498 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt); in MatchRotate()
8499 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
8500 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits)); in MatchRotate()
8503 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt); in MatchRotate()
8504 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
8505 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits)); in MatchRotate()
8508 Res = DAG.getNode(ISD::AND, DL, VT, Res, Mask); in MatchRotate()
8518 ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) { in MatchRotate()
8523 if (!Or.hasOneUse() || Or.getOpcode() != ISD::OR) in MatchRotate()
8541 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt); in MatchRotate()
8542 SDValue ShlY = DAG.getNode(ISD::SHL, DL, VT, Y, LHSShiftAmt); in MatchRotate()
8543 Res = DAG.getNode(ISD::OR, DL, VT, RotX, ShlY); in MatchRotate()
8546 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt); in MatchRotate()
8547 SDValue SrlY = DAG.getNode(ISD::SRL, DL, VT, Y, RHSShiftAmt); in MatchRotate()
8548 Res = DAG.getNode(ISD::OR, DL, VT, RotX, SrlY); in MatchRotate()
8564 if (ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) { in MatchRotate()
8568 Res = DAG.getNode(UseROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg, in MatchRotate()
8572 Res = DAG.getNode(UseFSHL ? ISD::FSHL : ISD::FSHR, DL, VT, LHSShiftArg, in MatchRotate()
8592 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || in MatchRotate()
8593 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND || in MatchRotate()
8594 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND || in MatchRotate()
8595 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) && in MatchRotate()
8596 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || in MatchRotate()
8597 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND || in MatchRotate()
8598 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND || in MatchRotate()
8599 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) { in MatchRotate()
8607 RExtOp0, HasROTL, ISD::ROTL, ISD::ROTR, DL); in MatchRotate()
8613 LExtOp0, HasROTR, ISD::ROTR, ISD::ROTL, DL); in MatchRotate()
8620 LExtOp0, RExtOp0, HasFSHL, ISD::FSHL, ISD::FSHR, DL); in MatchRotate()
8626 RExtOp0, LExtOp0, HasFSHR, ISD::FSHR, ISD::FSHL, DL); in MatchRotate()
8688 (Op.getOpcode() != ISD::LOAD || !Op.getValueType().isVector())) in calculateByteProvider()
8693 if (Op.getOpcode() != ISD::LOAD && VectorIndex.has_value()) in calculateByteProvider()
8704 case ISD::OR: { in calculateByteProvider()
8720 case ISD::SHL: { in calculateByteProvider()
8739 case ISD::ANY_EXTEND: in calculateByteProvider()
8740 case ISD::SIGN_EXTEND: in calculateByteProvider()
8741 case ISD::ZERO_EXTEND: { in calculateByteProvider()
8749 return Op.getOpcode() == ISD::ZERO_EXTEND in calculateByteProvider()
8756 case ISD::BSWAP: in calculateByteProvider()
8759 case ISD::EXTRACT_VECTOR_ELT: { in calculateByteProvider()
8790 case ISD::LOAD: { in calculateByteProvider()
8804 return L->getExtensionType() == ISD::ZEXTLOAD in calculateByteProvider()
8852 case ISD::TRUNCATE: in stripTruncAndExt()
8853 case ISD::ZERO_EXTEND: in stripTruncAndExt()
8854 case ISD::SIGN_EXTEND: in stripTruncAndExt()
8855 case ISD::ANY_EXTEND: in stripTruncAndExt()
8944 if (Trunc.getOpcode() != ISD::TRUNCATE) in mergeTruncStores()
8950 if ((WideVal.getOpcode() == ISD::SRL || WideVal.getOpcode() == ISD::SRA) && in mergeTruncStores()
9051 SourceValue = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SourceValue); in mergeTruncStores()
9058 SourceValue = DAG.getNode(ISD::BSWAP, DL, WideVT, SourceValue); in mergeTruncStores()
9062 SourceValue = DAG.getNode(ISD::ROTR, DL, WideVT, SourceValue, RotAmt); in mergeTruncStores()
9105 assert(N->getOpcode() == ISD::OR && in MatchLoadCombine()
9218 !TLI.isOperationLegal(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, in MatchLoadCombine()
9250 !TLI.isOperationLegal(ISD::BSWAP, VT)) in MatchLoadCombine()
9256 !TLI.isOperationLegal(ISD::SHL, VT)) in MatchLoadCombine()
9268 DAG.getExtLoad(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, in MatchLoadCombine()
9280 NeedsZext ? DAG.getNode(ISD::SHL, SDLoc(N), VT, NewLoad, in MatchLoadCombine()
9284 return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, ShiftedLoad); in MatchLoadCombine()
9304 assert(N->getOpcode() == ISD::XOR); in unfoldMaskedMerge()
9316 if (And.getOpcode() != ISD::AND || !And.hasOneUse()) in unfoldMaskedMerge()
9319 if (Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse()) in unfoldMaskedMerge()
9360 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M); in unfoldMaskedMerge()
9362 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y); in unfoldMaskedMerge()
9363 return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS); in unfoldMaskedMerge()
9372 SDValue LHS = DAG.getNode(ISD::OR, DL, VT, X, NotM); in unfoldMaskedMerge()
9374 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, NotM, NotY); in unfoldMaskedMerge()
9376 return DAG.getNode(ISD::AND, DL, VT, LHS, NotRHS); in unfoldMaskedMerge()
9379 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M); in unfoldMaskedMerge()
9381 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM); in unfoldMaskedMerge()
9383 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in unfoldMaskedMerge()
9403 if (SDValue C = DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, {N0, N1})) in visitXOR()
9409 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitXOR()
9417 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitXOR()
9429 if (SDValue RXOR = reassociateOps(ISD::XOR, DL, N0, N1, N->getFlags())) in visitXOR()
9434 reassociateReduction(ISD::VECREDUCE_XOR, ISD::XOR, DL, VT, N0, N1)) in visitXOR()
9438 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) && in visitXOR()
9442 return DAG.getNode(ISD::OR, DL, VT, N0, N1, Flags); in visitXOR()
9447 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && in visitXOR()
9457 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), in visitXOR()
9464 case ISD::SETCC: in visitXOR()
9466 case ISD::SELECT_CC: in visitXOR()
9469 case ISD::STRICT_FSETCC: in visitXOR()
9470 case ISD::STRICT_FSETCCS: { in visitXOR()
9476 N0.getOperand(0), N0Opcode == ISD::STRICT_FSETCCS); in visitXOR()
9489 if (isOneConstant(N1) && N0Opcode == ISD::ZERO_EXTEND && N0.hasOneUse() && in visitXOR()
9493 V = DAG.getNode(ISD::XOR, DL0, V.getValueType(), V, in visitXOR()
9496 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, V); in visitXOR()
9501 (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) { in visitXOR()
9504 unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND; in visitXOR()
9505 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
9506 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
9513 (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) { in visitXOR()
9516 unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND; in visitXOR()
9517 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
9518 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
9527 if (isAllOnesConstant(N1) && N0.getOpcode() == ISD::SUB && in visitXOR()
9529 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), in visitXOR()
9534 if (isAllOnesConstant(N1) && N0.getOpcode() == ISD::ADD && in visitXOR()
9540 if (N0Opcode == ISD::AND && N0.hasOneUse() && N0->getOperand(1) == N1) { in visitXOR()
9544 return DAG.getNode(ISD::AND, DL, VT, NotX, N1); in visitXOR()
9548 if (!LegalOperations || hasOperation(ISD::ABS, VT)) { in visitXOR()
9549 SDValue A = N0Opcode == ISD::ADD ? N0 : N1; in visitXOR()
9550 SDValue S = N0Opcode == ISD::SRA ? N0 : N1; in visitXOR()
9551 if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) { in visitXOR()
9557 return DAG.getNode(ISD::ABS, DL, VT, S0); in visitXOR()
9583 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0Opcode == ISD::SHL && in visitXOR()
9585 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT), in visitXOR()
9626 if (LogicOpcode != ISD::AND && LogicOpcode != ISD::OR && in combineShiftOfShiftedLogic()
9627 LogicOpcode != ISD::XOR) in combineShiftOfShiftedLogic()
9718 case ISD::OR: in visitShiftByConstant()
9719 case ISD::XOR: in visitShiftByConstant()
9720 case ISD::AND: in visitShiftByConstant()
9722 case ISD::ADD: in visitShiftByConstant()
9723 if (N->getOpcode() != ISD::SHL) in visitShiftByConstant()
9732 bool IsShiftByConstant = (BinOpLHSVal.getOpcode() == ISD::SHL || in visitShiftByConstant()
9733 BinOpLHSVal.getOpcode() == ISD::SRA || in visitShiftByConstant()
9734 BinOpLHSVal.getOpcode() == ISD::SRL) && in visitShiftByConstant()
9736 bool IsCopyOrSelect = BinOpLHSVal.getOpcode() == ISD::CopyFromReg || in visitShiftByConstant()
9737 BinOpLHSVal.getOpcode() == ISD::SELECT; in visitShiftByConstant()
9759 assert(N->getOpcode() == ISD::TRUNCATE); in distributeTruncateThroughAnd()
9760 assert(N->getOperand(0).getOpcode() == ISD::AND); in distributeTruncateThroughAnd()
9765 TLI.isTypeDesirableForOp(ISD::AND, TruncVT)) { in distributeTruncateThroughAnd()
9770 SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00); in distributeTruncateThroughAnd()
9771 SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01); in distributeTruncateThroughAnd()
9774 return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01); in distributeTruncateThroughAnd()
9805 if (ISD::matchUnaryPredicate(N1, MatchOutOfRange) && OutOfRange) { in visitRotate()
9809 DAG.FoldConstantArithmetic(ISD::UREM, dl, AmtVT, {N1, Bits})) in visitRotate()
9816 VT.getScalarSizeInBits() == 16 && hasOperation(ISD::BSWAP, VT)) in visitRotate()
9817 return DAG.getNode(ISD::BSWAP, dl, VT, N0); in visitRotate()
9824 if (N1.getOpcode() == ISD::TRUNCATE && in visitRotate()
9825 N1.getOperand(0).getOpcode() == ISD::AND) { in visitRotate()
9834 if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) { in visitRotate()
9840 unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB; in visitRotate()
9842 SDValue Norm1 = DAG.FoldConstantArithmetic(ISD::UREM, dl, ShiftVT, in visitRotate()
9844 SDValue Norm2 = DAG.FoldConstantArithmetic(ISD::UREM, dl, ShiftVT, in visitRotate()
9849 CombinedShift = DAG.FoldConstantArithmetic(ISD::ADD, dl, ShiftVT, in visitRotate()
9852 ISD::UREM, dl, ShiftVT, {CombinedShift, BitsizeC}); in visitRotate()
9873 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N0, N1})) in visitSHL()
9885 if (N0.getOpcode() == ISD::AND) { in visitSHL()
9890 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC && in visitSHL()
9894 DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N01, N1})) in visitSHL()
9895 return DAG.getNode(ISD::AND, DL, VT, N00, C); in visitSHL()
9909 if (N1.getOpcode() == ISD::TRUNCATE && in visitSHL()
9910 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSHL()
9912 return DAG.getNode(ISD::SHL, DL, VT, N0, NewOp1); in visitSHL()
9916 if (N0.getOpcode() == ISD::SHL) { in visitSHL()
9924 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange)) in visitSHL()
9934 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) { in visitSHL()
9935 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1)); in visitSHL()
9936 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum); in visitSHL()
9945 if ((N0.getOpcode() == ISD::ZERO_EXTEND || in visitSHL()
9946 N0.getOpcode() == ISD::ANY_EXTEND || in visitSHL()
9947 N0.getOpcode() == ISD::SIGN_EXTEND) && in visitSHL()
9948 N0.getOperand(0).getOpcode() == ISD::SHL) { in visitSHL()
9962 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchOutOfRange, in visitSHL()
9975 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchInRange, in visitSHL()
9980 Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, Sum, N1); in visitSHL()
9981 return DAG.getNode(ISD::SHL, DL, VT, Ext, Sum); in visitSHL()
9988 if (N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() && in visitSHL()
9989 N0.getOperand(0).getOpcode() == ISD::SRL) { in visitSHL()
9999 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchEqual, in visitSHL()
10004 NewSHL = DAG.getNode(ISD::SHL, DL, N0Op0.getValueType(), N0Op0, NewSHL); in visitSHL()
10006 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); in visitSHL()
10010 if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) { in visitSHL()
10022 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSHL()
10026 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01); in visitSHL()
10027 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10029 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSHL()
10033 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1); in visitSHL()
10042 if (N0.getOpcode() == ISD::SRL && in visitSHL()
10045 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSHL()
10049 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1); in visitSHL()
10051 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N01); in visitSHL()
10052 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, Diff); in visitSHL()
10053 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10054 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSHL()
10056 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSHL()
10060 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01); in visitSHL()
10062 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N1); in visitSHL()
10063 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10064 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSHL()
10070 if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) && in visitSHL()
10073 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1); in visitSHL()
10074 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask); in visitSHL()
10081 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) && in visitSHL()
10085 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) { in visitSHL()
10086 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1); in visitSHL()
10090 if (N0.getOpcode() == ISD::OR && N0->getFlags().hasDisjoint()) in visitSHL()
10099 if (N0.getOpcode() == ISD::SIGN_EXTEND && in visitSHL()
10100 N0.getOperand(0).getOpcode() == ISD::ADD && in visitSHL()
10109 DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {ExtC, N1})) { in visitSHL()
10111 SDValue ShlX = DAG.getNode(ISD::SHL, DL, VT, ExtX, N1); in visitSHL()
10112 return DAG.getNode(ISD::ADD, DL, VT, ShlX, ShlC); in visitSHL()
10118 if (N0.getOpcode() == ISD::MUL && N0->hasOneUse()) { in visitSHL()
10121 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) in visitSHL()
10122 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), Shl); in visitSHL()
10132 if (((N1.getOpcode() == ISD::CTTZ && in visitSHL()
10134 N1.getOpcode() == ISD::CTTZ_ZERO_UNDEF) && in visitSHL()
10135 N1.hasOneUse() && !TLI.isOperationLegalOrCustom(ISD::CTTZ, ShiftVT) && in visitSHL()
10136 TLI.isOperationLegalOrCustom(ISD::MUL, VT)) { in visitSHL()
10141 DAG.getZExtOrTrunc(DAG.getNode(ISD::AND, DL, ShiftVT, Y, NegY), DL, VT); in visitSHL()
10142 return DAG.getNode(ISD::MUL, DL, VT, And, N0); in visitSHL()
10149 if (N0.getOpcode() == ISD::VSCALE && N1C) { in visitSHL()
10157 if (N0.getOpcode() == ISD::STEP_VECTOR && in visitSHL()
10158 ISD::isConstantSplatVector(N1.getNode(), ShlVal)) { in visitSHL()
10175 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) && in combineShiftToMULH()
10186 if (ShiftOperand.getOpcode() != ISD::MUL) in combineShiftToMULH()
10193 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND; in combineShiftToMULH()
10194 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND; in combineShiftToMULH()
10204 if (U->getOpcode() != ISD::SRL && U->getOpcode() != ISD::SRA) { in combineShiftToMULH()
10217 unsigned MulLoHiOp = IsSignExt ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; in combineShiftToMULH()
10262 unsigned MulhOpcode = IsSignExt ? ISD::MULHS : ISD::MULHU; in combineShiftToMULH()
10279 bool IsSigned = N->getOpcode() == ISD::SRA; in combineShiftToMULH()
10287 if (Opcode != ISD::BSWAP && Opcode != ISD::BITREVERSE) in foldBitOrderCrossLogicOp()
10293 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && N0.hasOneUse()) { in foldBitOrderCrossLogicOp()
10330 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, DL, VT, {N0, N1})) in visitSRA()
10351 if (N0.getOpcode() == ISD::SRA) { in visitSRA()
10366 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), SumOfShifts)) { in visitSRA()
10368 if (N1.getOpcode() == ISD::BUILD_VECTOR) in visitSRA()
10370 else if (N1.getOpcode() == ISD::SPLAT_VECTOR) { in visitSRA()
10377 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue); in visitSRA()
10386 if (N0.getOpcode() == ISD::SHL && N1C) { in visitSRA()
10405 TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) && in visitSRA()
10406 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) && in visitSRA()
10409 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, in visitSRA()
10411 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, in visitSRA()
10413 return DAG.getNode(ISD::SIGN_EXTEND, DL, in visitSRA()
10424 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB) && N1C && in visitSRA()
10426 bool IsAdd = N0.getOpcode() == ISD::ADD; in visitSRA()
10428 if (Shl.getOpcode() == ISD::SHL && Shl.getOperand(1) == N1 && in visitSRA()
10454 Add = DAG.getNode(ISD::ADD, DL, TruncVT, Trunc, ShiftC); in visitSRA()
10456 Add = DAG.getNode(ISD::SUB, DL, TruncVT, ShiftC, Trunc); in visitSRA()
10464 if (N1.getOpcode() == ISD::TRUNCATE && in visitSRA()
10465 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSRA()
10467 return DAG.getNode(ISD::SRA, DL, VT, N0, NewOp1); in visitSRA()
10474 if (N0.getOpcode() == ISD::TRUNCATE && in visitSRA()
10475 (N0.getOperand(0).getOpcode() == ISD::SRL || in visitSRA()
10476 N0.getOperand(0).getOpcode() == ISD::SRA) && in visitSRA()
10486 Amt = DAG.getNode(ISD::ADD, DL, LargeShiftVT, Amt, in visitSRA()
10489 DAG.getNode(ISD::SRA, DL, LargeVT, N0Op0.getOperand(0), Amt); in visitSRA()
10490 return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA); in visitSRA()
10501 return DAG.getNode(ISD::SRL, DL, VT, N0, N1); in visitSRA()
10531 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, DL, VT, {N0, N1})) in visitSRL()
10549 if (N0.getOpcode() == ISD::SRL) { in visitSRL()
10557 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange)) in visitSRL()
10567 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) { in visitSRL()
10568 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1)); in visitSRL()
10569 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum); in visitSRL()
10573 if (N1C && N0.getOpcode() == ISD::TRUNCATE && in visitSRL()
10574 N0.getOperand(0).getOpcode() == ISD::SRL) { in visitSRL()
10589 SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT, in visitSRL()
10591 return DAG.getNode(ISD::TRUNCATE, DL, VT, NewShift); in visitSRL()
10598 SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT, in visitSRL()
10603 SDValue And = DAG.getNode(ISD::AND, DL, InnerShiftVT, NewShift, Mask); in visitSRL()
10604 return DAG.getNode(ISD::TRUNCATE, DL, VT, And); in visitSRL()
10611 if (N0.getOpcode() == ISD::SHL && in visitSRL()
10621 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSRL()
10625 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1); in visitSRL()
10627 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N01); in visitSRL()
10628 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, Diff); in visitSRL()
10629 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
10630 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSRL()
10632 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSRL()
10636 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01); in visitSRL()
10638 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N1); in visitSRL()
10639 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
10640 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSRL()
10646 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { in visitSRL()
10653 if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) { in visitSRL()
10657 DAG.getNode(ISD::SRL, DL0, SmallVT, N0.getOperand(0), in visitSRL()
10661 return DAG.getNode(ISD::AND, DL, VT, in visitSRL()
10662 DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift), in visitSRL()
10670 if (N0.getOpcode() == ISD::SRA) in visitSRL()
10671 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1); in visitSRL()
10676 if (N1C && N0.getOpcode() == ISD::CTLZ && in visitSRL()
10701 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in visitSRL()
10705 return DAG.getNode(ISD::XOR, DL, VT, Op, DAG.getConstant(1, DL, VT)); in visitSRL()
10710 if (N1.getOpcode() == ISD::TRUNCATE && in visitSRL()
10711 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSRL()
10713 return DAG.getNode(ISD::SRL, DL, VT, N0, NewOp1); in visitSRL()
10757 if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) in visitSRL()
10760 if (Use->getOpcode() == ISD::BRCOND || Use->getOpcode() == ISD::AND || in visitSRL()
10761 Use->getOpcode() == ISD::OR || Use->getOpcode() == ISD::XOR) in visitSRL()
10778 bool IsFSHL = N->getOpcode() == ISD::FSHL; in visitFunnelShift()
10814 ISD::SRL, DL, VT, N1, in visitFunnelShift()
10818 ISD::SHL, DL, VT, N0, in visitFunnelShift()
10832 (LHS->hasOneUse() || RHS->hasOneUse()) && ISD::isNON_EXTLoad(RHS) && in visitFunnelShift()
10833 ISD::isNON_EXTLoad(LHS)) { in visitFunnelShift()
10868 return DAG.getNode(ISD::SRL, DL, VT, N1, N2); in visitFunnelShift()
10870 return DAG.getNode(ISD::SHL, DL, VT, N0, N2); in visitFunnelShift()
10878 unsigned RotOpc = IsFSHL ? ISD::ROTL : ISD::ROTR; in visitFunnelShift()
10904 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) { in visitSHLSAT()
10906 if (N->getOpcode() == ISD::SSHLSAT && N1C && in visitSHLSAT()
10908 return DAG.getNode(ISD::SHL, DL, VT, N0, N1); in visitSHLSAT()
10911 if (N->getOpcode() == ISD::USHLSAT && N1C && in visitSHLSAT()
10914 return DAG.getNode(ISD::SHL, DL, VT, N0, N1); in visitSHLSAT()
10927 if (N->getOpcode() == ISD::TRUNCATE) in foldABSToABD()
10930 if (N->getOpcode() != ISD::ABS) in foldABSToABD()
10937 if (AbsOp1.getOpcode() != ISD::SUB) in foldABSToABD()
10948 (Opc0 != ISD::ZERO_EXTEND && Opc0 != ISD::SIGN_EXTEND && in foldABSToABD()
10949 Opc0 != ISD::SIGN_EXTEND_INREG)) { in foldABSToABD()
10951 if (AbsOp1->getFlags().hasNoSignedWrap() && hasOperation(ISD::ABDS, VT) && in foldABSToABD()
10953 SDValue ABD = DAG.getNode(ISD::ABDS, DL, VT, Op0, Op1); in foldABSToABD()
10960 if (Opc0 == ISD::SIGN_EXTEND_INREG) { in foldABSToABD()
10967 unsigned ABDOpcode = (Opc0 == ISD::ZERO_EXTEND) ? ISD::ABDU : ISD::ABDS; in foldABSToABD()
10975 DAG.getNode(ISD::TRUNCATE, DL, MaxVT, Op0), in foldABSToABD()
10976 DAG.getNode(ISD::TRUNCATE, DL, MaxVT, Op1)); in foldABSToABD()
10977 ABD = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, ABD); in foldABSToABD()
10997 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ABS, DL, VT, {N0})) in visitABS()
11000 if (N0.getOpcode() == ISD::ABS) in visitABS()
11011 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitABS()
11014 TLI.isTypeDesirableForOp(ISD::ABS, ExtVT) && in visitABS()
11015 hasOperation(ISD::ABS, ExtVT)) { in visitABS()
11017 ISD::ZERO_EXTEND, DL, VT, in visitABS()
11018 DAG.getNode(ISD::ABS, DL, ExtVT, in visitABS()
11019 DAG.getNode(ISD::TRUNCATE, DL, ExtVT, N0.getOperand(0)))); in visitABS()
11032 if (SDValue C = DAG.FoldConstantArithmetic(ISD::BSWAP, DL, VT, {N0})) in visitBSWAP()
11035 if (N0.getOpcode() == ISD::BSWAP) in visitBSWAP()
11042 if (N0.getOpcode() == ISD::BITREVERSE && N0.hasOneUse()) { in visitBSWAP()
11043 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
11044 return DAG.getNode(ISD::BITREVERSE, DL, VT, BSwap); in visitBSWAP()
11050 if (BW >= 32 && N0.getOpcode() == ISD::SHL && N0.hasOneUse()) { in visitBSWAP()
11057 (!LegalOperations || hasOperation(ISD::BSWAP, HalfVT))) { in visitBSWAP()
11060 Res = DAG.getNode(ISD::SHL, DL, VT, Res, in visitBSWAP()
11063 Res = DAG.getNode(ISD::BSWAP, DL, HalfVT, Res); in visitBSWAP()
11072 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) && in visitBSWAP()
11077 SDValue NewSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
11078 unsigned InverseShift = N0.getOpcode() == ISD::SHL ? ISD::SRL : ISD::SHL; in visitBSWAP()
11095 if (SDValue C = DAG.FoldConstantArithmetic(ISD::BITREVERSE, DL, VT, {N0})) in visitBITREVERSE()
11099 if (N0.getOpcode() == ISD::BITREVERSE) in visitBITREVERSE()
11105 if ((!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) && in visitBITREVERSE()
11107 return DAG.getNode(ISD::SHL, DL, VT, X, Y); in visitBITREVERSE()
11110 if ((!LegalOperations || TLI.isOperationLegal(ISD::SRL, VT)) && in visitBITREVERSE()
11112 return DAG.getNode(ISD::SRL, DL, VT, X, Y); in visitBITREVERSE()
11123 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTLZ, DL, VT, {N0})) in visitCTLZ()
11127 if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_UNDEF, VT)) in visitCTLZ()
11129 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, DL, VT, N0); in visitCTLZ()
11141 DAG.FoldConstantArithmetic(ISD::CTLZ_ZERO_UNDEF, DL, VT, {N0})) in visitCTLZ_ZERO_UNDEF()
11152 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTTZ, DL, VT, {N0})) in visitCTTZ()
11156 if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_UNDEF, VT)) in visitCTTZ()
11158 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, DL, VT, N0); in visitCTTZ()
11170 DAG.FoldConstantArithmetic(ISD::CTTZ_ZERO_UNDEF, DL, VT, {N0})) in visitCTTZ_ZERO_UNDEF()
11182 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTPOP, DL, VT, {N0})) in visitCTPOP()
11187 if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SHL) { in visitCTPOP()
11192 if ((N0.getOpcode() == ISD::SRL && in visitCTPOP()
11194 (N0.getOpcode() == ISD::SHL && in visitCTPOP()
11196 return DAG.getNode(ISD::CTPOP, DL, VT, N0.getOperand(0)); in visitCTPOP()
11206 if (hasOperation(ISD::CTPOP, HalfVT) && in visitCTPOP()
11207 TLI.isTypeDesirableForOp(ISD::CTPOP, HalfVT) && in visitCTPOP()
11211 SDValue PopCnt = DAG.getNode(ISD::CTPOP, DL, HalfVT, in visitCTPOP()
11238 ISD::CondCode CC, in combineMinNumMaxNumImpl()
11243 case ISD::SETOLT: in combineMinNumMaxNumImpl()
11244 case ISD::SETOLE: in combineMinNumMaxNumImpl()
11245 case ISD::SETLT: in combineMinNumMaxNumImpl()
11246 case ISD::SETLE: in combineMinNumMaxNumImpl()
11247 case ISD::SETULT: in combineMinNumMaxNumImpl()
11248 case ISD::SETULE: { in combineMinNumMaxNumImpl()
11252 unsigned IEEEOpcode = (LHS == True) ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE; in combineMinNumMaxNumImpl()
11256 unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM; in combineMinNumMaxNumImpl()
11261 case ISD::SETOGT: in combineMinNumMaxNumImpl()
11262 case ISD::SETOGE: in combineMinNumMaxNumImpl()
11263 case ISD::SETGT: in combineMinNumMaxNumImpl()
11264 case ISD::SETGE: in combineMinNumMaxNumImpl()
11265 case ISD::SETUGT: in combineMinNumMaxNumImpl()
11266 case ISD::SETUGE: { in combineMinNumMaxNumImpl()
11267 unsigned IEEEOpcode = (LHS == True) ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE; in combineMinNumMaxNumImpl()
11271 unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM; in combineMinNumMaxNumImpl()
11284 SDValue False, ISD::CondCode CC) { in combineMinNumMaxNum()
11314 return DAG.getNode(ISD::FNEG, DL, VT, Combined); in combineMinNumMaxNum()
11333 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse() || in foldSelectOfConstantsUsingSra()
11341 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in foldSelectOfConstantsUsingSra()
11342 if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(CondC) && in foldSelectOfConstantsUsingSra()
11346 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
11347 return DAG.getNode(ISD::OR, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
11349 if (CC == ISD::SETLT && isNullOrNullSplat(CondC) && isNullOrNullSplat(C2)) { in foldSelectOfConstantsUsingSra()
11352 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
11353 return DAG.getNode(ISD::AND, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
11363 if (Cond.getOpcode() != ISD::SETCC || !Cond->hasOneUse()) in shouldConvertSelectOfConstantsToMath()
11365 if (!TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT)) in shouldConvertSelectOfConstantsToMath()
11368 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in shouldConvertSelectOfConstantsToMath()
11369 if (CC == ISD::SETLT && isNullOrNullSplat(Cond.getOperand(1))) in shouldConvertSelectOfConstantsToMath()
11371 if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(Cond.getOperand(1))) in shouldConvertSelectOfConstantsToMath()
11410 DAG.getNode(ISD::XOR, DL, CondVT, Cond, DAG.getConstant(1, DL, CondVT)); in foldSelectOfConstants()
11460 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
11466 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
11474 return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC); in foldSelectOfConstants()
11480 return DAG.getNode(ISD::OR, DL, VT, Cond, N2); in foldSelectOfConstants()
11487 return DAG.getNode(ISD::OR, DL, VT, NotCond, N1); in foldSelectOfConstants()
11499 assert((N->getOpcode() == ISD::SELECT || N->getOpcode() == ISD::VSELECT || in foldBoolSelectToLogic()
11500 N->getOpcode() == ISD::VP_SELECT) && in foldBoolSelectToLogic()
11514 return matcher.getNode(ISD::OR, DL, VT, Cond, DAG.getFreeze(F)); in foldBoolSelectToLogic()
11519 return matcher.getNode(ISD::AND, DL, VT, Cond, DAG.getFreeze(T)); in foldBoolSelectToLogic()
11524 matcher.getNode(ISD::XOR, DL, VT, Cond, DAG.getAllOnesConstant(DL, VT)); in foldBoolSelectToLogic()
11525 return matcher.getNode(ISD::OR, DL, VT, NotCond, DAG.getFreeze(T)); in foldBoolSelectToLogic()
11531 matcher.getNode(ISD::XOR, DL, VT, Cond, DAG.getAllOnesConstant(DL, VT)); in foldBoolSelectToLogic()
11532 return matcher.getNode(ISD::AND, DL, VT, NotCond, DAG.getFreeze(F)); in foldBoolSelectToLogic()
11545 ISD::CondCode CC; in foldVSelectToSignBitSplatMask()
11553 if (CC == ISD::SETLT && isNullOrNullSplat(Cond1)) in foldVSelectToSignBitSplatMask()
11555 else if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(Cond1)) in foldVSelectToSignBitSplatMask()
11564 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
11565 return DAG.getNode(ISD::AND, DL, VT, Sra, DAG.getFreeze(N1)); in foldVSelectToSignBitSplatMask()
11572 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
11573 return DAG.getNode(ISD::OR, DL, VT, Sra, DAG.getFreeze(N2)); in foldVSelectToSignBitSplatMask()
11583 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
11585 return DAG.getNode(ISD::AND, DL, VT, Not, DAG.getFreeze(N2)); in foldVSelectToSignBitSplatMask()
11637 if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) { in visitSELECT()
11641 DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2, Flags); in visitSELECT()
11643 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, in visitSELECT()
11650 if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) { in visitSELECT()
11653 SDValue InnerSelect = DAG.getNode(ISD::SELECT, DL, N1.getValueType(), in visitSELECT()
11656 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1, in visitSELECT()
11664 if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) { in visitSELECT()
11671 SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0); in visitSELECT()
11672 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1, in visitSELECT()
11677 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1, in visitSELECT()
11683 if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) { in visitSELECT()
11690 SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0); in visitSELECT()
11691 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1, in visitSELECT()
11696 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1, in visitSELECT()
11703 if (N0.getOpcode() == ISD::SETCC) { in visitSELECT()
11705 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitSELECT()
11721 if (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::UADDO, VT) && in visitSELECT()
11722 CC == ISD::SETUGT && N0.hasOneUse() && isAllOnesConstant(N1) && in visitSELECT()
11723 N2.getOpcode() == ISD::ADD && Cond0 == N2.getOperand(0)) { in visitSELECT()
11740 SDValue UAO = DAG.getNode(ISD::UADDO, DL, VTs, Cond0, N2.getOperand(1)); in visitSELECT()
11745 if (TLI.isOperationLegal(ISD::SELECT_CC, VT) || in visitSELECT()
11747 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) { in visitSELECT()
11751 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, in visitSELECT()
11780 assert(LHS.getOpcode() == ISD::CONCAT_VECTORS && in ConvertSelectToConcatVector()
11781 RHS.getOpcode() == ISD::CONCAT_VECTORS && in ConvertSelectToConcatVector()
11782 Cond.getOpcode() == ISD::BUILD_VECTOR); in ConvertSelectToConcatVector()
11821 ISD::CONCAT_VECTORS, DL, VT, in ConvertSelectToConcatVector()
11841 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal); in refineUniformBase()
11846 if (Index.getOpcode() != ISD::ADD) in refineUniformBase()
11851 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal); in refineUniformBase()
11857 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal); in refineUniformBase()
11865 bool refineIndexType(SDValue &Index, ISD::MemIndexType &IndexType, EVT DataVT, in refineIndexType()
11870 if (Index.getOpcode() == ISD::ZERO_EXTEND) { in refineIndexType()
11872 IndexType = ISD::UNSIGNED_SCALED; in refineIndexType()
11876 if (ISD::isIndexTypeSigned(IndexType)) { in refineIndexType()
11877 IndexType = ISD::UNSIGNED_SCALED; in refineIndexType()
11883 if (Index.getOpcode() == ISD::SIGN_EXTEND && in refineIndexType()
11884 ISD::isIndexTypeSigned(IndexType) && in refineIndexType()
11902 ISD::MemIndexType IndexType = MSC->getIndexType(); in visitVPSCATTER()
11906 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitVPSCATTER()
11932 ISD::MemIndexType IndexType = MSC->getIndexType(); in visitMSCATTER()
11936 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMSCATTER()
11965 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMSTORE()
11975 ISD::isConstantSplatVectorAllOnes(Mask.getNode())) && in visitMSTORE()
11979 if (N->getOpcode() != ISD::DELETED_NODE) in visitMSTORE()
11987 if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MST->isUnindexed() && in visitMSTORE()
12013 if (N->getOpcode() != ISD::DELETED_NODE) in visitMSTORE()
12023 if ((Value.getOpcode() == ISD::TRUNCATE) && Value->hasOneUse() && in visitMSTORE()
12063 if (ISD::isConstantSplatVector(Mask.getNode(), SplatVal)) in visitVECTOR_COMPRESS()
12070 if (ISD::isBuildVectorOfConstantSDNodes(Mask.getNode())) { in visitVECTOR_COMPRESS()
12082 SDValue VecI = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Vec, in visitVECTOR_COMPRESS()
12091 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Passthru, in visitVECTOR_COMPRESS()
12110 ISD::MemIndexType IndexType = MGT->getIndexType(); in visitVPGATHER()
12138 ISD::MemIndexType IndexType = MGT->getIndexType(); in visitMGATHER()
12142 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMGATHER()
12168 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMLOAD()
12173 if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MLD->isUnindexed() && in visitMLOAD()
12174 !MLD->isExpandingLoad() && MLD->getExtensionType() == ISD::NON_EXTLOAD) { in visitMLOAD()
12214 !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()) || in foldVSelectOfConstants()
12215 !ISD::isBuildVectorOfConstantSDNodes(N2.getNode())) in foldVSelectOfConstants()
12249 auto ExtendOpcode = AllAddOne ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND; in foldVSelectOfConstants()
12251 return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2); in foldVSelectOfConstants()
12256 if (ISD::isConstantSplatVector(N1.getNode(), Pow2C) && Pow2C.isPowerOf2() && in foldVSelectOfConstants()
12260 return DAG.getNode(ISD::SHL, DL, VT, ZextCond, ShAmtC); in foldVSelectOfConstants()
12306 if (N1.getOpcode() == ISD::ADD && N1.getOperand(0) == N2 && N1->hasOneUse() && in visitVSELECT()
12312 ISD::ADD, DL, N1.getValueType(), N2, in visitVSELECT()
12313 DAG.getNode(ISD::AND, DL, N0.getValueType(), N1.getOperand(1), N0)); in visitVSELECT()
12321 if (N0.getOpcode() == ISD::SETCC) { in visitVSELECT()
12323 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitVSELECT()
12325 bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode()); in visitVSELECT()
12327 if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) || in visitVSELECT()
12328 (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) && in visitVSELECT()
12329 N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1)) in visitVSELECT()
12330 isAbs = ISD::isBuildVectorAllZeros(N2.getOperand(0).getNode()); in visitVSELECT()
12331 else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) && in visitVSELECT()
12332 N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1)) in visitVSELECT()
12333 isAbs = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode()); in visitVSELECT()
12336 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitVSELECT()
12337 return DAG.getNode(ISD::ABS, DL, VT, LHS); in visitVSELECT()
12340 ISD::SRA, DL, VT, LHS, in visitVSELECT()
12342 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift); in visitVSELECT()
12345 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift); in visitVSELECT()
12377 auto LoadExtOpcode = IsSigned ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in visitVSELECT()
12378 if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() && in visitVSELECT()
12381 TLI.isOperationLegalOrCustom(ISD::SETCC, WideVT)) { in visitVSELECT()
12386 auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in visitVSELECT()
12400 if (N1.getOpcode() == ISD::SUB && N2.getOpcode() == ISD::SUB && in visitVSELECT()
12404 unsigned ABDOpc = IsSigned ? ISD::ABDS : ISD::ABDU; in visitVSELECT()
12407 case ISD::SETGT: in visitVSELECT()
12408 case ISD::SETGE: in visitVSELECT()
12409 case ISD::SETUGT: in visitVSELECT()
12410 case ISD::SETUGE: in visitVSELECT()
12414 case ISD::SETLT: in visitVSELECT()
12415 case ISD::SETLE: in visitVSELECT()
12416 case ISD::SETULT: in visitVSELECT()
12417 case ISD::SETULE: in visitVSELECT()
12428 if (hasOperation(ISD::UADDSAT, VT)) { in visitVSELECT()
12432 ISD::CondCode SatCC = CC; in visitVSELECT()
12433 if (ISD::isConstantSplatVectorAllOnes(N1.getNode())) { in visitVSELECT()
12435 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
12436 } else if (ISD::isConstantSplatVectorAllOnes(N2.getNode())) { in visitVSELECT()
12440 if (Other && Other.getOpcode() == ISD::ADD) { in visitVSELECT()
12445 if (SatCC == ISD::SETUGE) { in visitVSELECT()
12447 SatCC = ISD::SETULE; in visitVSELECT()
12453 if (SatCC == ISD::SETULE && Other == CondRHS && in visitVSELECT()
12455 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12458 (OpRHS.getOpcode() == ISD::BUILD_VECTOR || in visitVSELECT()
12459 OpRHS.getOpcode() == ISD::SPLAT_VECTOR) && in visitVSELECT()
12467 if (SatCC == ISD::SETULE && in visitVSELECT()
12468 ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUADDSAT)) in visitVSELECT()
12469 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12475 if (hasOperation(ISD::USUBSAT, VT)) { in visitVSELECT()
12479 ISD::CondCode SatCC = CC; in visitVSELECT()
12480 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) { in visitVSELECT()
12482 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
12483 } else if (ISD::isConstantSplatVectorAllZeros(N2.getNode())) { in visitVSELECT()
12491 if (Other && Other.getOpcode() == ISD::TRUNCATE && in visitVSELECT()
12492 Other.getOperand(0).getOpcode() == ISD::SUB && in visitVSELECT()
12493 (SatCC == ISD::SETUGE || SatCC == ISD::SETUGT)) { in visitVSELECT()
12496 if (LHS == OpLHS && RHS == OpRHS && LHS.getOpcode() == ISD::ZERO_EXTEND) in visitVSELECT()
12510 if ((SatCC == ISD::SETUGE || SatCC == ISD::SETUGT) && in visitVSELECT()
12511 Other.getOpcode() == ISD::SUB && OpRHS == CondRHS) in visitVSELECT()
12512 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12514 if (OpRHS.getOpcode() == ISD::BUILD_VECTOR || in visitVSELECT()
12515 OpRHS.getOpcode() == ISD::SPLAT_VECTOR) { in visitVSELECT()
12516 if (CondRHS.getOpcode() == ISD::BUILD_VECTOR || in visitVSELECT()
12517 CondRHS.getOpcode() == ISD::SPLAT_VECTOR) { in visitVSELECT()
12526 if (SatCC == ISD::SETUGT && Other.getOpcode() == ISD::ADD && in visitVSELECT()
12527 ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUSUBSAT, in visitVSELECT()
12530 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12539 if (SatCC == ISD::SETLT && Other.getOpcode() == ISD::XOR && in visitVSELECT()
12540 ISD::isConstantSplatVector(OpRHS.getNode(), SplatValue) && in visitVSELECT()
12541 ISD::isConstantSplatVectorAllZeros(CondRHS.getNode()) && in visitVSELECT()
12546 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12559 if (ISD::isConstantSplatVectorAllOnes(N0.getNode())) in visitVSELECT()
12562 if (ISD::isConstantSplatVectorAllZeros(N0.getNode())) in visitVSELECT()
12568 if (N1.getOpcode() == ISD::CONCAT_VECTORS && in visitVSELECT()
12569 N2.getOpcode() == ISD::CONCAT_VECTORS && in visitVSELECT()
12570 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) { in visitVSELECT()
12578 if (hasOperation(ISD::SRA, VT)) in visitVSELECT()
12594 ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get(); in visitSELECT_CC()
12602 if (CC == ISD::SETEQ && !LegalTypes && N0.getValueType() == MVT::i1 && in visitSELECT_CC()
12622 if (SCC.getOpcode() == ISD::SETCC) { in visitSELECT_CC()
12624 DAG.getNode(ISD::SELECT_CC, DL, N2.getValueType(), SCC.getOperand(0), in visitSELECT_CC()
12644 N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BRCOND; in visitSETCC()
12646 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get(); in visitSETCC()
12654 if (PreferSetCC && Combined.getOpcode() != ISD::SETCC) { in visitSETCC()
12679 if (Cond == ISD::SETNE || Cond == ISD::SETEQ) { in visitSETCC()
12681 return A.getOpcode() == ISD::AND && in visitSETCC()
12682 (B.getOpcode() == ISD::SRL || B.getOpcode() == ISD::SHL) && in visitSETCC()
12686 return (B.getOpcode() == ISD::ROTL || B.getOpcode() == ISD::ROTR) && in visitSETCC()
12739 ShiftOpc == ISD::SHL ? (~*AndCMask).isMask() : AndCMask->isMask(); in visitSETCC()
12752 if (NewShiftOpc == ISD::SHL || NewShiftOpc == ISD::SRL) { in visitSETCC()
12754 NewShiftOpc == ISD::SHL in visitSETCC()
12760 DAG.getNode(ISD::AND, DL, OpVT, ShiftOrRotate.getOperand(0), in visitSETCC()
12782 return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond); in visitSETCCCARRY()
12802 ISD::LoadExtType LoadExt = Load->getExtensionType(); in isCompatibleLoad()
12803 if (LoadExt == ISD::NON_EXTLOAD || LoadExt == ISD::EXTLOAD) in isCompatibleLoad()
12808 if ((LoadExt == ISD::SEXTLOAD && ExtOpcode != ISD::SIGN_EXTEND) || in isCompatibleLoad()
12809 (LoadExt == ISD::ZEXTLOAD && ExtOpcode != ISD::ZERO_EXTEND)) in isCompatibleLoad()
12827 assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND || in tryToFoldExtendSelectLoad()
12828 Opcode == ISD::ANY_EXTEND) && in tryToFoldExtendSelectLoad()
12831 if (!(N0->getOpcode() == ISD::SELECT || N0->getOpcode() == ISD::VSELECT) || in tryToFoldExtendSelectLoad()
12840 auto ExtLoadOpcode = ISD::EXTLOAD; in tryToFoldExtendSelectLoad()
12841 if (Opcode == ISD::SIGN_EXTEND) in tryToFoldExtendSelectLoad()
12842 ExtLoadOpcode = ISD::SEXTLOAD; in tryToFoldExtendSelectLoad()
12843 else if (Opcode == ISD::ZERO_EXTEND) in tryToFoldExtendSelectLoad()
12844 ExtLoadOpcode = ISD::ZEXTLOAD; in tryToFoldExtendSelectLoad()
12852 (N0->getOpcode() == ISD::VSELECT && Level >= AfterLegalizeTypes && in tryToFoldExtendSelectLoad()
12853 TLI.getOperationAction(ISD::VSELECT, VT) != TargetLowering::Legal)) in tryToFoldExtendSelectLoad()
12874 assert((ISD::isExtOpcode(Opcode) || ISD::isExtVecInRegOpcode(Opcode)) && in tryToFoldExtendOfConstant()
12886 if (N0->getOpcode() == ISD::SELECT) { in tryToFoldExtendOfConstant()
12890 (Opcode != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0.getValueType(), VT))) { in tryToFoldExtendOfConstant()
12901 if (FoldOpc == ISD::ANY_EXTEND) in tryToFoldExtendOfConstant()
12902 FoldOpc = ISD::SIGN_EXTEND; in tryToFoldExtendOfConstant()
12914 ISD::isBuildVectorOfConstantSDNodes(N0.getNode()))) in tryToFoldExtendOfConstant()
12926 if (Opcode == ISD::ANY_EXTEND || Opcode == ISD::ANY_EXTEND_VECTOR_INREG) in tryToFoldExtendOfConstant()
12937 if (Opcode == ISD::SIGN_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG) in tryToFoldExtendOfConstant()
12964 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) { in ExtendUsesToFormExtLoad()
12965 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get(); in ExtendUsesToFormExtLoad()
12966 if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC)) in ExtendUsesToFormExtLoad()
12987 if (User->getOpcode() == ISD::CopyToReg) in ExtendUsesToFormExtLoad()
12996 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) { in ExtendUsesToFormExtLoad()
13011 ISD::NodeType ExtType) { in ExtendSetCCUses()
13026 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops)); in ExtendSetCCUses()
13036 assert((N->getOpcode() == ISD::SIGN_EXTEND || in CombineExtLoad()
13037 N->getOpcode() == ISD::ZERO_EXTEND) && in CombineExtLoad()
13057 if (N0->getOpcode() != ISD::LOAD) in CombineExtLoad()
13062 if (!ISD::isNON_EXTLoad(LN0) || !ISD::isUNINDEXEDLoad(LN0) || in CombineExtLoad()
13072 ISD::LoadExtType ExtType = in CombineExtLoad()
13073 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in CombineExtLoad()
13112 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in CombineExtLoad()
13113 SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads); in CombineExtLoad()
13123 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue); in CombineExtLoad()
13124 ExtendSetCCUses(SetCCs, N0, NewValue, (ISD::NodeType)N->getOpcode()); in CombineExtLoad()
13132 assert(N->getOpcode() == ISD::ZERO_EXTEND); in CombineZExtLogicopShiftLoad()
13140 if (!ISD::isBitwiseLogicOp(N0.getOpcode()) || in CombineZExtLogicopShiftLoad()
13141 N0.getOperand(1).getOpcode() != ISD::Constant || in CombineZExtLogicopShiftLoad()
13147 if (!(N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) || in CombineZExtLogicopShiftLoad()
13148 N1.getOperand(1).getOpcode() != ISD::Constant || in CombineZExtLogicopShiftLoad()
13157 if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) || in CombineZExtLogicopShiftLoad()
13158 Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed()) in CombineZExtLogicopShiftLoad()
13164 if (N1.getOpcode() == ISD::SHL && N0.getOpcode() != ISD::AND) in CombineZExtLogicopShiftLoad()
13172 ISD::ZERO_EXTEND, SetCCs, TLI)) in CombineZExtLogicopShiftLoad()
13176 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT, in CombineZExtLogicopShiftLoad()
13189 ExtendSetCCUses(SetCCs, N1.getOperand(0), ExtLoad, ISD::ZERO_EXTEND); in CombineZExtLogicopShiftLoad()
13194 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load), in CombineZExtLogicopShiftLoad()
13211 assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND || in matchVSelectOpSizesWithSetCC()
13212 CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND || in matchVSelectOpSizesWithSetCC()
13213 CastOpcode == ISD::FP_ROUND) && in matchVSelectOpSizesWithSetCC()
13220 if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT)) in matchVSelectOpSizesWithSetCC()
13224 if (VSel.getOpcode() != ISD::VSELECT || !VSel.hasOneUse() || in matchVSelectOpSizesWithSetCC()
13225 VSel.getOperand(0).getOpcode() != ISD::SETCC) in matchVSelectOpSizesWithSetCC()
13239 if (CastOpcode == ISD::FP_ROUND) { in matchVSelectOpSizesWithSetCC()
13247 return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB); in matchVSelectOpSizesWithSetCC()
13255 SDValue N0, ISD::LoadExtType ExtLoadType) { in tryToFoldExtOfExtload()
13257 bool isAExtLoad = (ExtLoadType == ISD::SEXTLOAD) ? ISD::isSEXTLoad(N0Node) in tryToFoldExtOfExtload()
13258 : ISD::isZEXTLoad(N0Node); in tryToFoldExtOfExtload()
13259 if ((!isAExtLoad && !ISD::isEXTLoad(N0Node)) || in tryToFoldExtOfExtload()
13260 !ISD::isUNINDEXEDLoad(N0Node) || !N0.hasOneUse()) in tryToFoldExtOfExtload()
13287 ISD::LoadExtType ExtLoadType, in tryToFoldExtOfLoad()
13288 ISD::NodeType ExtOpc, in tryToFoldExtOfLoad()
13290 if (!ISD::isNON_EXTLoad(N0.getNode()) || !ISD::isUNINDEXEDLoad(N0.getNode())) in tryToFoldExtOfLoad()
13295 assert(ExtLoadType == ISD::ZEXTLOAD && ExtOpc == ISD::ZERO_EXTEND && in tryToFoldExtOfLoad()
13298 if (User->getOpcode() == ISD::SETCC) { in tryToFoldExtOfLoad()
13299 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get(); in tryToFoldExtOfLoad()
13300 if (ISD::isSignedIntSetCC(CC)) { in tryToFoldExtOfLoad()
13301 ExtLoadType = ISD::SEXTLOAD; in tryToFoldExtOfLoad()
13302 ExtOpc = ISD::SIGN_EXTEND; in tryToFoldExtOfLoad()
13339 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); in tryToFoldExtOfLoad()
13348 ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc) { in tryToFoldExtOfMaskedLoad()
13353 if (!Ld || Ld->getExtensionType() != ISD::NON_EXTLOAD) in tryToFoldExtOfMaskedLoad()
13377 ISD::LoadExtType ExtLoadType) { in tryToFoldExtOfAtomicLoad()
13379 if (!ALoad || ALoad->getOpcode() != ISD::ATOMIC_LOAD) in tryToFoldExtOfAtomicLoad()
13385 ISD::LoadExtType ALoadExtTy = ALoad->getExtensionType(); in tryToFoldExtOfAtomicLoad()
13386 if ((ALoadExtTy == ISD::ZEXTLOAD && ExtLoadType == ISD::SEXTLOAD) || in tryToFoldExtOfAtomicLoad()
13387 (ALoadExtTy == ISD::SEXTLOAD && ExtLoadType == ISD::ZEXTLOAD)) in tryToFoldExtOfAtomicLoad()
13393 ISD::ATOMIC_LOAD, SDLoc(ALoad), MemoryVT, VT, ALoad->getChain(), in tryToFoldExtOfAtomicLoad()
13398 DAG.getNode(ISD::TRUNCATE, SDLoc(ALoad), OrigVT, SDValue(NewALoad, 0))); in tryToFoldExtOfAtomicLoad()
13406 assert((N->getOpcode() == ISD::SIGN_EXTEND || in foldExtendedSignBitTest()
13407 N->getOpcode() == ISD::ZERO_EXTEND) && "Expected sext or zext"); in foldExtendedSignBitTest()
13410 if (LegalOperations || SetCC.getOpcode() != ISD::SETCC || in foldExtendedSignBitTest()
13416 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC.getOperand(2))->get(); in foldExtendedSignBitTest()
13422 if (CC == ISD::SETGT && isAllOnesConstant(Ones) && VT == XVT) { in foldExtendedSignBitTest()
13433 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SRA : ISD::SRL; in foldExtendedSignBitTest()
13442 if (N0.getOpcode() != ISD::SETCC) in foldSextSetcc()
13447 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in foldSextSetcc()
13484 if (N0.hasOneUse() && TLI.isOperationLegalOrCustom(ISD::SETCC, VT) && in foldSextSetcc()
13485 !TLI.isOperationLegalOrCustom(ISD::SETCC, SVT)) { in foldSextSetcc()
13486 bool IsSignedCmp = ISD::isSignedIntSetCC(CC); in foldSextSetcc()
13487 unsigned LoadOpcode = IsSignedCmp ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in foldSextSetcc()
13488 unsigned ExtOpcode = IsSignedCmp ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in foldSextSetcc()
13499 if (!(ISD::isNON_EXTLoad(V.getNode()) && in foldSextSetcc()
13500 ISD::isUNINDEXEDLoad(V.getNode()) && in foldSextSetcc()
13556 (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, N00VT))) { in foldSextSetcc()
13583 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) in visitSIGN_EXTEND()
13584 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
13588 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG || in visitSIGN_EXTEND()
13589 N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) in visitSIGN_EXTEND()
13590 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT, in visitSIGN_EXTEND()
13594 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitSIGN_EXTEND()
13597 if ((N00.getOpcode() == ISD::TRUNCATE || TLI.isTruncateFree(N00, ExtVT)) && in visitSIGN_EXTEND()
13599 SDValue T = DAG.getNode(ISD::TRUNCATE, DL, ExtVT, N00); in visitSIGN_EXTEND()
13600 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, T); in visitSIGN_EXTEND()
13604 if (N0.getOpcode() == ISD::TRUNCATE) { in visitSIGN_EXTEND()
13634 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op); in visitSIGN_EXTEND()
13639 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in visitSIGN_EXTEND()
13643 if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, in visitSIGN_EXTEND()
13646 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
13648 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
13649 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op, in visitSIGN_EXTEND()
13657 ISD::SEXTLOAD, ISD::SIGN_EXTEND)) in visitSIGN_EXTEND()
13662 ISD::SEXTLOAD, ISD::SIGN_EXTEND)) in visitSIGN_EXTEND()
13672 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD)) in visitSIGN_EXTEND()
13677 tryToFoldExtOfAtomicLoad(DAG, TLI, VT, N0, ISD::SEXTLOAD)) in visitSIGN_EXTEND()
13682 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && in visitSIGN_EXTEND()
13684 N0.getOperand(1).getOpcode() == ISD::Constant && in visitSIGN_EXTEND()
13688 if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) && in visitSIGN_EXTEND()
13689 LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) { in visitSIGN_EXTEND()
13692 ISD::SIGN_EXTEND, SetCCs, TLI); in visitSIGN_EXTEND()
13694 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT, in visitSIGN_EXTEND()
13701 ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::SIGN_EXTEND); in visitSIGN_EXTEND()
13708 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And); in visitSIGN_EXTEND()
13714 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00), in visitSIGN_EXTEND()
13731 (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) && in visitSIGN_EXTEND()
13735 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0, Flags); in visitSIGN_EXTEND()
13743 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitSIGN_EXTEND()
13745 N0.getOperand(1).getOpcode() == ISD::ZERO_EXTEND && in visitSIGN_EXTEND()
13746 TLI.isOperationLegalOrCustom(ISD::SUB, VT)) { in visitSIGN_EXTEND()
13752 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() && in visitSIGN_EXTEND()
13754 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && in visitSIGN_EXTEND()
13755 TLI.isOperationLegalOrCustom(ISD::ADD, VT)) { in visitSIGN_EXTEND()
13757 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
13763 (!LegalOperations || (TLI.isOperationLegal(ISD::ZERO_EXTEND, VT) && in visitSIGN_EXTEND()
13764 TLI.isOperationLegal(ISD::ADD, VT)))) { in visitSIGN_EXTEND()
13776 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewXor); in visitSIGN_EXTEND()
13779 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
13780 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
13793 assert((Extend->getOpcode() == ISD::ZERO_EXTEND || in widenCtPop()
13794 Extend->getOpcode() == ISD::ANY_EXTEND) && in widenCtPop()
13798 if (CtPop.getOpcode() != ISD::CTPOP || !CtPop.hasOneUse()) in widenCtPop()
13803 if (TLI.isOperationLegalOrCustom(ISD::CTPOP, CtPop.getValueType()) || in widenCtPop()
13804 !TLI.isOperationLegalOrCustom(ISD::CTPOP, VT)) in widenCtPop()
13809 return DAG.getNode(ISD::CTPOP, DL, VT, NewZext); in widenCtPop()
13815 assert(Extend->getOpcode() == ISD::ZERO_EXTEND && "Expected zero extend."); in widenAbs()
13822 if (Abs.getOpcode() != ISD::ABS || !Abs.hasOneUse()) in widenAbs()
13834 DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Abs), LegalVT, Abs.getOperand(0)); in widenAbs()
13835 SDValue NewAbs = DAG.getNode(ISD::ABS, SDLoc(Abs), LegalVT, SExt); in widenAbs()
13857 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) { in visitZERO_EXTEND()
13859 if (N0.getOpcode() == ISD::ZERO_EXTEND) in visitZERO_EXTEND()
13861 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0), Flags); in visitZERO_EXTEND()
13866 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG || in visitZERO_EXTEND()
13867 N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) in visitZERO_EXTEND()
13868 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, N0.getOperand(0)); in visitZERO_EXTEND()
13892 if (N0.getOpcode() == ISD::TRUNCATE) { in visitZERO_EXTEND()
13925 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op); in visitZERO_EXTEND()
13930 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in visitZERO_EXTEND()
13937 if (!LegalOperations || (TLI.isOperationLegal(ISD::AND, SrcVT) && in visitZERO_EXTEND()
13938 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) { in visitZERO_EXTEND()
13949 if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) { in visitZERO_EXTEND()
13962 if (N0.getOpcode() == ISD::AND && in visitZERO_EXTEND()
13963 N0.getOperand(0).getOpcode() == ISD::TRUNCATE && in visitZERO_EXTEND()
13964 N0.getOperand(1).getOpcode() == ISD::Constant && in visitZERO_EXTEND()
13970 return DAG.getNode(ISD::AND, DL, VT, in visitZERO_EXTEND()
13976 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD, in visitZERO_EXTEND()
13977 ISD::ZERO_EXTEND, N->getFlags().hasNonNeg())) in visitZERO_EXTEND()
13982 ISD::ZEXTLOAD, ISD::ZERO_EXTEND)) in visitZERO_EXTEND()
13992 tryToFoldExtOfAtomicLoad(DAG, TLI, VT, N0, ISD::ZEXTLOAD)) in visitZERO_EXTEND()
13999 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && !TLI.isZExtFree(N0, VT) && in visitZERO_EXTEND()
14001 N0.getOperand(1).getOpcode() == ISD::Constant && in visitZERO_EXTEND()
14005 if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) && in visitZERO_EXTEND()
14006 LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) { in visitZERO_EXTEND()
14010 if (N0.getOpcode() == ISD::AND) { in visitZERO_EXTEND()
14020 ISD::ZERO_EXTEND, SetCCs, TLI); in visitZERO_EXTEND()
14022 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT, in visitZERO_EXTEND()
14029 ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::ZERO_EXTEND); in visitZERO_EXTEND()
14036 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And); in visitZERO_EXTEND()
14042 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00), in visitZERO_EXTEND()
14058 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD)) in visitZERO_EXTEND()
14064 if (N0.getOpcode() == ISD::SETCC) { in visitZERO_EXTEND()
14082 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0), in visitZERO_EXTEND()
14092 DAG.getNode(ISD::SETCC, DL, MatchingVectorType, N0.getOperand(0), in visitZERO_EXTEND()
14106 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, SCC); in visitZERO_EXTEND()
14110 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) && in visitZERO_EXTEND()
14115 if (ShVal.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse()) { in visitZERO_EXTEND()
14116 if (N0.getOpcode() == ISD::SHL) { in visitZERO_EXTEND()
14137 ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt); in visitZERO_EXTEND()
14140 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, ShVal), ShAmt); in visitZERO_EXTEND()
14159 SDNode *CSENode = DAG.getNodeIfExists(ISD::SIGN_EXTEND, N->getVTList(), N0); in visitZERO_EXTEND()
14182 if (N0.getOpcode() == ISD::ANY_EXTEND || N0.getOpcode() == ISD::ZERO_EXTEND || in visitANY_EXTEND()
14183 N0.getOpcode() == ISD::SIGN_EXTEND) { in visitANY_EXTEND()
14185 if (N0.getOpcode() == ISD::ZERO_EXTEND) in visitANY_EXTEND()
14193 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG || in visitANY_EXTEND()
14194 N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG || in visitANY_EXTEND()
14195 N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) in visitANY_EXTEND()
14200 if (N0.getOpcode() == ISD::TRUNCATE) { in visitANY_EXTEND()
14213 if (N0.getOpcode() == ISD::TRUNCATE) in visitANY_EXTEND()
14218 if (N0.getOpcode() == ISD::AND && in visitANY_EXTEND()
14219 N0.getOperand(0).getOpcode() == ISD::TRUNCATE && in visitANY_EXTEND()
14220 N0.getOperand(1).getOpcode() == ISD::Constant && in visitANY_EXTEND()
14223 SDValue Y = DAG.getNode(ISD::ANY_EXTEND, DL, VT, N0.getOperand(1)); in visitANY_EXTEND()
14225 return DAG.getNode(ISD::AND, DL, VT, X, Y); in visitANY_EXTEND()
14235 ISD::ZEXTLOAD, ISD::ZERO_EXTEND)) in visitANY_EXTEND()
14237 } else if (ISD::isNON_EXTLoad(N0.getNode()) && in visitANY_EXTEND()
14238 ISD::isUNINDEXEDLoad(N0.getNode()) && in visitANY_EXTEND()
14239 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitANY_EXTEND()
14244 ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI); in visitANY_EXTEND()
14247 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(), in visitANY_EXTEND()
14250 ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND); in visitANY_EXTEND()
14259 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); in visitANY_EXTEND()
14269 if (N0.getOpcode() == ISD::LOAD && !ISD::isNON_EXTLoad(N0.getNode()) && in visitANY_EXTEND()
14270 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { in visitANY_EXTEND()
14272 ISD::LoadExtType ExtType = LN0->getExtensionType(); in visitANY_EXTEND()
14285 if (N0.getOpcode() == ISD::SETCC) { in visitANY_EXTEND()
14346 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && in visitAssertExt()
14360 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert); in visitAssertExt()
14366 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && in visitAssertExt()
14367 N0.getOperand(0).getOpcode() == ISD::AssertSext && in visitAssertExt()
14368 Opcode == ISD::AssertZext) { in visitAssertExt()
14375 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert); in visitAssertExt()
14400 case ISD::ADD: in visitAssertAlign()
14401 case ISD::SUB: { in visitAssertAlign()
14427 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; in reduceLoadWidth()
14447 if (Opc == ISD::SIGN_EXTEND_INREG) { in reduceLoadWidth()
14448 ExtType = ISD::SEXTLOAD; in reduceLoadWidth()
14450 } else if (Opc == ISD::SRL || Opc == ISD::SRA) { in reduceLoadWidth()
14468 ExtType = Opc == ISD::SRL ? ISD::ZEXTLOAD : ISD::SEXTLOAD; in reduceLoadWidth()
14474 if ((LN->getExtensionType() == ISD::SEXTLOAD || in reduceLoadWidth()
14475 LN->getExtensionType() == ISD::ZEXTLOAD) && in reduceLoadWidth()
14478 } else if (Opc == ISD::AND) { in reduceLoadWidth()
14494 ExtType = ISD::ZEXTLOAD; in reduceLoadWidth()
14503 if (Opc == ISD::SRL || N0.getOpcode() == ISD::SRL) { in reduceLoadWidth()
14504 SDValue SRL = Opc == ISD::SRL ? SDValue(N, 0) : N0; in reduceLoadWidth()
14531 if (LN->getExtensionType() == ISD::SEXTLOAD) in reduceLoadWidth()
14541 if (ExtType == ISD::SEXTLOAD) in reduceLoadWidth()
14544 ExtType = ISD::ZEXTLOAD; in reduceLoadWidth()
14551 if (SRL.hasOneUse() && Mask->getOpcode() == ISD::AND && in reduceLoadWidth()
14562 } else if (ExtType == ISD::ZEXTLOAD && in reduceLoadWidth()
14586 if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() && in reduceLoadWidth()
14627 if (ExtType == ISD::NON_EXTLOAD) in reduceLoadWidth()
14652 Result = DAG.getNode(ISD::SHL, DL, VT, Result, in reduceLoadWidth()
14662 Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC); in reduceLoadWidth()
14684 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1); in visitSIGN_EXTEND_INREG()
14691 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in visitSIGN_EXTEND_INREG()
14693 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
14700 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) { in visitSIGN_EXTEND_INREG()
14705 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
14706 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
14712 if (ISD::isExtVecInRegOpcode(N0.getOpcode())) { in visitSIGN_EXTEND_INREG()
14717 bool IsZext = N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG; in visitSIGN_EXTEND_INREG()
14723 TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT))) in visitSIGN_EXTEND_INREG()
14724 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
14729 if (N0.getOpcode() == ISD::ZERO_EXTEND) { in visitSIGN_EXTEND_INREG()
14732 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
14733 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
14753 if (N0.getOpcode() == ISD::SRL) { in visitSIGN_EXTEND_INREG()
14760 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
14769 if (ISD::isEXTLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
14770 ISD::isUNINDEXEDLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
14774 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
14776 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
14787 if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
14791 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
14793 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
14806 Ld->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD && in visitSIGN_EXTEND_INREG()
14807 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT)) { in visitSIGN_EXTEND_INREG()
14811 Ld->getAddressingMode(), ISD::SEXTLOAD, Ld->isExpandingLoad()); in visitSIGN_EXTEND_INREG()
14828 GN0->getMemOperand(), GN0->getIndexType(), ISD::SEXTLOAD); in visitSIGN_EXTEND_INREG()
14838 if (ExtVTBits <= 16 && N0.getOpcode() == ISD::OR) { in visitSIGN_EXTEND_INREG()
14841 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, BSwap, N1); in visitSIGN_EXTEND_INREG()
14848 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() && in visitSIGN_EXTEND_INREG()
14849 ISD::isExtOpcode(N0.getOperand(0).getOpcode())) { in visitSIGN_EXTEND_INREG()
14856 TLI.isOperationLegal(ISD::SIGN_EXTEND, InnerExtVT))) { in visitSIGN_EXTEND_INREG()
14858 DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), InnerExtVT, Extendee); in visitSIGN_EXTEND_INREG()
14859 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, SignExtExtendee, in visitSIGN_EXTEND_INREG()
14879 assert(ISD::isExtVecInRegOpcode(InregOpcode) && in foldExtendVectorInregToExtendOfSubvector()
14884 if (!Src.hasOneUse() || Src.getOpcode() != ISD::CONCAT_VECTORS) in foldExtendVectorInregToExtendOfSubvector()
14907 return N->getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG in visitEXTEND_VECTOR_INREG()
14937 if (N0.getOpcode() == ISD::TRUNCATE) in visitTRUNCATE()
14938 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
14941 if (SDValue C = DAG.FoldConstantArithmetic(ISD::TRUNCATE, DL, VT, {N0})) in visitTRUNCATE()
14945 if (N0.getOpcode() == ISD::ZERO_EXTEND || in visitTRUNCATE()
14946 N0.getOpcode() == ISD::SIGN_EXTEND || in visitTRUNCATE()
14947 N0.getOpcode() == ISD::ANY_EXTEND) { in visitTRUNCATE()
14953 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
14961 if (!LegalTypes && N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in visitTRUNCATE()
14967 SDValue TrX = DAG.getNode(ISD::TRUNCATE, DL, VT, X); in visitTRUNCATE()
14968 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, TrX, ExtVal); in visitTRUNCATE()
14973 if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ANY_EXTEND)) in visitTRUNCATE()
14986 if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitTRUNCATE()
15003 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TrTy, in visitTRUNCATE()
15010 if (N0.getOpcode() == ISD::SELECT && N0.hasOneUse()) { in visitTRUNCATE()
15011 if ((!LegalOperations || TLI.isOperationLegal(ISD::SELECT, SrcVT)) && in visitTRUNCATE()
15015 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); in visitTRUNCATE()
15016 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2)); in visitTRUNCATE()
15017 return DAG.getNode(ISD::SELECT, DL, VT, Cond, TruncOp0, TruncOp1); in visitTRUNCATE()
15022 if (N0.getOpcode() == ISD::SHL && N0.hasOneUse() && in visitTRUNCATE()
15023 (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) && in visitTRUNCATE()
15024 TLI.isTypeDesirableForOp(ISD::SHL, VT)) { in visitTRUNCATE()
15030 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
15035 return DAG.getNode(ISD::SHL, DL, VT, Trunc, Amt); in visitTRUNCATE()
15046 if (N0.getOpcode() == ISD::BUILD_VECTOR && !LegalOperations && in visitTRUNCATE()
15054 SDValue TruncOp = DAG.getNode(ISD::TRUNCATE, DL, SVT, Op); in visitTRUNCATE()
15061 if (N0.getOpcode() == ISD::SPLAT_VECTOR && in visitTRUNCATE()
15063 (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, VT))) { in visitTRUNCATE()
15066 VT, DL, DAG.getNode(ISD::TRUNCATE, DL, SVT, N0->getOperand(0))); in visitTRUNCATE()
15074 N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && in visitTRUNCATE()
15075 N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR && in visitTRUNCATE()
15107 if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) { in visitTRUNCATE()
15121 if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) { in visitTRUNCATE()
15154 SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V); in visitTRUNCATE()
15158 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Opnds); in visitTRUNCATE()
15166 if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) { in visitTRUNCATE()
15171 TLI.isOperationLegal(ISD::EXTRACT_VECTOR_ELT, VecSrcVT))) { in visitTRUNCATE()
15173 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VecSrc, in visitTRUNCATE()
15186 if (!LegalTypes && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) { in visitTRUNCATE()
15188 if (N00.getOpcode() == ISD::SIGN_EXTEND || in visitTRUNCATE()
15189 N00.getOpcode() == ISD::ZERO_EXTEND || in visitTRUNCATE()
15190 N00.getOpcode() == ISD::ANY_EXTEND) { in visitTRUNCATE()
15193 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT, in visitTRUNCATE()
15206 case ISD::ADD: in visitTRUNCATE()
15207 case ISD::SUB: in visitTRUNCATE()
15208 case ISD::MUL: in visitTRUNCATE()
15209 case ISD::AND: in visitTRUNCATE()
15210 case ISD::OR: in visitTRUNCATE()
15211 case ISD::XOR: in visitTRUNCATE()
15219 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
15220 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
15225 case ISD::ADDE: in visitTRUNCATE()
15226 case ISD::UADDO_CARRY: in visitTRUNCATE()
15232 if (((!LegalOperations && N0.getOpcode() == ISD::UADDO_CARRY) || in visitTRUNCATE()
15235 SDValue X = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
15236 SDValue Y = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
15241 case ISD::USUBSAT: in visitTRUNCATE()
15246 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && in visitTRUNCATE()
15261 if (Elt.getOpcode() != ISD::MERGE_VALUES) in getBuildPairElt()
15269 assert(N->getOpcode() == ISD::BUILD_PAIR); in CombineConsecutiveLoads()
15280 if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !ISD::isNON_EXTLoad(LD2) || in CombineConsecutiveLoads()
15288 if ((!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) && in CombineConsecutiveLoads()
15323 case ISD::AND: in foldBitcastedFPLogic()
15324 FPOpcode = ISD::FABS; in foldBitcastedFPLogic()
15327 case ISD::XOR: in foldBitcastedFPLogic()
15328 FPOpcode = ISD::FNEG; in foldBitcastedFPLogic()
15331 case ISD::OR: in foldBitcastedFPLogic()
15332 FPOpcode = ISD::FABS; in foldBitcastedFPLogic()
15346 if (Op.getOpcode() == ISD::BITCAST && Op.getOperand(0).getValueType() == VT) in foldBitcastedFPLogic()
15349 return FPOpcode == ISD::FABS ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT); in foldBitcastedFPLogic()
15360 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, SDLoc(N), VT, LogicOp0); in foldBitcastedFPLogic()
15363 if (N0.getOpcode() == ISD::OR) in foldBitcastedFPLogic()
15364 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, FPOp); in foldBitcastedFPLogic()
15388 N0.getOpcode() == ISD::BUILD_VECTOR && N0->hasOneUse() && in visitBITCAST()
15400 TLI.isOperationLegal(ISD::ConstantFP, VT)) || in visitBITCAST()
15402 TLI.isOperationLegal(ISD::Constant, VT))) { in visitBITCAST()
15410 if (N0.getOpcode() == ISD::BITCAST) in visitBITCAST()
15415 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && VT.isInteger() && in visitBITCAST()
15418 return (V.getOpcode() == ISD::BITCAST && in visitBITCAST()
15420 (ISD::isBuildVectorOfConstantSDNodes(V.getNode()) && in visitBITCAST()
15431 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in visitBITCAST()
15441 TLI.isOperationLegal(ISD::LOAD, VT))) { in visitBITCAST()
15469 if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(N0.getValueType())) || in visitBITCAST()
15470 (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(N0.getValueType()))) && in visitBITCAST()
15482 if (N0.getOpcode() == ISD::FNEG) { in visitBITCAST()
15486 assert(N0.getOpcode() == ISD::FABS); in visitBITCAST()
15488 DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(NewConv), MVT::i64, NewConv, in visitBITCAST()
15492 FlipBit = DAG.getNode(ISD::AND, SDLoc(N0), MVT::i64, Hi, SignBit); in visitBITCAST()
15496 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
15498 return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits); in visitBITCAST()
15501 if (N0.getOpcode() == ISD::FNEG) in visitBITCAST()
15502 return DAG.getNode(ISD::XOR, DL, VT, in visitBITCAST()
15504 assert(N0.getOpcode() == ISD::FABS); in visitBITCAST()
15505 return DAG.getNode(ISD::AND, DL, VT, in visitBITCAST()
15520 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() && in visitBITCAST()
15532 X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X); in visitBITCAST()
15538 X = DAG.getNode(ISD::SRL, DL, in visitBITCAST()
15543 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X); in visitBITCAST()
15553 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X); in visitBITCAST()
15556 ISD::EXTRACT_ELEMENT, SDLoc(XorResult), MVT::i64, XorResult, in visitBITCAST()
15561 DAG.getNode(ISD::AND, SDLoc(XorResult64), MVT::i64, XorResult64, in visitBITCAST()
15565 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
15567 return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits); in visitBITCAST()
15570 X = DAG.getNode(ISD::AND, SDLoc(X), VT, in visitBITCAST()
15575 Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT, in visitBITCAST()
15579 return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst); in visitBITCAST()
15584 if (N0.getOpcode() == ISD::BUILD_PAIR) in visitBITCAST()
15593 N0->getOpcode() == ISD::VECTOR_SHUFFLE && N0.hasOneUse() && in visitBITCAST()
15601 if (Op.getOpcode() == ISD::BITCAST && in visitBITCAST()
15647 if (N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::SRL) in visitFREEZE()
15663 N0.getOpcode() == ISD::SELECT_CC || in visitFREEZE()
15664 N0.getOpcode() == ISD::SETCC || in visitFREEZE()
15665 N0.getOpcode() == ISD::BUILD_VECTOR || in visitFREEZE()
15666 N0.getOpcode() == ISD::BUILD_PAIR || in visitFREEZE()
15667 N0.getOpcode() == ISD::VECTOR_SHUFFLE || in visitFREEZE()
15668 N0.getOpcode() == ISD::CONCAT_VECTORS; in visitFREEZE()
15676 if (N0.getOpcode() == ISD::BUILD_VECTOR) { in visitFREEZE()
15679 if (llvm::ISD::isBuildVectorAllOnes(N0.getNode())) in visitFREEZE()
15681 if (llvm::ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) { in visitFREEZE()
15724 if (MaybePoisonOperand.getOpcode() == ISD::UNDEF) in visitFREEZE()
15730 if (FrozenMaybePoisonOperand.getOpcode() == ISD::FREEZE && in visitFREEZE()
15740 if (N->getOpcode() == ISD::DELETED_NODE) in visitFREEZE()
15752 if (Op.getOpcode() == ISD::UNDEF) in visitFREEZE()
15790 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(BV), SrcEltVT, Op); in ConstantFoldBITCASTofBUILD_VECTOR()
15850 assert(N.getOpcode() == ISD::FMUL); in isContractableFMUL()
15881 (!LegalOperations || matcher.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFADDForFMACombine()
15904 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA; in visitFADDForFMACombine()
15908 return matcher.match(N, ISD::FMA) || matcher.match(N, ISD::FMAD); in visitFADDForFMACombine()
15914 if (!matcher.match(N, ISD::FMUL)) in visitFADDForFMACombine()
15961 if (matcher.match(FMul, ISD::FMUL) && FMul.hasOneUse()) { in visitFADDForFMACombine()
15968 return FMA.getOpcode() == ISD::DELETED_NODE ? SDValue(N, 0) : FMA; in visitFADDForFMACombine()
15978 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFADDForFMACombine()
15985 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFADDForFMACombine()
15986 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), N1); in visitFADDForFMACombine()
15992 if (matcher.match(N1, ISD::FP_EXTEND)) { in visitFADDForFMACombine()
15999 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0)), in visitFADDForFMACombine()
16000 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0); in visitFADDForFMACombine()
16013 matcher.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
16014 matcher.getNode(ISD::FP_EXTEND, SL, VT, V), Z)); in visitFADDForFMACombine()
16018 if (matcher.match(N02, ISD::FP_EXTEND)) { in visitFADDForFMACombine()
16039 matcher.getNode(ISD::FP_EXTEND, SL, VT, X), in visitFADDForFMACombine()
16040 matcher.getNode(ISD::FP_EXTEND, SL, VT, Y), in visitFADDForFMACombine()
16042 matcher.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
16043 matcher.getNode(ISD::FP_EXTEND, SL, VT, V), Z)); in visitFADDForFMACombine()
16045 if (N0.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
16063 if (N12.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
16080 if (N1.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
16118 (!LegalOperations || matcher.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFSUBForFMACombine()
16136 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA; in visitFSUBForFMACombine()
16143 if (!matcher.match(N, ISD::FMUL)) in visitFSUBForFMACombine()
16153 matcher.getNode(ISD::FNEG, SL, VT, Z)); in visitFSUBForFMACombine()
16164 matcher.getNode(ISD::FNEG, SL, VT, YZ.getOperand(0)), in visitFSUBForFMACombine()
16190 if (matcher.match(N0, ISD::FNEG) && isContractableFMUL(N0.getOperand(0)) && in visitFSUBForFMACombine()
16195 matcher.getNode(ISD::FNEG, SL, VT, N00), N01, in visitFSUBForFMACombine()
16196 matcher.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
16203 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
16210 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
16211 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
16212 matcher.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
16219 if (matcher.match(N1, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
16227 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16228 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0))), in visitFSUBForFMACombine()
16229 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0); in visitFSUBForFMACombine()
16239 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
16241 if (matcher.match(N00, ISD::FNEG)) { in visitFSUBForFMACombine()
16247 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16250 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
16251 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
16263 if (matcher.match(N0, ISD::FNEG)) { in visitFSUBForFMACombine()
16265 if (matcher.match(N00, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
16271 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16274 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
16275 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
16291 return matcher.match(N, ISD::FMA) || matcher.match(N, ISD::FMAD); in visitFSUBForFMACombine()
16307 matcher.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
16319 matcher.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), in visitFSUBForFMACombine()
16322 matcher.getNode(ISD::FNEG, SL, VT, N20), N21, N0)); in visitFSUBForFMACombine()
16329 if (matcher.match(N02, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
16338 matcher.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(0)), in visitFSUBForFMACombine()
16339 matcher.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(1)), in visitFSUBForFMACombine()
16340 matcher.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
16351 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
16360 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
16361 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
16364 matcher.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(0)), in visitFSUBForFMACombine()
16365 matcher.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(1)), in visitFSUBForFMACombine()
16366 matcher.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
16373 if (isFusedOp(N1) && matcher.match(N1.getOperand(2), ISD::FP_EXTEND) && in visitFSUBForFMACombine()
16383 matcher.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), in visitFSUBForFMACombine()
16387 matcher.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16388 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1200)), in visitFSUBForFMACombine()
16389 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1201), N0)); in visitFSUBForFMACombine()
16399 if (matcher.match(N1, ISD::FP_EXTEND) && isFusedOp(N1.getOperand(0))) { in visitFSUBForFMACombine()
16411 matcher.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16412 matcher.getNode(ISD::FP_EXTEND, SL, VT, N100)), in visitFSUBForFMACombine()
16413 matcher.getNode(ISD::FP_EXTEND, SL, VT, N101), in visitFSUBForFMACombine()
16416 matcher.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16417 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1020)), in visitFSUBForFMACombine()
16418 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1021), N0)); in visitFSUBForFMACombine()
16435 assert(N->getOpcode() == ISD::FMUL && "Expected FMUL Operation"); in visitFMULForFMADistributiveCombine()
16441 SDValue FAdd = N0.getOpcode() == ISD::FADD ? N0 : N1; in visitFMULForFMADistributiveCombine()
16449 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFMULForFMADistributiveCombine()
16461 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA; in visitFMULForFMADistributiveCombine()
16467 if (X.getOpcode() == ISD::FADD && (Aggressive || X->hasOneUse())) { in visitFMULForFMADistributiveCombine()
16474 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
16490 if (X.getOpcode() == ISD::FSUB && (Aggressive || X->hasOneUse())) { in visitFMULForFMADistributiveCombine()
16494 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
16498 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
16499 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
16504 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
16526 if (Fused.getOpcode() != ISD::DELETED_NODE) in visitVP_FADD()
16548 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FADD, DL, VT, {N0, N1})) in visitFADD()
16553 return DAG.getNode(ISD::FADD, DL, VT, N1, N0); in visitFADD()
16570 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
16573 return DAG.getNode(ISD::FSUB, DL, VT, N0, NegN1); in visitFADD()
16576 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
16579 return DAG.getNode(ISD::FSUB, DL, VT, N1, NegN0); in visitFADD()
16582 if (!FMul.hasOneUse() || FMul.getOpcode() != ISD::FMUL) in visitFADD()
16591 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
16592 return DAG.getNode(ISD::FSUB, DL, VT, N1, Add); in visitFADD()
16597 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
16598 return DAG.getNode(ISD::FSUB, DL, VT, N0, Add); in visitFADD()
16608 if (N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1) in visitFADD()
16612 if (N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0) in visitFADD()
16623 if (N1CFP && N0.getOpcode() == ISD::FADD && in visitFADD()
16625 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1); in visitFADD()
16626 return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC); in visitFADD()
16632 if (TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && !N0CFP && !N1CFP) { in visitFADD()
16633 if (N0.getOpcode() == ISD::FMUL) { in visitFADD()
16641 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
16643 return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP); in visitFADD()
16647 if (CFP01 && !CFP00 && N1.getOpcode() == ISD::FADD && in visitFADD()
16650 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
16652 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP); in visitFADD()
16656 if (N1.getOpcode() == ISD::FMUL) { in visitFADD()
16664 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
16666 return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP); in visitFADD()
16670 if (CFP11 && !CFP10 && N0.getOpcode() == ISD::FADD && in visitFADD()
16673 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
16675 return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP); in visitFADD()
16679 if (N0.getOpcode() == ISD::FADD) { in visitFADD()
16685 return DAG.getNode(ISD::FMUL, DL, VT, N1, in visitFADD()
16690 if (N1.getOpcode() == ISD::FADD) { in visitFADD()
16696 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFADD()
16702 if (N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD && in visitFADD()
16706 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), in visitFADD()
16712 if (SDValue SD = reassociateReduction(ISD::VECREDUCE_FADD, ISD::FADD, DL, in visitFADD()
16719 if (Fused.getOpcode() != ISD::DELETED_NODE) in visitFADD()
16736 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
16739 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
16744 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
16747 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
16768 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FSUB, DL, VT, {N0, N1})) in visitFSUB()
16806 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFSUB()
16807 return DAG.getNode(ISD::FNEG, DL, VT, N1); in visitFSUB()
16814 N1.getOpcode() == ISD::FADD) { in visitFSUB()
16817 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1)); in visitFSUB()
16820 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(0)); in visitFSUB()
16826 return DAG.getNode(ISD::FADD, DL, VT, N0, NegN1); in visitFSUB()
16858 if (ConstOpIdx == 1 && N->getOpcode() == ISD::FDIV) in combineFMulOrFDivWithIntPow2()
16863 if (Pow2Op.getOpcode() != ISD::UINT_TO_FP && in combineFMulOrFDivWithIntPow2()
16864 (Pow2Op.getOpcode() != ISD::SINT_TO_FP || in combineFMulOrFDivWithIntPow2()
16889 N->getOpcode() == ISD::FMUL ? CurExp : (CurExp - MaxExpChange); in combineFMulOrFDivWithIntPow2()
16892 N->getOpcode() == ISD::FDIV ? CurExp : (CurExp + MaxExpChange); in combineFMulOrFDivWithIntPow2()
16906 return ISD::matchUnaryFpPredicate(ConstOp, IsFPConstValid); in combineFMulOrFDivWithIntPow2()
16935 SDValue Shift = DAG.getNode(ISD::SHL, DL, NewIntVT, Log2, MantissaShiftCnt); in combineFMulOrFDivWithIntPow2()
16937 DAG.getNode(N->getOpcode() == ISD::FMUL ? ISD::ADD : ISD::SUB, DL, in combineFMulOrFDivWithIntPow2()
16957 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FMUL, DL, VT, {N0, N1})) in visitFMUL()
16963 return DAG.getNode(ISD::FMUL, DL, VT, N1, N0); in visitFMUL()
16976 N0.getOpcode() == ISD::FMUL) { in visitFMUL()
16983 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1); in visitFMUL()
16984 return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts); in visitFMUL()
16990 if (N0.getOpcode() == ISD::FADD && N0.hasOneUse() && in visitFMUL()
16993 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1); in visitFMUL()
16994 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts); in visitFMUL()
16998 if (SDValue SD = reassociateReduction(ISD::VECREDUCE_FMUL, ISD::FMUL, DL, in visitFMUL()
17005 return DAG.getNode(ISD::FADD, DL, VT, N0, N0); in visitFMUL()
17009 if (!LegalOperations || TLI.isOperationLegal(ISD::FSUB, VT)) { in visitFMUL()
17010 return DAG.getNode(ISD::FSUB, DL, VT, in visitFMUL()
17028 return DAG.getNode(ISD::FMUL, DL, VT, NegN0, NegN1); in visitFMUL()
17034 (N0.getOpcode() == ISD::SELECT || N1.getOpcode() == ISD::SELECT) && in visitFMUL()
17035 TLI.isOperationLegal(ISD::FABS, VT)) { in visitFMUL()
17037 if (Select.getOpcode() != ISD::SELECT) in visitFMUL()
17045 Cond.getOpcode() == ISD::SETCC && Cond.getOperand(0) == X && in visitFMUL()
17048 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in visitFMUL()
17051 case ISD::SETOLT: in visitFMUL()
17052 case ISD::SETULT: in visitFMUL()
17053 case ISD::SETOLE: in visitFMUL()
17054 case ISD::SETULE: in visitFMUL()
17055 case ISD::SETLT: in visitFMUL()
17056 case ISD::SETLE: in visitFMUL()
17059 case ISD::SETOGT: in visitFMUL()
17060 case ISD::SETUGT: in visitFMUL()
17061 case ISD::SETOGE: in visitFMUL()
17062 case ISD::SETUGE: in visitFMUL()
17063 case ISD::SETGT: in visitFMUL()
17064 case ISD::SETGE: in visitFMUL()
17066 TLI.isOperationLegal(ISD::FNEG, VT)) in visitFMUL()
17067 return DAG.getNode(ISD::FNEG, DL, VT, in visitFMUL()
17068 DAG.getNode(ISD::FABS, DL, VT, X)); in visitFMUL()
17070 return DAG.getNode(ISD::FABS, DL, VT, X); in visitFMUL()
17108 return matcher.getNode(ISD::FMA, DL, VT, N0, N1, N2); in visitFMA()
17124 return matcher.getNode(ISD::FMA, DL, VT, NegN0, NegN1, N2); in visitFMA()
17137 return matcher.getNode(ISD::FADD, SDLoc(N), VT, N1, N2); in visitFMA()
17139 return matcher.getNode(ISD::FADD, SDLoc(N), VT, N0, N2); in visitFMA()
17144 return matcher.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2); in visitFMA()
17150 if (matcher.match(N2, ISD::FMUL) && N0 == N2.getOperand(0) && in visitFMA()
17154 ISD::FMUL, DL, VT, N0, in visitFMA()
17155 matcher.getNode(ISD::FADD, DL, VT, N1, N2.getOperand(1))); in visitFMA()
17159 if (matcher.match(N0, ISD::FMUL) && in visitFMA()
17163 ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
17164 matcher.getNode(ISD::FMUL, DL, VT, N1, N0.getOperand(1)), N2); in visitFMA()
17172 return matcher.getNode(ISD::FADD, DL, VT, N0, N2); in visitFMA()
17175 (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) { in visitFMA()
17176 SDValue RHSNeg = matcher.getNode(ISD::FNEG, DL, VT, N0); in visitFMA()
17178 return matcher.getNode(ISD::FADD, DL, VT, N2, RHSNeg); in visitFMA()
17182 if (matcher.match(N0, ISD::FNEG) && in visitFMA()
17183 (TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFMA()
17186 return matcher.getNode(ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
17187 matcher.getNode(ISD::FNEG, DL, VT, N1), N2); in visitFMA()
17195 return matcher.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
17196 matcher.getNode(ISD::FADD, DL, VT, N1, in visitFMA()
17201 if (N1CFP && matcher.match(N2, ISD::FNEG) && N2.getOperand(0) == N0) { in visitFMA()
17202 return matcher.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
17203 matcher.getNode(ISD::FADD, DL, VT, N1, in visitFMA()
17213 return matcher.getNode(ISD::FNEG, DL, VT, Neg); in visitFMA()
17227 return DAG.getNode(ISD::FMAD, DL, VT, N0, N1, N2); in visitFMAD()
17272 if (U->getOpcode() == ISD::FDIV && U->getOperand(1) == N1) { in combineRepeatedFPDivisors()
17274 if (U->getOperand(1).getOpcode() == ISD::FSQRT && in combineRepeatedFPDivisors()
17294 SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags); in combineRepeatedFPDivisors()
17300 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend, in combineRepeatedFPDivisors()
17325 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FDIV, DL, VT, {N0, N1})) in visitFDIV()
17355 TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFDIV()
17357 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFDIV()
17364 if (N1.getOpcode() == ISD::FSQRT) { in visitFDIV()
17366 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17367 } else if (N1.getOpcode() == ISD::FP_EXTEND && in visitFDIV()
17368 N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
17371 RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV); in visitFDIV()
17373 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17375 } else if (N1.getOpcode() == ISD::FP_ROUND && in visitFDIV()
17376 N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
17379 RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1)); in visitFDIV()
17381 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17383 } else if (N1.getOpcode() == ISD::FMUL) { in visitFDIV()
17387 if (N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
17390 } else if (N1.getOperand(1).getOpcode() == ISD::FSQRT) { in visitFDIV()
17400 if (Y.getOpcode() == ISD::FABS && Y.hasOneUse()) in visitFDIV()
17407 SDValue AA = DAG.getNode(ISD::FMUL, DL, VT, A, A); in visitFDIV()
17409 DAG.getNode(ISD::FMUL, DL, VT, AA, Sqrt.getOperand(0)); in visitFDIV()
17411 return DAG.getNode(ISD::FMUL, DL, VT, N0, Rsqrt); in visitFDIV()
17421 SDValue Div = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, Rsqrt, Y); in visitFDIV()
17423 return DAG.getNode(ISD::FMUL, DL, VT, N0, Div); in visitFDIV()
17437 if (N1.getOpcode() == ISD::FSQRT && N0 == N1.getOperand(0)) in visitFDIV()
17453 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, NegN0, NegN1); in visitFDIV()
17474 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FREM, DL, VT, {N0, N1})) in visitFREM()
17482 if (!TLI.isOperationLegal(ISD::FREM, VT) && in visitFREM()
17483 TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && in visitFREM()
17484 TLI.isOperationLegalOrCustom(ISD::FDIV, VT) && in visitFREM()
17485 TLI.isOperationLegalOrCustom(ISD::FTRUNC, VT) && in visitFREM()
17489 SDValue Div = DAG.getNode(ISD::FDIV, DL, VT, N0, N1); in visitFREM()
17490 SDValue Rnd = DAG.getNode(ISD::FTRUNC, DL, VT, Div); in visitFREM()
17493 MLA = DAG.getNode(ISD::FMA, DL, VT, DAG.getNode(ISD::FNEG, DL, VT, Rnd), in visitFREM()
17496 SDValue Mul = DAG.getNode(ISD::FMUL, DL, VT, Rnd, N1); in visitFREM()
17497 MLA = DAG.getNode(ISD::FSUB, DL, VT, N0, Mul); in visitFREM()
17499 return NeedsCopySign ? DAG.getNode(ISD::FCOPYSIGN, DL, VT, MLA, N0) : MLA; in visitFREM()
17547 if (N1.getOpcode() != ISD::FP_EXTEND && in CanCombineFCOPYSIGN_EXTEND_ROUND()
17548 N1.getOpcode() != ISD::FP_ROUND) in CanCombineFCOPYSIGN_EXTEND_ROUND()
17562 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FCOPYSIGN, DL, VT, {N0, N1})) in visitFCOPYSIGN()
17570 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT)) in visitFCOPYSIGN()
17571 return DAG.getNode(ISD::FABS, DL, VT, N0); in visitFCOPYSIGN()
17573 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFCOPYSIGN()
17574 return DAG.getNode(ISD::FNEG, DL, VT, in visitFCOPYSIGN()
17575 DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0)); in visitFCOPYSIGN()
17582 if (N0.getOpcode() == ISD::FABS || N0.getOpcode() == ISD::FNEG || in visitFCOPYSIGN()
17583 N0.getOpcode() == ISD::FCOPYSIGN) in visitFCOPYSIGN()
17584 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N0.getOperand(0), N1); in visitFCOPYSIGN()
17587 if (N1.getOpcode() == ISD::FABS) in visitFCOPYSIGN()
17588 return DAG.getNode(ISD::FABS, DL, VT, N0); in visitFCOPYSIGN()
17591 if (N1.getOpcode() == ISD::FCOPYSIGN) in visitFCOPYSIGN()
17592 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N0, N1.getOperand(1)); in visitFCOPYSIGN()
17597 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N0, N1.getOperand(0)); in visitFCOPYSIGN()
17640 (!DAG.getTargetLoweringInfo().isOperationExpand(ISD::FPOW, VT) && in visitFPOW()
17641 DAG.getTargetLoweringInfo().isOperationExpand(ISD::FCBRT, VT))) in visitFPOW()
17644 return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0)); in visitFPOW()
17669 if (!DAG.getTargetLoweringInfo().isOperationLegalOrCustom(ISD::FSQRT, VT)) in visitFPOW()
17679 SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0)); in visitFPOW()
17680 SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt); in visitFPOW()
17684 return DAG.getNode(ISD::FMUL, DL, VT, Sqrt, SqrtSqrt); in visitFPOW()
17699 if (!TLI.isOperationLegal(ISD::FTRUNC, VT) || in foldFPToIntToFP()
17706 if (N->getOpcode() == ISD::SINT_TO_FP && N0.getOpcode() == ISD::FP_TO_SINT && in foldFPToIntToFP()
17708 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
17710 if (N->getOpcode() == ISD::UINT_TO_FP && N0.getOpcode() == ISD::FP_TO_UINT && in foldFPToIntToFP()
17712 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
17730 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitSINT_TO_FP()
17731 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
17735 if (!hasOperation(ISD::SINT_TO_FP, OpVT) && in visitSINT_TO_FP()
17736 hasOperation(ISD::UINT_TO_FP, OpVT)) { in visitSINT_TO_FP()
17739 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
17744 if (N0.getOpcode() == ISD::SETCC && N0.getValueType() == MVT::i1 && in visitSINT_TO_FP()
17746 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
17754 if (N0.getOpcode() == ISD::ZERO_EXTEND && in visitSINT_TO_FP()
17755 N0.getOperand(0).getOpcode() == ISD::SETCC && !VT.isVector() && in visitSINT_TO_FP()
17756 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
17782 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitUINT_TO_FP()
17783 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
17787 if (!hasOperation(ISD::UINT_TO_FP, OpVT) && in visitUINT_TO_FP()
17788 hasOperation(ISD::SINT_TO_FP, OpVT)) { in visitUINT_TO_FP()
17791 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
17795 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() && in visitUINT_TO_FP()
17796 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitUINT_TO_FP()
17813 if (N0.getOpcode() != ISD::UINT_TO_FP && N0.getOpcode() != ISD::SINT_TO_FP) in FoldIntToFPToInt()
17818 bool IsInputSigned = N0.getOpcode() == ISD::SINT_TO_FP; in FoldIntToFPToInt()
17819 bool IsOutputSigned = N->getOpcode() == ISD::FP_TO_SINT; in FoldIntToFPToInt()
17839 unsigned ExtOp = IsInputSigned && IsOutputSigned ? ISD::SIGN_EXTEND in FoldIntToFPToInt()
17840 : ISD::ZERO_EXTEND; in FoldIntToFPToInt()
17844 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Src); in FoldIntToFPToInt()
17860 return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0); in visitFP_TO_SINT()
17875 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0); in visitFP_TO_UINT()
17902 DAG.FoldConstantArithmetic(ISD::FP_ROUND, SDLoc(N), VT, {N0, N1})) in visitFP_ROUND()
17906 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType()) in visitFP_ROUND()
17910 if (N0.getOpcode() == ISD::FP_ROUND) { in visitFP_ROUND()
17915 if (!hasOperation(ISD::FP_ROUND, VT)) in visitFP_ROUND()
17936 ISD::FP_ROUND, DL, VT, N0.getOperand(0), in visitFP_ROUND()
17946 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() && in visitFP_ROUND()
17949 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT, in visitFP_ROUND()
17952 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, in visitFP_ROUND()
17972 N->use_begin()->getOpcode() == ISD::FP_ROUND) in visitFP_EXTEND()
17977 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0); in visitFP_EXTEND()
17980 if (N0.getOpcode() == ISD::FP16_TO_FP && in visitFP_EXTEND()
17981 TLI.getOperationAction(ISD::FP16_TO_FP, VT) == TargetLowering::Legal) in visitFP_EXTEND()
17982 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0)); in visitFP_EXTEND()
17986 if (N0.getOpcode() == ISD::FP_ROUND in visitFP_EXTEND()
17991 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, in visitFP_EXTEND()
17993 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In); in visitFP_EXTEND()
17997 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in visitFP_EXTEND()
17998 TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, VT, N0.getValueType())) { in visitFP_EXTEND()
18000 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitFP_EXTEND()
18007 DAG.getNode(ISD::FP_ROUND, SDLoc(N0), N0.getValueType(), ExtLoad, in visitFP_EXTEND()
18025 return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0); in visitFCEIL()
18036 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0); in visitFTRUNC()
18043 case ISD::FRINT: in visitFTRUNC()
18044 case ISD::FTRUNC: in visitFTRUNC()
18045 case ISD::FNEARBYINT: in visitFTRUNC()
18046 case ISD::FROUNDEVEN: in visitFTRUNC()
18047 case ISD::FFLOOR: in visitFTRUNC()
18048 case ISD::FCEIL: in visitFTRUNC()
18060 return DAG.getNode(ISD::FFREXP, SDLoc(N), N->getVTList(), N0); in visitFFREXP()
18070 return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0); in visitFFLOOR()
18082 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); in visitFNEG()
18092 if (N0.getOpcode() == ISD::FSUB && in visitFNEG()
18095 return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0.getOperand(1), in visitFNEG()
18111 bool PropagatesNaN = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM; in visitFMinMax()
18112 bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM; in visitFMinMax()
18155 ? (IsMin ? ISD::VECREDUCE_FMINIMUM : ISD::VECREDUCE_FMAXIMUM) in visitFMinMax()
18156 : (IsMin ? ISD::VECREDUCE_FMIN : ISD::VECREDUCE_FMAX), in visitFMinMax()
18169 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFABS()
18172 if (N0.getOpcode() == ISD::FABS) in visitFABS()
18177 if (N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FCOPYSIGN) in visitFABS()
18178 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0)); in visitFABS()
18193 if (N1->getOpcode() == ISD::FREEZE && N1.hasOneUse()) { in visitBRCOND()
18194 return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other, Chain, in visitBRCOND()
18208 if (N1->getOpcode() == ISD::SETCC && N1.hasOneUse()) { in visitBRCOND()
18210 ISD::CondCode Cond = cast<CondCodeSDNode>(N1->getOperand(2))->get(); in visitBRCOND()
18216 auto IsAlwaysTrueOrFalse = [](ISD::CondCode Cond, ConstantSDNode *C) { in visitBRCOND()
18217 bool False = (Cond == ISD::SETULT && C->isZero()) || in visitBRCOND()
18218 (Cond == ISD::SETLT && C->isMinSignedValue()) || in visitBRCOND()
18219 (Cond == ISD::SETUGT && C->isAllOnes()) || in visitBRCOND()
18220 (Cond == ISD::SETGT && C->isMaxSignedValue()); in visitBRCOND()
18221 bool True = (Cond == ISD::SETULE && C->isAllOnes()) || in visitBRCOND()
18222 (Cond == ISD::SETLE && C->isMaxSignedValue()) || in visitBRCOND()
18223 (Cond == ISD::SETUGE && C->isZero()) || in visitBRCOND()
18224 (Cond == ISD::SETGE && C->isMinSignedValue()); in visitBRCOND()
18228 if (S0->getOpcode() == ISD::FREEZE && S0.hasOneUse() && S1C) { in visitBRCOND()
18234 if (S1->getOpcode() == ISD::FREEZE && S1.hasOneUse() && S0C) { in visitBRCOND()
18235 if (!IsAlwaysTrueOrFalse(ISD::getSetCCSwappedOperands(Cond), S0C)) { in visitBRCOND()
18243 ISD::BRCOND, SDLoc(N), MVT::Other, Chain, in visitBRCOND()
18255 if (N1.getOpcode() == ISD::SETCC && in visitBRCOND()
18256 TLI.isOperationLegalOrCustom(ISD::BR_CC, in visitBRCOND()
18258 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other, in visitBRCOND()
18268 return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other, in visitBRCOND()
18276 if (N.getOpcode() == ISD::SRL || in rebuildSetCC()
18277 (N.getOpcode() == ISD::TRUNCATE && in rebuildSetCC()
18279 N.getOperand(0).getOpcode() == ISD::SRL))) { in rebuildSetCC()
18281 if (N.getOpcode() == ISD::TRUNCATE) in rebuildSetCC()
18304 if (Op0.getOpcode() == ISD::AND && Op1.getOpcode() == ISD::Constant) { in rebuildSetCC()
18307 if (AndOp1.getOpcode() == ISD::Constant) { in rebuildSetCC()
18315 ISD::SETNE); in rebuildSetCC()
18323 if (N.getOpcode() == ISD::XOR) { in rebuildSetCC()
18330 while (N.getOpcode() == ISD::XOR) { in rebuildSetCC()
18343 if (N.getOpcode() != ISD::XOR) in rebuildSetCC()
18349 if (Op0.getOpcode() != ISD::SETCC && Op1.getOpcode() != ISD::SETCC) { in rebuildSetCC()
18352 if (isBitwiseNot(N) && Op0.hasOneUse() && Op0.getOpcode() == ISD::XOR && in rebuildSetCC()
18365 Equal ? ISD::SETEQ : ISD::SETNE); in rebuildSetCC()
18391 if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC) in visitBR_CC()
18392 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other, in visitBR_CC()
18455 if (!getCombineLoadStoreParts(N, ISD::PRE_INC, ISD::PRE_DEC, IsLoad, IsMasked, in CombineToPreIndexedLoadStore()
18461 if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) || in CombineToPreIndexedLoadStore()
18468 ISD::MemIndexedMode AM = ISD::UNINDEXED; in CombineToPreIndexedLoadStore()
18536 if (Use.getUser()->getOpcode() != ISD::ADD && in CombineToPreIndexedLoadStore()
18537 Use.getUser()->getOpcode() != ISD::SUB) { in CombineToPreIndexedLoadStore()
18633 int X0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 1) ? -1 : 1; in CombineToPreIndexedLoadStore()
18634 int Y0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 0) ? -1 : 1; in CombineToPreIndexedLoadStore()
18635 int X1 = (AM == ISD::PRE_DEC && !Swapped) ? -1 : 1; in CombineToPreIndexedLoadStore()
18636 int Y1 = (AM == ISD::PRE_DEC && Swapped) ? -1 : 1; in CombineToPreIndexedLoadStore()
18638 unsigned Opcode = (Y0 * Y1 < 0) ? ISD::SUB : ISD::ADD; in CombineToPreIndexedLoadStore()
18668 ISD::MemIndexedMode &AM, in shouldCombineToPostInc()
18672 (PtrUse->getOpcode() != ISD::ADD && PtrUse->getOpcode() != ISD::SUB)) in shouldCombineToPostInc()
18695 if (getCombineLoadStoreParts(Use, ISD::POST_INC, ISD::POST_DEC, IsLoad, in shouldCombineToPostInc()
18706 if (Use->getOpcode() == ISD::ADD || Use->getOpcode() == ISD::SUB) { in shouldCombineToPostInc()
18718 ISD::MemIndexedMode &AM, in getPostIndexedLoadStoreOp()
18721 if (!getCombineLoadStoreParts(N, ISD::POST_INC, ISD::POST_DEC, IsLoad, in getPostIndexedLoadStoreOp()
18765 ISD::MemIndexedMode AM = ISD::UNINDEXED; in CombineToPostIndexedLoadStore()
18806 ISD::MemIndexedMode AM = LD->getAddressingMode(); in SplitIndexingFromLoad()
18807 assert(AM != ISD::UNINDEXED); in SplitIndexingFromLoad()
18814 assert((Inc.getOpcode() != ISD::TargetConstant || in SplitIndexingFromLoad()
18817 if (Inc.getOpcode() == ISD::TargetConstant) { in SplitIndexingFromLoad()
18824 (AM == ISD::PRE_INC || AM == ISD::POST_INC ? ISD::ADD : ISD::SUB); in SplitIndexingFromLoad()
18840 TLI.isOperationLegal(ISD::FTRUNC, STMemType)) { in getTruncatedStoreValue()
18841 Val = DAG.getNode(ISD::FTRUNC, SDLoc(ST), STMemType, Val); in getTruncatedStoreValue()
18846 Val = DAG.getNode(ISD::TRUNCATE, SDLoc(ST), STMemType, Val); in getTruncatedStoreValue()
18865 case ISD::NON_EXTLOAD: in extendLoadedValueToExtension()
18868 case ISD::EXTLOAD: in extendLoadedValueToExtension()
18869 Val = DAG.getNode(ISD::ANY_EXTEND, SDLoc(LD), LDType, Val); in extendLoadedValueToExtension()
18871 case ISD::SEXTLOAD: in extendLoadedValueToExtension()
18872 Val = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(LD), LDType, Val); in extendLoadedValueToExtension()
18874 case ISD::ZEXTLOAD: in extendLoadedValueToExtension()
18875 Val = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(LD), LDType, Val); in extendLoadedValueToExtension()
18887 if (Chain.getOpcode() == ISD::CALLSEQ_START) in getUniqueStoreFeeding()
18892 if (Chain.getOpcode() == ISD::TokenFactor) { in getUniqueStoreFeeding()
19001 !LDMemType.isVector() && LD->getExtensionType() != ISD::SEXTLOAD) { in ForwardStoreValueToDirectLoad()
19007 auto Val = DAG.getNode(ISD::AND, SDLoc(LD), LDType, ST->getValue(), Mask); in ForwardStoreValueToDirectLoad()
19018 TLI.isOperationLegal(ISD::SRL, STType)) { in ForwardStoreValueToDirectLoad()
19019 Val = DAG.getNode(ISD::SRL, SDLoc(LD), STType, Val, in ForwardStoreValueToDirectLoad()
19039 Val = DAG.getNode(ISD::TRUNCATE, SDLoc(LD), LDMemType, Val); in ForwardStoreValueToDirectLoad()
19148 if (LD->getExtensionType() == ISD::NON_EXTLOAD) { in visitLOAD()
19159 SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N), in visitLOAD()
19352 if (!TLI.isOperationLegal(ISD::LOAD, SliceType)) in isLegal()
19366 if (!TLI.isOperationLegal(ISD::ADD, PtrType)) in isLegal()
19372 !TLI.isOperationLegal(ISD::ZERO_EXTEND, TruncateType)) in isLegal()
19416 BaseAddr = DAG->getNode(ISD::ADD, DL, ArithType, BaseAddr, in loadSlice()
19433 DAG->getNode(ISD::ZERO_EXTEND, SDLoc(LastInst), FinalType, LastInst); in loadSlice()
19445 if (Use->getOpcode() != ISD::BITCAST) in canMergeExpensiveCrossRegisterBankCopy()
19455 if (ArgRC == ResRC || !TLI.isOperationLegal(ISD::LOAD, ResVT)) in canMergeExpensiveCrossRegisterBankCopy()
19476 if (!TLI.isOperationLegal(ISD::LOAD, ResVT)) in canMergeExpensiveCrossRegisterBankCopy()
19634 if (!LD->isSimple() || !ISD::isNormalLoad(LD) || in SliceUpLoad()
19663 if (User->getOpcode() == ISD::SRL && User->hasOneUse() && in SliceUpLoad()
19671 if (User->getOpcode() != ISD::TRUNCATE) in SliceUpLoad()
19714 if (SliceInst.getOpcode() != ISD::LOAD) in SliceUpLoad()
19716 assert(SliceInst->getOpcode() == ISD::LOAD && in SliceUpLoad()
19721 SDValue Chain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, in SliceUpLoad()
19736 if (V->getOpcode() != ISD::AND || in CheckForMaskedLoad()
19738 !ISD::isNormalLoad(V->getOperand(0).getNode())) in CheckForMaskedLoad()
19785 else if (Chain->getOpcode() == ISD::TokenFactor && in CheckForMaskedLoad()
19845 ISD::SRL, DL, IVal.getValueType(), IVal, in ShrinkLoadReplaceStoreWithStore()
19869 IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal); in ShrinkLoadReplaceStoreWithStore()
19896 if ((Opc != ISD::OR && Opc != ISD::XOR && Opc != ISD::AND) || in ReduceLoadOpStoreWidth()
19905 if (Opc == ISD::OR && EnableShrinkLoadReplaceStoreWithStore) { in ReduceLoadOpStoreWidth()
19924 if (Value.getOperand(1).getOpcode() != ISD::Constant) in ReduceLoadOpStoreWidth()
19928 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in ReduceLoadOpStoreWidth()
19940 if (Opc == ISD::AND) in ReduceLoadOpStoreWidth()
19968 if (Opc == ISD::AND) in ReduceLoadOpStoreWidth()
20016 if (ISD::isNormalStore(ST) && ISD::isNormalLoad(Value.getNode()) && in TransformFPLoadStorePair()
20037 if (!TLI.isOperationLegal(ISD::LOAD, IntVT) || in TransformFPLoadStorePair()
20038 !TLI.isOperationLegal(ISD::STORE, IntVT) || in TransformFPLoadStorePair()
20039 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) || in TransformFPLoadStorePair()
20040 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT) || in TransformFPLoadStorePair()
20095 if (Use->getOpcode() == ISD::MUL) { // We have another multiply use. in isMulAddWithConstProfitable()
20131 if (OtherOp->getOpcode() == ISD::ADD && in isMulAddWithConstProfitable()
20258 StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS in mergeStoresOfConstantsOrVecElts()
20259 : ISD::BUILD_VECTOR, in mergeStoresOfConstantsOrVecElts()
20273 (Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT || in mergeStoresOfConstantsOrVecElts()
20274 Val.getOpcode() == ISD::EXTRACT_SUBVECTOR)) { in mergeStoresOfConstantsOrVecElts()
20280 Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in mergeStoresOfConstantsOrVecElts()
20281 Val = DAG.getNode(ISD::BUILD_VECTOR, DL, MemVT, Val); in mergeStoresOfConstantsOrVecElts()
20283 unsigned OpC = MemVT.isVector() ? ISD::EXTRACT_SUBVECTOR in mergeStoresOfConstantsOrVecElts()
20284 : ISD::EXTRACT_VECTOR_ELT; in mergeStoresOfConstantsOrVecElts()
20294 StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS in mergeStoresOfConstantsOrVecElts()
20295 : ISD::BUILD_VECTOR, in mergeStoresOfConstantsOrVecElts()
20327 } else if (ISD::isBuildVectorOfConstantSDNodes(Val.getNode()) || in mergeStoresOfConstantsOrVecElts()
20328 ISD::isBuildVectorOfConstantFPSDNodes(Val.getNode())) { in mergeStoresOfConstantsOrVecElts()
20470 if (OtherBC.getOpcode() != ISD::EXTRACT_VECTOR_ELT && in getStoreMergeCandidates()
20471 OtherBC.getOpcode() != ISD::EXTRACT_SUBVECTOR) in getStoreMergeCandidates()
20575 if (N->getOpcode() == ISD::TokenFactor) { in checkMergeStoreCandidatesForDependencies()
20691 else if (ISD::isBuildVectorAllZeros(StoredVal.getNode())) in tryStoreMergeOfConstants()
20918 (hasOperation(ISD::ROTL, PairVT) || in tryStoreMergeOfLoads()
20919 hasOperation(ISD::ROTR, PairVT))) { in tryStoreMergeOfLoads()
21002 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValTy, StoreTy) && in tryStoreMergeOfLoads()
21003 TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValTy, StoreTy) && in tryStoreMergeOfLoads()
21004 TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValTy, StoreTy) && in tryStoreMergeOfLoads()
21107 StoreOp = DAG.getNode(ISD::ROTL, LoadDL, JointMemOpVT, NewLoad, RotAmt); in tryStoreMergeOfLoads()
21117 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, LoadDL, ExtendedTy, in tryStoreMergeOfLoads()
21260 SDValue Token = DAG.getNode(ISD::TokenFactor, SL, in replaceStoreChain()
21272 if (Value.getOpcode() == ISD::TargetConstantFP) in replaceStoreOfFPConstant()
21275 if (!ISD::isNormalStore(ST)) in replaceStoreOfFPConstant()
21302 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { in replaceStoreOfFPConstant()
21313 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) { in replaceStoreOfFPConstant()
21320 if (ST->isSimple() && TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32) && in replaceStoreOfFPConstant()
21340 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in replaceStoreOfFPConstant()
21358 if (Value.getOpcode() != ISD::INSERT_VECTOR_ELT || !Value.hasOneUse()) in replaceStoreOfInsertLoad()
21374 !ISD::isNormalStore(ST) || in replaceStoreOfInsertLoad()
21430 if (Value.getOpcode() == ISD::BITCAST && !ST->isTruncatingStore() && in visitSTORE()
21440 TLI.isOperationLegal(ISD::STORE, SVT)) && in visitSTORE()
21495 if ((Value.getOpcode() == ISD::ZERO_EXTEND || in visitSTORE()
21496 Value.getOpcode() == ISD::SIGN_EXTEND || in visitSTORE()
21497 Value.getOpcode() == ISD::ANY_EXTEND) && in visitSTORE()
21499 TLI.isOperationLegalOrCustom(ISD::STORE, ST->getMemoryVT())) in visitSTORE()
21515 if (N->getOpcode() != ISD::DELETED_NODE) in visitSTORE()
21613 if ((Value.getOpcode() == ISD::FP_ROUND || in visitSTORE()
21614 Value.getOpcode() == ISD::TRUNCATE) && in visitSTORE()
21634 if (N->getOpcode() == ISD::DELETED_NODE || !isa<StoreSDNode>(N)) in visitSTORE()
21674 case ISD::TokenFactor: in visitLIFETIME_END()
21678 case ISD::LIFETIME_START: in visitLIFETIME_END()
21679 case ISD::LIFETIME_END: in visitLIFETIME_END()
21685 case ISD::STORE: { in visitLIFETIME_END()
21753 if (!Val.getValueType().isScalarInteger() || Val.getOpcode() != ISD::OR) in splitMergedValStore()
21760 if (Op1.getOpcode() != ISD::SHL) { in splitMergedValStore()
21762 if (Op1.getOpcode() != ISD::SHL) in splitMergedValStore()
21778 if (Lo.getOpcode() != ISD::ZERO_EXTEND || !Lo.hasOneUse() || in splitMergedValStore()
21781 Hi.getOpcode() != ISD::ZERO_EXTEND || !Hi.hasOneUse() || in splitMergedValStore()
21788 EVT LowTy = (Lo.getOperand(0).getOpcode() == ISD::BITCAST) in splitMergedValStore()
21791 EVT HighTy = (Hi.getOperand(0).getOpcode() == ISD::BITCAST) in splitMergedValStore()
21803 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Lo.getOperand(0)); in splitMergedValStore()
21804 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Hi.getOperand(0)); in splitMergedValStore()
21828 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in mergeEltWithShuffle()
21856 if (ArgVal.getOpcode() == ISD::CONCAT_VECTORS) { in mergeEltWithShuffle()
21893 assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT && in mergeInsertEltWithShuffle()
21923 assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT && in combineInsertEltToShuffle()
21927 if (InsertVal.getOpcode() != ISD::BITCAST || !InsertVal.hasOneUse() || in combineInsertEltToShuffle()
21967 SDValue PaddedSubV = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShufVT, ConcatOps); in combineInsertEltToShuffle()
22004 if (Scalar.getOpcode() == ISD::ZERO_EXTEND || in combineInsertEltToLoad()
22005 Scalar.getOpcode() == ISD::SIGN_EXTEND || in combineInsertEltToLoad()
22006 Scalar.getOpcode() == ISD::ANY_EXTEND) { in combineInsertEltToLoad()
22027 !VecLoad->isSimple() || VecLoad->getExtensionType() != ISD::NON_EXTLOAD || in combineInsertEltToLoad()
22028 ScalarLoad->getExtensionType() != ISD::NON_EXTLOAD || in combineInsertEltToLoad()
22058 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), VecLoad->getBasePtr(), in combineInsertEltToLoad()
22087 if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitINSERT_VECTOR_ELT()
22110 if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitINSERT_VECTOR_ELT()
22123 if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT && InVec.hasOneUse() in visitINSERT_VECTOR_ELT()
22128 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in visitINSERT_VECTOR_ELT()
22131 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(InVec.getNode()), in visitINSERT_VECTOR_ELT()
22146 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) { in visitINSERT_VECTOR_ELT()
22187 if (CurVec.getOpcode() == ISD::BUILD_VECTOR && CurVec.hasOneUse()) { in visitINSERT_VECTOR_ELT()
22194 if (CurVec.getOpcode() == ISD::SCALAR_TO_VECTOR && CurVec.hasOneUse()) { in visitINSERT_VECTOR_ELT()
22200 if (CurVec.getOpcode() == ISD::INSERT_VECTOR_ELT && CurVec.hasOneUse()) in visitINSERT_VECTOR_ELT()
22217 if (CurVec.getOpcode() == ISD::VECTOR_SHUFFLE && CurVec.hasOneUse()) { in visitINSERT_VECTOR_ELT()
22250 return DAG.getNode(ISD::AND, DL, VT, CurVec, in visitINSERT_VECTOR_ELT()
22293 ISD::LoadExtType ExtTy = in scalarizeExtractedVectorLoad()
22294 ResultVT.bitsGT(VecEltVT) ? ISD::NON_EXTLOAD : ISD::EXTLOAD; in scalarizeExtractedVectorLoad()
22295 if (!TLI.isOperationLegalOrCustom(ISD::LOAD, VecEltVT) || in scalarizeExtractedVectorLoad()
22331 ISD::LoadExtType ExtType = in scalarizeExtractedVectorLoad()
22332 TLI.isLoadExtLegal(ISD::ZEXTLOAD, ResultVT, VecEltVT) ? ISD::ZEXTLOAD in scalarizeExtractedVectorLoad()
22333 : ISD::EXTLOAD; in scalarizeExtractedVectorLoad()
22346 Load = DAG.getNode(ISD::TRUNCATE, DL, ResultVT, Load); in scalarizeExtractedVectorLoad()
22377 ISD::isConstantSplatVector(Op0.getNode(), SplatVal) || in scalarizeExtractedBinop()
22379 ISD::isConstantSplatVector(Op1.getNode(), SplatVal)) { in scalarizeExtractedBinop()
22383 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index); in scalarizeExtractedBinop()
22384 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index); in scalarizeExtractedBinop()
22473 case ISD::TRUNCATE: in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22480 case ISD::SRL: in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22497 if (User->getOpcode() != ISD::BUILD_VECTOR) in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22535 !(TLI.isOperationLegalOrCustom(ISD::BITCAST, NewVecVT) && in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22536 TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, NewVecVT))) in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22545 SDValue V = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, NewScalarVT, NewVecOp, in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22566 if (VecOp.getOpcode() == ISD::INSERT_VECTOR_ELT && in visitEXTRACT_VECTOR_ELT()
22573 if (VecOp.getOpcode() == ISD::SCALAR_TO_VECTOR) { in visitEXTRACT_VECTOR_ELT()
22585 return DAG.getNode(ISD::TRUNCATE, DL, ScalarVT, InOp); in visitEXTRACT_VECTOR_ELT()
22586 return DAG.getNode(ISD::ANY_EXTEND, DL, ScalarVT, InOp); in visitEXTRACT_VECTOR_ELT()
22598 if (((IndexC && VecOp.getOpcode() == ISD::BUILD_VECTOR) || in visitEXTRACT_VECTOR_ELT()
22599 VecOp.getOpcode() == ISD::SPLAT_VECTOR) && in visitEXTRACT_VECTOR_ELT()
22601 assert((VecOp.getOpcode() != ISD::BUILD_VECTOR || in visitEXTRACT_VECTOR_ELT()
22605 VecOp.getOpcode() == ISD::BUILD_VECTOR ? IndexC->getZExtValue() : 0; in visitEXTRACT_VECTOR_ELT()
22648 if (IndexC && VecOp.getOpcode() == ISD::BITCAST && VecVT.isInteger() && in visitEXTRACT_VECTOR_ELT()
22660 BCSrc.getOpcode() == ISD::SCALAR_TO_VECTOR) { in visitEXTRACT_VECTOR_ELT()
22687 if (IndexC && VecOp.getOpcode() == ISD::VECTOR_SHUFFLE) { in visitEXTRACT_VECTOR_ELT()
22705 if (SVInVec.getOpcode() == ISD::BUILD_VECTOR) { in visitEXTRACT_VECTOR_ELT()
22720 TLI.isOperationLegal(ISD::EXTRACT_VECTOR_ELT, VecVT) || in visitEXTRACT_VECTOR_ELT()
22721 TLI.isOperationExpand(ISD::VECTOR_SHUFFLE, VecVT)) { in visitEXTRACT_VECTOR_ELT()
22722 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, SVInVec, in visitEXTRACT_VECTOR_ELT()
22730 return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitEXTRACT_VECTOR_ELT()
22743 if (N->getOpcode() != ISD::DELETED_NODE) in visitEXTRACT_VECTOR_ELT()
22751 if (N->getOpcode() != ISD::DELETED_NODE) in visitEXTRACT_VECTOR_ELT()
22769 if (VecOp.getOpcode() == ISD::BITCAST) { in visitEXTRACT_VECTOR_ELT()
22785 ISD::isNormalLoad(VecOp.getNode()) && in visitEXTRACT_VECTOR_ELT()
22802 if (ISD::isNormalLoad(VecOp.getNode())) { in visitEXTRACT_VECTOR_ELT()
22804 } else if (VecOp.getOpcode() == ISD::SCALAR_TO_VECTOR && in visitEXTRACT_VECTOR_ELT()
22806 ISD::isNormalLoad(VecOp.getOperand(0).getNode())) { in visitEXTRACT_VECTOR_ELT()
22831 if (VecOp.getOpcode() == ISD::BITCAST) { in visitEXTRACT_VECTOR_ELT()
22838 if (ISD::isNormalLoad(VecOp.getNode())) { in visitEXTRACT_VECTOR_ELT()
22843 } else if (VecOp.getOpcode() == ISD::CONCAT_VECTORS && !BCNumEltsChanged && in visitEXTRACT_VECTOR_ELT()
22862 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, in visitEXTRACT_VECTOR_ELT()
22865 return DAG.getNode(ISD::BITCAST, DL, ScalarVT, Elt); in visitEXTRACT_VECTOR_ELT()
22908 bool AnyExt = In.getOpcode() == ISD::ANY_EXTEND; in reduceBuildVecExtToExtBuildVec()
22909 bool ZeroExt = In.getOpcode() == ISD::ZERO_EXTEND; in reduceBuildVecExtToExtBuildVec()
22965 assert((Cast.getOpcode() == ISD::ANY_EXTEND || in reduceBuildVecExtToExtBuildVec()
22966 Cast.getOpcode() == ISD::ZERO_EXTEND || in reduceBuildVecExtToExtBuildVec()
22986 (!TLI.isOperationLegal(ISD::BUILD_VECTOR, VecVT) && in reduceBuildVecExtToExtBuildVec()
22987 TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))) in reduceBuildVecExtToExtBuildVec()
23004 assert(N->getOpcode() == ISD::BUILD_VECTOR && "Expected build vector"); in reduceBuildVecTruncToBitCast()
23029 if (Op.getOpcode() == ISD::BITCAST) in reduceBuildVecTruncToBitCast()
23041 if (In.getOpcode() != ISD::TRUNCATE) in reduceBuildVecTruncToBitCast()
23046 if (In.getOpcode() != ISD::SRL) { in reduceBuildVecTruncToBitCast()
23118 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
23127 VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, in createBuildVecShuffle()
23129 VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, ZeroIdx); in createBuildVecShuffle()
23142 !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, InVT1)) in createBuildVecShuffle()
23151 VecIn2 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT1, in createBuildVecShuffle()
23159 VecIn2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
23175 if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, InVT1)) in createBuildVecShuffle()
23179 VecIn2 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT1, in createBuildVecShuffle()
23220 Shuffle = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuffle, ZeroIdx); in createBuildVecShuffle()
23226 assert(BV->getOpcode() == ISD::BUILD_VECTOR && "Expected build vector"); in reduceBuildVecToShuffleWithZero()
23253 if (Zext.getOpcode() != ISD::ZERO_EXTEND || !Zext.hasOneUse() || in reduceBuildVecToShuffleWithZero()
23254 Zext.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT || in reduceBuildVecToShuffleWithZero()
23325 if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, VT)) in reduceBuildVecToShuffle()
23358 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in reduceBuildVecToShuffle()
23420 SDValue VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec, in reduceBuildVecToShuffle()
23422 SDValue VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec, in reduceBuildVecToShuffle()
23533 bool IsLeftShuffle = L.getOpcode() == ISD::VECTOR_SHUFFLE && in reduceBuildVecToShuffle()
23542 bool IsRightShuffle = R.getOpcode() == ISD::VECTOR_SHUFFLE && in reduceBuildVecToShuffle()
23583 FoundZeroExtend |= (Opc == ISD::ZERO_EXTEND); in convertBuildVecZextToZext()
23584 if ((Opc == ISD::ZERO_EXTEND || Opc == ISD::ANY_EXTEND) && in convertBuildVecZextToZext()
23585 Op.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT && in convertBuildVecZextToZext()
23616 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InVT, In, in convertBuildVecZextToZext()
23618 return DAG.getNode(FoundZeroExtend ? ISD::ZERO_EXTEND : ISD::ANY_EXTEND, DL, in convertBuildVecZextToZext()
23645 (LegalOperations && !TLI.isOperationLegalOrCustom(ISD::BITCAST, OpIntVT))) in convertBuildVecZextToBuildVecWithZeros()
23674 if (Op.getOpcode() != ISD::ZERO_EXTEND) in convertBuildVecZextToBuildVecWithZeros()
23709 !(TLI.isOperationLegalOrCustom(ISD::TRUNCATE, NewScalarIntVT) && in convertBuildVecZextToBuildVecWithZeros()
23710 TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, NewIntVT)))) in convertBuildVecZextToBuildVecWithZeros()
23733 Op = DAG.getNode(ISD::TRUNCATE, DL, NewScalarIntVT, Op); in convertBuildVecZextToBuildVecWithZeros()
23747 if (ISD::allOperandsUndef(N)) in visitBUILD_VECTOR()
23772 DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), NewVT, Ops); in visitBUILD_VECTOR()
23783 if ((Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT) && in visitBUILD_VECTOR()
23804 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), N->getValueType(0), in visitBUILD_VECTOR()
23825 if (TLI.getOperationAction(ISD::SPLAT_VECTOR, VT) != TargetLowering::Expand) in visitBUILD_VECTOR()
23828 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V); in visitBUILD_VECTOR()
23851 if (ISD::BITCAST == Op.getOpcode() && in combineConcatVectorOfScalars()
23854 else if (ISD::UNDEF == Op.getOpcode()) in combineConcatVectorOfScalars()
23855 Ops.push_back(DAG.getNode(ISD::UNDEF, DL, SVT)); in combineConcatVectorOfScalars()
23880 Op = DAG.getNode(ISD::UNDEF, DL, SVT); in combineConcatVectorOfScalars()
23905 if (Op.getOpcode() != ISD::CONCAT_VECTORS) in combineConcatVectorOfConcatVectors()
23927 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, ConcatOps); in combineConcatVectorOfConcatVectors()
23957 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR) in combineConcatVectorOfExtracts()
24011 case ISD::SINT_TO_FP: in combineConcatVectorOfCasts()
24012 case ISD::UINT_TO_FP: in combineConcatVectorOfCasts()
24013 case ISD::FP_TO_SINT: in combineConcatVectorOfCasts()
24014 case ISD::FP_TO_UINT: in combineConcatVectorOfCasts()
24049 case ISD::SINT_TO_FP: in combineConcatVectorOfCasts()
24050 case ISD::UINT_TO_FP: in combineConcatVectorOfCasts()
24055 case ISD::FP_TO_SINT: in combineConcatVectorOfCasts()
24056 case ISD::FP_TO_UINT: in combineConcatVectorOfCasts()
24067 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatSrcVT, SrcOps); in combineConcatVectorOfCasts()
24089 !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) in combineConcatVectorOfShuffleAndItsOperands()
24167 NewShufOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, ShufOpParts); in combineConcatVectorOfShuffleAndItsOperands()
24181 if (ISD::allOperandsUndef(N)) in visitCONCAT_VECTORS()
24197 if (In.getOpcode() == ISD::CONCAT_VECTORS && In.hasOneUse() && in visitCONCAT_VECTORS()
24202 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitCONCAT_VECTORS()
24209 if (!LegalOperations && Scalar.getOpcode() == ISD::SCALAR_TO_VECTOR && in visitCONCAT_VECTORS()
24221 if (Scalar->getOpcode() == ISD::TRUNCATE && in visitCONCAT_VECTORS()
24243 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), NVT, Scalar); in visitCONCAT_VECTORS()
24253 return ISD::UNDEF == Op.getOpcode() || ISD::BUILD_VECTOR == Op.getOpcode(); in visitCONCAT_VECTORS()
24265 if (ISD::BUILD_VECTOR == Op.getOpcode()) { in visitCONCAT_VECTORS()
24277 if (ISD::UNDEF == Op.getOpcode()) in visitCONCAT_VECTORS()
24280 if (ISD::BUILD_VECTOR == Op.getOpcode()) { in visitCONCAT_VECTORS()
24287 DAG.getNode(ISD::TRUNCATE, SDLoc(N), MinVT, Op.getOperand(i))); in visitCONCAT_VECTORS()
24336 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR) in visitCONCAT_VECTORS()
24368 if (V.getOpcode() == ISD::INSERT_SUBVECTOR && in getSubVectorSrc()
24373 if (IndexC && V.getOpcode() == ISD::CONCAT_VECTORS && in getSubVectorSrc()
24443 if (BOpcode == ISD::FSUB) { in narrowExtractedVectorBinOp()
24493 SDValue X = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
24495 SDValue Y = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
24512 if (BOpcode != ISD::AND && BOpcode != ISD::OR && BOpcode != ISD::XOR) in narrowExtractedVectorBinOp()
24518 if (V.getOpcode() == ISD::CONCAT_VECTORS && V.getNumOperands() == 2) in narrowExtractedVectorBinOp()
24534 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
24538 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
24620 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && in foldExtractSubvectorFromShuffleVector()
24642 !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, NarrowVT)) in foldExtractSubvectorFromShuffleVector()
24750 NewOps.emplace_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowVT, in foldExtractSubvectorFromShuffleVector()
24773 if (TLI.isOperationLegalOrCustomOrPromote(ISD::LOAD, NVT)) in visitEXTRACT_SUBVECTOR()
24779 if (ExtIdx == 0 && V.getOpcode() == ISD::EXTRACT_SUBVECTOR && V.hasOneUse()) { in visitEXTRACT_SUBVECTOR()
24782 TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NVT)) { in visitEXTRACT_SUBVECTOR()
24783 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, V.getOperand(0), in visitEXTRACT_SUBVECTOR()
24789 if (V.getOpcode() == ISD::SPLAT_VECTOR) in visitEXTRACT_SUBVECTOR()
24791 if (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, NVT)) in visitEXTRACT_SUBVECTOR()
24797 if (V.getOpcode() == ISD::INSERT_SUBVECTOR) { in visitEXTRACT_SUBVECTOR()
24807 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, InsSub, in visitEXTRACT_SUBVECTOR()
24813 if (V.getOpcode() == ISD::BITCAST && in visitEXTRACT_SUBVECTOR()
24815 (!LegalOperations || TLI.isOperationLegal(ISD::BITCAST, NVT))) { in visitEXTRACT_SUBVECTOR()
24825 if (TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NewExtVT)) { in visitEXTRACT_SUBVECTOR()
24827 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT, in visitEXTRACT_SUBVECTOR()
24842 if (TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NewExtVT)) { in visitEXTRACT_SUBVECTOR()
24845 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT, in visitEXTRACT_SUBVECTOR()
24850 TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, ScalarVT)) { in visitEXTRACT_SUBVECTOR()
24853 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, in visitEXTRACT_SUBVECTOR()
24862 if (V.getOpcode() == ISD::CONCAT_VECTORS) { in visitEXTRACT_SUBVECTOR()
24892 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, in visitEXTRACT_SUBVECTOR()
24904 if (V.getOpcode() == ISD::BUILD_VECTOR) { in visitEXTRACT_SUBVECTOR()
24917 TLI.isOperationLegal(ISD::BUILD_VECTOR, ExtractVT))) && in visitEXTRACT_SUBVECTOR()
24924 Src = DAG.getNode(ISD::TRUNCATE, DL, EltVT, Src); in visitEXTRACT_SUBVECTOR()
24936 if (V.getOpcode() == ISD::INSERT_SUBVECTOR) { in visitEXTRACT_SUBVECTOR()
24951 if (LegalOperations && !TLI.isOperationLegal(ISD::BITCAST, NVT)) in visitEXTRACT_SUBVECTOR()
24957 ISD::EXTRACT_SUBVECTOR, DL, NVT, in visitEXTRACT_SUBVECTOR()
24978 if (N0.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 || in foldShuffleOfConcatUndefs()
24979 N1.getOpcode() != ISD::CONCAT_VECTORS || N1.getNumOperands() != 2 || in foldShuffleOfConcatUndefs()
25018 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Shuf0, Shuf1); in foldShuffleOfConcatUndefs()
25049 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1); in partitionShuffleOfConcats()
25083 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in partitionShuffleOfConcats()
25121 if (N0AnyConst && !N1AnyConst && !ISD::isBuildVectorAllZeros(N0.getNode())) in combineShuffleOfScalars()
25123 if (!N0AnyConst && N1AnyConst && !ISD::isBuildVectorAllZeros(N1.getNode())) in combineShuffleOfScalars()
25143 if (S.getOpcode() == ISD::BUILD_VECTOR) { in combineShuffleOfScalars()
25145 } else if (S.getOpcode() == ISD::SCALAR_TO_VECTOR) { in combineShuffleOfScalars()
25247 unsigned Opcode = ISD::ANY_EXTEND_VECTOR_INREG; in combineShuffleToAnyExtendVectorInreg()
25367 unsigned Opcode = ISD::ZERO_EXTEND_VECTOR_INREG; in combineShuffleToZeroExtendVectorInReg()
25399 if (!ISD::isExtVecInRegOpcode(Opcode)) in combineTruncationShuffle()
25555 if (Op0.getOpcode() != ISD::BITCAST) in combineShuffleOfBitcast()
25559 (!Op1.isUndef() && (Op1.getOpcode() != ISD::BITCAST || in combineShuffleOfBitcast()
25570 !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, InVT))) in combineShuffleOfBitcast()
25699 if (Op0.getOpcode() != ISD::INSERT_VECTOR_ELT) in replaceShuffleOfInsert()
25717 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Shuf), Op0.getValueType(), in replaceShuffleOfInsert()
25811 SDValue ExtL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, L, Index); in visitVECTOR_SHUFFLE()
25812 SDValue ExtR = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, R, Index); in visitVECTOR_SHUFFLE()
25815 SDValue Insert = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, NewBO); in visitVECTOR_SHUFFLE()
25822 if ((!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) && in visitVECTOR_SHUFFLE()
25824 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && SplatIndex == 0) in visitVECTOR_SHUFFLE()
25827 if (N0.getOpcode() == ISD::INSERT_VECTOR_ELT) in visitVECTOR_SHUFFLE()
25834 if (N0.getOpcode() == ISD::BITCAST && N0.getOperand(0).hasOneUse() && in visitVECTOR_SHUFFLE()
25836 (N0.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR || in visitVECTOR_SHUFFLE()
25837 N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR)) { in visitVECTOR_SHUFFLE()
25854 if (V->getOpcode() == ISD::BITCAST) { in visitVECTOR_SHUFFLE()
25861 if (V->getOpcode() == ISD::BUILD_VECTOR) { in visitVECTOR_SHUFFLE()
25916 if (N0.getOpcode() == ISD::CONCAT_VECTORS && in visitVECTOR_SHUFFLE()
25919 (N1.getOpcode() == ISD::CONCAT_VECTORS && in visitVECTOR_SHUFFLE()
25928 if (N0.getOpcode() == ISD::CONCAT_VECTORS && N1.isUndef() && in visitVECTOR_SHUFFLE()
25943 SDValue NewCat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, in visitVECTOR_SHUFFLE()
25954 TLI.isOperationLegalOrCustom(ISD::INSERT_SUBVECTOR, VT)) { in visitVECTOR_SHUFFLE()
25957 assert(RHS.getOpcode() == ISD::CONCAT_VECTORS && "Can't find subvectors"); in visitVECTOR_SHUFFLE()
25994 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, LHS, in visitVECTOR_SHUFFLE()
26002 if (N1.getOpcode() == ISD::CONCAT_VECTORS) in visitVECTOR_SHUFFLE()
26005 if (N0.getOpcode() == ISD::CONCAT_VECTORS) { in visitVECTOR_SHUFFLE()
26059 if (TLI.isOperationLegalOrCustom(ISD::AND, IntVT)) in visitVECTOR_SHUFFLE()
26061 VT, DAG.getNode(ISD::AND, DL, IntVT, DAG.getBitcast(IntVT, N0), in visitVECTOR_SHUFFLE()
26076 if (N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && in visitVECTOR_SHUFFLE()
26081 if (BC0.getOpcode() == ISD::VECTOR_SHUFFLE && BC0.hasOneUse()) { in visitVECTOR_SHUFFLE()
26255 if (N1.getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
26256 N0.getOpcode() != ISD::VECTOR_SHUFFLE) { in visitVECTOR_SHUFFLE()
26273 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
26274 N1.getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
26288 if (N->getOperand(i).getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
26329 (Op00.getOpcode() == ISD::VECTOR_SHUFFLE || in visitVECTOR_SHUFFLE()
26330 Op10.getOpcode() == ISD::VECTOR_SHUFFLE || in visitVECTOR_SHUFFLE()
26331 Op01.getOpcode() == ISD::VECTOR_SHUFFLE || in visitVECTOR_SHUFFLE()
26332 Op11.getOpcode() == ISD::VECTOR_SHUFFLE)) { in visitVECTOR_SHUFFLE()
26431 if (C && EE.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitSCALAR_TO_VECTOR()
26451 if (Opcode != ISD::EXTRACT_VECTOR_ELT || in visitSCALAR_TO_VECTOR()
26458 SDValue Val = DAG.getNode(ISD::TRUNCATE, SDLoc(Scalar), VecEltVT, Scalar); in visitSCALAR_TO_VECTOR()
26459 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Val); in visitSCALAR_TO_VECTOR()
26488 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), SubVT, LegalShuffle, in visitSCALAR_TO_VECTOR()
26510 if (N0.isUndef() && N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && in visitINSERT_SUBVECTOR()
26520 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), in visitINSERT_SUBVECTOR()
26524 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), in visitINSERT_SUBVECTOR()
26532 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && N1.getOperand(0) == N0 && in visitINSERT_SUBVECTOR()
26538 if (N0.isUndef() && N1.getOpcode() == ISD::SPLAT_VECTOR) in visitINSERT_SUBVECTOR()
26540 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, N1.getOperand(0)); in visitINSERT_SUBVECTOR()
26546 if (N0.isUndef() && N1.getOpcode() == ISD::BITCAST && in visitINSERT_SUBVECTOR()
26547 N1.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR && in visitINSERT_SUBVECTOR()
26560 if (N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST) { in visitINSERT_SUBVECTOR()
26568 SDValue NewINSERT = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), in visitINSERT_SUBVECTOR()
26577 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && in visitINSERT_SUBVECTOR()
26580 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0), in visitINSERT_SUBVECTOR()
26586 if (N0.isUndef() && N1.getOpcode() == ISD::INSERT_SUBVECTOR && in visitINSERT_SUBVECTOR()
26589 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0, in visitINSERT_SUBVECTOR()
26595 if ((N0.isUndef() || N0.getOpcode() == ISD::BITCAST) && in visitINSERT_SUBVECTOR()
26596 N1.getOpcode() == ISD::BITCAST) { in visitINSERT_SUBVECTOR()
26621 if (NewIdx && hasOperation(ISD::INSERT_SUBVECTOR, NewVT)) { in visitINSERT_SUBVECTOR()
26623 Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, NewVT, Res, N1Src, NewIdx); in visitINSERT_SUBVECTOR()
26633 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && N0.hasOneUse() && in visitINSERT_SUBVECTOR()
26638 SDValue NewOp = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, in visitINSERT_SUBVECTOR()
26641 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N0.getNode()), in visitINSERT_SUBVECTOR()
26648 if (N0.getOpcode() == ISD::CONCAT_VECTORS && N0.hasOneUse() && in visitINSERT_SUBVECTOR()
26655 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitINSERT_SUBVECTOR()
26669 if (N0->getOpcode() == ISD::FP16_TO_FP) in visitFP_TO_FP16()
26677 assert((Op == ISD::FP16_TO_FP || Op == ISD::BF16_TO_FP) && in visitFP16_TO_FP()
26683 if (!TLI.shouldKeepZExtForFP16Conv() && N0->getOpcode() == ISD::AND) { in visitFP16_TO_FP()
26702 if (N0->getOpcode() == ISD::BF16_TO_FP) in visitFP_TO_BF16()
26722 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getVectorElementType(), N0, in visitVECREDUCE()
26725 Res = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Res); in visitVECREDUCE()
26731 if (Opcode == ISD::VECREDUCE_AND || Opcode == ISD::VECREDUCE_OR) { in visitVECREDUCE()
26732 unsigned NewOpcode = Opcode == ISD::VECREDUCE_AND in visitVECREDUCE()
26733 ? ISD::VECREDUCE_UMIN : ISD::VECREDUCE_UMAX; in visitVECREDUCE()
26742 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && in visitVECREDUCE()
26746 if ((Opcode == ISD::VECREDUCE_OR && in visitVECREDUCE()
26748 (Opcode == ISD::VECREDUCE_AND && in visitVECREDUCE()
26769 if (N->getOpcode() == ISD::VP_GATHER) in visitVPOp()
26773 if (N->getOpcode() == ISD::VP_SCATTER) in visitVPOp()
26777 if (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD) in visitVPOp()
26781 if (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE) in visitVPOp()
26789 if (auto EVLIdx = ISD::getVPExplicitVectorLengthIdx(N->getOpcode())) in visitVPOp()
26791 if (auto MaskIdx = ISD::getVPMaskIdx(N->getOpcode())) in visitVPOp()
26793 ISD::isConstantSplatVectorAllZeros(N->getOperand(*MaskIdx).getNode()); in visitVPOp()
26798 case ISD::VP_FADD: in visitVPOp()
26800 case ISD::VP_FSUB: in visitVPOp()
26802 case ISD::VP_FMA: in visitVPOp()
26804 case ISD::VP_SELECT: in visitVPOp()
26806 case ISD::VP_MUL: in visitVPOp()
26815 if (ISD::isVPBinaryOp(N->getOpcode())) in visitVPOp()
26827 if (ISD::isVPReduction(N->getOpcode())) in visitVPOp()
26929 assert(N->getOpcode() == ISD::AND && "Unexpected opcode!"); in XformToShuffleWithZero()
26941 if (RHS.getOpcode() != ISD::BUILD_VECTOR) in XformToShuffleWithZero()
27032 bool IsBothSplatVector = N0.getOpcode() == ISD::SPLAT_VECTOR && in scalarizeBinOpOfSplats()
27033 N1.getOpcode() == ISD::SPLAT_VECTOR; in scalarizeBinOpOfSplats()
27042 SDValue X = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src0, IndexC); in scalarizeBinOpOfSplats()
27043 SDValue Y = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src1, IndexC); in scalarizeBinOpOfSplats()
27048 if (N0.getOpcode() == ISD::BUILD_VECTOR && N0.getOpcode() == N1.getOpcode() && in scalarizeBinOpOfSplats()
27076 (N0.getOpcode() == ISD::SPLAT_VECTOR || in SimplifyVCastOp()
27084 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcEltVT, Src0, IndexC); in SimplifyVCastOp()
27132 Shuf0->getOperand(0).getOpcode() != ISD::INSERT_VECTOR_ELT) { in SimplifyVBinOp()
27141 Shuf1->getOperand(0).getOpcode() != ISD::INSERT_VECTOR_ELT) { in SimplifyVBinOp()
27154 if (LHS.getOpcode() == ISD::INSERT_SUBVECTOR && LHS.getOperand(0).isUndef() && in SimplifyVBinOp()
27155 RHS.getOpcode() == ISD::INSERT_SUBVECTOR && RHS.getOperand(0).isUndef() && in SimplifyVBinOp()
27169 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, VecC, NarrowBO, Z); in SimplifyVBinOp()
27175 return Concat.getOpcode() == ISD::CONCAT_VECTORS && in SimplifyVBinOp()
27178 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()); in SimplifyVBinOp()
27200 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in SimplifyVBinOp()
27212 assert(N0.getOpcode() == ISD::SETCC && in SimplifySelect()
27224 if (SCC.getOpcode() == ISD::SELECT_CC) { in SimplifySelect()
27226 SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0), in SimplifySelect()
27252 if (NaN->isNaN() && RHS.getOpcode() == ISD::FSQRT) { in SimplifySelectOps()
27255 ISD::CondCode CC; in SimplifySelectOps()
27259 if (TheSelect->getOpcode() == ISD::SELECT_CC) { in SimplifySelectOps()
27266 if (Cmp.getOpcode() == ISD::SETCC) { in SimplifySelectOps()
27273 Sqrt.getOperand(0) == CmpLHS && (CC == ISD::SETOLT || in SimplifySelectOps()
27274 CC == ISD::SETULT || CC == ISD::SETLT)) { in SimplifySelectOps()
27294 if (LHS.getOpcode() == ISD::LOAD) { in SimplifySelectOps()
27312 LLD->getExtensionType() != ISD::EXTLOAD && in SimplifySelectOps()
27313 RLD->getExtensionType() != ISD::EXTLOAD) || in SimplifySelectOps()
27323 LLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex || in SimplifySelectOps()
27324 RLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex || in SimplifySelectOps()
27352 if (TheSelect->getOpcode() == ISD::SELECT) { in SimplifySelectOps()
27389 Addr = DAG.getNode(ISD::SELECT_CC, SDLoc(TheSelect), in SimplifySelectOps()
27407 if (LLD->getExtensionType() == ISD::NON_EXTLOAD) { in SimplifySelectOps()
27415 LLD->getExtensionType() == ISD::EXTLOAD ? RLD->getExtensionType() in SimplifySelectOps()
27438 ISD::CondCode CC) { in foldSelectCCToShiftAnd()
27451 if (CC == ISD::SETGT && TLI.hasAndNot(N2)) { in foldSelectCCToShiftAnd()
27456 } else if (CC == ISD::SETLT) { in foldSelectCCToShiftAnd()
27472 SDValue Shift = DAG.getNode(ISD::SRL, DL, XType, N0, ShiftAmt); in foldSelectCCToShiftAnd()
27476 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift); in foldSelectCCToShiftAnd()
27480 if (CC == ISD::SETGT) in foldSelectCCToShiftAnd()
27483 return DAG.getNode(ISD::AND, DL, AType, Shift, N2); in foldSelectCCToShiftAnd()
27492 SDValue Shift = DAG.getNode(ISD::SRA, DL, XType, N0, ShiftAmt); in foldSelectCCToShiftAnd()
27496 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift); in foldSelectCCToShiftAnd()
27500 if (CC == ISD::SETGT) in foldSelectCCToShiftAnd()
27503 return DAG.getNode(ISD::AND, DL, AType, Shift, N2); in foldSelectCCToShiftAnd()
27565 bool IsFabs = N->getOpcode() == ISD::FABS; in foldSignChangeInBitcast()
27568 if (IsFree || N0.getOpcode() != ISD::BITCAST || !N0.hasOneUse()) in foldSignChangeInBitcast()
27595 Int = DAG.getNode(IsFabs ? ISD::AND : ISD::XOR, DL, IntVT, Int, in foldSignChangeInBitcast()
27607 ISD::CondCode CC) { in convertSelectOfFPConstantsToLoadOffset()
27620 if (TLI.getOperationAction(ISD::ConstantFP, VT) == TargetLowering::Legal || in convertSelectOfFPConstantsToLoadOffset()
27651 CPIdx = DAG.getNode(ISD::ADD, DL, CPIdx.getValueType(), CPIdx, CstOffset); in convertSelectOfFPConstantsToLoadOffset()
27661 SDValue N2, SDValue N3, ISD::CondCode CC, in SimplifySelectCC()
27696 if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND && in SimplifySelectCC()
27707 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt); in SimplifySelectCC()
27712 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt); in SimplifySelectCC()
27714 return DAG.getNode(ISD::AND, DL, VT, Shr, N3); in SimplifySelectCC()
27726 (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, CmpOpVT))) { in SimplifySelectCC()
27729 CC = ISD::getSetCCInverse(CC, CmpOpVT); in SimplifySelectCC()
27745 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC); in SimplifySelectCC()
27760 ISD::SHL, DL, N2.getValueType(), Temp, in SimplifySelectCC()
27772 if (N1C && N1C->isZero() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { in SimplifySelectCC()
27776 if (CC == ISD::SETNE) in SimplifySelectCC()
27783 if ((Count.getOpcode() == ISD::CTTZ || in SimplifySelectCC()
27784 Count.getOpcode() == ISD::CTTZ_ZERO_UNDEF) && in SimplifySelectCC()
27786 (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ, VT))) in SimplifySelectCC()
27787 return DAG.getNode(ISD::CTTZ, DL, VT, N0); in SimplifySelectCC()
27790 if ((Count.getOpcode() == ISD::CTLZ || in SimplifySelectCC()
27791 Count.getOpcode() == ISD::CTLZ_ZERO_UNDEF) && in SimplifySelectCC()
27793 (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ, VT))) in SimplifySelectCC()
27794 return DAG.getNode(ISD::CTLZ, DL, VT, N0); in SimplifySelectCC()
27803 ((N1C->isAllOnes() && CC == ISD::SETGT) || in SimplifySelectCC()
27804 (N1C->isZero() && CC == ISD::SETLT)) && in SimplifySelectCC()
27807 ISD::SRA, DL, CmpOpVT, N0, in SimplifySelectCC()
27809 return DAG.getNode(ISD::XOR, DL, VT, DAG.getSExtOrTrunc(ASR, DL, VT), in SimplifySelectCC()
27810 DAG.getSExtOrTrunc(CC == ISD::SETLT ? N3 : N2, DL, VT)); in SimplifySelectCC()
27823 ISD::CondCode Cond, const SDLoc &DL, in SimplifySetCC()
27930 case ISD::TRUNCATE: in takeInexpensiveLog2()
27931 case ISD::ZERO_EXTEND: in takeInexpensiveLog2()
27959 if (ISD::matchUnaryPredicate(Op, IsPowerOfTwo)) { in takeInexpensiveLog2()
27963 if (Op.getOpcode() == ISD::SPLAT_VECTOR) in takeInexpensiveLog2()
27990 if (Op.getOpcode() == ISD::SHL) { in takeInexpensiveLog2()
27996 return DAG.getNode(ISD::ADD, DL, VT, LogX, in takeInexpensiveLog2()
28001 if ((Op.getOpcode() == ISD::SELECT || Op.getOpcode() == ISD::VSELECT) && in takeInexpensiveLog2()
28012 if ((Op.getOpcode() == ISD::UMIN || Op.getOpcode() == ISD::UMAX) && in takeInexpensiveLog2()
28039 SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V); in BuildLogBase2()
28041 SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz); in BuildLogBase2()
28086 MulEst = DAG.getNode(ISD::FMUL, DL, VT, N, Est, Flags); in BuildDivEstimate()
28090 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, MulEst, Flags); in BuildDivEstimate()
28093 NewEst = DAG.getNode(ISD::FSUB, DL, VT, in BuildDivEstimate()
28097 NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in BuildDivEstimate()
28100 Est = DAG.getNode(ISD::FADD, DL, VT, MulEst, NewEst, Flags); in BuildDivEstimate()
28105 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, N, Flags); in BuildDivEstimate()
28130 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags); in buildSqrtNROneConst()
28131 HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg, Flags); in buildSqrtNROneConst()
28135 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags); in buildSqrtNROneConst()
28136 NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst, Flags); in buildSqrtNROneConst()
28137 NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst, Flags); in buildSqrtNROneConst()
28138 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in buildSqrtNROneConst()
28143 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg, Flags); in buildSqrtNROneConst()
28168 SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags); in buildSqrtNRTwoConst()
28169 SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags); in buildSqrtNRTwoConst()
28170 SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags); in buildSqrtNRTwoConst()
28178 LHS = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf, Flags); in buildSqrtNRTwoConst()
28181 LHS = DAG.getNode(ISD::FMUL, DL, VT, AE, MinusHalf, Flags); in buildSqrtNRTwoConst()
28184 Est = DAG.getNode(ISD::FMUL, DL, VT, LHS, RHS, Flags); in buildSqrtNRTwoConst()
28233 Test.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT, in buildSqrtEstimateImpl()
28266 Offset = (LSN->getAddressingMode() == ISD::PRE_INC) ? C->getSExtValue() in mayAlias()
28267 : (LSN->getAddressingMode() == ISD::PRE_DEC) in mayAlias()
28425 case ISD::EntryToken: in GatherAllAliases()
28429 case ISD::LOAD: in GatherAllAliases()
28430 case ISD::STORE: { in GatherAllAliases()
28444 case ISD::CopyFromReg: in GatherAllAliases()
28449 case ISD::LIFETIME_START: in GatherAllAliases()
28450 case ISD::LIFETIME_END: { in GatherAllAliases()
28487 if (Chain.getOpcode() == ISD::TokenFactor) { in GatherAllAliases()