Lines Matching refs:Intrinsic
452 AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, in getIntImmCostIntrin()
466 if (IID >= Intrinsic::aarch64_addg && IID <= Intrinsic::aarch64_udiv) in getIntImmCostIntrin()
472 case Intrinsic::sadd_with_overflow: in getIntImmCostIntrin()
473 case Intrinsic::uadd_with_overflow: in getIntImmCostIntrin()
474 case Intrinsic::ssub_with_overflow: in getIntImmCostIntrin()
475 case Intrinsic::usub_with_overflow: in getIntImmCostIntrin()
476 case Intrinsic::smul_with_overflow: in getIntImmCostIntrin()
477 case Intrinsic::umul_with_overflow: in getIntImmCostIntrin()
486 case Intrinsic::experimental_stackmap: in getIntImmCostIntrin()
490 case Intrinsic::experimental_patchpoint_void: in getIntImmCostIntrin()
491 case Intrinsic::experimental_patchpoint: in getIntImmCostIntrin()
495 case Intrinsic::experimental_gc_statepoint: in getIntImmCostIntrin()
554 case Intrinsic::experimental_vector_histogram_add: in getIntrinsicInstrCost()
558 case Intrinsic::umin: in getIntrinsicInstrCost()
559 case Intrinsic::umax: in getIntrinsicInstrCost()
560 case Intrinsic::smin: in getIntrinsicInstrCost()
561 case Intrinsic::smax: { in getIntrinsicInstrCost()
574 case Intrinsic::sadd_sat: in getIntrinsicInstrCost()
575 case Intrinsic::ssub_sat: in getIntrinsicInstrCost()
576 case Intrinsic::uadd_sat: in getIntrinsicInstrCost()
577 case Intrinsic::usub_sat: { in getIntrinsicInstrCost()
590 case Intrinsic::abs: { in getIntrinsicInstrCost()
599 case Intrinsic::bswap: { in getIntrinsicInstrCost()
608 case Intrinsic::experimental_stepvector: { in getIntrinsicInstrCost()
621 case Intrinsic::vector_extract: in getIntrinsicInstrCost()
622 case Intrinsic::vector_insert: { in getIntrinsicInstrCost()
635 bool IsExtract = ICA.getID() == Intrinsic::vector_extract; in getIntrinsicInstrCost()
654 case Intrinsic::bitreverse: { in getIntrinsicInstrCost()
656 {Intrinsic::bitreverse, MVT::i32, 1}, in getIntrinsicInstrCost()
657 {Intrinsic::bitreverse, MVT::i64, 1}, in getIntrinsicInstrCost()
658 {Intrinsic::bitreverse, MVT::v8i8, 1}, in getIntrinsicInstrCost()
659 {Intrinsic::bitreverse, MVT::v16i8, 1}, in getIntrinsicInstrCost()
660 {Intrinsic::bitreverse, MVT::v4i16, 2}, in getIntrinsicInstrCost()
661 {Intrinsic::bitreverse, MVT::v8i16, 2}, in getIntrinsicInstrCost()
662 {Intrinsic::bitreverse, MVT::v2i32, 2}, in getIntrinsicInstrCost()
663 {Intrinsic::bitreverse, MVT::v4i32, 2}, in getIntrinsicInstrCost()
664 {Intrinsic::bitreverse, MVT::v1i64, 2}, in getIntrinsicInstrCost()
665 {Intrinsic::bitreverse, MVT::v2i64, 2}, in getIntrinsicInstrCost()
681 case Intrinsic::ctpop: { in getIntrinsicInstrCost()
710 case Intrinsic::sadd_with_overflow: in getIntrinsicInstrCost()
711 case Intrinsic::uadd_with_overflow: in getIntrinsicInstrCost()
712 case Intrinsic::ssub_with_overflow: in getIntrinsicInstrCost()
713 case Intrinsic::usub_with_overflow: in getIntrinsicInstrCost()
714 case Intrinsic::smul_with_overflow: in getIntrinsicInstrCost()
715 case Intrinsic::umul_with_overflow: { in getIntrinsicInstrCost()
717 {Intrinsic::sadd_with_overflow, MVT::i8, 3}, in getIntrinsicInstrCost()
718 {Intrinsic::uadd_with_overflow, MVT::i8, 3}, in getIntrinsicInstrCost()
719 {Intrinsic::sadd_with_overflow, MVT::i16, 3}, in getIntrinsicInstrCost()
720 {Intrinsic::uadd_with_overflow, MVT::i16, 3}, in getIntrinsicInstrCost()
721 {Intrinsic::sadd_with_overflow, MVT::i32, 1}, in getIntrinsicInstrCost()
722 {Intrinsic::uadd_with_overflow, MVT::i32, 1}, in getIntrinsicInstrCost()
723 {Intrinsic::sadd_with_overflow, MVT::i64, 1}, in getIntrinsicInstrCost()
724 {Intrinsic::uadd_with_overflow, MVT::i64, 1}, in getIntrinsicInstrCost()
725 {Intrinsic::ssub_with_overflow, MVT::i8, 3}, in getIntrinsicInstrCost()
726 {Intrinsic::usub_with_overflow, MVT::i8, 3}, in getIntrinsicInstrCost()
727 {Intrinsic::ssub_with_overflow, MVT::i16, 3}, in getIntrinsicInstrCost()
728 {Intrinsic::usub_with_overflow, MVT::i16, 3}, in getIntrinsicInstrCost()
729 {Intrinsic::ssub_with_overflow, MVT::i32, 1}, in getIntrinsicInstrCost()
730 {Intrinsic::usub_with_overflow, MVT::i32, 1}, in getIntrinsicInstrCost()
731 {Intrinsic::ssub_with_overflow, MVT::i64, 1}, in getIntrinsicInstrCost()
732 {Intrinsic::usub_with_overflow, MVT::i64, 1}, in getIntrinsicInstrCost()
733 {Intrinsic::smul_with_overflow, MVT::i8, 5}, in getIntrinsicInstrCost()
734 {Intrinsic::umul_with_overflow, MVT::i8, 4}, in getIntrinsicInstrCost()
735 {Intrinsic::smul_with_overflow, MVT::i16, 5}, in getIntrinsicInstrCost()
736 {Intrinsic::umul_with_overflow, MVT::i16, 4}, in getIntrinsicInstrCost()
737 {Intrinsic::smul_with_overflow, MVT::i32, 2}, // eg umull;tst in getIntrinsicInstrCost()
738 {Intrinsic::umul_with_overflow, MVT::i32, 2}, // eg umull;cmp sxtw in getIntrinsicInstrCost()
739 {Intrinsic::smul_with_overflow, MVT::i64, 3}, // eg mul;smulh;cmp in getIntrinsicInstrCost()
740 {Intrinsic::umul_with_overflow, MVT::i64, 3}, // eg mul;umulh;cmp asr in getIntrinsicInstrCost()
749 case Intrinsic::fptosi_sat: in getIntrinsicInstrCost()
750 case Intrinsic::fptoui_sat: { in getIntrinsicInstrCost()
753 bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat; in getIntrinsicInstrCost()
782 IntrinsicCostAttributes Attrs1(IsSigned ? Intrinsic::smin : Intrinsic::umin, in getIntrinsicInstrCost()
785 IntrinsicCostAttributes Attrs2(IsSigned ? Intrinsic::smax : Intrinsic::umax, in getIntrinsicInstrCost()
792 case Intrinsic::fshl: in getIntrinsicInstrCost()
793 case Intrinsic::fshr: { in getIntrinsicInstrCost()
806 {Intrinsic::fshl, MVT::v4i32, 3}, // ushr + shl + orr in getIntrinsicInstrCost()
807 {Intrinsic::fshl, MVT::v2i64, 3}, {Intrinsic::fshl, MVT::v16i8, 4}, in getIntrinsicInstrCost()
808 {Intrinsic::fshl, MVT::v8i16, 4}, {Intrinsic::fshl, MVT::v2i32, 3}, in getIntrinsicInstrCost()
809 {Intrinsic::fshl, MVT::v8i8, 4}, {Intrinsic::fshl, MVT::v4i16, 4}}; in getIntrinsicInstrCost()
813 CostTableLookup(FshlTbl, Intrinsic::fshl, LegalisationCost.second); in getIntrinsicInstrCost()
838 case Intrinsic::get_active_lane_mask: { in getIntrinsicInstrCost()
883 Intrinsic::aarch64_sve_convert_to_svbool || in processPhiNode()
920 case Intrinsic::aarch64_sve_and_z: in tryCombineFromSVBoolBinOp()
921 case Intrinsic::aarch64_sve_bic_z: in tryCombineFromSVBoolBinOp()
922 case Intrinsic::aarch64_sve_eor_z: in tryCombineFromSVBoolBinOp()
923 case Intrinsic::aarch64_sve_nand_z: in tryCombineFromSVBoolBinOp()
924 case Intrinsic::aarch64_sve_nor_z: in tryCombineFromSVBoolBinOp()
925 case Intrinsic::aarch64_sve_orn_z: in tryCombineFromSVBoolBinOp()
926 case Intrinsic::aarch64_sve_orr_z: in tryCombineFromSVBoolBinOp()
938 PredIntr->getIntrinsicID() != Intrinsic::aarch64_sve_convert_to_svbool) in tryCombineFromSVBoolBinOp()
948 Intrinsic::aarch64_sve_convert_from_svbool, {PredOpTy}, {BinOpOp1}); in tryCombineFromSVBoolBinOp()
954 Intrinsic::aarch64_sve_convert_from_svbool, {PredOpTy}, {BinOpOp2})); in tryCombineFromSVBoolBinOp()
997 Intrinsic::aarch64_sve_convert_to_svbool || in instCombineConvertFromSVBool()
999 Intrinsic::aarch64_sve_convert_from_svbool)) in instCombineConvertFromSVBool()
1017 if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_convert_from_svbool>( in isAllActivePredicate()
1018 m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>( in isAllActivePredicate()
1026 return match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>( in isAllActivePredicate()
1085 if (Pg->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue) in instCombineSVEDup()
1119 if (!Pg || Pg->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue) in instCombineSVECmpNE()
1136 DupQLane->getIntrinsicID() != Intrinsic::aarch64_sve_dupq_lane) in instCombineSVECmpNE()
1144 if (!VecIns || VecIns->getIntrinsicID() != Intrinsic::vector_insert) in instCombineSVECmpNE()
1200 auto *PTrue = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue, in instCombineSVECmpNE()
1203 Intrinsic::aarch64_sve_convert_to_svbool, {PredType}, {PTrue}); in instCombineSVECmpNE()
1205 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, in instCombineSVECmpNE()
1217 bool IsAfter = IntrinsicID == Intrinsic::aarch64_sve_lasta; in instCombineSVELast()
1254 if (IntrPG->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue) in instCombineSVELast()
1335 auto *PTrue = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue, in instCombineRDFFR()
1338 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_rdffr_z, {}, {PTrue}); in instCombineRDFFR()
1370 (II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_first || in instCombineSVEPTest()
1371 II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_last)) { in instCombineSVEPTest()
1376 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptest_any, Tys, Ops); in instCombineSVEPTest()
1388 Intrinsic::ID OpIID = Op->getIntrinsicID(); in instCombineSVEPTest()
1390 if (Pg->getIntrinsicID() == Intrinsic::aarch64_sve_convert_to_svbool && in instCombineSVEPTest()
1391 OpIID == Intrinsic::aarch64_sve_convert_to_svbool && in instCombineSVEPTest()
1405 if ((Pg == Op) && (II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_any) && in instCombineSVEPTest()
1406 ((OpIID == Intrinsic::aarch64_sve_brka_z) || in instCombineSVEPTest()
1407 (OpIID == Intrinsic::aarch64_sve_brkb_z) || in instCombineSVEPTest()
1408 (OpIID == Intrinsic::aarch64_sve_brkpa_z) || in instCombineSVEPTest()
1409 (OpIID == Intrinsic::aarch64_sve_brkpb_z) || in instCombineSVEPTest()
1410 (OpIID == Intrinsic::aarch64_sve_rdffr_z) || in instCombineSVEPTest()
1411 (OpIID == Intrinsic::aarch64_sve_and_z) || in instCombineSVEPTest()
1412 (OpIID == Intrinsic::aarch64_sve_bic_z) || in instCombineSVEPTest()
1413 (OpIID == Intrinsic::aarch64_sve_eor_z) || in instCombineSVEPTest()
1414 (OpIID == Intrinsic::aarch64_sve_nand_z) || in instCombineSVEPTest()
1415 (OpIID == Intrinsic::aarch64_sve_nor_z) || in instCombineSVEPTest()
1416 (OpIID == Intrinsic::aarch64_sve_orn_z) || in instCombineSVEPTest()
1417 (OpIID == Intrinsic::aarch64_sve_orr_z))) { in instCombineSVEPTest()
1430 template <Intrinsic::ID MulOpc, typename Intrinsic::ID FuseOpc>
1515 static Instruction::BinaryOps intrinsicIDToBinOpCode(unsigned Intrinsic) { in intrinsicIDToBinOpCode() argument
1516 switch (Intrinsic) { in intrinsicIDToBinOpCode()
1517 case Intrinsic::aarch64_sve_fmul_u: in intrinsicIDToBinOpCode()
1519 case Intrinsic::aarch64_sve_fadd_u: in intrinsicIDToBinOpCode()
1521 case Intrinsic::aarch64_sve_fsub_u: in intrinsicIDToBinOpCode()
1537 !match(OpPredicate, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>( in instCombineSVEVectorBinOp()
1550 Intrinsic::ID IID) { in instCombineSVEAllActive()
1552 if (!match(OpPredicate, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>( in instCombineSVEAllActive()
1557 auto *NewDecl = Intrinsic::getDeclaration(Mod, IID, {II.getType()}); in instCombineSVEAllActive()
1567 Intrinsic::ID IID) { in instCombineSVEAllOrNoActive()
1579 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_add_u)) in instCombineSVEVectorAdd()
1581 if (auto MLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul, in instCombineSVEVectorAdd()
1582 Intrinsic::aarch64_sve_mla>( in instCombineSVEVectorAdd()
1585 if (auto MAD = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul, in instCombineSVEVectorAdd()
1586 Intrinsic::aarch64_sve_mad>( in instCombineSVEVectorAdd()
1595 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fadd_u)) in instCombineSVEVectorFAdd()
1598 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, in instCombineSVEVectorFAdd()
1599 Intrinsic::aarch64_sve_fmla>(IC, II, in instCombineSVEVectorFAdd()
1603 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, in instCombineSVEVectorFAdd()
1604 Intrinsic::aarch64_sve_fmad>(IC, II, in instCombineSVEVectorFAdd()
1608 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u, in instCombineSVEVectorFAdd()
1609 Intrinsic::aarch64_sve_fmla>(IC, II, in instCombineSVEVectorFAdd()
1618 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, in instCombineSVEVectorFAddU()
1619 Intrinsic::aarch64_sve_fmla>(IC, II, in instCombineSVEVectorFAddU()
1623 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, in instCombineSVEVectorFAddU()
1624 Intrinsic::aarch64_sve_fmad>(IC, II, in instCombineSVEVectorFAddU()
1628 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u, in instCombineSVEVectorFAddU()
1629 Intrinsic::aarch64_sve_fmla_u>( in instCombineSVEVectorFAddU()
1638 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fsub_u)) in instCombineSVEVectorFSub()
1641 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, in instCombineSVEVectorFSub()
1642 Intrinsic::aarch64_sve_fmls>(IC, II, in instCombineSVEVectorFSub()
1646 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, in instCombineSVEVectorFSub()
1647 Intrinsic::aarch64_sve_fnmsb>( in instCombineSVEVectorFSub()
1651 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u, in instCombineSVEVectorFSub()
1652 Intrinsic::aarch64_sve_fmls>(IC, II, in instCombineSVEVectorFSub()
1661 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, in instCombineSVEVectorFSubU()
1662 Intrinsic::aarch64_sve_fmls>(IC, II, in instCombineSVEVectorFSubU()
1666 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, in instCombineSVEVectorFSubU()
1667 Intrinsic::aarch64_sve_fnmsb>( in instCombineSVEVectorFSubU()
1671 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u, in instCombineSVEVectorFSubU()
1672 Intrinsic::aarch64_sve_fmls_u>( in instCombineSVEVectorFSubU()
1681 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sub_u)) in instCombineSVEVectorSub()
1683 if (auto MLS = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul, in instCombineSVEVectorSub()
1684 Intrinsic::aarch64_sve_mls>( in instCombineSVEVectorSub()
1692 Intrinsic::ID IID) { in instCombineSVEVectorMul()
1709 if (!IntrI || IntrI->getIntrinsicID() != Intrinsic::aarch64_sve_dup) in instCombineSVEVectorMul()
1739 bool IsSigned = II.getIntrinsicID() == Intrinsic::aarch64_sve_sunpkhi || in instCombineSVEUnpack()
1740 II.getIntrinsicID() == Intrinsic::aarch64_sve_sunpklo; in instCombineSVEUnpack()
1782 constexpr Intrinsic::ID FromSVB = Intrinsic::aarch64_sve_convert_from_svbool; in instCombineSVEUzp1()
1783 constexpr Intrinsic::ID ToSVB = Intrinsic::aarch64_sve_convert_to_svbool; in instCombineSVEUzp1()
1814 m_Intrinsic<Intrinsic::aarch64_sve_uzp1>(m_Value(A), m_Value(B))) && in instCombineSVEZip()
1815 match(II.getArgOperand(1), m_Intrinsic<Intrinsic::aarch64_sve_uzp2>( in instCombineSVEZip()
1818 II, (II.getIntrinsicID() == Intrinsic::aarch64_sve_zip1 ? A : B)); in instCombineSVEZip()
1839 if (match(Index, m_Intrinsic<Intrinsic::aarch64_sve_index>( in instCombineLD1GatherIndex()
1869 if (match(Index, m_Intrinsic<Intrinsic::aarch64_sve_index>( in instCombineST1ScatterIndex()
1903 Intrinsic::aarch64_sve_asrd, {II.getType()}, {Pred, Vec, DivisorLog2}); in instCombineSVESDIV()
1910 Intrinsic::aarch64_sve_asrd, {II.getType()}, {Pred, Vec, DivisorLog2}); in instCombineSVESDIV()
1912 Intrinsic::aarch64_sve_neg, {ASRD->getType()}, {ASRD, Pred, ASRD}); in instCombineSVESDIV()
1952 m_Intrinsic<Intrinsic::vector_insert>( in instCombineSVEDupqLane()
2028 if (!match(Vec, m_Intrinsic<Intrinsic::aarch64_sve_sqabs>( in instCombineSVESrshl()
2030 !match(Vec, m_Intrinsic<Intrinsic::aarch64_sve_abs>( in instCombineSVESrshl()
2048 auto LSL = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_lsl, in instCombineSVESrshl()
2057 Intrinsic::ID IID = II.getIntrinsicID(); in instCombineIntrinsic()
2062 case Intrinsic::aarch64_sve_st1_scatter: in instCombineIntrinsic()
2063 case Intrinsic::aarch64_sve_st1_scatter_scalar_offset: in instCombineIntrinsic()
2064 case Intrinsic::aarch64_sve_st1_scatter_sxtw: in instCombineIntrinsic()
2065 case Intrinsic::aarch64_sve_st1_scatter_sxtw_index: in instCombineIntrinsic()
2066 case Intrinsic::aarch64_sve_st1_scatter_uxtw: in instCombineIntrinsic()
2067 case Intrinsic::aarch64_sve_st1_scatter_uxtw_index: in instCombineIntrinsic()
2068 case Intrinsic::aarch64_sve_st1dq: in instCombineIntrinsic()
2069 case Intrinsic::aarch64_sve_st1q_scatter_index: in instCombineIntrinsic()
2070 case Intrinsic::aarch64_sve_st1q_scatter_scalar_offset: in instCombineIntrinsic()
2071 case Intrinsic::aarch64_sve_st1q_scatter_vector_offset: in instCombineIntrinsic()
2072 case Intrinsic::aarch64_sve_st1wq: in instCombineIntrinsic()
2073 case Intrinsic::aarch64_sve_stnt1: in instCombineIntrinsic()
2074 case Intrinsic::aarch64_sve_stnt1_scatter: in instCombineIntrinsic()
2075 case Intrinsic::aarch64_sve_stnt1_scatter_index: in instCombineIntrinsic()
2076 case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset: in instCombineIntrinsic()
2077 case Intrinsic::aarch64_sve_stnt1_scatter_uxtw: in instCombineIntrinsic()
2079 case Intrinsic::aarch64_sve_st2: in instCombineIntrinsic()
2080 case Intrinsic::aarch64_sve_st2q: in instCombineIntrinsic()
2082 case Intrinsic::aarch64_sve_st3: in instCombineIntrinsic()
2083 case Intrinsic::aarch64_sve_st3q: in instCombineIntrinsic()
2085 case Intrinsic::aarch64_sve_st4: in instCombineIntrinsic()
2086 case Intrinsic::aarch64_sve_st4q: in instCombineIntrinsic()
2088 case Intrinsic::aarch64_sve_ld1_gather: in instCombineIntrinsic()
2089 case Intrinsic::aarch64_sve_ld1_gather_scalar_offset: in instCombineIntrinsic()
2090 case Intrinsic::aarch64_sve_ld1_gather_sxtw: in instCombineIntrinsic()
2091 case Intrinsic::aarch64_sve_ld1_gather_sxtw_index: in instCombineIntrinsic()
2092 case Intrinsic::aarch64_sve_ld1_gather_uxtw: in instCombineIntrinsic()
2093 case Intrinsic::aarch64_sve_ld1_gather_uxtw_index: in instCombineIntrinsic()
2094 case Intrinsic::aarch64_sve_ld1q_gather_index: in instCombineIntrinsic()
2095 case Intrinsic::aarch64_sve_ld1q_gather_scalar_offset: in instCombineIntrinsic()
2096 case Intrinsic::aarch64_sve_ld1q_gather_vector_offset: in instCombineIntrinsic()
2097 case Intrinsic::aarch64_sve_ld1ro: in instCombineIntrinsic()
2098 case Intrinsic::aarch64_sve_ld1rq: in instCombineIntrinsic()
2099 case Intrinsic::aarch64_sve_ld1udq: in instCombineIntrinsic()
2100 case Intrinsic::aarch64_sve_ld1uwq: in instCombineIntrinsic()
2101 case Intrinsic::aarch64_sve_ld2_sret: in instCombineIntrinsic()
2102 case Intrinsic::aarch64_sve_ld2q_sret: in instCombineIntrinsic()
2103 case Intrinsic::aarch64_sve_ld3_sret: in instCombineIntrinsic()
2104 case Intrinsic::aarch64_sve_ld3q_sret: in instCombineIntrinsic()
2105 case Intrinsic::aarch64_sve_ld4_sret: in instCombineIntrinsic()
2106 case Intrinsic::aarch64_sve_ld4q_sret: in instCombineIntrinsic()
2107 case Intrinsic::aarch64_sve_ldff1: in instCombineIntrinsic()
2108 case Intrinsic::aarch64_sve_ldff1_gather: in instCombineIntrinsic()
2109 case Intrinsic::aarch64_sve_ldff1_gather_index: in instCombineIntrinsic()
2110 case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset: in instCombineIntrinsic()
2111 case Intrinsic::aarch64_sve_ldff1_gather_sxtw: in instCombineIntrinsic()
2112 case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index: in instCombineIntrinsic()
2113 case Intrinsic::aarch64_sve_ldff1_gather_uxtw: in instCombineIntrinsic()
2114 case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index: in instCombineIntrinsic()
2115 case Intrinsic::aarch64_sve_ldnf1: in instCombineIntrinsic()
2116 case Intrinsic::aarch64_sve_ldnt1: in instCombineIntrinsic()
2117 case Intrinsic::aarch64_sve_ldnt1_gather: in instCombineIntrinsic()
2118 case Intrinsic::aarch64_sve_ldnt1_gather_index: in instCombineIntrinsic()
2119 case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset: in instCombineIntrinsic()
2120 case Intrinsic::aarch64_sve_ldnt1_gather_uxtw: in instCombineIntrinsic()
2122 case Intrinsic::aarch64_neon_fmaxnm: in instCombineIntrinsic()
2123 case Intrinsic::aarch64_neon_fminnm: in instCombineIntrinsic()
2125 case Intrinsic::aarch64_sve_convert_from_svbool: in instCombineIntrinsic()
2127 case Intrinsic::aarch64_sve_dup: in instCombineIntrinsic()
2129 case Intrinsic::aarch64_sve_dup_x: in instCombineIntrinsic()
2131 case Intrinsic::aarch64_sve_cmpne: in instCombineIntrinsic()
2132 case Intrinsic::aarch64_sve_cmpne_wide: in instCombineIntrinsic()
2134 case Intrinsic::aarch64_sve_rdffr: in instCombineIntrinsic()
2136 case Intrinsic::aarch64_sve_lasta: in instCombineIntrinsic()
2137 case Intrinsic::aarch64_sve_lastb: in instCombineIntrinsic()
2139 case Intrinsic::aarch64_sve_clasta_n: in instCombineIntrinsic()
2140 case Intrinsic::aarch64_sve_clastb_n: in instCombineIntrinsic()
2142 case Intrinsic::aarch64_sve_cntd: in instCombineIntrinsic()
2144 case Intrinsic::aarch64_sve_cntw: in instCombineIntrinsic()
2146 case Intrinsic::aarch64_sve_cnth: in instCombineIntrinsic()
2148 case Intrinsic::aarch64_sve_cntb: in instCombineIntrinsic()
2150 case Intrinsic::aarch64_sve_ptest_any: in instCombineIntrinsic()
2151 case Intrinsic::aarch64_sve_ptest_first: in instCombineIntrinsic()
2152 case Intrinsic::aarch64_sve_ptest_last: in instCombineIntrinsic()
2154 case Intrinsic::aarch64_sve_fabd: in instCombineIntrinsic()
2155 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fabd_u); in instCombineIntrinsic()
2156 case Intrinsic::aarch64_sve_fadd: in instCombineIntrinsic()
2158 case Intrinsic::aarch64_sve_fadd_u: in instCombineIntrinsic()
2160 case Intrinsic::aarch64_sve_fdiv: in instCombineIntrinsic()
2161 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fdiv_u); in instCombineIntrinsic()
2162 case Intrinsic::aarch64_sve_fmax: in instCombineIntrinsic()
2163 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmax_u); in instCombineIntrinsic()
2164 case Intrinsic::aarch64_sve_fmaxnm: in instCombineIntrinsic()
2165 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmaxnm_u); in instCombineIntrinsic()
2166 case Intrinsic::aarch64_sve_fmin: in instCombineIntrinsic()
2167 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmin_u); in instCombineIntrinsic()
2168 case Intrinsic::aarch64_sve_fminnm: in instCombineIntrinsic()
2169 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fminnm_u); in instCombineIntrinsic()
2170 case Intrinsic::aarch64_sve_fmla: in instCombineIntrinsic()
2171 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmla_u); in instCombineIntrinsic()
2172 case Intrinsic::aarch64_sve_fmls: in instCombineIntrinsic()
2173 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmls_u); in instCombineIntrinsic()
2174 case Intrinsic::aarch64_sve_fmul: in instCombineIntrinsic()
2176 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmul_u)) in instCombineIntrinsic()
2178 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_fmul_u); in instCombineIntrinsic()
2179 case Intrinsic::aarch64_sve_fmul_u: in instCombineIntrinsic()
2180 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_fmul_u); in instCombineIntrinsic()
2181 case Intrinsic::aarch64_sve_fmulx: in instCombineIntrinsic()
2182 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmulx_u); in instCombineIntrinsic()
2183 case Intrinsic::aarch64_sve_fnmla: in instCombineIntrinsic()
2184 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fnmla_u); in instCombineIntrinsic()
2185 case Intrinsic::aarch64_sve_fnmls: in instCombineIntrinsic()
2186 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fnmls_u); in instCombineIntrinsic()
2187 case Intrinsic::aarch64_sve_fsub: in instCombineIntrinsic()
2189 case Intrinsic::aarch64_sve_fsub_u: in instCombineIntrinsic()
2191 case Intrinsic::aarch64_sve_add: in instCombineIntrinsic()
2193 case Intrinsic::aarch64_sve_add_u: in instCombineIntrinsic()
2194 return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u, in instCombineIntrinsic()
2195 Intrinsic::aarch64_sve_mla_u>( in instCombineIntrinsic()
2197 case Intrinsic::aarch64_sve_mla: in instCombineIntrinsic()
2198 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mla_u); in instCombineIntrinsic()
2199 case Intrinsic::aarch64_sve_mls: in instCombineIntrinsic()
2200 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mls_u); in instCombineIntrinsic()
2201 case Intrinsic::aarch64_sve_mul: in instCombineIntrinsic()
2203 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mul_u)) in instCombineIntrinsic()
2205 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_mul_u); in instCombineIntrinsic()
2206 case Intrinsic::aarch64_sve_mul_u: in instCombineIntrinsic()
2207 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_mul_u); in instCombineIntrinsic()
2208 case Intrinsic::aarch64_sve_sabd: in instCombineIntrinsic()
2209 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sabd_u); in instCombineIntrinsic()
2210 case Intrinsic::aarch64_sve_smax: in instCombineIntrinsic()
2211 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smax_u); in instCombineIntrinsic()
2212 case Intrinsic::aarch64_sve_smin: in instCombineIntrinsic()
2213 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smin_u); in instCombineIntrinsic()
2214 case Intrinsic::aarch64_sve_smulh: in instCombineIntrinsic()
2215 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smulh_u); in instCombineIntrinsic()
2216 case Intrinsic::aarch64_sve_sub: in instCombineIntrinsic()
2218 case Intrinsic::aarch64_sve_sub_u: in instCombineIntrinsic()
2219 return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u, in instCombineIntrinsic()
2220 Intrinsic::aarch64_sve_mls_u>( in instCombineIntrinsic()
2222 case Intrinsic::aarch64_sve_uabd: in instCombineIntrinsic()
2223 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_uabd_u); in instCombineIntrinsic()
2224 case Intrinsic::aarch64_sve_umax: in instCombineIntrinsic()
2225 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umax_u); in instCombineIntrinsic()
2226 case Intrinsic::aarch64_sve_umin: in instCombineIntrinsic()
2227 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umin_u); in instCombineIntrinsic()
2228 case Intrinsic::aarch64_sve_umulh: in instCombineIntrinsic()
2229 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umulh_u); in instCombineIntrinsic()
2230 case Intrinsic::aarch64_sve_asr: in instCombineIntrinsic()
2231 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_asr_u); in instCombineIntrinsic()
2232 case Intrinsic::aarch64_sve_lsl: in instCombineIntrinsic()
2233 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_lsl_u); in instCombineIntrinsic()
2234 case Intrinsic::aarch64_sve_lsr: in instCombineIntrinsic()
2235 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_lsr_u); in instCombineIntrinsic()
2236 case Intrinsic::aarch64_sve_and: in instCombineIntrinsic()
2237 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_and_u); in instCombineIntrinsic()
2238 case Intrinsic::aarch64_sve_bic: in instCombineIntrinsic()
2239 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_bic_u); in instCombineIntrinsic()
2240 case Intrinsic::aarch64_sve_eor: in instCombineIntrinsic()
2241 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_eor_u); in instCombineIntrinsic()
2242 case Intrinsic::aarch64_sve_orr: in instCombineIntrinsic()
2243 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_orr_u); in instCombineIntrinsic()
2244 case Intrinsic::aarch64_sve_sqsub: in instCombineIntrinsic()
2245 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sqsub_u); in instCombineIntrinsic()
2246 case Intrinsic::aarch64_sve_uqsub: in instCombineIntrinsic()
2247 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_uqsub_u); in instCombineIntrinsic()
2248 case Intrinsic::aarch64_sve_tbl: in instCombineIntrinsic()
2250 case Intrinsic::aarch64_sve_uunpkhi: in instCombineIntrinsic()
2251 case Intrinsic::aarch64_sve_uunpklo: in instCombineIntrinsic()
2252 case Intrinsic::aarch64_sve_sunpkhi: in instCombineIntrinsic()
2253 case Intrinsic::aarch64_sve_sunpklo: in instCombineIntrinsic()
2255 case Intrinsic::aarch64_sve_uzp1: in instCombineIntrinsic()
2257 case Intrinsic::aarch64_sve_zip1: in instCombineIntrinsic()
2258 case Intrinsic::aarch64_sve_zip2: in instCombineIntrinsic()
2260 case Intrinsic::aarch64_sve_ld1_gather_index: in instCombineIntrinsic()
2262 case Intrinsic::aarch64_sve_st1_scatter_index: in instCombineIntrinsic()
2264 case Intrinsic::aarch64_sve_ld1: in instCombineIntrinsic()
2266 case Intrinsic::aarch64_sve_st1: in instCombineIntrinsic()
2268 case Intrinsic::aarch64_sve_sdiv: in instCombineIntrinsic()
2270 case Intrinsic::aarch64_sve_sel: in instCombineIntrinsic()
2272 case Intrinsic::aarch64_sve_srshl: in instCombineIntrinsic()
2274 case Intrinsic::aarch64_sve_dupq_lane: in instCombineIntrinsic()
2289 case Intrinsic::aarch64_neon_fcvtxn: in simplifyDemandedVectorEltsIntrinsic()
2290 case Intrinsic::aarch64_neon_rshrn: in simplifyDemandedVectorEltsIntrinsic()
2291 case Intrinsic::aarch64_neon_sqrshrn: in simplifyDemandedVectorEltsIntrinsic()
2292 case Intrinsic::aarch64_neon_sqrshrun: in simplifyDemandedVectorEltsIntrinsic()
2293 case Intrinsic::aarch64_neon_sqshrn: in simplifyDemandedVectorEltsIntrinsic()
2294 case Intrinsic::aarch64_neon_sqshrun: in simplifyDemandedVectorEltsIntrinsic()
2295 case Intrinsic::aarch64_neon_sqxtn: in simplifyDemandedVectorEltsIntrinsic()
2296 case Intrinsic::aarch64_neon_sqxtun: in simplifyDemandedVectorEltsIntrinsic()
2297 case Intrinsic::aarch64_neon_uqrshrn: in simplifyDemandedVectorEltsIntrinsic()
2298 case Intrinsic::aarch64_neon_uqshrn: in simplifyDemandedVectorEltsIntrinsic()
2299 case Intrinsic::aarch64_neon_uqxtn: in simplifyDemandedVectorEltsIntrinsic()
3680 case Intrinsic::aarch64_neon_st2: in getOrCreateResultFromMemIntrinsic()
3681 case Intrinsic::aarch64_neon_st3: in getOrCreateResultFromMemIntrinsic()
3682 case Intrinsic::aarch64_neon_st4: { in getOrCreateResultFromMemIntrinsic()
3702 case Intrinsic::aarch64_neon_ld2: in getOrCreateResultFromMemIntrinsic()
3703 case Intrinsic::aarch64_neon_ld3: in getOrCreateResultFromMemIntrinsic()
3704 case Intrinsic::aarch64_neon_ld4: in getOrCreateResultFromMemIntrinsic()
3716 case Intrinsic::aarch64_neon_ld2: in getTgtMemIntrinsic()
3717 case Intrinsic::aarch64_neon_ld3: in getTgtMemIntrinsic()
3718 case Intrinsic::aarch64_neon_ld4: in getTgtMemIntrinsic()
3723 case Intrinsic::aarch64_neon_st2: in getTgtMemIntrinsic()
3724 case Intrinsic::aarch64_neon_st3: in getTgtMemIntrinsic()
3725 case Intrinsic::aarch64_neon_st4: in getTgtMemIntrinsic()
3735 case Intrinsic::aarch64_neon_ld2: in getTgtMemIntrinsic()
3736 case Intrinsic::aarch64_neon_st2: in getTgtMemIntrinsic()
3739 case Intrinsic::aarch64_neon_ld3: in getTgtMemIntrinsic()
3740 case Intrinsic::aarch64_neon_st3: in getTgtMemIntrinsic()
3743 case Intrinsic::aarch64_neon_ld4: in getTgtMemIntrinsic()
3744 case Intrinsic::aarch64_neon_st4: in getTgtMemIntrinsic()
3814 AArch64TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, in getMinMaxReductionCost()