Lines Matching refs:Intrinsic
56 static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, in upgradePTESTIntrinsic()
66 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradePTESTIntrinsic()
72 static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, in upgradeX86IntrinsicsWith8BitMask()
82 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86IntrinsicsWith8BitMask()
88 static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, in upgradeX86MaskedFPCompare()
95 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86MaskedFPCompare()
99 static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, in upgradeX86BF16Intrinsic()
105 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86BF16Intrinsic()
109 static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, in upgradeX86BF16DPIntrinsic()
115 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86BF16DPIntrinsic()
503 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeX86IntrinsicFunction()
504 Intrinsic::x86_rdtscp); in upgradeX86IntrinsicFunction()
508 Intrinsic::ID ID; in upgradeX86IntrinsicFunction()
512 ID = StringSwitch<Intrinsic::ID>(Name) in upgradeX86IntrinsicFunction()
513 .Case("c", Intrinsic::x86_sse41_ptestc) in upgradeX86IntrinsicFunction()
514 .Case("z", Intrinsic::x86_sse41_ptestz) in upgradeX86IntrinsicFunction()
515 .Case("nzc", Intrinsic::x86_sse41_ptestnzc) in upgradeX86IntrinsicFunction()
516 .Default(Intrinsic::not_intrinsic); in upgradeX86IntrinsicFunction()
517 if (ID != Intrinsic::not_intrinsic) in upgradeX86IntrinsicFunction()
527 ID = StringSwitch<Intrinsic::ID>(Name) in upgradeX86IntrinsicFunction()
528 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps) in upgradeX86IntrinsicFunction()
529 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd) in upgradeX86IntrinsicFunction()
530 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps) in upgradeX86IntrinsicFunction()
531 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw) in upgradeX86IntrinsicFunction()
532 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256) in upgradeX86IntrinsicFunction()
533 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw) in upgradeX86IntrinsicFunction()
534 .Default(Intrinsic::not_intrinsic); in upgradeX86IntrinsicFunction()
535 if (ID != Intrinsic::not_intrinsic) in upgradeX86IntrinsicFunction()
540 ID = StringSwitch<Intrinsic::ID>(Name) in upgradeX86IntrinsicFunction()
541 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128) in upgradeX86IntrinsicFunction()
542 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256) in upgradeX86IntrinsicFunction()
543 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512) in upgradeX86IntrinsicFunction()
544 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128) in upgradeX86IntrinsicFunction()
545 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256) in upgradeX86IntrinsicFunction()
546 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512) in upgradeX86IntrinsicFunction()
547 .Default(Intrinsic::not_intrinsic); in upgradeX86IntrinsicFunction()
548 if (ID != Intrinsic::not_intrinsic) in upgradeX86IntrinsicFunction()
555 ID = StringSwitch<Intrinsic::ID>(Name) in upgradeX86IntrinsicFunction()
557 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128) in upgradeX86IntrinsicFunction()
559 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256) in upgradeX86IntrinsicFunction()
561 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512) in upgradeX86IntrinsicFunction()
563 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128) in upgradeX86IntrinsicFunction()
565 Intrinsic::x86_avx512bf16_cvtneps2bf16_256) in upgradeX86IntrinsicFunction()
567 Intrinsic::x86_avx512bf16_cvtneps2bf16_512) in upgradeX86IntrinsicFunction()
568 .Default(Intrinsic::not_intrinsic); in upgradeX86IntrinsicFunction()
569 if (ID != Intrinsic::not_intrinsic) in upgradeX86IntrinsicFunction()
573 ID = StringSwitch<Intrinsic::ID>(Name) in upgradeX86IntrinsicFunction()
574 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128) in upgradeX86IntrinsicFunction()
575 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256) in upgradeX86IntrinsicFunction()
576 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512) in upgradeX86IntrinsicFunction()
577 .Default(Intrinsic::not_intrinsic); in upgradeX86IntrinsicFunction()
578 if (ID != Intrinsic::not_intrinsic) in upgradeX86IntrinsicFunction()
584 Intrinsic::ID ID = Intrinsic::not_intrinsic; in upgradeX86IntrinsicFunction()
593 ID = Intrinsic::x86_xop_vpermil2pd; in upgradeX86IntrinsicFunction()
595 ID = Intrinsic::x86_xop_vpermil2ps; in upgradeX86IntrinsicFunction()
597 ID = Intrinsic::x86_xop_vpermil2pd_256; in upgradeX86IntrinsicFunction()
599 ID = Intrinsic::x86_xop_vpermil2ps_256; in upgradeX86IntrinsicFunction()
603 ID = StringSwitch<Intrinsic::ID>(Name) in upgradeX86IntrinsicFunction()
604 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss) in upgradeX86IntrinsicFunction()
605 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd) in upgradeX86IntrinsicFunction()
606 .Default(Intrinsic::not_intrinsic); in upgradeX86IntrinsicFunction()
608 if (ID != Intrinsic::not_intrinsic) { in upgradeX86IntrinsicFunction()
610 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeX86IntrinsicFunction()
617 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp); in upgradeX86IntrinsicFunction()
631 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse, in upgradeArmOrAarch64IntrinsicFunction()
639 Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer); in upgradeArmOrAarch64IntrinsicFunction()
650 Intrinsic::ID ID = in upgradeArmOrAarch64IntrinsicFunction()
651 StringSwitch<Intrinsic::ID>(Name) in upgradeArmOrAarch64IntrinsicFunction()
653 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot in upgradeArmOrAarch64IntrinsicFunction()
654 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot) in upgradeArmOrAarch64IntrinsicFunction()
655 .Default(Intrinsic::not_intrinsic); in upgradeArmOrAarch64IntrinsicFunction()
656 if (ID != Intrinsic::not_intrinsic) { in upgradeArmOrAarch64IntrinsicFunction()
664 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); in upgradeArmOrAarch64IntrinsicFunction()
676 Intrinsic::ID ID = in upgradeArmOrAarch64IntrinsicFunction()
677 StringSwitch<Intrinsic::ID>(Name) in upgradeArmOrAarch64IntrinsicFunction()
679 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla in upgradeArmOrAarch64IntrinsicFunction()
680 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla) in upgradeArmOrAarch64IntrinsicFunction()
682 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb in upgradeArmOrAarch64IntrinsicFunction()
683 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb) in upgradeArmOrAarch64IntrinsicFunction()
685 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt in upgradeArmOrAarch64IntrinsicFunction()
686 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt) in upgradeArmOrAarch64IntrinsicFunction()
687 .Default(Intrinsic::not_intrinsic); in upgradeArmOrAarch64IntrinsicFunction()
688 if (ID != Intrinsic::not_intrinsic) { in upgradeArmOrAarch64IntrinsicFunction()
689 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeArmOrAarch64IntrinsicFunction()
704 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name) in upgradeArmOrAarch64IntrinsicFunction()
705 .StartsWith("vclz.", Intrinsic::ctlz) in upgradeArmOrAarch64IntrinsicFunction()
706 .StartsWith("vcnt.", Intrinsic::ctpop) in upgradeArmOrAarch64IntrinsicFunction()
707 .StartsWith("vqadds.", Intrinsic::sadd_sat) in upgradeArmOrAarch64IntrinsicFunction()
708 .StartsWith("vqaddu.", Intrinsic::uadd_sat) in upgradeArmOrAarch64IntrinsicFunction()
709 .StartsWith("vqsubs.", Intrinsic::ssub_sat) in upgradeArmOrAarch64IntrinsicFunction()
710 .StartsWith("vqsubu.", Intrinsic::usub_sat) in upgradeArmOrAarch64IntrinsicFunction()
711 .Default(Intrinsic::not_intrinsic); in upgradeArmOrAarch64IntrinsicFunction()
712 if (ID != Intrinsic::not_intrinsic) { in upgradeArmOrAarch64IntrinsicFunction()
713 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, in upgradeArmOrAarch64IntrinsicFunction()
723 static const Intrinsic::ID StoreInts[] = { in upgradeArmOrAarch64IntrinsicFunction()
724 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2, in upgradeArmOrAarch64IntrinsicFunction()
725 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4}; in upgradeArmOrAarch64IntrinsicFunction()
727 static const Intrinsic::ID StoreLaneInts[] = { in upgradeArmOrAarch64IntrinsicFunction()
728 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane, in upgradeArmOrAarch64IntrinsicFunction()
729 Intrinsic::arm_neon_vst4lane}; in upgradeArmOrAarch64IntrinsicFunction()
734 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
737 NewFn = Intrinsic::getDeclaration( in upgradeArmOrAarch64IntrinsicFunction()
806 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name) in upgradeArmOrAarch64IntrinsicFunction()
807 .StartsWith("frintn", Intrinsic::roundeven) in upgradeArmOrAarch64IntrinsicFunction()
808 .StartsWith("rbit", Intrinsic::bitreverse) in upgradeArmOrAarch64IntrinsicFunction()
809 .Default(Intrinsic::not_intrinsic); in upgradeArmOrAarch64IntrinsicFunction()
810 if (ID != Intrinsic::not_intrinsic) { in upgradeArmOrAarch64IntrinsicFunction()
811 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, in upgradeArmOrAarch64IntrinsicFunction()
822 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
823 Intrinsic::aarch64_neon_faddp, Ty); in upgradeArmOrAarch64IntrinsicFunction()
834 Intrinsic::ID ID = in upgradeArmOrAarch64IntrinsicFunction()
835 StringSwitch<Intrinsic::ID>(Name) in upgradeArmOrAarch64IntrinsicFunction()
836 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2) in upgradeArmOrAarch64IntrinsicFunction()
837 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2) in upgradeArmOrAarch64IntrinsicFunction()
838 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2) in upgradeArmOrAarch64IntrinsicFunction()
839 .Default(Intrinsic::not_intrinsic); in upgradeArmOrAarch64IntrinsicFunction()
840 if (ID != Intrinsic::not_intrinsic) { in upgradeArmOrAarch64IntrinsicFunction()
841 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeArmOrAarch64IntrinsicFunction()
856 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
857 Intrinsic::aarch64_sve_faddqv, Tys); in upgradeArmOrAarch64IntrinsicFunction()
870 static const Intrinsic::ID LoadIDs[] = { in upgradeArmOrAarch64IntrinsicFunction()
871 Intrinsic::aarch64_sve_ld2_sret, in upgradeArmOrAarch64IntrinsicFunction()
872 Intrinsic::aarch64_sve_ld3_sret, in upgradeArmOrAarch64IntrinsicFunction()
873 Intrinsic::aarch64_sve_ld4_sret, in upgradeArmOrAarch64IntrinsicFunction()
875 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
887 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
888 Intrinsic::vector_extract, Tys); in upgradeArmOrAarch64IntrinsicFunction()
896 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
897 Intrinsic::vector_insert, Tys); in upgradeArmOrAarch64IntrinsicFunction()
906 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
907 Intrinsic::vector_insert, Tys); in upgradeArmOrAarch64IntrinsicFunction()
918 static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) { in shouldUpgradeNVPTXBF16Intrinsic()
920 return StringSwitch<Intrinsic::ID>(Name) in shouldUpgradeNVPTXBF16Intrinsic()
921 .Case("bf16", Intrinsic::nvvm_abs_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
922 .Case("bf16x2", Intrinsic::nvvm_abs_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
923 .Default(Intrinsic::not_intrinsic); in shouldUpgradeNVPTXBF16Intrinsic()
926 return StringSwitch<Intrinsic::ID>(Name) in shouldUpgradeNVPTXBF16Intrinsic()
927 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
928 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
929 .Case("ftz.bf16", Intrinsic::nvvm_fma_rn_ftz_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
930 .Case("ftz.bf16x2", Intrinsic::nvvm_fma_rn_ftz_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
931 .Case("ftz.relu.bf16", Intrinsic::nvvm_fma_rn_ftz_relu_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
932 .Case("ftz.relu.bf16x2", Intrinsic::nvvm_fma_rn_ftz_relu_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
933 .Case("ftz.sat.bf16", Intrinsic::nvvm_fma_rn_ftz_sat_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
934 .Case("ftz.sat.bf16x2", Intrinsic::nvvm_fma_rn_ftz_sat_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
935 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
936 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
937 .Case("sat.bf16", Intrinsic::nvvm_fma_rn_sat_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
938 .Case("sat.bf16x2", Intrinsic::nvvm_fma_rn_sat_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
939 .Default(Intrinsic::not_intrinsic); in shouldUpgradeNVPTXBF16Intrinsic()
942 return StringSwitch<Intrinsic::ID>(Name) in shouldUpgradeNVPTXBF16Intrinsic()
943 .Case("bf16", Intrinsic::nvvm_fmax_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
944 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
945 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
946 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
947 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
948 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
950 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
952 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
953 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
955 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
956 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
957 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
958 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
960 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
961 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
962 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
963 .Default(Intrinsic::not_intrinsic); in shouldUpgradeNVPTXBF16Intrinsic()
966 return StringSwitch<Intrinsic::ID>(Name) in shouldUpgradeNVPTXBF16Intrinsic()
967 .Case("bf16", Intrinsic::nvvm_fmin_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
968 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
969 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
970 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
971 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
972 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
974 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
976 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
977 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
979 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
980 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
981 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
982 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
984 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
985 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
986 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
987 .Default(Intrinsic::not_intrinsic); in shouldUpgradeNVPTXBF16Intrinsic()
990 return StringSwitch<Intrinsic::ID>(Name) in shouldUpgradeNVPTXBF16Intrinsic()
991 .Case("bf16", Intrinsic::nvvm_neg_bf16) in shouldUpgradeNVPTXBF16Intrinsic()
992 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2) in shouldUpgradeNVPTXBF16Intrinsic()
993 .Default(Intrinsic::not_intrinsic); in shouldUpgradeNVPTXBF16Intrinsic()
995 return Intrinsic::not_intrinsic; in shouldUpgradeNVPTXBF16Intrinsic()
1021 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::fshr, in upgradeIntrinsicFunction1()
1046 NewFn = Intrinsic::getDeclaration( in upgradeIntrinsicFunction1()
1047 F->getParent(), Intrinsic::ldexp, in upgradeIntrinsicFunction1()
1058 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name) in upgradeIntrinsicFunction1()
1059 .StartsWith("ctlz.", Intrinsic::ctlz) in upgradeIntrinsicFunction1()
1060 .StartsWith("cttz.", Intrinsic::cttz) in upgradeIntrinsicFunction1()
1061 .Default(Intrinsic::not_intrinsic); in upgradeIntrinsicFunction1()
1062 if (ID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1064 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, in upgradeIntrinsicFunction1()
1072 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::coro_end); in upgradeIntrinsicFunction1()
1095 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value); in upgradeIntrinsicFunction1()
1103 Intrinsic::ID ID = in upgradeIntrinsicFunction1()
1104 StringSwitch<Intrinsic::ID>(Name) in upgradeIntrinsicFunction1()
1105 .StartsWith("extract.", Intrinsic::vector_extract) in upgradeIntrinsicFunction1()
1106 .StartsWith("insert.", Intrinsic::vector_insert) in upgradeIntrinsicFunction1()
1107 .StartsWith("splice.", Intrinsic::vector_splice) in upgradeIntrinsicFunction1()
1108 .StartsWith("reverse.", Intrinsic::vector_reverse) in upgradeIntrinsicFunction1()
1109 .StartsWith("interleave2.", Intrinsic::vector_interleave2) in upgradeIntrinsicFunction1()
1110 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2) in upgradeIntrinsicFunction1()
1111 .Default(Intrinsic::not_intrinsic); in upgradeIntrinsicFunction1()
1112 if (ID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1115 if (ID == Intrinsic::vector_extract || in upgradeIntrinsicFunction1()
1116 ID == Intrinsic::vector_interleave2) in upgradeIntrinsicFunction1()
1119 if (ID != Intrinsic::vector_interleave2) in upgradeIntrinsicFunction1()
1121 if (ID == Intrinsic::vector_insert) in upgradeIntrinsicFunction1()
1125 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); in upgradeIntrinsicFunction1()
1133 ID = StringSwitch<Intrinsic::ID>(Groups[1]) in upgradeIntrinsicFunction1()
1134 .Case("add", Intrinsic::vector_reduce_add) in upgradeIntrinsicFunction1()
1135 .Case("mul", Intrinsic::vector_reduce_mul) in upgradeIntrinsicFunction1()
1136 .Case("and", Intrinsic::vector_reduce_and) in upgradeIntrinsicFunction1()
1137 .Case("or", Intrinsic::vector_reduce_or) in upgradeIntrinsicFunction1()
1138 .Case("xor", Intrinsic::vector_reduce_xor) in upgradeIntrinsicFunction1()
1139 .Case("smax", Intrinsic::vector_reduce_smax) in upgradeIntrinsicFunction1()
1140 .Case("smin", Intrinsic::vector_reduce_smin) in upgradeIntrinsicFunction1()
1141 .Case("umax", Intrinsic::vector_reduce_umax) in upgradeIntrinsicFunction1()
1142 .Case("umin", Intrinsic::vector_reduce_umin) in upgradeIntrinsicFunction1()
1143 .Case("fmax", Intrinsic::vector_reduce_fmax) in upgradeIntrinsicFunction1()
1144 .Case("fmin", Intrinsic::vector_reduce_fmin) in upgradeIntrinsicFunction1()
1145 .Default(Intrinsic::not_intrinsic); in upgradeIntrinsicFunction1()
1148 if (ID == Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1153 ID = StringSwitch<Intrinsic::ID>(Groups[1]) in upgradeIntrinsicFunction1()
1154 .Case("fadd", Intrinsic::vector_reduce_fadd) in upgradeIntrinsicFunction1()
1155 .Case("fmul", Intrinsic::vector_reduce_fmul) in upgradeIntrinsicFunction1()
1156 .Default(Intrinsic::not_intrinsic); in upgradeIntrinsicFunction1()
1158 if (ID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1162 Intrinsic::getDeclaration(F->getParent(), ID, {Args[V2 ? 1 : 0]}); in upgradeIntrinsicFunction1()
1173 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::get_rounding); in upgradeIntrinsicFunction1()
1183 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeIntrinsicFunction1()
1184 Intrinsic::launder_invariant_group, ObjectPtr); in upgradeIntrinsicFunction1()
1193 .StartsWith("memcpy.", Intrinsic::memcpy) in upgradeIntrinsicFunction1()
1194 .StartsWith("memmove.", Intrinsic::memmove) in upgradeIntrinsicFunction1()
1201 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ParamTypes); in upgradeIntrinsicFunction1()
1213 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset, in upgradeIntrinsicFunction1()
1223 Intrinsic::ID IID = in upgradeIntrinsicFunction1()
1224 StringSwitch<Intrinsic::ID>(Name) in upgradeIntrinsicFunction1()
1225 .Cases("brev32", "brev64", Intrinsic::bitreverse) in upgradeIntrinsicFunction1()
1226 .Case("clz.i", Intrinsic::ctlz) in upgradeIntrinsicFunction1()
1227 .Case("popc.i", Intrinsic::ctpop) in upgradeIntrinsicFunction1()
1228 .Default(Intrinsic::not_intrinsic); in upgradeIntrinsicFunction1()
1229 if (IID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1230 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, in upgradeIntrinsicFunction1()
1238 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name); in upgradeIntrinsicFunction1()
1239 if (IID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1280 Intrinsic::getName(Intrinsic::objectsize, Tys, F->getParent())) { in upgradeIntrinsicFunction1()
1282 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize, in upgradeIntrinsicFunction1()
1292 NewFn = Intrinsic::getDeclaration( in upgradeIntrinsicFunction1()
1293 F->getParent(), Intrinsic::ptr_annotation, in upgradeIntrinsicFunction1()
1301 Intrinsic::ID ID; in upgradeIntrinsicFunction1()
1302 ID = StringSwitch<Intrinsic::ID>(Name) in upgradeIntrinsicFunction1()
1303 .Case("aes32dsi", Intrinsic::riscv_aes32dsi) in upgradeIntrinsicFunction1()
1304 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi) in upgradeIntrinsicFunction1()
1305 .Case("aes32esi", Intrinsic::riscv_aes32esi) in upgradeIntrinsicFunction1()
1306 .Case("aes32esmi", Intrinsic::riscv_aes32esmi) in upgradeIntrinsicFunction1()
1307 .Default(Intrinsic::not_intrinsic); in upgradeIntrinsicFunction1()
1308 if (ID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1311 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeIntrinsicFunction1()
1317 ID = StringSwitch<Intrinsic::ID>(Name) in upgradeIntrinsicFunction1()
1318 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks) in upgradeIntrinsicFunction1()
1319 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed) in upgradeIntrinsicFunction1()
1320 .Default(Intrinsic::not_intrinsic); in upgradeIntrinsicFunction1()
1321 if (ID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1325 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeIntrinsicFunction1()
1331 ID = StringSwitch<Intrinsic::ID>(Name) in upgradeIntrinsicFunction1()
1332 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0) in upgradeIntrinsicFunction1()
1333 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1) in upgradeIntrinsicFunction1()
1334 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0) in upgradeIntrinsicFunction1()
1335 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1) in upgradeIntrinsicFunction1()
1336 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0) in upgradeIntrinsicFunction1()
1337 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1) in upgradeIntrinsicFunction1()
1338 .Default(Intrinsic::not_intrinsic); in upgradeIntrinsicFunction1()
1339 if (ID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1342 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeIntrinsicFunction1()
1361 NewFn = Intrinsic::getDeclaration( in upgradeIntrinsicFunction1()
1362 F->getParent(), Intrinsic::var_annotation, in upgradeIntrinsicFunction1()
1371 Intrinsic::ID ID = in upgradeIntrinsicFunction1()
1372 StringSwitch<Intrinsic::ID>(Name) in upgradeIntrinsicFunction1()
1373 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd) in upgradeIntrinsicFunction1()
1374 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd) in upgradeIntrinsicFunction1()
1375 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect) in upgradeIntrinsicFunction1()
1376 .Default(Intrinsic::not_intrinsic); in upgradeIntrinsicFunction1()
1377 if (ID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1380 Intrinsic::getDeclaration(F->getParent(), ID, F->getReturnType()); in upgradeIntrinsicFunction1()
1385 ID = StringSwitch<Intrinsic::ID>(Name) in upgradeIntrinsicFunction1()
1386 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed) in upgradeIntrinsicFunction1()
1388 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed) in upgradeIntrinsicFunction1()
1389 .Default(Intrinsic::not_intrinsic); in upgradeIntrinsicFunction1()
1390 if (ID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1392 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeIntrinsicFunction1()
1408 F->getIntrinsicID() != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1413 SmallVector<Intrinsic::IITDescriptor> Desc; in upgradeIntrinsicFunction1()
1414 Intrinsic::getIntrinsicInfoTableEntries(F->getIntrinsicID(), Desc); in upgradeIntrinsicFunction1()
1415 if (Desc.front().Kind == Intrinsic::IITDescriptor::Struct) { in upgradeIntrinsicFunction1()
1425 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn)) in upgradeIntrinsicFunction1()
1432 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F); in upgradeIntrinsicFunction1()
1455 if (Intrinsic::ID id = F->getIntrinsicID()) in UpgradeIntrinsicFunction()
1456 F->setAttributes(Intrinsic::getAttributes(F->getContext(), id)); in UpgradeIntrinsicFunction()
1659 Intrinsic::ID IID; in upgradeX86VPERMT2Intrinsics()
1661 IID = Intrinsic::x86_avx512_vpermi2var_ps_128; in upgradeX86VPERMT2Intrinsics()
1663 IID = Intrinsic::x86_avx512_vpermi2var_d_128; in upgradeX86VPERMT2Intrinsics()
1665 IID = Intrinsic::x86_avx512_vpermi2var_pd_128; in upgradeX86VPERMT2Intrinsics()
1667 IID = Intrinsic::x86_avx512_vpermi2var_q_128; in upgradeX86VPERMT2Intrinsics()
1669 IID = Intrinsic::x86_avx512_vpermi2var_ps_256; in upgradeX86VPERMT2Intrinsics()
1671 IID = Intrinsic::x86_avx512_vpermi2var_d_256; in upgradeX86VPERMT2Intrinsics()
1673 IID = Intrinsic::x86_avx512_vpermi2var_pd_256; in upgradeX86VPERMT2Intrinsics()
1675 IID = Intrinsic::x86_avx512_vpermi2var_q_256; in upgradeX86VPERMT2Intrinsics()
1677 IID = Intrinsic::x86_avx512_vpermi2var_ps_512; in upgradeX86VPERMT2Intrinsics()
1679 IID = Intrinsic::x86_avx512_vpermi2var_d_512; in upgradeX86VPERMT2Intrinsics()
1681 IID = Intrinsic::x86_avx512_vpermi2var_pd_512; in upgradeX86VPERMT2Intrinsics()
1683 IID = Intrinsic::x86_avx512_vpermi2var_q_512; in upgradeX86VPERMT2Intrinsics()
1685 IID = Intrinsic::x86_avx512_vpermi2var_hi_128; in upgradeX86VPERMT2Intrinsics()
1687 IID = Intrinsic::x86_avx512_vpermi2var_hi_256; in upgradeX86VPERMT2Intrinsics()
1689 IID = Intrinsic::x86_avx512_vpermi2var_hi_512; in upgradeX86VPERMT2Intrinsics()
1691 IID = Intrinsic::x86_avx512_vpermi2var_qi_128; in upgradeX86VPERMT2Intrinsics()
1693 IID = Intrinsic::x86_avx512_vpermi2var_qi_256; in upgradeX86VPERMT2Intrinsics()
1695 IID = Intrinsic::x86_avx512_vpermi2var_qi_512; in upgradeX86VPERMT2Intrinsics()
1706 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in upgradeX86VPERMT2Intrinsics()
1715 Intrinsic::ID IID) { in upgradeX86BinaryIntrinsics()
1719 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86BinaryIntrinsics()
1745 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl; in upgradeX86Rotate()
1746 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86Rotate()
1815 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl; in upgradeX86ConcatShift()
1816 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86ConcatShift()
1877 Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty); in upgradeAbs()
1969 Intrinsic::ID IID) { in upgradeX86MaskedShift()
1970 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID); in upgradeX86MaskedShift()
2005 Intrinsic::ID IID; in upgradeAVX512MaskToSelect()
2008 IID = Intrinsic::x86_sse_max_ps; in upgradeAVX512MaskToSelect()
2010 IID = Intrinsic::x86_sse2_max_pd; in upgradeAVX512MaskToSelect()
2012 IID = Intrinsic::x86_avx_max_ps_256; in upgradeAVX512MaskToSelect()
2014 IID = Intrinsic::x86_avx_max_pd_256; in upgradeAVX512MaskToSelect()
2019 IID = Intrinsic::x86_sse_min_ps; in upgradeAVX512MaskToSelect()
2021 IID = Intrinsic::x86_sse2_min_pd; in upgradeAVX512MaskToSelect()
2023 IID = Intrinsic::x86_avx_min_ps_256; in upgradeAVX512MaskToSelect()
2025 IID = Intrinsic::x86_avx_min_pd_256; in upgradeAVX512MaskToSelect()
2030 IID = Intrinsic::x86_ssse3_pshuf_b_128; in upgradeAVX512MaskToSelect()
2032 IID = Intrinsic::x86_avx2_pshuf_b; in upgradeAVX512MaskToSelect()
2034 IID = Intrinsic::x86_avx512_pshuf_b_512; in upgradeAVX512MaskToSelect()
2039 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128; in upgradeAVX512MaskToSelect()
2041 IID = Intrinsic::x86_avx2_pmul_hr_sw; in upgradeAVX512MaskToSelect()
2043 IID = Intrinsic::x86_avx512_pmul_hr_sw_512; in upgradeAVX512MaskToSelect()
2048 IID = Intrinsic::x86_sse2_pmulh_w; in upgradeAVX512MaskToSelect()
2050 IID = Intrinsic::x86_avx2_pmulh_w; in upgradeAVX512MaskToSelect()
2052 IID = Intrinsic::x86_avx512_pmulh_w_512; in upgradeAVX512MaskToSelect()
2057 IID = Intrinsic::x86_sse2_pmulhu_w; in upgradeAVX512MaskToSelect()
2059 IID = Intrinsic::x86_avx2_pmulhu_w; in upgradeAVX512MaskToSelect()
2061 IID = Intrinsic::x86_avx512_pmulhu_w_512; in upgradeAVX512MaskToSelect()
2066 IID = Intrinsic::x86_sse2_pmadd_wd; in upgradeAVX512MaskToSelect()
2068 IID = Intrinsic::x86_avx2_pmadd_wd; in upgradeAVX512MaskToSelect()
2070 IID = Intrinsic::x86_avx512_pmaddw_d_512; in upgradeAVX512MaskToSelect()
2075 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128; in upgradeAVX512MaskToSelect()
2077 IID = Intrinsic::x86_avx2_pmadd_ub_sw; in upgradeAVX512MaskToSelect()
2079 IID = Intrinsic::x86_avx512_pmaddubs_w_512; in upgradeAVX512MaskToSelect()
2084 IID = Intrinsic::x86_sse2_packsswb_128; in upgradeAVX512MaskToSelect()
2086 IID = Intrinsic::x86_avx2_packsswb; in upgradeAVX512MaskToSelect()
2088 IID = Intrinsic::x86_avx512_packsswb_512; in upgradeAVX512MaskToSelect()
2093 IID = Intrinsic::x86_sse2_packssdw_128; in upgradeAVX512MaskToSelect()
2095 IID = Intrinsic::x86_avx2_packssdw; in upgradeAVX512MaskToSelect()
2097 IID = Intrinsic::x86_avx512_packssdw_512; in upgradeAVX512MaskToSelect()
2102 IID = Intrinsic::x86_sse2_packuswb_128; in upgradeAVX512MaskToSelect()
2104 IID = Intrinsic::x86_avx2_packuswb; in upgradeAVX512MaskToSelect()
2106 IID = Intrinsic::x86_avx512_packuswb_512; in upgradeAVX512MaskToSelect()
2111 IID = Intrinsic::x86_sse41_packusdw; in upgradeAVX512MaskToSelect()
2113 IID = Intrinsic::x86_avx2_packusdw; in upgradeAVX512MaskToSelect()
2115 IID = Intrinsic::x86_avx512_packusdw_512; in upgradeAVX512MaskToSelect()
2120 IID = Intrinsic::x86_avx_vpermilvar_ps; in upgradeAVX512MaskToSelect()
2122 IID = Intrinsic::x86_avx_vpermilvar_pd; in upgradeAVX512MaskToSelect()
2124 IID = Intrinsic::x86_avx_vpermilvar_ps_256; in upgradeAVX512MaskToSelect()
2126 IID = Intrinsic::x86_avx_vpermilvar_pd_256; in upgradeAVX512MaskToSelect()
2128 IID = Intrinsic::x86_avx512_vpermilvar_ps_512; in upgradeAVX512MaskToSelect()
2130 IID = Intrinsic::x86_avx512_vpermilvar_pd_512; in upgradeAVX512MaskToSelect()
2134 IID = Intrinsic::x86_avx_cvt_pd2dq_256; in upgradeAVX512MaskToSelect()
2136 IID = Intrinsic::x86_avx_cvt_pd2_ps_256; in upgradeAVX512MaskToSelect()
2138 IID = Intrinsic::x86_avx_cvtt_pd2dq_256; in upgradeAVX512MaskToSelect()
2140 IID = Intrinsic::x86_sse2_cvttps2dq; in upgradeAVX512MaskToSelect()
2142 IID = Intrinsic::x86_avx_cvtt_ps2dq_256; in upgradeAVX512MaskToSelect()
2146 IID = Intrinsic::x86_avx2_permps; in upgradeAVX512MaskToSelect()
2148 IID = Intrinsic::x86_avx2_permd; in upgradeAVX512MaskToSelect()
2150 IID = Intrinsic::x86_avx512_permvar_df_256; in upgradeAVX512MaskToSelect()
2152 IID = Intrinsic::x86_avx512_permvar_di_256; in upgradeAVX512MaskToSelect()
2154 IID = Intrinsic::x86_avx512_permvar_sf_512; in upgradeAVX512MaskToSelect()
2156 IID = Intrinsic::x86_avx512_permvar_si_512; in upgradeAVX512MaskToSelect()
2158 IID = Intrinsic::x86_avx512_permvar_df_512; in upgradeAVX512MaskToSelect()
2160 IID = Intrinsic::x86_avx512_permvar_di_512; in upgradeAVX512MaskToSelect()
2162 IID = Intrinsic::x86_avx512_permvar_hi_128; in upgradeAVX512MaskToSelect()
2164 IID = Intrinsic::x86_avx512_permvar_hi_256; in upgradeAVX512MaskToSelect()
2166 IID = Intrinsic::x86_avx512_permvar_hi_512; in upgradeAVX512MaskToSelect()
2168 IID = Intrinsic::x86_avx512_permvar_qi_128; in upgradeAVX512MaskToSelect()
2170 IID = Intrinsic::x86_avx512_permvar_qi_256; in upgradeAVX512MaskToSelect()
2172 IID = Intrinsic::x86_avx512_permvar_qi_512; in upgradeAVX512MaskToSelect()
2177 IID = Intrinsic::x86_avx512_dbpsadbw_128; in upgradeAVX512MaskToSelect()
2179 IID = Intrinsic::x86_avx512_dbpsadbw_256; in upgradeAVX512MaskToSelect()
2181 IID = Intrinsic::x86_avx512_dbpsadbw_512; in upgradeAVX512MaskToSelect()
2186 IID = Intrinsic::x86_avx512_pmultishift_qb_128; in upgradeAVX512MaskToSelect()
2188 IID = Intrinsic::x86_avx512_pmultishift_qb_256; in upgradeAVX512MaskToSelect()
2190 IID = Intrinsic::x86_avx512_pmultishift_qb_512; in upgradeAVX512MaskToSelect()
2195 IID = Intrinsic::x86_avx512_conflict_d_128; in upgradeAVX512MaskToSelect()
2197 IID = Intrinsic::x86_avx512_conflict_d_256; in upgradeAVX512MaskToSelect()
2199 IID = Intrinsic::x86_avx512_conflict_d_512; in upgradeAVX512MaskToSelect()
2201 IID = Intrinsic::x86_avx512_conflict_q_128; in upgradeAVX512MaskToSelect()
2203 IID = Intrinsic::x86_avx512_conflict_q_256; in upgradeAVX512MaskToSelect()
2205 IID = Intrinsic::x86_avx512_conflict_q_512; in upgradeAVX512MaskToSelect()
2210 IID = Intrinsic::x86_sse2_pavg_b; in upgradeAVX512MaskToSelect()
2212 IID = Intrinsic::x86_avx2_pavg_b; in upgradeAVX512MaskToSelect()
2214 IID = Intrinsic::x86_avx512_pavg_b_512; in upgradeAVX512MaskToSelect()
2216 IID = Intrinsic::x86_sse2_pavg_w; in upgradeAVX512MaskToSelect()
2218 IID = Intrinsic::x86_avx2_pavg_w; in upgradeAVX512MaskToSelect()
2220 IID = Intrinsic::x86_avx512_pavg_w_512; in upgradeAVX512MaskToSelect()
2229 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in upgradeAVX512MaskToSelect()
2335 Function *Intr = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sqrt, in upgradeX86IntrinsicCall()
2343 Builder.CreateCall(Intrinsic::getDeclaration( in upgradeX86IntrinsicCall()
2344 F->getParent(), Intrinsic::sqrt, CI->getType()), in upgradeX86IntrinsicCall()
2350 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512 in upgradeX86IntrinsicCall()
2351 : Intrinsic::x86_avx512_sqrt_pd_512; in upgradeX86IntrinsicCall()
2354 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
2357 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), in upgradeX86IntrinsicCall()
2358 Intrinsic::sqrt, in upgradeX86IntrinsicCall()
2469 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
2474 IID = Intrinsic::x86_avx512_vpshufbitqmb_128; in upgradeX86IntrinsicCall()
2477 IID = Intrinsic::x86_avx512_vpshufbitqmb_256; in upgradeX86IntrinsicCall()
2480 IID = Intrinsic::x86_avx512_vpshufbitqmb_512; in upgradeX86IntrinsicCall()
2484 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
2491 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
2493 IID = Intrinsic::x86_avx512_fpclass_ps_128; in upgradeX86IntrinsicCall()
2495 IID = Intrinsic::x86_avx512_fpclass_ps_256; in upgradeX86IntrinsicCall()
2497 IID = Intrinsic::x86_avx512_fpclass_ps_512; in upgradeX86IntrinsicCall()
2499 IID = Intrinsic::x86_avx512_fpclass_pd_128; in upgradeX86IntrinsicCall()
2501 IID = Intrinsic::x86_avx512_fpclass_pd_256; in upgradeX86IntrinsicCall()
2503 IID = Intrinsic::x86_avx512_fpclass_pd_512; in upgradeX86IntrinsicCall()
2507 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
2515 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
2517 IID = Intrinsic::x86_avx512_mask_cmp_ps_128; in upgradeX86IntrinsicCall()
2519 IID = Intrinsic::x86_avx512_mask_cmp_ps_256; in upgradeX86IntrinsicCall()
2521 IID = Intrinsic::x86_avx512_mask_cmp_ps_512; in upgradeX86IntrinsicCall()
2523 IID = Intrinsic::x86_avx512_mask_cmp_pd_128; in upgradeX86IntrinsicCall()
2525 IID = Intrinsic::x86_avx512_mask_cmp_pd_256; in upgradeX86IntrinsicCall()
2527 IID = Intrinsic::x86_avx512_mask_cmp_pd_512; in upgradeX86IntrinsicCall()
2536 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
2560 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax); in upgradeX86IntrinsicCall()
2564 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax); in upgradeX86IntrinsicCall()
2568 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin); in upgradeX86IntrinsicCall()
2572 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin); in upgradeX86IntrinsicCall()
2629 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round in upgradeX86IntrinsicCall()
2630 : Intrinsic::x86_avx512_sitofp_round; in upgradeX86IntrinsicCall()
2632 Intrinsic::getDeclaration(CI->getModule(), IID, {DstTy, SrcTy}); in upgradeX86IntrinsicCall()
2674 Function *ELd = Intrinsic::getDeclaration( in upgradeX86IntrinsicCall()
2675 F->getParent(), Intrinsic::masked_expandload, ResultTy); in upgradeX86IntrinsicCall()
2689 Function *CSt = Intrinsic::getDeclaration( in upgradeX86IntrinsicCall()
2690 F->getParent(), Intrinsic::masked_compressstore, ResultTy); in upgradeX86IntrinsicCall()
2700 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress in upgradeX86IntrinsicCall()
2701 : Intrinsic::x86_avx512_mask_expand; in upgradeX86IntrinsicCall()
2702 Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy); in upgradeX86IntrinsicCall()
2765 Function *CRC32 = Intrinsic::getDeclaration( in upgradeX86IntrinsicCall()
2766 F->getParent(), Intrinsic::x86_sse42_crc32_32_8); in upgradeX86IntrinsicCall()
2883 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat); in upgradeX86IntrinsicCall()
2888 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat); in upgradeX86IntrinsicCall()
2892 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat); in upgradeX86IntrinsicCall()
2896 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat); in upgradeX86IntrinsicCall()
3253 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3255 IID = Intrinsic::x86_avx512_add_ps_512; in upgradeX86IntrinsicCall()
3257 IID = Intrinsic::x86_avx512_add_pd_512; in upgradeX86IntrinsicCall()
3260 Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3269 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3271 IID = Intrinsic::x86_avx512_div_ps_512; in upgradeX86IntrinsicCall()
3273 IID = Intrinsic::x86_avx512_div_pd_512; in upgradeX86IntrinsicCall()
3276 Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3285 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3287 IID = Intrinsic::x86_avx512_mul_ps_512; in upgradeX86IntrinsicCall()
3289 IID = Intrinsic::x86_avx512_mul_pd_512; in upgradeX86IntrinsicCall()
3292 Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3301 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3303 IID = Intrinsic::x86_avx512_sub_ps_512; in upgradeX86IntrinsicCall()
3305 IID = Intrinsic::x86_avx512_sub_pd_512; in upgradeX86IntrinsicCall()
3308 Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3320 static const Intrinsic::ID MinMaxTbl[2][2] = { in upgradeX86IntrinsicCall()
3321 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512}, in upgradeX86IntrinsicCall()
3322 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}}; in upgradeX86IntrinsicCall()
3323 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble]; in upgradeX86IntrinsicCall()
3326 Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3332 Builder.CreateCall(Intrinsic::getDeclaration( in upgradeX86IntrinsicCall()
3333 F->getParent(), Intrinsic::ctlz, CI->getType()), in upgradeX86IntrinsicCall()
3345 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3348 IID = Intrinsic::x86_avx2_psllv_q; in upgradeX86IntrinsicCall()
3350 IID = Intrinsic::x86_avx2_psllv_q_256; in upgradeX86IntrinsicCall()
3352 IID = Intrinsic::x86_avx2_psllv_d; in upgradeX86IntrinsicCall()
3354 IID = Intrinsic::x86_avx2_psllv_d_256; in upgradeX86IntrinsicCall()
3356 IID = Intrinsic::x86_avx512_psllv_w_128; in upgradeX86IntrinsicCall()
3358 IID = Intrinsic::x86_avx512_psllv_w_256; in upgradeX86IntrinsicCall()
3360 IID = Intrinsic::x86_avx512_psllv_w_512; in upgradeX86IntrinsicCall()
3365 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d in upgradeX86IntrinsicCall()
3366 : Intrinsic::x86_sse2_psll_d; in upgradeX86IntrinsicCall()
3368 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q in upgradeX86IntrinsicCall()
3369 : Intrinsic::x86_sse2_psll_q; in upgradeX86IntrinsicCall()
3371 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w in upgradeX86IntrinsicCall()
3372 : Intrinsic::x86_sse2_psll_w; in upgradeX86IntrinsicCall()
3377 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d in upgradeX86IntrinsicCall()
3378 : Intrinsic::x86_avx2_psll_d; in upgradeX86IntrinsicCall()
3380 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q in upgradeX86IntrinsicCall()
3381 : Intrinsic::x86_avx2_psll_q; in upgradeX86IntrinsicCall()
3383 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w in upgradeX86IntrinsicCall()
3384 : Intrinsic::x86_avx2_psll_w; in upgradeX86IntrinsicCall()
3389 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 in upgradeX86IntrinsicCall()
3390 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512 in upgradeX86IntrinsicCall()
3391 : Intrinsic::x86_avx512_psll_d_512; in upgradeX86IntrinsicCall()
3393 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 in upgradeX86IntrinsicCall()
3394 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512 in upgradeX86IntrinsicCall()
3395 : Intrinsic::x86_avx512_psll_q_512; in upgradeX86IntrinsicCall()
3397 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512 in upgradeX86IntrinsicCall()
3398 : Intrinsic::x86_avx512_psll_w_512; in upgradeX86IntrinsicCall()
3412 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3415 IID = Intrinsic::x86_avx2_psrlv_q; in upgradeX86IntrinsicCall()
3417 IID = Intrinsic::x86_avx2_psrlv_q_256; in upgradeX86IntrinsicCall()
3419 IID = Intrinsic::x86_avx2_psrlv_d; in upgradeX86IntrinsicCall()
3421 IID = Intrinsic::x86_avx2_psrlv_d_256; in upgradeX86IntrinsicCall()
3423 IID = Intrinsic::x86_avx512_psrlv_w_128; in upgradeX86IntrinsicCall()
3425 IID = Intrinsic::x86_avx512_psrlv_w_256; in upgradeX86IntrinsicCall()
3427 IID = Intrinsic::x86_avx512_psrlv_w_512; in upgradeX86IntrinsicCall()
3432 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d in upgradeX86IntrinsicCall()
3433 : Intrinsic::x86_sse2_psrl_d; in upgradeX86IntrinsicCall()
3435 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q in upgradeX86IntrinsicCall()
3436 : Intrinsic::x86_sse2_psrl_q; in upgradeX86IntrinsicCall()
3438 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w in upgradeX86IntrinsicCall()
3439 : Intrinsic::x86_sse2_psrl_w; in upgradeX86IntrinsicCall()
3444 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d in upgradeX86IntrinsicCall()
3445 : Intrinsic::x86_avx2_psrl_d; in upgradeX86IntrinsicCall()
3447 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q in upgradeX86IntrinsicCall()
3448 : Intrinsic::x86_avx2_psrl_q; in upgradeX86IntrinsicCall()
3450 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w in upgradeX86IntrinsicCall()
3451 : Intrinsic::x86_avx2_psrl_w; in upgradeX86IntrinsicCall()
3456 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 in upgradeX86IntrinsicCall()
3457 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 in upgradeX86IntrinsicCall()
3458 : Intrinsic::x86_avx512_psrl_d_512; in upgradeX86IntrinsicCall()
3460 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 in upgradeX86IntrinsicCall()
3461 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 in upgradeX86IntrinsicCall()
3462 : Intrinsic::x86_avx512_psrl_q_512; in upgradeX86IntrinsicCall()
3464 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512 in upgradeX86IntrinsicCall()
3465 : Intrinsic::x86_avx512_psrl_w_512; in upgradeX86IntrinsicCall()
3479 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3482 IID = Intrinsic::x86_avx2_psrav_d; in upgradeX86IntrinsicCall()
3484 IID = Intrinsic::x86_avx2_psrav_d_256; in upgradeX86IntrinsicCall()
3486 IID = Intrinsic::x86_avx512_psrav_w_128; in upgradeX86IntrinsicCall()
3488 IID = Intrinsic::x86_avx512_psrav_w_256; in upgradeX86IntrinsicCall()
3490 IID = Intrinsic::x86_avx512_psrav_w_512; in upgradeX86IntrinsicCall()
3495 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d in upgradeX86IntrinsicCall()
3496 : Intrinsic::x86_sse2_psra_d; in upgradeX86IntrinsicCall()
3498 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 in upgradeX86IntrinsicCall()
3499 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128 in upgradeX86IntrinsicCall()
3500 : Intrinsic::x86_avx512_psra_q_128; in upgradeX86IntrinsicCall()
3502 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w in upgradeX86IntrinsicCall()
3503 : Intrinsic::x86_sse2_psra_w; in upgradeX86IntrinsicCall()
3508 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d in upgradeX86IntrinsicCall()
3509 : Intrinsic::x86_avx2_psra_d; in upgradeX86IntrinsicCall()
3511 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 in upgradeX86IntrinsicCall()
3512 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256 in upgradeX86IntrinsicCall()
3513 : Intrinsic::x86_avx512_psra_q_256; in upgradeX86IntrinsicCall()
3515 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w in upgradeX86IntrinsicCall()
3516 : Intrinsic::x86_avx2_psra_w; in upgradeX86IntrinsicCall()
3521 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 in upgradeX86IntrinsicCall()
3522 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512 in upgradeX86IntrinsicCall()
3523 : Intrinsic::x86_avx512_psra_d_512; in upgradeX86IntrinsicCall()
3525 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 in upgradeX86IntrinsicCall()
3526 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512 in upgradeX86IntrinsicCall()
3527 : Intrinsic::x86_avx512_psra_q_512; in upgradeX86IntrinsicCall()
3529 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512 in upgradeX86IntrinsicCall()
3530 : Intrinsic::x86_avx512_psra_w_512; in upgradeX86IntrinsicCall()
3578 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in upgradeX86IntrinsicCall()
3579 Intrinsic::fma, in upgradeX86IntrinsicCall()
3593 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in upgradeX86IntrinsicCall()
3594 Intrinsic::fma, in upgradeX86IntrinsicCall()
3631 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3633 IID = Intrinsic::x86_avx512_vfmadd_f64; in upgradeX86IntrinsicCall()
3635 IID = Intrinsic::x86_avx512_vfmadd_f32; in upgradeX86IntrinsicCall()
3636 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID); in upgradeX86IntrinsicCall()
3639 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in upgradeX86IntrinsicCall()
3685 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3688 IID = Intrinsic::x86_avx512_vfmadd_ps_512; in upgradeX86IntrinsicCall()
3690 IID = Intrinsic::x86_avx512_vfmadd_pd_512; in upgradeX86IntrinsicCall()
3692 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3695 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in upgradeX86IntrinsicCall()
3708 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3710 IID = Intrinsic::x86_fma_vfmaddsub_ps; in upgradeX86IntrinsicCall()
3712 IID = Intrinsic::x86_fma_vfmaddsub_ps_256; in upgradeX86IntrinsicCall()
3714 IID = Intrinsic::x86_fma_vfmaddsub_pd; in upgradeX86IntrinsicCall()
3716 IID = Intrinsic::x86_fma_vfmaddsub_pd_256; in upgradeX86IntrinsicCall()
3724 Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), Ops); in upgradeX86IntrinsicCall()
3735 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3738 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512; in upgradeX86IntrinsicCall()
3740 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512; in upgradeX86IntrinsicCall()
3747 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3755 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in upgradeX86IntrinsicCall()
3781 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3783 IID = Intrinsic::x86_avx512_pternlog_d_128; in upgradeX86IntrinsicCall()
3785 IID = Intrinsic::x86_avx512_pternlog_d_256; in upgradeX86IntrinsicCall()
3787 IID = Intrinsic::x86_avx512_pternlog_d_512; in upgradeX86IntrinsicCall()
3789 IID = Intrinsic::x86_avx512_pternlog_q_128; in upgradeX86IntrinsicCall()
3791 IID = Intrinsic::x86_avx512_pternlog_q_256; in upgradeX86IntrinsicCall()
3793 IID = Intrinsic::x86_avx512_pternlog_q_512; in upgradeX86IntrinsicCall()
3799 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3809 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3811 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128; in upgradeX86IntrinsicCall()
3813 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256; in upgradeX86IntrinsicCall()
3815 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512; in upgradeX86IntrinsicCall()
3817 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128; in upgradeX86IntrinsicCall()
3819 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256; in upgradeX86IntrinsicCall()
3821 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512; in upgradeX86IntrinsicCall()
3827 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3845 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3847 IID = Intrinsic::x86_avx512_vpdpbusd_128; in upgradeX86IntrinsicCall()
3849 IID = Intrinsic::x86_avx512_vpdpbusd_256; in upgradeX86IntrinsicCall()
3851 IID = Intrinsic::x86_avx512_vpdpbusd_512; in upgradeX86IntrinsicCall()
3853 IID = Intrinsic::x86_avx512_vpdpbusds_128; in upgradeX86IntrinsicCall()
3855 IID = Intrinsic::x86_avx512_vpdpbusds_256; in upgradeX86IntrinsicCall()
3857 IID = Intrinsic::x86_avx512_vpdpbusds_512; in upgradeX86IntrinsicCall()
3863 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3875 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3877 IID = Intrinsic::x86_avx512_vpdpwssd_128; in upgradeX86IntrinsicCall()
3879 IID = Intrinsic::x86_avx512_vpdpwssd_256; in upgradeX86IntrinsicCall()
3881 IID = Intrinsic::x86_avx512_vpdpwssd_512; in upgradeX86IntrinsicCall()
3883 IID = Intrinsic::x86_avx512_vpdpwssds_128; in upgradeX86IntrinsicCall()
3885 IID = Intrinsic::x86_avx512_vpdpwssds_256; in upgradeX86IntrinsicCall()
3887 IID = Intrinsic::x86_avx512_vpdpwssds_512; in upgradeX86IntrinsicCall()
3893 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3901 Intrinsic::ID IID; in upgradeX86IntrinsicCall()
3903 IID = Intrinsic::x86_addcarry_32; in upgradeX86IntrinsicCall()
3905 IID = Intrinsic::x86_addcarry_64; in upgradeX86IntrinsicCall()
3907 IID = Intrinsic::x86_subborrow_32; in upgradeX86IntrinsicCall()
3909 IID = Intrinsic::x86_subborrow_64; in upgradeX86IntrinsicCall()
3917 Intrinsic::getDeclaration(CI->getModule(), IID), Args); in upgradeX86IntrinsicCall()
3944 Intrinsic::getDeclaration(F->getParent(), Intrinsic::arm_mve_vctp64), in upgradeARMIntrinsicCall()
3947 Intrinsic::getDeclaration( in upgradeARMIntrinsicCall()
3948 F->getParent(), Intrinsic::arm_mve_pred_v2i, in upgradeARMIntrinsicCall()
3952 Intrinsic::getDeclaration( in upgradeARMIntrinsicCall()
3953 F->getParent(), Intrinsic::arm_mve_pred_i2v, in upgradeARMIntrinsicCall()
3978 case Intrinsic::arm_mve_mull_int_predicated: in upgradeARMIntrinsicCall()
3979 case Intrinsic::arm_mve_vqdmull_predicated: in upgradeARMIntrinsicCall()
3980 case Intrinsic::arm_mve_vldr_gather_base_predicated: in upgradeARMIntrinsicCall()
3983 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: in upgradeARMIntrinsicCall()
3984 case Intrinsic::arm_mve_vstr_scatter_base_predicated: in upgradeARMIntrinsicCall()
3985 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: in upgradeARMIntrinsicCall()
3989 case Intrinsic::arm_mve_vldr_gather_offset_predicated: in upgradeARMIntrinsicCall()
3993 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: in upgradeARMIntrinsicCall()
3997 case Intrinsic::arm_cde_vcx1q_predicated: in upgradeARMIntrinsicCall()
3998 case Intrinsic::arm_cde_vcx1qa_predicated: in upgradeARMIntrinsicCall()
3999 case Intrinsic::arm_cde_vcx2q_predicated: in upgradeARMIntrinsicCall()
4000 case Intrinsic::arm_cde_vcx2qa_predicated: in upgradeARMIntrinsicCall()
4001 case Intrinsic::arm_cde_vcx3q_predicated: in upgradeARMIntrinsicCall()
4002 case Intrinsic::arm_cde_vcx3qa_predicated: in upgradeARMIntrinsicCall()
4014 Intrinsic::getDeclaration( in upgradeARMIntrinsicCall()
4015 F->getParent(), Intrinsic::arm_mve_pred_v2i, in upgradeARMIntrinsicCall()
4019 Intrinsic::getDeclaration(F->getParent(), in upgradeARMIntrinsicCall()
4020 Intrinsic::arm_mve_pred_i2v, {V2I1Ty}), in upgradeARMIntrinsicCall()
4026 Function *Fn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); in upgradeARMIntrinsicCall()
4228 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, in UpgradeIntrinsicCall()
4237 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, in UpgradeIntrinsicCall()
4244 Builder.CreateCall(Intrinsic::getDeclaration( in UpgradeIntrinsicCall()
4245 F->getParent(), Intrinsic::convert_from_fp16, in UpgradeIntrinsicCall()
4249 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name); in UpgradeIntrinsicCall()
4250 if (IID != Intrinsic::not_intrinsic && in UpgradeIntrinsicCall()
4253 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in UpgradeIntrinsicCall()
4343 case Intrinsic::arm_neon_vst1: in UpgradeIntrinsicCall()
4344 case Intrinsic::arm_neon_vst2: in UpgradeIntrinsicCall()
4345 case Intrinsic::arm_neon_vst3: in UpgradeIntrinsicCall()
4346 case Intrinsic::arm_neon_vst4: in UpgradeIntrinsicCall()
4347 case Intrinsic::arm_neon_vst2lane: in UpgradeIntrinsicCall()
4348 case Intrinsic::arm_neon_vst3lane: in UpgradeIntrinsicCall()
4349 case Intrinsic::arm_neon_vst4lane: { in UpgradeIntrinsicCall()
4354 case Intrinsic::aarch64_sve_bfmlalb_lane_v2: in UpgradeIntrinsicCall()
4355 case Intrinsic::aarch64_sve_bfmlalt_lane_v2: in UpgradeIntrinsicCall()
4356 case Intrinsic::aarch64_sve_bfdot_lane_v2: { in UpgradeIntrinsicCall()
4364 case Intrinsic::aarch64_sve_ld3_sret: in UpgradeIntrinsicCall()
4365 case Intrinsic::aarch64_sve_ld4_sret: in UpgradeIntrinsicCall()
4366 case Intrinsic::aarch64_sve_ld2_sret: { in UpgradeIntrinsicCall()
4388 case Intrinsic::coro_end: { in UpgradeIntrinsicCall()
4395 case Intrinsic::vector_extract: { in UpgradeIntrinsicCall()
4410 case Intrinsic::vector_insert: { in UpgradeIntrinsicCall()
4446 case Intrinsic::arm_neon_bfdot: in UpgradeIntrinsicCall()
4447 case Intrinsic::arm_neon_bfmmla: in UpgradeIntrinsicCall()
4448 case Intrinsic::arm_neon_bfmlalb: in UpgradeIntrinsicCall()
4449 case Intrinsic::arm_neon_bfmlalt: in UpgradeIntrinsicCall()
4450 case Intrinsic::aarch64_neon_bfdot: in UpgradeIntrinsicCall()
4451 case Intrinsic::aarch64_neon_bfmmla: in UpgradeIntrinsicCall()
4452 case Intrinsic::aarch64_neon_bfmlalb: in UpgradeIntrinsicCall()
4453 case Intrinsic::aarch64_neon_bfmlalt: { in UpgradeIntrinsicCall()
4470 case Intrinsic::bitreverse: in UpgradeIntrinsicCall()
4474 case Intrinsic::ctlz: in UpgradeIntrinsicCall()
4475 case Intrinsic::cttz: in UpgradeIntrinsicCall()
4482 case Intrinsic::objectsize: { in UpgradeIntrinsicCall()
4492 case Intrinsic::ctpop: in UpgradeIntrinsicCall()
4496 case Intrinsic::convert_from_fp16: in UpgradeIntrinsicCall()
4500 case Intrinsic::dbg_value: { in UpgradeIntrinsicCall()
4528 case Intrinsic::ptr_annotation: in UpgradeIntrinsicCall()
4545 case Intrinsic::var_annotation: in UpgradeIntrinsicCall()
4561 case Intrinsic::riscv_aes32dsi: in UpgradeIntrinsicCall()
4562 case Intrinsic::riscv_aes32dsmi: in UpgradeIntrinsicCall()
4563 case Intrinsic::riscv_aes32esi: in UpgradeIntrinsicCall()
4564 case Intrinsic::riscv_aes32esmi: in UpgradeIntrinsicCall()
4565 case Intrinsic::riscv_sm4ks: in UpgradeIntrinsicCall()
4566 case Intrinsic::riscv_sm4ed: { in UpgradeIntrinsicCall()
4592 case Intrinsic::riscv_sha256sig0: in UpgradeIntrinsicCall()
4593 case Intrinsic::riscv_sha256sig1: in UpgradeIntrinsicCall()
4594 case Intrinsic::riscv_sha256sum0: in UpgradeIntrinsicCall()
4595 case Intrinsic::riscv_sha256sum1: in UpgradeIntrinsicCall()
4596 case Intrinsic::riscv_sm3p0: in UpgradeIntrinsicCall()
4597 case Intrinsic::riscv_sm3p1: { in UpgradeIntrinsicCall()
4615 case Intrinsic::x86_xop_vfrcz_ss: in UpgradeIntrinsicCall()
4616 case Intrinsic::x86_xop_vfrcz_sd: in UpgradeIntrinsicCall()
4620 case Intrinsic::x86_xop_vpermil2pd: in UpgradeIntrinsicCall()
4621 case Intrinsic::x86_xop_vpermil2ps: in UpgradeIntrinsicCall()
4622 case Intrinsic::x86_xop_vpermil2pd_256: in UpgradeIntrinsicCall()
4623 case Intrinsic::x86_xop_vpermil2ps_256: { in UpgradeIntrinsicCall()
4632 case Intrinsic::x86_sse41_ptestc: in UpgradeIntrinsicCall()
4633 case Intrinsic::x86_sse41_ptestz: in UpgradeIntrinsicCall()
4634 case Intrinsic::x86_sse41_ptestnzc: { in UpgradeIntrinsicCall()
4655 case Intrinsic::x86_rdtscp: { in UpgradeIntrinsicCall()
4677 case Intrinsic::x86_sse41_insertps: in UpgradeIntrinsicCall()
4678 case Intrinsic::x86_sse41_dppd: in UpgradeIntrinsicCall()
4679 case Intrinsic::x86_sse41_dpps: in UpgradeIntrinsicCall()
4680 case Intrinsic::x86_sse41_mpsadbw: in UpgradeIntrinsicCall()
4681 case Intrinsic::x86_avx_dp_ps_256: in UpgradeIntrinsicCall()
4682 case Intrinsic::x86_avx2_mpsadbw: { in UpgradeIntrinsicCall()
4693 case Intrinsic::x86_avx512_mask_cmp_pd_128: in UpgradeIntrinsicCall()
4694 case Intrinsic::x86_avx512_mask_cmp_pd_256: in UpgradeIntrinsicCall()
4695 case Intrinsic::x86_avx512_mask_cmp_pd_512: in UpgradeIntrinsicCall()
4696 case Intrinsic::x86_avx512_mask_cmp_ps_128: in UpgradeIntrinsicCall()
4697 case Intrinsic::x86_avx512_mask_cmp_ps_256: in UpgradeIntrinsicCall()
4698 case Intrinsic::x86_avx512_mask_cmp_ps_512: { in UpgradeIntrinsicCall()
4713 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128: in UpgradeIntrinsicCall()
4714 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256: in UpgradeIntrinsicCall()
4715 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512: in UpgradeIntrinsicCall()
4716 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128: in UpgradeIntrinsicCall()
4717 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256: in UpgradeIntrinsicCall()
4718 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: { in UpgradeIntrinsicCall()
4722 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128) in UpgradeIntrinsicCall()
4735 case Intrinsic::x86_avx512bf16_dpbf16ps_128: in UpgradeIntrinsicCall()
4736 case Intrinsic::x86_avx512bf16_dpbf16ps_256: in UpgradeIntrinsicCall()
4737 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{ in UpgradeIntrinsicCall()
4750 case Intrinsic::thread_pointer: { in UpgradeIntrinsicCall()
4755 case Intrinsic::memcpy: in UpgradeIntrinsicCall()
4756 case Intrinsic::memmove: in UpgradeIntrinsicCall()
4757 case Intrinsic::memset: { in UpgradeIntrinsicCall()
4942 llvm::Intrinsic::ID IntrinsicFunc) { in UpgradeARCRuntime()
4948 Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc); in UpgradeARCRuntime()
5008 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use); in UpgradeARCRuntime()
5016 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = { in UpgradeARCRuntime()
5017 {"objc_autorelease", llvm::Intrinsic::objc_autorelease}, in UpgradeARCRuntime()
5018 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop}, in UpgradeARCRuntime()
5019 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush}, in UpgradeARCRuntime()
5021 llvm::Intrinsic::objc_autoreleaseReturnValue}, in UpgradeARCRuntime()
5022 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak}, in UpgradeARCRuntime()
5023 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak}, in UpgradeARCRuntime()
5024 {"objc_initWeak", llvm::Intrinsic::objc_initWeak}, in UpgradeARCRuntime()
5025 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak}, in UpgradeARCRuntime()
5026 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained}, in UpgradeARCRuntime()
5027 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak}, in UpgradeARCRuntime()
5028 {"objc_release", llvm::Intrinsic::objc_release}, in UpgradeARCRuntime()
5029 {"objc_retain", llvm::Intrinsic::objc_retain}, in UpgradeARCRuntime()
5030 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease}, in UpgradeARCRuntime()
5032 llvm::Intrinsic::objc_retainAutoreleaseReturnValue}, in UpgradeARCRuntime()
5034 llvm::Intrinsic::objc_retainAutoreleasedReturnValue}, in UpgradeARCRuntime()
5035 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock}, in UpgradeARCRuntime()
5036 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong}, in UpgradeARCRuntime()
5037 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak}, in UpgradeARCRuntime()
5039 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue}, in UpgradeARCRuntime()
5040 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject}, in UpgradeARCRuntime()
5041 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject}, in UpgradeARCRuntime()
5042 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer}, in UpgradeARCRuntime()
5043 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease}, in UpgradeARCRuntime()
5044 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter}, in UpgradeARCRuntime()
5045 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit}, in UpgradeARCRuntime()
5047 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart}, in UpgradeARCRuntime()
5049 llvm::Intrinsic::objc_arc_annotation_topdown_bbend}, in UpgradeARCRuntime()
5051 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart}, in UpgradeARCRuntime()
5053 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}}; in UpgradeARCRuntime()