Lines Matching full:ci
1653 static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, in upgradeX86VPERMT2Intrinsics() argument
1655 Type *Ty = CI.getType(); in upgradeX86VPERMT2Intrinsics()
1699 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1), in upgradeX86VPERMT2Intrinsics()
1700 CI.getArgOperand(2) }; in upgradeX86VPERMT2Intrinsics()
1706 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in upgradeX86VPERMT2Intrinsics()
1709 : Builder.CreateBitCast(CI.getArgOperand(1), in upgradeX86VPERMT2Intrinsics()
1711 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru); in upgradeX86VPERMT2Intrinsics()
1714 static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, in upgradeX86BinaryIntrinsics() argument
1716 Type *Ty = CI.getType(); in upgradeX86BinaryIntrinsics()
1717 Value *Op0 = CI.getOperand(0); in upgradeX86BinaryIntrinsics()
1718 Value *Op1 = CI.getOperand(1); in upgradeX86BinaryIntrinsics()
1719 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86BinaryIntrinsics()
1722 if (CI.arg_size() == 4) { // For masked intrinsics. in upgradeX86BinaryIntrinsics()
1723 Value *VecSrc = CI.getOperand(2); in upgradeX86BinaryIntrinsics()
1724 Value *Mask = CI.getOperand(3); in upgradeX86BinaryIntrinsics()
1730 static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, in upgradeX86Rotate() argument
1732 Type *Ty = CI.getType(); in upgradeX86Rotate()
1733 Value *Src = CI.getArgOperand(0); in upgradeX86Rotate()
1734 Value *Amt = CI.getArgOperand(1); in upgradeX86Rotate()
1746 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86Rotate()
1749 if (CI.arg_size() == 4) { // For masked intrinsics. in upgradeX86Rotate()
1750 Value *VecSrc = CI.getOperand(2); in upgradeX86Rotate()
1751 Value *Mask = CI.getOperand(3); in upgradeX86Rotate()
1757 static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, in upgradeX86vpcom() argument
1759 Type *Ty = CI.getType(); in upgradeX86vpcom()
1760 Value *LHS = CI.getArgOperand(0); in upgradeX86vpcom()
1761 Value *RHS = CI.getArgOperand(1); in upgradeX86vpcom()
1796 static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, in upgradeX86ConcatShift() argument
1798 Type *Ty = CI.getType(); in upgradeX86ConcatShift()
1799 Value *Op0 = CI.getArgOperand(0); in upgradeX86ConcatShift()
1800 Value *Op1 = CI.getArgOperand(1); in upgradeX86ConcatShift()
1801 Value *Amt = CI.getArgOperand(2); in upgradeX86ConcatShift()
1816 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86ConcatShift()
1819 unsigned NumArgs = CI.arg_size(); in upgradeX86ConcatShift()
1821 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) : in upgradeX86ConcatShift()
1822 ZeroMask ? ConstantAggregateZero::get(CI.getType()) : in upgradeX86ConcatShift()
1823 CI.getArgOperand(0); in upgradeX86ConcatShift()
1824 Value *Mask = CI.getOperand(NumArgs - 1); in upgradeX86ConcatShift()
1874 static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) { in upgradeAbs() argument
1875 Type *Ty = CI.getType(); in upgradeAbs()
1876 Value *Op0 = CI.getArgOperand(0); in upgradeAbs()
1877 Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty); in upgradeAbs()
1879 if (CI.arg_size() == 3) in upgradeAbs()
1880 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1)); in upgradeAbs()
1884 static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) { in upgradePMULDQ() argument
1885 Type *Ty = CI.getType(); in upgradePMULDQ()
1888 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty); in upgradePMULDQ()
1889 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty); in upgradePMULDQ()
1907 if (CI.arg_size() == 4) in upgradePMULDQ()
1908 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2)); in upgradePMULDQ()
1936 static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, in upgradeMaskedCompare() argument
1938 Value *Op0 = CI.getArgOperand(0); in upgradeMaskedCompare()
1959 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1)); in upgradeMaskedCompare()
1962 Value *Mask = CI.getArgOperand(CI.arg_size() - 1); in upgradeMaskedCompare()
1968 static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, in upgradeX86MaskedShift() argument
1970 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID); in upgradeX86MaskedShift()
1972 { CI.getArgOperand(0), CI.getArgOperand(1) }); in upgradeX86MaskedShift()
1973 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2)); in upgradeX86MaskedShift()
1976 static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) { in upgradeMaskedMove() argument
1977 Value* A = CI.getArgOperand(0); in upgradeMaskedMove()
1978 Value* B = CI.getArgOperand(1); in upgradeMaskedMove()
1979 Value* Src = CI.getArgOperand(2); in upgradeMaskedMove()
1980 Value* Mask = CI.getArgOperand(3); in upgradeMaskedMove()
1990 static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) { in upgradeMaskToInt() argument
1991 Value* Op = CI.getArgOperand(0); in upgradeMaskToInt()
1992 Type* ReturnOp = CI.getType(); in upgradeMaskToInt()
1993 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements(); in upgradeMaskToInt()
2000 CallBase &CI, Value *&Rep) { in upgradeAVX512MaskToSelect() argument
2003 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits(); in upgradeAVX512MaskToSelect()
2004 unsigned EltWidth = CI.getType()->getScalarSizeInBits(); in upgradeAVX512MaskToSelect()
2144 bool IsFloat = CI.getType()->isFPOrFPVectorTy(); in upgradeAVX512MaskToSelect()
2226 SmallVector<Value *, 4> Args(CI.args()); in upgradeAVX512MaskToSelect()
2229 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in upgradeAVX512MaskToSelect()
2231 unsigned NumArgs = CI.arg_size(); in upgradeAVX512MaskToSelect()
2232 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep, in upgradeAVX512MaskToSelect()
2233 CI.getArgOperand(NumArgs - 2)); in upgradeAVX512MaskToSelect()
2248 static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, in upgradeX86IntrinsicCall() argument
2259 Value *Arg0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2260 Value *Arg1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2279 Value *Arg0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2280 Value *Arg1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2290 Value *Arg0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2291 Value *Arg1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2302 Value *Arg0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2303 Value *Arg1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2309 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1)); in upgradeX86IntrinsicCall()
2310 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2315 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2316 CI->getArgOperand(2), Aligned); in upgradeX86IntrinsicCall()
2322 CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2323 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); in upgradeX86IntrinsicCall()
2326 if (CI->getOperand(0)->getType()->isIntegerTy(8)) in upgradeX86IntrinsicCall()
2328 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / in upgradeX86IntrinsicCall()
2330 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy); in upgradeX86IntrinsicCall()
2333 Value *Vec = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2344 F->getParent(), Intrinsic::sqrt, CI->getType()), in upgradeX86IntrinsicCall()
2345 {CI->getArgOperand(0)}); in upgradeX86IntrinsicCall()
2347 if (CI->arg_size() == 4 && in upgradeX86IntrinsicCall()
2348 (!isa<ConstantInt>(CI->getArgOperand(3)) || in upgradeX86IntrinsicCall()
2349 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { in upgradeX86IntrinsicCall()
2353 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)}; in upgradeX86IntrinsicCall()
2354 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
2359 CI->getType()), in upgradeX86IntrinsicCall()
2360 {CI->getArgOperand(0)}); in upgradeX86IntrinsicCall()
2363 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2366 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2367 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2368 Value *Mask = CI->getArgOperand(2); in upgradeX86IntrinsicCall()
2378 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType()) in upgradeX86IntrinsicCall()
2380 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0)); in upgradeX86IntrinsicCall()
2382 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2384 unsigned NumElts = CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
2385 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts); in upgradeX86IntrinsicCall()
2386 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts); in upgradeX86IntrinsicCall()
2398 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2400 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2401 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2403 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2405 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2406 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2409 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2411 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2412 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2414 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2416 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2417 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2419 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2421 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2422 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2425 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2427 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2429 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2431 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2432 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2447 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2449 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2460 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp, in upgradeX86IntrinsicCall()
2465 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true); in upgradeX86IntrinsicCall()
2467 Type *OpTy = CI->getArgOperand(0)->getType(); in upgradeX86IntrinsicCall()
2485 {CI->getOperand(0), CI->getArgOperand(1)}); in upgradeX86IntrinsicCall()
2486 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
2488 Type *OpTy = CI->getArgOperand(0)->getType(); in upgradeX86IntrinsicCall()
2508 {CI->getOperand(0), CI->getArgOperand(1)}); in upgradeX86IntrinsicCall()
2509 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
2511 SmallVector<Value *, 4> Args(CI->args()); in upgradeX86IntrinsicCall()
2531 Value *Mask = Constant::getAllOnesValue(CI->getType()); in upgradeX86IntrinsicCall()
2540 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2541 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true); in upgradeX86IntrinsicCall()
2543 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2544 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false); in upgradeX86IntrinsicCall()
2549 Value *Op = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2556 Rep = upgradeAbs(Builder, *CI); in upgradeX86IntrinsicCall()
2560 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax); in upgradeX86IntrinsicCall()
2564 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax); in upgradeX86IntrinsicCall()
2568 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin); in upgradeX86IntrinsicCall()
2572 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin); in upgradeX86IntrinsicCall()
2576 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false); in upgradeX86IntrinsicCall()
2580 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true); in upgradeX86IntrinsicCall()
2584 Builder.CreateSIToFP(CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2585 cast<VectorType>(CI->getType())->getElementType()); in upgradeX86IntrinsicCall()
2586 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in upgradeX86IntrinsicCall()
2589 Builder.CreateUIToFP(CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2590 cast<VectorType>(CI->getType())->getElementType()); in upgradeX86IntrinsicCall()
2591 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in upgradeX86IntrinsicCall()
2593 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0); in upgradeX86IntrinsicCall()
2595 Rep, cast<VectorType>(CI->getType())->getElementType()); in upgradeX86IntrinsicCall()
2596 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in upgradeX86IntrinsicCall()
2612 auto *DstTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2613 Rep = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2626 else if (CI->arg_size() == 4 && in upgradeX86IntrinsicCall()
2627 (!isa<ConstantInt>(CI->getArgOperand(3)) || in upgradeX86IntrinsicCall()
2628 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { in upgradeX86IntrinsicCall()
2632 Intrinsic::getDeclaration(CI->getModule(), IID, {DstTy, SrcTy}); in upgradeX86IntrinsicCall()
2633 Rep = Builder.CreateCall(F, {Rep, CI->getArgOperand(3)}); in upgradeX86IntrinsicCall()
2639 if (CI->arg_size() >= 3) in upgradeX86IntrinsicCall()
2640 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in upgradeX86IntrinsicCall()
2641 CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2644 auto *DstTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2645 Rep = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2655 if (CI->arg_size() >= 3) in upgradeX86IntrinsicCall()
2656 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in upgradeX86IntrinsicCall()
2657 CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2661 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2662 CI->getArgOperand(2), Aligned); in upgradeX86IntrinsicCall()
2664 auto *ResultTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2668 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), in upgradeX86IntrinsicCall()
2671 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), in upgradeX86IntrinsicCall()
2676 Rep = Builder.CreateCall(ELd, {Ptr, MaskVec, CI->getOperand(1)}); in upgradeX86IntrinsicCall()
2678 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType()); in upgradeX86IntrinsicCall()
2682 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), in upgradeX86IntrinsicCall()
2686 getX86MaskVec(Builder, CI->getArgOperand(2), in upgradeX86IntrinsicCall()
2691 Rep = Builder.CreateCall(CSt, {CI->getArgOperand(1), Ptr, MaskVec}); in upgradeX86IntrinsicCall()
2694 auto *ResultTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2696 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), in upgradeX86IntrinsicCall()
2704 {CI->getOperand(0), CI->getOperand(1), MaskVec}); in upgradeX86IntrinsicCall()
2717 if (CI->arg_size() == 3) { in upgradeX86IntrinsicCall()
2718 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2741 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned); in upgradeX86IntrinsicCall()
2743 Value *Sel = CI->getArgOperand(2); in upgradeX86IntrinsicCall()
2745 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel); in upgradeX86IntrinsicCall()
2746 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel); in upgradeX86IntrinsicCall()
2750 Rep = upgradeX86Rotate(Builder, *CI, false); in upgradeX86IntrinsicCall()
2753 Rep = upgradeX86Rotate(Builder, *CI, true); in upgradeX86IntrinsicCall()
2758 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask); in upgradeX86IntrinsicCall()
2763 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask); in upgradeX86IntrinsicCall()
2768 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C)); in upgradeX86IntrinsicCall()
2769 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)}); in upgradeX86IntrinsicCall()
2770 Rep = Builder.CreateZExt(Rep, CI->getType(), ""); in upgradeX86IntrinsicCall()
2774 auto *VecTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2777 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0)); in upgradeX86IntrinsicCall()
2788 auto *DstTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2796 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask); in upgradeX86IntrinsicCall()
2802 if (CI->arg_size() == 3) in upgradeX86IntrinsicCall()
2803 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in upgradeX86IntrinsicCall()
2804 CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2809 Type *Ty = CI->getArgOperand(1)->getType(); in upgradeX86IntrinsicCall()
2810 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty); in upgradeX86IntrinsicCall()
2812 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2816 Type *EltTy = cast<VectorType>(CI->getType())->getElementType(); in upgradeX86IntrinsicCall()
2819 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2829 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2830 Type *VT = CI->getType(); in upgradeX86IntrinsicCall()
2845 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2846 CI->getArgOperand(1), ShuffleMask); in upgradeX86IntrinsicCall()
2848 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3)); in upgradeX86IntrinsicCall()
2851 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType()) in upgradeX86IntrinsicCall()
2854 cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
2860 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2861 CI->getArgOperand(0), ShuffleMask); in upgradeX86IntrinsicCall()
2863 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2869 Value *Op = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2870 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount(); in upgradeX86IntrinsicCall()
2876 if (CI->arg_size() == 3) in upgradeX86IntrinsicCall()
2877 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in upgradeX86IntrinsicCall()
2878 CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2883 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat); in upgradeX86IntrinsicCall()
2888 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat); in upgradeX86IntrinsicCall()
2892 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat); in upgradeX86IntrinsicCall()
2896 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat); in upgradeX86IntrinsicCall()
2898 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2899 CI->getArgOperand(1), CI->getArgOperand(2), in upgradeX86IntrinsicCall()
2900 CI->getArgOperand(3), CI->getArgOperand(4), in upgradeX86IntrinsicCall()
2904 Builder, CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2905 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true); in upgradeX86IntrinsicCall()
2908 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
2909 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2913 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
2914 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2919 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
2920 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); in upgradeX86IntrinsicCall()
2924 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
2925 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); in upgradeX86IntrinsicCall()
2929 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2930 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2931 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2932 auto *VecTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2943 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2944 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2945 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2947 cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
2983 if (CI->arg_size() == 5) in upgradeX86IntrinsicCall()
2984 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, in upgradeX86IntrinsicCall()
2985 CI->getArgOperand(3)); in upgradeX86IntrinsicCall()
2989 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2990 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
2992 cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3008 if (CI->arg_size() == 4) in upgradeX86IntrinsicCall()
3009 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in upgradeX86IntrinsicCall()
3010 CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3013 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3014 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
3015 auto *VecTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3024 if (CI->arg_size() == 4) in upgradeX86IntrinsicCall()
3025 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in upgradeX86IntrinsicCall()
3026 CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3036 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
3038 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3043 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3044 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3047 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0; in upgradeX86IntrinsicCall()
3048 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1; in upgradeX86IntrinsicCall()
3065 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3066 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
3067 auto *VecTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3082 if (CI->arg_size() == 4) in upgradeX86IntrinsicCall()
3083 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in upgradeX86IntrinsicCall()
3084 CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3087 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3088 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
3089 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3101 if (CI->arg_size() == 4) in upgradeX86IntrinsicCall()
3102 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in upgradeX86IntrinsicCall()
3103 CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3106 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3107 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
3108 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3120 if (CI->arg_size() == 4) in upgradeX86IntrinsicCall()
3121 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in upgradeX86IntrinsicCall()
3122 CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3124 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3125 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
3126 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
3127 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3129 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3147 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3)); in upgradeX86IntrinsicCall()
3151 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3152 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3153 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3169 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3172 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3173 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
3174 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3175 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3185 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3188 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3189 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
3190 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3191 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3201 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3204 VectorType *FTy = cast<VectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3206 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in upgradeX86IntrinsicCall()
3207 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in upgradeX86IntrinsicCall()
3210 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3213 VectorType *FTy = cast<VectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3215 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy)); in upgradeX86IntrinsicCall()
3217 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in upgradeX86IntrinsicCall()
3220 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3223 VectorType *FTy = cast<VectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3225 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in upgradeX86IntrinsicCall()
3226 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in upgradeX86IntrinsicCall()
3229 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3232 VectorType *FTy = cast<VectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3234 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in upgradeX86IntrinsicCall()
3235 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in upgradeX86IntrinsicCall()
3238 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3240 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3242 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3244 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3246 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3248 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3250 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3261 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3263 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3266 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3277 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3279 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3282 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3293 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3295 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3298 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3309 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3311 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3314 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3327 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3329 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3333 F->getParent(), Intrinsic::ctlz, CI->getType()), in upgradeX86IntrinsicCall()
3334 {CI->getArgOperand(0), Builder.getInt1(false)}); in upgradeX86IntrinsicCall()
3336 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3403 Rep = upgradeX86MaskedShift(Builder, *CI, IID); in upgradeX86IntrinsicCall()
3470 Rep = upgradeX86MaskedShift(Builder, *CI, IID); in upgradeX86IntrinsicCall()
3535 Rep = upgradeX86MaskedShift(Builder, *CI, IID); in upgradeX86IntrinsicCall()
3537 Rep = upgradeMaskedMove(Builder, *CI); in upgradeX86IntrinsicCall()
3539 Rep = upgradeMaskToInt(Builder, *CI); in upgradeX86IntrinsicCall()
3544 Value *Ptr = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3548 Ptr, PointerType::getUnqual(CI->getType()), "cast"); in upgradeX86IntrinsicCall()
3550 CI->getType(), BC, in upgradeX86IntrinsicCall()
3551 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)); in upgradeX86IntrinsicCall()
3562 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3563 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3578 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in upgradeX86IntrinsicCall()
3584 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in upgradeX86IntrinsicCall()
3586 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3587 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3593 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in upgradeX86IntrinsicCall()
3598 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()), in upgradeX86IntrinsicCall()
3612 Value *A = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3613 Value *B = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
3614 Value *C = CI->getArgOperand(2); in upgradeX86IntrinsicCall()
3627 if (!isa<ConstantInt>(CI->getArgOperand(4)) || in upgradeX86IntrinsicCall()
3628 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) { in upgradeX86IntrinsicCall()
3629 Value *Ops[] = {A, B, C, CI->getArgOperand(4)}; in upgradeX86IntrinsicCall()
3636 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID); in upgradeX86IntrinsicCall()
3639 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in upgradeX86IntrinsicCall()
3652 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0); in upgradeX86IntrinsicCall()
3654 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3655 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep, in upgradeX86IntrinsicCall()
3671 Value *A = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3672 Value *B = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
3673 Value *C = CI->getArgOperand(2); in upgradeX86IntrinsicCall()
3682 if (CI->arg_size() == 5 && in upgradeX86IntrinsicCall()
3683 (!isa<ConstantInt>(CI->getArgOperand(4)) || in upgradeX86IntrinsicCall()
3684 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) { in upgradeX86IntrinsicCall()
3693 {A, B, C, CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3695 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in upgradeX86IntrinsicCall()
3700 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) in upgradeX86IntrinsicCall()
3701 : IsMask3 ? CI->getArgOperand(2) in upgradeX86IntrinsicCall()
3702 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3704 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3706 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
3707 unsigned EltWidth = CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3720 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3721 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3734 if (CI->arg_size() == 5) { in upgradeX86IntrinsicCall()
3742 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3743 CI->getArgOperand(2), CI->getArgOperand(4)}; in upgradeX86IntrinsicCall()
3750 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3752 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3753 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3755 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in upgradeX86IntrinsicCall()
3771 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) in upgradeX86IntrinsicCall()
3772 : IsMask3 ? CI->getArgOperand(2) in upgradeX86IntrinsicCall()
3773 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3775 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3779 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
3780 unsigned EltWidth = CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3797 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3798 CI->getArgOperand(2), CI->getArgOperand(3)}; in upgradeX86IntrinsicCall()
3799 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3801 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in upgradeX86IntrinsicCall()
3802 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3803 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru); in upgradeX86IntrinsicCall()
3808 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
3825 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3826 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3827 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3829 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in upgradeX86IntrinsicCall()
3830 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3831 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3837 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm); in upgradeX86IntrinsicCall()
3844 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
3861 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3862 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3863 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3865 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in upgradeX86IntrinsicCall()
3866 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3867 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3874 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
3891 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3892 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3893 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3895 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in upgradeX86IntrinsicCall()
3896 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3897 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3914 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3915 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3917 Intrinsic::getDeclaration(CI->getModule(), IID), Args); in upgradeX86IntrinsicCall()
3923 CI->getArgOperand(3), llvm::PointerType::getUnqual(Data->getType())); in upgradeX86IntrinsicCall()
3928 CI->replaceAllUsesWith(CF); in upgradeX86IntrinsicCall()
3931 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) { in upgradeX86IntrinsicCall()
3938 static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, in upgradeARMIntrinsicCall() argument
3945 CI->getArgOperand(0), CI->getName()); in upgradeARMIntrinsicCall()
3975 unsigned ID = CI->getIntrinsicID(); in upgradeARMIntrinsicCall()
3981 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
3986 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(), in upgradeARMIntrinsicCall()
3990 Tys = {CI->getType(), CI->getOperand(0)->getType(), in upgradeARMIntrinsicCall()
3991 CI->getOperand(1)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
3994 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(), in upgradeARMIntrinsicCall()
3995 CI->getOperand(2)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
4003 Tys = {CI->getOperand(1)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
4010 for (Value *Op : CI->args()) { in upgradeARMIntrinsicCall()
4027 return Builder.CreateCall(Fn, Ops, CI->getName()); in upgradeARMIntrinsicCall()
4037 static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, in upgradeAMDGCNIntrinsicCall() argument
4047 unsigned NumOperands = CI->getNumOperands(); in upgradeAMDGCNIntrinsicCall()
4051 Value *Ptr = CI->getArgOperand(0); in upgradeAMDGCNIntrinsicCall()
4056 Value *Val = CI->getArgOperand(1); in upgradeAMDGCNIntrinsicCall()
4057 if (Val->getType() != CI->getType()) // Malformed. in upgradeAMDGCNIntrinsicCall()
4066 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2)); in upgradeAMDGCNIntrinsicCall()
4071 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4)); in upgradeAMDGCNIntrinsicCall()
4084 Type *RetTy = CI->getType(); in upgradeAMDGCNIntrinsicCall()
4112 static MDType *unwrapMAVOp(CallBase *CI, unsigned Op) { in unwrapMAVOp() argument
4113 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) in unwrapMAVOp()
4120 /// \p CI - The debug intrinsic call.
4121 static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) { in upgradeDbgIntrinsicToDbgRecord() argument
4124 DR = new DbgLabelRecord(unwrapMAVOp<DILabel>(CI, 0), CI->getDebugLoc()); in upgradeDbgIntrinsicToDbgRecord()
4127 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1), in upgradeDbgIntrinsicToDbgRecord()
4128 unwrapMAVOp<DIExpression>(CI, 2), unwrapMAVOp<DIAssignID>(CI, 3), in upgradeDbgIntrinsicToDbgRecord()
4129 unwrapMAVOp<Metadata>(CI, 4), unwrapMAVOp<DIExpression>(CI, 5), in upgradeDbgIntrinsicToDbgRecord()
4130 CI->getDebugLoc()); in upgradeDbgIntrinsicToDbgRecord()
4133 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1), in upgradeDbgIntrinsicToDbgRecord()
4134 unwrapMAVOp<DIExpression>(CI, 2), CI->getDebugLoc(), in upgradeDbgIntrinsicToDbgRecord()
4138 DIExpression *Expr = unwrapMAVOp<DIExpression>(CI, 2); in upgradeDbgIntrinsicToDbgRecord()
4140 DR = new DbgVariableRecord(unwrapMAVOp<Metadata>(CI, 0), in upgradeDbgIntrinsicToDbgRecord()
4141 unwrapMAVOp<DILocalVariable>(CI, 1), Expr, in upgradeDbgIntrinsicToDbgRecord()
4142 CI->getDebugLoc()); in upgradeDbgIntrinsicToDbgRecord()
4147 if (CI->arg_size() == 4) { in upgradeDbgIntrinsicToDbgRecord()
4148 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)); in upgradeDbgIntrinsicToDbgRecord()
4156 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, VarOp), in upgradeDbgIntrinsicToDbgRecord()
4157 unwrapMAVOp<DIExpression>(CI, ExprOp), CI->getDebugLoc()); in upgradeDbgIntrinsicToDbgRecord()
4160 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator()); in upgradeDbgIntrinsicToDbgRecord()
4165 void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { in UpgradeIntrinsicCall() argument
4169 Function *F = dyn_cast<Function>(CI->getCalledOperand()); in UpgradeIntrinsicCall()
4173 LLVMContext &C = CI->getContext(); in UpgradeIntrinsicCall()
4175 Builder.SetInsertPoint(CI->getParent(), CI->getIterator()); in UpgradeIntrinsicCall()
4195 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4202 Value *Ptr = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4203 Value *Val = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4209 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4210 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4218 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4219 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4226 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4235 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4247 CI->getArgOperand(0), "h2f"); in UpgradeIntrinsicCall()
4256 Value *Arg = CI->getArgOperand(I); in UpgradeIntrinsicCall()
4270 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder); in UpgradeIntrinsicCall()
4272 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder); in UpgradeIntrinsicCall()
4274 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder); in UpgradeIntrinsicCall()
4280 if (!CI->getModule()->IsNewDbgInfoFormat) { in UpgradeIntrinsicCall()
4282 upgradeIntrinsicFunction1(CI->getCalledFunction(), NewFn, false); in UpgradeIntrinsicCall()
4287 upgradeDbgIntrinsicToDbgRecord(Name, CI); in UpgradeIntrinsicCall()
4295 CI->replaceAllUsesWith(Rep); in UpgradeIntrinsicCall()
4296 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4302 if (CI->getFunctionType() == NewFn->getFunctionType()) { in UpgradeIntrinsicCall()
4305 (CI->getCalledFunction()->getName() != NewFn->getName()) && in UpgradeIntrinsicCall()
4307 CI->setCalledFunction(NewFn); in UpgradeIntrinsicCall()
4312 if (auto *OldST = dyn_cast<StructType>(CI->getType())) { in UpgradeIntrinsicCall()
4319 SmallVector<Value *> Args(CI->args()); in UpgradeIntrinsicCall()
4326 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4327 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4333 CI->setCalledOperand( in UpgradeIntrinsicCall()
4334 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType())); in UpgradeIntrinsicCall()
4350 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4358 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4376 SmallVector<Value *, 2> Args(CI->args()); in UpgradeIntrinsicCall()
4389 SmallVector<Value *, 3> Args(CI->args()); in UpgradeIntrinsicCall()
4390 Args.push_back(ConstantTokenNone::get(CI->getContext())); in UpgradeIntrinsicCall()
4404 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
4406 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx}); in UpgradeIntrinsicCall()
4418 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
4419 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType()); in UpgradeIntrinsicCall()
4423 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx}); in UpgradeIntrinsicCall()
4438 Value *V = CI->getArgOperand(I); in UpgradeIntrinsicCall()
4455 assert(CI->arg_size() == 3 && in UpgradeIntrinsicCall()
4458 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
4462 auto Iter = CI->args().begin(); in UpgradeIntrinsicCall()
4471 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
4476 assert(CI->arg_size() == 1 && in UpgradeIntrinsicCall()
4479 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()}); in UpgradeIntrinsicCall()
4484 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2); in UpgradeIntrinsicCall()
4486 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3); in UpgradeIntrinsicCall()
4488 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic}); in UpgradeIntrinsicCall()
4493 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
4497 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
4506 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata()); in UpgradeIntrinsicCall()
4509 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4515 assert(CI->arg_size() == 4); in UpgradeIntrinsicCall()
4517 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1))) in UpgradeIntrinsicCall()
4521 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)}); in UpgradeIntrinsicCall()
4524 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4530 if (CI->arg_size() != 4) { in UpgradeIntrinsicCall()
4537 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4538 CI->getArgOperand(2), CI->getArgOperand(3), in UpgradeIntrinsicCall()
4540 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4541 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
4542 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4547 if (CI->arg_size() != 4) { in UpgradeIntrinsicCall()
4553 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4554 CI->getArgOperand(2), CI->getArgOperand(3), in UpgradeIntrinsicCall()
4556 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4557 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
4558 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4569 Value *Arg2 = CI->getArgOperand(2); in UpgradeIntrinsicCall()
4570 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64)) in UpgradeIntrinsicCall()
4573 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4574 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4575 if (CI->getType()->isIntegerTy(64)) { in UpgradeIntrinsicCall()
4585 if (Res->getType() != CI->getType()) in UpgradeIntrinsicCall()
4586 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true); in UpgradeIntrinsicCall()
4587 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4588 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4589 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4600 if (!CI->getType()->isIntegerTy(64)) in UpgradeIntrinsicCall()
4604 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty()); in UpgradeIntrinsicCall()
4608 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true); in UpgradeIntrinsicCall()
4609 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4610 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4611 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4617 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)}); in UpgradeIntrinsicCall()
4624 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4639 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4644 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4658 if (CI->getNumOperands() == 0) in UpgradeIntrinsicCall()
4665 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0), in UpgradeIntrinsicCall()
4671 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4672 CI->replaceAllUsesWith(TSC); in UpgradeIntrinsicCall()
4673 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4685 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4699 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4707 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4708 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4709 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4719 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4720 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
4730 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4731 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4732 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4738 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4740 cast<FixedVectorType>(CI->getType())->getNumElements() * 2; in UpgradeIntrinsicCall()
4765 if (CI->arg_size() != 5) { in UpgradeIntrinsicCall()
4771 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4772 CI->getArgOperand(2), CI->getArgOperand(4)}; in UpgradeIntrinsicCall()
4774 AttributeList OldAttrs = CI->getAttributes(); in UpgradeIntrinsicCall()
4782 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3)); in UpgradeIntrinsicCall()
4792 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4793 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
4794 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4951 CallInst *CI = dyn_cast<CallInst>(U); in UpgradeARCRuntime() local
4952 if (!CI || CI->getCalledFunction() != Fn) in UpgradeARCRuntime()
4955 IRBuilder<> Builder(CI->getParent(), CI->getIterator()); in UpgradeARCRuntime()
4961 if (NewFuncTy->getReturnType() != CI->getType() && in UpgradeARCRuntime()
4962 !CastInst::castIsValid(Instruction::BitCast, CI, in UpgradeARCRuntime()
4968 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) { in UpgradeARCRuntime()
4969 Value *Arg = CI->getArgOperand(I); in UpgradeARCRuntime()
4991 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind()); in UpgradeARCRuntime()
4992 NewCall->takeName(CI); in UpgradeARCRuntime()
4995 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType()); in UpgradeARCRuntime()
4997 if (!CI->use_empty()) in UpgradeARCRuntime()
4998 CI->replaceAllUsesWith(NewRetVal); in UpgradeARCRuntime()
4999 CI->eraseFromParent(); in UpgradeARCRuntime()