Lines Matching refs:Ops
6402 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, in EmitNeonCall() argument
6412 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); in EmitNeonCall()
6414 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); in EmitNeonCall()
6418 return Builder.CreateConstrainedFPCall(F, Ops, name); in EmitNeonCall()
6420 return Builder.CreateCall(F, Ops, name); in EmitNeonCall()
7599 SmallVectorImpl<Value *> &Ops, const CallExpr *E) { in EmitCommonNeonSISDBuiltinExpr() argument
7621 std::swap(Ops[0], Ops[1]); in EmitCommonNeonSISDBuiltinExpr()
7637 if (Ops[j]->getType()->getPrimitiveSizeInBits() == in EmitCommonNeonSISDBuiltinExpr()
7641 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); in EmitCommonNeonSISDBuiltinExpr()
7644 Ops[j] = CGF.Builder.CreateTruncOrBitCast( in EmitCommonNeonSISDBuiltinExpr()
7645 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType()); in EmitCommonNeonSISDBuiltinExpr()
7646 Ops[j] = in EmitCommonNeonSISDBuiltinExpr()
7647 CGF.Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0); in EmitCommonNeonSISDBuiltinExpr()
7650 Value *Result = CGF.EmitNeonCall(F, Ops, s); in EmitCommonNeonSISDBuiltinExpr()
7662 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1, in EmitCommonNeonBuiltinExpr() argument
7705 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
7706 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements); in EmitCommonNeonBuiltinExpr()
7718 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
7719 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
7723 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
7724 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitCommonNeonBuiltinExpr()
7725 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7726 return Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7733 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
7734 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
7735 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); in EmitCommonNeonBuiltinExpr()
7740 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); in EmitCommonNeonBuiltinExpr()
7743 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); in EmitCommonNeonBuiltinExpr()
7749 std::swap(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7771 return EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7775 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, in EmitCommonNeonBuiltinExpr()
7779 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, in EmitCommonNeonBuiltinExpr()
7783 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, in EmitCommonNeonBuiltinExpr()
7787 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, in EmitCommonNeonBuiltinExpr()
7791 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, in EmitCommonNeonBuiltinExpr()
7797 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); in EmitCommonNeonBuiltinExpr()
7801 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7804 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
7805 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
7810 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7813 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
7814 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
7821 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
7830 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
7846 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
7860 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); in EmitCommonNeonBuiltinExpr()
7861 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
7862 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
7913 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7917 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7922 int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); in EmitCommonNeonBuiltinExpr()
7927 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7928 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7929 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext"); in EmitCommonNeonBuiltinExpr()
7933 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7934 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7935 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
7940 {Ops[1], Ops[2], Ops[0]}); in EmitCommonNeonBuiltinExpr()
7945 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
7946 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); in EmitCommonNeonBuiltinExpr()
7956 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); in EmitCommonNeonBuiltinExpr()
7957 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
7974 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); in EmitCommonNeonBuiltinExpr()
7975 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
7983 Ops[0] = Builder.CreateInsertElement(V, Ld, CI); in EmitCommonNeonBuiltinExpr()
7984 return EmitNeonSplat(Ops[0], CI); in EmitCommonNeonBuiltinExpr()
7994 for (unsigned I = 2; I < Ops.size() - 1; ++I) in EmitCommonNeonBuiltinExpr()
7995 Ops[I] = Builder.CreateBitCast(Ops[I], Ty); in EmitCommonNeonBuiltinExpr()
7996 Ops.push_back(getAlignmentValue32(PtrOp1)); in EmitCommonNeonBuiltinExpr()
7997 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), NameHint); in EmitCommonNeonBuiltinExpr()
7998 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
8003 Ops[0] = Builder.CreateBitCast(Ops[0], DTy); in EmitCommonNeonBuiltinExpr()
8005 return Builder.CreateZExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
8006 return Builder.CreateSExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
8011 Ops[0] = Builder.CreateBitCast(Ops[0], QTy); in EmitCommonNeonBuiltinExpr()
8012 return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); in EmitCommonNeonBuiltinExpr()
8022 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitCommonNeonBuiltinExpr()
8032 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
8042 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); in EmitCommonNeonBuiltinExpr()
8046 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
8047 Ops[1] = in EmitCommonNeonBuiltinExpr()
8049 Ops.resize(2); in EmitCommonNeonBuiltinExpr()
8050 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
8064 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
8073 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
8077 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", in EmitCommonNeonBuiltinExpr()
8081 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", in EmitCommonNeonBuiltinExpr()
8088 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
8094 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
8097 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", in EmitCommonNeonBuiltinExpr()
8104 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
8108 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); in EmitCommonNeonBuiltinExpr()
8109 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], in EmitCommonNeonBuiltinExpr()
8114 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
8116 Ops[0] = Builder.CreateZExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
8118 Ops[0] = Builder.CreateSExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
8119 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); in EmitCommonNeonBuiltinExpr()
8120 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); in EmitCommonNeonBuiltinExpr()
8125 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
8126 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); in EmitCommonNeonBuiltinExpr()
8128 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
8130 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
8131 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); in EmitCommonNeonBuiltinExpr()
8135 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); in EmitCommonNeonBuiltinExpr()
8151 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
8152 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
8160 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
8167 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitCommonNeonBuiltinExpr()
8168 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
8181 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
8182 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
8185 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
8192 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
8193 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
8194 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); in EmitCommonNeonBuiltinExpr()
8199 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); in EmitCommonNeonBuiltinExpr()
8202 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); in EmitCommonNeonBuiltinExpr()
8206 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
8207 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
8216 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
8217 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitCommonNeonBuiltinExpr()
8224 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
8225 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
8226 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
8227 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitCommonNeonBuiltinExpr()
8229 return Builder.CreateSExt(Ops[0], Ty, "vtst"); in EmitCommonNeonBuiltinExpr()
8233 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
8234 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
8242 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
8243 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitCommonNeonBuiltinExpr()
8250 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitCommonNeonBuiltinExpr()
8251 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
8255 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
8256 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
8265 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
8266 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitCommonNeonBuiltinExpr()
8278 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vdot"); in EmitCommonNeonBuiltinExpr()
8285 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_low"); in EmitCommonNeonBuiltinExpr()
8292 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_low"); in EmitCommonNeonBuiltinExpr()
8299 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_high"); in EmitCommonNeonBuiltinExpr()
8306 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_high"); in EmitCommonNeonBuiltinExpr()
8313 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vmmla"); in EmitCommonNeonBuiltinExpr()
8319 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusmmla"); in EmitCommonNeonBuiltinExpr()
8326 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusdot"); in EmitCommonNeonBuiltinExpr()
8333 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfdot"); in EmitCommonNeonBuiltinExpr()
8338 return EmitNeonCall(F, Ops, "vcvtfp2bf"); in EmitCommonNeonBuiltinExpr()
8348 Value *Result = EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
8379 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in packTBLDVectorList() argument
8389 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType()); in packTBLDVectorList()
8395 int PairPos = 0, End = Ops.size() - 1; in packTBLDVectorList()
8397 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
8398 Ops[PairPos+1], Indices, in packTBLDVectorList()
8407 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
8501 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; in EmitSpecialRegisterBuiltin() local
8502 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitSpecialRegisterBuiltin()
8665 Value *Ops[2]; in EmitARMBuiltinExpr() local
8667 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitARMBuiltinExpr()
8671 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitARMBuiltinExpr()
8973 SmallVector<Value*, 4> Ops; in EmitARMBuiltinExpr() local
9004 Ops.push_back(PtrOp0.emitRawPointer(*this)); in EmitARMBuiltinExpr()
9031 Ops.push_back(PtrOp1.emitRawPointer(*this)); in EmitARMBuiltinExpr()
9036 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E)); in EmitARMBuiltinExpr()
9056 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); in EmitARMBuiltinExpr()
9076 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitARMBuiltinExpr()
9079 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, in EmitARMBuiltinExpr()
9082 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, in EmitARMBuiltinExpr()
9085 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, in EmitARMBuiltinExpr()
9088 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, in EmitARMBuiltinExpr()
9092 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops, in EmitARMBuiltinExpr()
9103 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], in EmitARMBuiltinExpr()
9104 Ops[3], Ops[4], Ops[5]}); in EmitARMBuiltinExpr()
9131 return Builder.CreateCall(F, Ops, "vcvtr"); in EmitARMBuiltinExpr()
9154 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch); in EmitARMBuiltinExpr()
9164 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
9165 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitARMBuiltinExpr()
9167 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
9173 Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); in EmitARMBuiltinExpr()
9176 return Builder.CreateShuffleVector(Ops[1], Ld, Indices, "vld1q_lane"); in EmitARMBuiltinExpr()
9180 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
9183 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); in EmitARMBuiltinExpr()
9188 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", in EmitARMBuiltinExpr()
9192 Ops, "vqrshrun_n", 1, true); in EmitARMBuiltinExpr()
9195 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", in EmitARMBuiltinExpr()
9199 Ops, "vqshrun_n", 1, true); in EmitARMBuiltinExpr()
9203 Ops, "vrecpe"); in EmitARMBuiltinExpr()
9206 Ops, "vrshrn_n", 1, true); in EmitARMBuiltinExpr()
9209 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
9210 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
9211 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); in EmitARMBuiltinExpr()
9213 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); in EmitARMBuiltinExpr()
9214 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); in EmitARMBuiltinExpr()
9221 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); in EmitARMBuiltinExpr()
9223 Ops, "vsli_n"); in EmitARMBuiltinExpr()
9226 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
9227 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitARMBuiltinExpr()
9228 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitARMBuiltinExpr()
9233 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
9234 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); in EmitARMBuiltinExpr()
9235 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
9236 Ops[2] = getAlignmentValue32(PtrOp0); in EmitARMBuiltinExpr()
9237 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; in EmitARMBuiltinExpr()
9239 Tys), Ops); in EmitARMBuiltinExpr()
9243 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
9244 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitARMBuiltinExpr()
9245 return Builder.CreateStore(Ops[1], in EmitARMBuiltinExpr()
9246 PtrOp0.withElementType(Ops[1]->getType())); in EmitARMBuiltinExpr()
9250 Ops, "vtbl1"); in EmitARMBuiltinExpr()
9253 Ops, "vtbl2"); in EmitARMBuiltinExpr()
9256 Ops, "vtbl3"); in EmitARMBuiltinExpr()
9259 Ops, "vtbl4"); in EmitARMBuiltinExpr()
9262 Ops, "vtbx1"); in EmitARMBuiltinExpr()
9265 Ops, "vtbx2"); in EmitARMBuiltinExpr()
9268 Ops, "vtbx3"); in EmitARMBuiltinExpr()
9271 Ops, "vtbx4"); in EmitARMBuiltinExpr()
9419 llvm::SmallVector<Value *, 4> Ops; in EmitARMMVEBuiltinExpr() local
9438 Ops.push_back(EmitScalarExpr(Addr)); in EmitARMMVEBuiltinExpr()
9442 Value *LoadResult = Builder.CreateCall(F, Ops); in EmitARMMVEBuiltinExpr()
9456 llvm::SmallVector<Value *, 4> Ops; in EmitARMMVEBuiltinExpr() local
9460 Ops.push_back(EmitScalarExpr(Addr)); in EmitARMMVEBuiltinExpr()
9481 Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i})); in EmitARMMVEBuiltinExpr()
9486 Ops.push_back(llvm::ConstantInt::get(Int32Ty, i)); in EmitARMMVEBuiltinExpr()
9487 ToReturn = Builder.CreateCall(F, Ops); in EmitARMMVEBuiltinExpr()
9488 Ops.pop_back(); in EmitARMMVEBuiltinExpr()
9509 SmallVectorImpl<Value *> &Ops, in EmitAArch64TblBuiltinExpr() argument
9566 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 1), nullptr, Ops[1], in EmitAArch64TblBuiltinExpr()
9570 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 2), nullptr, Ops[2], in EmitAArch64TblBuiltinExpr()
9574 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 3), nullptr, Ops[3], in EmitAArch64TblBuiltinExpr()
9578 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 4), nullptr, Ops[4], in EmitAArch64TblBuiltinExpr()
9583 packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 1), nullptr, Ops[2], Ty, in EmitAArch64TblBuiltinExpr()
9587 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); in EmitAArch64TblBuiltinExpr()
9590 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
9595 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 2), Ops[0], Ops[3], in EmitAArch64TblBuiltinExpr()
9600 packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 3), nullptr, Ops[4], Ty, in EmitAArch64TblBuiltinExpr()
9604 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], in EmitAArch64TblBuiltinExpr()
9608 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
9613 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 4), Ops[0], Ops[5], in EmitAArch64TblBuiltinExpr()
9647 return CGF.EmitNeonCall(F, Ops, s); in EmitAArch64TblBuiltinExpr()
9836 SmallVectorImpl<Value *> &Ops, in EmitSVEGatherLoad() argument
9843 if (Ops[1]->getType()->isVectorTy()) in EmitSVEGatherLoad()
9847 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[1]->getType()}); in EmitSVEGatherLoad()
9862 Ops[0] = EmitSVEPredicateCast( in EmitSVEGatherLoad()
9863 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType())); in EmitSVEGatherLoad()
9868 if (Ops.size() == 2) { in EmitSVEGatherLoad()
9869 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset"); in EmitSVEGatherLoad()
9870 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEGatherLoad()
9875 if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) { in EmitSVEGatherLoad()
9878 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt)); in EmitSVEGatherLoad()
9881 Value *Call = Builder.CreateCall(F, Ops); in EmitSVEGatherLoad()
9890 SmallVectorImpl<Value *> &Ops, in EmitSVEScatterStore() argument
9898 Ops.insert(Ops.begin(), Ops.pop_back_val()); in EmitSVEScatterStore()
9901 if (Ops[2]->getType()->isVectorTy()) in EmitSVEScatterStore()
9905 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[2]->getType()}); in EmitSVEScatterStore()
9916 if (Ops.size() == 3) { in EmitSVEScatterStore()
9917 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset"); in EmitSVEScatterStore()
9918 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEScatterStore()
9923 Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy); in EmitSVEScatterStore()
9932 Ops[1] = EmitSVEPredicateCast( in EmitSVEScatterStore()
9933 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType())); in EmitSVEScatterStore()
9937 if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) { in EmitSVEScatterStore()
9940 Ops[3] = Builder.CreateShl(Ops[3], Log2_32(BytesPerElt)); in EmitSVEScatterStore()
9943 return Builder.CreateCall(F, Ops); in EmitSVEScatterStore()
9947 SmallVectorImpl<Value *> &Ops, in EmitSVEGatherPrefetch() argument
9951 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType()); in EmitSVEGatherPrefetch()
9953 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType()); in EmitSVEGatherPrefetch()
9956 Ops[0] = EmitSVEPredicateCast(Ops[0], OverloadedTy); in EmitSVEGatherPrefetch()
9959 if (Ops[1]->getType()->isVectorTy()) { in EmitSVEGatherPrefetch()
9960 if (Ops.size() == 3) { in EmitSVEGatherPrefetch()
9962 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEGatherPrefetch()
9965 std::swap(Ops[2], Ops[3]); in EmitSVEGatherPrefetch()
9971 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt)); in EmitSVEGatherPrefetch()
9976 return Builder.CreateCall(F, Ops); in EmitSVEGatherPrefetch()
9980 SmallVectorImpl<Value*> &Ops, in EmitSVEStructLoad() argument
10008 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy); in EmitSVEStructLoad()
10009 Value *BasePtr = Ops[1]; in EmitSVEStructLoad()
10012 if (Ops.size() > 2) in EmitSVEStructLoad()
10013 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]); in EmitSVEStructLoad()
10028 SmallVectorImpl<Value*> &Ops, in EmitSVEStructStore() argument
10054 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy); in EmitSVEStructStore()
10055 Value *BasePtr = Ops[1]; in EmitSVEStructStore()
10058 if (Ops.size() > (2 + N)) in EmitSVEStructStore()
10059 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]); in EmitSVEStructStore()
10064 for (unsigned I = Ops.size() - N; I < Ops.size(); ++I) in EmitSVEStructStore()
10065 Operands.push_back(Ops[I]); in EmitSVEStructStore()
10076 SmallVectorImpl<Value *> &Ops, in EmitSVEPMull() argument
10081 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]); in EmitSVEPMull()
10085 Function *F = CGM.getIntrinsic(BuiltinID, Ops[0]->getType()); in EmitSVEPMull()
10086 Value *Call = Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitSVEPMull()
10094 ArrayRef<Value *> Ops, unsigned BuiltinID) { in EmitSVEMovl() argument
10097 return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)}); in EmitSVEMovl()
10101 SmallVectorImpl<Value *> &Ops, in EmitSVEPrefetchLoad() argument
10107 Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy); in EmitSVEPrefetchLoad()
10108 Value *BasePtr = Ops[1]; in EmitSVEPrefetchLoad()
10111 if (Ops.size() > 3) in EmitSVEPrefetchLoad()
10112 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]); in EmitSVEPrefetchLoad()
10114 Value *PrfOp = Ops.back(); in EmitSVEPrefetchLoad()
10122 SmallVectorImpl<Value *> &Ops, in EmitSVEMaskedLoad() argument
10149 Value *Predicate = EmitSVEPredicateCast(Ops[0], PredTy); in EmitSVEMaskedLoad()
10150 Value *BasePtr = Ops[1]; in EmitSVEMaskedLoad()
10153 if (Ops.size() > 2) in EmitSVEMaskedLoad()
10154 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]); in EmitSVEMaskedLoad()
10170 SmallVectorImpl<Value *> &Ops, in EmitSVEMaskedStore() argument
10178 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType()); in EmitSVEMaskedStore()
10196 Value *Predicate = EmitSVEPredicateCast(Ops[0], PredTy); in EmitSVEMaskedStore()
10197 Value *BasePtr = Ops[1]; in EmitSVEMaskedStore()
10200 if (Ops.size() == 4) in EmitSVEMaskedStore()
10201 BasePtr = Builder.CreateGEP(AddrMemoryTy, BasePtr, Ops[2]); in EmitSVEMaskedStore()
10205 IsQuadStore ? Ops.back() : Builder.CreateTrunc(Ops.back(), MemoryTy); in EmitSVEMaskedStore()
10217 SmallVectorImpl<Value *> &Ops, in EmitSMELd1St1() argument
10219 Ops[2] = EmitSVEPredicateCast( in EmitSMELd1St1()
10220 Ops[2], getSVEVectorForElementType(SVEBuiltinMemEltTy(TypeFlags))); in EmitSMELd1St1()
10223 NewOps.push_back(Ops[2]); in EmitSMELd1St1()
10225 llvm::Value *BasePtr = Ops[3]; in EmitSMELd1St1()
10229 if (Ops.size() == 5) { in EmitSMELd1St1()
10235 Builder.CreateMul(StreamingVectorLengthCall, Ops[4], "mulvl"); in EmitSMELd1St1()
10237 BasePtr = Builder.CreateGEP(Int8Ty, Ops[3], Mulvl); in EmitSMELd1St1()
10240 NewOps.push_back(Ops[0]); in EmitSMELd1St1()
10241 NewOps.push_back(Ops[1]); in EmitSMELd1St1()
10247 SmallVectorImpl<Value *> &Ops, in EmitSMEReadWrite() argument
10252 Ops[1] = EmitSVEPredicateCast(Ops[1], VecTy); in EmitSMEReadWrite()
10254 Ops[2] = EmitSVEPredicateCast(Ops[2], VecTy); in EmitSMEReadWrite()
10255 return Builder.CreateCall(F, Ops); in EmitSMEReadWrite()
10259 SmallVectorImpl<Value *> &Ops, in EmitSMEZero() argument
10262 if (Ops.size() == 0) in EmitSMEZero()
10263 Ops.push_back(llvm::ConstantInt::get(Int32Ty, 255)); in EmitSMEZero()
10265 return Builder.CreateCall(F, Ops); in EmitSMEZero()
10269 SmallVectorImpl<Value *> &Ops, in EmitSMELdrStr() argument
10271 if (Ops.size() == 2) in EmitSMELdrStr()
10272 Ops.push_back(Builder.getInt32(0)); in EmitSMELdrStr()
10274 Ops[2] = Builder.CreateIntCast(Ops[2], Int32Ty, true); in EmitSMELdrStr()
10276 return Builder.CreateCall(F, Ops); in EmitSMELdrStr()
10301 SmallVectorImpl<Value *> &Ops) { in InsertExplicitZeroOperand() argument
10303 Ops.insert(Ops.begin(), SplatZero); in InsertExplicitZeroOperand()
10307 SmallVectorImpl<Value *> &Ops) { in InsertExplicitUndefOperand() argument
10309 Ops.insert(Ops.begin(), SplatUndef); in InsertExplicitUndefOperand()
10315 ArrayRef<Value *> Ops) { in getSVEOverloadTypes() argument
10322 return {DefaultType, Ops[1]->getType()}; in getSVEOverloadTypes()
10325 return {getSVEPredType(TypeFlags), Ops[0]->getType()}; in getSVEOverloadTypes()
10328 return {Ops[0]->getType(), Ops.back()->getType()}; in getSVEOverloadTypes()
10332 return {ResultType, Ops[1]->getType()}; in getSVEOverloadTypes()
10340 ArrayRef<Value *> Ops) { in EmitSVETupleSetOrGet() argument
10344 unsigned I = cast<ConstantInt>(Ops[1])->getSExtValue(); in EmitSVETupleSetOrGet()
10346 TypeFlags.isTupleSet() ? Ops[2]->getType() : Ty); in EmitSVETupleSetOrGet()
10355 return Builder.CreateInsertVector(Ty, Ops[0], Ops[2], Idx); in EmitSVETupleSetOrGet()
10356 return Builder.CreateExtractVector(Ty, Ops[0], Idx); in EmitSVETupleSetOrGet()
10361 ArrayRef<Value *> Ops) { in EmitSVETupleCreate() argument
10364 auto *SrcTy = dyn_cast<llvm::ScalableVectorType>(Ops[0]->getType()); in EmitSVETupleCreate()
10371 for (unsigned I = 0; I < Ops.size(); I++) { in EmitSVETupleCreate()
10373 Call = Builder.CreateInsertVector(Ty, Call, Ops[I], Idx); in EmitSVETupleCreate()
10415 unsigned BuiltinID, const CallExpr *E, SmallVectorImpl<Value *> &Ops, in GetAArch64SVEProcessedOperands() argument
10442 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), *Result)); in GetAArch64SVEProcessedOperands()
10447 Ops.push_back(Arg); in GetAArch64SVEProcessedOperands()
10457 Ops.push_back(Arg); in GetAArch64SVEProcessedOperands()
10465 Ops.push_back(Builder.CreateExtractVector(NewVTy, Arg, Idx)); in GetAArch64SVEProcessedOperands()
10482 llvm::SmallVector<Value *, 4> Ops; in EmitAArch64SVEBuiltinExpr() local
10484 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags); in EmitAArch64SVEBuiltinExpr()
10487 return EmitSVEMaskedLoad(E, Ty, Ops, Builtin->LLVMIntrinsic, in EmitAArch64SVEBuiltinExpr()
10490 return EmitSVEMaskedStore(E, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10492 return EmitSVEGatherLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10494 return EmitSVEScatterStore(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10496 return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10498 return EmitSVEGatherPrefetch(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10500 return EmitSVEStructLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10502 return EmitSVEStructStore(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
10504 return EmitSVETupleSetOrGet(TypeFlags, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
10506 return EmitSVETupleCreate(TypeFlags, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
10511 InsertExplicitZeroOperand(Builder, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
10514 InsertExplicitUndefOperand(Builder, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
10519 Ops.push_back(Builder.getInt32(/*SV_ALL*/ 31)); in EmitAArch64SVEBuiltinExpr()
10521 Ops.insert(&Ops[1], Builder.getInt32(/*SV_ALL*/ 31)); in EmitAArch64SVEBuiltinExpr()
10524 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in EmitAArch64SVEBuiltinExpr()
10525 if (auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType())) in EmitAArch64SVEBuiltinExpr()
10527 Ops[i] = EmitSVEPredicateCast(Ops[i], getSVEType(TypeFlags)); in EmitAArch64SVEBuiltinExpr()
10532 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]); in EmitAArch64SVEBuiltinExpr()
10536 std::swap(Ops[1], Ops[2]); in EmitAArch64SVEBuiltinExpr()
10538 std::swap(Ops[1], Ops[2]); in EmitAArch64SVEBuiltinExpr()
10541 std::swap(Ops[1], Ops[2]); in EmitAArch64SVEBuiltinExpr()
10544 std::swap(Ops[1], Ops[3]); in EmitAArch64SVEBuiltinExpr()
10548 llvm::Type *OpndTy = Ops[1]->getType(); in EmitAArch64SVEBuiltinExpr()
10550 Ops[1] = Builder.CreateSelect(Ops[0], Ops[1], SplatZero); in EmitAArch64SVEBuiltinExpr()
10554 getSVEOverloadTypes(TypeFlags, Ty, Ops)); in EmitAArch64SVEBuiltinExpr()
10555 Value *Call = Builder.CreateCall(F, Ops); in EmitAArch64SVEBuiltinExpr()
10574 return Builder.CreateCall(CastFromSVCountF, Ops[0]); in EmitAArch64SVEBuiltinExpr()
10581 return Builder.CreateCall(CastToSVCountF, Ops[0]); in EmitAArch64SVEBuiltinExpr()
10592 bool IsSVCount = isa<TargetExtType>(Ops[0]->getType()); in EmitAArch64SVEBuiltinExpr()
10593 assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->getName() == in EmitAArch64SVEBuiltinExpr()
10606 IsSVCount ? Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0]; in EmitAArch64SVEBuiltinExpr()
10607 llvm::Value *Ops1 = EmitSVEPredicateCast(Ops[1], OverloadedTy); in EmitAArch64SVEBuiltinExpr()
10608 llvm::Value *PSel = Builder.CreateCall(F, {Ops0, Ops1, Ops[2]}); in EmitAArch64SVEBuiltinExpr()
10616 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]}); in EmitAArch64SVEBuiltinExpr()
10624 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]}); in EmitAArch64SVEBuiltinExpr()
10630 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb); in EmitAArch64SVEBuiltinExpr()
10635 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb); in EmitAArch64SVEBuiltinExpr()
10640 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt); in EmitAArch64SVEBuiltinExpr()
10645 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt); in EmitAArch64SVEBuiltinExpr()
10651 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair); in EmitAArch64SVEBuiltinExpr()
10657 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair); in EmitAArch64SVEBuiltinExpr()
10664 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType())); in EmitAArch64SVEBuiltinExpr()
10688 unsigned NumOpnds = Ops.size(); in EmitAArch64SVEBuiltinExpr()
10696 llvm::Type *EltTy = Ops[0]->getType(); in EmitAArch64SVEBuiltinExpr()
10702 VecOps.push_back(Builder.CreateZExt(Ops[I], EltTy)); in EmitAArch64SVEBuiltinExpr()
10775 return Builder.CreateCall(F, Ops); in EmitAArch64SVEBuiltinExpr()
10790 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], Builder.getInt64(0)); in EmitAArch64SVEBuiltinExpr()
10805 return Builder.CreateExtractVector(Ty, Ops[0], Builder.getInt64(0)); in EmitAArch64SVEBuiltinExpr()
10820 Value *Insert = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0], in EmitAArch64SVEBuiltinExpr()
10832 SmallVectorImpl<Value *> &Ops) { in swapCommutativeSMEOperands() argument
10852 std::swap(Ops[I + 1], Ops[I + 1 + MultiVec]); in swapCommutativeSMEOperands()
10860 llvm::SmallVector<Value *, 4> Ops; in EmitAArch64SMEBuiltinExpr() local
10862 GetAArch64SVEProcessedOperands(BuiltinID, E, Ops, TypeFlags); in EmitAArch64SMEBuiltinExpr()
10865 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SMEBuiltinExpr()
10867 return EmitSMEReadWrite(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SMEBuiltinExpr()
10870 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SMEBuiltinExpr()
10875 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SMEBuiltinExpr()
10878 swapCommutativeSMEOperands(BuiltinID, Ops); in EmitAArch64SMEBuiltinExpr()
10885 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in EmitAArch64SMEBuiltinExpr()
10886 if (auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType())) in EmitAArch64SMEBuiltinExpr()
10888 Ops[i] = EmitSVEPredicateCast(Ops[i], getSVEType(TypeFlags)); in EmitAArch64SMEBuiltinExpr()
10894 Value *Call = Builder.CreateCall(F, Ops); in EmitAArch64SMEBuiltinExpr()
11110 Value *Ops[2]; in EmitAArch64BuiltinExpr() local
11112 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitAArch64BuiltinExpr()
11116 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitAArch64BuiltinExpr()
11227 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)}; in EmitAArch64BuiltinExpr() local
11228 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
11435 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) }; in EmitAArch64BuiltinExpr() local
11436 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
11495 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")}; in EmitAArch64BuiltinExpr() local
11496 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
11519 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")}; in EmitAArch64BuiltinExpr() local
11520 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
11627 llvm::SmallVector<Value*, 4> Ops; in EmitAArch64BuiltinExpr() local
11649 Ops.push_back(PtrOp0.emitRawPointer(*this)); in EmitAArch64BuiltinExpr()
11653 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E)); in EmitAArch64BuiltinExpr()
11661 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); in EmitAArch64BuiltinExpr()
11662 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); in EmitAArch64BuiltinExpr()
11681 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11682 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, HalfTy), Ops, "vabs"); in EmitAArch64BuiltinExpr()
11685 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11686 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11687 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11688 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11690 return Builder.CreateBitCast(Ops[0], Int128Ty); in EmitAArch64BuiltinExpr()
11699 Value *Ptr = Ops[0]; in EmitAArch64BuiltinExpr()
11708 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11709 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; in EmitAArch64BuiltinExpr()
11712 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
11714 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
11715 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
11725 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11728 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64) in EmitAArch64BuiltinExpr()
11730 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32) in EmitAArch64BuiltinExpr()
11734 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
11736 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
11737 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
11753 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11777 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt"); in EmitAArch64BuiltinExpr()
11778 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11788 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11796 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break; in EmitAArch64BuiltinExpr()
11798 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break; in EmitAArch64BuiltinExpr()
11800 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg"); in EmitAArch64BuiltinExpr()
11801 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11809 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11817 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n"); in EmitAArch64BuiltinExpr()
11818 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11826 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11831 Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext"); in EmitAArch64BuiltinExpr()
11835 Ops[0] = Builder.CreateZExt(Ops[0], InTy); in EmitAArch64BuiltinExpr()
11838 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n"); in EmitAArch64BuiltinExpr()
11880 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11882 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
11888 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11890 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
11896 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11898 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
11904 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11906 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
11912 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11914 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
11918 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11919 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
11920 Ops[0] = in EmitAArch64BuiltinExpr()
11921 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); in EmitAArch64BuiltinExpr()
11922 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); in EmitAArch64BuiltinExpr()
11938 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11939 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
11940 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
11942 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11944 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11945 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
11961 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11962 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); in EmitAArch64BuiltinExpr()
11963 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); in EmitAArch64BuiltinExpr()
11965 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11967 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11968 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
11984 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11985 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11986 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy); in EmitAArch64BuiltinExpr()
11988 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11990 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11991 return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
12017 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12018 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
12019 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
12020 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
12021 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); in EmitAArch64BuiltinExpr()
12025 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12026 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
12027 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
12028 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
12029 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitAArch64BuiltinExpr()
12031 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); in EmitAArch64BuiltinExpr()
12045 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
12046 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
12049 Ops[1] = in EmitAArch64BuiltinExpr()
12050 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
12051 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
12052 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
12055 Ops[1] = in EmitAArch64BuiltinExpr()
12056 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 2)); in EmitAArch64BuiltinExpr()
12057 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
12058 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
12062 Ops[0] = in EmitAArch64BuiltinExpr()
12063 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 8)); in EmitAArch64BuiltinExpr()
12064 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12068 Ops[0] = in EmitAArch64BuiltinExpr()
12069 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 16)); in EmitAArch64BuiltinExpr()
12070 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12074 Ops[0] = in EmitAArch64BuiltinExpr()
12075 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 4)); in EmitAArch64BuiltinExpr()
12076 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12080 Ops[0] = in EmitAArch64BuiltinExpr()
12081 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 8)); in EmitAArch64BuiltinExpr()
12082 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12086 Ops[0] = in EmitAArch64BuiltinExpr()
12087 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 2)); in EmitAArch64BuiltinExpr()
12088 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12091 Ops[0] = in EmitAArch64BuiltinExpr()
12092 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2)); in EmitAArch64BuiltinExpr()
12093 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12097 Ops[0] = in EmitAArch64BuiltinExpr()
12098 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4)); in EmitAArch64BuiltinExpr()
12099 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12103 Ops[0] = in EmitAArch64BuiltinExpr()
12104 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 1)); in EmitAArch64BuiltinExpr()
12105 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12108 Ops[0] = in EmitAArch64BuiltinExpr()
12109 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
12110 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12114 Ops[0] = in EmitAArch64BuiltinExpr()
12115 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2)); in EmitAArch64BuiltinExpr()
12116 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12119 Ops[0] = in EmitAArch64BuiltinExpr()
12120 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2)); in EmitAArch64BuiltinExpr()
12121 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12124 Ops[0] = in EmitAArch64BuiltinExpr()
12125 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
12126 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12130 Ops[0] = in EmitAArch64BuiltinExpr()
12131 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 4)); in EmitAArch64BuiltinExpr()
12132 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12136 Ops[0] = in EmitAArch64BuiltinExpr()
12137 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 2)); in EmitAArch64BuiltinExpr()
12138 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12141 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12142 return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh"); in EmitAArch64BuiltinExpr()
12144 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12145 return Builder.CreateFSub(Ops[0], Ops[1], "vsubh"); in EmitAArch64BuiltinExpr()
12147 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12148 return Builder.CreateFMul(Ops[0], Ops[1], "vmulh"); in EmitAArch64BuiltinExpr()
12150 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12151 return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh"); in EmitAArch64BuiltinExpr()
12156 {EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)), Ops[0]}); in EmitAArch64BuiltinExpr()
12163 {Neg, EmitScalarExpr(E->getArg(2)), Ops[0]}); in EmitAArch64BuiltinExpr()
12167 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); in EmitAArch64BuiltinExpr()
12170 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); in EmitAArch64BuiltinExpr()
12174 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
12177 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
12180 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
12185 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
12188 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12189 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
12191 Ops, "vqshlu_n"); in EmitAArch64BuiltinExpr()
12198 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12199 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
12200 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); in EmitAArch64BuiltinExpr()
12207 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12208 int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); in EmitAArch64BuiltinExpr()
12209 Ops[1] = ConstantInt::get(Int64Ty, -SV); in EmitAArch64BuiltinExpr()
12210 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); in EmitAArch64BuiltinExpr()
12217 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
12218 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); in EmitAArch64BuiltinExpr()
12219 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), in EmitAArch64BuiltinExpr()
12220 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); in EmitAArch64BuiltinExpr()
12221 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); in EmitAArch64BuiltinExpr()
12227 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); in EmitAArch64BuiltinExpr()
12232 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
12242 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
12247 Ops[1] = Builder.CreateAShr( in EmitAArch64BuiltinExpr()
12248 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
12251 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
12259 return Ops[0]; in EmitAArch64BuiltinExpr()
12260 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
12262 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
12268 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
12271 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
12272 ProductOps.push_back(vectorWrapScalar16(Ops[2])); in EmitAArch64BuiltinExpr()
12274 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
12277 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
12278 Ops.pop_back(); in EmitAArch64BuiltinExpr()
12284 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
12289 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
12291 Ops[1] = in EmitAArch64BuiltinExpr()
12298 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
12304 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
12307 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
12308 ProductOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
12309 Ops[1] = in EmitAArch64BuiltinExpr()
12312 Ops.pop_back(); in EmitAArch64BuiltinExpr()
12318 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
12323 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12329 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
12357 Builtin->NameHint, Builtin->TypeModifier, E, Ops, in EmitAArch64BuiltinExpr()
12360 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops, Arch)) in EmitAArch64BuiltinExpr()
12369 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
12370 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
12371 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
12373 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); in EmitAArch64BuiltinExpr()
12374 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
12375 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
12376 return Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12382 Value *Addend = Ops[0]; in EmitAArch64BuiltinExpr()
12383 Value *Multiplicand = Ops[1]; in EmitAArch64BuiltinExpr()
12384 Value *LaneSource = Ops[2]; in EmitAArch64BuiltinExpr()
12385 Ops[0] = Multiplicand; in EmitAArch64BuiltinExpr()
12386 Ops[1] = LaneSource; in EmitAArch64BuiltinExpr()
12387 Ops[2] = Addend; in EmitAArch64BuiltinExpr()
12394 llvm::Constant *cst = cast<Constant>(Ops[3]); in EmitAArch64BuiltinExpr()
12396 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); in EmitAArch64BuiltinExpr()
12397 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); in EmitAArch64BuiltinExpr()
12399 Ops.pop_back(); in EmitAArch64BuiltinExpr()
12402 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); in EmitAArch64BuiltinExpr()
12408 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
12409 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
12412 Ops[2] = Builder.CreateBitCast(Ops[2], VTy); in EmitAArch64BuiltinExpr()
12413 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
12417 DoubleTy, {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
12420 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12421 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12425 Ops[2] = Builder.CreateBitCast(Ops[2], STy); in EmitAArch64BuiltinExpr()
12427 cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
12428 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); in EmitAArch64BuiltinExpr()
12432 {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
12435 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12436 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12438 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
12439 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
12442 {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
12450 Ops.push_back(EmitScalarExpr(E->getArg(3))); in EmitAArch64BuiltinExpr()
12452 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
12455 {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
12461 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitAArch64BuiltinExpr()
12467 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); in EmitAArch64BuiltinExpr()
12469 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12471 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmax"); in EmitAArch64BuiltinExpr()
12478 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); in EmitAArch64BuiltinExpr()
12480 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12482 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmin"); in EmitAArch64BuiltinExpr()
12489 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); in EmitAArch64BuiltinExpr()
12500 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
12503 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); in EmitAArch64BuiltinExpr()
12511 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); in EmitAArch64BuiltinExpr()
12517 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); in EmitAArch64BuiltinExpr()
12521 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
12523 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12525 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
12529 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
12531 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12533 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
12535 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12537 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
12540 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12542 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
12544 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
12546 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
12549 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); in EmitAArch64BuiltinExpr()
12552 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); in EmitAArch64BuiltinExpr()
12555 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); in EmitAArch64BuiltinExpr()
12558 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); in EmitAArch64BuiltinExpr()
12561 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); in EmitAArch64BuiltinExpr()
12563 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12567 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
12574 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
12577 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12581 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndi"); in EmitAArch64BuiltinExpr()
12584 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12588 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
12595 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
12598 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12602 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
12609 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
12612 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12616 return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
12619 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12623 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
12630 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
12633 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12637 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
12644 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
12647 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12651 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
12657 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12659 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd32x"); in EmitAArch64BuiltinExpr()
12665 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12667 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd32z"); in EmitAArch64BuiltinExpr()
12673 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12675 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd64x"); in EmitAArch64BuiltinExpr()
12681 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12683 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd64z"); in EmitAArch64BuiltinExpr()
12690 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
12694 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12696 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitAArch64BuiltinExpr()
12697 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
12702 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
12704 return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
12710 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
12712 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
12729 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtz"); in EmitAArch64BuiltinExpr()
12745 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); in EmitAArch64BuiltinExpr()
12761 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); in EmitAArch64BuiltinExpr()
12777 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); in EmitAArch64BuiltinExpr()
12793 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); in EmitAArch64BuiltinExpr()
12798 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
12804 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
12805 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
12806 Ops.pop_back(); in EmitAArch64BuiltinExpr()
12808 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
12816 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
12819 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
12820 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
12821 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
12831 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); in EmitAArch64BuiltinExpr()
12836 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); in EmitAArch64BuiltinExpr()
12839 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12843 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
12850 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12851 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
12856 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); in EmitAArch64BuiltinExpr()
12867 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12868 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
12869 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12879 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12880 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
12881 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12891 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12892 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
12893 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12903 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12904 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
12905 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12912 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12913 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12914 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12921 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12922 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12923 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12930 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12931 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12932 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12939 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12940 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12941 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12948 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12949 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12950 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12957 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12958 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12959 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12966 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12967 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12968 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
12975 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12976 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12977 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
12984 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12985 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12986 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
12993 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
12994 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
12995 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
13002 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13003 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
13004 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
13011 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13012 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
13013 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
13020 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13021 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
13022 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
13029 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13030 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
13031 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
13038 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13039 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
13040 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
13047 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13048 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
13049 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
13056 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13057 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
13058 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
13065 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13066 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
13067 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
13074 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13075 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
13076 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
13083 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13084 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
13085 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
13092 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13093 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); in EmitAArch64BuiltinExpr()
13094 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
13101 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13102 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); in EmitAArch64BuiltinExpr()
13103 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
13110 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13111 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); in EmitAArch64BuiltinExpr()
13112 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
13119 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13120 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); in EmitAArch64BuiltinExpr()
13121 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
13124 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
13126 return Builder.CreateFMul(Ops[0], RHS); in EmitAArch64BuiltinExpr()
13133 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13134 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
13135 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
13142 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13143 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
13150 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13151 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
13152 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
13159 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13160 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
13167 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13168 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
13169 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
13176 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13177 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
13184 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13185 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
13186 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
13193 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
13194 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
13200 return EmitNeonCall(Intrin, Ops, "vsri_n"); in EmitAArch64BuiltinExpr()
13206 return EmitNeonCall(Intrin, Ops, "vsli_n"); in EmitAArch64BuiltinExpr()
13210 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
13211 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitAArch64BuiltinExpr()
13212 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
13217 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
13218 TmpOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
13221 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitAArch64BuiltinExpr()
13222 return Builder.CreateAdd(Ops[0], tmp); in EmitAArch64BuiltinExpr()
13226 return Builder.CreateAlignedLoad(VTy, Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
13230 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
13231 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
13234 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13235 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], in EmitAArch64BuiltinExpr()
13237 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); in EmitAArch64BuiltinExpr()
13241 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13243 VTy->getElementType(), Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
13245 Ops[0] = LI; in EmitAArch64BuiltinExpr()
13246 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vldap1_lane"); in EmitAArch64BuiltinExpr()
13251 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], in EmitAArch64BuiltinExpr()
13254 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); in EmitAArch64BuiltinExpr()
13255 return EmitNeonSplat(Ops[0], CI); in EmitAArch64BuiltinExpr()
13259 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13260 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitAArch64BuiltinExpr()
13261 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
13264 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13265 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitAArch64BuiltinExpr()
13267 Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
13275 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
13276 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13282 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
13283 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13289 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
13290 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13296 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
13297 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13303 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
13304 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13310 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
13311 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13315 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
13317 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
13318 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13319 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13320 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
13321 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld2_lane"); in EmitAArch64BuiltinExpr()
13322 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13326 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
13328 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
13329 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13330 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13331 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
13332 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
13333 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld3_lane"); in EmitAArch64BuiltinExpr()
13334 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13338 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
13340 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
13341 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13342 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13343 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
13344 Ops[4] = Builder.CreateBitCast(Ops[4], Ty); in EmitAArch64BuiltinExpr()
13345 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); in EmitAArch64BuiltinExpr()
13346 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld4_lane"); in EmitAArch64BuiltinExpr()
13347 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
13351 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13352 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; in EmitAArch64BuiltinExpr()
13354 Ops, ""); in EmitAArch64BuiltinExpr()
13358 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13359 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitAArch64BuiltinExpr()
13360 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
13362 Ops, ""); in EmitAArch64BuiltinExpr()
13366 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13367 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
13369 Ops, ""); in EmitAArch64BuiltinExpr()
13373 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13374 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
13375 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
13377 Ops, ""); in EmitAArch64BuiltinExpr()
13381 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13382 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
13384 Ops, ""); in EmitAArch64BuiltinExpr()
13388 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
13389 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
13390 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; in EmitAArch64BuiltinExpr()
13392 Ops, ""); in EmitAArch64BuiltinExpr()
13396 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13397 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13406 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
13407 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitAArch64BuiltinExpr()
13414 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13415 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13423 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
13424 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitAArch64BuiltinExpr()
13431 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
13432 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
13441 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
13442 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitAArch64BuiltinExpr()
13449 Ops, "vtbl1"); in EmitAArch64BuiltinExpr()
13453 Ops, "vtbl2"); in EmitAArch64BuiltinExpr()
13457 Ops, "vtbl3"); in EmitAArch64BuiltinExpr()
13461 Ops, "vtbl4"); in EmitAArch64BuiltinExpr()
13465 Ops, "vtbx1"); in EmitAArch64BuiltinExpr()
13469 Ops, "vtbx2"); in EmitAArch64BuiltinExpr()
13473 Ops, "vtbx3"); in EmitAArch64BuiltinExpr()
13477 Ops, "vtbx4"); in EmitAArch64BuiltinExpr()
13482 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); in EmitAArch64BuiltinExpr()
13487 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); in EmitAArch64BuiltinExpr()
13601 BuildVector(ArrayRef<llvm::Value*> Ops) { in BuildVector() argument
13602 assert((Ops.size() & (Ops.size() - 1)) == 0 && in BuildVector()
13605 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) in BuildVector()
13606 AllConstants &= isa<Constant>(Ops[i]); in BuildVector()
13611 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
13612 CstOps.push_back(cast<Constant>(Ops[i])); in BuildVector()
13618 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size())); in BuildVector()
13620 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
13621 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt64(i)); in BuildVector()
13647 static Value *EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86MaskedStore() argument
13649 Value *Ptr = Ops[0]; in EmitX86MaskedStore()
13652 CGF, Ops[2], in EmitX86MaskedStore()
13653 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements()); in EmitX86MaskedStore()
13655 return CGF.Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec); in EmitX86MaskedStore()
13658 static Value *EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86MaskedLoad() argument
13660 llvm::Type *Ty = Ops[1]->getType(); in EmitX86MaskedLoad()
13661 Value *Ptr = Ops[0]; in EmitX86MaskedLoad()
13664 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements()); in EmitX86MaskedLoad()
13666 return CGF.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]); in EmitX86MaskedLoad()
13670 ArrayRef<Value *> Ops) { in EmitX86ExpandLoad() argument
13671 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType()); in EmitX86ExpandLoad()
13672 Value *Ptr = Ops[0]; in EmitX86ExpandLoad()
13675 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements()); in EmitX86ExpandLoad()
13679 return CGF.Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] }); in EmitX86ExpandLoad()
13683 ArrayRef<Value *> Ops, in EmitX86CompressExpand() argument
13685 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType()); in EmitX86CompressExpand()
13687 Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements()); in EmitX86CompressExpand()
13692 return CGF.Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec }); in EmitX86CompressExpand()
13696 ArrayRef<Value *> Ops) { in EmitX86CompressStore() argument
13697 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType()); in EmitX86CompressStore()
13698 Value *Ptr = Ops[0]; in EmitX86CompressStore()
13700 Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements()); in EmitX86CompressStore()
13704 return CGF.Builder.CreateCall(F, { Ops[1], Ptr, MaskVec }); in EmitX86CompressStore()
13708 ArrayRef<Value *> Ops, in EmitX86MaskLogic() argument
13710 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86MaskLogic()
13711 Value *LHS = getMaskVecValue(CGF, Ops[0], NumElts); in EmitX86MaskLogic()
13712 Value *RHS = getMaskVecValue(CGF, Ops[1], NumElts); in EmitX86MaskLogic()
13718 Ops[0]->getType()); in EmitX86MaskLogic()
13739 static Value *EmitX86vpcom(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86vpcom() argument
13741 Value *Op0 = Ops[0]; in EmitX86vpcom()
13742 Value *Op1 = Ops[1]; in EmitX86vpcom()
13744 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86vpcom()
13831 bool Signed, ArrayRef<Value *> Ops) { in EmitX86MaskedCompare() argument
13832 assert((Ops.size() == 2 || Ops.size() == 4) && in EmitX86MaskedCompare()
13835 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86MaskedCompare()
13855 Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]); in EmitX86MaskedCompare()
13859 if (Ops.size() == 4) in EmitX86MaskedCompare()
13860 MaskIn = Ops[3]; in EmitX86MaskedCompare()
13871 ArrayRef<Value *> Ops, bool IsSigned) { in EmitX86ConvertIntToFp() argument
13872 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue(); in EmitX86ConvertIntToFp()
13873 llvm::Type *Ty = Ops[1]->getType(); in EmitX86ConvertIntToFp()
13879 Function *F = CGF.CGM.getIntrinsic(IID, { Ty, Ops[0]->getType() }); in EmitX86ConvertIntToFp()
13880 Res = CGF.Builder.CreateCall(F, { Ops[0], Ops[3] }); in EmitX86ConvertIntToFp()
13883 Res = IsSigned ? CGF.Builder.CreateSIToFP(Ops[0], Ty) in EmitX86ConvertIntToFp()
13884 : CGF.Builder.CreateUIToFP(Ops[0], Ty); in EmitX86ConvertIntToFp()
13887 return EmitX86Select(CGF, Ops[2], Res, Ops[1]); in EmitX86ConvertIntToFp()
13892 ArrayRef<Value *> Ops, unsigned BuiltinID, in EmitX86FMAExpr() argument
13947 Value *A = Ops[0]; in EmitX86FMAExpr()
13948 Value *B = Ops[1]; in EmitX86FMAExpr()
13949 Value *C = Ops[2]; in EmitX86FMAExpr()
13958 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 || in EmitX86FMAExpr()
13961 Res = CGF.Builder.CreateCall(Intr, {A, B, C, Ops.back() }); in EmitX86FMAExpr()
13984 MaskFalseVal = Ops[0]; in EmitX86FMAExpr()
13992 MaskFalseVal = Constant::getNullValue(Ops[0]->getType()); in EmitX86FMAExpr()
14006 MaskFalseVal = Ops[2]; in EmitX86FMAExpr()
14011 return EmitX86Select(CGF, Ops[3], Res, MaskFalseVal); in EmitX86FMAExpr()
14017 MutableArrayRef<Value *> Ops, Value *Upper, in EmitScalarFMAExpr() argument
14021 if (Ops.size() > 4) in EmitScalarFMAExpr()
14022 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue(); in EmitScalarFMAExpr()
14025 Ops[2] = CGF.Builder.CreateFNeg(Ops[2]); in EmitScalarFMAExpr()
14027 Ops[0] = CGF.Builder.CreateExtractElement(Ops[0], (uint64_t)0); in EmitScalarFMAExpr()
14028 Ops[1] = CGF.Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitScalarFMAExpr()
14029 Ops[2] = CGF.Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitScalarFMAExpr()
14034 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) { in EmitScalarFMAExpr()
14048 {Ops[0], Ops[1], Ops[2], Ops[4]}); in EmitScalarFMAExpr()
14052 Intrinsic::experimental_constrained_fma, Ops[0]->getType()); in EmitScalarFMAExpr()
14053 Res = CGF.Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3)); in EmitScalarFMAExpr()
14055 Function *FMA = CGF.CGM.getIntrinsic(Intrinsic::fma, Ops[0]->getType()); in EmitScalarFMAExpr()
14056 Res = CGF.Builder.CreateCall(FMA, Ops.slice(0, 3)); in EmitScalarFMAExpr()
14059 if (Ops.size() > 3) { in EmitScalarFMAExpr()
14061 : Ops[PTIdx]; in EmitScalarFMAExpr()
14069 Res = EmitX86ScalarSelect(CGF, Ops[3], Res, PassThru); in EmitScalarFMAExpr()
14075 ArrayRef<Value *> Ops) { in EmitX86Muldq() argument
14076 llvm::Type *Ty = Ops[0]->getType(); in EmitX86Muldq()
14080 Value *LHS = CGF.Builder.CreateBitCast(Ops[0], Ty); in EmitX86Muldq()
14081 Value *RHS = CGF.Builder.CreateBitCast(Ops[1], Ty); in EmitX86Muldq()
14104 ArrayRef<Value *> Ops) { in EmitX86Ternlog() argument
14105 llvm::Type *Ty = Ops[0]->getType(); in EmitX86Ternlog()
14126 Ops.drop_back()); in EmitX86Ternlog()
14127 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0]; in EmitX86Ternlog()
14128 return EmitX86Select(CGF, Ops[4], Ternlog, PassThru); in EmitX86Ternlog()
14147 ArrayRef<Value *> Ops, in EmitX86CvtF16ToFloatExpr() argument
14149 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) && in EmitX86CvtF16ToFloatExpr()
14153 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) { in EmitX86CvtF16ToFloatExpr()
14156 return CGF.Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]}); in EmitX86CvtF16ToFloatExpr()
14160 Value *Src = Ops[0]; in EmitX86CvtF16ToFloatExpr()
14177 if (Ops.size() >= 3) in EmitX86CvtF16ToFloatExpr()
14178 Res = EmitX86Select(CGF, Ops[2], Res, Ops[1]); in EmitX86CvtF16ToFloatExpr()
14372 SmallVector<Value*, 4> Ops; in EmitX86BuiltinExpr() local
14383 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E)); in EmitX86BuiltinExpr()
14392 auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) { in EmitX86BuiltinExpr()
14393 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm)); in EmitX86BuiltinExpr()
14395 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
14403 auto getVectorFCmpIR = [this, &Ops, E](CmpInst::Predicate Pred, in EmitX86BuiltinExpr()
14408 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
14410 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
14411 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
14420 Value *Address = Ops[0]; in EmitX86BuiltinExpr()
14421 ConstantInt *C = cast<ConstantInt>(Ops[1]); in EmitX86BuiltinExpr()
14430 Ops[0]); in EmitX86BuiltinExpr()
14450 Ops[0]); in EmitX86BuiltinExpr()
14456 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitX86BuiltinExpr()
14457 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitX86BuiltinExpr()
14462 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType()); in EmitX86BuiltinExpr()
14463 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitX86BuiltinExpr()
14477 return Builder.CreateBitCast(BuildVector(Ops), in EmitX86BuiltinExpr()
14490 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14491 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
14495 return Builder.CreateExtractElement(Ops[0], Index); in EmitX86BuiltinExpr()
14506 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14507 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
14511 return Builder.CreateInsertElement(Ops[0], Ops[1], Index); in EmitX86BuiltinExpr()
14516 Builder.CreateStore(Ops[0], Tmp); in EmitX86BuiltinExpr()
14567 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); in EmitX86BuiltinExpr()
14568 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); in EmitX86BuiltinExpr()
14569 Ops[1] = Mhi; in EmitX86BuiltinExpr()
14570 Ops.push_back(Mlo); in EmitX86BuiltinExpr()
14571 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
14575 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_xgetbv), Ops); in EmitX86BuiltinExpr()
14594 return EmitX86MaskedStore(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
14599 return EmitX86MaskedStore(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
14615 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
14629 return EmitX86SExtMask(*this, Ops[0], ConvertType(E->getType())); in EmitX86BuiltinExpr()
14643 return EmitX86ConvertToMask(*this, Ops[0]); in EmitX86BuiltinExpr()
14651 return EmitX86ConvertIntToFp(*this, E, Ops, /*IsSigned*/ true); in EmitX86BuiltinExpr()
14658 return EmitX86ConvertIntToFp(*this, E, Ops, /*IsSigned*/ false); in EmitX86BuiltinExpr()
14665 return EmitScalarFMAExpr(*this, E, Ops, Ops[0]); in EmitX86BuiltinExpr()
14668 return EmitScalarFMAExpr(*this, E, Ops, in EmitX86BuiltinExpr()
14669 Constant::getNullValue(Ops[0]->getType())); in EmitX86BuiltinExpr()
14673 return EmitScalarFMAExpr(*this, E, Ops, Ops[0], /*ZeroMask*/ true); in EmitX86BuiltinExpr()
14677 return EmitScalarFMAExpr(*this, E, Ops, Ops[2], /*ZeroMask*/ false, 2); in EmitX86BuiltinExpr()
14681 return EmitScalarFMAExpr(*this, E, Ops, Ops[2], /*ZeroMask*/ false, 2, in EmitX86BuiltinExpr()
14701 return EmitX86FMAExpr(*this, E, Ops, BuiltinID, /*IsAddSub*/ false); in EmitX86BuiltinExpr()
14714 return EmitX86FMAExpr(*this, E, Ops, BuiltinID, /*IsAddSub*/ true); in EmitX86BuiltinExpr()
14729 *this, Ops, in EmitX86BuiltinExpr()
14750 return EmitX86MaskedLoad(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
14755 return EmitX86MaskedLoad(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
14770 *this, Ops, in EmitX86BuiltinExpr()
14791 return EmitX86ExpandLoad(*this, Ops); in EmitX86BuiltinExpr()
14811 return EmitX86CompressStore(*this, Ops); in EmitX86BuiltinExpr()
14831 return EmitX86CompressExpand(*this, Ops, /*IsCompress*/false); in EmitX86BuiltinExpr()
14851 return EmitX86CompressExpand(*this, Ops, /*IsCompress*/true); in EmitX86BuiltinExpr()
14955 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(), in EmitX86BuiltinExpr()
14956 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements()); in EmitX86BuiltinExpr()
14957 Ops[3] = getMaskVecValue(*this, Ops[3], MinElts); in EmitX86BuiltinExpr()
14959 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
15064 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(), in EmitX86BuiltinExpr()
15065 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements()); in EmitX86BuiltinExpr()
15066 Ops[1] = getMaskVecValue(*this, Ops[1], MinElts); in EmitX86BuiltinExpr()
15068 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
15090 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15092 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
15101 Value *Res = Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
15104 if (Ops.size() == 4) in EmitX86BuiltinExpr()
15105 Res = EmitX86Select(*this, Ops[3], Res, Ops[2]); in EmitX86BuiltinExpr()
15126 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15128 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15130 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
15140 Ops[1], ArrayRef(Indices, DstNumElts), "widen"); in EmitX86BuiltinExpr()
15149 return Builder.CreateShuffleVector(Ops[0], Op1, in EmitX86BuiltinExpr()
15154 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType()); in EmitX86BuiltinExpr()
15155 return EmitX86Select(*this, Ops[2], Res, Ops[1]); in EmitX86BuiltinExpr()
15160 if (const auto *C = dyn_cast<Constant>(Ops[2])) in EmitX86BuiltinExpr()
15162 return Builder.CreateTrunc(Ops[0], Ops[1]->getType()); in EmitX86BuiltinExpr()
15179 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
15190 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15191 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
15199 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
15205 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
15206 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15222 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
15228 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
15229 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15245 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
15257 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
15258 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15274 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
15283 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
15284 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15303 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
15310 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
15311 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15320 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
15326 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15329 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15341 Ops[1] = Ops[0]; in EmitX86BuiltinExpr()
15342 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
15356 return Builder.CreateShuffleVector(Ops[1], Ops[0], in EmitX86BuiltinExpr()
15366 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15367 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15376 return Builder.CreateShuffleVector(Ops[1], Ops[0], in EmitX86BuiltinExpr()
15387 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
15388 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15404 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
15412 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
15414 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15426 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType()); in EmitX86BuiltinExpr()
15428 OutOps[l] = Ops[1]; in EmitX86BuiltinExpr()
15430 OutOps[l] = Ops[0]; in EmitX86BuiltinExpr()
15450 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15451 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15470 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
15474 return Builder.CreateBitCast(SV, Ops[0]->getType(), "cast"); in EmitX86BuiltinExpr()
15479 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15480 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
15499 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
15509 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15510 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15513 return llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
15515 Value *In = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15524 return Builder.CreateBitCast(SV, Ops[0]->getType()); in EmitX86BuiltinExpr()
15530 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
15531 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15534 return llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
15536 Value *In = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15545 return Builder.CreateBitCast(SV, Ops[0]->getType()); in EmitX86BuiltinExpr()
15554 Value *Ptr = Ops[0]; in EmitX86BuiltinExpr()
15555 Value *Src = Ops[1]; in EmitX86BuiltinExpr()
15589 return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], false); in EmitX86BuiltinExpr()
15602 return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], true); in EmitX86BuiltinExpr()
15627 return EmitX86Select(*this, Ops[0], Ops[1], Ops[2]); in EmitX86BuiltinExpr()
15632 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitX86BuiltinExpr()
15633 Value *B = Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitX86BuiltinExpr()
15634 A = EmitX86ScalarSelect(*this, Ops[0], A, B); in EmitX86BuiltinExpr()
15635 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0); in EmitX86BuiltinExpr()
15649 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
15650 return EmitX86MaskedCompare(*this, CC, true, Ops); in EmitX86BuiltinExpr()
15664 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
15665 return EmitX86MaskedCompare(*this, CC, false, Ops); in EmitX86BuiltinExpr()
15671 return EmitX86vpcom(*this, Ops, true); in EmitX86BuiltinExpr()
15676 return EmitX86vpcom(*this, Ops, false); in EmitX86BuiltinExpr()
15682 Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
15683 Value *C = llvm::Constant::getAllOnesValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
15691 Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
15692 Value *C = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
15734 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15735 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15736 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
15762 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15763 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15764 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
15767 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
15773 return EmitX86MaskLogic(*this, Instruction::And, Ops); in EmitX86BuiltinExpr()
15778 return EmitX86MaskLogic(*this, Instruction::And, Ops, true); in EmitX86BuiltinExpr()
15783 return EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
15788 return EmitX86MaskLogic(*this, Instruction::Xor, Ops, true); in EmitX86BuiltinExpr()
15793 return EmitX86MaskLogic(*this, Instruction::Xor, Ops); in EmitX86BuiltinExpr()
15798 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15799 Value *Res = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15801 Ops[0]->getType()); in EmitX86BuiltinExpr()
15810 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15811 Value *Res = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15812 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
15818 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
15819 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
15820 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
15833 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
15842 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitX86BuiltinExpr()
15843 return Builder.CreateCall(F, {Ops[0],Builder.getInt1(false)}); in EmitX86BuiltinExpr()
15847 Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0); in EmitX86BuiltinExpr()
15858 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0); in EmitX86BuiltinExpr()
15863 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue(); in EmitX86BuiltinExpr()
15882 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
15884 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitX86BuiltinExpr()
15895 Value *Src = Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitX86BuiltinExpr()
15896 A = EmitX86ScalarSelect(*this, Ops[3], A, Src); in EmitX86BuiltinExpr()
15897 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0); in EmitX86BuiltinExpr()
15908 if (Ops.size() == 2) { in EmitX86BuiltinExpr()
15909 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
15928 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
15934 Ops[0]->getType()); in EmitX86BuiltinExpr()
15935 return Builder.CreateConstrainedFPCall(F, Ops[0]); in EmitX86BuiltinExpr()
15937 Function *F = CGM.getIntrinsic(Intrinsic::sqrt, Ops[0]->getType()); in EmitX86BuiltinExpr()
15938 return Builder.CreateCall(F, Ops[0]); in EmitX86BuiltinExpr()
15945 return EmitX86Muldq(*this, /*IsSigned*/false, Ops); in EmitX86BuiltinExpr()
15950 return EmitX86Muldq(*this, /*IsSigned*/true, Ops); in EmitX86BuiltinExpr()
15958 return EmitX86Ternlog(*this, /*ZeroMask*/false, Ops); in EmitX86BuiltinExpr()
15966 return EmitX86Ternlog(*this, /*ZeroMask*/true, Ops); in EmitX86BuiltinExpr()
15977 return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false); in EmitX86BuiltinExpr()
15989 return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true); in EmitX86BuiltinExpr()
16000 return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false); in EmitX86BuiltinExpr()
16012 return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true); in EmitX86BuiltinExpr()
16021 CGM.getIntrinsic(Intrinsic::vector_reduce_fadd, Ops[1]->getType()); in EmitX86BuiltinExpr()
16024 return Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
16032 CGM.getIntrinsic(Intrinsic::vector_reduce_fmul, Ops[1]->getType()); in EmitX86BuiltinExpr()
16035 return Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
16043 CGM.getIntrinsic(Intrinsic::vector_reduce_fmax, Ops[0]->getType()); in EmitX86BuiltinExpr()
16046 return Builder.CreateCall(F, {Ops[0]}); in EmitX86BuiltinExpr()
16054 CGM.getIntrinsic(Intrinsic::vector_reduce_fmin, Ops[0]->getType()); in EmitX86BuiltinExpr()
16057 return Builder.CreateCall(F, {Ops[0]}); in EmitX86BuiltinExpr()
16091 Ops[0]); in EmitX86BuiltinExpr()
16116 { Ops[0], Ops[1], Ops[2] }); in EmitX86BuiltinExpr()
16118 Ops[3]); in EmitX86BuiltinExpr()
16132 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
16133 Value *MaskIn = Ops[2]; in EmitX86BuiltinExpr()
16134 Ops.erase(&Ops[2]); in EmitX86BuiltinExpr()
16168 Value *Fpclass = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
16179 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
16204 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID), {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
16207 Builder.CreateDefaultAlignedStore(Result, Ops[2]); in EmitX86BuiltinExpr()
16211 return Builder.CreateDefaultAlignedStore(Result, Ops[3]); in EmitX86BuiltinExpr()
16231 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
16238 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
16239 Value *MaskIn = Ops[2]; in EmitX86BuiltinExpr()
16240 Ops.erase(&Ops[2]); in EmitX86BuiltinExpr()
16256 Value *Shufbit = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
16307 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f; in EmitX86BuiltinExpr()
16396 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
16397 Ops[3] = getMaskVecValue(*this, Ops[3], NumElts); in EmitX86BuiltinExpr()
16398 Value *Cmp = Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
16402 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
16413 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
16416 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
16418 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
16419 return EmitX86MaskedCompareResult(*this, Cmp, NumElts, Ops[3]); in EmitX86BuiltinExpr()
16466 return EmitX86CvtF16ToFloatExpr(*this, Ops, ConvertType(E->getType())); in EmitX86BuiltinExpr()
16471 Ops[2] = getMaskVecValue( in EmitX86BuiltinExpr()
16472 *this, Ops[2], in EmitX86BuiltinExpr()
16473 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements()); in EmitX86BuiltinExpr()
16475 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
16478 return Builder.CreateFPExt(Ops[0], Builder.getFloatTy()); in EmitX86BuiltinExpr()
16492 Value *Res = Builder.CreateCall(CGM.getIntrinsic(IID), Ops[0]); in EmitX86BuiltinExpr()
16493 return EmitX86Select(*this, Ops[2], Res, Ops[1]); in EmitX86BuiltinExpr()
16540 Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned); in EmitX86BuiltinExpr()
16541 Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned); in EmitX86BuiltinExpr()
16552 Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned); in EmitX86BuiltinExpr()
16553 Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned); in EmitX86BuiltinExpr()
16585 std::swap(Ops[0], Ops[1]); in EmitX86BuiltinExpr()
16586 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitX86BuiltinExpr()
16587 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
16604 return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], Align(1), true); in EmitX86BuiltinExpr()
16627 Ops[0], llvm::PointerType::get(getLLVMContext(), 257)); in EmitX86BuiltinExpr()
16639 Ops[0], llvm::PointerType::get(getLLVMContext(), 256)); in EmitX86BuiltinExpr()
16648 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
16652 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[2], i * 16); in EmitX86BuiltinExpr()
16662 Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1], Ops[2]}); in EmitX86BuiltinExpr()
16666 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[3], i * 16); in EmitX86BuiltinExpr()
16699 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[1], Ops[2]}); in EmitX86BuiltinExpr()
16712 Builder.CreateDefaultAlignedStore(Out, Ops[0]); in EmitX86BuiltinExpr()
16717 Builder.CreateDefaultAlignedStore(Zero, Ops[0]); in EmitX86BuiltinExpr()
16750 InOps[0] = Ops[2]; in EmitX86BuiltinExpr()
16752 Value *Ptr = Builder.CreateConstGEP1_32(Ty, Ops[1], i); in EmitX86BuiltinExpr()
16770 Value *Ptr = Builder.CreateConstGEP1_32(Extract->getType(), Ops[0], i); in EmitX86BuiltinExpr()
16779 Value *Ptr = Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i); in EmitX86BuiltinExpr()
16794 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
16795 return EmitX86Select(*this, Ops[3], Call, Ops[0]); in EmitX86BuiltinExpr()
16803 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
16804 Value *And = Builder.CreateAnd(Ops[3], llvm::ConstantInt::get(Int8Ty, 1)); in EmitX86BuiltinExpr()
16805 return EmitX86Select(*this, And, Call, Ops[0]); in EmitX86BuiltinExpr()
16813 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
16815 return Builder.CreateShuffleVector(Call, Ops[2], Mask); in EmitX86BuiltinExpr()
16819 CGM.getIntrinsic(Intrinsic::prefetch, Ops[0]->getType()), in EmitX86BuiltinExpr()
16820 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1], in EmitX86BuiltinExpr()
16999 SmallVector<Value *, 2> Ops; in EmitPPCBuiltinExpr() local
17000 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitPPCBuiltinExpr()
17001 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitPPCBuiltinExpr()
17004 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]); in EmitPPCBuiltinExpr()
17005 Ops.pop_back(); in EmitPPCBuiltinExpr()
17051 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
17067 SmallVector<Value *, 3> Ops; in EmitPPCBuiltinExpr() local
17068 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitPPCBuiltinExpr()
17069 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitPPCBuiltinExpr()
17070 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitPPCBuiltinExpr()
17073 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]); in EmitPPCBuiltinExpr()
17074 Ops.pop_back(); in EmitPPCBuiltinExpr()
17114 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
17371 SmallVector<Value *, 2> Ops; in EmitPPCBuiltinExpr() local
17376 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty)); in EmitPPCBuiltinExpr()
17377 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty)); in EmitPPCBuiltinExpr()
17381 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, ""); in EmitPPCBuiltinExpr()
17387 SmallVector<Value *, 3> Ops; in EmitPPCBuiltinExpr() local
17393 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty)); in EmitPPCBuiltinExpr()
17394 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty)); in EmitPPCBuiltinExpr()
17395 Ops.push_back(Builder.CreateBitCast(Op2, V1I128Ty)); in EmitPPCBuiltinExpr()
17412 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, ""); in EmitPPCBuiltinExpr()
17846 SmallVector<Value *, 4> Ops; in EmitPPCBuiltinExpr() local
17849 Ops.push_back( in EmitPPCBuiltinExpr()
17852 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitPPCBuiltinExpr()
17871 Value *Ptr = Ops[0]; in EmitPPCBuiltinExpr()
17889 std::reverse(Ops.begin() + 1, Ops.end()); in EmitPPCBuiltinExpr()
17906 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]); in EmitPPCBuiltinExpr()
17908 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]); in EmitPPCBuiltinExpr()
17910 Ops.pop_back(); in EmitPPCBuiltinExpr()
17912 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
17920 for (unsigned i=1; i<Ops.size(); i++) in EmitPPCBuiltinExpr()
17921 CallOps.push_back(Ops[i]); in EmitPPCBuiltinExpr()
17924 return Builder.CreateAlignedStore(Call, Ops[0], MaybeAlign(64)); in EmitPPCBuiltinExpr()
21363 Value *Ops[18]; in EmitWebAssemblyBuiltinExpr() local
21365 Ops[OpIdx++] = EmitScalarExpr(E->getArg(0)); in EmitWebAssemblyBuiltinExpr()
21366 Ops[OpIdx++] = EmitScalarExpr(E->getArg(1)); in EmitWebAssemblyBuiltinExpr()
21371 Ops[OpIdx++] = llvm::ConstantInt::get(getLLVMContext(), *LaneConst); in EmitWebAssemblyBuiltinExpr()
21374 return Builder.CreateCall(Callee, Ops); in EmitWebAssemblyBuiltinExpr()
21680 SmallVector<llvm::Value*,5> Ops = { Base }; in EmitHexagonBuiltinExpr() local
21682 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitHexagonBuiltinExpr()
21684 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops); in EmitHexagonBuiltinExpr()
21795 SmallVector<llvm::Value*,4> Ops; in EmitHexagonBuiltinExpr() local
21801 Ops.push_back(V2Q(EmitScalarExpr(PredOp))); in EmitHexagonBuiltinExpr()
21804 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitHexagonBuiltinExpr()
21805 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitHexagonBuiltinExpr()
21852 SmallVector<Value *, 4> Ops; in EmitRISCVBuiltinExpr() local
21881 Ops.push_back(AggValue); in EmitRISCVBuiltinExpr()
21884 Ops.push_back(EmitScalarOrConstFoldImmArg(ICEArguments, i, E)); in EmitRISCVBuiltinExpr()
21929 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitRISCVBuiltinExpr()
21930 Value *Result = Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitRISCVBuiltinExpr()
21938 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType()); in EmitRISCVBuiltinExpr()
21939 Value *Result = Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitRISCVBuiltinExpr()
22023 if (Ops.size() == 2) in EmitRISCVBuiltinExpr()
22024 DomainVal = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitRISCVBuiltinExpr()
22041 Address(Ops[0], ResTy, CharUnits::fromQuantity(Width / 8))); in EmitRISCVBuiltinExpr()
22051 if (Ops.size() == 3) in EmitRISCVBuiltinExpr()
22052 DomainVal = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitRISCVBuiltinExpr()
22060 StoreInst *Store = Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitRISCVBuiltinExpr()
22077 return Builder.CreateCall(F, Ops, ""); in EmitRISCVBuiltinExpr()