xref: /freebsd/contrib/llvm-project/llvm/lib/IR/IRBuilder.cpp (revision 6966ac055c3b7a39266fb982493330df7a097997) 
1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IRBuilder class, which is used as a convenient way
10 // to create LLVM instructions with a consistent and simplified interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/IR/Constant.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DerivedTypes.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/GlobalVariable.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Operator.h"
27 #include "llvm/IR/Statepoint.h"
28 #include "llvm/IR/Type.h"
29 #include "llvm/IR/Value.h"
30 #include "llvm/Support/Casting.h"
31 #include "llvm/Support/MathExtras.h"
32 #include <cassert>
33 #include <cstdint>
34 #include <vector>
35 
36 using namespace llvm;
37 
38 /// CreateGlobalString - Make a new global variable with an initializer that
39 /// has array of i8 type filled in with the nul terminated string value
40 /// specified.  If Name is specified, it is the name of the global variable
41 /// created.
42 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
43                                                   const Twine &Name,
44                                                   unsigned AddressSpace) {
45   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
46   Module &M = *BB->getParent()->getParent();
47   auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
48                                 GlobalValue::PrivateLinkage, StrConstant, Name,
49                                 nullptr, GlobalVariable::NotThreadLocal,
50                                 AddressSpace);
51   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
52   GV->setAlignment(1);
53   return GV;
54 }
55 
56 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
57   assert(BB && BB->getParent() && "No current function!");
58   return BB->getParent()->getReturnType();
59 }
60 
61 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
62   auto *PT = cast<PointerType>(Ptr->getType());
63   if (PT->getElementType()->isIntegerTy(8))
64     return Ptr;
65 
66   // Otherwise, we need to insert a bitcast.
67   PT = getInt8PtrTy(PT->getAddressSpace());
68   BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
69   BB->getInstList().insert(InsertPt, BCI);
70   SetInstDebugLocation(BCI);
71   return BCI;
72 }
73 
74 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
75                                   IRBuilderBase *Builder,
76                                   const Twine &Name = "",
77                                   Instruction *FMFSource = nullptr) {
78   CallInst *CI = CallInst::Create(Callee, Ops, Name);
79   if (FMFSource)
80     CI->copyFastMathFlags(FMFSource);
81   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
82   Builder->SetInstDebugLocation(CI);
83   return CI;
84 }
85 
86 static InvokeInst *createInvokeHelper(Function *Invokee, BasicBlock *NormalDest,
87                                       BasicBlock *UnwindDest,
88                                       ArrayRef<Value *> Ops,
89                                       IRBuilderBase *Builder,
90                                       const Twine &Name = "") {
91   InvokeInst *II =
92       InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
93   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
94                                                   II);
95   Builder->SetInstDebugLocation(II);
96   return II;
97 }
98 
99 CallInst *IRBuilderBase::
100 CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
101              bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
102              MDNode *NoAliasTag) {
103   Ptr = getCastedInt8PtrValue(Ptr);
104   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
105   Type *Tys[] = { Ptr->getType(), Size->getType() };
106   Module *M = BB->getParent()->getParent();
107   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
108 
109   CallInst *CI = createCallHelper(TheFn, Ops, this);
110 
111   if (Align > 0)
112     cast<MemSetInst>(CI)->setDestAlignment(Align);
113 
114   // Set the TBAA info if present.
115   if (TBAATag)
116     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
117 
118   if (ScopeTag)
119     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
120 
121   if (NoAliasTag)
122     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
123 
124   return CI;
125 }
126 
127 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
128     Value *Ptr, Value *Val, Value *Size, unsigned Align, uint32_t ElementSize,
129     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
130   assert(Align >= ElementSize &&
131          "Pointer alignment must be at least element size.");
132 
133   Ptr = getCastedInt8PtrValue(Ptr);
134   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
135   Type *Tys[] = {Ptr->getType(), Size->getType()};
136   Module *M = BB->getParent()->getParent();
137   Function *TheFn = Intrinsic::getDeclaration(
138       M, Intrinsic::memset_element_unordered_atomic, Tys);
139 
140   CallInst *CI = createCallHelper(TheFn, Ops, this);
141 
142   cast<AtomicMemSetInst>(CI)->setDestAlignment(Align);
143 
144   // Set the TBAA info if present.
145   if (TBAATag)
146     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
147 
148   if (ScopeTag)
149     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
150 
151   if (NoAliasTag)
152     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
153 
154   return CI;
155 }
156 
157 CallInst *IRBuilderBase::
158 CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
159              Value *Size, bool isVolatile, MDNode *TBAATag,
160              MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
161   assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
162   assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
163   Dst = getCastedInt8PtrValue(Dst);
164   Src = getCastedInt8PtrValue(Src);
165 
166   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
167   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
168   Module *M = BB->getParent()->getParent();
169   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
170 
171   CallInst *CI = createCallHelper(TheFn, Ops, this);
172 
173   auto* MCI = cast<MemCpyInst>(CI);
174   if (DstAlign > 0)
175     MCI->setDestAlignment(DstAlign);
176   if (SrcAlign > 0)
177     MCI->setSourceAlignment(SrcAlign);
178 
179   // Set the TBAA info if present.
180   if (TBAATag)
181     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
182 
183   // Set the TBAA Struct info if present.
184   if (TBAAStructTag)
185     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
186 
187   if (ScopeTag)
188     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
189 
190   if (NoAliasTag)
191     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
192 
193   return CI;
194 }
195 
196 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
197     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
198     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
199     MDNode *ScopeTag, MDNode *NoAliasTag) {
200   assert(DstAlign >= ElementSize &&
201          "Pointer alignment must be at least element size");
202   assert(SrcAlign >= ElementSize &&
203          "Pointer alignment must be at least element size");
204   Dst = getCastedInt8PtrValue(Dst);
205   Src = getCastedInt8PtrValue(Src);
206 
207   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
208   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
209   Module *M = BB->getParent()->getParent();
210   Function *TheFn = Intrinsic::getDeclaration(
211       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
212 
213   CallInst *CI = createCallHelper(TheFn, Ops, this);
214 
215   // Set the alignment of the pointer args.
216   auto *AMCI = cast<AtomicMemCpyInst>(CI);
217   AMCI->setDestAlignment(DstAlign);
218   AMCI->setSourceAlignment(SrcAlign);
219 
220   // Set the TBAA info if present.
221   if (TBAATag)
222     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
223 
224   // Set the TBAA Struct info if present.
225   if (TBAAStructTag)
226     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
227 
228   if (ScopeTag)
229     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
230 
231   if (NoAliasTag)
232     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
233 
234   return CI;
235 }
236 
237 CallInst *IRBuilderBase::
238 CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
239               Value *Size, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
240               MDNode *NoAliasTag) {
241   assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
242   assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
243   Dst = getCastedInt8PtrValue(Dst);
244   Src = getCastedInt8PtrValue(Src);
245 
246   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
247   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
248   Module *M = BB->getParent()->getParent();
249   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
250 
251   CallInst *CI = createCallHelper(TheFn, Ops, this);
252 
253   auto *MMI = cast<MemMoveInst>(CI);
254   if (DstAlign > 0)
255     MMI->setDestAlignment(DstAlign);
256   if (SrcAlign > 0)
257     MMI->setSourceAlignment(SrcAlign);
258 
259   // Set the TBAA info if present.
260   if (TBAATag)
261     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
262 
263   if (ScopeTag)
264     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
265 
266   if (NoAliasTag)
267     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
268 
269   return CI;
270 }
271 
272 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
273     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
274     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
275     MDNode *ScopeTag, MDNode *NoAliasTag) {
276   assert(DstAlign >= ElementSize &&
277          "Pointer alignment must be at least element size");
278   assert(SrcAlign >= ElementSize &&
279          "Pointer alignment must be at least element size");
280   Dst = getCastedInt8PtrValue(Dst);
281   Src = getCastedInt8PtrValue(Src);
282 
283   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
284   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
285   Module *M = BB->getParent()->getParent();
286   Function *TheFn = Intrinsic::getDeclaration(
287       M, Intrinsic::memmove_element_unordered_atomic, Tys);
288 
289   CallInst *CI = createCallHelper(TheFn, Ops, this);
290 
291   // Set the alignment of the pointer args.
292   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
293   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
294 
295   // Set the TBAA info if present.
296   if (TBAATag)
297     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
298 
299   // Set the TBAA Struct info if present.
300   if (TBAAStructTag)
301     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
302 
303   if (ScopeTag)
304     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
305 
306   if (NoAliasTag)
307     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
308 
309   return CI;
310 }
311 
312 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
313                                     Value *Src) {
314   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
315   Value *Ops[] = {Src};
316   Type *Tys[] = { Src->getType() };
317   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
318   return createCallHelper(Decl, Ops, Builder);
319 }
320 
321 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
322   Module *M = GetInsertBlock()->getParent()->getParent();
323   Value *Ops[] = {Acc, Src};
324   Type *Tys[] = {Acc->getType(), Src->getType()};
325   auto Decl = Intrinsic::getDeclaration(
326       M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys);
327   return createCallHelper(Decl, Ops, this);
328 }
329 
330 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
331   Module *M = GetInsertBlock()->getParent()->getParent();
332   Value *Ops[] = {Acc, Src};
333   Type *Tys[] = {Acc->getType(), Src->getType()};
334   auto Decl = Intrinsic::getDeclaration(
335       M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys);
336   return createCallHelper(Decl, Ops, this);
337 }
338 
339 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
340   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
341                                Src);
342 }
343 
344 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
345   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
346                                Src);
347 }
348 
349 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
350   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
351                                Src);
352 }
353 
354 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
355   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
356                                Src);
357 }
358 
359 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
360   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
361                                Src);
362 }
363 
364 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
365   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
366                      : Intrinsic::experimental_vector_reduce_umax;
367   return getReductionIntrinsic(this, ID, Src);
368 }
369 
370 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
371   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
372                      : Intrinsic::experimental_vector_reduce_umin;
373   return getReductionIntrinsic(this, ID, Src);
374 }
375 
376 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
377   auto Rdx = getReductionIntrinsic(
378       this, Intrinsic::experimental_vector_reduce_fmax, Src);
379   if (NoNaN) {
380     FastMathFlags FMF;
381     FMF.setNoNaNs();
382     Rdx->setFastMathFlags(FMF);
383   }
384   return Rdx;
385 }
386 
387 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
388   auto Rdx = getReductionIntrinsic(
389       this, Intrinsic::experimental_vector_reduce_fmin, Src);
390   if (NoNaN) {
391     FastMathFlags FMF;
392     FMF.setNoNaNs();
393     Rdx->setFastMathFlags(FMF);
394   }
395   return Rdx;
396 }
397 
398 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
399   assert(isa<PointerType>(Ptr->getType()) &&
400          "lifetime.start only applies to pointers.");
401   Ptr = getCastedInt8PtrValue(Ptr);
402   if (!Size)
403     Size = getInt64(-1);
404   else
405     assert(Size->getType() == getInt64Ty() &&
406            "lifetime.start requires the size to be an i64");
407   Value *Ops[] = { Size, Ptr };
408   Module *M = BB->getParent()->getParent();
409   Function *TheFn =
410       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
411   return createCallHelper(TheFn, Ops, this);
412 }
413 
414 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
415   assert(isa<PointerType>(Ptr->getType()) &&
416          "lifetime.end only applies to pointers.");
417   Ptr = getCastedInt8PtrValue(Ptr);
418   if (!Size)
419     Size = getInt64(-1);
420   else
421     assert(Size->getType() == getInt64Ty() &&
422            "lifetime.end requires the size to be an i64");
423   Value *Ops[] = { Size, Ptr };
424   Module *M = BB->getParent()->getParent();
425   Function *TheFn =
426       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
427   return createCallHelper(TheFn, Ops, this);
428 }
429 
430 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
431 
432   assert(isa<PointerType>(Ptr->getType()) &&
433          "invariant.start only applies to pointers.");
434   Ptr = getCastedInt8PtrValue(Ptr);
435   if (!Size)
436     Size = getInt64(-1);
437   else
438     assert(Size->getType() == getInt64Ty() &&
439            "invariant.start requires the size to be an i64");
440 
441   Value *Ops[] = {Size, Ptr};
442   // Fill in the single overloaded type: memory object type.
443   Type *ObjectPtr[1] = {Ptr->getType()};
444   Module *M = BB->getParent()->getParent();
445   Function *TheFn =
446       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
447   return createCallHelper(TheFn, Ops, this);
448 }
449 
450 CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
451   assert(Cond->getType() == getInt1Ty() &&
452          "an assumption condition must be of type i1");
453 
454   Value *Ops[] = { Cond };
455   Module *M = BB->getParent()->getParent();
456   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
457   return createCallHelper(FnAssume, Ops, this);
458 }
459 
460 /// Create a call to a Masked Load intrinsic.
461 /// \p Ptr      - base pointer for the load
462 /// \p Align    - alignment of the source location
463 /// \p Mask     - vector of booleans which indicates what vector lanes should
464 ///               be accessed in memory
465 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
466 ///               of the result
467 /// \p Name     - name of the result variable
468 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
469                                           Value *Mask, Value *PassThru,
470                                           const Twine &Name) {
471   auto *PtrTy = cast<PointerType>(Ptr->getType());
472   Type *DataTy = PtrTy->getElementType();
473   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
474   assert(Mask && "Mask should not be all-ones (null)");
475   if (!PassThru)
476     PassThru = UndefValue::get(DataTy);
477   Type *OverloadedTypes[] = { DataTy, PtrTy };
478   Value *Ops[] = { Ptr, getInt32(Align), Mask,  PassThru};
479   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
480                                OverloadedTypes, Name);
481 }
482 
483 /// Create a call to a Masked Store intrinsic.
484 /// \p Val   - data to be stored,
485 /// \p Ptr   - base pointer for the store
486 /// \p Align - alignment of the destination location
487 /// \p Mask  - vector of booleans which indicates what vector lanes should
488 ///            be accessed in memory
489 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
490                                            unsigned Align, Value *Mask) {
491   auto *PtrTy = cast<PointerType>(Ptr->getType());
492   Type *DataTy = PtrTy->getElementType();
493   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
494   assert(Mask && "Mask should not be all-ones (null)");
495   Type *OverloadedTypes[] = { DataTy, PtrTy };
496   Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
497   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
498 }
499 
500 /// Create a call to a Masked intrinsic, with given intrinsic Id,
501 /// an array of operands - Ops, and an array of overloaded types -
502 /// OverloadedTypes.
503 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
504                                                ArrayRef<Value *> Ops,
505                                                ArrayRef<Type *> OverloadedTypes,
506                                                const Twine &Name) {
507   Module *M = BB->getParent()->getParent();
508   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
509   return createCallHelper(TheFn, Ops, this, Name);
510 }
511 
512 /// Create a call to a Masked Gather intrinsic.
513 /// \p Ptrs     - vector of pointers for loading
514 /// \p Align    - alignment for one element
515 /// \p Mask     - vector of booleans which indicates what vector lanes should
516 ///               be accessed in memory
517 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
518 ///               of the result
519 /// \p Name     - name of the result variable
520 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
521                                             Value *Mask,  Value *PassThru,
522                                             const Twine& Name) {
523   auto PtrsTy = cast<VectorType>(Ptrs->getType());
524   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
525   unsigned NumElts = PtrsTy->getVectorNumElements();
526   Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
527 
528   if (!Mask)
529     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
530                                      NumElts));
531 
532   if (!PassThru)
533     PassThru = UndefValue::get(DataTy);
534 
535   Type *OverloadedTypes[] = {DataTy, PtrsTy};
536   Value * Ops[] = {Ptrs, getInt32(Align), Mask, PassThru};
537 
538   // We specify only one type when we create this intrinsic. Types of other
539   // arguments are derived from this type.
540   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
541                                Name);
542 }
543 
544 /// Create a call to a Masked Scatter intrinsic.
545 /// \p Data  - data to be stored,
546 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
547 ///            stored
548 /// \p Align - alignment for one element
549 /// \p Mask  - vector of booleans which indicates what vector lanes should
550 ///            be accessed in memory
551 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
552                                              unsigned Align, Value *Mask) {
553   auto PtrsTy = cast<VectorType>(Ptrs->getType());
554   auto DataTy = cast<VectorType>(Data->getType());
555   unsigned NumElts = PtrsTy->getVectorNumElements();
556 
557 #ifndef NDEBUG
558   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
559   assert(NumElts == DataTy->getVectorNumElements() &&
560          PtrTy->getElementType() == DataTy->getElementType() &&
561          "Incompatible pointer and data types");
562 #endif
563 
564   if (!Mask)
565     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
566                                      NumElts));
567 
568   Type *OverloadedTypes[] = {DataTy, PtrsTy};
569   Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
570 
571   // We specify only one type when we create this intrinsic. Types of other
572   // arguments are derived from this type.
573   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
574 }
575 
576 template <typename T0, typename T1, typename T2, typename T3>
577 static std::vector<Value *>
578 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
579                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
580                   ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
581                   ArrayRef<T3> GCArgs) {
582   std::vector<Value *> Args;
583   Args.push_back(B.getInt64(ID));
584   Args.push_back(B.getInt32(NumPatchBytes));
585   Args.push_back(ActualCallee);
586   Args.push_back(B.getInt32(CallArgs.size()));
587   Args.push_back(B.getInt32(Flags));
588   Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
589   Args.push_back(B.getInt32(TransitionArgs.size()));
590   Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
591   Args.push_back(B.getInt32(DeoptArgs.size()));
592   Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
593   Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
594 
595   return Args;
596 }
597 
598 template <typename T0, typename T1, typename T2, typename T3>
599 static CallInst *CreateGCStatepointCallCommon(
600     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
601     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
602     ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
603     const Twine &Name) {
604   // Extract out the type of the callee.
605   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
606   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
607          "actual callee must be a callable value");
608 
609   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
610   // Fill in the one generic type'd argument (the function is also vararg)
611   Type *ArgTypes[] = { FuncPtrType };
612   Function *FnStatepoint =
613     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
614                               ArgTypes);
615 
616   std::vector<Value *> Args =
617       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
618                         CallArgs, TransitionArgs, DeoptArgs, GCArgs);
619   return createCallHelper(FnStatepoint, Args, Builder, Name);
620 }
621 
622 CallInst *IRBuilderBase::CreateGCStatepointCall(
623     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
624     ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
625     ArrayRef<Value *> GCArgs, const Twine &Name) {
626   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
627       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
628       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
629 }
630 
631 CallInst *IRBuilderBase::CreateGCStatepointCall(
632     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
633     ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
634     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
635   return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
636       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
637       DeoptArgs, GCArgs, Name);
638 }
639 
640 CallInst *IRBuilderBase::CreateGCStatepointCall(
641     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
642     ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
643     ArrayRef<Value *> GCArgs, const Twine &Name) {
644   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
645       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
646       CallArgs, None, DeoptArgs, GCArgs, Name);
647 }
648 
649 template <typename T0, typename T1, typename T2, typename T3>
650 static InvokeInst *CreateGCStatepointInvokeCommon(
651     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
652     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
653     uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
654     ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
655   // Extract out the type of the callee.
656   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
657   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
658          "actual callee must be a callable value");
659 
660   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
661   // Fill in the one generic type'd argument (the function is also vararg)
662   Function *FnStatepoint = Intrinsic::getDeclaration(
663       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
664 
665   std::vector<Value *> Args =
666       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
667                         InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
668   return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
669                             Name);
670 }
671 
672 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
673     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
674     BasicBlock *NormalDest, BasicBlock *UnwindDest,
675     ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
676     ArrayRef<Value *> GCArgs, const Twine &Name) {
677   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
678       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
679       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
680       DeoptArgs, GCArgs, Name);
681 }
682 
683 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
684     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
685     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
686     ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
687     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
688   return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
689       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
690       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
691 }
692 
693 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
694     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
695     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
696     ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
697   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
698       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
699       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
700       Name);
701 }
702 
703 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
704                                        Type *ResultType,
705                                        const Twine &Name) {
706  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
707  Module *M = BB->getParent()->getParent();
708  Type *Types[] = {ResultType};
709  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
710 
711  Value *Args[] = {Statepoint};
712  return createCallHelper(FnGCResult, Args, this, Name);
713 }
714 
715 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
716                                          int BaseOffset,
717                                          int DerivedOffset,
718                                          Type *ResultType,
719                                          const Twine &Name) {
720  Module *M = BB->getParent()->getParent();
721  Type *Types[] = {ResultType};
722  Function *FnGCRelocate =
723      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
724 
725  Value *Args[] = {Statepoint,
726                   getInt32(BaseOffset),
727                   getInt32(DerivedOffset)};
728  return createCallHelper(FnGCRelocate, Args, this, Name);
729 }
730 
731 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
732                                               Instruction *FMFSource,
733                                               const Twine &Name) {
734   Module *M = BB->getModule();
735   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
736   return createCallHelper(Fn, {V}, this, Name, FMFSource);
737 }
738 
739 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
740                                                Value *RHS,
741                                                Instruction *FMFSource,
742                                                const Twine &Name) {
743   Module *M = BB->getModule();
744   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
745   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
746 }
747 
748 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
749                                          ArrayRef<Type *> Types,
750                                          ArrayRef<Value *> Args,
751                                          Instruction *FMFSource,
752                                          const Twine &Name) {
753   Module *M = BB->getModule();
754   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
755   return createCallHelper(Fn, Args, this, Name, FMFSource);
756 }
757