xref: /freebsd/contrib/llvm-project/llvm/lib/IR/IRBuilder.cpp (revision 05ab65497e06edd12683163480841aa9630b9d4c) 
1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IRBuilder class, which is used as a convenient way
10 // to create LLVM instructions with a consistent and simplified interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/IR/Constant.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DerivedTypes.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/GlobalVariable.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Operator.h"
27 #include "llvm/IR/Statepoint.h"
28 #include "llvm/IR/Type.h"
29 #include "llvm/IR/Value.h"
30 #include "llvm/Support/Casting.h"
31 #include "llvm/Support/MathExtras.h"
32 #include <cassert>
33 #include <cstdint>
34 #include <vector>
35 
36 using namespace llvm;
37 
38 /// CreateGlobalString - Make a new global variable with an initializer that
39 /// has array of i8 type filled in with the nul terminated string value
40 /// specified.  If Name is specified, it is the name of the global variable
41 /// created.
42 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
43                                                   const Twine &Name,
44                                                   unsigned AddressSpace) {
45   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
46   Module &M = *BB->getParent()->getParent();
47   auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
48                                 GlobalValue::PrivateLinkage, StrConstant, Name,
49                                 nullptr, GlobalVariable::NotThreadLocal,
50                                 AddressSpace);
51   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
52   GV->setAlignment(Align::None());
53   return GV;
54 }
55 
56 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
57   assert(BB && BB->getParent() && "No current function!");
58   return BB->getParent()->getReturnType();
59 }
60 
61 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
62   auto *PT = cast<PointerType>(Ptr->getType());
63   if (PT->getElementType()->isIntegerTy(8))
64     return Ptr;
65 
66   // Otherwise, we need to insert a bitcast.
67   PT = getInt8PtrTy(PT->getAddressSpace());
68   BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
69   BB->getInstList().insert(InsertPt, BCI);
70   SetInstDebugLocation(BCI);
71   return BCI;
72 }
73 
74 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
75                                   IRBuilderBase *Builder,
76                                   const Twine &Name = "",
77                                   Instruction *FMFSource = nullptr) {
78   CallInst *CI = CallInst::Create(Callee, Ops, Name);
79   if (FMFSource)
80     CI->copyFastMathFlags(FMFSource);
81   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
82   Builder->SetInstDebugLocation(CI);
83   return CI;
84 }
85 
86 static InvokeInst *createInvokeHelper(Function *Invokee, BasicBlock *NormalDest,
87                                       BasicBlock *UnwindDest,
88                                       ArrayRef<Value *> Ops,
89                                       IRBuilderBase *Builder,
90                                       const Twine &Name = "") {
91   InvokeInst *II =
92       InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
93   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
94                                                   II);
95   Builder->SetInstDebugLocation(II);
96   return II;
97 }
98 
99 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
100                                       MaybeAlign Align, bool isVolatile,
101                                       MDNode *TBAATag, MDNode *ScopeTag,
102                                       MDNode *NoAliasTag) {
103   Ptr = getCastedInt8PtrValue(Ptr);
104   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
105   Type *Tys[] = { Ptr->getType(), Size->getType() };
106   Module *M = BB->getParent()->getParent();
107   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
108 
109   CallInst *CI = createCallHelper(TheFn, Ops, this);
110 
111   if (Align)
112     cast<MemSetInst>(CI)->setDestAlignment(Align->value());
113 
114   // Set the TBAA info if present.
115   if (TBAATag)
116     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
117 
118   if (ScopeTag)
119     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
120 
121   if (NoAliasTag)
122     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
123 
124   return CI;
125 }
126 
127 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
128     Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
129     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
130 
131   Ptr = getCastedInt8PtrValue(Ptr);
132   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
133   Type *Tys[] = {Ptr->getType(), Size->getType()};
134   Module *M = BB->getParent()->getParent();
135   Function *TheFn = Intrinsic::getDeclaration(
136       M, Intrinsic::memset_element_unordered_atomic, Tys);
137 
138   CallInst *CI = createCallHelper(TheFn, Ops, this);
139 
140   cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
141 
142   // Set the TBAA info if present.
143   if (TBAATag)
144     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
145 
146   if (ScopeTag)
147     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
148 
149   if (NoAliasTag)
150     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
151 
152   return CI;
153 }
154 
155 CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,
156                                       unsigned SrcAlign, Value *Size,
157                                       bool isVolatile, MDNode *TBAATag,
158                                       MDNode *TBAAStructTag, MDNode *ScopeTag,
159                                       MDNode *NoAliasTag) {
160   return CreateMemCpy(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign),
161                       Size, isVolatile, TBAATag, TBAAStructTag, ScopeTag,
162                       NoAliasTag);
163 }
164 
165 CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, MaybeAlign DstAlign,
166                                       Value *Src, MaybeAlign SrcAlign,
167                                       Value *Size, bool isVolatile,
168                                       MDNode *TBAATag, MDNode *TBAAStructTag,
169                                       MDNode *ScopeTag, MDNode *NoAliasTag) {
170   Dst = getCastedInt8PtrValue(Dst);
171   Src = getCastedInt8PtrValue(Src);
172 
173   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
174   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
175   Module *M = BB->getParent()->getParent();
176   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
177 
178   CallInst *CI = createCallHelper(TheFn, Ops, this);
179 
180   auto* MCI = cast<MemCpyInst>(CI);
181   if (DstAlign)
182     MCI->setDestAlignment(*DstAlign);
183   if (SrcAlign)
184     MCI->setSourceAlignment(*SrcAlign);
185 
186   // Set the TBAA info if present.
187   if (TBAATag)
188     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
189 
190   // Set the TBAA Struct info if present.
191   if (TBAAStructTag)
192     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
193 
194   if (ScopeTag)
195     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
196 
197   if (NoAliasTag)
198     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
199 
200   return CI;
201 }
202 
203 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
204     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
205     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
206     MDNode *ScopeTag, MDNode *NoAliasTag) {
207   assert(DstAlign >= ElementSize &&
208          "Pointer alignment must be at least element size");
209   assert(SrcAlign >= ElementSize &&
210          "Pointer alignment must be at least element size");
211   Dst = getCastedInt8PtrValue(Dst);
212   Src = getCastedInt8PtrValue(Src);
213 
214   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
215   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
216   Module *M = BB->getParent()->getParent();
217   Function *TheFn = Intrinsic::getDeclaration(
218       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
219 
220   CallInst *CI = createCallHelper(TheFn, Ops, this);
221 
222   // Set the alignment of the pointer args.
223   auto *AMCI = cast<AtomicMemCpyInst>(CI);
224   AMCI->setDestAlignment(DstAlign);
225   AMCI->setSourceAlignment(SrcAlign);
226 
227   // Set the TBAA info if present.
228   if (TBAATag)
229     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
230 
231   // Set the TBAA Struct info if present.
232   if (TBAAStructTag)
233     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
234 
235   if (ScopeTag)
236     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
237 
238   if (NoAliasTag)
239     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
240 
241   return CI;
242 }
243 
244 CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
245                                        Value *Src, MaybeAlign SrcAlign,
246                                        Value *Size, bool isVolatile,
247                                        MDNode *TBAATag, MDNode *ScopeTag,
248                                        MDNode *NoAliasTag) {
249   Dst = getCastedInt8PtrValue(Dst);
250   Src = getCastedInt8PtrValue(Src);
251 
252   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
253   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
254   Module *M = BB->getParent()->getParent();
255   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
256 
257   CallInst *CI = createCallHelper(TheFn, Ops, this);
258 
259   auto *MMI = cast<MemMoveInst>(CI);
260   if (DstAlign)
261     MMI->setDestAlignment(*DstAlign);
262   if (SrcAlign)
263     MMI->setSourceAlignment(*SrcAlign);
264 
265   // Set the TBAA info if present.
266   if (TBAATag)
267     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
268 
269   if (ScopeTag)
270     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
271 
272   if (NoAliasTag)
273     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
274 
275   return CI;
276 }
277 
278 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
279     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
280     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
281     MDNode *ScopeTag, MDNode *NoAliasTag) {
282   assert(DstAlign >= ElementSize &&
283          "Pointer alignment must be at least element size");
284   assert(SrcAlign >= ElementSize &&
285          "Pointer alignment must be at least element size");
286   Dst = getCastedInt8PtrValue(Dst);
287   Src = getCastedInt8PtrValue(Src);
288 
289   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
290   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
291   Module *M = BB->getParent()->getParent();
292   Function *TheFn = Intrinsic::getDeclaration(
293       M, Intrinsic::memmove_element_unordered_atomic, Tys);
294 
295   CallInst *CI = createCallHelper(TheFn, Ops, this);
296 
297   // Set the alignment of the pointer args.
298   CI->addParamAttr(
299       0, Attribute::getWithAlignment(CI->getContext(), Align(DstAlign)));
300   CI->addParamAttr(
301       1, Attribute::getWithAlignment(CI->getContext(), Align(SrcAlign)));
302 
303   // Set the TBAA info if present.
304   if (TBAATag)
305     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
306 
307   // Set the TBAA Struct info if present.
308   if (TBAAStructTag)
309     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
310 
311   if (ScopeTag)
312     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
313 
314   if (NoAliasTag)
315     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
316 
317   return CI;
318 }
319 
320 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
321                                     Value *Src) {
322   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
323   Value *Ops[] = {Src};
324   Type *Tys[] = { Src->getType() };
325   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
326   return createCallHelper(Decl, Ops, Builder);
327 }
328 
329 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
330   Module *M = GetInsertBlock()->getParent()->getParent();
331   Value *Ops[] = {Acc, Src};
332   Type *Tys[] = {Acc->getType(), Src->getType()};
333   auto Decl = Intrinsic::getDeclaration(
334       M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys);
335   return createCallHelper(Decl, Ops, this);
336 }
337 
338 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
339   Module *M = GetInsertBlock()->getParent()->getParent();
340   Value *Ops[] = {Acc, Src};
341   Type *Tys[] = {Acc->getType(), Src->getType()};
342   auto Decl = Intrinsic::getDeclaration(
343       M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys);
344   return createCallHelper(Decl, Ops, this);
345 }
346 
347 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
348   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
349                                Src);
350 }
351 
352 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
353   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
354                                Src);
355 }
356 
357 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
358   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
359                                Src);
360 }
361 
362 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
363   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
364                                Src);
365 }
366 
367 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
368   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
369                                Src);
370 }
371 
372 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
373   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
374                      : Intrinsic::experimental_vector_reduce_umax;
375   return getReductionIntrinsic(this, ID, Src);
376 }
377 
378 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
379   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
380                      : Intrinsic::experimental_vector_reduce_umin;
381   return getReductionIntrinsic(this, ID, Src);
382 }
383 
384 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
385   auto Rdx = getReductionIntrinsic(
386       this, Intrinsic::experimental_vector_reduce_fmax, Src);
387   if (NoNaN) {
388     FastMathFlags FMF;
389     FMF.setNoNaNs();
390     Rdx->setFastMathFlags(FMF);
391   }
392   return Rdx;
393 }
394 
395 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
396   auto Rdx = getReductionIntrinsic(
397       this, Intrinsic::experimental_vector_reduce_fmin, Src);
398   if (NoNaN) {
399     FastMathFlags FMF;
400     FMF.setNoNaNs();
401     Rdx->setFastMathFlags(FMF);
402   }
403   return Rdx;
404 }
405 
406 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
407   assert(isa<PointerType>(Ptr->getType()) &&
408          "lifetime.start only applies to pointers.");
409   Ptr = getCastedInt8PtrValue(Ptr);
410   if (!Size)
411     Size = getInt64(-1);
412   else
413     assert(Size->getType() == getInt64Ty() &&
414            "lifetime.start requires the size to be an i64");
415   Value *Ops[] = { Size, Ptr };
416   Module *M = BB->getParent()->getParent();
417   Function *TheFn =
418       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
419   return createCallHelper(TheFn, Ops, this);
420 }
421 
422 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
423   assert(isa<PointerType>(Ptr->getType()) &&
424          "lifetime.end only applies to pointers.");
425   Ptr = getCastedInt8PtrValue(Ptr);
426   if (!Size)
427     Size = getInt64(-1);
428   else
429     assert(Size->getType() == getInt64Ty() &&
430            "lifetime.end requires the size to be an i64");
431   Value *Ops[] = { Size, Ptr };
432   Module *M = BB->getParent()->getParent();
433   Function *TheFn =
434       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
435   return createCallHelper(TheFn, Ops, this);
436 }
437 
438 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
439 
440   assert(isa<PointerType>(Ptr->getType()) &&
441          "invariant.start only applies to pointers.");
442   Ptr = getCastedInt8PtrValue(Ptr);
443   if (!Size)
444     Size = getInt64(-1);
445   else
446     assert(Size->getType() == getInt64Ty() &&
447            "invariant.start requires the size to be an i64");
448 
449   Value *Ops[] = {Size, Ptr};
450   // Fill in the single overloaded type: memory object type.
451   Type *ObjectPtr[1] = {Ptr->getType()};
452   Module *M = BB->getParent()->getParent();
453   Function *TheFn =
454       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
455   return createCallHelper(TheFn, Ops, this);
456 }
457 
458 CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
459   assert(Cond->getType() == getInt1Ty() &&
460          "an assumption condition must be of type i1");
461 
462   Value *Ops[] = { Cond };
463   Module *M = BB->getParent()->getParent();
464   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
465   return createCallHelper(FnAssume, Ops, this);
466 }
467 
468 /// Create a call to a Masked Load intrinsic.
469 /// \p Ptr      - base pointer for the load
470 /// \p Align    - alignment of the source location
471 /// \p Mask     - vector of booleans which indicates what vector lanes should
472 ///               be accessed in memory
473 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
474 ///               of the result
475 /// \p Name     - name of the result variable
476 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
477                                           Value *Mask, Value *PassThru,
478                                           const Twine &Name) {
479   auto *PtrTy = cast<PointerType>(Ptr->getType());
480   Type *DataTy = PtrTy->getElementType();
481   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
482   assert(Mask && "Mask should not be all-ones (null)");
483   if (!PassThru)
484     PassThru = UndefValue::get(DataTy);
485   Type *OverloadedTypes[] = { DataTy, PtrTy };
486   Value *Ops[] = { Ptr, getInt32(Align), Mask,  PassThru};
487   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
488                                OverloadedTypes, Name);
489 }
490 
491 /// Create a call to a Masked Store intrinsic.
492 /// \p Val   - data to be stored,
493 /// \p Ptr   - base pointer for the store
494 /// \p Align - alignment of the destination location
495 /// \p Mask  - vector of booleans which indicates what vector lanes should
496 ///            be accessed in memory
497 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
498                                            unsigned Align, Value *Mask) {
499   auto *PtrTy = cast<PointerType>(Ptr->getType());
500   Type *DataTy = PtrTy->getElementType();
501   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
502   assert(Mask && "Mask should not be all-ones (null)");
503   Type *OverloadedTypes[] = { DataTy, PtrTy };
504   Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
505   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
506 }
507 
508 /// Create a call to a Masked intrinsic, with given intrinsic Id,
509 /// an array of operands - Ops, and an array of overloaded types -
510 /// OverloadedTypes.
511 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
512                                                ArrayRef<Value *> Ops,
513                                                ArrayRef<Type *> OverloadedTypes,
514                                                const Twine &Name) {
515   Module *M = BB->getParent()->getParent();
516   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
517   return createCallHelper(TheFn, Ops, this, Name);
518 }
519 
520 /// Create a call to a Masked Gather intrinsic.
521 /// \p Ptrs     - vector of pointers for loading
522 /// \p Align    - alignment for one element
523 /// \p Mask     - vector of booleans which indicates what vector lanes should
524 ///               be accessed in memory
525 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
526 ///               of the result
527 /// \p Name     - name of the result variable
528 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
529                                             Value *Mask,  Value *PassThru,
530                                             const Twine& Name) {
531   auto PtrsTy = cast<VectorType>(Ptrs->getType());
532   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
533   unsigned NumElts = PtrsTy->getVectorNumElements();
534   Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
535 
536   if (!Mask)
537     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
538                                      NumElts));
539 
540   if (!PassThru)
541     PassThru = UndefValue::get(DataTy);
542 
543   Type *OverloadedTypes[] = {DataTy, PtrsTy};
544   Value * Ops[] = {Ptrs, getInt32(Align), Mask, PassThru};
545 
546   // We specify only one type when we create this intrinsic. Types of other
547   // arguments are derived from this type.
548   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
549                                Name);
550 }
551 
552 /// Create a call to a Masked Scatter intrinsic.
553 /// \p Data  - data to be stored,
554 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
555 ///            stored
556 /// \p Align - alignment for one element
557 /// \p Mask  - vector of booleans which indicates what vector lanes should
558 ///            be accessed in memory
559 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
560                                              unsigned Align, Value *Mask) {
561   auto PtrsTy = cast<VectorType>(Ptrs->getType());
562   auto DataTy = cast<VectorType>(Data->getType());
563   unsigned NumElts = PtrsTy->getVectorNumElements();
564 
565 #ifndef NDEBUG
566   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
567   assert(NumElts == DataTy->getVectorNumElements() &&
568          PtrTy->getElementType() == DataTy->getElementType() &&
569          "Incompatible pointer and data types");
570 #endif
571 
572   if (!Mask)
573     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
574                                      NumElts));
575 
576   Type *OverloadedTypes[] = {DataTy, PtrsTy};
577   Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
578 
579   // We specify only one type when we create this intrinsic. Types of other
580   // arguments are derived from this type.
581   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
582 }
583 
584 template <typename T0, typename T1, typename T2, typename T3>
585 static std::vector<Value *>
586 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
587                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
588                   ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
589                   ArrayRef<T3> GCArgs) {
590   std::vector<Value *> Args;
591   Args.push_back(B.getInt64(ID));
592   Args.push_back(B.getInt32(NumPatchBytes));
593   Args.push_back(ActualCallee);
594   Args.push_back(B.getInt32(CallArgs.size()));
595   Args.push_back(B.getInt32(Flags));
596   Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
597   Args.push_back(B.getInt32(TransitionArgs.size()));
598   Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
599   Args.push_back(B.getInt32(DeoptArgs.size()));
600   Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
601   Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
602 
603   return Args;
604 }
605 
606 template <typename T0, typename T1, typename T2, typename T3>
607 static CallInst *CreateGCStatepointCallCommon(
608     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
609     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
610     ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
611     const Twine &Name) {
612   // Extract out the type of the callee.
613   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
614   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
615          "actual callee must be a callable value");
616 
617   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
618   // Fill in the one generic type'd argument (the function is also vararg)
619   Type *ArgTypes[] = { FuncPtrType };
620   Function *FnStatepoint =
621     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
622                               ArgTypes);
623 
624   std::vector<Value *> Args =
625       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
626                         CallArgs, TransitionArgs, DeoptArgs, GCArgs);
627   return createCallHelper(FnStatepoint, Args, Builder, Name);
628 }
629 
630 CallInst *IRBuilderBase::CreateGCStatepointCall(
631     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
632     ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
633     ArrayRef<Value *> GCArgs, const Twine &Name) {
634   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
635       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
636       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
637 }
638 
639 CallInst *IRBuilderBase::CreateGCStatepointCall(
640     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
641     ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
642     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
643   return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
644       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
645       DeoptArgs, GCArgs, Name);
646 }
647 
648 CallInst *IRBuilderBase::CreateGCStatepointCall(
649     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
650     ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
651     ArrayRef<Value *> GCArgs, const Twine &Name) {
652   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
653       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
654       CallArgs, None, DeoptArgs, GCArgs, Name);
655 }
656 
657 template <typename T0, typename T1, typename T2, typename T3>
658 static InvokeInst *CreateGCStatepointInvokeCommon(
659     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
660     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
661     uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
662     ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
663   // Extract out the type of the callee.
664   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
665   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
666          "actual callee must be a callable value");
667 
668   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
669   // Fill in the one generic type'd argument (the function is also vararg)
670   Function *FnStatepoint = Intrinsic::getDeclaration(
671       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
672 
673   std::vector<Value *> Args =
674       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
675                         InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
676   return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
677                             Name);
678 }
679 
680 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
681     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
682     BasicBlock *NormalDest, BasicBlock *UnwindDest,
683     ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
684     ArrayRef<Value *> GCArgs, const Twine &Name) {
685   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
686       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
687       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
688       DeoptArgs, GCArgs, Name);
689 }
690 
691 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
692     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
693     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
694     ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
695     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
696   return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
697       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
698       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
699 }
700 
701 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
702     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
703     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
704     ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
705   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
706       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
707       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
708       Name);
709 }
710 
711 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
712                                        Type *ResultType,
713                                        const Twine &Name) {
714  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
715  Module *M = BB->getParent()->getParent();
716  Type *Types[] = {ResultType};
717  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
718 
719  Value *Args[] = {Statepoint};
720  return createCallHelper(FnGCResult, Args, this, Name);
721 }
722 
723 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
724                                          int BaseOffset,
725                                          int DerivedOffset,
726                                          Type *ResultType,
727                                          const Twine &Name) {
728  Module *M = BB->getParent()->getParent();
729  Type *Types[] = {ResultType};
730  Function *FnGCRelocate =
731      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
732 
733  Value *Args[] = {Statepoint,
734                   getInt32(BaseOffset),
735                   getInt32(DerivedOffset)};
736  return createCallHelper(FnGCRelocate, Args, this, Name);
737 }
738 
739 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
740                                               Instruction *FMFSource,
741                                               const Twine &Name) {
742   Module *M = BB->getModule();
743   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
744   return createCallHelper(Fn, {V}, this, Name, FMFSource);
745 }
746 
747 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
748                                                Value *RHS,
749                                                Instruction *FMFSource,
750                                                const Twine &Name) {
751   Module *M = BB->getModule();
752   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
753   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
754 }
755 
756 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
757                                          ArrayRef<Type *> Types,
758                                          ArrayRef<Value *> Args,
759                                          Instruction *FMFSource,
760                                          const Twine &Name) {
761   Module *M = BB->getModule();
762   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
763   return createCallHelper(Fn, Args, this, Name, FMFSource);
764 }
765