xref: /freebsd/contrib/llvm-project/llvm/lib/IR/IRBuilder.cpp (revision 3a9a9c0ca44ec535dcf73fe8462bee458e54814b) 
1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IRBuilder class, which is used as a convenient way
10 // to create LLVM instructions with a consistent and simplified interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/IR/Constant.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DerivedTypes.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/GlobalVariable.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/NoFolder.h"
27 #include "llvm/IR/Operator.h"
28 #include "llvm/IR/Statepoint.h"
29 #include "llvm/IR/Type.h"
30 #include "llvm/IR/Value.h"
31 #include "llvm/Support/Casting.h"
32 #include <cassert>
33 #include <cstdint>
34 #include <vector>
35 
36 using namespace llvm;
37 
38 /// CreateGlobalString - Make a new global variable with an initializer that
39 /// has array of i8 type filled in with the nul terminated string value
40 /// specified.  If Name is specified, it is the name of the global variable
41 /// created.
42 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
43                                                   const Twine &Name,
44                                                   unsigned AddressSpace,
45                                                   Module *M) {
46   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
47   if (!M)
48     M = BB->getParent()->getParent();
49   auto *GV = new GlobalVariable(
50       *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
51       StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
52   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
53   GV->setAlignment(Align(1));
54   return GV;
55 }
56 
57 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
58   assert(BB && BB->getParent() && "No current function!");
59   return BB->getParent()->getReturnType();
60 }
61 
62 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
63   auto *PT = cast<PointerType>(Ptr->getType());
64   if (PT->isOpaqueOrPointeeTypeMatches(getInt8Ty()))
65     return Ptr;
66 
67   // Otherwise, we need to insert a bitcast.
68   return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace()));
69 }
70 
71 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
72                                   IRBuilderBase *Builder,
73                                   const Twine &Name = "",
74                                   Instruction *FMFSource = nullptr,
75                                   ArrayRef<OperandBundleDef> OpBundles = {}) {
76   CallInst *CI = Builder->CreateCall(Callee, Ops, OpBundles, Name);
77   if (FMFSource)
78     CI->copyFastMathFlags(FMFSource);
79   return CI;
80 }
81 
82 Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
83   assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
84   if (cast<ConstantInt>(Scaling)->isZero())
85     return Scaling;
86   Module *M = GetInsertBlock()->getParent()->getParent();
87   Function *TheFn =
88       Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
89   CallInst *CI = createCallHelper(TheFn, {}, this, Name);
90   return cast<ConstantInt>(Scaling)->getSExtValue() == 1
91              ? CI
92              : CreateMul(CI, Scaling);
93 }
94 
95 Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
96   Type *STy = DstType->getScalarType();
97   if (isa<ScalableVectorType>(DstType)) {
98     Type *StepVecType = DstType;
99     // TODO: We expect this special case (element type < 8 bits) to be
100     // temporary - once the intrinsic properly supports < 8 bits this code
101     // can be removed.
102     if (STy->getScalarSizeInBits() < 8)
103       StepVecType =
104           VectorType::get(getInt8Ty(), cast<ScalableVectorType>(DstType));
105     Value *Res = CreateIntrinsic(Intrinsic::experimental_stepvector,
106                                  {StepVecType}, {}, nullptr, Name);
107     if (StepVecType != DstType)
108       Res = CreateTrunc(Res, DstType);
109     return Res;
110   }
111 
112   unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();
113 
114   // Create a vector of consecutive numbers from zero to VF.
115   SmallVector<Constant *, 8> Indices;
116   for (unsigned i = 0; i < NumEls; ++i)
117     Indices.push_back(ConstantInt::get(STy, i));
118 
119   // Add the consecutive indices to the vector value.
120   return ConstantVector::get(Indices);
121 }
122 
123 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
124                                       MaybeAlign Align, bool isVolatile,
125                                       MDNode *TBAATag, MDNode *ScopeTag,
126                                       MDNode *NoAliasTag) {
127   Ptr = getCastedInt8PtrValue(Ptr);
128   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
129   Type *Tys[] = { Ptr->getType(), Size->getType() };
130   Module *M = BB->getParent()->getParent();
131   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
132 
133   CallInst *CI = createCallHelper(TheFn, Ops, this);
134 
135   if (Align)
136     cast<MemSetInst>(CI)->setDestAlignment(Align->value());
137 
138   // Set the TBAA info if present.
139   if (TBAATag)
140     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
141 
142   if (ScopeTag)
143     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
144 
145   if (NoAliasTag)
146     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
147 
148   return CI;
149 }
150 
151 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
152     Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
153     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
154 
155   Ptr = getCastedInt8PtrValue(Ptr);
156   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
157   Type *Tys[] = {Ptr->getType(), Size->getType()};
158   Module *M = BB->getParent()->getParent();
159   Function *TheFn = Intrinsic::getDeclaration(
160       M, Intrinsic::memset_element_unordered_atomic, Tys);
161 
162   CallInst *CI = createCallHelper(TheFn, Ops, this);
163 
164   cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
165 
166   // Set the TBAA info if present.
167   if (TBAATag)
168     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
169 
170   if (ScopeTag)
171     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
172 
173   if (NoAliasTag)
174     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
175 
176   return CI;
177 }
178 
179 CallInst *IRBuilderBase::CreateMemTransferInst(
180     Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
181     MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
182     MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
183   Dst = getCastedInt8PtrValue(Dst);
184   Src = getCastedInt8PtrValue(Src);
185 
186   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
187   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
188   Module *M = BB->getParent()->getParent();
189   Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
190 
191   CallInst *CI = createCallHelper(TheFn, Ops, this);
192 
193   auto* MCI = cast<MemTransferInst>(CI);
194   if (DstAlign)
195     MCI->setDestAlignment(*DstAlign);
196   if (SrcAlign)
197     MCI->setSourceAlignment(*SrcAlign);
198 
199   // Set the TBAA info if present.
200   if (TBAATag)
201     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
202 
203   // Set the TBAA Struct info if present.
204   if (TBAAStructTag)
205     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
206 
207   if (ScopeTag)
208     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
209 
210   if (NoAliasTag)
211     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
212 
213   return CI;
214 }
215 
216 CallInst *IRBuilderBase::CreateMemCpyInline(
217     Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign,
218     Value *Size, bool IsVolatile, MDNode *TBAATag, MDNode *TBAAStructTag,
219     MDNode *ScopeTag, MDNode *NoAliasTag) {
220   Dst = getCastedInt8PtrValue(Dst);
221   Src = getCastedInt8PtrValue(Src);
222 
223   Value *Ops[] = {Dst, Src, Size, getInt1(IsVolatile)};
224   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
225   Function *F = BB->getParent();
226   Module *M = F->getParent();
227   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys);
228 
229   CallInst *CI = createCallHelper(TheFn, Ops, this);
230 
231   auto *MCI = cast<MemCpyInlineInst>(CI);
232   if (DstAlign)
233     MCI->setDestAlignment(*DstAlign);
234   if (SrcAlign)
235     MCI->setSourceAlignment(*SrcAlign);
236 
237   // Set the TBAA info if present.
238   if (TBAATag)
239     MCI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
240 
241   // Set the TBAA Struct info if present.
242   if (TBAAStructTag)
243     MCI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
244 
245   if (ScopeTag)
246     MCI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
247 
248   if (NoAliasTag)
249     MCI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
250 
251   return CI;
252 }
253 
254 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
255     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
256     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
257     MDNode *ScopeTag, MDNode *NoAliasTag) {
258   assert(DstAlign >= ElementSize &&
259          "Pointer alignment must be at least element size");
260   assert(SrcAlign >= ElementSize &&
261          "Pointer alignment must be at least element size");
262   Dst = getCastedInt8PtrValue(Dst);
263   Src = getCastedInt8PtrValue(Src);
264 
265   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
266   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
267   Module *M = BB->getParent()->getParent();
268   Function *TheFn = Intrinsic::getDeclaration(
269       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
270 
271   CallInst *CI = createCallHelper(TheFn, Ops, this);
272 
273   // Set the alignment of the pointer args.
274   auto *AMCI = cast<AtomicMemCpyInst>(CI);
275   AMCI->setDestAlignment(DstAlign);
276   AMCI->setSourceAlignment(SrcAlign);
277 
278   // Set the TBAA info if present.
279   if (TBAATag)
280     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
281 
282   // Set the TBAA Struct info if present.
283   if (TBAAStructTag)
284     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
285 
286   if (ScopeTag)
287     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
288 
289   if (NoAliasTag)
290     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
291 
292   return CI;
293 }
294 
295 CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
296                                        Value *Src, MaybeAlign SrcAlign,
297                                        Value *Size, bool isVolatile,
298                                        MDNode *TBAATag, MDNode *ScopeTag,
299                                        MDNode *NoAliasTag) {
300   Dst = getCastedInt8PtrValue(Dst);
301   Src = getCastedInt8PtrValue(Src);
302 
303   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
304   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
305   Module *M = BB->getParent()->getParent();
306   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
307 
308   CallInst *CI = createCallHelper(TheFn, Ops, this);
309 
310   auto *MMI = cast<MemMoveInst>(CI);
311   if (DstAlign)
312     MMI->setDestAlignment(*DstAlign);
313   if (SrcAlign)
314     MMI->setSourceAlignment(*SrcAlign);
315 
316   // Set the TBAA info if present.
317   if (TBAATag)
318     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
319 
320   if (ScopeTag)
321     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
322 
323   if (NoAliasTag)
324     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
325 
326   return CI;
327 }
328 
329 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
330     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
331     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
332     MDNode *ScopeTag, MDNode *NoAliasTag) {
333   assert(DstAlign >= ElementSize &&
334          "Pointer alignment must be at least element size");
335   assert(SrcAlign >= ElementSize &&
336          "Pointer alignment must be at least element size");
337   Dst = getCastedInt8PtrValue(Dst);
338   Src = getCastedInt8PtrValue(Src);
339 
340   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
341   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
342   Module *M = BB->getParent()->getParent();
343   Function *TheFn = Intrinsic::getDeclaration(
344       M, Intrinsic::memmove_element_unordered_atomic, Tys);
345 
346   CallInst *CI = createCallHelper(TheFn, Ops, this);
347 
348   // Set the alignment of the pointer args.
349   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
350   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
351 
352   // Set the TBAA info if present.
353   if (TBAATag)
354     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
355 
356   // Set the TBAA Struct info if present.
357   if (TBAAStructTag)
358     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
359 
360   if (ScopeTag)
361     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
362 
363   if (NoAliasTag)
364     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
365 
366   return CI;
367 }
368 
369 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
370                                     Value *Src) {
371   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
372   Value *Ops[] = {Src};
373   Type *Tys[] = { Src->getType() };
374   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
375   return createCallHelper(Decl, Ops, Builder);
376 }
377 
378 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
379   Module *M = GetInsertBlock()->getParent()->getParent();
380   Value *Ops[] = {Acc, Src};
381   auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
382                                         {Src->getType()});
383   return createCallHelper(Decl, Ops, this);
384 }
385 
386 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
387   Module *M = GetInsertBlock()->getParent()->getParent();
388   Value *Ops[] = {Acc, Src};
389   auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
390                                         {Src->getType()});
391   return createCallHelper(Decl, Ops, this);
392 }
393 
394 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
395   return getReductionIntrinsic(this, Intrinsic::vector_reduce_add, Src);
396 }
397 
398 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
399   return getReductionIntrinsic(this, Intrinsic::vector_reduce_mul, Src);
400 }
401 
402 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
403   return getReductionIntrinsic(this, Intrinsic::vector_reduce_and, Src);
404 }
405 
406 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
407   return getReductionIntrinsic(this, Intrinsic::vector_reduce_or, Src);
408 }
409 
410 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
411   return getReductionIntrinsic(this, Intrinsic::vector_reduce_xor, Src);
412 }
413 
414 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
415   auto ID =
416       IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
417   return getReductionIntrinsic(this, ID, Src);
418 }
419 
420 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
421   auto ID =
422       IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
423   return getReductionIntrinsic(this, ID, Src);
424 }
425 
426 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
427   return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmax, Src);
428 }
429 
430 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) {
431   return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmin, Src);
432 }
433 
434 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
435   assert(isa<PointerType>(Ptr->getType()) &&
436          "lifetime.start only applies to pointers.");
437   Ptr = getCastedInt8PtrValue(Ptr);
438   if (!Size)
439     Size = getInt64(-1);
440   else
441     assert(Size->getType() == getInt64Ty() &&
442            "lifetime.start requires the size to be an i64");
443   Value *Ops[] = { Size, Ptr };
444   Module *M = BB->getParent()->getParent();
445   Function *TheFn =
446       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
447   return createCallHelper(TheFn, Ops, this);
448 }
449 
450 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
451   assert(isa<PointerType>(Ptr->getType()) &&
452          "lifetime.end only applies to pointers.");
453   Ptr = getCastedInt8PtrValue(Ptr);
454   if (!Size)
455     Size = getInt64(-1);
456   else
457     assert(Size->getType() == getInt64Ty() &&
458            "lifetime.end requires the size to be an i64");
459   Value *Ops[] = { Size, Ptr };
460   Module *M = BB->getParent()->getParent();
461   Function *TheFn =
462       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
463   return createCallHelper(TheFn, Ops, this);
464 }
465 
466 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
467 
468   assert(isa<PointerType>(Ptr->getType()) &&
469          "invariant.start only applies to pointers.");
470   Ptr = getCastedInt8PtrValue(Ptr);
471   if (!Size)
472     Size = getInt64(-1);
473   else
474     assert(Size->getType() == getInt64Ty() &&
475            "invariant.start requires the size to be an i64");
476 
477   Value *Ops[] = {Size, Ptr};
478   // Fill in the single overloaded type: memory object type.
479   Type *ObjectPtr[1] = {Ptr->getType()};
480   Module *M = BB->getParent()->getParent();
481   Function *TheFn =
482       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
483   return createCallHelper(TheFn, Ops, this);
484 }
485 
486 CallInst *
487 IRBuilderBase::CreateAssumption(Value *Cond,
488                                 ArrayRef<OperandBundleDef> OpBundles) {
489   assert(Cond->getType() == getInt1Ty() &&
490          "an assumption condition must be of type i1");
491 
492   Value *Ops[] = { Cond };
493   Module *M = BB->getParent()->getParent();
494   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
495   return createCallHelper(FnAssume, Ops, this, "", nullptr, OpBundles);
496 }
497 
498 Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
499   Module *M = BB->getModule();
500   auto *FnIntrinsic = Intrinsic::getDeclaration(
501       M, Intrinsic::experimental_noalias_scope_decl, {});
502   return createCallHelper(FnIntrinsic, {Scope}, this);
503 }
504 
505 /// Create a call to a Masked Load intrinsic.
506 /// \p Ty        - vector type to load
507 /// \p Ptr       - base pointer for the load
508 /// \p Alignment - alignment of the source location
509 /// \p Mask      - vector of booleans which indicates what vector lanes should
510 ///                be accessed in memory
511 /// \p PassThru  - pass-through value that is used to fill the masked-off lanes
512 ///                of the result
513 /// \p Name      - name of the result variable
514 CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
515                                           Value *Mask, Value *PassThru,
516                                           const Twine &Name) {
517   auto *PtrTy = cast<PointerType>(Ptr->getType());
518   assert(Ty->isVectorTy() && "Type should be vector");
519   assert(PtrTy->isOpaqueOrPointeeTypeMatches(Ty) && "Wrong element type");
520   assert(Mask && "Mask should not be all-ones (null)");
521   if (!PassThru)
522     PassThru = UndefValue::get(Ty);
523   Type *OverloadedTypes[] = { Ty, PtrTy };
524   Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
525   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
526                                OverloadedTypes, Name);
527 }
528 
529 /// Create a call to a Masked Store intrinsic.
530 /// \p Val       - data to be stored,
531 /// \p Ptr       - base pointer for the store
532 /// \p Alignment - alignment of the destination location
533 /// \p Mask      - vector of booleans which indicates what vector lanes should
534 ///                be accessed in memory
535 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
536                                            Align Alignment, Value *Mask) {
537   auto *PtrTy = cast<PointerType>(Ptr->getType());
538   Type *DataTy = Val->getType();
539   assert(DataTy->isVectorTy() && "Val should be a vector");
540   assert(PtrTy->isOpaqueOrPointeeTypeMatches(DataTy) && "Wrong element type");
541   assert(Mask && "Mask should not be all-ones (null)");
542   Type *OverloadedTypes[] = { DataTy, PtrTy };
543   Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
544   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
545 }
546 
547 /// Create a call to a Masked intrinsic, with given intrinsic Id,
548 /// an array of operands - Ops, and an array of overloaded types -
549 /// OverloadedTypes.
550 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
551                                                ArrayRef<Value *> Ops,
552                                                ArrayRef<Type *> OverloadedTypes,
553                                                const Twine &Name) {
554   Module *M = BB->getParent()->getParent();
555   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
556   return createCallHelper(TheFn, Ops, this, Name);
557 }
558 
559 /// Create a call to a Masked Gather intrinsic.
560 /// \p Ty       - vector type to gather
561 /// \p Ptrs     - vector of pointers for loading
562 /// \p Align    - alignment for one element
563 /// \p Mask     - vector of booleans which indicates what vector lanes should
564 ///               be accessed in memory
565 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
566 ///               of the result
567 /// \p Name     - name of the result variable
568 CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
569                                             Align Alignment, Value *Mask,
570                                             Value *PassThru,
571                                             const Twine &Name) {
572   auto *VecTy = cast<VectorType>(Ty);
573   ElementCount NumElts = VecTy->getElementCount();
574   auto *PtrsTy = cast<VectorType>(Ptrs->getType());
575   assert(cast<PointerType>(PtrsTy->getElementType())
576              ->isOpaqueOrPointeeTypeMatches(
577                  cast<VectorType>(Ty)->getElementType()) &&
578          "Element type mismatch");
579   assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
580 
581   if (!Mask)
582     Mask = Constant::getAllOnesValue(
583         VectorType::get(Type::getInt1Ty(Context), NumElts));
584 
585   if (!PassThru)
586     PassThru = UndefValue::get(Ty);
587 
588   Type *OverloadedTypes[] = {Ty, PtrsTy};
589   Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
590 
591   // We specify only one type when we create this intrinsic. Types of other
592   // arguments are derived from this type.
593   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
594                                Name);
595 }
596 
597 /// Create a call to a Masked Scatter intrinsic.
598 /// \p Data  - data to be stored,
599 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
600 ///            stored
601 /// \p Align - alignment for one element
602 /// \p Mask  - vector of booleans which indicates what vector lanes should
603 ///            be accessed in memory
604 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
605                                              Align Alignment, Value *Mask) {
606   auto *PtrsTy = cast<VectorType>(Ptrs->getType());
607   auto *DataTy = cast<VectorType>(Data->getType());
608   ElementCount NumElts = PtrsTy->getElementCount();
609 
610 #ifndef NDEBUG
611   auto *PtrTy = cast<PointerType>(PtrsTy->getElementType());
612   assert(NumElts == DataTy->getElementCount() &&
613          PtrTy->isOpaqueOrPointeeTypeMatches(DataTy->getElementType()) &&
614          "Incompatible pointer and data types");
615 #endif
616 
617   if (!Mask)
618     Mask = Constant::getAllOnesValue(
619         VectorType::get(Type::getInt1Ty(Context), NumElts));
620 
621   Type *OverloadedTypes[] = {DataTy, PtrsTy};
622   Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
623 
624   // We specify only one type when we create this intrinsic. Types of other
625   // arguments are derived from this type.
626   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
627 }
628 
629 template <typename T0>
630 static std::vector<Value *>
631 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
632                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
633   std::vector<Value *> Args;
634   Args.push_back(B.getInt64(ID));
635   Args.push_back(B.getInt32(NumPatchBytes));
636   Args.push_back(ActualCallee);
637   Args.push_back(B.getInt32(CallArgs.size()));
638   Args.push_back(B.getInt32(Flags));
639   llvm::append_range(Args, CallArgs);
640   // GC Transition and Deopt args are now always handled via operand bundle.
641   // They will be removed from the signature of gc.statepoint shortly.
642   Args.push_back(B.getInt32(0));
643   Args.push_back(B.getInt32(0));
644   // GC args are now encoded in the gc-live operand bundle
645   return Args;
646 }
647 
648 template<typename T1, typename T2, typename T3>
649 static std::vector<OperandBundleDef>
650 getStatepointBundles(Optional<ArrayRef<T1>> TransitionArgs,
651                      Optional<ArrayRef<T2>> DeoptArgs,
652                      ArrayRef<T3> GCArgs) {
653   std::vector<OperandBundleDef> Rval;
654   if (DeoptArgs) {
655     SmallVector<Value*, 16> DeoptValues;
656     llvm::append_range(DeoptValues, *DeoptArgs);
657     Rval.emplace_back("deopt", DeoptValues);
658   }
659   if (TransitionArgs) {
660     SmallVector<Value*, 16> TransitionValues;
661     llvm::append_range(TransitionValues, *TransitionArgs);
662     Rval.emplace_back("gc-transition", TransitionValues);
663   }
664   if (GCArgs.size()) {
665     SmallVector<Value*, 16> LiveValues;
666     llvm::append_range(LiveValues, GCArgs);
667     Rval.emplace_back("gc-live", LiveValues);
668   }
669   return Rval;
670 }
671 
672 template <typename T0, typename T1, typename T2, typename T3>
673 static CallInst *CreateGCStatepointCallCommon(
674     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
675     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
676     Optional<ArrayRef<T1>> TransitionArgs,
677     Optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
678     const Twine &Name) {
679   // Extract out the type of the callee.
680   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
681   assert(isa<FunctionType>(FuncPtrType->getPointerElementType()) &&
682          "actual callee must be a callable value");
683 
684   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
685   // Fill in the one generic type'd argument (the function is also vararg)
686   Type *ArgTypes[] = { FuncPtrType };
687   Function *FnStatepoint =
688     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
689                               ArgTypes);
690 
691   std::vector<Value *> Args =
692       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
693                         CallArgs);
694 
695   return Builder->CreateCall(FnStatepoint, Args,
696                              getStatepointBundles(TransitionArgs, DeoptArgs,
697                                                   GCArgs),
698                              Name);
699 }
700 
701 CallInst *IRBuilderBase::CreateGCStatepointCall(
702     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
703     ArrayRef<Value *> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
704     ArrayRef<Value *> GCArgs, const Twine &Name) {
705   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
706       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
707       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
708 }
709 
710 CallInst *IRBuilderBase::CreateGCStatepointCall(
711     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
712     ArrayRef<Value *> CallArgs, Optional<ArrayRef<Use>> TransitionArgs,
713     Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
714     const Twine &Name) {
715   return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
716       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
717       DeoptArgs, GCArgs, Name);
718 }
719 
720 CallInst *IRBuilderBase::CreateGCStatepointCall(
721     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
722     ArrayRef<Use> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
723     ArrayRef<Value *> GCArgs, const Twine &Name) {
724   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
725       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
726       CallArgs, None, DeoptArgs, GCArgs, Name);
727 }
728 
729 template <typename T0, typename T1, typename T2, typename T3>
730 static InvokeInst *CreateGCStatepointInvokeCommon(
731     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
732     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
733     uint32_t Flags, ArrayRef<T0> InvokeArgs,
734     Optional<ArrayRef<T1>> TransitionArgs, Optional<ArrayRef<T2>> DeoptArgs,
735     ArrayRef<T3> GCArgs, const Twine &Name) {
736   // Extract out the type of the callee.
737   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
738   assert(isa<FunctionType>(FuncPtrType->getPointerElementType()) &&
739          "actual callee must be a callable value");
740 
741   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
742   // Fill in the one generic type'd argument (the function is also vararg)
743   Function *FnStatepoint = Intrinsic::getDeclaration(
744       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
745 
746   std::vector<Value *> Args =
747       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
748                         InvokeArgs);
749 
750   return Builder->CreateInvoke(FnStatepoint, NormalDest, UnwindDest, Args,
751                                getStatepointBundles(TransitionArgs, DeoptArgs,
752                                                     GCArgs),
753                                Name);
754 }
755 
756 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
757     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
758     BasicBlock *NormalDest, BasicBlock *UnwindDest,
759     ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Value *>> DeoptArgs,
760     ArrayRef<Value *> GCArgs, const Twine &Name) {
761   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
762       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
763       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
764       DeoptArgs, GCArgs, Name);
765 }
766 
767 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
768     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
769     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
770     ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
771     Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
772   return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
773       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
774       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
775 }
776 
777 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
778     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
779     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
780     Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
781   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
782       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
783       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
784       Name);
785 }
786 
787 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
788                                        Type *ResultType,
789                                        const Twine &Name) {
790  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
791  Module *M = BB->getParent()->getParent();
792  Type *Types[] = {ResultType};
793  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
794 
795  Value *Args[] = {Statepoint};
796  return createCallHelper(FnGCResult, Args, this, Name);
797 }
798 
799 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
800                                          int BaseOffset,
801                                          int DerivedOffset,
802                                          Type *ResultType,
803                                          const Twine &Name) {
804  Module *M = BB->getParent()->getParent();
805  Type *Types[] = {ResultType};
806  Function *FnGCRelocate =
807      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
808 
809  Value *Args[] = {Statepoint,
810                   getInt32(BaseOffset),
811                   getInt32(DerivedOffset)};
812  return createCallHelper(FnGCRelocate, Args, this, Name);
813 }
814 
815 CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
816                                                 const Twine &Name) {
817   Module *M = BB->getParent()->getParent();
818   Type *PtrTy = DerivedPtr->getType();
819   Function *FnGCFindBase = Intrinsic::getDeclaration(
820       M, Intrinsic::experimental_gc_get_pointer_base, {PtrTy, PtrTy});
821   return createCallHelper(FnGCFindBase, {DerivedPtr}, this, Name);
822 }
823 
824 CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
825                                                   const Twine &Name) {
826   Module *M = BB->getParent()->getParent();
827   Type *PtrTy = DerivedPtr->getType();
828   Function *FnGCGetOffset = Intrinsic::getDeclaration(
829       M, Intrinsic::experimental_gc_get_pointer_offset, {PtrTy});
830   return createCallHelper(FnGCGetOffset, {DerivedPtr}, this, Name);
831 }
832 
833 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
834                                               Instruction *FMFSource,
835                                               const Twine &Name) {
836   Module *M = BB->getModule();
837   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
838   return createCallHelper(Fn, {V}, this, Name, FMFSource);
839 }
840 
841 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
842                                                Value *RHS,
843                                                Instruction *FMFSource,
844                                                const Twine &Name) {
845   Module *M = BB->getModule();
846   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
847   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
848 }
849 
850 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
851                                          ArrayRef<Type *> Types,
852                                          ArrayRef<Value *> Args,
853                                          Instruction *FMFSource,
854                                          const Twine &Name) {
855   Module *M = BB->getModule();
856   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
857   return createCallHelper(Fn, Args, this, Name, FMFSource);
858 }
859 
860 CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
861     Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
862     const Twine &Name, MDNode *FPMathTag,
863     Optional<RoundingMode> Rounding,
864     Optional<fp::ExceptionBehavior> Except) {
865   Value *RoundingV = getConstrainedFPRounding(Rounding);
866   Value *ExceptV = getConstrainedFPExcept(Except);
867 
868   FastMathFlags UseFMF = FMF;
869   if (FMFSource)
870     UseFMF = FMFSource->getFastMathFlags();
871 
872   CallInst *C = CreateIntrinsic(ID, {L->getType()},
873                                 {L, R, RoundingV, ExceptV}, nullptr, Name);
874   setConstrainedFPCallAttr(C);
875   setFPAttrs(C, FPMathTag, UseFMF);
876   return C;
877 }
878 
879 Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
880                                    const Twine &Name, MDNode *FPMathTag) {
881   if (Instruction::isBinaryOp(Opc)) {
882     assert(Ops.size() == 2 && "Invalid number of operands!");
883     return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
884                        Ops[0], Ops[1], Name, FPMathTag);
885   }
886   if (Instruction::isUnaryOp(Opc)) {
887     assert(Ops.size() == 1 && "Invalid number of operands!");
888     return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
889                       Ops[0], Name, FPMathTag);
890   }
891   llvm_unreachable("Unexpected opcode!");
892 }
893 
894 CallInst *IRBuilderBase::CreateConstrainedFPCast(
895     Intrinsic::ID ID, Value *V, Type *DestTy,
896     Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
897     Optional<RoundingMode> Rounding,
898     Optional<fp::ExceptionBehavior> Except) {
899   Value *ExceptV = getConstrainedFPExcept(Except);
900 
901   FastMathFlags UseFMF = FMF;
902   if (FMFSource)
903     UseFMF = FMFSource->getFastMathFlags();
904 
905   CallInst *C;
906   bool HasRoundingMD = false;
907   switch (ID) {
908   default:
909     break;
910 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
911   case Intrinsic::INTRINSIC:                                \
912     HasRoundingMD = ROUND_MODE;                             \
913     break;
914 #include "llvm/IR/ConstrainedOps.def"
915   }
916   if (HasRoundingMD) {
917     Value *RoundingV = getConstrainedFPRounding(Rounding);
918     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
919                         nullptr, Name);
920   } else
921     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
922                         Name);
923 
924   setConstrainedFPCallAttr(C);
925 
926   if (isa<FPMathOperator>(C))
927     setFPAttrs(C, FPMathTag, UseFMF);
928   return C;
929 }
930 
931 Value *IRBuilderBase::CreateFCmpHelper(
932     CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
933     MDNode *FPMathTag, bool IsSignaling) {
934   if (IsFPConstrained) {
935     auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
936                           : Intrinsic::experimental_constrained_fcmp;
937     return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
938   }
939 
940   if (auto *LC = dyn_cast<Constant>(LHS))
941     if (auto *RC = dyn_cast<Constant>(RHS))
942       return Insert(Folder.CreateFCmp(P, LC, RC), Name);
943   return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
944 }
945 
946 CallInst *IRBuilderBase::CreateConstrainedFPCmp(
947     Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
948     const Twine &Name, Optional<fp::ExceptionBehavior> Except) {
949   Value *PredicateV = getConstrainedFPPredicate(P);
950   Value *ExceptV = getConstrainedFPExcept(Except);
951 
952   CallInst *C = CreateIntrinsic(ID, {L->getType()},
953                                 {L, R, PredicateV, ExceptV}, nullptr, Name);
954   setConstrainedFPCallAttr(C);
955   return C;
956 }
957 
958 CallInst *IRBuilderBase::CreateConstrainedFPCall(
959     Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
960     Optional<RoundingMode> Rounding,
961     Optional<fp::ExceptionBehavior> Except) {
962   llvm::SmallVector<Value *, 6> UseArgs;
963 
964   append_range(UseArgs, Args);
965   bool HasRoundingMD = false;
966   switch (Callee->getIntrinsicID()) {
967   default:
968     break;
969 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
970   case Intrinsic::INTRINSIC:                                \
971     HasRoundingMD = ROUND_MODE;                             \
972     break;
973 #include "llvm/IR/ConstrainedOps.def"
974   }
975   if (HasRoundingMD)
976     UseArgs.push_back(getConstrainedFPRounding(Rounding));
977   UseArgs.push_back(getConstrainedFPExcept(Except));
978 
979   CallInst *C = CreateCall(Callee, UseArgs, Name);
980   setConstrainedFPCallAttr(C);
981   return C;
982 }
983 
984 Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
985                                    const Twine &Name, Instruction *MDFrom) {
986   if (auto *V = Folder.FoldSelect(C, True, False))
987     return V;
988 
989   SelectInst *Sel = SelectInst::Create(C, True, False);
990   if (MDFrom) {
991     MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
992     MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
993     Sel = addBranchMetadata(Sel, Prof, Unpred);
994   }
995   if (isa<FPMathOperator>(Sel))
996     setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
997   return Insert(Sel, Name);
998 }
999 
1000 Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
1001                                     const Twine &Name) {
1002   assert(LHS->getType() == RHS->getType() &&
1003          "Pointer subtraction operand types must match!");
1004   assert(cast<PointerType>(LHS->getType())
1005              ->isOpaqueOrPointeeTypeMatches(ElemTy) &&
1006          "Pointer type must match element type");
1007   Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context));
1008   Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context));
1009   Value *Difference = CreateSub(LHS_int, RHS_int);
1010   return CreateExactSDiv(Difference, ConstantExpr::getSizeOf(ElemTy),
1011                          Name);
1012 }
1013 
1014 Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
1015   assert(isa<PointerType>(Ptr->getType()) &&
1016          "launder.invariant.group only applies to pointers.");
1017   // FIXME: we could potentially avoid casts to/from i8*.
1018   auto *PtrType = Ptr->getType();
1019   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
1020   if (PtrType != Int8PtrTy)
1021     Ptr = CreateBitCast(Ptr, Int8PtrTy);
1022   Module *M = BB->getParent()->getParent();
1023   Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
1024       M, Intrinsic::launder_invariant_group, {Int8PtrTy});
1025 
1026   assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&
1027          FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
1028              Int8PtrTy &&
1029          "LaunderInvariantGroup should take and return the same type");
1030 
1031   CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr});
1032 
1033   if (PtrType != Int8PtrTy)
1034     return CreateBitCast(Fn, PtrType);
1035   return Fn;
1036 }
1037 
1038 Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
1039   assert(isa<PointerType>(Ptr->getType()) &&
1040          "strip.invariant.group only applies to pointers.");
1041 
1042   // FIXME: we could potentially avoid casts to/from i8*.
1043   auto *PtrType = Ptr->getType();
1044   auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
1045   if (PtrType != Int8PtrTy)
1046     Ptr = CreateBitCast(Ptr, Int8PtrTy);
1047   Module *M = BB->getParent()->getParent();
1048   Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
1049       M, Intrinsic::strip_invariant_group, {Int8PtrTy});
1050 
1051   assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&
1052          FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
1053              Int8PtrTy &&
1054          "StripInvariantGroup should take and return the same type");
1055 
1056   CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr});
1057 
1058   if (PtrType != Int8PtrTy)
1059     return CreateBitCast(Fn, PtrType);
1060   return Fn;
1061 }
1062 
1063 Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
1064   auto *Ty = cast<VectorType>(V->getType());
1065   if (isa<ScalableVectorType>(Ty)) {
1066     Module *M = BB->getParent()->getParent();
1067     Function *F = Intrinsic::getDeclaration(
1068         M, Intrinsic::experimental_vector_reverse, Ty);
1069     return Insert(CallInst::Create(F, V), Name);
1070   }
1071   // Keep the original behaviour for fixed vector
1072   SmallVector<int, 8> ShuffleMask;
1073   int NumElts = Ty->getElementCount().getKnownMinValue();
1074   for (int i = 0; i < NumElts; ++i)
1075     ShuffleMask.push_back(NumElts - i - 1);
1076   return CreateShuffleVector(V, ShuffleMask, Name);
1077 }
1078 
1079 Value *IRBuilderBase::CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,
1080                                          const Twine &Name) {
1081   assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1082   assert(V1->getType() == V2->getType() &&
1083          "Splice expects matching operand types!");
1084 
1085   if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) {
1086     Module *M = BB->getParent()->getParent();
1087     Function *F = Intrinsic::getDeclaration(
1088         M, Intrinsic::experimental_vector_splice, VTy);
1089 
1090     Value *Ops[] = {V1, V2, getInt32(Imm)};
1091     return Insert(CallInst::Create(F, Ops), Name);
1092   }
1093 
1094   unsigned NumElts = cast<FixedVectorType>(V1->getType())->getNumElements();
1095   assert(((-Imm <= NumElts) || (Imm < NumElts)) &&
1096          "Invalid immediate for vector splice!");
1097 
1098   // Keep the original behaviour for fixed vector
1099   unsigned Idx = (NumElts + Imm) % NumElts;
1100   SmallVector<int, 8> Mask;
1101   for (unsigned I = 0; I < NumElts; ++I)
1102     Mask.push_back(Idx + I);
1103 
1104   return CreateShuffleVector(V1, V2, Mask);
1105 }
1106 
1107 Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
1108                                         const Twine &Name) {
1109   auto EC = ElementCount::getFixed(NumElts);
1110   return CreateVectorSplat(EC, V, Name);
1111 }
1112 
1113 Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
1114                                         const Twine &Name) {
1115   assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1116 
1117   // First insert it into a poison vector so we can shuffle it.
1118   Type *I32Ty = getInt32Ty();
1119   Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
1120   V = CreateInsertElement(Poison, V, ConstantInt::get(I32Ty, 0),
1121                           Name + ".splatinsert");
1122 
1123   // Shuffle the value across the desired number of elements.
1124   SmallVector<int, 16> Zeros;
1125   Zeros.resize(EC.getKnownMinValue());
1126   return CreateShuffleVector(V, Zeros, Name + ".splat");
1127 }
1128 
1129 Value *IRBuilderBase::CreateExtractInteger(
1130     const DataLayout &DL, Value *From, IntegerType *ExtractedTy,
1131     uint64_t Offset, const Twine &Name) {
1132   auto *IntTy = cast<IntegerType>(From->getType());
1133   assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=
1134              DL.getTypeStoreSize(IntTy) &&
1135          "Element extends past full value");
1136   uint64_t ShAmt = 8 * Offset;
1137   Value *V = From;
1138   if (DL.isBigEndian())
1139     ShAmt = 8 * (DL.getTypeStoreSize(IntTy) -
1140                  DL.getTypeStoreSize(ExtractedTy) - Offset);
1141   if (ShAmt) {
1142     V = CreateLShr(V, ShAmt, Name + ".shift");
1143   }
1144   assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&
1145          "Cannot extract to a larger integer!");
1146   if (ExtractedTy != IntTy) {
1147     V = CreateTrunc(V, ExtractedTy, Name + ".trunc");
1148   }
1149   return V;
1150 }
1151 
1152 Value *IRBuilderBase::CreatePreserveArrayAccessIndex(
1153     Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
1154     MDNode *DbgInfo) {
1155   auto *BaseType = Base->getType();
1156   assert(isa<PointerType>(BaseType) &&
1157          "Invalid Base ptr type for preserve.array.access.index.");
1158   assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) &&
1159          "Pointer element type mismatch");
1160 
1161   Value *LastIndexV = getInt32(LastIndex);
1162   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1163   SmallVector<Value *, 4> IdxList(Dimension, Zero);
1164   IdxList.push_back(LastIndexV);
1165 
1166   Type *ResultType =
1167       GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList);
1168 
1169   Module *M = BB->getParent()->getParent();
1170   Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
1171       M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
1172 
1173   Value *DimV = getInt32(Dimension);
1174   CallInst *Fn =
1175       CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1176   Fn->addParamAttr(
1177       0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1178   if (DbgInfo)
1179     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1180 
1181   return Fn;
1182 }
1183 
1184 Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1185     Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1186   assert(isa<PointerType>(Base->getType()) &&
1187          "Invalid Base ptr type for preserve.union.access.index.");
1188   auto *BaseType = Base->getType();
1189 
1190   Module *M = BB->getParent()->getParent();
1191   Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
1192       M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
1193 
1194   Value *DIIndex = getInt32(FieldIndex);
1195   CallInst *Fn =
1196       CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
1197   if (DbgInfo)
1198     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1199 
1200   return Fn;
1201 }
1202 
1203 Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1204     Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1205     MDNode *DbgInfo) {
1206   auto *BaseType = Base->getType();
1207   assert(isa<PointerType>(BaseType) &&
1208          "Invalid Base ptr type for preserve.struct.access.index.");
1209   assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) &&
1210          "Pointer element type mismatch");
1211 
1212   Value *GEPIndex = getInt32(Index);
1213   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1214   Type *ResultType =
1215       GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex});
1216 
1217   Module *M = BB->getParent()->getParent();
1218   Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
1219       M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
1220 
1221   Value *DIIndex = getInt32(FieldIndex);
1222   CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
1223                             {Base, GEPIndex, DIIndex});
1224   Fn->addParamAttr(
1225       0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1226   if (DbgInfo)
1227     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1228 
1229   return Fn;
1230 }
1231 
1232 CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1233                                                          Value *PtrValue,
1234                                                          Value *AlignValue,
1235                                                          Value *OffsetValue) {
1236   SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1237   if (OffsetValue)
1238     Vals.push_back(OffsetValue);
1239   OperandBundleDefT<Value *> AlignOpB("align", Vals);
1240   return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});
1241 }
1242 
1243 CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1244                                                    Value *PtrValue,
1245                                                    unsigned Alignment,
1246                                                    Value *OffsetValue) {
1247   assert(isa<PointerType>(PtrValue->getType()) &&
1248          "trying to create an alignment assumption on a non-pointer?");
1249   assert(Alignment != 0 && "Invalid Alignment");
1250   auto *PtrTy = cast<PointerType>(PtrValue->getType());
1251   Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1252   Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);
1253   return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1254 }
1255 
1256 CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1257                                                    Value *PtrValue,
1258                                                    Value *Alignment,
1259                                                    Value *OffsetValue) {
1260   assert(isa<PointerType>(PtrValue->getType()) &&
1261          "trying to create an alignment assumption on a non-pointer?");
1262   return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
1263 }
1264 
1265 IRBuilderDefaultInserter::~IRBuilderDefaultInserter() {}
1266 IRBuilderCallbackInserter::~IRBuilderCallbackInserter() {}
1267 IRBuilderFolder::~IRBuilderFolder() {}
1268 void ConstantFolder::anchor() {}
1269 void NoFolder::anchor() {}
1270