Lines Matching full:builder
1 //===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
62 #define DEBUG_TYPE "openmp-ir-builder"
68 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
74 "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
300 void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) { in spliceBB() argument
301 DebugLoc DebugLoc = Builder.getCurrentDebugLocation(); in spliceBB()
302 BasicBlock *Old = Builder.GetInsertBlock(); in spliceBB()
304 spliceBB(Builder.saveIP(), New, CreateBranch); in spliceBB()
306 Builder.SetInsertPoint(Old->getTerminator()); in spliceBB()
308 Builder.SetInsertPoint(Old); in spliceBB()
310 // SetInsertPoint also updates the Builder's debug location, but we want to in spliceBB()
311 // keep the one the Builder was configured to use. in spliceBB()
312 Builder.SetCurrentDebugLocation(DebugLoc); in spliceBB()
326 BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch, in splitBB() argument
328 DebugLoc DebugLoc = Builder.getCurrentDebugLocation(); in splitBB()
329 BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name); in splitBB()
331 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator()); in splitBB()
333 Builder.SetInsertPoint(Builder.GetInsertBlock()); in splitBB()
334 // SetInsertPoint also updates the Builder's debug location, but we want to in splitBB()
335 // keep the one the Builder was configured to use. in splitBB()
336 Builder.SetCurrentDebugLocation(DebugLoc); in splitBB()
340 BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch, in splitBB() argument
342 DebugLoc DebugLoc = Builder.getCurrentDebugLocation(); in splitBB()
343 BasicBlock *New = splitBB(Builder.saveIP(), CreateBranch, Name); in splitBB()
345 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator()); in splitBB()
347 Builder.SetInsertPoint(Builder.GetInsertBlock()); in splitBB()
348 // SetInsertPoint also updates the Builder's debug location, but we want to in splitBB()
349 // keep the one the Builder was configured to use. in splitBB()
350 Builder.SetCurrentDebugLocation(DebugLoc); in splitBB()
354 BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch, in splitBBWithSuffix() argument
356 BasicBlock *Old = Builder.GetInsertBlock(); in splitBBWithSuffix()
357 return splitBB(Builder, CreateBranch, Old->getName() + Suffix); in splitBBWithSuffix()
363 Value *createFakeIntVal(IRBuilderBase &Builder, in createFakeIntVal() argument
368 Builder.restoreIP(OuterAllocaIP); in createFakeIntVal()
371 Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, Name + ".addr"); in createFakeIntVal()
378 Builder.CreateLoad(Builder.getInt32Ty(), FakeValAddr, Name + ".val"); in createFakeIntVal()
383 Builder.restoreIP(InnerAllocaIP); in createFakeIntVal()
387 Builder.CreateLoad(Builder.getInt32Ty(), FakeVal, Name + ".use"); in createFakeIntVal()
390 cast<BinaryOperator>(Builder.CreateAdd(FakeVal, Builder.getInt32(10))); in createFakeIntVal()
495 IRBuilderBase &Builder, in getKernelArgsVector() argument
497 Value *Version = Builder.getInt32(OMP_KERNEL_ARG_VERSION); in getKernelArgsVector()
498 Value *PointerNum = Builder.getInt32(KernelArgs.NumTargetItems); in getKernelArgsVector()
499 auto Int32Ty = Type::getInt32Ty(Builder.getContext()); in getKernelArgsVector()
501 Value *Flags = Builder.getInt64(KernelArgs.HasNoWait); in getKernelArgsVector()
504 Builder.CreateInsertValue(ZeroArray, KernelArgs.NumTeams, {0}); in getKernelArgsVector()
506 Builder.CreateInsertValue(ZeroArray, KernelArgs.NumThreads, {0}); in getKernelArgsVector()
639 static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder, in raiseUserConstantDataAllocasToEntryBlock() argument
786 raiseUserConstantDataAllocasToEntryBlock(Builder, F); in finalize()
876 SrcLocStr = Builder.CreateGlobalStringPtr(LocStr, /* Name */ "", in getOrCreateSrcLocStr()
930 return Builder.CreateCall( in getOrCreateThreadID()
976 Builder.CreateCall(getOrCreateRuntimeFunctionPtr( in createBarrier()
984 return Builder.saveIP(); in createBarrier()
995 auto *UI = Builder.CreateUnreachable(); in createCancel()
1000 Builder.SetInsertPoint(ThenTI); in createCancel()
1006 CancelKind = Builder.getInt32(Value); \ in createCancel()
1017 Value *Result = Builder.CreateCall( in createCancel()
1021 IRBuilder<>::InsertPointGuard IPG(Builder); in createCancel()
1022 Builder.restoreIP(IP); in createCancel()
1023 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), in createCancel()
1033 Builder.SetInsertPoint(UI->getParent()); in createCancel()
1036 return Builder.saveIP(); in createCancel()
1046 Builder.restoreIP(AllocaIP); in emitTargetKernel()
1048 Builder.CreateAlloca(OpenMPIRBuilder::KernelArgs, nullptr, "kernel_args"); in emitTargetKernel()
1049 Builder.restoreIP(Loc.IP); in emitTargetKernel()
1053 Builder.CreateStructGEP(OpenMPIRBuilder::KernelArgs, KernelArgsPtr, I); in emitTargetKernel()
1054 Builder.CreateAlignedStore( in emitTargetKernel()
1062 Return = Builder.CreateCall( in emitTargetKernel()
1066 return Builder.saveIP(); in emitTargetKernel()
1077 Builder.restoreIP(Loc.IP); in emitKernelLaunch()
1096 getKernelArgsVector(Args, Builder, ArgsVector); in emitKernelLaunch()
1111 Builder.restoreIP(emitTargetKernel(Builder, AllocaIP, Return, RTLoc, DeviceID, in emitKernelLaunch()
1116 BasicBlock::Create(Builder.getContext(), "omp_offload.failed"); in emitKernelLaunch()
1118 BasicBlock::Create(Builder.getContext(), "omp_offload.cont"); in emitKernelLaunch()
1119 Value *Failed = Builder.CreateIsNotNull(Return); in emitKernelLaunch()
1120 Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock); in emitKernelLaunch()
1122 auto CurFn = Builder.GetInsertBlock()->getParent(); in emitKernelLaunch()
1124 Builder.restoreIP(emitTargetCallFallbackCB(Builder.saveIP())); in emitKernelLaunch()
1127 return Builder.saveIP(); in emitKernelLaunch()
1137 BasicBlock *BB = Builder.GetInsertBlock(); in emitCancelationCheckImpl()
1139 if (Builder.GetInsertPoint() == BB->end()) { in emitCancelationCheckImpl()
1145 NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint()); in emitCancelationCheckImpl()
1147 Builder.SetInsertPoint(BB); in emitCancelationCheckImpl()
1153 Value *Cmp = Builder.CreateIsNull(CancelFlag); in emitCancelationCheckImpl()
1154 Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock, in emitCancelationCheckImpl()
1159 Builder.SetInsertPoint(CancellationBlock); in emitCancelationCheckImpl()
1161 ExitCB(Builder.saveIP()); in emitCancelationCheckImpl()
1163 FI.FiniCB(Builder.saveIP()); in emitCancelationCheckImpl()
1166 Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin()); in emitCancelationCheckImpl()
1179 IRBuilder<> &Builder = OMPIRBuilder->Builder; in targetParallelCallback() local
1194 Builder.SetInsertPoint(CI); in targetParallelCallback()
1199 OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.saveIP(); in targetParallelCallback()
1200 Builder.SetInsertPoint(OuterAllocaBB, OuterAllocaBB->getFirstInsertionPt()); in targetParallelCallback()
1202 Builder.CreateAlloca(ArrayType::get(PtrTy, NumCapturedVars)); in targetParallelCallback()
1207 Args = Builder.CreatePointerCast(ArgsAlloca, PtrTy); in targetParallelCallback()
1208 Builder.restoreIP(CurrentIP); in targetParallelCallback()
1213 Value *StoreAddress = Builder.CreateConstInBoundsGEP2_64( in targetParallelCallback()
1215 Builder.CreateStore(V, StoreAddress); in targetParallelCallback()
1219 IfCondition ? Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32) in targetParallelCallback()
1220 : Builder.getInt32(1); in targetParallelCallback()
1227 /* number of threads */ NumThreads ? NumThreads : Builder.getInt32(-1), in targetParallelCallback()
1228 /* Proc bind */ Builder.getInt32(-1), in targetParallelCallback()
1230 Builder.CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr), in targetParallelCallback()
1233 /* number of arguments */ Builder.getInt64(NumCapturedVars)}; in targetParallelCallback()
1238 Builder.CreateCall(RTLFn, Parallel51CallArgs); in targetParallelCallback()
1241 << *Builder.GetInsertBlock()->getParent() << "\n"); in targetParallelCallback()
1244 Builder.SetInsertPoint(PrivTID); in targetParallelCallback()
1246 Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI), in targetParallelCallback()
1266 IRBuilder<> &Builder = OMPIRBuilder->Builder; in hostParallelCallback() local
1301 Builder.SetInsertPoint(CI); in hostParallelCallback()
1305 Ident, Builder.getInt32(NumCapturedVars), in hostParallelCallback()
1306 Builder.CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr)}; in hostParallelCallback()
1311 Value *Cond = Builder.CreateSExtOrTrunc(IfCondition, OMPIRBuilder->Int32); in hostParallelCallback()
1324 RealArgs.back() = Builder.CreateBitCast(RealArgs.back(), PtrTy); in hostParallelCallback()
1326 Builder.CreateCall(RTLFn, RealArgs); in hostParallelCallback()
1329 << *Builder.GetInsertBlock()->getParent() << "\n"); in hostParallelCallback()
1332 Builder.SetInsertPoint(PrivTID); in hostParallelCallback()
1334 Builder.CreateStore(Builder.CreateLoad(OMPIRBuilder->Int32, OutlinedAI), in hostParallelCallback()
1371 Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)}; in createParallel()
1372 Builder.CreateCall( in createParallel()
1381 Builder.CreateCall( in createParallel()
1385 BasicBlock *InsertBB = Builder.GetInsertBlock(); in createParallel()
1399 Builder.restoreIP(NewOuter); in createParallel()
1400 AllocaInst *TIDAddrAlloca = Builder.CreateAlloca(Int32, nullptr, "tid.addr"); in createParallel()
1402 Builder.CreateAlloca(Int32, nullptr, "zero.addr"); in createParallel()
1425 auto *UI = new UnreachableInst(Builder.getContext(), InsertBB); in createParallel()
1438 IRBuilder<>::InsertPointGuard IPG(Builder); in createParallel()
1439 Builder.restoreIP(IP); in createParallel()
1440 Instruction *I = Builder.CreateBr(PRegExitBB); in createParallel()
1453 Builder.SetInsertPoint(PRegEntryBB->getTerminator()); in createParallel()
1454 InsertPointTy InnerAllocaIP = Builder.saveIP(); in createParallel()
1457 Builder.CreateAlloca(Int32, nullptr, "tid.addr.local"); in createParallel()
1458 Instruction *PrivTID = Builder.CreateLoad(Int32, PrivTIDAddr, "tid"); in createParallel()
1461 ToBeDeleted.push_back(Builder.CreateLoad(Int32, TIDAddr, "tid.addr.use")); in createParallel()
1463 Builder.CreateLoad(Int32, ZeroAddr, "zero.addr.use"); in createParallel()
1566 IRBuilder<>::InsertPointGuard Guard(Builder); in createParallel()
1569 Builder.restoreIP(OuterAllocaIP); in createParallel()
1571 Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded"); in createParallel()
1575 Builder.SetInsertPoint(InsertBB, in createParallel()
1577 Builder.CreateStore(&V, Ptr); in createParallel()
1580 Builder.restoreIP(InnerAllocaIP); in createParallel()
1581 Inner = Builder.CreateLoad(V.getType(), Ptr); in createParallel()
1589 Builder.restoreIP( in createParallel()
1590 PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue)); in createParallel()
1664 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush), Args); in emitFlush()
1682 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait), in emitTaskwaitImpl()
1700 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield), in emitTaskyieldImpl()
1738 IRBuilderBase &Builder = OMPBuilder.Builder; in emitTaskDependencies() local
1743 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP(); in emitTaskDependencies()
1744 Builder.SetInsertPoint( in emitTaskDependencies()
1748 DepArray = Builder.CreateAlloca(DepArrayTy, nullptr, ".dep.arr.addr"); in emitTaskDependencies()
1752 Builder.CreateConstInBoundsGEP2_64(DepArrayTy, DepArray, 0, DepIdx); in emitTaskDependencies()
1754 Value *Addr = Builder.CreateStructGEP( in emitTaskDependencies()
1757 Value *DepValPtr = Builder.CreatePtrToInt(Dep.DepVal, Builder.getInt64Ty()); in emitTaskDependencies()
1758 Builder.CreateStore(DepValPtr, Addr); in emitTaskDependencies()
1760 Value *Size = Builder.CreateStructGEP( in emitTaskDependencies()
1762 Builder.CreateStore( in emitTaskDependencies()
1763 Builder.getInt64(M.getDataLayout().getTypeStoreSize(Dep.DepValueType)), in emitTaskDependencies()
1766 Value *Flags = Builder.CreateStructGEP( in emitTaskDependencies()
1769 Builder.CreateStore( in emitTaskDependencies()
1770 ConstantInt::get(Builder.getInt8Ty(), in emitTaskDependencies()
1774 Builder.restoreIP(OldIP); in emitTaskDependencies()
1806 BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, "task.exit"); in createTask()
1807 BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, "task.body"); in createTask()
1809 splitBB(Builder, /*CreateBranch=*/true, "task.alloca"); in createTask()
1824 Builder, AllocaIP, ToBeDeleted, TaskAllocaIP, "global.tid", false)); in createTask()
1836 Builder.SetInsertPoint(StaleCI); in createTask()
1853 Value *Flags = Builder.getInt32(Tied); in createTask()
1856 Builder.CreateSelect(Final, Builder.getInt32(2), Builder.getInt32(0)); in createTask()
1857 Flags = Builder.CreateOr(FinalFlag, Flags); in createTask()
1864 Value *TaskSize = Builder.getInt64( in createTask()
1870 Value *SharedsSize = Builder.getInt64(0); in createTask()
1882 Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType)); in createTask()
1887 CallInst *TaskData = Builder.CreateCall( in createTask()
1896 Value *TaskShareds = Builder.CreateLoad(VoidPtr, TaskData); in createTask()
1897 Builder.CreateMemCpy(TaskShareds, Alignment, Shareds, Alignment, in createTask()
1903 InsertPointTy OldIP = Builder.saveIP(); in createTask()
1904 Builder.SetInsertPoint( in createTask()
1908 DepArray = Builder.CreateAlloca(DepArrayTy, nullptr, ".dep.arr.addr"); in createTask()
1913 Builder.CreateConstInBoundsGEP2_64(DepArrayTy, DepArray, 0, P); in createTask()
1915 Value *Addr = Builder.CreateStructGEP( in createTask()
1919 Builder.CreatePtrToInt(Dep.DepVal, Builder.getInt64Ty()); in createTask()
1920 Builder.CreateStore(DepValPtr, Addr); in createTask()
1922 Value *Size = Builder.CreateStructGEP( in createTask()
1925 Builder.CreateStore(Builder.getInt64(M.getDataLayout().getTypeStoreSize( in createTask()
1929 Value *Flags = Builder.CreateStructGEP( in createTask()
1932 Builder.CreateStore( in createTask()
1933 ConstantInt::get(Builder.getInt8Ty(), in createTask()
1939 Builder.restoreIP(OldIP); in createTask()
1962 splitBB(Builder, /*CreateBranch=*/true, "if.end"); in createTask()
1964 Builder.GetInsertPoint()->getParent()->getTerminator(); in createTask()
1966 Builder.SetInsertPoint(IfTerminator); in createTask()
1969 Builder.SetInsertPoint(ElseTI); in createTask()
1974 Builder.CreateCall( in createTask()
1976 {Ident, ThreadID, Builder.getInt32(Dependencies.size()), DepArray, in createTask()
1977 ConstantInt::get(Builder.getInt32Ty(), 0), in createTask()
1984 Builder.CreateCall(TaskBeginFn, {Ident, ThreadID, TaskData}); in createTask()
1987 CI = Builder.CreateCall(&OutlinedFn, {ThreadID, TaskData}); in createTask()
1989 CI = Builder.CreateCall(&OutlinedFn, {ThreadID}); in createTask()
1991 Builder.CreateCall(TaskCompleteFn, {Ident, ThreadID, TaskData}); in createTask()
1992 Builder.SetInsertPoint(ThenTI); in createTask()
1998 Builder.CreateCall( in createTask()
2000 {Ident, ThreadID, TaskData, Builder.getInt32(Dependencies.size()), in createTask()
2001 DepArray, ConstantInt::get(Builder.getInt32Ty(), 0), in createTask()
2007 Builder.CreateCall(TaskFn, {Ident, ThreadID, TaskData}); in createTask()
2012 Builder.SetInsertPoint(TaskAllocaBB, TaskAllocaBB->begin()); in createTask()
2014 LoadInst *Shareds = Builder.CreateLoad(VoidPtr, OutlinedFn.getArg(1)); in createTask()
2024 Builder.SetInsertPoint(TaskExitBB, TaskExitBB->begin()); in createTask()
2026 return Builder.saveIP(); in createTask()
2044 Builder.CreateCall(TaskgroupFn, {Ident, ThreadID}); in createTaskgroup()
2046 BasicBlock *TaskgroupExitBB = splitBB(Builder, true, "taskgroup.exit"); in createTaskgroup()
2047 BodyGenCB(AllocaIP, Builder.saveIP()); in createTaskgroup()
2049 Builder.SetInsertPoint(TaskgroupExitBB); in createTaskgroup()
2053 Builder.CreateCall(EndTaskgroupFn, {Ident, ThreadID}); in createTaskgroup()
2055 return Builder.saveIP(); in createTaskgroup()
2077 IRBuilder<>::InsertPointGuard IPG(Builder); in createSections()
2078 Builder.restoreIP(IP); in createSections()
2082 Instruction *I = Builder.CreateBr(ExitBB); in createSections()
2106 Builder.restoreIP(CodeGenIP); in createSections()
2108 splitBBWithSuffix(Builder, /*CreateBranch=*/false, ".sections.after"); in createSections()
2110 SwitchInst *SwitchStmt = Builder.CreateSwitch(IndVar, Continue); in createSections()
2116 SwitchStmt->addCase(Builder.getInt32(CaseNumber), CaseBB); in createSections()
2117 Builder.SetInsertPoint(CaseBB); in createSections()
2118 BranchInst *CaseEndBr = Builder.CreateBr(Continue); in createSections()
2142 Builder.restoreIP(AfterIP); in createSections()
2144 splitBBWithSuffix(Builder, /*CreateBranch=*/true, "sections.fini"); in createSections()
2145 CB(Builder.saveIP()); in createSections()
2169 IRBuilder<>::InsertPointGuard IPG(Builder); in createSection()
2170 Builder.restoreIP(IP); in createSection()
2174 Instruction *I = Builder.CreateBr(ExitBB); in createSection()
2203 cast<Constant>(&*List[I]), Builder.getPtrTy()); in emitUsed()
2207 ArrayType *ATy = ArrayType::get(Builder.getPtrTy(), UsedArray.size()); in emitUsed()
2216 return Builder.CreateCall( in getGPUThreadID()
2223 return Builder.CreateCall( in getGPUWarpSize()
2229 return Builder.CreateAShr(getGPUThreadID(), LaneIDBits, "nvptx_warp_id"); in getNVPTXWarpID()
2236 return Builder.CreateAnd(getGPUThreadID(), Builder.getInt32(LaneIDMask), in getNVPTXLaneID()
2250 return Builder.CreateBitCast(From, ToType); in castValueToType()
2252 return Builder.CreateIntCast(From, ToType, /*isSigned*/ true); in castValueToType()
2253 InsertPointTy SaveIP = Builder.saveIP(); in castValueToType()
2254 Builder.restoreIP(AllocaIP); in castValueToType()
2255 Value *CastItem = Builder.CreateAlloca(ToType); in castValueToType()
2256 Builder.restoreIP(SaveIP); in castValueToType()
2258 Value *ValCastItem = Builder.CreatePointerBitCastOrAddrSpaceCast( in castValueToType()
2260 Builder.CreateStore(From, ValCastItem); in castValueToType()
2261 return Builder.CreateLoad(ToType, CastItem); in castValueToType()
2272 Type *CastTy = Builder.getIntNTy(Size <= 4 ? 32 : 64); in createRuntimeShuffleFunction()
2275 Builder.CreateIntCast(getGPUWarpSize(), Builder.getInt16Ty(), true); in createRuntimeShuffleFunction()
2280 Builder.CreateIntCast(WarpSize, Builder.getInt16Ty(), /*isSigned=*/true); in createRuntimeShuffleFunction()
2282 Builder.CreateCall(ShuffleFunc, {ElemCast, Offset, WarpSizeCast}); in createRuntimeShuffleFunction()
2300 Type *IndexTy = Builder.getIndexTy( in shuffleAndStore()
2307 Type *IntType = Builder.getIntNTy(IntSize * 8); in shuffleAndStore()
2308 Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast( in shuffleAndStore()
2311 Builder.CreateGEP(ElemType, SrcAddr, {ConstantInt::get(IndexTy, 1)}); in shuffleAndStore()
2312 ElemPtr = Builder.CreatePointerBitCastOrAddrSpaceCast( in shuffleAndStore()
2315 Function *CurFunc = Builder.GetInsertBlock()->getParent(); in shuffleAndStore()
2317 Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast( in shuffleAndStore()
2318 SrcAddrGEP, Builder.getPtrTy()); in shuffleAndStore()
2323 BasicBlock *CurrentBB = Builder.GetInsertBlock(); in shuffleAndStore()
2326 Builder.CreatePHI(Ptr->getType(), /*NumReservedValues=*/2); in shuffleAndStore()
2329 Builder.CreatePHI(ElemPtr->getType(), /*NumReservedValues=*/2); in shuffleAndStore()
2333 Value *PtrDiff = Builder.CreatePtrDiff( in shuffleAndStore()
2334 Builder.getInt8Ty(), PtrEnd, in shuffleAndStore()
2335 Builder.CreatePointerBitCastOrAddrSpaceCast(Ptr, Builder.getPtrTy())); in shuffleAndStore()
2336 Builder.CreateCondBr( in shuffleAndStore()
2337 Builder.CreateICmpSGT(PtrDiff, Builder.getInt64(IntSize - 1)), ThenBB, in shuffleAndStore()
2342 Builder.CreateAlignedLoad( in shuffleAndStore()
2345 Builder.CreateAlignedStore(Res, ElemPtr, in shuffleAndStore()
2348 Builder.CreateGEP(IntType, Ptr, {ConstantInt::get(IndexTy, 1)}); in shuffleAndStore()
2350 Builder.CreateGEP(IntType, ElemPtr, {ConstantInt::get(IndexTy, 1)}); in shuffleAndStore()
2357 AllocaIP, Builder.CreateLoad(IntType, Ptr), IntType, Offset); in shuffleAndStore()
2360 Res = Builder.CreateTrunc(Res, ElemType); in shuffleAndStore()
2361 Builder.CreateStore(Res, ElemPtr); in shuffleAndStore()
2362 Ptr = Builder.CreateGEP(IntType, Ptr, {ConstantInt::get(IndexTy, 1)}); in shuffleAndStore()
2364 Builder.CreateGEP(IntType, ElemPtr, {ConstantInt::get(IndexTy, 1)}); in shuffleAndStore()
2374 Type *IndexTy = Builder.getIndexTy( in emitReductionListCopy()
2392 Value *SrcElementPtrAddr = Builder.CreateInBoundsGEP( in emitReductionListCopy()
2395 SrcElementAddr = Builder.CreateLoad(Builder.getPtrTy(), SrcElementPtrAddr); in emitReductionListCopy()
2399 DestElementPtrAddr = Builder.CreateInBoundsGEP( in emitReductionListCopy()
2404 InsertPointTy CurIP = Builder.saveIP(); in emitReductionListCopy()
2405 Builder.restoreIP(AllocaIP); in emitReductionListCopy()
2406 AllocaInst *DestAlloca = Builder.CreateAlloca(RI.ElementType, nullptr, in emitReductionListCopy()
2412 Builder.CreateAddrSpaceCast(DestElementAddr, Builder.getPtrTy(), in emitReductionListCopy()
2414 Builder.restoreIP(CurIP); in emitReductionListCopy()
2421 Builder.CreateLoad(Builder.getPtrTy(), DestElementPtrAddr); in emitReductionListCopy()
2434 Value *Elem = Builder.CreateLoad(RI.ElementType, SrcElementAddr); in emitReductionListCopy()
2436 Builder.CreateStore(Elem, DestElementAddr); in emitReductionListCopy()
2440 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32( in emitReductionListCopy()
2442 Value *SrcReal = Builder.CreateLoad( in emitReductionListCopy()
2444 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32( in emitReductionListCopy()
2446 Value *SrcImg = Builder.CreateLoad( in emitReductionListCopy()
2449 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32( in emitReductionListCopy()
2451 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32( in emitReductionListCopy()
2453 Builder.CreateStore(SrcReal, DestRealPtr); in emitReductionListCopy()
2454 Builder.CreateStore(SrcImg, DestImgPtr); in emitReductionListCopy()
2458 Value *SizeVal = Builder.getInt64( in emitReductionListCopy()
2460 Builder.CreateMemCpy( in emitReductionListCopy()
2475 Value *CastDestAddr = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionListCopy()
2476 DestElementAddr, Builder.getPtrTy(), in emitReductionListCopy()
2478 Builder.CreateStore(CastDestAddr, DestElementPtrAddr); in emitReductionListCopy()
2486 InsertPointTy SavedIP = Builder.saveIP(); in emitInterWarpCopyFunction()
2489 Builder.getVoidTy(), {Builder.getPtrTy(), Builder.getInt32Ty()}, in emitInterWarpCopyFunction()
2498 Builder.SetInsertPoint(EntryBB); in emitInterWarpCopyFunction()
2519 ArrayType *ArrayTy = ArrayType::get(Builder.getInt32Ty(), WarpSize); in emitInterWarpCopyFunction()
2536 InsertPointTy(Builder.GetInsertBlock(), in emitInterWarpCopyFunction()
2537 Builder.GetInsertBlock()->getFirstInsertionPt()); in emitInterWarpCopyFunction()
2540 Builder.restoreIP(AllocaIP); in emitInterWarpCopyFunction()
2541 AllocaInst *ReduceListAlloca = Builder.CreateAlloca( in emitInterWarpCopyFunction()
2544 Builder.CreateAlloca(Arg1Type, nullptr, NumWarpsArg->getName() + ".addr"); in emitInterWarpCopyFunction()
2545 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitInterWarpCopyFunction()
2547 Value *NumWarpsAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitInterWarpCopyFunction()
2550 Builder.CreateStore(ReduceListArg, ReduceListAddrCast); in emitInterWarpCopyFunction()
2551 Builder.CreateStore(NumWarpsArg, NumWarpsAddrCast); in emitInterWarpCopyFunction()
2554 getInsertPointAfterInstr(&Builder.GetInsertBlock()->back()); in emitInterWarpCopyFunction()
2555 Builder.restoreIP(CodeGenIP); in emitInterWarpCopyFunction()
2558 Builder.CreateLoad(Builder.getPtrTy(), ReduceListAddrCast); in emitInterWarpCopyFunction()
2568 Type *CType = Builder.getIntNTy(TySize * 8); in emitInterWarpCopyFunction()
2578 CodeGenIP = Builder.saveIP(); in emitInterWarpCopyFunction()
2579 Builder.restoreIP(AllocaIP); in emitInterWarpCopyFunction()
2581 Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, ".cnt.addr"); in emitInterWarpCopyFunction()
2583 CntAddr = Builder.CreateAddrSpaceCast(CntAddr, Builder.getPtrTy(), in emitInterWarpCopyFunction()
2585 Builder.restoreIP(CodeGenIP); in emitInterWarpCopyFunction()
2586 Builder.CreateStore(Constant::getNullValue(Builder.getInt32Ty()), in emitInterWarpCopyFunction()
2592 emitBlock(PrecondBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2593 Cnt = Builder.CreateLoad(Builder.getInt32Ty(), CntAddr, in emitInterWarpCopyFunction()
2595 Value *Cmp = Builder.CreateICmpULT( in emitInterWarpCopyFunction()
2596 Cnt, ConstantInt::get(Builder.getInt32Ty(), NumIters)); in emitInterWarpCopyFunction()
2597 Builder.CreateCondBr(Cmp, BodyBB, ExitBB); in emitInterWarpCopyFunction()
2598 emitBlock(BodyBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2602 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), in emitInterWarpCopyFunction()
2611 Value *IsWarpMaster = Builder.CreateIsNull(LaneID, "warp_master"); in emitInterWarpCopyFunction()
2612 Builder.CreateCondBr(IsWarpMaster, ThenBB, ElseBB); in emitInterWarpCopyFunction()
2613 emitBlock(ThenBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2617 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size()); in emitInterWarpCopyFunction()
2618 Type *IndexTy = Builder.getIndexTy( in emitInterWarpCopyFunction()
2621 Builder.CreateInBoundsGEP(RedListArrayTy, ReduceList, in emitInterWarpCopyFunction()
2625 Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr); in emitInterWarpCopyFunction()
2627 ElemPtr = Builder.CreateGEP(Builder.getInt32Ty(), ElemPtr, Cnt); in emitInterWarpCopyFunction()
2631 Value *MediumPtr = Builder.CreateInBoundsGEP( in emitInterWarpCopyFunction()
2632 ArrayTy, TransferMedium, {Builder.getInt64(0), WarpID}); in emitInterWarpCopyFunction()
2635 Value *Elem = Builder.CreateLoad(CType, ElemPtr); in emitInterWarpCopyFunction()
2637 Builder.CreateStore(Elem, MediumPtr, in emitInterWarpCopyFunction()
2639 Builder.CreateBr(MergeBB); in emitInterWarpCopyFunction()
2642 emitBlock(ElseBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2643 Builder.CreateBr(MergeBB); in emitInterWarpCopyFunction()
2646 emitBlock(MergeBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2647 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), in emitInterWarpCopyFunction()
2658 Builder.CreateLoad(Builder.getInt32Ty(), NumWarpsAddrCast); in emitInterWarpCopyFunction()
2661 Builder.CreateICmpULT(GPUThreadID, NumWarpsVal, "is_active_thread"); in emitInterWarpCopyFunction()
2662 Builder.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB); in emitInterWarpCopyFunction()
2664 emitBlock(W0ThenBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2668 Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP( in emitInterWarpCopyFunction()
2669 ArrayTy, TransferMedium, {Builder.getInt64(0), GPUThreadID}); in emitInterWarpCopyFunction()
2672 Builder.CreateInBoundsGEP(RedListArrayTy, ReduceList, in emitInterWarpCopyFunction()
2676 Builder.CreateLoad(Builder.getPtrTy(), TargetElemPtrPtr); in emitInterWarpCopyFunction()
2680 Builder.CreateGEP(Builder.getInt32Ty(), TargetElemPtr, Cnt); in emitInterWarpCopyFunction()
2684 Builder.CreateLoad(CType, SrcMediumPtrVal, /*IsVolatile*/ true); in emitInterWarpCopyFunction()
2685 Builder.CreateStore(SrcMediumValue, TargetElemPtr); in emitInterWarpCopyFunction()
2686 Builder.CreateBr(W0MergeBB); in emitInterWarpCopyFunction()
2688 emitBlock(W0ElseBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2689 Builder.CreateBr(W0MergeBB); in emitInterWarpCopyFunction()
2691 emitBlock(W0MergeBB, Builder.GetInsertBlock()->getParent()); in emitInterWarpCopyFunction()
2694 Cnt = Builder.CreateNSWAdd( in emitInterWarpCopyFunction()
2695 Cnt, ConstantInt::get(Builder.getInt32Ty(), /*V=*/1)); in emitInterWarpCopyFunction()
2696 Builder.CreateStore(Cnt, CntAddr, /*Volatile=*/false); in emitInterWarpCopyFunction()
2698 auto *CurFn = Builder.GetInsertBlock()->getParent(); in emitInterWarpCopyFunction()
2706 Builder.CreateRetVoid(); in emitInterWarpCopyFunction()
2707 Builder.restoreIP(SavedIP); in emitInterWarpCopyFunction()
2717 FunctionType::get(Builder.getVoidTy(), in emitShuffleAndReduceFunction()
2718 {Builder.getPtrTy(), Builder.getInt16Ty(), in emitShuffleAndReduceFunction()
2719 Builder.getInt16Ty(), Builder.getInt16Ty()}, in emitShuffleAndReduceFunction()
2733 Builder.SetInsertPoint(EntryBB); in emitShuffleAndReduceFunction()
2747 Value *ReduceListAlloca = Builder.CreateAlloca( in emitShuffleAndReduceFunction()
2749 Value *LaneIdAlloca = Builder.CreateAlloca(LaneIDArgType, nullptr, in emitShuffleAndReduceFunction()
2751 Value *RemoteLaneOffsetAlloca = Builder.CreateAlloca( in emitShuffleAndReduceFunction()
2753 Value *AlgoVerAlloca = Builder.CreateAlloca(LaneIDArgType, nullptr, in emitShuffleAndReduceFunction()
2756 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size()); in emitShuffleAndReduceFunction()
2760 Instruction *RemoteReductionListAlloca = Builder.CreateAlloca( in emitShuffleAndReduceFunction()
2763 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2766 Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2768 Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2771 Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2773 Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2774 RemoteReductionListAlloca, Builder.getPtrTy(), in emitShuffleAndReduceFunction()
2777 Builder.CreateStore(ReduceListArg, ReduceListAddrCast); in emitShuffleAndReduceFunction()
2778 Builder.CreateStore(LaneIDArg, LaneIdAddrCast); in emitShuffleAndReduceFunction()
2779 Builder.CreateStore(RemoteLaneOffsetArg, RemoteLaneOffsetAddrCast); in emitShuffleAndReduceFunction()
2780 Builder.CreateStore(AlgoVerArg, AlgoVerAddrCast); in emitShuffleAndReduceFunction()
2782 Value *ReduceList = Builder.CreateLoad(ReduceListArgType, ReduceListAddrCast); in emitShuffleAndReduceFunction()
2783 Value *LaneId = Builder.CreateLoad(LaneIDArgType, LaneIdAddrCast); in emitShuffleAndReduceFunction()
2785 Builder.CreateLoad(LaneIDArgType, RemoteLaneOffsetAddrCast); in emitShuffleAndReduceFunction()
2786 Value *AlgoVer = Builder.CreateLoad(LaneIDArgType, AlgoVerAddrCast); in emitShuffleAndReduceFunction()
2818 Value *CondAlgo0 = Builder.CreateIsNull(AlgoVer); in emitShuffleAndReduceFunction()
2819 Value *Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1)); in emitShuffleAndReduceFunction()
2820 Value *LaneComp = Builder.CreateICmpULT(LaneId, RemoteLaneOffset); in emitShuffleAndReduceFunction()
2821 Value *CondAlgo1 = Builder.CreateAnd(Algo1, LaneComp); in emitShuffleAndReduceFunction()
2822 Value *Algo2 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(2)); in emitShuffleAndReduceFunction()
2823 Value *LaneIdAnd1 = Builder.CreateAnd(LaneId, Builder.getInt16(1)); in emitShuffleAndReduceFunction()
2824 Value *LaneIdComp = Builder.CreateIsNull(LaneIdAnd1); in emitShuffleAndReduceFunction()
2825 Value *Algo2AndLaneIdComp = Builder.CreateAnd(Algo2, LaneIdComp); in emitShuffleAndReduceFunction()
2827 Builder.CreateICmpSGT(RemoteLaneOffset, Builder.getInt16(0)); in emitShuffleAndReduceFunction()
2828 Value *CondAlgo2 = Builder.CreateAnd(Algo2AndLaneIdComp, RemoteOffsetComp); in emitShuffleAndReduceFunction()
2829 Value *CA0OrCA1 = Builder.CreateOr(CondAlgo0, CondAlgo1); in emitShuffleAndReduceFunction()
2830 Value *CondReduce = Builder.CreateOr(CA0OrCA1, CondAlgo2); in emitShuffleAndReduceFunction()
2836 Builder.CreateCondBr(CondReduce, ThenBB, ElseBB); in emitShuffleAndReduceFunction()
2837 emitBlock(ThenBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2838 Value *LocalReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2839 ReduceList, Builder.getPtrTy()); in emitShuffleAndReduceFunction()
2840 Value *RemoteReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitShuffleAndReduceFunction()
2841 RemoteListAddrCast, Builder.getPtrTy()); in emitShuffleAndReduceFunction()
2842 Builder.CreateCall(ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr}) in emitShuffleAndReduceFunction()
2844 Builder.CreateBr(MergeBB); in emitShuffleAndReduceFunction()
2846 emitBlock(ElseBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2847 Builder.CreateBr(MergeBB); in emitShuffleAndReduceFunction()
2849 emitBlock(MergeBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2853 Algo1 = Builder.CreateICmpEQ(AlgoVer, Builder.getInt16(1)); in emitShuffleAndReduceFunction()
2854 Value *LaneIdGtOffset = Builder.CreateICmpUGE(LaneId, RemoteLaneOffset); in emitShuffleAndReduceFunction()
2855 Value *CondCopy = Builder.CreateAnd(Algo1, LaneIdGtOffset); in emitShuffleAndReduceFunction()
2860 Builder.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB); in emitShuffleAndReduceFunction()
2862 emitBlock(CpyThenBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2865 Builder.CreateBr(CpyMergeBB); in emitShuffleAndReduceFunction()
2867 emitBlock(CpyElseBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2868 Builder.CreateBr(CpyMergeBB); in emitShuffleAndReduceFunction()
2870 emitBlock(CpyMergeBB, Builder.GetInsertBlock()->getParent()); in emitShuffleAndReduceFunction()
2872 Builder.CreateRetVoid(); in emitShuffleAndReduceFunction()
2880 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP(); in emitListToGlobalCopyFunction()
2883 Builder.getVoidTy(), in emitListToGlobalCopyFunction()
2884 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()}, in emitListToGlobalCopyFunction()
2895 Builder.SetInsertPoint(EntryBlock); in emitListToGlobalCopyFunction()
2904 Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr, in emitListToGlobalCopyFunction()
2906 Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, in emitListToGlobalCopyFunction()
2908 Value *ReduceListArgAlloca = Builder.CreateAlloca( in emitListToGlobalCopyFunction()
2909 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr"); in emitListToGlobalCopyFunction()
2910 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
2911 BufferArgAlloca, Builder.getPtrTy(), in emitListToGlobalCopyFunction()
2913 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
2914 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast"); in emitListToGlobalCopyFunction()
2915 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalCopyFunction()
2916 ReduceListArgAlloca, Builder.getPtrTy(), in emitListToGlobalCopyFunction()
2919 Builder.CreateStore(BufferArg, BufferArgAddrCast); in emitListToGlobalCopyFunction()
2920 Builder.CreateStore(IdxArg, IdxArgAddrCast); in emitListToGlobalCopyFunction()
2921 Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast); in emitListToGlobalCopyFunction()
2924 Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast); in emitListToGlobalCopyFunction()
2926 Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast); in emitListToGlobalCopyFunction()
2927 Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)}; in emitListToGlobalCopyFunction()
2928 Type *IndexTy = Builder.getIndexTy( in emitListToGlobalCopyFunction()
2933 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size()); in emitListToGlobalCopyFunction()
2935 Value *ElemPtrPtr = Builder.CreateInBoundsGEP( in emitListToGlobalCopyFunction()
2939 Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr); in emitListToGlobalCopyFunction()
2943 Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferArgVal, Idxs); in emitListToGlobalCopyFunction()
2944 Value *GlobVal = Builder.CreateConstInBoundsGEP2_32( in emitListToGlobalCopyFunction()
2949 Value *TargetElement = Builder.CreateLoad(RI.ElementType, ElemPtr); in emitListToGlobalCopyFunction()
2950 Builder.CreateStore(TargetElement, GlobVal); in emitListToGlobalCopyFunction()
2954 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32( in emitListToGlobalCopyFunction()
2956 Value *SrcReal = Builder.CreateLoad( in emitListToGlobalCopyFunction()
2958 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32( in emitListToGlobalCopyFunction()
2960 Value *SrcImg = Builder.CreateLoad( in emitListToGlobalCopyFunction()
2963 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32( in emitListToGlobalCopyFunction()
2965 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32( in emitListToGlobalCopyFunction()
2967 Builder.CreateStore(SrcReal, DestRealPtr); in emitListToGlobalCopyFunction()
2968 Builder.CreateStore(SrcImg, DestImgPtr); in emitListToGlobalCopyFunction()
2973 Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType)); in emitListToGlobalCopyFunction()
2974 Builder.CreateMemCpy( in emitListToGlobalCopyFunction()
2982 Builder.CreateRetVoid(); in emitListToGlobalCopyFunction()
2983 Builder.restoreIP(OldIP); in emitListToGlobalCopyFunction()
2990 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP(); in emitListToGlobalReduceFunction()
2993 Builder.getVoidTy(), in emitListToGlobalReduceFunction()
2994 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()}, in emitListToGlobalReduceFunction()
3005 Builder.SetInsertPoint(EntryBlock); in emitListToGlobalReduceFunction()
3014 Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr, in emitListToGlobalReduceFunction()
3016 Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, in emitListToGlobalReduceFunction()
3018 Value *ReduceListArgAlloca = Builder.CreateAlloca( in emitListToGlobalReduceFunction()
3019 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr"); in emitListToGlobalReduceFunction()
3021 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size()); in emitListToGlobalReduceFunction()
3026 Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list"); in emitListToGlobalReduceFunction()
3028 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalReduceFunction()
3029 BufferArgAlloca, Builder.getPtrTy(), in emitListToGlobalReduceFunction()
3031 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalReduceFunction()
3032 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast"); in emitListToGlobalReduceFunction()
3033 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalReduceFunction()
3034 ReduceListArgAlloca, Builder.getPtrTy(), in emitListToGlobalReduceFunction()
3036 Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitListToGlobalReduceFunction()
3037 LocalReduceList, Builder.getPtrTy(), in emitListToGlobalReduceFunction()
3040 Builder.CreateStore(BufferArg, BufferArgAddrCast); in emitListToGlobalReduceFunction()
3041 Builder.CreateStore(IdxArg, IdxArgAddrCast); in emitListToGlobalReduceFunction()
3042 Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast); in emitListToGlobalReduceFunction()
3044 Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast); in emitListToGlobalReduceFunction()
3045 Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)}; in emitListToGlobalReduceFunction()
3046 Type *IndexTy = Builder.getIndexTy( in emitListToGlobalReduceFunction()
3049 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP( in emitListToGlobalReduceFunction()
3053 Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs); in emitListToGlobalReduceFunction()
3055 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32( in emitListToGlobalReduceFunction()
3057 Builder.CreateStore(GlobValPtr, TargetElementPtrPtr); in emitListToGlobalReduceFunction()
3062 Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast); in emitListToGlobalReduceFunction()
3063 Builder.CreateCall(ReduceFn, {LocalReduceListAddrCast, ReduceList}) in emitListToGlobalReduceFunction()
3065 Builder.CreateRetVoid(); in emitListToGlobalReduceFunction()
3066 Builder.restoreIP(OldIP); in emitListToGlobalReduceFunction()
3073 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP(); in emitGlobalToListCopyFunction()
3076 Builder.getVoidTy(), in emitGlobalToListCopyFunction()
3077 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()}, in emitGlobalToListCopyFunction()
3088 Builder.SetInsertPoint(EntryBlock); in emitGlobalToListCopyFunction()
3097 Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr, in emitGlobalToListCopyFunction()
3099 Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, in emitGlobalToListCopyFunction()
3101 Value *ReduceListArgAlloca = Builder.CreateAlloca( in emitGlobalToListCopyFunction()
3102 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr"); in emitGlobalToListCopyFunction()
3103 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
3104 BufferArgAlloca, Builder.getPtrTy(), in emitGlobalToListCopyFunction()
3106 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
3107 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast"); in emitGlobalToListCopyFunction()
3108 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListCopyFunction()
3109 ReduceListArgAlloca, Builder.getPtrTy(), in emitGlobalToListCopyFunction()
3111 Builder.CreateStore(BufferArg, BufferArgAddrCast); in emitGlobalToListCopyFunction()
3112 Builder.CreateStore(IdxArg, IdxArgAddrCast); in emitGlobalToListCopyFunction()
3113 Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast); in emitGlobalToListCopyFunction()
3116 Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast); in emitGlobalToListCopyFunction()
3117 Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast); in emitGlobalToListCopyFunction()
3118 Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)}; in emitGlobalToListCopyFunction()
3119 Type *IndexTy = Builder.getIndexTy( in emitGlobalToListCopyFunction()
3124 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size()); in emitGlobalToListCopyFunction()
3126 Value *ElemPtrPtr = Builder.CreateInBoundsGEP( in emitGlobalToListCopyFunction()
3130 Value *ElemPtr = Builder.CreateLoad(Builder.getPtrTy(), ElemPtrPtr); in emitGlobalToListCopyFunction()
3133 Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs); in emitGlobalToListCopyFunction()
3134 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32( in emitGlobalToListCopyFunction()
3139 Value *TargetElement = Builder.CreateLoad(RI.ElementType, GlobValPtr); in emitGlobalToListCopyFunction()
3140 Builder.CreateStore(TargetElement, ElemPtr); in emitGlobalToListCopyFunction()
3144 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32( in emitGlobalToListCopyFunction()
3146 Value *SrcReal = Builder.CreateLoad( in emitGlobalToListCopyFunction()
3148 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32( in emitGlobalToListCopyFunction()
3150 Value *SrcImg = Builder.CreateLoad( in emitGlobalToListCopyFunction()
3153 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32( in emitGlobalToListCopyFunction()
3155 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32( in emitGlobalToListCopyFunction()
3157 Builder.CreateStore(SrcReal, DestRealPtr); in emitGlobalToListCopyFunction()
3158 Builder.CreateStore(SrcImg, DestImgPtr); in emitGlobalToListCopyFunction()
3163 Builder.getInt64(M.getDataLayout().getTypeStoreSize(RI.ElementType)); in emitGlobalToListCopyFunction()
3164 Builder.CreateMemCpy( in emitGlobalToListCopyFunction()
3173 Builder.CreateRetVoid(); in emitGlobalToListCopyFunction()
3174 Builder.restoreIP(OldIP); in emitGlobalToListCopyFunction()
3181 OpenMPIRBuilder::InsertPointTy OldIP = Builder.saveIP(); in emitGlobalToListReduceFunction()
3184 Builder.getVoidTy(), in emitGlobalToListReduceFunction()
3185 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()}, in emitGlobalToListReduceFunction()
3196 Builder.SetInsertPoint(EntryBlock); in emitGlobalToListReduceFunction()
3205 Value *BufferArgAlloca = Builder.CreateAlloca(Builder.getPtrTy(), nullptr, in emitGlobalToListReduceFunction()
3207 Value *IdxArgAlloca = Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, in emitGlobalToListReduceFunction()
3209 Value *ReduceListArgAlloca = Builder.CreateAlloca( in emitGlobalToListReduceFunction()
3210 Builder.getPtrTy(), nullptr, ReduceListArg->getName() + ".addr"); in emitGlobalToListReduceFunction()
3212 ArrayType::get(Builder.getPtrTy(), ReductionInfos.size()); in emitGlobalToListReduceFunction()
3217 Builder.CreateAlloca(RedListArrayTy, nullptr, ".omp.reduction.red_list"); in emitGlobalToListReduceFunction()
3219 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListReduceFunction()
3220 BufferArgAlloca, Builder.getPtrTy(), in emitGlobalToListReduceFunction()
3222 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListReduceFunction()
3223 IdxArgAlloca, Builder.getPtrTy(), IdxArgAlloca->getName() + ".ascast"); in emitGlobalToListReduceFunction()
3224 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListReduceFunction()
3225 ReduceListArgAlloca, Builder.getPtrTy(), in emitGlobalToListReduceFunction()
3227 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitGlobalToListReduceFunction()
3228 LocalReduceList, Builder.getPtrTy(), in emitGlobalToListReduceFunction()
3231 Builder.CreateStore(BufferArg, BufferArgAddrCast); in emitGlobalToListReduceFunction()
3232 Builder.CreateStore(IdxArg, IdxArgAddrCast); in emitGlobalToListReduceFunction()
3233 Builder.CreateStore(ReduceListArg, ReduceListArgAddrCast); in emitGlobalToListReduceFunction()
3235 Value *BufferVal = Builder.CreateLoad(Builder.getPtrTy(), BufferArgAddrCast); in emitGlobalToListReduceFunction()
3236 Value *Idxs[] = {Builder.CreateLoad(Builder.getInt32Ty(), IdxArgAddrCast)}; in emitGlobalToListReduceFunction()
3237 Type *IndexTy = Builder.getIndexTy( in emitGlobalToListReduceFunction()
3240 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP( in emitGlobalToListReduceFunction()
3245 Builder.CreateInBoundsGEP(ReductionsBufferTy, BufferVal, Idxs); in emitGlobalToListReduceFunction()
3246 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32( in emitGlobalToListReduceFunction()
3248 Builder.CreateStore(GlobValPtr, TargetElementPtrPtr); in emitGlobalToListReduceFunction()
3253 Builder.CreateLoad(Builder.getPtrTy(), ReduceListArgAddrCast); in emitGlobalToListReduceFunction()
3254 Builder.CreateCall(ReduceFn, {ReduceList, ReductionList}) in emitGlobalToListReduceFunction()
3256 Builder.CreateRetVoid(); in emitGlobalToListReduceFunction()
3257 Builder.restoreIP(OldIP); in emitGlobalToListReduceFunction()
3270 auto *FuncTy = FunctionType::get(Builder.getVoidTy(), in createReductionFunction()
3271 {Builder.getPtrTy(), Builder.getPtrTy()}, in createReductionFunction()
3281 Builder.SetInsertPoint(EntryBB); in createReductionFunction()
3293 Builder.CreateAlloca(Arg0Type, nullptr, Arg0->getName() + ".addr"); in createReductionFunction()
3295 Builder.CreateAlloca(Arg1Type, nullptr, Arg1->getName() + ".addr"); in createReductionFunction()
3296 Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in createReductionFunction()
3298 Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast( in createReductionFunction()
3300 Builder.CreateStore(Arg0, LHSAddrCast); in createReductionFunction()
3301 Builder.CreateStore(Arg1, RHSAddrCast); in createReductionFunction()
3302 LHSArrayPtr = Builder.CreateLoad(Arg0Type, LHSAddrCast); in createReductionFunction()
3303 RHSArrayPtr = Builder.CreateLoad(Arg1Type, RHSAddrCast); in createReductionFunction()
3305 Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), ReductionInfos.size()); in createReductionFunction()
3306 Type *IndexTy = Builder.getIndexTy( in createReductionFunction()
3311 Value *RHSI8PtrPtr = Builder.CreateInBoundsGEP( in createReductionFunction()
3314 Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr); in createReductionFunction()
3315 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast( in createReductionFunction()
3319 Value *LHSI8PtrPtr = Builder.CreateInBoundsGEP( in createReductionFunction()
3322 Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr); in createReductionFunction()
3323 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast( in createReductionFunction()
3330 Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr); in createReductionFunction()
3331 Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr); in createReductionFunction()
3333 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced); in createReductionFunction()
3334 if (!Builder.GetInsertBlock()) in createReductionFunction()
3336 Builder.CreateStore(Reduced, LHSPtr); in createReductionFunction()
3345 Builder.restoreIP(RI.ReductionGenClang( in createReductionFunction()
3346 Builder.saveIP(), Index, &LHSFixupPtr, &RHSFixupPtr, ReductionFunc)); in createReductionFunction()
3362 Builder.CreateRetVoid(); in createReductionFunction()
3394 Builder.restoreIP(CodeGenIP); in createReductionsGPU()
3406 return Builder.saveIP(); in createReductionsGPU()
3408 Function *CurFunc = Builder.GetInsertBlock()->getParent(); in createReductionsGPU()
3417 CodeGenIP = Builder.saveIP(); in createReductionsGPU()
3419 createReductionFunction(Builder.GetInsertBlock()->getParent()->getName(), in createReductionsGPU()
3421 Builder.restoreIP(CodeGenIP); in createReductionsGPU()
3445 CodeGenIP = Builder.saveIP(); in createReductionsGPU()
3446 Builder.restoreIP(AllocaIP); in createReductionsGPU()
3448 Builder.CreateAlloca(RedArrayTy, nullptr, ".omp.reduction.red_list"); in createReductionsGPU()
3449 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast( in createReductionsGPU()
3451 Builder.restoreIP(CodeGenIP); in createReductionsGPU()
3452 Type *IndexTy = Builder.getIndexTy( in createReductionsGPU()
3456 Value *ElemPtr = Builder.CreateInBoundsGEP( in createReductionsGPU()
3460 Builder.CreatePointerBitCastOrAddrSpaceCast(RI.PrivateVariable, PtrTy); in createReductionsGPU()
3461 Builder.CreateStore(CastElem, ElemPtr); in createReductionsGPU()
3463 CodeGenIP = Builder.saveIP(); in createReductionsGPU()
3467 Builder.restoreIP(CodeGenIP); in createReductionsGPU()
3469 Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, PtrTy); in createReductionsGPU()
3480 Builder.getInt64(MaxDataSize * ReductionInfos.size()); in createReductionsGPU()
3483 Builder.CreatePointerBitCastOrAddrSpaceCast(SarFunc, PtrTy); in createReductionsGPU()
3485 Builder.CreatePointerBitCastOrAddrSpaceCast(WcFunc, PtrTy); in createReductionsGPU()
3489 Res = Builder.CreateCall(Pv2Ptr, Args); in createReductionsGPU()
3491 CodeGenIP = Builder.saveIP(); in createReductionsGPU()
3504 Builder.restoreIP(CodeGenIP); in createReductionsGPU()
3506 Value *KernelTeamsReductionPtr = Builder.CreateCall( in createReductionsGPU()
3511 Builder.getInt32(ReductionBufNum), in createReductionsGPU()
3523 Res = Builder.CreateCall(TeamsReduceFn, Args3); in createReductionsGPU()
3529 Value *Cond = Builder.CreateICmpEQ(Res, Builder.getInt32(1)); in createReductionsGPU()
3530 Builder.CreateCondBr(Cond, ThenBB, ExitBB); in createReductionsGPU()
3543 Builder.CreatePointerBitCastOrAddrSpaceCast(RI.PrivateVariable, PtrTy); in createReductionsGPU()
3547 Builder.restoreIP(RI.ReductionGenClang(Builder.saveIP(), En.index(), in createReductionsGPU()
3568 return Builder.saveIP(); in createReductionsGPU()
3609 Type *RedArrayTy = ArrayType::get(Builder.getPtrTy(), NumReductions); in createReductions()
3610 Builder.SetInsertPoint(AllocaIP.getBlock()->getTerminator()); in createReductions()
3611 Value *RedArray = Builder.CreateAlloca(RedArrayTy, nullptr, "red.array"); in createReductions()
3613 Builder.SetInsertPoint(InsertBlock, InsertBlock->end()); in createReductions()
3618 Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64( in createReductions()
3620 Builder.CreateStore(RI.PrivateVariable, RedArrayElemPtr); in createReductions()
3625 Function *Func = Builder.GetInsertBlock()->getParent(); in createReductions()
3637 Constant *NumVariables = Builder.getInt32(NumReductions); in createReductions()
3640 Constant *RedArraySize = Builder.getInt64(RedArrayByteSize); in createReductions()
3647 Builder.CreateCall(ReduceFunc, in createReductions()
3660 Builder.CreateSwitch(ReduceCall, ContinuationBlock, /* NumCases */ 2); in createReductions()
3661 Switch->addCase(Builder.getInt32(1), NonAtomicRedBlock); in createReductions()
3662 Switch->addCase(Builder.getInt32(2), AtomicRedBlock); in createReductions()
3667 Builder.SetInsertPoint(NonAtomicRedBlock); in createReductions()
3675 RedValue = Builder.CreateLoad(ValueType, RI.Variable, in createReductions()
3679 Builder.CreateLoad(ValueType, RI.PrivateVariable, in createReductions()
3683 Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RI.Variable, in createReductions()
3686 Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), RedValue, in createReductions()
3689 if (!Builder.GetInsertBlock()) in createReductions()
3693 Builder.CreateStore(Reduced, RI.Variable); in createReductions()
3698 Builder.CreateCall(EndReduceFunc, {Ident, ThreadId, Lock}); in createReductions()
3699 Builder.CreateBr(ContinuationBlock); in createReductions()
3704 Builder.SetInsertPoint(AtomicRedBlock); in createReductions()
3707 Builder.restoreIP(RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType, in createReductions()
3709 if (!Builder.GetInsertBlock()) in createReductions()
3712 Builder.CreateBr(ContinuationBlock); in createReductions()
3714 Builder.CreateUnreachable(); in createReductions()
3722 Builder.SetInsertPoint(ReductionFuncBlock); in createReductions()
3728 Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64( in createReductions()
3730 Value *LHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), LHSI8PtrPtr); in createReductions()
3731 Value *LHSPtr = Builder.CreateBitCast(LHSI8Ptr, RI.Variable->getType()); in createReductions()
3732 Value *LHS = Builder.CreateLoad(RI.ElementType, LHSPtr); in createReductions()
3733 Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64( in createReductions()
3735 Value *RHSI8Ptr = Builder.CreateLoad(Builder.getPtrTy(), RHSI8PtrPtr); in createReductions()
3737 Builder.CreateBitCast(RHSI8Ptr, RI.PrivateVariable->getType()); in createReductions()
3738 Value *RHS = Builder.CreateLoad(RI.ElementType, RHSPtr); in createReductions()
3740 Builder.restoreIP(RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced)); in createReductions()
3741 if (!Builder.GetInsertBlock()) in createReductions()
3745 Builder.CreateStore(Reduced, LHSPtr); in createReductions()
3747 Builder.CreateRetVoid(); in createReductions()
3749 Builder.SetInsertPoint(ContinuationBlock); in createReductions()
3750 return Builder.saveIP(); in createReductions()
3769 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args); in createMaster()
3772 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args); in createMaster()
3794 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args); in createMasked()
3797 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, ArgsEnd); in createMasked()
3827 Builder.SetCurrentDebugLocation(DL); in createLoopSkeleton()
3829 Builder.SetInsertPoint(Preheader); in createLoopSkeleton()
3830 Builder.CreateBr(Header); in createLoopSkeleton()
3832 Builder.SetInsertPoint(Header); in createLoopSkeleton()
3833 PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_" + Name + ".iv"); in createLoopSkeleton()
3835 Builder.CreateBr(Cond); in createLoopSkeleton()
3837 Builder.SetInsertPoint(Cond); in createLoopSkeleton()
3839 Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_" + Name + ".cmp"); in createLoopSkeleton()
3840 Builder.CreateCondBr(Cmp, Body, Exit); in createLoopSkeleton()
3842 Builder.SetInsertPoint(Body); in createLoopSkeleton()
3843 Builder.CreateBr(Latch); in createLoopSkeleton()
3845 Builder.SetInsertPoint(Latch); in createLoopSkeleton()
3846 Value *Next = Builder.CreateAdd(IndVarPHI, ConstantInt::get(IndVarTy, 1), in createLoopSkeleton()
3848 Builder.CreateBr(Header); in createLoopSkeleton()
3851 Builder.SetInsertPoint(Exit); in createLoopSkeleton()
3852 Builder.CreateBr(After); in createLoopSkeleton()
3885 spliceBB(Builder, After, /*CreateBranch=*/false); in createCanonicalLoop()
3886 Builder.CreateBr(CL->getPreheader()); in createCanonicalLoop()
3934 Value *IsNeg = Builder.CreateICmpSLT(Step, Zero); in createCanonicalLoop()
3935 Incr = Builder.CreateSelect(IsNeg, Builder.CreateNeg(Step), Step); in createCanonicalLoop()
3936 Value *LB = Builder.CreateSelect(IsNeg, Stop, Start); in createCanonicalLoop()
3937 Value *UB = Builder.CreateSelect(IsNeg, Start, Stop); in createCanonicalLoop()
3938 Span = Builder.CreateSub(UB, LB, "", false, true); in createCanonicalLoop()
3939 ZeroCmp = Builder.CreateICmp( in createCanonicalLoop()
3942 Span = Builder.CreateSub(Stop, Start, "", true); in createCanonicalLoop()
3943 ZeroCmp = Builder.CreateICmp( in createCanonicalLoop()
3949 CountIfLooping = Builder.CreateAdd(Builder.CreateUDiv(Span, Incr), One); in createCanonicalLoop()
3952 Value *CountIfTwo = Builder.CreateAdd( in createCanonicalLoop()
3953 Builder.CreateUDiv(Builder.CreateSub(Span, One), Incr), One); in createCanonicalLoop()
3954 Value *OneCmp = Builder.CreateICmp(CmpInst::ICMP_ULE, Span, Incr); in createCanonicalLoop()
3955 CountIfLooping = Builder.CreateSelect(OneCmp, One, CountIfTwo); in createCanonicalLoop()
3957 Value *TripCount = Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping, in createCanonicalLoop()
3961 Builder.restoreIP(CodeGenIP); in createCanonicalLoop()
3962 Value *Span = Builder.CreateMul(IV, Step); in createCanonicalLoop()
3963 Value *IndVar = Builder.CreateAdd(Span, Start); in createCanonicalLoop()
3964 BodyGenCB(Builder.saveIP(), IndVar); in createCanonicalLoop()
3966 LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP(); in createCanonicalLoop()
3995 Builder.restoreIP(CLI->getPreheaderIP()); in applyStaticWorkshareLoop()
3996 Builder.SetCurrentDebugLocation(DL); in applyStaticWorkshareLoop()
4010 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca()); in applyStaticWorkshareLoop()
4013 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter"); in applyStaticWorkshareLoop()
4014 Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound"); in applyStaticWorkshareLoop()
4015 Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound"); in applyStaticWorkshareLoop()
4016 Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride"); in applyStaticWorkshareLoop()
4022 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator()); in applyStaticWorkshareLoop()
4025 Builder.CreateStore(Zero, PLowerBound); in applyStaticWorkshareLoop()
4026 Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One); in applyStaticWorkshareLoop()
4027 Builder.CreateStore(UpperBound, PUpperBound); in applyStaticWorkshareLoop()
4028 Builder.CreateStore(One, PStride); in applyStaticWorkshareLoop()
4037 Builder.CreateCall(StaticInit, in applyStaticWorkshareLoop()
4040 Value *LowerBound = Builder.CreateLoad(IVTy, PLowerBound); in applyStaticWorkshareLoop()
4041 Value *InclusiveUpperBound = Builder.CreateLoad(IVTy, PUpperBound); in applyStaticWorkshareLoop()
4042 Value *TripCountMinusOne = Builder.CreateSub(InclusiveUpperBound, LowerBound); in applyStaticWorkshareLoop()
4043 Value *TripCount = Builder.CreateAdd(TripCountMinusOne, One); in applyStaticWorkshareLoop()
4051 Builder.SetInsertPoint(CLI->getBody(), in applyStaticWorkshareLoop()
4053 Builder.SetCurrentDebugLocation(DL); in applyStaticWorkshareLoop()
4054 return Builder.CreateAdd(OldIV, LowerBound); in applyStaticWorkshareLoop()
4058 Builder.SetInsertPoint(CLI->getExit(), in applyStaticWorkshareLoop()
4060 Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum}); in applyStaticWorkshareLoop()
4064 createBarrier(LocationDescription(Builder.saveIP(), DL), in applyStaticWorkshareLoop()
4099 Builder.restoreIP(AllocaIP); in applyStaticChunkedWorkshareLoop()
4100 Builder.SetCurrentDebugLocation(DL); in applyStaticChunkedWorkshareLoop()
4101 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter"); in applyStaticChunkedWorkshareLoop()
4103 Builder.CreateAlloca(InternalIVTy, nullptr, "p.lowerbound"); in applyStaticChunkedWorkshareLoop()
4105 Builder.CreateAlloca(InternalIVTy, nullptr, "p.upperbound"); in applyStaticChunkedWorkshareLoop()
4106 Value *PStride = Builder.CreateAlloca(InternalIVTy, nullptr, "p.stride"); in applyStaticChunkedWorkshareLoop()
4109 Builder.restoreIP(CLI->getPreheaderIP()); in applyStaticChunkedWorkshareLoop()
4110 Builder.SetCurrentDebugLocation(DL); in applyStaticChunkedWorkshareLoop()
4114 Builder.CreateZExtOrTrunc(ChunkSize, InternalIVTy, "chunksize"); in applyStaticChunkedWorkshareLoop()
4116 Builder.CreateZExt(OrigTripCount, InternalIVTy, "tripcount"); in applyStaticChunkedWorkshareLoop()
4120 Builder.CreateStore(Zero, PLowerBound); in applyStaticChunkedWorkshareLoop()
4121 Value *OrigUpperBound = Builder.CreateSub(CastedTripCount, One); in applyStaticChunkedWorkshareLoop()
4122 Builder.CreateStore(OrigUpperBound, PUpperBound); in applyStaticChunkedWorkshareLoop()
4123 Builder.CreateStore(One, PStride); in applyStaticChunkedWorkshareLoop()
4131 Builder.CreateCall(StaticInit, in applyStaticChunkedWorkshareLoop()
4140 Builder.CreateLoad(InternalIVTy, PLowerBound, "omp_firstchunk.lb"); in applyStaticChunkedWorkshareLoop()
4142 Builder.CreateLoad(InternalIVTy, PUpperBound, "omp_firstchunk.ub"); in applyStaticChunkedWorkshareLoop()
4143 Value *FirstChunkEnd = Builder.CreateAdd(FirstChunkStop, One); in applyStaticChunkedWorkshareLoop()
4145 Builder.CreateSub(FirstChunkEnd, FirstChunkStart, "omp_chunk.range"); in applyStaticChunkedWorkshareLoop()
4147 Builder.CreateLoad(InternalIVTy, PStride, "omp_dispatch.stride"); in applyStaticChunkedWorkshareLoop()
4150 BasicBlock *DispatchEnter = splitBB(Builder, true); in applyStaticChunkedWorkshareLoop()
4153 {Builder.saveIP(), DL}, in applyStaticChunkedWorkshareLoop()
4173 Builder.restoreIP(CLI->getPreheaderIP()); in applyStaticChunkedWorkshareLoop()
4174 Builder.SetCurrentDebugLocation(DL); in applyStaticChunkedWorkshareLoop()
4177 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator()); in applyStaticChunkedWorkshareLoop()
4178 Value *ChunkEnd = Builder.CreateAdd(DispatchCounter, ChunkRange); in applyStaticChunkedWorkshareLoop()
4180 Builder.CreateICmpUGE(ChunkEnd, CastedTripCount, "omp_chunk.is_last"); in applyStaticChunkedWorkshareLoop()
4182 Builder.CreateSub(CastedTripCount, DispatchCounter); in applyStaticChunkedWorkshareLoop()
4183 Value *ChunkTripCount = Builder.CreateSelect( in applyStaticChunkedWorkshareLoop()
4186 Builder.CreateTrunc(ChunkTripCount, IVTy, "omp_chunk.tripcount.trunc"); in applyStaticChunkedWorkshareLoop()
4193 Builder.CreateTrunc(DispatchCounter, IVTy, "omp_dispatch.iv.trunc"); in applyStaticChunkedWorkshareLoop()
4195 Builder.restoreIP(CLI->getBodyIP()); in applyStaticChunkedWorkshareLoop()
4196 return Builder.CreateAdd(IV, BackcastedDispatchCounter); in applyStaticChunkedWorkshareLoop()
4200 Builder.SetInsertPoint(DispatchExit, DispatchExit->getFirstInsertionPt()); in applyStaticChunkedWorkshareLoop()
4201 Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum}); in applyStaticChunkedWorkshareLoop()
4205 createBarrier(LocationDescription(Builder.saveIP(), DL), OMPD_for, in applyStaticChunkedWorkshareLoop()
4265 IRBuilder<> &Builder = OMPBuilder->Builder; in createTargetLoopWorkshareCall() local
4270 RealArgs.push_back(Builder.CreateBitCast(&LoopBodyFn, ParallelTaskPtr)); in createTargetLoopWorkshareCall()
4275 Builder.CreateCall(RTLFn, RealArgs); in createTargetLoopWorkshareCall()
4280 Builder.restoreIP({InsertBlock, std::prev(InsertBlock->end())}); in createTargetLoopWorkshareCall()
4281 Value *NumThreads = Builder.CreateCall(RTLNumThreads, {}); in createTargetLoopWorkshareCall()
4284 Builder.CreateZExtOrTrunc(NumThreads, TripCountTy, "num.threads.cast")); in createTargetLoopWorkshareCall()
4290 Builder.CreateCall(RTLFn, RealArgs); in createTargetLoopWorkshareCall()
4299 IRBuilder<> &Builder = OMPIRBuilder->Builder; in workshareLoopTargetCallback() local
4313 Builder.restoreIP({Preheader, Preheader->end()}); in workshareLoopTargetCallback()
4315 Builder.CreateBr(CLI->getExit()); in workshareLoopTargetCallback()
4340 LoopBodyArg = Constant::getNullValue(Builder.getPtrTy()); in workshareLoopTargetCallback()
4375 Builder.restoreIP({CLI->getPreheader(), CLI->getPreheader()->begin()}); in applyWorkshareLoopTarget()
4379 AllocaInst *NewLoopCnt = Builder.CreateAlloca(CLI->getIndVarType(), 0, ""); in applyWorkshareLoopTarget()
4381 Builder.CreateLoad(CLI->getIndVarType(), NewLoopCnt); in applyWorkshareLoopTarget()
4560 Builder.SetCurrentDebugLocation(DL); in applyDynamicWorkshareLoop()
4573 Builder.SetInsertPoint(AllocaIP.getBlock()->getFirstNonPHIOrDbgOrAlloca()); in applyDynamicWorkshareLoop()
4575 Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter"); in applyDynamicWorkshareLoop()
4576 Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound"); in applyDynamicWorkshareLoop()
4577 Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound"); in applyDynamicWorkshareLoop()
4578 Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride"); in applyDynamicWorkshareLoop()
4585 Builder.SetInsertPoint(PreHeader->getTerminator()); in applyDynamicWorkshareLoop()
4587 Builder.CreateStore(One, PLowerBound); in applyDynamicWorkshareLoop()
4589 Builder.CreateStore(UpperBound, PUpperBound); in applyDynamicWorkshareLoop()
4590 Builder.CreateStore(One, PStride); in applyDynamicWorkshareLoop()
4610 Builder.CreateCall(DynamicInit, in applyDynamicWorkshareLoop()
4619 Builder.SetInsertPoint(OuterCond, OuterCond->getFirstInsertionPt()); in applyDynamicWorkshareLoop()
4621 Builder.CreateCall(DynamicNext, {SrcLoc, ThreadNum, PLastIter, in applyDynamicWorkshareLoop()
4624 Value *MoreWork = Builder.CreateCmp(CmpInst::ICMP_NE, Res, Zero32); in applyDynamicWorkshareLoop()
4626 Builder.CreateSub(Builder.CreateLoad(IVTy, PLowerBound), One, "lb"); in applyDynamicWorkshareLoop()
4627 Builder.CreateCondBr(MoreWork, Header, Exit); in applyDynamicWorkshareLoop()
4644 Builder.SetInsertPoint(Cond, Cond->getFirstInsertionPt()); in applyDynamicWorkshareLoop()
4645 UpperBound = Builder.CreateLoad(IVTy, PUpperBound, "ub"); in applyDynamicWorkshareLoop()
4646 Instruction *Comp = &*Builder.GetInsertPoint(); in applyDynamicWorkshareLoop()
4657 Builder.SetInsertPoint(&Latch->back()); in applyDynamicWorkshareLoop()
4659 Builder.CreateCall(DynamicFini, {SrcLoc, ThreadNum}); in applyDynamicWorkshareLoop()
4664 Builder.SetInsertPoint(&Exit->back()); in applyDynamicWorkshareLoop()
4665 createBarrier(LocationDescription(Builder.saveIP(), DL), in applyDynamicWorkshareLoop()
4729 Builder.SetCurrentDebugLocation(DL); in collapseLoops()
4731 Builder.restoreIP(ComputeIP); in collapseLoops()
4733 Builder.restoreIP(Outermost->getPreheaderIP()); in collapseLoops()
4748 CollapsedTripCount = Builder.CreateMul(CollapsedTripCount, OrigTripCount, in collapseLoops()
4761 Builder.restoreIP(Result->getBodyIP()); in collapseLoops()
4769 Value *NewIndVar = Builder.CreateURem(Leftover, OrigTripCount); in collapseLoops()
4772 Leftover = Builder.CreateUDiv(Leftover, OrigTripCount); in collapseLoops()
4882 Builder.SetCurrentDebugLocation(DL); in tileLoops()
4883 Builder.restoreIP(OutermostLoop->getPreheaderIP()); in tileLoops()
4890 Value *FloorTripCount = Builder.CreateUDiv(OrigTripCount, TileSize); in tileLoops()
4891 Value *FloorTripRem = Builder.CreateURem(OrigTripCount, TileSize); in tileLoops()
4901 Builder.CreateICmpNE(FloorTripRem, ConstantInt::get(IVType, 0)); in tileLoops()
4903 FloorTripOverflow = Builder.CreateZExt(FloorTripOverflow, IVType); in tileLoops()
4905 Builder.CreateAdd(FloorTripCount, FloorTripOverflow, in tileLoops()
4956 Builder.SetInsertPoint(Enter->getTerminator()); in tileLoops()
4963 Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCount[i]); in tileLoops()
4965 Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize); in tileLoops()
4998 Builder.restoreIP(Result.back()->getBodyIP()); in tileLoops()
5006 Builder.CreateMul(Size, FloorLoop->getIndVar(), {}, /*HasNUW=*/true); in tileLoops()
5008 Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true); in tileLoops()
5075 LLVMContext &Ctx = Builder.getContext(); in unrollLoopFull()
5082 LLVMContext &Ctx = Builder.getContext(); in unrollLoopHeuristic()
5126 Builder.SetInsertPoint(HeadOldTerm); in createIfVersion()
5128 Builder.CreateCondBr(IfCond, ThenBlock, /*ifFalse*/ ElseBlock); in createIfVersion()
5134 Builder.SetInsertPoint(ElseBlock); in createIfVersion()
5147 Builder.CreateBr(NewBlocks.front()); in createIfVersion()
5171 LLVMContext &Ctx = Builder.getContext(); in applySimd()
5189 InsertPointTy IP = Builder.saveIP(); in applySimd()
5190 Builder.SetInsertPoint(CanonicalLoop->getPreheader()->getTerminator()); in applySimd()
5194 Builder.CreateAlignmentAssumption(F->getDataLayout(), in applySimd()
5197 Builder.restoreIP(IP); in applySimd()
5516 llvm::Value *DidItLD = Builder.CreateLoad(Builder.getInt32Ty(), DidIt); in createCopyPrivate()
5521 Builder.CreateCall(Fn, Args); in createCopyPrivate()
5523 return Builder.saveIP(); in createCopyPrivate()
5538 DidIt = Builder.CreateAlloca(llvm::Type::getInt32Ty(Builder.getContext())); in createSingle()
5539 Builder.CreateStore(Builder.getInt32(0), DidIt); in createSingle()
5550 Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args); in createSingle()
5553 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args); in createSingle()
5562 Builder.CreateStore(Builder.getInt32(1), DidIt); in createSingle()
5580 createCopyPrivate(LocationDescription(Builder.saveIP(), Loc.DL), in createSingle()
5585 createBarrier(LocationDescription(Builder.saveIP(), Loc.DL), in createSingle()
5588 return Builder.saveIP(); in createSingle()
5615 Instruction *EntryCall = Builder.CreateCall(RTFn, EnterArgs); in createCritical()
5619 Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args); in createCritical()
5640 Builder.restoreIP(AllocaIP); in createOrderedDepend()
5641 AllocaInst *ArgsBase = Builder.CreateAlloca(ArrI64Ty, nullptr, Name); in createOrderedDepend()
5643 Builder.restoreIP(Loc.IP); in createOrderedDepend()
5647 Value *DependAddrGEPIter = Builder.CreateInBoundsGEP( in createOrderedDepend()
5648 ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(I)}); in createOrderedDepend()
5649 StoreInst *STInst = Builder.CreateStore(StoreValues[I], DependAddrGEPIter); in createOrderedDepend()
5653 Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP( in createOrderedDepend()
5654 ArrI64Ty, ArgsBase, {Builder.getInt64(0), Builder.getInt64(0)}); in createOrderedDepend()
5667 Builder.CreateCall(RTLFn, Args); in createOrderedDepend()
5669 return Builder.saveIP(); in createOrderedDepend()
5690 EntryCall = Builder.CreateCall(EntryRTLFn, Args); in createOrderedThreadsSimd()
5694 ExitCall = Builder.CreateCall(ExitRTLFn, Args); in createOrderedThreadsSimd()
5711 BasicBlock *EntryBB = Builder.GetInsertBlock(); in EmitOMPInlinedRegion()
5714 SplitPos = new UnreachableInst(Builder.getContext(), EntryBB); in EmitOMPInlinedRegion()
5719 Builder.SetInsertPoint(EntryBB->getTerminator()); in EmitOMPInlinedRegion()
5724 /* CodeGenIP */ Builder.saveIP()); in EmitOMPInlinedRegion()
5737 // block, and clear the builder's insertion point. in EmitOMPInlinedRegion()
5745 Builder.SetInsertPoint(InsertBB); in EmitOMPInlinedRegion()
5747 return Builder.saveIP(); in EmitOMPInlinedRegion()
5754 return Builder.saveIP(); in emitCommonDirectiveEntry()
5756 BasicBlock *EntryBB = Builder.GetInsertBlock(); in emitCommonDirectiveEntry()
5757 Value *CallBool = Builder.CreateIsNotNull(EntryCall); in emitCommonDirectiveEntry()
5759 auto *UI = new UnreachableInst(Builder.getContext(), ThenBB); in emitCommonDirectiveEntry()
5761 // Emit thenBB and set the Builder's insertion point there for in emitCommonDirectiveEntry()
5769 Builder.CreateCondBr(CallBool, ThenBB, ExitBB); in emitCommonDirectiveEntry()
5771 Builder.SetInsertPoint(UI); in emitCommonDirectiveEntry()
5772 Builder.Insert(EntryBBTI); in emitCommonDirectiveEntry()
5774 Builder.SetInsertPoint(ThenBB->getTerminator()); in emitCommonDirectiveEntry()
5784 Builder.restoreIP(FinIP); in emitCommonDirectiveExit()
5799 // set Builder IP for call creation in emitCommonDirectiveExit()
5800 Builder.SetInsertPoint(FiniBBTI); in emitCommonDirectiveExit()
5804 return Builder.saveIP(); in emitCommonDirectiveExit()
5808 Builder.Insert(ExitCall); in emitCommonDirectiveExit()
5820 IRBuilder<>::InsertPointGuard IPG(Builder); in createCopyinClauseBlocks()
5851 Builder.SetInsertPoint(OMP_Entry); in createCopyinClauseBlocks()
5852 Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy); in createCopyinClauseBlocks()
5853 Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy); in createCopyinClauseBlocks()
5854 Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr); in createCopyinClauseBlocks()
5855 Builder.CreateCondBr(cmp, CopyBegin, CopyEnd); in createCopyinClauseBlocks()
5857 Builder.SetInsertPoint(CopyBegin); in createCopyinClauseBlocks()
5859 Builder.SetInsertPoint(Builder.CreateBr(CopyEnd)); in createCopyinClauseBlocks()
5861 return Builder.saveIP(); in createCopyinClauseBlocks()
5867 IRBuilder<>::InsertPointGuard IPG(Builder); in createOMPAlloc()
5878 return Builder.CreateCall(Fn, Args, Name); in createOMPAlloc()
5884 IRBuilder<>::InsertPointGuard IPG(Builder); in createOMPFree()
5893 return Builder.CreateCall(Fn, Args, Name); in createOMPFree()
5900 IRBuilder<>::InsertPointGuard IPG(Builder); in createOMPInteropInit()
5922 return Builder.CreateCall(Fn, Args); in createOMPInteropInit()
5928 IRBuilder<>::InsertPointGuard IPG(Builder); in createOMPInteropDestroy()
5949 return Builder.CreateCall(Fn, Args); in createOMPInteropDestroy()
5957 IRBuilder<>::InsertPointGuard IPG(Builder); in createOMPInteropUse()
5977 return Builder.CreateCall(Fn, Args); in createOMPInteropUse()
5983 IRBuilder<>::InsertPointGuard IPG(Builder); in createCachedThreadPrivate()
5997 return Builder.CreateCall(Fn, Args); in createCachedThreadPrivate()
6016 Function *Kernel = Builder.GetInsertBlock()->getParent(); in createTargetInit()
6098 Builder.CreateCall(Fn, {KernelEnvironment, KernelLaunchEnvironment}); in createTargetInit()
6100 Value *ExecUserCode = Builder.CreateICmpEQ( in createTargetInit()
6110 auto *UI = Builder.CreateUnreachable(); in createTargetInit()
6116 Builder.SetInsertPoint(WorkerExitBB); in createTargetInit()
6117 Builder.CreateRetVoid(); in createTargetInit()
6120 Builder.SetInsertPoint(CheckBBTI); in createTargetInit()
6121 Builder.CreateCondBr(ExecUserCode, UI->getParent(), WorkerExitBB); in createTargetInit()
6140 Builder.CreateCall(Fn, {}); in createTargetDeinit()
6145 Function *Kernel = Builder.GetInsertBlock()->getParent(); in createTargetDeinit()
6283 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::WeakAnyLinkage, in createOutlinedFunctionID()
6284 Constant::getNullValue(Builder.getInt8Ty()), EntryFnIDName); in createOutlinedFunctionID()
6295 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::InternalLinkage, in createTargetRegionEntryAddr()
6296 Constant::getNullValue(Builder.getInt8Ty()), EntryFnName); in createTargetRegionEntryAddr()
6354 Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv)); in createTargetData()
6355 return Builder.saveIP(); in createTargetData()
6358 Builder.restoreIP(CodeGenIP); in createTargetData()
6365 MapInfo = &GenMapInfoCB(Builder.saveIP()); in createTargetData()
6366 emitOffloadingArrays(AllocaIP, Builder.saveIP(), *MapInfo, Info, in createTargetData()
6371 emitOffloadingArraysArgument(Builder, RTArgs, Info, in createTargetData()
6375 Value *PointerNum = Builder.getInt32(Info.NumberOfPtrs); in createTargetData()
6392 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(*MapperFunc), in createTargetData()
6398 Builder.CreateCall(BeginMapperFunc, OffloadingArgs); in createTargetData()
6403 Builder.CreateLoad(Builder.getPtrTy(), DeviceMap.second.first); in createTargetData()
6404 Builder.CreateStore(LI, DeviceMap.second.second); in createTargetData()
6411 Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::Priv)); in createTargetData()
6419 Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv)); in createTargetData()
6425 emitOffloadingArraysArgument(Builder, RTArgs, Info, !MapInfo->Names.empty(), in createTargetData()
6429 Value *PointerNum = Builder.getInt32(Info.NumberOfPtrs); in createTargetData()
6446 Builder.CreateCall(EndMapperFunc, OffloadingArgs); in createTargetData()
6457 BeginThenGen(AllocaIP, Builder.saveIP()); in createTargetData()
6462 Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv)); in createTargetData()
6467 EndThenGen(AllocaIP, Builder.saveIP()); in createTargetData()
6473 BeginThenGen(AllocaIP, Builder.saveIP()); in createTargetData()
6477 return Builder.saveIP(); in createTargetData()
6545 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, StringRef FuncName, in createOutlinedFunction() argument
6553 auto *Int8PtrTy = PointerType::getUnqual(Builder.getContext()); in createOutlinedFunction()
6561 : Type::getInt64Ty(Builder.getContext())); in createOutlinedFunction()
6567 auto FuncType = FunctionType::get(Builder.getVoidTy(), ParameterTypes, in createOutlinedFunction()
6570 Builder.GetInsertBlock()->getModule()); in createOutlinedFunction()
6573 auto OldInsertPoint = Builder.saveIP(); in createOutlinedFunction()
6576 BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func); in createOutlinedFunction()
6577 Builder.SetInsertPoint(EntryBB); in createOutlinedFunction()
6581 Builder.restoreIP(OMPBuilder.createTargetInit(Builder, /*IsSPMD*/ false)); in createOutlinedFunction()
6583 BasicBlock *UserCodeEntryBB = Builder.GetInsertBlock(); in createOutlinedFunction()
6592 Builder.restoreIP(CBFunc(Builder.saveIP(), Builder.saveIP())); in createOutlinedFunction()
6594 OMPBuilder.createTargetDeinit(Builder); in createOutlinedFunction()
6597 Builder.CreateRetVoid(); in createOutlinedFunction()
6600 Builder.SetInsertPoint(EntryBB->getFirstNonPHI()); in createOutlinedFunction()
6601 auto AllocaIP = Builder.saveIP(); in createOutlinedFunction()
6603 Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrDbg()); in createOutlinedFunction()
6647 Builder.restoreIP( in createOutlinedFunction()
6648 ArgAccessorFuncCB(Arg, Input, InputCopy, AllocaIP, Builder.saveIP())); in createOutlinedFunction()
6681 Builder.restoreIP(OldInsertPoint); in createOutlinedFunction()
6692 IRBuilderBase &Builder, in emitTargetTaskProxyFunction() argument
6724 FunctionType::get(Builder.getVoidTy(), {ThreadIDTy, TaskPtrTy}, in emitTargetTaskProxyFunction()
6728 Builder.GetInsertBlock()->getModule()); in emitTargetTaskProxyFunction()
6733 BasicBlock::Create(Builder.getContext(), "entry", ProxyFn); in emitTargetTaskProxyFunction()
6734 Builder.SetInsertPoint(EntryBB); in emitTargetTaskProxyFunction()
6752 Builder.CreateAlloca(ArgStructType, nullptr, "structArg"); in emitTargetTaskProxyFunction()
6756 Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType)); in emitTargetTaskProxyFunction()
6758 Value *Shareds = Builder.CreateStructGEP(TaskTy, TaskT, 0); in emitTargetTaskProxyFunction()
6760 Builder.CreateLoad(PointerType::getUnqual(Ctx), Shareds); in emitTargetTaskProxyFunction()
6762 Builder.CreateMemCpy( in emitTargetTaskProxyFunction()
6766 Builder.CreateCall(KernelLaunchFunction, {ThreadId, NewArgStructAlloca}); in emitTargetTaskProxyFunction()
6768 Builder.CreateRetVoid(); in emitTargetTaskProxyFunction()
6772 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, in emitTargetOutlinedFunction() argument
6779 [&OMPBuilder, &Builder, &Inputs, &CBFunc, in emitTargetOutlinedFunction()
6781 return createOutlinedFunction(OMPBuilder, Builder, EntryFnName, Inputs, in emitTargetOutlinedFunction()
6883 splitBB(Builder, /*CreateBranch=*/true, "target.task.body"); in emitTargetTask()
6885 splitBB(Builder, /*CreateBranch=*/true, "target.task.alloca"); in emitTargetTask()
6898 Builder, AllocaIP, ToBeDeleted, TargetTaskAllocaIP, "global.tid", false)); in emitTargetTask()
6900 Builder.restoreIP(TargetTaskBodyIP); in emitTargetTask()
6908 Builder.restoreIP(emitKernelLaunch(Builder, OutlinedFn, OutlinedFnID, in emitTargetTask()
6912 OI.ExitBB = Builder.saveIP().getBlock(); in emitTargetTask()
6921 Function *ProxyFn = emitTargetTaskProxyFunction(*this, Builder, StaleCI); in emitTargetTask()
6926 Builder.SetInsertPoint(StaleCI); in emitTargetTask()
6931 getOrCreateSrcLocStr(LocationDescription(Builder), SrcLocStrSize); in emitTargetTask()
6947 Builder.getInt64(M.getDataLayout().getTypeStoreSize(Task)); in emitTargetTask()
6952 Value *SharedsSize = Builder.getInt64(0); in emitTargetTask()
6963 Builder.getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType)); in emitTargetTask()
6972 Value *Flags = Builder.getInt32(0); in emitTargetTask()
6977 CallInst *TaskData = Builder.CreateCall( in emitTargetTask()
6985 Value *TaskShareds = Builder.CreateLoad(VoidPtr, TaskData); in emitTargetTask()
6986 Builder.CreateMemCpy(TaskShareds, Alignment, Shareds, Alignment, in emitTargetTask()
7004 Builder.CreateCall( in emitTargetTask()
7007 /*ndeps=*/Builder.getInt32(Dependencies.size()), in emitTargetTask()
7009 /*ndeps_noalias=*/ConstantInt::get(Builder.getInt32Ty(), 0), in emitTargetTask()
7018 Builder.CreateCall(TaskBeginFn, {Ident, ThreadID, TaskData}); in emitTargetTask()
7021 CI = Builder.CreateCall(ProxyFn, {ThreadID, TaskData}); in emitTargetTask()
7023 CI = Builder.CreateCall(ProxyFn, {ThreadID}); in emitTargetTask()
7025 Builder.CreateCall(TaskCompleteFn, {Ident, ThreadID, TaskData}); in emitTargetTask()
7032 Builder.CreateCall( in emitTargetTask()
7034 {Ident, ThreadID, TaskData, Builder.getInt32(Dependencies.size()), in emitTargetTask()
7035 DepArray, ConstantInt::get(Builder.getInt32Ty(), 0), in emitTargetTask()
7040 Builder.CreateCall(TaskFn, {Ident, ThreadID, TaskData}); in emitTargetTask()
7050 << *(Builder.GetInsertBlock()) << "\n"); in emitTargetTask()
7052 << *(Builder.GetInsertBlock()->getParent()->getParent()) in emitTargetTask()
7054 return Builder.saveIP(); in emitTargetTask()
7057 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, in emitTargetCall() argument
7068 OpenMPIRBuilder::MapInfosTy &MapInfo = GenMapInfoCB(Builder.saveIP());
7069 OMPBuilder.emitOffloadingArrays(AllocaIP, Builder.saveIP(), MapInfo, Info,
7073 OMPBuilder.emitOffloadingArraysArgument(Builder, RTArgs, Info,
7079 Builder.restoreIP(IP); in __anon46338d6b3702()
7080 Builder.CreateCall(OutlinedFn, Args); in __anon46338d6b3702()
7081 return Builder.saveIP(); in __anon46338d6b3702()
7086 Value *DeviceID = Builder.getInt64(OMP_DEVICEID_UNDEF);
7087 Value *NumTeamsVal = Builder.getInt32(NumTeams);
7088 Value *NumThreadsVal = Builder.getInt32(NumThreads);
7094 Value *NumIterations = Builder.getInt64(0);
7096 Value *DynCGGroupMem = Builder.getInt32(0);
7109 Builder.restoreIP(OMPBuilder.emitTargetTask(
7113 Builder.restoreIP(OMPBuilder.emitKernelLaunch(
7114 Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, KArgs,
7130 Builder.restoreIP(CodeGenIP); in createTarget()
7137 emitTargetOutlinedFunction(*this, Builder, EntryInfo, OutlinedFn, in createTarget()
7144 emitTargetCall(*this, Builder, AllocaIP, OutlinedFn, OutlinedFnID, NumTeams, in createTarget()
7146 return Builder.saveIP(); in createTarget()
7204 LLVMContext &Ctx = Builder.getContext(); in getSizeInBytes()
7208 Builder.CreateGEP(BasePtr->getType(), Null, Builder.getInt32(1)); in getSizeInBytes()
7209 Value *SizePtrToInt = Builder.CreatePtrToInt(SizeGep, Type::getInt64Ty(Ctx)); in getSizeInBytes()
7235 Builder.restoreIP(AllocaIP); in createMapperAllocas()
7236 AllocaInst *ArgsBase = Builder.CreateAlloca( in createMapperAllocas()
7238 AllocaInst *Args = Builder.CreateAlloca(ArrI8PtrTy, /* ArraySize = */ nullptr, in createMapperAllocas()
7240 AllocaInst *ArgSizes = Builder.CreateAlloca( in createMapperAllocas()
7242 Builder.restoreIP(Loc.IP); in createMapperAllocas()
7259 Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.ArgsBase, in emitMapperCall()
7260 {Builder.getInt32(0), Builder.getInt32(0)}); in emitMapperCall()
7262 Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.Args, in emitMapperCall()
7263 {Builder.getInt32(0), Builder.getInt32(0)}); in emitMapperCall()
7265 Builder.CreateInBoundsGEP(ArrI64Ty, MapperAllocas.ArgSizes, in emitMapperCall()
7266 {Builder.getInt32(0), Builder.getInt32(0)}); in emitMapperCall()
7269 Builder.CreateCall(MapperFunc, in emitMapperCall()
7270 {SrcLocInfo, Builder.getInt64(DeviceID), in emitMapperCall()
7271 Builder.getInt32(NumOperands), ArgsBaseGEP, ArgsGEP, in emitMapperCall()
7275 void OpenMPIRBuilder::emitOffloadingArraysArgument(IRBuilderBase &Builder, in emitOffloadingArraysArgument() argument
7298 RTArgs.BasePointersArray = Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
7302 RTArgs.PointersArray = Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
7306 RTArgs.SizesArray = Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
7309 RTArgs.MapTypesArray = Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
7321 RTArgs.MapNamesArray = Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
7331 Builder.CreatePointerCast(Info.RTArgs.MappersArray, VoidPtrPtrTy); in emitOffloadingArraysArgument()
7349 Type *Int64Ty = Builder.getInt64Ty(); in emitNonContiguousDescriptor()
7363 Builder.restoreIP(AllocaIP); in emitNonContiguousDescriptor()
7366 Builder.CreateAlloca(ArrayTy, /* ArraySize = */ nullptr, "dims"); in emitNonContiguousDescriptor()
7367 Builder.restoreIP(CodeGenIP); in emitNonContiguousDescriptor()
7370 Value *DimsLVal = Builder.CreateInBoundsGEP( in emitNonContiguousDescriptor()
7372 {Builder.getInt64(0), Builder.getInt64(II)}); in emitNonContiguousDescriptor()
7374 Value *OffsetLVal = Builder.CreateStructGEP(DimTy, DimsLVal, OffsetFD); in emitNonContiguousDescriptor()
7375 Builder.CreateAlignedStore( in emitNonContiguousDescriptor()
7379 Value *CountLVal = Builder.CreateStructGEP(DimTy, DimsLVal, CountFD); in emitNonContiguousDescriptor()
7380 Builder.CreateAlignedStore( in emitNonContiguousDescriptor()
7384 Value *StrideLVal = Builder.CreateStructGEP(DimTy, DimsLVal, StrideFD); in emitNonContiguousDescriptor()
7385 Builder.CreateAlignedStore( in emitNonContiguousDescriptor()
7390 Builder.restoreIP(CodeGenIP); in emitNonContiguousDescriptor()
7391 Value *DAddr = Builder.CreatePointerBitCastOrAddrSpaceCast( in emitNonContiguousDescriptor()
7392 DimsAddr, Builder.getPtrTy()); in emitNonContiguousDescriptor()
7393 Value *P = Builder.CreateConstInBoundsGEP2_32( in emitNonContiguousDescriptor()
7394 ArrayType::get(Builder.getPtrTy(), Info.NumberOfPtrs), in emitNonContiguousDescriptor()
7396 Builder.CreateAlignedStore( in emitNonContiguousDescriptor()
7397 DAddr, P, M.getDataLayout().getPrefTypeAlign(Builder.getPtrTy())); in emitNonContiguousDescriptor()
7415 Builder.restoreIP(AllocaIP); in emitOffloadingArrays()
7419 ArrayType::get(Builder.getPtrTy(), Info.NumberOfPtrs); in emitOffloadingArrays()
7421 Info.RTArgs.BasePointersArray = Builder.CreateAlloca( in emitOffloadingArrays()
7424 Info.RTArgs.PointersArray = Builder.CreateAlloca( in emitOffloadingArrays()
7426 AllocaInst *MappersArray = Builder.CreateAlloca( in emitOffloadingArrays()
7433 Type *Int64Ty = Builder.getInt64Ty(); in emitOffloadingArrays()
7456 Info.RTArgs.SizesArray = Builder.CreateAlloca( in emitOffloadingArrays()
7458 Builder.restoreIP(CodeGenIP); in emitOffloadingArrays()
7474 AllocaInst *Buffer = Builder.CreateAlloca( in emitOffloadingArrays()
7477 Builder.restoreIP(CodeGenIP); in emitOffloadingArrays()
7478 Builder.CreateMemCpy( in emitOffloadingArrays()
7481 Builder.getIntN( in emitOffloadingArrays()
7487 Builder.restoreIP(CodeGenIP); in emitOffloadingArrays()
7509 Constant::getNullValue(PointerType::getUnqual(Builder.getContext())); in emitOffloadingArrays()
7530 PointerType *PtrTy = Builder.getPtrTy(); in emitOffloadingArrays()
7533 Value *BP = Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
7536 Builder.CreateAlignedStore(BPVal, BP, in emitOffloadingArrays()
7541 CodeGenIP = Builder.saveIP(); in emitOffloadingArrays()
7542 Builder.restoreIP(AllocaIP); in emitOffloadingArrays()
7543 Info.DevicePtrInfoMap[BPVal] = {BP, Builder.CreateAlloca(PtrTy)}; in emitOffloadingArrays()
7544 Builder.restoreIP(CodeGenIP); in emitOffloadingArrays()
7555 Value *P = Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
7559 Builder.CreateAlignedStore(PVal, P, in emitOffloadingArrays()
7563 Value *S = Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
7567 Builder.CreateAlignedStore(Builder.CreateIntCast(CombinedInfo.Sizes[I], in emitOffloadingArrays()
7577 MFunc = Builder.CreatePointerCast(CustomMFunc, PtrTy); in emitOffloadingArrays()
7578 Value *MAddr = Builder.CreateInBoundsGEP( in emitOffloadingArrays()
7580 {Builder.getIntN(IndexSize, 0), Builder.getIntN(IndexSize, I)}); in emitOffloadingArrays()
7581 Builder.CreateAlignedStore( in emitOffloadingArrays()
7592 BasicBlock *CurBB = Builder.GetInsertBlock(); in emitBranch()
7599 Builder.CreateBr(Target); in emitBranch()
7602 Builder.ClearInsertionPoint(); in emitBranch()
7607 BasicBlock *CurBB = Builder.GetInsertBlock(); in emitBlock()
7623 Builder.SetInsertPoint(BB); in emitBlock()
7634 ThenGen(AllocaIP, Builder.saveIP()); in emitIfClause()
7636 ElseGen(AllocaIP, Builder.saveIP()); in emitIfClause()
7640 Function *CurFn = Builder.GetInsertBlock()->getParent(); in emitIfClause()
7647 Builder.CreateCondBr(Cond, ThenBlock, ElseBlock); in emitIfClause()
7650 ThenGen(AllocaIP, Builder.saveIP()); in emitIfClause()
7655 ElseGen(AllocaIP, Builder.saveIP()); in emitIfClause()
7741 Builder.CreateLoad(XElemTy, X.Var, X.IsVolatile, "omp.atomic.read"); in createAtomicRead()
7749 Builder.CreateLoad(IntCastTy, X.Var, X.IsVolatile, "omp.atomic.load"); in createAtomicRead()
7752 XRead = Builder.CreateBitCast(XLoad, XElemTy, "atomic.flt.cast"); in createAtomicRead()
7754 XRead = Builder.CreateIntToPtr(XLoad, XElemTy, "atomic.ptr.cast"); in createAtomicRead()
7758 Builder.CreateStore(XRead, V.Var, V.IsVolatile); in createAtomicRead()
7759 return Builder.saveIP(); in createAtomicRead()
7777 StoreInst *XSt = Builder.CreateStore(Expr, X.Var, X.IsVolatile); in createAtomicWrite()
7784 Builder.CreateBitCast(Expr, IntCastTy, "atomic.src.int.cast"); in createAtomicWrite()
7785 StoreInst *XSt = Builder.CreateStore(ExprCast, X.Var, X.IsVolatile); in createAtomicWrite()
7790 return Builder.saveIP(); in createAtomicWrite()
7817 return Builder.saveIP(); in createAtomicUpdate()
7825 return Builder.CreateAdd(Src1, Src2); in emitRMWOpAsInstruction()
7827 return Builder.CreateSub(Src1, Src2); in emitRMWOpAsInstruction()
7829 return Builder.CreateAnd(Src1, Src2); in emitRMWOpAsInstruction()
7831 return Builder.CreateNeg(Builder.CreateAnd(Src1, Src2)); in emitRMWOpAsInstruction()
7833 return Builder.CreateOr(Src1, Src2); in emitRMWOpAsInstruction()
7835 return Builder.CreateXor(Src1, Src2); in emitRMWOpAsInstruction()
7879 Res.first = Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO); in emitAtomicUpdate()
7891 Builder.CreateLoad(IntCastTy, X, X->getName() + ".atomic.load"); in emitAtomicUpdate()
7898 BasicBlock *CurBB = Builder.GetInsertBlock(); in emitAtomicUpdate()
7900 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable(); in emitAtomicUpdate()
7906 Builder.restoreIP(AllocaIP); in emitAtomicUpdate()
7907 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(XElemTy); in emitAtomicUpdate()
7909 Builder.SetInsertPoint(ContBB); in emitAtomicUpdate()
7910 llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(), 2); in emitAtomicUpdate()
7916 OldExprVal = Builder.CreateBitCast(PHI, XElemTy, in emitAtomicUpdate()
7919 OldExprVal = Builder.CreateIntToPtr(PHI, XElemTy, in emitAtomicUpdate()
7924 Value *Upd = UpdateOp(OldExprVal, Builder); in emitAtomicUpdate()
7925 Builder.CreateStore(Upd, NewAtomicAddr); in emitAtomicUpdate()
7926 LoadInst *DesiredVal = Builder.CreateLoad(IntCastTy, NewAtomicAddr); in emitAtomicUpdate()
7929 AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg( in emitAtomicUpdate()
7932 Value *PreviousVal = Builder.CreateExtractValue(Result, /*Idxs=*/0); in emitAtomicUpdate()
7933 Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1); in emitAtomicUpdate()
7934 PHI->addIncoming(PreviousVal, Builder.GetInsertBlock()); in emitAtomicUpdate()
7935 Builder.CreateCondBr(SuccessFailureVal, ExitBB, ContBB); in emitAtomicUpdate()
7944 Builder.SetInsertPoint(ExitBB); in emitAtomicUpdate()
7946 Builder.SetInsertPoint(ExitTI); in emitAtomicUpdate()
7981 Builder.CreateStore(CapturedVal, V.Var, V.IsVolatile); in createAtomicCapture()
7984 return Builder.saveIP(); in createAtomicCapture()
8022 Value *EBCast = Builder.CreateBitCast(E, IntCastTy); in createAtomicCompare()
8023 Value *DBCast = Builder.CreateBitCast(D, IntCastTy); in createAtomicCompare()
8024 Result = Builder.CreateAtomicCmpXchg(X.Var, EBCast, DBCast, MaybeAlign(), in createAtomicCompare()
8028 Builder.CreateAtomicCmpXchg(X.Var, E, D, MaybeAlign(), AO, Failure); in createAtomicCompare()
8032 Value *OldValue = Builder.CreateExtractValue(Result, /*Idxs=*/0); in createAtomicCompare()
8034 OldValue = Builder.CreateBitCast(OldValue, X.ElemTy); in createAtomicCompare()
8038 Builder.CreateStore(OldValue, V.Var, V.IsVolatile); in createAtomicCompare()
8040 Value *SuccessOrFail = Builder.CreateExtractValue(Result, /*Idxs=*/1); in createAtomicCompare()
8051 BasicBlock *CurBB = Builder.GetInsertBlock(); in createAtomicCompare()
8053 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable(); in createAtomicCompare()
8061 Builder.CreateCondBr(SuccessOrFail, ExitBB, ContBB); in createAtomicCompare()
8063 Builder.SetInsertPoint(ContBB); in createAtomicCompare()
8064 Builder.CreateStore(OldValue, V.Var); in createAtomicCompare()
8065 Builder.CreateBr(ExitBB); in createAtomicCompare()
8070 Builder.SetInsertPoint(ExitBB); in createAtomicCompare()
8072 Builder.SetInsertPoint(ExitTI); in createAtomicCompare()
8076 Builder.CreateSelect(SuccessOrFail, E, OldValue); in createAtomicCompare()
8077 Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile); in createAtomicCompare()
8087 Value *SuccessFailureVal = Builder.CreateExtractValue(Result, /*Idxs=*/1); in createAtomicCompare()
8089 ? Builder.CreateSExt(SuccessFailureVal, R.ElemTy) in createAtomicCompare()
8090 : Builder.CreateZExt(SuccessFailureVal, R.ElemTy); in createAtomicCompare()
8091 Builder.CreateStore(ResultCast, R.Var, R.IsVolatile); in createAtomicCompare()
8134 Builder.CreateAtomicRMW(NewOp, X.Var, E, MaybeAlign(), AO); in createAtomicCompare()
8163 Value *NonAtomicCmp = Builder.CreateCmp(Pred, OldValue, E); in createAtomicCompare()
8164 CapturedValue = Builder.CreateSelect(NonAtomicCmp, E, OldValue); in createAtomicCompare()
8166 Builder.CreateStore(CapturedValue, V.Var, V.IsVolatile); in createAtomicCompare()
8172 return Builder.saveIP(); in createAtomicCompare()
8186 Function *CurrentFunction = Builder.GetInsertBlock()->getParent(); in createTeams()
8190 if (&OuterAllocaBB == Builder.GetInsertBlock()) { in createTeams()
8191 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "teams.entry"); in createTeams()
8192 Builder.SetInsertPoint(BodyBB, BodyBB->begin()); in createTeams()
8212 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, "teams.exit"); in createTeams()
8213 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "teams.body"); in createTeams()
8215 splitBB(Builder, /*CreateBranch=*/true, "teams.alloca"); in createTeams()
8226 NumTeamsUpper = Builder.getInt32(0); in createTeams()
8237 IfExpr = Builder.CreateICmpNE(IfExpr, in createTeams()
8239 NumTeamsUpper = Builder.CreateSelect( in createTeams()
8240 IfExpr, NumTeamsUpper, Builder.getInt32(1), "numTeamsUpper"); in createTeams()
8243 NumTeamsLower = Builder.CreateSelect( in createTeams()
8244 IfExpr, NumTeamsLower, Builder.getInt32(1), "numTeamsLower"); in createTeams()
8248 ThreadLimit = Builder.getInt32(0); in createTeams()
8251 Builder.CreateCall( in createTeams()
8269 Builder, OuterAllocaIP, ToBeDeleted, AllocaIP, "gid", true)); in createTeams()
8271 Builder, OuterAllocaIP, ToBeDeleted, AllocaIP, "tid", true)); in createTeams()
8296 Builder.SetInsertPoint(StaleCI); in createTeams()
8298 Ident, Builder.getInt32(StaleCI->arg_size() - 2), &OutlinedFn}; in createTeams()
8301 Builder.CreateCall(getOrCreateRuntimeFunctionPtr( in createTeams()
8315 Builder.SetInsertPoint(ExitBB, ExitBB->begin()); in createTeams()
8317 return Builder.saveIP(); in createTeams()
8423 return ConstantAsMetadata::get(ConstantInt::get(Builder.getInt32Ty(), V)); in createOffloadEntriesAndInfoMetadata()