Lines Matching refs:CGF
47 void Enter(CodeGenFunction &CGF) override { in Enter() argument
48 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
50 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
51 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
52 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
54 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
55 CGF.EmitBlock(ThenBlock); in Enter()
58 void Done(CodeGenFunction &CGF) { in Done() argument
60 CGF.EmitBranch(ContBlock); in Done()
61 CGF.EmitBlock(ContBlock, true); in Done()
63 void Exit(CodeGenFunction &CGF) override { in Exit() argument
64 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
185 CodeGenFunction &CGF; member in __anon61bd5b670111::CheckVarsEscapingDeclContext
206 if (auto *CSI = CGF.CapturedStmtInfo) { in markAsEscaped()
232 if ((!CGF.CapturedStmtInfo || in markAsEscaped()
233 (IsForCombinedParallelRegion && CGF.CapturedStmtInfo)) && in markAsEscaped()
311 unsigned WarpSize = CGF.getTarget().getGridValue().GV_Warp_Size; in buildRecordForGlobalizedVars()
317 CGF.getContext(), EscapedDeclsForParallel, EscapedDeclsForTeams, in buildRecordForGlobalizedVars()
322 CheckVarsEscapingDeclContext(CodeGenFunction &CGF, in CheckVarsEscapingDeclContext() argument
324 : CGF(CGF), EscapedDecls(TeamsReductions.begin(), TeamsReductions.end()) { in CheckVarsEscapingDeclContext()
726 void Enter(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
727 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
728 RT.emitKernelInit(D, CGF, EST, /* IsSPMD */ false); in emitNonSPMDKernel()
730 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitNonSPMDKernel()
732 void Exit(CodeGenFunction &CGF) override { in emitNonSPMDKernel() argument
733 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitNonSPMDKernel()
734 RT.clearLocThreadIdInsertPt(CGF); in emitNonSPMDKernel()
735 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ false); in emitNonSPMDKernel()
746 CodeGenFunction &CGF, in emitKernelInit() argument
750 computeMinAndMaxThreadsAndTeams(D, CGF, MinThreadsVal, MaxThreadsVal, in emitKernelInit()
753 CGBuilderTy &Bld = CGF.Builder; in emitKernelInit()
757 emitGenericVarsProlog(CGF, EST.Loc); in emitKernelInit()
760 void CGOpenMPRuntimeGPU::emitKernelDeinit(CodeGenFunction &CGF, in emitKernelDeinit() argument
764 emitGenericVarsEpilog(CGF); in emitKernelDeinit()
790 CGBuilderTy &Bld = CGF.Builder; in emitKernelDeinit()
821 void Enter(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
826 RT.emitKernelInit(D, CGF, EST, /* IsSPMD */ true); in emitSPMDKernel()
828 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true); in emitSPMDKernel()
830 void Exit(CodeGenFunction &CGF) override { in emitSPMDKernel() argument
835 RT.clearLocThreadIdInsertPt(CGF); in emitSPMDKernel()
836 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ true); in emitSPMDKernel()
896 void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
902 void CGOpenMPRuntimeGPU::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
908 void CGOpenMPRuntimeGPU::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
914 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitParallelOutlinedFunction() argument
922 CGF, D, ThreadIDVar, InnermostKind, CodeGen)); in emitParallelOutlinedFunction()
972 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitTeamsOutlinedFunction() argument
1012 void Enter(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1014 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitTeamsOutlinedFunction()
1016 auto I = Rt.FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitTeamsOutlinedFunction()
1026 Rt.emitGenericVarsProlog(CGF, Loc); in emitTeamsOutlinedFunction()
1028 void Exit(CodeGenFunction &CGF) override { in emitTeamsOutlinedFunction() argument
1029 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitTeamsOutlinedFunction()
1030 .emitGenericVarsEpilog(CGF); in emitTeamsOutlinedFunction()
1035 CGF, D, ThreadIDVar, InnermostKind, CodeGen); in emitTeamsOutlinedFunction()
1040 void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF, in emitGenericVarsProlog() argument
1045 CGBuilderTy &Bld = CGF.Builder; in emitGenericVarsProlog()
1047 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsProlog()
1060 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitGenericVarsProlog()
1061 ParValue = CGF.EmitLoadOfScalar(ParLVal, Loc); in emitGenericVarsProlog()
1065 llvm::Value *AllocArgs[] = {CGF.getTypeSize(VD->getType())}; in emitGenericVarsProlog()
1067 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsProlog()
1076 llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo(); in emitGenericVarsProlog()
1080 CGF.MakeNaturalAlignPointeeRawAddrLValue(CastedVoidPtr, VarTy); in emitGenericVarsProlog()
1086 CGF.EmitStoreOfScalar(ParValue, VarAddr); in emitGenericVarsProlog()
1087 I->getSecond().MappedParams->setVarAddr(CGF, VD, VarAddr.getAddress()); in emitGenericVarsProlog()
1089 if (auto *DI = CGF.getDebugInfo()) in emitGenericVarsProlog()
1096 getKmpcAllocShared(CGF, VD); in emitGenericVarsProlog()
1098 LValue Base = CGF.MakeAddrLValue(AddrSizePair.first, VD->getType(), in emitGenericVarsProlog()
1101 I->getSecond().MappedParams->setVarAddr(CGF, VD, Base.getAddress()); in emitGenericVarsProlog()
1103 I->getSecond().MappedParams->apply(CGF); in emitGenericVarsProlog()
1106 bool CGOpenMPRuntimeGPU::isDelayedVariableLengthDecl(CodeGenFunction &CGF, in isDelayedVariableLengthDecl() argument
1108 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in isDelayedVariableLengthDecl()
1117 CGOpenMPRuntimeGPU::getKmpcAllocShared(CodeGenFunction &CGF, in getKmpcAllocShared() argument
1119 CGBuilderTy &Bld = CGF.Builder; in getKmpcAllocShared()
1122 llvm::Value *Size = CGF.getTypeSize(VD->getType()); in getKmpcAllocShared()
1125 Size, llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity() - 1)); in getKmpcAllocShared()
1127 llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity()); in getKmpcAllocShared()
1134 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in getKmpcAllocShared()
1144 CodeGenFunction &CGF, in getKmpcFreeShared() argument
1147 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in getKmpcFreeShared()
1152 void CGOpenMPRuntimeGPU::emitGenericVarsEpilog(CodeGenFunction &CGF) { in emitGenericVarsEpilog() argument
1156 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in emitGenericVarsEpilog()
1162 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsEpilog()
1169 I->getSecond().MappedParams->restore(CGF); in emitGenericVarsEpilog()
1172 CGF.getTypeSize(VD->getType())}; in emitGenericVarsEpilog()
1173 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitGenericVarsEpilog()
1180 void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
1185 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
1190 RawAddress ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in emitTeamsCall()
1192 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); in emitTeamsCall()
1199 OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).emitRawPointer(CGF)); in emitTeamsCall()
1202 emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); in emitTeamsCall()
1205 void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF, in emitParallelCall() argument
1211 if (!CGF.HaveInsertPoint()) in emitParallelCall()
1215 NumThreads](CodeGenFunction &CGF, in emitParallelCall()
1217 CGBuilderTy &Bld = CGF.Builder; in emitParallelCall()
1228 CodeGenFunction::OMPPrivateScope PrivateArgScope(CGF); in emitParallelCall()
1230 Address CapturedVarsAddrs = CGF.CreateDefaultAlignTempAlloca( in emitParallelCall()
1236 ASTContext &Ctx = CGF.getContext(); in emitParallelCall()
1242 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy); in emitParallelCall()
1244 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy); in emitParallelCall()
1245 CGF.EmitStoreOfScalar(PtrV, Dst, /*Volatile=*/false, in emitParallelCall()
1253 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty, in emitParallelCall()
1256 IfCondVal = llvm::ConstantInt::get(CGF.Int32Ty, 1); in emitParallelCall()
1259 NumThreadsVal = llvm::ConstantInt::get(CGF.Int32Ty, -1); in emitParallelCall()
1261 NumThreadsVal = Bld.CreateZExtOrTrunc(NumThreadsVal, CGF.Int32Ty), in emitParallelCall()
1264 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
1267 getThreadID(CGF, Loc), in emitParallelCall()
1270 llvm::ConstantInt::get(CGF.Int32Ty, -1), in emitParallelCall()
1273 Bld.CreateBitOrPointerCast(CapturedVarsAddrs.emitRawPointer(CGF), in emitParallelCall()
1274 CGF.VoidPtrPtrTy), in emitParallelCall()
1276 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitParallelCall()
1282 RCG(CGF); in emitParallelCall()
1285 void CGOpenMPRuntimeGPU::syncCTAThreads(CodeGenFunction &CGF) { in syncCTAThreads() argument
1287 if (!CGF.HaveInsertPoint()) in syncCTAThreads()
1294 llvm::ConstantInt::get(CGF.Int32Ty, /*V=*/0, /*isSigned=*/true)}; in syncCTAThreads()
1295 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in syncCTAThreads()
1300 void CGOpenMPRuntimeGPU::emitBarrierCall(CodeGenFunction &CGF, in emitBarrierCall() argument
1305 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
1309 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
1310 getThreadID(CGF, Loc)}; in emitBarrierCall()
1312 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitBarrierCall()
1318 CodeGenFunction &CGF, StringRef CriticalName, in emitCriticalRegion() argument
1321 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.critical.loop"); in emitCriticalRegion()
1322 llvm::BasicBlock *TestBB = CGF.createBasicBlock("omp.critical.test"); in emitCriticalRegion()
1323 llvm::BasicBlock *SyncBB = CGF.createBasicBlock("omp.critical.sync"); in emitCriticalRegion()
1324 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.critical.body"); in emitCriticalRegion()
1325 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.critical.exit"); in emitCriticalRegion()
1327 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in emitCriticalRegion()
1330 llvm::Value *Mask = CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
1333 llvm::Value *ThreadID = RT.getGPUThreadID(CGF); in emitCriticalRegion()
1336 llvm::Value *TeamWidth = RT.getGPUNumThreads(CGF); in emitCriticalRegion()
1340 CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/0); in emitCriticalRegion()
1341 Address Counter = CGF.CreateMemTemp(Int32Ty, "critical_counter"); in emitCriticalRegion()
1342 LValue CounterLVal = CGF.MakeAddrLValue(Counter, Int32Ty); in emitCriticalRegion()
1343 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty), CounterLVal, in emitCriticalRegion()
1347 CGF.EmitBlock(LoopBB); in emitCriticalRegion()
1348 llvm::Value *CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
1349 llvm::Value *CmpLoopBound = CGF.Builder.CreateICmpSLT(CounterVal, TeamWidth); in emitCriticalRegion()
1350 CGF.Builder.CreateCondBr(CmpLoopBound, TestBB, ExitBB); in emitCriticalRegion()
1354 CGF.EmitBlock(TestBB); in emitCriticalRegion()
1355 CounterVal = CGF.EmitLoadOfScalar(CounterLVal, Loc); in emitCriticalRegion()
1357 CGF.Builder.CreateICmpEQ(ThreadID, CounterVal); in emitCriticalRegion()
1358 CGF.Builder.CreateCondBr(CmpThreadToCounter, BodyBB, SyncBB); in emitCriticalRegion()
1361 CGF.EmitBlock(BodyBB); in emitCriticalRegion()
1364 CGOpenMPRuntime::emitCriticalRegion(CGF, CriticalName, CriticalOpGen, Loc, in emitCriticalRegion()
1371 CGF.EmitBlock(SyncBB); in emitCriticalRegion()
1373 (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in emitCriticalRegion()
1378 CGF.Builder.CreateNSWAdd(CounterVal, CGF.Builder.getInt32(1)); in emitCriticalRegion()
1379 CGF.EmitStoreOfScalar(IncCounterVal, CounterLVal); in emitCriticalRegion()
1380 CGF.EmitBranch(LoopBB); in emitCriticalRegion()
1383 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in emitCriticalRegion()
1387 static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val, in castValueToType() argument
1390 assert(!CGF.getContext().getTypeSizeInChars(CastTy).isZero() && in castValueToType()
1392 assert(!CGF.getContext().getTypeSizeInChars(ValTy).isZero() && in castValueToType()
1394 llvm::Type *LLVMCastTy = CGF.ConvertTypeForMem(CastTy); in castValueToType()
1397 if (CGF.getContext().getTypeSizeInChars(ValTy) == in castValueToType()
1398 CGF.getContext().getTypeSizeInChars(CastTy)) in castValueToType()
1399 return CGF.Builder.CreateBitCast(Val, LLVMCastTy); in castValueToType()
1401 return CGF.Builder.CreateIntCast(Val, LLVMCastTy, in castValueToType()
1403 Address CastItem = CGF.CreateMemTemp(CastTy); in castValueToType()
1405 CGF.EmitStoreOfScalar(Val, ValCastItem, /*Volatile=*/false, ValTy, in castValueToType()
1408 return CGF.EmitLoadOfScalar(CastItem, /*Volatile=*/false, CastTy, Loc, in castValueToType()
1656 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, in emitReduction() argument
1659 if (!CGF.HaveInsertPoint()) in emitReduction()
1672 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, in emitReduction()
1691 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); in emitReduction()
1694 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(), in emitReduction()
1695 CGF.AllocaInsertPt->getIterator()); in emitReduction()
1696 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(), in emitReduction()
1697 CGF.Builder.GetInsertPoint()); in emitReduction()
1699 CodeGenIP, CGF.SourceLocToDebugLoc(Loc)); in emitReduction()
1702 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitReduction()
1709 ElementType = CGF.ConvertTypeForMem(Private->getType()); in emitReduction()
1712 PrivateVariable = CGF.GetAddrOfLocalVar(RHSVar).emitRawPointer(CGF); in emitReduction()
1715 Variable = CGF.GetAddrOfLocalVar(LHSVar).emitRawPointer(CGF); in emitReduction()
1717 switch (CGF.getEvaluationKind(Private->getType())) { in emitReduction()
1731 CGF.Builder.restoreIP(CodeGenIP); in emitReduction()
1732 auto *CurFn = CGF.CurFn; in emitReduction()
1733 CGF.CurFn = NewFunc; in emitReduction()
1735 *LHSPtr = CGF.GetAddrOfLocalVar( in emitReduction()
1737 .emitRawPointer(CGF); in emitReduction()
1738 *RHSPtr = CGF.GetAddrOfLocalVar( in emitReduction()
1740 .emitRawPointer(CGF); in emitReduction()
1742 emitSingleReductionCombiner(CGF, ReductionOps[I], Privates[I], in emitReduction()
1746 CGF.CurFn = CurFn; in emitReduction()
1748 return InsertPointTy(CGF.Builder.GetInsertBlock(), in emitReduction()
1749 CGF.Builder.GetInsertPoint()); in emitReduction()
1757 CGF.Builder.restoreIP(OMPBuilder.createReductionsGPU( in emitReduction()
1760 CGF.getTarget().getGridValue(), C.getLangOpts().OpenMPCUDAReductionBufNum, in emitReduction()
1798 CGOpenMPRuntimeGPU::getParameterAddress(CodeGenFunction &CGF, in getParameterAddress() argument
1804 Address LocalAddr = CGF.GetAddrOfLocalVar(TargetParam); in getParameterAddress()
1810 CGF.getTypes().getTargetAddressSpace(NativePointeeTy); in getParameterAddress()
1812 llvm::Value *TargetAddr = CGF.EmitLoadOfScalar(LocalAddr, /*Volatile=*/false, in getParameterAddress()
1815 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getParameterAddress()
1817 llvm::PointerType::get(CGF.getLLVMContext(), NativePointeeAddrSpace)); in getParameterAddress()
1818 Address NativeParamAddr = CGF.CreateMemTemp(NativeParamType); in getParameterAddress()
1819 CGF.EmitStoreOfScalar(TargetAddr, NativeParamAddr, /*Volatile=*/false, in getParameterAddress()
1825 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, in emitOutlinedFunctionCall() argument
1842 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(NativeArg, TargetType)); in emitOutlinedFunctionCall()
1844 CGOpenMPRuntime::emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, TargetArgs); in emitOutlinedFunctionCall()
1890 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in createParallelDataSharingWrapper() local
1891 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, Fn, CGFI, WrapperArgs, in createParallelDataSharingWrapper()
1897 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, in createParallelDataSharingWrapper()
1899 CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr); in createParallelDataSharingWrapper()
1903 Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).emitRawPointer(CGF)); in createParallelDataSharingWrapper()
1904 Args.emplace_back(ZeroAddr.emitRawPointer(CGF)); in createParallelDataSharingWrapper()
1906 CGBuilderTy &Bld = CGF.Builder; in createParallelDataSharingWrapper()
1912 CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args"); in createParallelDataSharingWrapper()
1915 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( in createParallelDataSharingWrapper()
1924 SharedArgListAddress = CGF.EmitLoadOfPointer( in createParallelDataSharingWrapper()
1925 GlobalArgs, CGF.getContext() in createParallelDataSharingWrapper()
1926 .getPointerType(CGF.getContext().VoidPtrTy) in createParallelDataSharingWrapper()
1933 Src, CGF.SizeTy->getPointerTo(), CGF.SizeTy); in createParallelDataSharingWrapper()
1934 llvm::Value *LB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
1937 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
1943 Src, CGF.SizeTy->getPointerTo(), CGF.SizeTy); in createParallelDataSharingWrapper()
1944 llvm::Value *UB = CGF.EmitLoadOfScalar( in createParallelDataSharingWrapper()
1947 CGF.getContext().getPointerType(CGF.getContext().getSizeType()), in createParallelDataSharingWrapper()
1953 ASTContext &CGFContext = CGF.getContext(); in createParallelDataSharingWrapper()
1958 Src, CGF.ConvertTypeForMem(CGFContext.getPointerType(ElemTy)), in createParallelDataSharingWrapper()
1959 CGF.ConvertTypeForMem(ElemTy)); in createParallelDataSharingWrapper()
1960 llvm::Value *Arg = CGF.EmitLoadOfScalar(TypedAddress, in createParallelDataSharingWrapper()
1966 Arg = castValueToType(CGF, Arg, ElemTy, CGFContext.getUIntPtrType(), in createParallelDataSharingWrapper()
1973 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedParallelFn, Args); in createParallelDataSharingWrapper()
1974 CGF.FinishFunction(); in createParallelDataSharingWrapper()
1978 void CGOpenMPRuntimeGPU::emitFunctionProlog(CodeGenFunction &CGF, in emitFunctionProlog() argument
1984 assert(FunctionGlobalizedDecls.count(CGF.CurFn) == 0 && in emitFunctionProlog()
1996 NeedToDelayGlobalization = CGF.CapturedStmtInfo->getKind() == CR_OpenMP; in emitFunctionProlog()
2003 CheckVarsEscapingDeclContext VarChecker(CGF, TeamAndReductions.second); in emitFunctionProlog()
2016 auto I = FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first; in emitFunctionProlog()
2032 emitGenericVarsProlog(CGF, D->getBeginLoc()); in emitFunctionProlog()
2036 void Emit(CodeGenFunction &CGF, Flags flags) override { in emitFunctionProlog()
2037 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()) in emitFunctionProlog()
2038 .emitGenericVarsEpilog(CGF); in emitFunctionProlog()
2041 CGF.EHStack.pushCleanup<GlobalizationScope>(NormalAndEHCleanup); in emitFunctionProlog()
2045 Address CGOpenMPRuntimeGPU::getAddressOfLocalVariable(CodeGenFunction &CGF, in getAddressOfLocalVariable() argument
2073 llvm::Type *VarTy = CGF.ConvertTypeForMem(VD->getType()); in getAddressOfLocalVariable()
2083 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
2093 auto I = FunctionGlobalizedDecls.find(CGF.CurFn); in getAddressOfLocalVariable()
2114 void CGOpenMPRuntimeGPU::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
2115 FunctionGlobalizedDecls.erase(CGF.CurFn); in functionFinished()
2116 CGOpenMPRuntime::functionFinished(CGF); in functionFinished()
2120 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultDistScheduleAndChunk() argument
2123 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime()); in getDefaultDistScheduleAndChunk()
2126 Chunk = CGF.EmitScalarConversion( in getDefaultDistScheduleAndChunk()
2127 RT.getGPUNumThreads(CGF), in getDefaultDistScheduleAndChunk()
2128 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultDistScheduleAndChunk()
2133 CGF, S, ScheduleKind, Chunk); in getDefaultDistScheduleAndChunk()
2137 CodeGenFunction &CGF, const OMPLoopDirective &S, in getDefaultScheduleAndChunk() argument
2143 ChunkExpr = IntegerLiteral::Create(CGF.getContext(), ChunkSize, in getDefaultScheduleAndChunk()
2144 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), in getDefaultScheduleAndChunk()
2149 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { in adjustTargetSpecificDataForLambdas() argument
2165 Address VDAddr = CGF.GetAddrOfLocalVar(VD); in adjustTargetSpecificDataForLambdas()
2168 VDLVal = CGF.EmitLoadOfReferenceLValue(VDAddr, VD->getType()); in adjustTargetSpecificDataForLambdas()
2170 VDLVal = CGF.MakeAddrLValue( in adjustTargetSpecificDataForLambdas()
2175 if (ThisCapture && CGF.CapturedStmtInfo->isCXXThisExprCaptured()) { in adjustTargetSpecificDataForLambdas()
2177 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); in adjustTargetSpecificDataForLambdas()
2178 llvm::Value *CXXThis = CGF.LoadCXXThis(); in adjustTargetSpecificDataForLambdas()
2179 CGF.EmitStoreOfScalar(CXXThis, ThisLVal); in adjustTargetSpecificDataForLambdas()
2191 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); in adjustTargetSpecificDataForLambdas()
2192 Address VDAddr = CGF.GetAddrOfLocalVar(cast<VarDecl>(VD)); in adjustTargetSpecificDataForLambdas()
2194 VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr, in adjustTargetSpecificDataForLambdas()
2197 CGF.EmitStoreOfScalar(VDAddr.emitRawPointer(CGF), VarLVal); in adjustTargetSpecificDataForLambdas()
2343 llvm::Value *CGOpenMPRuntimeGPU::getGPUNumThreads(CodeGenFunction &CGF) { in getGPUNumThreads() argument
2344 CGBuilderTy &Bld = CGF.Builder; in getGPUNumThreads()
2345 llvm::Module *M = &CGF.CGM.getModule(); in getGPUNumThreads()
2350 llvm::FunctionType::get(CGF.Int32Ty, std::nullopt, false), in getGPUNumThreads()
2351 llvm::GlobalVariable::ExternalLinkage, LocSize, &CGF.CGM.getModule()); in getGPUNumThreads()
2356 llvm::Value *CGOpenMPRuntimeGPU::getGPUThreadID(CodeGenFunction &CGF) { in getGPUThreadID() argument
2358 return CGF.EmitRuntimeCall( in getGPUThreadID()