Lines Matching refs:CGF
52 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitPreInitStmt() argument
59 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
62 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in emitPreInitStmt()
63 CGF.EmitAutoVarCleanups(Emission); in emitPreInitStmt()
72 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
73 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
74 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
75 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
76 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
81 CodeGenFunction &CGF, const OMPExecutableDirective &S, in OMPLexicalScope() argument
84 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPLexicalScope()
85 InlinedShareds(CGF) { in OMPLexicalScope()
87 emitPreInitStmt(CGF, S); in OMPLexicalScope()
99 CGF.getContext(), const_cast<VarDecl *>(VD), in OMPLexicalScope()
100 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && in OMPLexicalScope()
103 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress()); in OMPLexicalScope()
121 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPParallelScope() argument
122 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt, in OMPParallelScope()
136 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPTeamsScope() argument
137 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt, in OMPTeamsScope()
144 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) { in emitPreInitStmt() argument
153 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); in emitPreInitStmt()
163 CGF, OrigVD, in emitPreInitStmt()
164 Address(llvm::UndefValue::get(CGF.ConvertTypeForMem( in emitPreInitStmt()
165 CGF.getContext().getPointerType(OrigVDTy))), in emitPreInitStmt()
166 CGF.ConvertTypeForMem(OrigVDTy), in emitPreInitStmt()
167 CGF.getContext().getDeclAlign(OrigVD))); in emitPreInitStmt()
171 (void)PreCondVars.apply(CGF); in emitPreInitStmt()
176 [&CGF](unsigned Cnt, const Stmt *CurStmt) { in emitPreInitStmt()
179 CGF.EmitStmt(Init); in emitPreInitStmt()
180 CGF.EmitStmt(CXXFor->getRangeStmt()); in emitPreInitStmt()
181 CGF.EmitStmt(CXXFor->getEndStmt()); in emitPreInitStmt()
213 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
216 CGF.EmitStmt(S); in emitPreInitStmt()
219 PreCondVars.restore(CGF); in emitPreInitStmt()
223 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) in OMPLoopScope() argument
224 : CodeGenFunction::RunCleanupsScope(CGF) { in OMPLoopScope()
225 emitPreInitStmt(CGF, S); in OMPLoopScope()
232 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
233 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
234 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
235 (isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
236 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
240 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPSimdLexicalScope() argument
241 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPSimdLexicalScope()
242 InlinedShareds(CGF) { in OMPSimdLexicalScope()
249 CGF.EmitVarDecl(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
252 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
253 CGF.EmitAutoVarCleanups(Emission); in OMPSimdLexicalScope()
261 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
267 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
272 CGF.EmitOMPPrivateClause(S, InlinedShareds); in OMPSimdLexicalScope()
275 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl())); in OMPSimdLexicalScope()
295 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in OMPSimdLexicalScope()
296 isCapturedVar(CGF, VD) || in OMPSimdLexicalScope()
297 (CGF.CapturedStmtInfo && in OMPSimdLexicalScope()
301 InlinedShareds.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress()); in OMPSimdLexicalScope()
312 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
399 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, in castValueFromUintptr() argument
402 ASTContext &Ctx = CGF.getContext(); in castValueFromUintptr()
404 llvm::Value *CastedPtr = CGF.EmitScalarConversion( in castValueFromUintptr()
405 AddrLV.getAddress().emitRawPointer(CGF), Ctx.getUIntPtrType(), in castValueFromUintptr()
409 CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(); in castValueFromUintptr()
454 CodeGenFunction &CGF, FunctionArgList &Args, in emitOutlinedFunctionPrologue() argument
466 CodeGenModule &CGM = CGF.CGM; in emitOutlinedFunctionPrologue()
561 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, in emitOutlinedFunctionPrologue()
571 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], in emitOutlinedFunctionPrologue()
574 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); in emitOutlinedFunctionPrologue()
587 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), in emitOutlinedFunctionPrologue()
591 ArgLVal = CGF.MakeAddrLValue( in emitOutlinedFunctionPrologue()
592 castValueFromUintptr(CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
596 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
604 ArgAddr = CGF.EmitLoadOfReference(ArgLVal); in emitOutlinedFunctionPrologue()
607 ArgAddr = CGF.EmitLoadOfPointer( in emitOutlinedFunctionPrologue()
621 CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
627 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
1263 [&Emission](CodeGenFunction &CGF) { in EmitOMPReductionClauseInit() argument
1264 CGF.EmitAutoVarInit(Emission); in EmitOMPReductionClauseInit()
1473 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitPostUpdateForReductionClause() argument
1475 if (!CGF.HaveInsertPoint()) in emitPostUpdateForReductionClause()
1481 if (llvm::Value *Cond = CondGen(CGF)) { in emitPostUpdateForReductionClause()
1484 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); in emitPostUpdateForReductionClause()
1485 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done"); in emitPostUpdateForReductionClause()
1486 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB); in emitPostUpdateForReductionClause()
1487 CGF.EmitBlock(ThenBB); in emitPostUpdateForReductionClause()
1490 CGF.EmitIgnoredExpr(PostUpdate); in emitPostUpdateForReductionClause()
1494 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitPostUpdateForReductionClause()
1508 checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, in checkForLastprivateConditionalUpdate() argument
1510 if (CGF.getLangOpts().OpenMP < 50) in checkForLastprivateConditionalUpdate()
1521 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1532 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1543 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1560 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional( in checkForLastprivateConditionalUpdate()
1561 CGF, S, PrivateDecls); in checkForLastprivateConditionalUpdate()
1565 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitCommonOMPParallelDirective() argument
1571 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( in emitCommonOMPParallelDirective()
1572 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind, in emitCommonOMPParallelDirective()
1575 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitCommonOMPParallelDirective()
1576 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), in emitCommonOMPParallelDirective()
1578 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( in emitCommonOMPParallelDirective()
1579 CGF, NumThreads, NumThreadsClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1582 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF); in emitCommonOMPParallelDirective()
1583 CGF.CGM.getOpenMPRuntime().emitProcBindClause( in emitCommonOMPParallelDirective()
1584 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1595 OMPParallelScope Scope(CGF, S); in emitCommonOMPParallelDirective()
1601 CodeGenBoundParameters(CGF, S, CapturedVars); in emitCommonOMPParallelDirective()
1602 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPParallelDirective()
1603 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn, in emitCommonOMPParallelDirective()
1622 static void emitOMPCopyinClause(CodeGenFunction &CGF, in emitOMPCopyinClause() argument
1624 bool Copyins = CGF.EmitOMPCopyinClause(S); in emitOMPCopyinClause()
1629 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in emitOMPCopyinClause()
1630 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in emitOMPCopyinClause()
1636 CodeGenFunction &CGF, const VarDecl *VD) { in getAddressOfLocalVariable() argument
1637 CodeGenModule &CGM = CGF.CGM; in getAddressOfLocalVariable()
1648 Size = CGF.getTypeSize(CVD->getType()); in getAddressOfLocalVariable()
1650 Size = CGF.Builder.CreateNUWAdd( in getAddressOfLocalVariable()
1652 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1653 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1662 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator()); in getAddressOfLocalVariable()
1666 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy); in getAddressOfLocalVariable()
1668 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator, in getAddressOfLocalVariable()
1672 CGF.Builder, Size, Allocator, in getAddressOfLocalVariable()
1675 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator); in getAddressOfLocalVariable()
1677 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI); in getAddressOfLocalVariable()
1678 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
1680 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())), in getAddressOfLocalVariable()
1682 return Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align); in getAddressOfLocalVariable()
1686 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, in getAddrOfThreadPrivate() argument
1688 CodeGenModule &CGM = CGF.CGM; in getAddrOfThreadPrivate()
1697 CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy); in getAddrOfThreadPrivate()
1703 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName); in getAddrOfThreadPrivate()
1721 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, in EmitOMPInlinedRegionBody() argument
1723 CGBuilderTy &Builder = CGF.Builder; in EmitOMPInlinedRegionBody()
1729 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB); in EmitOMPInlinedRegionBody()
1730 CGF.EmitStmt(RegionBodyStmt); in EmitOMPInlinedRegionBody()
1738 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP, in EmitOMPOutlinedRegionBody() argument
1740 CGBuilderTy &Builder = CGF.Builder; in EmitOMPOutlinedRegionBody()
1746 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB); in EmitOMPOutlinedRegionBody()
1747 CGF.EmitStmt(RegionBodyStmt); in EmitOMPOutlinedRegionBody()
1813 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelDirective() argument
1814 Action.Enter(CGF); in EmitOMPParallelDirective()
1815 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelDirective()
1816 emitOMPCopyinClause(CGF, S); in EmitOMPParallelDirective()
1817 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1818 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1819 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelDirective()
1821 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); in EmitOMPParallelDirective()
1822 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelDirective()
1853 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) { in OMPTransformDirectiveScopeRAII() argument
1855 Scope = new OMPLoopScope(CGF, *Dir); in OMPTransformDirectiveScopeRAII()
1857 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI); in OMPTransformDirectiveScopeRAII()
1870 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, in emitBody() argument
1876 CGF.getContext().getSourceManager(), CS->getLBracLoc(), in emitBody()
1880 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange()); in emitBody()
1882 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level); in emitBody()
1896 CGF.EmitStmt(CXXFor->getLoopVarStmt()); in emitBody()
1902 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1); in emitBody()
1906 CGF.EmitStmt(S); in emitBody()
1984 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true); in emitCapturedStmtFunc() local
1987 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get()); in emitCapturedStmtFunc()
1988 llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S); in emitCapturedStmtFunc()
2245 static void emitAlignedClause(CodeGenFunction &CGF, in emitAlignedClause() argument
2247 if (!CGF.HaveInsertPoint()) in emitAlignedClause()
2253 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); in emitAlignedClause()
2263 CGF.getContext() in emitAlignedClause()
2264 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( in emitAlignedClause()
2271 llvm::Value *PtrValue = CGF.EmitScalarExpr(E); in emitAlignedClause()
2272 CGF.emitAlignmentAssumption( in emitAlignedClause()
2274 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment)); in emitAlignedClause()
2322 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitPreCond() argument
2325 if (!CGF.HaveInsertPoint()) in emitPreCond()
2328 CodeGenFunction::OMPPrivateScope PreCondScope(CGF); in emitPreCond()
2329 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope); in emitPreCond()
2333 CGF.EmitIgnoredExpr(I); in emitPreCond()
2346 CGF.CreateMemTemp(VD->getType().getNonReferenceType()); in emitPreCond()
2347 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr); in emitPreCond()
2349 (void)PreCondVars.apply(CGF); in emitPreCond()
2353 CGF.EmitIgnoredExpr(E); in emitPreCond()
2356 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); in emitPreCond()
2357 PreCondVars.restore(CGF); in emitPreCond()
2394 static void emitSimdlenSafelenClause(CodeGenFunction &CGF, in emitSimdlenSafelenClause() argument
2396 if (!CGF.HaveInsertPoint()) in emitSimdlenSafelenClause()
2399 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2402 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2406 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>()); in emitSimdlenSafelenClause()
2408 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2411 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2415 CGF.LoopStack.setParallel(/*Enable=*/false); in emitSimdlenSafelenClause()
2483 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, in emitOMPLoopBodyWithStopPoint() argument
2486 CGF.EmitOMPLoopBody(S, LoopExit); in emitOMPLoopBodyWithStopPoint()
2487 CGF.EmitStopPoint(&S); in emitOMPLoopBodyWithStopPoint()
2491 static LValue EmitOMPHelperVar(CodeGenFunction &CGF, in EmitOMPHelperVar() argument
2494 CGF.EmitVarDecl(*VDecl); in EmitOMPHelperVar()
2495 return CGF.EmitLValue(Helper); in EmitOMPHelperVar()
2498 static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitCommonSimdLoop() argument
2501 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF, in emitCommonSimdLoop()
2503 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S); in emitCommonSimdLoop()
2504 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2505 SimdInitGen(CGF); in emitCommonSimdLoop()
2507 BodyCodeGen(CGF); in emitCommonSimdLoop()
2509 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonSimdLoop() argument
2510 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2511 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false); in emitCommonSimdLoop()
2513 BodyCodeGen(CGF); in emitCommonSimdLoop()
2518 if (CGF.getLangOpts().OpenMP >= 50 && in emitCommonSimdLoop()
2527 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen); in emitCommonSimdLoop()
2530 ThenRCG(CGF); in emitCommonSimdLoop()
2534 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitOMPSimdRegion() argument
2536 Action.Enter(CGF); in emitOMPSimdRegion()
2539 OMPLoopScope PreInitScope(CGF, S); in emitOMPSimdRegion()
2548 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable())); in emitOMPSimdRegion()
2549 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable())); in emitOMPSimdRegion()
2557 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in emitOMPSimdRegion()
2561 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); in emitOMPSimdRegion()
2562 ContBlock = CGF.createBasicBlock("simd.if.end"); in emitOMPSimdRegion()
2563 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in emitOMPSimdRegion()
2564 CGF.getProfileCount(&S)); in emitOMPSimdRegion()
2565 CGF.EmitBlock(ThenBlock); in emitOMPSimdRegion()
2566 CGF.incrementProfileCounter(&S); in emitOMPSimdRegion()
2572 CGF.EmitVarDecl(*IVDecl); in emitOMPSimdRegion()
2573 CGF.EmitIgnoredExpr(S.getInit()); in emitOMPSimdRegion()
2579 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in emitOMPSimdRegion()
2581 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in emitOMPSimdRegion()
2584 emitAlignedClause(CGF, S); in emitOMPSimdRegion()
2585 (void)CGF.EmitOMPLinearClauseInit(S); in emitOMPSimdRegion()
2587 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in emitOMPSimdRegion()
2588 CGF.EmitOMPPrivateClause(S, LoopScope); in emitOMPSimdRegion()
2589 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in emitOMPSimdRegion()
2590 CGF.EmitOMPLinearClause(S, LoopScope); in emitOMPSimdRegion()
2591 CGF.EmitOMPReductionClauseInit(S, LoopScope); in emitOMPSimdRegion()
2593 CGF, S, CGF.EmitLValue(S.getIterationVariable())); in emitOMPSimdRegion()
2594 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in emitOMPSimdRegion()
2597 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitOMPSimdRegion()
2600 CGF, S, in emitOMPSimdRegion()
2601 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2602 CGF.EmitOMPSimdInit(S); in emitOMPSimdRegion()
2604 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2605 CGF.EmitOMPInnerLoop( in emitOMPSimdRegion()
2607 [&S](CodeGenFunction &CGF) { in emitOMPSimdRegion() argument
2608 emitOMPLoopBodyWithStopPoint(CGF, S, in emitOMPSimdRegion()
2613 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2616 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); in emitOMPSimdRegion()
2617 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); in emitOMPSimdRegion()
2618 emitPostUpdateForReductionClause(CGF, S, in emitOMPSimdRegion()
2621 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2625 CGF.EmitBranch(ContBlock); in emitOMPSimdRegion()
2626 CGF.EmitBlock(ContBlock, true); in emitOMPSimdRegion()
2662 GetAlignedMapping(const OMPSimdDirective &S, CodeGenFunction &CGF) { in GetAlignedMapping() argument
2668 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); in GetAlignedMapping()
2678 CGF.getContext() in GetAlignedMapping()
2679 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( in GetAlignedMapping()
2685 llvm::Value *PtrValue = CGF.EmitScalarExpr(E); in GetAlignedMapping()
2686 AlignedVars[PtrValue] = CGF.Builder.getInt64(Alignment.getSExtValue()); in GetAlignedMapping()
2696 auto &&CodeGenIRBuilder = [this, &S, UseOMPIRBuilder](CodeGenFunction &CGF, in EmitOMPSimdDirective()
2701 GetAlignedMapping(S, CGF); in EmitOMPSimdDirective()
2751 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSimdDirective() argument
2752 emitOMPSimdRegion(CGF, S, Action); in EmitOMPSimdDirective()
2908 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2912 CGF.LoopStack.setParallel(!IsMonotonic); in EmitOMPOuterLoop()
2915 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPOuterLoop()
2917 CGF.EmitOMPSimdInit(S); in EmitOMPOuterLoop()
2921 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2928 CGF.EmitOMPInnerLoop( in EmitOMPOuterLoop()
2930 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2931 CodeGenLoop(CGF, S, LoopExit); in EmitOMPOuterLoop()
2933 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2934 CodeGenOrdered(CGF, Loc, IVSize, IVSigned); in EmitOMPOuterLoop()
2953 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2955 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPOuterLoop()
3048 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc, in EmitOMPForOuterLoop()
3052 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize, in EmitOMPForOuterLoop()
3137 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, in emitDistributeParallelForInnerBounds() argument
3141 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitDistributeParallelForInnerBounds()
3143 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitDistributeParallelForInnerBounds()
3151 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable()); in emitDistributeParallelForInnerBounds()
3152 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable()); in emitDistributeParallelForInnerBounds()
3153 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
3155 PrevLBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
3159 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
3161 PrevUBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
3166 CGF.EmitStoreOfScalar(PrevLBVal, LB); in emitDistributeParallelForInnerBounds()
3167 CGF.EmitStoreOfScalar(PrevUBVal, UB); in emitDistributeParallelForInnerBounds()
3180 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, in emitDistributeParallelForDispatchBounds() argument
3191 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
3193 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
3198 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDistributeParallelForDistributeInnerBoundParams() argument
3202 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
3203 llvm::Value *LBCast = CGF.Builder.CreateIntCast( in emitDistributeParallelForDistributeInnerBoundParams()
3204 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
3207 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
3209 llvm::Value *UBCast = CGF.Builder.CreateIntCast( in emitDistributeParallelForDistributeInnerBoundParams()
3210 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
3215 emitInnerParallelForWhenCombined(CodeGenFunction &CGF, in emitInnerParallelForWhenCombined() argument
3218 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF, in emitInnerParallelForWhenCombined()
3220 Action.Enter(CGF); in emitInnerParallelForWhenCombined()
3231 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitInnerParallelForWhenCombined()
3233 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(), in emitInnerParallelForWhenCombined()
3239 CGF, S, in emitInnerParallelForWhenCombined()
3247 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForDirective() argument
3248 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForDirective()
3257 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForSimdDirective() argument
3258 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForSimdDirective()
3267 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeSimdDirective() argument
3268 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeSimdDirective()
3277 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDeviceFunction() argument
3278 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDeviceFunction()
3290 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDirective() argument
3291 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDirective()
3445 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3447 CGF.EmitOMPSimdInit(S); in EmitOMPWorksharingLoop()
3450 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPWorksharingLoop()
3455 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3465 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitOMPWorksharingLoop()
3466 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, in EmitOMPWorksharingLoop()
3470 CGF.EmitIgnoredExpr(S.getEnsureUpperBound()); in EmitOMPWorksharingLoop()
3472 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPWorksharingLoop()
3486 CGF.EmitOMPInnerLoop( in EmitOMPWorksharingLoop()
3491 [&S, LoopExit](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3492 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit); in EmitOMPWorksharingLoop()
3498 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3499 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPWorksharingLoop()
3514 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3515 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3516 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3525 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3526 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3527 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3535 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3536 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3537 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3554 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitForLoopBounds() argument
3557 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitForLoopBounds()
3559 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitForLoopBounds()
3568 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDispatchForLoopBounds() argument
3572 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType()); in emitDispatchForLoopBounds()
3573 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0); in emitDispatchForLoopBounds()
3574 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration()); in emitDispatchForLoopBounds()
3586 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirectiveDecls() argument
3588 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirectiveDecls()
3589 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirectiveDecls()
3615 RedCG.emitSharedOrigLValue(CGF, Count); in emitScanBasedDirectiveDecls()
3616 RedCG.emitAggregateType(CGF, Count); in emitScanBasedDirectiveDecls()
3619 CGF, in emitScanBasedDirectiveDecls()
3625 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl())); in emitScanBasedDirectiveDecls()
3638 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirectiveFinals() argument
3640 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirectiveFinals()
3641 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirectiveFinals()
3661 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub( in emitScanBasedDirectiveFinals()
3663 llvm::ConstantInt::get(CGF.SizeTy, 1, /*isSigned=*/false)); in emitScanBasedDirectiveFinals()
3669 CGF, in emitScanBasedDirectiveFinals()
3673 LValue DestLVal = CGF.EmitLValue(OrigExpr); in emitScanBasedDirectiveFinals()
3674 LValue SrcLVal = CGF.EmitLValue(CopyArrayElem); in emitScanBasedDirectiveFinals()
3675 CGF.EmitOMPCopy( in emitScanBasedDirectiveFinals()
3702 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirective() argument
3706 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirective()
3707 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirective()
3723 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S); in emitScanBasedDirective()
3731 CGF.OMPFirstScanLoop = true; in emitScanBasedDirective()
3732 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitScanBasedDirective()
3733 FirstGen(CGF); in emitScanBasedDirective()
3738 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitScanBasedDirective() argument
3739 Action.Enter(CGF); in emitScanBasedDirective()
3743 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock(); in emitScanBasedDirective()
3744 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body"); in emitScanBasedDirective()
3745 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit"); in emitScanBasedDirective()
3747 CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy); in emitScanBasedDirective()
3749 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy); in emitScanBasedDirective()
3750 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg); in emitScanBasedDirective()
3751 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy); in emitScanBasedDirective()
3752 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal); in emitScanBasedDirective()
3753 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy); in emitScanBasedDirective()
3754 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub( in emitScanBasedDirective()
3755 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3756 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc()); in emitScanBasedDirective()
3757 CGF.EmitBlock(LoopBB); in emitScanBasedDirective()
3758 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2); in emitScanBasedDirective()
3760 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3761 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB); in emitScanBasedDirective()
3762 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB); in emitScanBasedDirective()
3766 CGF.createBasicBlock("omp.inner.log.scan.body"); in emitScanBasedDirective()
3768 CGF.createBasicBlock("omp.inner.log.scan.exit"); in emitScanBasedDirective()
3769 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K); in emitScanBasedDirective()
3770 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3771 CGF.EmitBlock(InnerLoopBB); in emitScanBasedDirective()
3772 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3775 CodeGenFunction::OMPPrivateScope PrivScope(CGF); in emitScanBasedDirective()
3784 CGF, in emitScanBasedDirective()
3788 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(); in emitScanBasedDirective()
3793 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K); in emitScanBasedDirective()
3795 CGF, in emitScanBasedDirective()
3799 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(); in emitScanBasedDirective()
3806 CGF.CGM.getOpenMPRuntime().emitReduction( in emitScanBasedDirective()
3807 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps, in emitScanBasedDirective()
3811 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3812 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3813 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K); in emitScanBasedDirective()
3814 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3815 CGF.EmitBlock(InnerExitBB); in emitScanBasedDirective()
3817 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1)); in emitScanBasedDirective()
3818 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3821 CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true); in emitScanBasedDirective()
3822 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3823 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal); in emitScanBasedDirective()
3824 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB); in emitScanBasedDirective()
3825 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc()); in emitScanBasedDirective()
3826 CGF.EmitBlock(ExitBB); in emitScanBasedDirective()
3829 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); in emitScanBasedDirective()
3830 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in emitScanBasedDirective()
3831 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in emitScanBasedDirective()
3835 RCG(CGF); in emitScanBasedDirective()
3838 CGF.OMPFirstScanLoop = false; in emitScanBasedDirective()
3839 SecondGen(CGF); in emitScanBasedDirective()
3842 static bool emitWorksharingDirective(CodeGenFunction &CGF, in emitWorksharingDirective() argument
3850 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3851 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitWorksharingDirective()
3852 OMPLoopScope LoopScope(CGF, S); in emitWorksharingDirective()
3853 return CGF.EmitScalarExpr(S.getNumIterations()); in emitWorksharingDirective()
3855 const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3857 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3858 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3862 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(), in emitWorksharingDirective()
3866 &HasLastprivates](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3868 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3869 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3874 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen); in emitWorksharingDirective()
3875 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen); in emitWorksharingDirective()
3877 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen); in emitWorksharingDirective()
3879 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitWorksharingDirective()
3881 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3942 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPForDirective() argument
3973 HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPForDirective()
3994 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, in EmitOMPForSimdDirective()
3996 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPForSimdDirective()
4012 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, in createSectionLVal() argument
4015 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); in createSectionLVal()
4017 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); in createSectionLVal()
4026 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { in EmitSections() argument
4027 const ASTContext &C = CGF.getContext(); in EmitSections()
4031 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", in EmitSections()
4032 CGF.Builder.getInt32(0)); in EmitSections()
4034 ? CGF.Builder.getInt32(CS->size() - 1) in EmitSections()
4035 : CGF.Builder.getInt32(0); in EmitSections()
4037 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); in EmitSections()
4038 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", in EmitSections()
4039 CGF.Builder.getInt32(1)); in EmitSections()
4040 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", in EmitSections()
4041 CGF.Builder.getInt32(0)); in EmitSections()
4043 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); in EmitSections()
4045 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); in EmitSections()
4047 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); in EmitSections()
4056 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { in EmitSections() argument
4068 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); in EmitSections()
4070 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()), in EmitSections()
4075 auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
4076 CGF.EmitBlock(CaseBB); in EmitSections()
4077 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); in EmitSections()
4078 CGF.EmitStmt(SubStmt); in EmitSections()
4079 CGF.EmitBranch(ExitBB); in EmitSections()
4083 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
4084 CGF.EmitBlock(CaseBB); in EmitSections()
4085 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB); in EmitSections()
4086 CGF.EmitStmt(CapturedStmt); in EmitSections()
4087 CGF.EmitBranch(ExitBB); in EmitSections()
4089 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in EmitSections()
4092 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in EmitSections()
4093 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { in EmitSections()
4097 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitSections()
4098 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitSections()
4101 CGF.EmitOMPPrivateClause(S, LoopScope); in EmitSections()
4102 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV); in EmitSections()
4103 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitSections()
4104 CGF.EmitOMPReductionClauseInit(S, LoopScope); in EmitSections()
4107 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in EmitSections()
4115 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitSections()
4116 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit); in EmitSections()
4118 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc()); in EmitSections()
4119 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( in EmitSections()
4120 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); in EmitSections()
4121 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); in EmitSections()
4123 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV); in EmitSections()
4125 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen, in EmitSections()
4128 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitSections() argument
4129 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitSections()
4132 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); in EmitSections()
4133 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitSections()
4135 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { in EmitSections() argument
4136 return CGF.Builder.CreateIsNotNull( in EmitSections()
4137 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitSections()
4142 CGF.EmitOMPLastprivateClauseFinal( in EmitSections()
4144 CGF.Builder.CreateIsNotNull( in EmitSections()
4145 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()))); in EmitSections()
4281 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSingleDirective() argument
4282 Action.Enter(CGF); in EmitOMPSingleDirective()
4283 OMPPrivateScope SingleScope(CGF); in EmitOMPSingleDirective()
4284 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope); in EmitOMPSingleDirective()
4285 CGF.EmitOMPPrivateClause(S, SingleScope); in EmitOMPSingleDirective()
4287 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPSingleDirective()
4308 static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitMaster() argument
4309 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitMaster() argument
4310 Action.Enter(CGF); in emitMaster()
4311 CGF.EmitStmt(S.getRawStmt()); in emitMaster()
4313 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); in emitMaster()
4344 static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitMasked() argument
4345 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitMasked() argument
4346 Action.Enter(CGF); in emitMasked()
4347 CGF.EmitStmt(S.getRawStmt()); in emitMasked()
4352 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(), in emitMasked()
4428 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPCriticalDirective() argument
4429 Action.Enter(CGF); in EmitOMPCriticalDirective()
4430 CGF.EmitStmt(S.getAssociatedStmt()); in EmitOMPCriticalDirective()
4446 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForDirective() argument
4447 Action.Enter(CGF); in EmitOMPParallelForDirective()
4448 emitOMPCopyinClause(CGF, S); in EmitOMPParallelForDirective()
4449 (void)emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPParallelForDirective()
4452 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in EmitOMPParallelForDirective() argument
4453 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in EmitOMPParallelForDirective()
4455 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI); in EmitOMPParallelForDirective()
4456 OMPLoopScope LoopScope(CGF, S); in EmitOMPParallelForDirective()
4457 return CGF.EmitScalarExpr(S.getNumIterations()); in EmitOMPParallelForDirective()
4480 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForSimdDirective() argument
4481 Action.Enter(CGF); in EmitOMPParallelForSimdDirective()
4482 emitOMPCopyinClause(CGF, S); in EmitOMPParallelForSimdDirective()
4483 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPParallelForSimdDirective()
4486 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in EmitOMPParallelForSimdDirective() argument
4487 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in EmitOMPParallelForSimdDirective()
4489 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI); in EmitOMPParallelForSimdDirective()
4490 OMPLoopScope LoopScope(CGF, S); in EmitOMPParallelForSimdDirective()
4491 return CGF.EmitScalarExpr(S.getNumIterations()); in EmitOMPParallelForSimdDirective()
4514 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterDirective() argument
4515 Action.Enter(CGF); in EmitOMPParallelMasterDirective()
4516 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelMasterDirective()
4517 emitOMPCopyinClause(CGF, S); in EmitOMPParallelMasterDirective()
4518 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
4519 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
4520 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelMasterDirective()
4522 emitMaster(CGF, S); in EmitOMPParallelMasterDirective()
4523 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelMasterDirective()
4541 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMaskedDirective() argument
4542 Action.Enter(CGF); in EmitOMPParallelMaskedDirective()
4543 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelMaskedDirective()
4544 emitOMPCopyinClause(CGF, S); in EmitOMPParallelMaskedDirective()
4545 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelMaskedDirective()
4546 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelMaskedDirective()
4547 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelMaskedDirective()
4549 emitMasked(CGF, S); in EmitOMPParallelMaskedDirective()
4550 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelMaskedDirective()
4568 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelSectionsDirective() argument
4569 Action.Enter(CGF); in EmitOMPParallelSectionsDirective()
4570 emitOMPCopyinClause(CGF, S); in EmitOMPParallelSectionsDirective()
4571 CGF.EmitSections(S); in EmitOMPParallelSectionsDirective()
4757 CapturedRegion](CodeGenFunction &CGF, in EmitOMPTaskBasedDirective()
4763 OMPPrivateScope Scope(CGF); in EmitOMPTaskBasedDirective()
4765 if (auto *DI = CGF.getDebugInfo()) { in EmitOMPTaskBasedDirective()
4767 CGF.CapturedStmtInfo->getCaptureFields(); in EmitOMPTaskBasedDirective()
4768 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue(); in EmitOMPTaskBasedDirective()
4770 unsigned CharWidth = CGF.getContext().getCharWidth(); in EmitOMPTaskBasedDirective()
4788 CGF.getContext().getASTRecordLayout(CaptureRecord); in EmitOMPTaskBasedDirective()
4791 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo()) in EmitOMPTaskBasedDirective()
4793 CGF.Builder, false); in EmitOMPTaskBasedDirective()
4807 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back(); in EmitOMPTaskBasedDirective()
4814 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) { in EmitOMPTaskBasedDirective()
4828 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
4829 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTaskBasedDirective()
4830 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTaskBasedDirective()
4840 RawAddress PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
4841 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); in EmitOMPTaskBasedDirective()
4849 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
4859 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
4868 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
4870 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
4871 RawAddress PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
4872 CGF.getContext().getPointerType(Ty), ".local.ptr.addr"); in EmitOMPTaskBasedDirective()
4882 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(), in EmitOMPTaskBasedDirective()
4884 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTaskBasedDirective()
4885 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTaskBasedDirective()
4888 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD), in EmitOMPTaskBasedDirective()
4890 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, in EmitOMPTaskBasedDirective()
4893 Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress()); in EmitOMPTaskBasedDirective()
4897 CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4898 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()), in EmitOMPTaskBasedDirective()
4899 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4901 if (auto *DI = CGF.getDebugInfo()) in EmitOMPTaskBasedDirective()
4902 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo()) in EmitOMPTaskBasedDirective()
4904 Pair.first, Pair.second.getBasePointer(), CGF.Builder, in EmitOMPTaskBasedDirective()
4912 VDType = CGF.getContext().getPointerType(VDType); in EmitOMPTaskBasedDirective()
4914 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4917 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)), in EmitOMPTaskBasedDirective()
4918 CGF.getPointerAlign()); in EmitOMPTaskBasedDirective()
4920 Ptr = CGF.Builder.CreateLoad(Replacement); in EmitOMPTaskBasedDirective()
4921 Replacement = Address(Ptr, CGF.ConvertTypeForMem(VDType), in EmitOMPTaskBasedDirective()
4922 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4925 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4926 Address Replacement(Ptr, CGF.ConvertTypeForMem(VDType), in EmitOMPTaskBasedDirective()
4927 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4933 OMPPrivateScope FirstprivateScope(CGF); in EmitOMPTaskBasedDirective()
4936 CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4937 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()), in EmitOMPTaskBasedDirective()
4938 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4942 OMPLexicalScope LexScope(CGF, S, CapturedRegion); in EmitOMPTaskBasedDirective()
4945 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
4946 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9))); in EmitOMPTaskBasedDirective()
4948 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4949 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
4953 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
4955 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
4956 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
4958 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF), in EmitOMPTaskBasedDirective()
4959 CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
4960 CGF.getContext().getPointerType( in EmitOMPTaskBasedDirective()
4963 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), in EmitOMPTaskBasedDirective()
4965 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
4991 OMPPrivateScope InRedScope(CGF); in EmitOMPTaskBasedDirective()
4995 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4996 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
5002 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
5006 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), in EmitOMPTaskBasedDirective()
5009 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in EmitOMPTaskBasedDirective()
5011 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
5012 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
5014 CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
5015 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
5016 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), in EmitOMPTaskBasedDirective()
5018 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), in EmitOMPTaskBasedDirective()
5020 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
5026 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF, in EmitOMPTaskBasedDirective()
5028 Action.Enter(CGF); in EmitOMPTaskBasedDirective()
5029 BodyGen(CGF); in EmitOMPTaskBasedDirective()
5143 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTaskBasedDirective() argument
5145 OMPPrivateScope Scope(CGF); in EmitOMPTargetTaskBasedDirective()
5148 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTargetTaskBasedDirective()
5149 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTargetTaskBasedDirective()
5150 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTargetTaskBasedDirective()
5161 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTargetTaskBasedDirective()
5167 auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(), in EmitOMPTargetTaskBasedDirective()
5169 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTargetTaskBasedDirective()
5170 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTargetTaskBasedDirective()
5173 CGF.Builder.CreateLoad(Pair.second), in EmitOMPTargetTaskBasedDirective()
5174 CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()), in EmitOMPTargetTaskBasedDirective()
5175 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTargetTaskBasedDirective()
5179 CGF.processInReduction(S, Data, CGF, CS, Scope); in EmitOMPTargetTaskBasedDirective()
5181 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5182 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5183 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5184 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5185 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5186 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5189 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
5190 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
5193 Action.Enter(CGF); in EmitOMPTargetTaskBasedDirective()
5194 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); in EmitOMPTargetTaskBasedDirective()
5196 if (CGF.CGM.getLangOpts().OpenMP >= 51 && in EmitOMPTargetTaskBasedDirective()
5201 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause( in EmitOMPTargetTaskBasedDirective()
5202 CGF, TL->getThreadLimit(), S.getBeginLoc()); in EmitOMPTargetTaskBasedDirective()
5204 BodyGen(CGF); in EmitOMPTargetTaskBasedDirective()
5219 CodeGenFunction &CGF, in processInReduction() argument
5224 OMPLexicalScope LexScope(CGF, S, CapturedRegion); in processInReduction()
5227 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( in processInReduction()
5228 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(4))); in processInReduction()
5230 RedCG.emitSharedOrigLValue(CGF, Cnt); in processInReduction()
5231 RedCG.emitAggregateType(CGF, Cnt); in processInReduction()
5235 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in processInReduction()
5237 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in processInReduction()
5238 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in processInReduction()
5240 CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF), in processInReduction()
5241 CGF.getContext().VoidPtrTy, in processInReduction()
5242 CGF.getContext().getPointerType( in processInReduction()
5245 CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()), in processInReduction()
5247 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in processInReduction()
5270 OMPPrivateScope InRedScope(CGF); in processInReduction()
5274 RedCG.emitSharedOrigLValue(CGF, Cnt); in processInReduction()
5275 RedCG.emitAggregateType(CGF, Cnt); in processInReduction()
5279 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in processInReduction()
5284 CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), TRExpr->getExprLoc()); in processInReduction()
5286 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in processInReduction()
5288 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in processInReduction()
5289 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in processInReduction()
5291 CGF.EmitScalarConversion( in processInReduction()
5292 Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy, in processInReduction()
5293 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), in processInReduction()
5295 CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()), in processInReduction()
5297 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in processInReduction()
5321 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskDirective() argument
5322 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPTaskDirective()
5325 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskDirective()
5327 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn, in EmitOMPTaskDirective()
5387 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTaskgroupDirective() argument
5388 Action.Enter(CGF); in EmitOMPTaskgroupDirective()
5403 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(), in EmitOMPTaskgroupDirective()
5406 CGF.EmitVarDecl(*VD); in EmitOMPTaskgroupDirective()
5407 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), in EmitOMPTaskgroupDirective()
5410 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTaskgroupDirective()
5817 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
5819 CGF.EmitOMPSimdInit(S); in EmitOMPDistributeLoop()
5822 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
5823 CGF.EmitOMPInnerLoop( in EmitOMPDistributeLoop()
5825 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5826 CodeGenLoop(CGF, S, LoopExit); in EmitOMPDistributeLoop()
5828 [&S, StaticChunked](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5830 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound()); in EmitOMPDistributeLoop()
5831 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound()); in EmitOMPDistributeLoop()
5832 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound()); in EmitOMPDistributeLoop()
5833 CGF.EmitIgnoredExpr(S.getCombinedInit()); in EmitOMPDistributeLoop()
5850 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5851 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
5852 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
5861 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
5862 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
5863 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
5884 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeDirective() argument
5885 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeDirective()
5894 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in emitOutlinedOrderedFunction() local
5896 CGF.CapturedStmtInfo = &CapStmtInfo; in emitOutlinedOrderedFunction()
5897 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, Loc); in emitOutlinedOrderedFunction()
5903 static void emitRestoreIP(CodeGenFunction &CGF, const T *C, in emitRestoreIP() argument
5908 QualType Int64Ty = CGF.CGM.getContext().getIntTypeForBitwidth( in emitRestoreIP()
5914 llvm::Value *StoreValue = CGF.EmitScalarConversion( in emitRestoreIP()
5915 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, in emitRestoreIP()
5921 CGF.Builder.restoreIP( in emitRestoreIP()
5922 OMPBuilder.createOrderedDepend(CGF.Builder, AllocaIP, NumLoops, in emitRestoreIP()
5996 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, in EmitOMPOrderedDirective()
6001 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in EmitOMPOrderedDirective()
6004 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(), in EmitOMPOrderedDirective()
6007 Action.Enter(CGF); in EmitOMPOrderedDirective()
6008 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPOrderedDirective()
6015 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, in convertToScalarValue() argument
6018 assert(CGF.hasScalarEvaluationKind(DestType) && in convertToScalarValue()
6021 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, in convertToScalarValue()
6023 : CGF.EmitComplexToScalarConversion( in convertToScalarValue()
6028 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, in convertToComplexValue() argument
6030 assert(CGF.getEvaluationKind(DestType) == TEK_Complex && in convertToComplexValue()
6037 llvm::Value *ScalarVal = CGF.EmitScalarConversion( in convertToComplexValue()
6046 ComplexVal.first = CGF.EmitScalarConversion( in convertToComplexValue()
6048 ComplexVal.second = CGF.EmitScalarConversion( in convertToComplexValue()
6054 static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitSimpleAtomicStore() argument
6057 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); in emitSimpleAtomicStore()
6059 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false); in emitSimpleAtomicStore()
6062 static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, in emitSimpleAtomicLoad() argument
6066 return CGF.EmitLoadOfLValue(LVal, Loc); in emitSimpleAtomicLoad()
6067 return CGF.EmitAtomicLoad( in emitSimpleAtomicLoad()
6090 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitOMPAtomicReadExpr() argument
6096 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicReadExpr()
6097 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicReadExpr()
6098 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc); in emitOMPAtomicReadExpr()
6107 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc, in emitOMPAtomicReadExpr()
6117 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc); in emitOMPAtomicReadExpr()
6118 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicReadExpr()
6121 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, in emitOMPAtomicWriteExpr() argument
6126 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); in emitOMPAtomicWriteExpr()
6127 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicWriteExpr()
6136 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc, in emitOMPAtomicWriteExpr()
6148 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, in emitOMPAtomicRMW() argument
6153 ASTContext &Context = CGF.getContext(); in emitOMPAtomicRMW()
6164 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) { in emitOMPAtomicRMW()
6169 return llvm::isPowerOf2_64(CGF.CGM.getDataLayout().getTypeStoreSize(T)); in emitOMPAtomicRMW()
6254 UpdateVal = CGF.Builder.CreateIntCast( in emitOMPAtomicRMW()
6258 UpdateVal = CGF.Builder.CreateCast(llvm::Instruction::CastOps::UIToFP, IC, in emitOMPAtomicRMW()
6262 CGF.Builder.CreateAtomicRMW(RMWOp, X.getAddress(), UpdateVal, AO); in emitOMPAtomicRMW()
6290 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, in emitOMPAtomicUpdateExpr() argument
6304 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicUpdateExpr()
6305 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicUpdateExpr()
6310 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { in emitOMPAtomicUpdateExpr()
6311 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicUpdateExpr()
6312 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicUpdateExpr()
6313 return CGF.EmitAnyExpr(UE); in emitOMPAtomicUpdateExpr()
6315 (void)CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicUpdateExpr()
6317 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicUpdateExpr()
6326 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc, in emitOMPAtomicUpdateExpr()
6338 static RValue convertToType(CodeGenFunction &CGF, RValue Value, in convertToType() argument
6341 switch (CGF.getEvaluationKind(ResType)) { in convertToType()
6344 convertToScalarValue(CGF, Value, SourceType, ResType, Loc)); in convertToType()
6346 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc); in convertToType()
6355 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, in emitOMPAtomicCaptureExpr() argument
6364 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicCaptureExpr()
6365 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicCaptureExpr()
6366 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicCaptureExpr()
6384 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, in emitOMPAtomicCaptureExpr()
6386 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
6387 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicCaptureExpr()
6388 RValue Res = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
6392 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
6394 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
6403 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
6404 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); in emitOMPAtomicCaptureExpr()
6405 NewVVal = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
6411 ExprRValue = convertToType(CGF, ExprRValue, E->getType(), in emitOMPAtomicCaptureExpr()
6418 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
6421 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
6428 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc); in emitOMPAtomicCaptureExpr()
6429 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicCaptureExpr()
6431 if (CGF.CGM.getLangOpts().OpenMP < 51) { in emitOMPAtomicCaptureExpr()
6441 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc, in emitOMPAtomicCaptureExpr()
6445 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, std::nullopt, Loc, in emitOMPAtomicCaptureExpr()
6450 CGF.CGM.getOpenMPRuntime().emitFlush( in emitOMPAtomicCaptureExpr()
6451 CGF, std::nullopt, Loc, llvm::AtomicOrdering::AcquireRelease); in emitOMPAtomicCaptureExpr()
6463 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO, in emitOMPAtomicCompareExpr() argument
6468 CGF.CGM.getOpenMPRuntime().getOMPBuilder(); in emitOMPAtomicCompareExpr()
6486 LValue XLVal = CGF.EmitLValue(X); in emitOMPAtomicCompareExpr()
6489 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) { in emitOMPAtomicCompareExpr()
6491 return CGF.EmitScalarExpr(E); in emitOMPAtomicCompareExpr()
6493 llvm::Value *V = CGF.EmitScalarExpr(NewE); in emitOMPAtomicCompareExpr()
6496 return CGF.EmitScalarConversion(V, NewE->getType(), X->getType(), Loc); in emitOMPAtomicCompareExpr()
6502 EVal = CGF.Builder.CreateIntCast( in emitOMPAtomicCompareExpr()
6507 DVal = CGF.Builder.CreateIntCast( in emitOMPAtomicCompareExpr()
6512 XAddr.emitRawPointer(CGF), XAddr.getElementType(), in emitOMPAtomicCompareExpr()
6517 LValue LV = CGF.EmitLValue(V); in emitOMPAtomicCompareExpr()
6519 VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(), in emitOMPAtomicCompareExpr()
6524 LValue LV = CGF.EmitLValue(R); in emitOMPAtomicCompareExpr()
6526 ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(), in emitOMPAtomicCompareExpr()
6534 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare( in emitOMPAtomicCompareExpr()
6535 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr, in emitOMPAtomicCompareExpr()
6538 CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare( in emitOMPAtomicCompareExpr()
6539 CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr, in emitOMPAtomicCompareExpr()
6543 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, in emitOMPAtomicExpr() argument
6552 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc); in emitOMPAtomicExpr()
6555 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc); in emitOMPAtomicExpr()
6559 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc); in emitOMPAtomicExpr()
6562 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE, in emitOMPAtomicExpr()
6566 emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE, in emitOMPAtomicExpr()
6658 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, in emitCommonOMPTargetDirective() argument
6662 CodeGenModule &CGM = CGF.CGM; in emitCommonOMPTargetDirective()
6666 OMPLexicalScope Scope(CGF, S, OMPD_target); in emitCommonOMPTargetDirective()
6668 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonOMPTargetDirective() argument
6669 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in emitCommonOMPTargetDirective()
6674 auto LPCRegion = CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S); in emitCommonOMPTargetDirective()
6700 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val) in emitCommonOMPTargetDirective()
6713 assert(CGF.CurFuncDecl && "No parent declaration for target region!"); in emitCommonOMPTargetDirective()
6717 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
6719 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
6723 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl))); in emitCommonOMPTargetDirective()
6728 OMPLexicalScope Scope(CGF, S, OMPD_task); in emitCommonOMPTargetDirective()
6730 [IsOffloadEntry](CodeGenFunction &CGF, in emitCommonOMPTargetDirective()
6733 OMPLoopScope(CGF, D); in emitCommonOMPTargetDirective()
6735 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations()); in emitCommonOMPTargetDirective()
6736 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty, in emitCommonOMPTargetDirective()
6742 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device, in emitCommonOMPTargetDirective()
6746 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, in emitTargetRegion() argument
6748 Action.Enter(CGF); in emitTargetRegion()
6749 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetRegion()
6750 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetRegion()
6751 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetRegion()
6754 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetRegion()
6756 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); in emitTargetRegion()
6757 CGF.EnsureInsertPoint(); in emitTargetRegion()
6763 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDeviceFunction() argument
6764 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDeviceFunction()
6775 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDirective() argument
6776 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDirective()
6781 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, in emitCommonOMPTeamsDirective() argument
6787 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( in emitCommonOMPTeamsDirective()
6788 CGF, S, *CS->getCapturedDecl()->param_begin(), InnermostKind, in emitCommonOMPTeamsDirective()
6797 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit, in emitCommonOMPTeamsDirective()
6801 OMPTeamsScope Scope(CGF, S); in emitCommonOMPTeamsDirective()
6803 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPTeamsDirective()
6804 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn, in emitCommonOMPTeamsDirective()
6810 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTeamsDirective() argument
6811 Action.Enter(CGF); in EmitOMPTeamsDirective()
6812 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDirective()
6813 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
6814 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
6815 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDirective()
6817 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); in EmitOMPTeamsDirective()
6818 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDirective()
6825 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsRegion() argument
6828 Action.Enter(CGF); in emitTargetTeamsRegion()
6830 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetTeamsRegion() argument
6831 Action.Enter(CGF); in emitTargetTeamsRegion()
6832 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsRegion()
6833 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetTeamsRegion()
6834 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetTeamsRegion()
6835 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsRegion()
6838 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetTeamsRegion()
6839 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetTeamsRegion()
6840 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsRegion()
6842 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen); in emitTargetTeamsRegion()
6843 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsRegion()
6850 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDeviceFunction() argument
6851 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDeviceFunction()
6863 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDirective() argument
6864 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDirective()
6870 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeRegion() argument
6872 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
6873 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeRegion() argument
6874 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeRegion()
6878 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeRegion()
6880 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
6881 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeRegion()
6882 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeRegion()
6884 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeRegion()
6886 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeRegion()
6888 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); in emitTargetTeamsDistributeRegion()
6889 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeRegion()
6896 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDeviceFunction() argument
6897 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDeviceFunction()
6909 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDirective() argument
6910 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDirective()
6916 CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeSimdRegion() argument
6918 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
6919 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeSimdRegion() argument
6920 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeSimdRegion()
6924 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeSimdRegion()
6926 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
6927 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeSimdRegion()
6928 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeSimdRegion()
6930 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeSimdRegion()
6932 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeSimdRegion()
6934 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); in emitTargetTeamsDistributeSimdRegion()
6935 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeSimdRegion()
6942 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDeviceFunction() argument
6943 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDeviceFunction()
6955 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDirective() argument
6956 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDirective()
6964 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeDirective() argument
6965 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeDirective()
6969 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeDirective()
6971 Action.Enter(CGF); in EmitOMPTeamsDistributeDirective()
6972 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeDirective()
6973 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeDirective()
6975 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeDirective()
6977 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeDirective()
6986 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeSimdDirective() argument
6987 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeSimdDirective()
6991 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeSimdDirective()
6993 Action.Enter(CGF); in EmitOMPTeamsDistributeSimdDirective()
6994 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeSimdDirective()
6995 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeSimdDirective()
6997 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, in EmitOMPTeamsDistributeSimdDirective()
6999 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeSimdDirective()
7008 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForDirective() argument
7009 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForDirective()
7014 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForDirective()
7016 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForDirective()
7017 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForDirective()
7018 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForDirective()
7020 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeParallelForDirective()
7022 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForDirective()
7031 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForSimdDirective() argument
7032 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForSimdDirective()
7037 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForSimdDirective()
7039 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
7040 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
7041 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForSimdDirective()
7043 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in EmitOMPTeamsDistributeParallelForSimdDirective()
7044 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in EmitOMPTeamsDistributeParallelForSimdDirective()
7045 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForSimdDirective()
7124 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, in emitTargetTeamsDistributeParallelForRegion() argument
7126 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
7127 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForRegion() argument
7128 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForRegion()
7133 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForRegion()
7135 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
7136 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForRegion()
7137 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForRegion()
7139 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForRegion()
7140 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForRegion()
7141 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForRegion()
7144 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, in emitTargetTeamsDistributeParallelForRegion()
7146 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForRegion()
7155 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDeviceFunction() argument
7156 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDeviceFunction()
7168 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDirective() argument
7169 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDirective()
7175 CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion() argument
7178 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
7179 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForSimdRegion() argument
7180 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForSimdRegion()
7185 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion()
7187 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
7188 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
7189 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForSimdRegion()
7191 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForSimdRegion()
7192 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForSimdRegion()
7193 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForSimdRegion()
7196 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, in emitTargetTeamsDistributeParallelForSimdRegion()
7198 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForSimdRegion()
7207 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction() argument
7208 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction()
7220 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDirective() argument
7221 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDirective()
7391 void Enter(CodeGenFunction &CGF) override { in EmitOMPTargetDataDirective() argument
7397 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDataDirective() argument
7398 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTargetDataDirective() argument
7399 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTargetDataDirective()
7403 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDataDirective() argument
7409 Action.Enter(CGF); in EmitOMPTargetDataDirective()
7412 OMPPrivateScope PrivateScope(CGF); in EmitOMPTargetDataDirective()
7415 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
7418 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
7421 RCG(CGF); in EmitOMPTargetDataDirective()
7432 CGF.EmitVarDecl(*OED); in EmitOMPTargetDataDirective()
7438 CGF.EmitVarDecl(*OED); in EmitOMPTargetDataDirective()
7444 OMPLexicalScope Scope(CGF, S, CaptureRegion); in EmitOMPTargetDataDirective()
7445 RCG(CGF); in EmitOMPTargetDataDirective()
7456 OMPLexicalScope Scope(CGF, S); in EmitOMPTargetDataDirective()
7457 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, in EmitOMPTargetDataDirective()
7530 static void emitTargetParallelRegion(CodeGenFunction &CGF, in emitTargetParallelRegion() argument
7535 Action.Enter(CGF); in emitTargetParallelRegion()
7536 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelRegion() argument
7537 Action.Enter(CGF); in emitTargetParallelRegion()
7538 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetParallelRegion()
7539 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetParallelRegion()
7540 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetParallelRegion()
7541 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetParallelRegion()
7544 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetParallelRegion()
7546 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetParallelRegion()
7547 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in emitTargetParallelRegion()
7549 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen, in emitTargetParallelRegion()
7551 emitPostUpdateForReductionClause(CGF, S, in emitTargetParallelRegion()
7558 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDeviceFunction() argument
7559 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDeviceFunction()
7571 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDirective() argument
7572 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDirective()
7577 static void emitTargetParallelForRegion(CodeGenFunction &CGF, in emitTargetParallelForRegion() argument
7580 Action.Enter(CGF); in emitTargetParallelForRegion()
7583 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForRegion() argument
7584 Action.Enter(CGF); in emitTargetParallelForRegion()
7586 CGF, OMPD_target_parallel_for, S.hasCancel()); in emitTargetParallelForRegion()
7587 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForRegion()
7590 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, in emitTargetParallelForRegion()
7598 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDeviceFunction() argument
7599 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDeviceFunction()
7611 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDirective() argument
7612 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDirective()
7618 emitTargetParallelForSimdRegion(CodeGenFunction &CGF, in emitTargetParallelForSimdRegion() argument
7621 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
7624 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForSimdRegion() argument
7625 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
7626 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForSimdRegion()
7629 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, in emitTargetParallelForSimdRegion()
7637 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDeviceFunction() argument
7638 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDeviceFunction()
7650 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDirective() argument
7651 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDirective()
7657 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, in mapParam() argument
7661 Privates.addPrivate(VDecl, CGF.GetAddrOfLocalVar(PVD)); in mapParam()
7699 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7711 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
7712 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in EmitOMPTaskLoopBasedDirective()
7716 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); in EmitOMPTaskLoopBasedDirective()
7717 ContBlock = CGF.createBasicBlock("taskloop.if.end"); in EmitOMPTaskLoopBasedDirective()
7718 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in EmitOMPTaskLoopBasedDirective()
7719 CGF.getProfileCount(&S)); in EmitOMPTaskLoopBasedDirective()
7720 CGF.EmitBlock(ThenBlock); in EmitOMPTaskLoopBasedDirective()
7721 CGF.incrementProfileCounter(&S); in EmitOMPTaskLoopBasedDirective()
7724 (void)CGF.EmitOMPLinearClauseInit(S); in EmitOMPTaskLoopBasedDirective()
7726 OMPPrivateScope LoopScope(CGF); in EmitOMPTaskLoopBasedDirective()
7734 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP, in EmitOMPTaskLoopBasedDirective()
7736 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP, in EmitOMPTaskLoopBasedDirective()
7738 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope); in EmitOMPTaskLoopBasedDirective()
7739 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP, in EmitOMPTaskLoopBasedDirective()
7741 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
7742 CGF.EmitOMPLinearClause(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
7743 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
7748 CGF.EmitVarDecl(*IVDecl); in EmitOMPTaskLoopBasedDirective()
7749 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPTaskLoopBasedDirective()
7755 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in EmitOMPTaskLoopBasedDirective()
7757 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in EmitOMPTaskLoopBasedDirective()
7761 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false); in EmitOMPTaskLoopBasedDirective()
7763 CGF, S, in EmitOMPTaskLoopBasedDirective()
7764 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7766 CGF.EmitOMPSimdInit(S); in EmitOMPTaskLoopBasedDirective()
7768 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7769 CGF.EmitOMPInnerLoop( in EmitOMPTaskLoopBasedDirective()
7771 [&S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
7772 emitOMPLoopBodyWithStopPoint(CGF, S, in EmitOMPTaskLoopBasedDirective()
7780 CGF.EmitBranch(ContBlock); in EmitOMPTaskLoopBasedDirective()
7781 CGF.EmitBlock(ContBlock, true); in EmitOMPTaskLoopBasedDirective()
7785 CGF.EmitOMPLastprivateClauseFinal( in EmitOMPTaskLoopBasedDirective()
7787 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar( in EmitOMPTaskLoopBasedDirective()
7788 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
7792 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
7793 return CGF.Builder.CreateIsNotNull( in EmitOMPTaskLoopBasedDirective()
7794 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
7799 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskLoopBasedDirective()
7802 &Data](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
7803 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
7804 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S, in EmitOMPTaskLoopBasedDirective()
7808 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, in EmitOMPTaskLoopBasedDirective()
7816 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF, in EmitOMPTaskLoopBasedDirective()
7818 Action.Enter(CGF); in EmitOMPTaskLoopBasedDirective()
7819 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, in EmitOMPTaskLoopBasedDirective()
7842 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopDirective() argument
7843 Action.Enter(CGF); in EmitOMPMasterTaskLoopDirective()
7854 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopSimdDirective() argument
7855 Action.Enter(CGF); in EmitOMPMasterTaskLoopSimdDirective()
7866 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopDirective() argument
7867 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopDirective()
7869 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopDirective()
7870 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopDirective()
7872 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopDirective()
7873 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopDirective()
7884 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopSimdDirective() argument
7885 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopSimdDirective()
7887 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopSimdDirective()
7888 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopSimdDirective()
7890 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopSimdDirective()
7891 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopSimdDirective()
7925 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPGenericLoopDirective() argument
7931 OMPPrivateScope LoopScope(CGF); in EmitOMPGenericLoopDirective()
7932 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in EmitOMPGenericLoopDirective()
7934 CGF.EmitStmt(CS); in EmitOMPGenericLoopDirective()
7937 CGF.EmitStmt(CS); in EmitOMPGenericLoopDirective()
7948 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelGenericLoopDirective() argument
7949 Action.Enter(CGF); in EmitOMPParallelGenericLoopDirective()
7950 emitOMPCopyinClause(CGF, S); in EmitOMPParallelGenericLoopDirective()
7951 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPParallelGenericLoopDirective()
7967 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsGenericLoopDirective() argument
7968 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsGenericLoopDirective()
7972 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsGenericLoopDirective()
7974 Action.Enter(CGF); in EmitOMPTeamsGenericLoopDirective()
7975 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsGenericLoopDirective()
7976 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsGenericLoopDirective()
7978 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsGenericLoopDirective()
7980 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsGenericLoopDirective()
7988 static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF, in emitTargetTeamsLoopCodegenStatus() argument
7991 bool IsDevice = CGF.CGM.getLangOpts().OpenMPIsTargetDevice; in emitTargetTeamsLoopCodegenStatus()
7997 auto &SM = CGF.getContext().getSourceManager(); in emitTargetTeamsLoopCodegenStatus()
8007 CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsGenericLoopRegionAsParallel() argument
8009 Action.Enter(CGF); in emitTargetTeamsGenericLoopRegionAsParallel()
8012 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsGenericLoopRegionAsParallel() argument
8013 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsGenericLoopRegionAsParallel()
8018 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsGenericLoopRegionAsParallel()
8020 Action.Enter(CGF); in emitTargetTeamsGenericLoopRegionAsParallel()
8021 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsGenericLoopRegionAsParallel()
8022 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsGenericLoopRegionAsParallel()
8024 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsGenericLoopRegionAsParallel()
8025 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsGenericLoopRegionAsParallel()
8026 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsGenericLoopRegionAsParallel()
8030 CGF, TTL_CODEGEN_TYPE " as parallel for", S)); in emitTargetTeamsGenericLoopRegionAsParallel()
8031 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, in emitTargetTeamsGenericLoopRegionAsParallel()
8033 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsGenericLoopRegionAsParallel()
8038 CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsGenericLoopRegionAsDistribute() argument
8040 Action.Enter(CGF); in emitTargetTeamsGenericLoopRegionAsDistribute()
8042 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsGenericLoopRegionAsDistribute() argument
8043 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsGenericLoopRegionAsDistribute()
8047 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsGenericLoopRegionAsDistribute()
8049 Action.Enter(CGF); in emitTargetTeamsGenericLoopRegionAsDistribute()
8050 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsGenericLoopRegionAsDistribute()
8051 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsGenericLoopRegionAsDistribute()
8053 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsGenericLoopRegionAsDistribute()
8054 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsGenericLoopRegionAsDistribute()
8055 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsGenericLoopRegionAsDistribute()
8059 CGF, TTL_CODEGEN_TYPE " as distribute", S)); in emitTargetTeamsGenericLoopRegionAsDistribute()
8060 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); in emitTargetTeamsGenericLoopRegionAsDistribute()
8061 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsGenericLoopRegionAsDistribute()
8067 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsGenericLoopDirective() argument
8069 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S); in EmitOMPTargetTeamsGenericLoopDirective()
8071 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S); in EmitOMPTargetTeamsGenericLoopDirective()
8080 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsGenericLoopDeviceFunction() argument
8082 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S); in EmitOMPTargetTeamsGenericLoopDeviceFunction()
8084 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S); in EmitOMPTargetTeamsGenericLoopDeviceFunction()
8096 CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S, in emitTargetParallelGenericLoopRegion() argument
8098 Action.Enter(CGF); in emitTargetParallelGenericLoopRegion()
8100 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelGenericLoopRegion() argument
8101 Action.Enter(CGF); in emitTargetParallelGenericLoopRegion()
8103 CGF, OMPD_target_parallel_loop, /*hasCancel=*/false); in emitTargetParallelGenericLoopRegion()
8104 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelGenericLoopRegion()
8107 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, in emitTargetParallelGenericLoopRegion()
8115 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelGenericLoopDeviceFunction() argument
8116 emitTargetParallelGenericLoopRegion(CGF, S, Action); in EmitOMPTargetParallelGenericLoopDeviceFunction()
8130 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelGenericLoopDirective() argument
8131 emitTargetParallelGenericLoopRegion(CGF, S, Action); in EmitOMPTargetParallelGenericLoopDirective()
8144 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitSimpleOMPExecutableDirective() argument
8145 OMPPrivateScope GlobalsScope(CGF); in EmitSimpleOMPExecutableDirective()
8156 if (!CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
8157 LValue GlobLVal = CGF.EmitLValue(Ref); in EmitSimpleOMPExecutableDirective()
8165 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D); in EmitSimpleOMPExecutableDirective()
8166 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action); in EmitSimpleOMPExecutableDirective()
8171 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
8172 LValue GlobLVal = CGF.EmitLValue(E); in EmitSimpleOMPExecutableDirective()
8177 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
8178 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
8190 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
8191 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
8197 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); in EmitSimpleOMPExecutableDirective()