1 //===----- CGCoroutine.cpp - Emit LLVM Code for C++ coroutines ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This contains code dealing with C++ code generation of coroutines. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CGCleanup.h" 14 #include "CodeGenFunction.h" 15 #include "llvm/ADT/ScopeExit.h" 16 #include "clang/AST/StmtCXX.h" 17 #include "clang/AST/StmtVisitor.h" 18 19 using namespace clang; 20 using namespace CodeGen; 21 22 using llvm::Value; 23 using llvm::BasicBlock; 24 25 namespace { 26 enum class AwaitKind { Init, Normal, Yield, Final }; 27 static constexpr llvm::StringLiteral AwaitKindStr[] = {"init", "await", "yield", 28 "final"}; 29 } 30 31 struct clang::CodeGen::CGCoroData { 32 // What is the current await expression kind and how many 33 // await/yield expressions were encountered so far. 34 // These are used to generate pretty labels for await expressions in LLVM IR. 35 AwaitKind CurrentAwaitKind = AwaitKind::Init; 36 unsigned AwaitNum = 0; 37 unsigned YieldNum = 0; 38 39 // How many co_return statements are in the coroutine. Used to decide whether 40 // we need to add co_return; equivalent at the end of the user authored body. 41 unsigned CoreturnCount = 0; 42 43 // A branch to this block is emitted when coroutine needs to suspend. 44 llvm::BasicBlock *SuspendBB = nullptr; 45 46 // The promise type's 'unhandled_exception' handler, if it defines one. 47 Stmt *ExceptionHandler = nullptr; 48 49 // A temporary i1 alloca that stores whether 'await_resume' threw an 50 // exception. If it did, 'true' is stored in this variable, and the coroutine 51 // body must be skipped. If the promise type does not define an exception 52 // handler, this is null. 53 llvm::Value *ResumeEHVar = nullptr; 54 55 // Stores the jump destination just before the coroutine memory is freed. 56 // This is the destination that every suspend point jumps to for the cleanup 57 // branch. 58 CodeGenFunction::JumpDest CleanupJD; 59 60 // Stores the jump destination just before the final suspend. The co_return 61 // statements jumps to this point after calling return_xxx promise member. 62 CodeGenFunction::JumpDest FinalJD; 63 64 // Stores the llvm.coro.id emitted in the function so that we can supply it 65 // as the first argument to coro.begin, coro.alloc and coro.free intrinsics. 66 // Note: llvm.coro.id returns a token that cannot be directly expressed in a 67 // builtin. 68 llvm::CallInst *CoroId = nullptr; 69 70 // Stores the llvm.coro.begin emitted in the function so that we can replace 71 // all coro.frame intrinsics with direct SSA value of coro.begin that returns 72 // the address of the coroutine frame of the current coroutine. 73 llvm::CallInst *CoroBegin = nullptr; 74 75 // Stores the last emitted coro.free for the deallocate expressions, we use it 76 // to wrap dealloc code with if(auto mem = coro.free) dealloc(mem). 77 llvm::CallInst *LastCoroFree = nullptr; 78 79 // If coro.id came from the builtin, remember the expression to give better 80 // diagnostic. If CoroIdExpr is nullptr, the coro.id was created by 81 // EmitCoroutineBody. 82 CallExpr const *CoroIdExpr = nullptr; 83 }; 84 85 // Defining these here allows to keep CGCoroData private to this file. 86 clang::CodeGen::CodeGenFunction::CGCoroInfo::CGCoroInfo() {} 87 CodeGenFunction::CGCoroInfo::~CGCoroInfo() {} 88 89 static void createCoroData(CodeGenFunction &CGF, 90 CodeGenFunction::CGCoroInfo &CurCoro, 91 llvm::CallInst *CoroId, 92 CallExpr const *CoroIdExpr = nullptr) { 93 if (CurCoro.Data) { 94 if (CurCoro.Data->CoroIdExpr) 95 CGF.CGM.Error(CoroIdExpr->getBeginLoc(), 96 "only one __builtin_coro_id can be used in a function"); 97 else if (CoroIdExpr) 98 CGF.CGM.Error(CoroIdExpr->getBeginLoc(), 99 "__builtin_coro_id shall not be used in a C++ coroutine"); 100 else 101 llvm_unreachable("EmitCoroutineBodyStatement called twice?"); 102 103 return; 104 } 105 106 CurCoro.Data = std::unique_ptr<CGCoroData>(new CGCoroData); 107 CurCoro.Data->CoroId = CoroId; 108 CurCoro.Data->CoroIdExpr = CoroIdExpr; 109 } 110 111 // Synthesize a pretty name for a suspend point. 112 static SmallString<32> buildSuspendPrefixStr(CGCoroData &Coro, AwaitKind Kind) { 113 unsigned No = 0; 114 switch (Kind) { 115 case AwaitKind::Init: 116 case AwaitKind::Final: 117 break; 118 case AwaitKind::Normal: 119 No = ++Coro.AwaitNum; 120 break; 121 case AwaitKind::Yield: 122 No = ++Coro.YieldNum; 123 break; 124 } 125 SmallString<32> Prefix(AwaitKindStr[static_cast<unsigned>(Kind)]); 126 if (No > 1) { 127 Twine(No).toVector(Prefix); 128 } 129 return Prefix; 130 } 131 132 static bool memberCallExpressionCanThrow(const Expr *E) { 133 if (const auto *CE = dyn_cast<CXXMemberCallExpr>(E)) 134 if (const auto *Proto = 135 CE->getMethodDecl()->getType()->getAs<FunctionProtoType>()) 136 if (isNoexceptExceptionSpec(Proto->getExceptionSpecType()) && 137 Proto->canThrow() == CT_Cannot) 138 return false; 139 return true; 140 } 141 142 // Emit suspend expression which roughly looks like: 143 // 144 // auto && x = CommonExpr(); 145 // if (!x.await_ready()) { 146 // llvm_coro_save(); 147 // x.await_suspend(...); (*) 148 // llvm_coro_suspend(); (**) 149 // } 150 // x.await_resume(); 151 // 152 // where the result of the entire expression is the result of x.await_resume() 153 // 154 // (*) If x.await_suspend return type is bool, it allows to veto a suspend: 155 // if (x.await_suspend(...)) 156 // llvm_coro_suspend(); 157 // 158 // (**) llvm_coro_suspend() encodes three possible continuations as 159 // a switch instruction: 160 // 161 // %where-to = call i8 @llvm.coro.suspend(...) 162 // switch i8 %where-to, label %coro.ret [ ; jump to epilogue to suspend 163 // i8 0, label %yield.ready ; go here when resumed 164 // i8 1, label %yield.cleanup ; go here when destroyed 165 // ] 166 // 167 // See llvm's docs/Coroutines.rst for more details. 168 // 169 namespace { 170 struct LValueOrRValue { 171 LValue LV; 172 RValue RV; 173 }; 174 } 175 static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Coro, 176 CoroutineSuspendExpr const &S, 177 AwaitKind Kind, AggValueSlot aggSlot, 178 bool ignoreResult, bool forLValue) { 179 auto *E = S.getCommonExpr(); 180 181 auto Binder = 182 CodeGenFunction::OpaqueValueMappingData::bind(CGF, S.getOpaqueValue(), E); 183 auto UnbindOnExit = llvm::make_scope_exit([&] { Binder.unbind(CGF); }); 184 185 auto Prefix = buildSuspendPrefixStr(Coro, Kind); 186 BasicBlock *ReadyBlock = CGF.createBasicBlock(Prefix + Twine(".ready")); 187 BasicBlock *SuspendBlock = CGF.createBasicBlock(Prefix + Twine(".suspend")); 188 BasicBlock *CleanupBlock = CGF.createBasicBlock(Prefix + Twine(".cleanup")); 189 190 // If expression is ready, no need to suspend. 191 CGF.EmitBranchOnBoolExpr(S.getReadyExpr(), ReadyBlock, SuspendBlock, 0); 192 193 // Otherwise, emit suspend logic. 194 CGF.EmitBlock(SuspendBlock); 195 196 auto &Builder = CGF.Builder; 197 llvm::Function *CoroSave = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_save); 198 auto *NullPtr = llvm::ConstantPointerNull::get(CGF.CGM.Int8PtrTy); 199 auto *SaveCall = Builder.CreateCall(CoroSave, {NullPtr}); 200 201 auto *SuspendRet = CGF.EmitScalarExpr(S.getSuspendExpr()); 202 if (SuspendRet != nullptr && SuspendRet->getType()->isIntegerTy(1)) { 203 // Veto suspension if requested by bool returning await_suspend. 204 BasicBlock *RealSuspendBlock = 205 CGF.createBasicBlock(Prefix + Twine(".suspend.bool")); 206 CGF.Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock); 207 CGF.EmitBlock(RealSuspendBlock); 208 } 209 210 // Emit the suspend point. 211 const bool IsFinalSuspend = (Kind == AwaitKind::Final); 212 llvm::Function *CoroSuspend = 213 CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_suspend); 214 auto *SuspendResult = Builder.CreateCall( 215 CoroSuspend, {SaveCall, Builder.getInt1(IsFinalSuspend)}); 216 217 // Create a switch capturing three possible continuations. 218 auto *Switch = Builder.CreateSwitch(SuspendResult, Coro.SuspendBB, 2); 219 Switch->addCase(Builder.getInt8(0), ReadyBlock); 220 Switch->addCase(Builder.getInt8(1), CleanupBlock); 221 222 // Emit cleanup for this suspend point. 223 CGF.EmitBlock(CleanupBlock); 224 CGF.EmitBranchThroughCleanup(Coro.CleanupJD); 225 226 // Emit await_resume expression. 227 CGF.EmitBlock(ReadyBlock); 228 229 // Exception handling requires additional IR. If the 'await_resume' function 230 // is marked as 'noexcept', we avoid generating this additional IR. 231 CXXTryStmt *TryStmt = nullptr; 232 if (Coro.ExceptionHandler && Kind == AwaitKind::Init && 233 memberCallExpressionCanThrow(S.getResumeExpr())) { 234 Coro.ResumeEHVar = 235 CGF.CreateTempAlloca(Builder.getInt1Ty(), Prefix + Twine("resume.eh")); 236 Builder.CreateFlagStore(true, Coro.ResumeEHVar); 237 238 auto Loc = S.getResumeExpr()->getExprLoc(); 239 auto *Catch = new (CGF.getContext()) 240 CXXCatchStmt(Loc, /*exDecl=*/nullptr, Coro.ExceptionHandler); 241 auto *TryBody = CompoundStmt::Create(CGF.getContext(), S.getResumeExpr(), 242 FPOptionsOverride(), Loc, Loc); 243 TryStmt = CXXTryStmt::Create(CGF.getContext(), Loc, TryBody, Catch); 244 CGF.EnterCXXTryStmt(*TryStmt); 245 } 246 247 LValueOrRValue Res; 248 if (forLValue) 249 Res.LV = CGF.EmitLValue(S.getResumeExpr()); 250 else 251 Res.RV = CGF.EmitAnyExpr(S.getResumeExpr(), aggSlot, ignoreResult); 252 253 if (TryStmt) { 254 Builder.CreateFlagStore(false, Coro.ResumeEHVar); 255 CGF.ExitCXXTryStmt(*TryStmt); 256 } 257 258 return Res; 259 } 260 261 RValue CodeGenFunction::EmitCoawaitExpr(const CoawaitExpr &E, 262 AggValueSlot aggSlot, 263 bool ignoreResult) { 264 return emitSuspendExpression(*this, *CurCoro.Data, E, 265 CurCoro.Data->CurrentAwaitKind, aggSlot, 266 ignoreResult, /*forLValue*/false).RV; 267 } 268 RValue CodeGenFunction::EmitCoyieldExpr(const CoyieldExpr &E, 269 AggValueSlot aggSlot, 270 bool ignoreResult) { 271 return emitSuspendExpression(*this, *CurCoro.Data, E, AwaitKind::Yield, 272 aggSlot, ignoreResult, /*forLValue*/false).RV; 273 } 274 275 void CodeGenFunction::EmitCoreturnStmt(CoreturnStmt const &S) { 276 ++CurCoro.Data->CoreturnCount; 277 const Expr *RV = S.getOperand(); 278 if (RV && RV->getType()->isVoidType() && !isa<InitListExpr>(RV)) { 279 // Make sure to evaluate the non initlist expression of a co_return 280 // with a void expression for side effects. 281 RunCleanupsScope cleanupScope(*this); 282 EmitIgnoredExpr(RV); 283 } 284 EmitStmt(S.getPromiseCall()); 285 EmitBranchThroughCleanup(CurCoro.Data->FinalJD); 286 } 287 288 289 #ifndef NDEBUG 290 static QualType getCoroutineSuspendExprReturnType(const ASTContext &Ctx, 291 const CoroutineSuspendExpr *E) { 292 const auto *RE = E->getResumeExpr(); 293 // Is it possible for RE to be a CXXBindTemporaryExpr wrapping 294 // a MemberCallExpr? 295 assert(isa<CallExpr>(RE) && "unexpected suspend expression type"); 296 return cast<CallExpr>(RE)->getCallReturnType(Ctx); 297 } 298 #endif 299 300 LValue 301 CodeGenFunction::EmitCoawaitLValue(const CoawaitExpr *E) { 302 assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() && 303 "Can't have a scalar return unless the return type is a " 304 "reference type!"); 305 return emitSuspendExpression(*this, *CurCoro.Data, *E, 306 CurCoro.Data->CurrentAwaitKind, AggValueSlot::ignored(), 307 /*ignoreResult*/false, /*forLValue*/true).LV; 308 } 309 310 LValue 311 CodeGenFunction::EmitCoyieldLValue(const CoyieldExpr *E) { 312 assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() && 313 "Can't have a scalar return unless the return type is a " 314 "reference type!"); 315 return emitSuspendExpression(*this, *CurCoro.Data, *E, 316 AwaitKind::Yield, AggValueSlot::ignored(), 317 /*ignoreResult*/false, /*forLValue*/true).LV; 318 } 319 320 // Hunts for the parameter reference in the parameter copy/move declaration. 321 namespace { 322 struct GetParamRef : public StmtVisitor<GetParamRef> { 323 public: 324 DeclRefExpr *Expr = nullptr; 325 GetParamRef() {} 326 void VisitDeclRefExpr(DeclRefExpr *E) { 327 assert(Expr == nullptr && "multilple declref in param move"); 328 Expr = E; 329 } 330 void VisitStmt(Stmt *S) { 331 for (auto *C : S->children()) { 332 if (C) 333 Visit(C); 334 } 335 } 336 }; 337 } 338 339 // This class replaces references to parameters to their copies by changing 340 // the addresses in CGF.LocalDeclMap and restoring back the original values in 341 // its destructor. 342 343 namespace { 344 struct ParamReferenceReplacerRAII { 345 CodeGenFunction::DeclMapTy SavedLocals; 346 CodeGenFunction::DeclMapTy& LocalDeclMap; 347 348 ParamReferenceReplacerRAII(CodeGenFunction::DeclMapTy &LocalDeclMap) 349 : LocalDeclMap(LocalDeclMap) {} 350 351 void addCopy(DeclStmt const *PM) { 352 // Figure out what param it refers to. 353 354 assert(PM->isSingleDecl()); 355 VarDecl const*VD = static_cast<VarDecl const*>(PM->getSingleDecl()); 356 Expr const *InitExpr = VD->getInit(); 357 GetParamRef Visitor; 358 Visitor.Visit(const_cast<Expr*>(InitExpr)); 359 assert(Visitor.Expr); 360 DeclRefExpr *DREOrig = Visitor.Expr; 361 auto *PD = DREOrig->getDecl(); 362 363 auto it = LocalDeclMap.find(PD); 364 assert(it != LocalDeclMap.end() && "parameter is not found"); 365 SavedLocals.insert({ PD, it->second }); 366 367 auto copyIt = LocalDeclMap.find(VD); 368 assert(copyIt != LocalDeclMap.end() && "parameter copy is not found"); 369 it->second = copyIt->getSecond(); 370 } 371 372 ~ParamReferenceReplacerRAII() { 373 for (auto&& SavedLocal : SavedLocals) { 374 LocalDeclMap.insert({SavedLocal.first, SavedLocal.second}); 375 } 376 } 377 }; 378 } 379 380 // For WinEH exception representation backend needs to know what funclet coro.end 381 // belongs to. That information is passed in a funclet bundle. 382 static SmallVector<llvm::OperandBundleDef, 1> 383 getBundlesForCoroEnd(CodeGenFunction &CGF) { 384 SmallVector<llvm::OperandBundleDef, 1> BundleList; 385 386 if (llvm::Instruction *EHPad = CGF.CurrentFuncletPad) 387 BundleList.emplace_back("funclet", EHPad); 388 389 return BundleList; 390 } 391 392 namespace { 393 // We will insert coro.end to cut any of the destructors for objects that 394 // do not need to be destroyed once the coroutine is resumed. 395 // See llvm/docs/Coroutines.rst for more details about coro.end. 396 struct CallCoroEnd final : public EHScopeStack::Cleanup { 397 void Emit(CodeGenFunction &CGF, Flags flags) override { 398 auto &CGM = CGF.CGM; 399 auto *NullPtr = llvm::ConstantPointerNull::get(CGF.Int8PtrTy); 400 llvm::Function *CoroEndFn = CGM.getIntrinsic(llvm::Intrinsic::coro_end); 401 // See if we have a funclet bundle to associate coro.end with. (WinEH) 402 auto Bundles = getBundlesForCoroEnd(CGF); 403 auto *CoroEnd = CGF.Builder.CreateCall( 404 CoroEndFn, {NullPtr, CGF.Builder.getTrue()}, Bundles); 405 if (Bundles.empty()) { 406 // Otherwise, (landingpad model), create a conditional branch that leads 407 // either to a cleanup block or a block with EH resume instruction. 408 auto *ResumeBB = CGF.getEHResumeBlock(/*isCleanup=*/true); 409 auto *CleanupContBB = CGF.createBasicBlock("cleanup.cont"); 410 CGF.Builder.CreateCondBr(CoroEnd, ResumeBB, CleanupContBB); 411 CGF.EmitBlock(CleanupContBB); 412 } 413 } 414 }; 415 } 416 417 namespace { 418 // Make sure to call coro.delete on scope exit. 419 struct CallCoroDelete final : public EHScopeStack::Cleanup { 420 Stmt *Deallocate; 421 422 // Emit "if (coro.free(CoroId, CoroBegin)) Deallocate;" 423 424 // Note: That deallocation will be emitted twice: once for a normal exit and 425 // once for exceptional exit. This usage is safe because Deallocate does not 426 // contain any declarations. The SubStmtBuilder::makeNewAndDeleteExpr() 427 // builds a single call to a deallocation function which is safe to emit 428 // multiple times. 429 void Emit(CodeGenFunction &CGF, Flags) override { 430 // Remember the current point, as we are going to emit deallocation code 431 // first to get to coro.free instruction that is an argument to a delete 432 // call. 433 BasicBlock *SaveInsertBlock = CGF.Builder.GetInsertBlock(); 434 435 auto *FreeBB = CGF.createBasicBlock("coro.free"); 436 CGF.EmitBlock(FreeBB); 437 CGF.EmitStmt(Deallocate); 438 439 auto *AfterFreeBB = CGF.createBasicBlock("after.coro.free"); 440 CGF.EmitBlock(AfterFreeBB); 441 442 // We should have captured coro.free from the emission of deallocate. 443 auto *CoroFree = CGF.CurCoro.Data->LastCoroFree; 444 if (!CoroFree) { 445 CGF.CGM.Error(Deallocate->getBeginLoc(), 446 "Deallocation expressoin does not refer to coro.free"); 447 return; 448 } 449 450 // Get back to the block we were originally and move coro.free there. 451 auto *InsertPt = SaveInsertBlock->getTerminator(); 452 CoroFree->moveBefore(InsertPt); 453 CGF.Builder.SetInsertPoint(InsertPt); 454 455 // Add if (auto *mem = coro.free) Deallocate; 456 auto *NullPtr = llvm::ConstantPointerNull::get(CGF.Int8PtrTy); 457 auto *Cond = CGF.Builder.CreateICmpNE(CoroFree, NullPtr); 458 CGF.Builder.CreateCondBr(Cond, FreeBB, AfterFreeBB); 459 460 // No longer need old terminator. 461 InsertPt->eraseFromParent(); 462 CGF.Builder.SetInsertPoint(AfterFreeBB); 463 } 464 explicit CallCoroDelete(Stmt *DeallocStmt) : Deallocate(DeallocStmt) {} 465 }; 466 } 467 468 static void emitBodyAndFallthrough(CodeGenFunction &CGF, 469 const CoroutineBodyStmt &S, Stmt *Body) { 470 CGF.EmitStmt(Body); 471 const bool CanFallthrough = CGF.Builder.GetInsertBlock(); 472 if (CanFallthrough) 473 if (Stmt *OnFallthrough = S.getFallthroughHandler()) 474 CGF.EmitStmt(OnFallthrough); 475 } 476 477 void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) { 478 auto *NullPtr = llvm::ConstantPointerNull::get(Builder.getInt8PtrTy()); 479 auto &TI = CGM.getContext().getTargetInfo(); 480 unsigned NewAlign = TI.getNewAlign() / TI.getCharWidth(); 481 482 auto *EntryBB = Builder.GetInsertBlock(); 483 auto *AllocBB = createBasicBlock("coro.alloc"); 484 auto *InitBB = createBasicBlock("coro.init"); 485 auto *FinalBB = createBasicBlock("coro.final"); 486 auto *RetBB = createBasicBlock("coro.ret"); 487 488 auto *CoroId = Builder.CreateCall( 489 CGM.getIntrinsic(llvm::Intrinsic::coro_id), 490 {Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr}); 491 createCoroData(*this, CurCoro, CoroId); 492 CurCoro.Data->SuspendBB = RetBB; 493 assert(ShouldEmitLifetimeMarkers && 494 "Must emit lifetime intrinsics for coroutines"); 495 496 // Backend is allowed to elide memory allocations, to help it, emit 497 // auto mem = coro.alloc() ? 0 : ... allocation code ...; 498 auto *CoroAlloc = Builder.CreateCall( 499 CGM.getIntrinsic(llvm::Intrinsic::coro_alloc), {CoroId}); 500 501 Builder.CreateCondBr(CoroAlloc, AllocBB, InitBB); 502 503 EmitBlock(AllocBB); 504 auto *AllocateCall = EmitScalarExpr(S.getAllocate()); 505 auto *AllocOrInvokeContBB = Builder.GetInsertBlock(); 506 507 // Handle allocation failure if 'ReturnStmtOnAllocFailure' was provided. 508 if (auto *RetOnAllocFailure = S.getReturnStmtOnAllocFailure()) { 509 auto *RetOnFailureBB = createBasicBlock("coro.ret.on.failure"); 510 511 // See if allocation was successful. 512 auto *NullPtr = llvm::ConstantPointerNull::get(Int8PtrTy); 513 auto *Cond = Builder.CreateICmpNE(AllocateCall, NullPtr); 514 Builder.CreateCondBr(Cond, InitBB, RetOnFailureBB); 515 516 // If not, return OnAllocFailure object. 517 EmitBlock(RetOnFailureBB); 518 EmitStmt(RetOnAllocFailure); 519 } 520 else { 521 Builder.CreateBr(InitBB); 522 } 523 524 EmitBlock(InitBB); 525 526 // Pass the result of the allocation to coro.begin. 527 auto *Phi = Builder.CreatePHI(VoidPtrTy, 2); 528 Phi->addIncoming(NullPtr, EntryBB); 529 Phi->addIncoming(AllocateCall, AllocOrInvokeContBB); 530 auto *CoroBegin = Builder.CreateCall( 531 CGM.getIntrinsic(llvm::Intrinsic::coro_begin), {CoroId, Phi}); 532 CurCoro.Data->CoroBegin = CoroBegin; 533 534 CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB); 535 { 536 CGDebugInfo *DI = getDebugInfo(); 537 ParamReferenceReplacerRAII ParamReplacer(LocalDeclMap); 538 CodeGenFunction::RunCleanupsScope ResumeScope(*this); 539 EHStack.pushCleanup<CallCoroDelete>(NormalAndEHCleanup, S.getDeallocate()); 540 541 // Create mapping between parameters and copy-params for coroutine function. 542 llvm::ArrayRef<const Stmt *> ParamMoves = S.getParamMoves(); 543 assert( 544 (ParamMoves.size() == 0 || (ParamMoves.size() == FnArgs.size())) && 545 "ParamMoves and FnArgs should be the same size for coroutine function"); 546 if (ParamMoves.size() == FnArgs.size() && DI) 547 for (const auto Pair : llvm::zip(FnArgs, ParamMoves)) 548 DI->getCoroutineParameterMappings().insert( 549 {std::get<0>(Pair), std::get<1>(Pair)}); 550 551 // Create parameter copies. We do it before creating a promise, since an 552 // evolution of coroutine TS may allow promise constructor to observe 553 // parameter copies. 554 for (auto *PM : S.getParamMoves()) { 555 EmitStmt(PM); 556 ParamReplacer.addCopy(cast<DeclStmt>(PM)); 557 // TODO: if(CoroParam(...)) need to surround ctor and dtor 558 // for the copy, so that llvm can elide it if the copy is 559 // not needed. 560 } 561 562 EmitStmt(S.getPromiseDeclStmt()); 563 564 Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl()); 565 auto *PromiseAddrVoidPtr = 566 new llvm::BitCastInst(PromiseAddr.getPointer(), VoidPtrTy, "", CoroId); 567 // Update CoroId to refer to the promise. We could not do it earlier because 568 // promise local variable was not emitted yet. 569 CoroId->setArgOperand(1, PromiseAddrVoidPtr); 570 571 // ReturnValue should be valid as long as the coroutine's return type 572 // is not void. The assertion could help us to reduce the check later. 573 assert(ReturnValue.isValid() == (bool)S.getReturnStmt()); 574 // Now we have the promise, initialize the GRO. 575 // We need to emit `get_return_object` first. According to: 576 // [dcl.fct.def.coroutine]p7 577 // The call to get_return_object is sequenced before the call to 578 // initial_suspend and is invoked at most once. 579 // 580 // So we couldn't emit return value when we emit return statment, 581 // otherwise the call to get_return_object wouldn't be in front 582 // of initial_suspend. 583 if (ReturnValue.isValid()) { 584 EmitAnyExprToMem(S.getReturnValue(), ReturnValue, 585 S.getReturnValue()->getType().getQualifiers(), 586 /*IsInit*/ true); 587 } 588 589 EHStack.pushCleanup<CallCoroEnd>(EHCleanup); 590 591 CurCoro.Data->CurrentAwaitKind = AwaitKind::Init; 592 CurCoro.Data->ExceptionHandler = S.getExceptionHandler(); 593 EmitStmt(S.getInitSuspendStmt()); 594 CurCoro.Data->FinalJD = getJumpDestInCurrentScope(FinalBB); 595 596 CurCoro.Data->CurrentAwaitKind = AwaitKind::Normal; 597 598 if (CurCoro.Data->ExceptionHandler) { 599 // If we generated IR to record whether an exception was thrown from 600 // 'await_resume', then use that IR to determine whether the coroutine 601 // body should be skipped. 602 // If we didn't generate the IR (perhaps because 'await_resume' was marked 603 // as 'noexcept'), then we skip this check. 604 BasicBlock *ContBB = nullptr; 605 if (CurCoro.Data->ResumeEHVar) { 606 BasicBlock *BodyBB = createBasicBlock("coro.resumed.body"); 607 ContBB = createBasicBlock("coro.resumed.cont"); 608 Value *SkipBody = Builder.CreateFlagLoad(CurCoro.Data->ResumeEHVar, 609 "coro.resumed.eh"); 610 Builder.CreateCondBr(SkipBody, ContBB, BodyBB); 611 EmitBlock(BodyBB); 612 } 613 614 auto Loc = S.getBeginLoc(); 615 CXXCatchStmt Catch(Loc, /*exDecl=*/nullptr, 616 CurCoro.Data->ExceptionHandler); 617 auto *TryStmt = 618 CXXTryStmt::Create(getContext(), Loc, S.getBody(), &Catch); 619 620 EnterCXXTryStmt(*TryStmt); 621 emitBodyAndFallthrough(*this, S, TryStmt->getTryBlock()); 622 ExitCXXTryStmt(*TryStmt); 623 624 if (ContBB) 625 EmitBlock(ContBB); 626 } 627 else { 628 emitBodyAndFallthrough(*this, S, S.getBody()); 629 } 630 631 // See if we need to generate final suspend. 632 const bool CanFallthrough = Builder.GetInsertBlock(); 633 const bool HasCoreturns = CurCoro.Data->CoreturnCount > 0; 634 if (CanFallthrough || HasCoreturns) { 635 EmitBlock(FinalBB); 636 CurCoro.Data->CurrentAwaitKind = AwaitKind::Final; 637 EmitStmt(S.getFinalSuspendStmt()); 638 } else { 639 // We don't need FinalBB. Emit it to make sure the block is deleted. 640 EmitBlock(FinalBB, /*IsFinished=*/true); 641 } 642 } 643 644 EmitBlock(RetBB); 645 // Emit coro.end before getReturnStmt (and parameter destructors), since 646 // resume and destroy parts of the coroutine should not include them. 647 llvm::Function *CoroEnd = CGM.getIntrinsic(llvm::Intrinsic::coro_end); 648 Builder.CreateCall(CoroEnd, {NullPtr, Builder.getFalse()}); 649 650 if (Stmt *Ret = S.getReturnStmt()) { 651 // Since we already emitted the return value above, so we shouldn't 652 // emit it again here. 653 cast<ReturnStmt>(Ret)->setRetValue(nullptr); 654 EmitStmt(Ret); 655 } 656 657 // LLVM require the frontend to mark the coroutine. 658 CurFn->setPresplitCoroutine(); 659 } 660 661 // Emit coroutine intrinsic and patch up arguments of the token type. 662 RValue CodeGenFunction::EmitCoroutineIntrinsic(const CallExpr *E, 663 unsigned int IID) { 664 SmallVector<llvm::Value *, 8> Args; 665 switch (IID) { 666 default: 667 break; 668 // The coro.frame builtin is replaced with an SSA value of the coro.begin 669 // intrinsic. 670 case llvm::Intrinsic::coro_frame: { 671 if (CurCoro.Data && CurCoro.Data->CoroBegin) { 672 return RValue::get(CurCoro.Data->CoroBegin); 673 } 674 CGM.Error(E->getBeginLoc(), "this builtin expect that __builtin_coro_begin " 675 "has been used earlier in this function"); 676 auto *NullPtr = llvm::ConstantPointerNull::get(Builder.getInt8PtrTy()); 677 return RValue::get(NullPtr); 678 } 679 case llvm::Intrinsic::coro_size: { 680 auto &Context = getContext(); 681 CanQualType SizeTy = Context.getSizeType(); 682 llvm::IntegerType *T = Builder.getIntNTy(Context.getTypeSize(SizeTy)); 683 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::coro_size, T); 684 return RValue::get(Builder.CreateCall(F)); 685 } 686 case llvm::Intrinsic::coro_align: { 687 auto &Context = getContext(); 688 CanQualType SizeTy = Context.getSizeType(); 689 llvm::IntegerType *T = Builder.getIntNTy(Context.getTypeSize(SizeTy)); 690 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::coro_align, T); 691 return RValue::get(Builder.CreateCall(F)); 692 } 693 // The following three intrinsics take a token parameter referring to a token 694 // returned by earlier call to @llvm.coro.id. Since we cannot represent it in 695 // builtins, we patch it up here. 696 case llvm::Intrinsic::coro_alloc: 697 case llvm::Intrinsic::coro_begin: 698 case llvm::Intrinsic::coro_free: { 699 if (CurCoro.Data && CurCoro.Data->CoroId) { 700 Args.push_back(CurCoro.Data->CoroId); 701 break; 702 } 703 CGM.Error(E->getBeginLoc(), "this builtin expect that __builtin_coro_id has" 704 " been used earlier in this function"); 705 // Fallthrough to the next case to add TokenNone as the first argument. 706 [[fallthrough]]; 707 } 708 // @llvm.coro.suspend takes a token parameter. Add token 'none' as the first 709 // argument. 710 case llvm::Intrinsic::coro_suspend: 711 Args.push_back(llvm::ConstantTokenNone::get(getLLVMContext())); 712 break; 713 } 714 for (const Expr *Arg : E->arguments()) 715 Args.push_back(EmitScalarExpr(Arg)); 716 717 llvm::Function *F = CGM.getIntrinsic(IID); 718 llvm::CallInst *Call = Builder.CreateCall(F, Args); 719 720 // Note: The following code is to enable to emit coro.id and coro.begin by 721 // hand to experiment with coroutines in C. 722 // If we see @llvm.coro.id remember it in the CoroData. We will update 723 // coro.alloc, coro.begin and coro.free intrinsics to refer to it. 724 if (IID == llvm::Intrinsic::coro_id) { 725 createCoroData(*this, CurCoro, Call, E); 726 } 727 else if (IID == llvm::Intrinsic::coro_begin) { 728 if (CurCoro.Data) 729 CurCoro.Data->CoroBegin = Call; 730 } 731 else if (IID == llvm::Intrinsic::coro_free) { 732 // Remember the last coro_free as we need it to build the conditional 733 // deletion of the coroutine frame. 734 if (CurCoro.Data) 735 CurCoro.Data->LastCoroFree = Call; 736 } 737 return RValue::get(Call); 738 } 739