/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGCoroutine.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===----- CGCoroutine.cpp - Emit LLVM Code for C++ coroutines ------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // This contains code dealing with C++ code generation of coroutines. |
10 | | // |
11 | | //===----------------------------------------------------------------------===// |
12 | | |
13 | | #include "CGCleanup.h" |
14 | | #include "CodeGenFunction.h" |
15 | | #include "llvm/ADT/ScopeExit.h" |
16 | | #include "clang/AST/StmtCXX.h" |
17 | | #include "clang/AST/StmtVisitor.h" |
18 | | |
19 | | using namespace clang; |
20 | | using namespace CodeGen; |
21 | | |
22 | | using llvm::Value; |
23 | | using llvm::BasicBlock; |
24 | | |
25 | | namespace { |
26 | | enum class AwaitKind { Init, Normal, Yield, Final }; |
27 | | static constexpr llvm::StringLiteral AwaitKindStr[] = {"init", "await", "yield", |
28 | | "final"}; |
29 | | } |
30 | | |
31 | | struct clang::CodeGen::CGCoroData { |
32 | | // What is the current await expression kind and how many |
33 | | // await/yield expressions were encountered so far. |
34 | | // These are used to generate pretty labels for await expressions in LLVM IR. |
35 | | AwaitKind CurrentAwaitKind = AwaitKind::Init; |
36 | | unsigned AwaitNum = 0; |
37 | | unsigned YieldNum = 0; |
38 | | |
39 | | // How many co_return statements are in the coroutine. Used to decide whether |
40 | | // we need to add co_return; equivalent at the end of the user authored body. |
41 | | unsigned CoreturnCount = 0; |
42 | | |
43 | | // A branch to this block is emitted when coroutine needs to suspend. |
44 | | llvm::BasicBlock *SuspendBB = nullptr; |
45 | | |
46 | | // The promise type's 'unhandled_exception' handler, if it defines one. |
47 | | Stmt *ExceptionHandler = nullptr; |
48 | | |
49 | | // A temporary i1 alloca that stores whether 'await_resume' threw an |
50 | | // exception. If it did, 'true' is stored in this variable, and the coroutine |
51 | | // body must be skipped. If the promise type does not define an exception |
52 | | // handler, this is null. |
53 | | llvm::Value *ResumeEHVar = nullptr; |
54 | | |
55 | | // Stores the jump destination just before the coroutine memory is freed. |
56 | | // This is the destination that every suspend point jumps to for the cleanup |
57 | | // branch. |
58 | | CodeGenFunction::JumpDest CleanupJD; |
59 | | |
60 | | // Stores the jump destination just before the final suspend. The co_return |
61 | | // statements jumps to this point after calling return_xxx promise member. |
62 | | CodeGenFunction::JumpDest FinalJD; |
63 | | |
64 | | // Stores the llvm.coro.id emitted in the function so that we can supply it |
65 | | // as the first argument to coro.begin, coro.alloc and coro.free intrinsics. |
66 | | // Note: llvm.coro.id returns a token that cannot be directly expressed in a |
67 | | // builtin. |
68 | | llvm::CallInst *CoroId = nullptr; |
69 | | |
70 | | // Stores the llvm.coro.begin emitted in the function so that we can replace |
71 | | // all coro.frame intrinsics with direct SSA value of coro.begin that returns |
72 | | // the address of the coroutine frame of the current coroutine. |
73 | | llvm::CallInst *CoroBegin = nullptr; |
74 | | |
75 | | // Stores the last emitted coro.free for the deallocate expressions, we use it |
76 | | // to wrap dealloc code with if(auto mem = coro.free) dealloc(mem). |
77 | | llvm::CallInst *LastCoroFree = nullptr; |
78 | | |
79 | | // If coro.id came from the builtin, remember the expression to give better |
80 | | // diagnostic. If CoroIdExpr is nullptr, the coro.id was created by |
81 | | // EmitCoroutineBody. |
82 | | CallExpr const *CoroIdExpr = nullptr; |
83 | | }; |
84 | | |
85 | | // Defining these here allows to keep CGCoroData private to this file. |
86 | 315k | clang::CodeGen::CodeGenFunction::CGCoroInfo::CGCoroInfo() {} |
87 | 315k | CodeGenFunction::CGCoroInfo::~CGCoroInfo() {} |
88 | | |
89 | | static void createCoroData(CodeGenFunction &CGF, |
90 | | CodeGenFunction::CGCoroInfo &CurCoro, |
91 | | llvm::CallInst *CoroId, |
92 | 121 | CallExpr const *CoroIdExpr = nullptr) { |
93 | 121 | if (CurCoro.Data) { |
94 | 1 | if (CurCoro.Data->CoroIdExpr) |
95 | 1 | CGF.CGM.Error(CoroIdExpr->getBeginLoc(), |
96 | 1 | "only one __builtin_coro_id can be used in a function"); |
97 | 0 | else if (CoroIdExpr) |
98 | 0 | CGF.CGM.Error(CoroIdExpr->getBeginLoc(), |
99 | 0 | "__builtin_coro_id shall not be used in a C++ coroutine"); |
100 | 0 | else |
101 | 0 | llvm_unreachable("EmitCoroutineBodyStatement called twice?"); |
102 | | |
103 | 1 | return; |
104 | 1 | } |
105 | | |
106 | 120 | CurCoro.Data = std::unique_ptr<CGCoroData>(new CGCoroData); |
107 | 120 | CurCoro.Data->CoroId = CoroId; |
108 | 120 | CurCoro.Data->CoroIdExpr = CoroIdExpr; |
109 | 120 | } |
110 | | |
111 | | // Synthesize a pretty name for a suspend point. |
112 | 290 | static SmallString<32> buildSuspendPrefixStr(CGCoroData &Coro, AwaitKind Kind) { |
113 | 290 | unsigned No = 0; |
114 | 290 | switch (Kind) { |
115 | 118 | case AwaitKind::Init: |
116 | 231 | case AwaitKind::Final: |
117 | 231 | break; |
118 | 51 | case AwaitKind::Normal: |
119 | 51 | No = ++Coro.AwaitNum; |
120 | 51 | break; |
121 | 8 | case AwaitKind::Yield: |
122 | 8 | No = ++Coro.YieldNum; |
123 | 8 | break; |
124 | 290 | } |
125 | 290 | SmallString<32> Prefix(AwaitKindStr[static_cast<unsigned>(Kind)]); |
126 | 290 | if (No > 1) { |
127 | 18 | Twine(No).toVector(Prefix); |
128 | 18 | } |
129 | 290 | return Prefix; |
130 | 290 | } |
131 | | |
132 | 22 | static bool memberCallExpressionCanThrow(const Expr *E) { |
133 | 22 | if (const auto *CE = dyn_cast<CXXMemberCallExpr>(E)) |
134 | 22 | if (const auto *Proto = |
135 | 22 | CE->getMethodDecl()->getType()->getAs<FunctionProtoType>()) |
136 | 22 | if (isNoexceptExceptionSpec(Proto->getExceptionSpecType()) && |
137 | 22 | Proto->canThrow() == CT_Cannot19 ) |
138 | 19 | return false; |
139 | 3 | return true; |
140 | 22 | } |
141 | | |
142 | | // Emit suspend expression which roughly looks like: |
143 | | // |
144 | | // auto && x = CommonExpr(); |
145 | | // if (!x.await_ready()) { |
146 | | // llvm_coro_save(); |
147 | | // x.await_suspend(...); (*) |
148 | | // llvm_coro_suspend(); (**) |
149 | | // } |
150 | | // x.await_resume(); |
151 | | // |
152 | | // where the result of the entire expression is the result of x.await_resume() |
153 | | // |
154 | | // (*) If x.await_suspend return type is bool, it allows to veto a suspend: |
155 | | // if (x.await_suspend(...)) |
156 | | // llvm_coro_suspend(); |
157 | | // |
158 | | // (**) llvm_coro_suspend() encodes three possible continuations as |
159 | | // a switch instruction: |
160 | | // |
161 | | // %where-to = call i8 @llvm.coro.suspend(...) |
162 | | // switch i8 %where-to, label %coro.ret [ ; jump to epilogue to suspend |
163 | | // i8 0, label %yield.ready ; go here when resumed |
164 | | // i8 1, label %yield.cleanup ; go here when destroyed |
165 | | // ] |
166 | | // |
167 | | // See llvm's docs/Coroutines.rst for more details. |
168 | | // |
169 | | namespace { |
170 | | struct LValueOrRValue { |
171 | | LValue LV; |
172 | | RValue RV; |
173 | | }; |
174 | | } |
175 | | static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Coro, |
176 | | CoroutineSuspendExpr const &S, |
177 | | AwaitKind Kind, AggValueSlot aggSlot, |
178 | 290 | bool ignoreResult, bool forLValue) { |
179 | 290 | auto *E = S.getCommonExpr(); |
180 | | |
181 | 290 | auto Binder = |
182 | 290 | CodeGenFunction::OpaqueValueMappingData::bind(CGF, S.getOpaqueValue(), E); |
183 | 290 | auto UnbindOnExit = llvm::make_scope_exit([&] { Binder.unbind(CGF); }); |
184 | | |
185 | 290 | auto Prefix = buildSuspendPrefixStr(Coro, Kind); |
186 | 290 | BasicBlock *ReadyBlock = CGF.createBasicBlock(Prefix + Twine(".ready")); |
187 | 290 | BasicBlock *SuspendBlock = CGF.createBasicBlock(Prefix + Twine(".suspend")); |
188 | 290 | BasicBlock *CleanupBlock = CGF.createBasicBlock(Prefix + Twine(".cleanup")); |
189 | | |
190 | | // If expression is ready, no need to suspend. |
191 | 290 | CGF.EmitBranchOnBoolExpr(S.getReadyExpr(), ReadyBlock, SuspendBlock, 0); |
192 | | |
193 | | // Otherwise, emit suspend logic. |
194 | 290 | CGF.EmitBlock(SuspendBlock); |
195 | | |
196 | 290 | auto &Builder = CGF.Builder; |
197 | 290 | llvm::Function *CoroSave = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_save); |
198 | 290 | auto *NullPtr = llvm::ConstantPointerNull::get(CGF.CGM.Int8PtrTy); |
199 | 290 | auto *SaveCall = Builder.CreateCall(CoroSave, {NullPtr}); |
200 | | |
201 | 290 | auto *SuspendRet = CGF.EmitScalarExpr(S.getSuspendExpr()); |
202 | 290 | if (SuspendRet != nullptr && SuspendRet->getType()->isIntegerTy(1)20 ) { |
203 | | // Veto suspension if requested by bool returning await_suspend. |
204 | 3 | BasicBlock *RealSuspendBlock = |
205 | 3 | CGF.createBasicBlock(Prefix + Twine(".suspend.bool")); |
206 | 3 | CGF.Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock); |
207 | 3 | CGF.EmitBlock(RealSuspendBlock); |
208 | 3 | } |
209 | | |
210 | | // Emit the suspend point. |
211 | 290 | const bool IsFinalSuspend = (Kind == AwaitKind::Final); |
212 | 290 | llvm::Function *CoroSuspend = |
213 | 290 | CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_suspend); |
214 | 290 | auto *SuspendResult = Builder.CreateCall( |
215 | 290 | CoroSuspend, {SaveCall, Builder.getInt1(IsFinalSuspend)}); |
216 | | |
217 | | // Create a switch capturing three possible continuations. |
218 | 290 | auto *Switch = Builder.CreateSwitch(SuspendResult, Coro.SuspendBB, 2); |
219 | 290 | Switch->addCase(Builder.getInt8(0), ReadyBlock); |
220 | 290 | Switch->addCase(Builder.getInt8(1), CleanupBlock); |
221 | | |
222 | | // Emit cleanup for this suspend point. |
223 | 290 | CGF.EmitBlock(CleanupBlock); |
224 | 290 | CGF.EmitBranchThroughCleanup(Coro.CleanupJD); |
225 | | |
226 | | // Emit await_resume expression. |
227 | 290 | CGF.EmitBlock(ReadyBlock); |
228 | | |
229 | | // Exception handling requires additional IR. If the 'await_resume' function |
230 | | // is marked as 'noexcept', we avoid generating this additional IR. |
231 | 290 | CXXTryStmt *TryStmt = nullptr; |
232 | 290 | if (Coro.ExceptionHandler && Kind == AwaitKind::Init46 && |
233 | 290 | memberCallExpressionCanThrow(S.getResumeExpr())22 ) { |
234 | 3 | Coro.ResumeEHVar = |
235 | 3 | CGF.CreateTempAlloca(Builder.getInt1Ty(), Prefix + Twine("resume.eh")); |
236 | 3 | Builder.CreateFlagStore(true, Coro.ResumeEHVar); |
237 | | |
238 | 3 | auto Loc = S.getResumeExpr()->getExprLoc(); |
239 | 3 | auto *Catch = new (CGF.getContext()) |
240 | 3 | CXXCatchStmt(Loc, /*exDecl=*/nullptr, Coro.ExceptionHandler); |
241 | 3 | auto *TryBody = CompoundStmt::Create(CGF.getContext(), S.getResumeExpr(), |
242 | 3 | FPOptionsOverride(), Loc, Loc); |
243 | 3 | TryStmt = CXXTryStmt::Create(CGF.getContext(), Loc, TryBody, Catch); |
244 | 3 | CGF.EnterCXXTryStmt(*TryStmt); |
245 | 3 | } |
246 | | |
247 | 290 | LValueOrRValue Res; |
248 | 290 | if (forLValue) |
249 | 6 | Res.LV = CGF.EmitLValue(S.getResumeExpr()); |
250 | 284 | else |
251 | 284 | Res.RV = CGF.EmitAnyExpr(S.getResumeExpr(), aggSlot, ignoreResult); |
252 | | |
253 | 290 | if (TryStmt) { |
254 | 3 | Builder.CreateFlagStore(false, Coro.ResumeEHVar); |
255 | 3 | CGF.ExitCXXTryStmt(*TryStmt); |
256 | 3 | } |
257 | | |
258 | 290 | return Res; |
259 | 290 | } |
260 | | |
261 | | RValue CodeGenFunction::EmitCoawaitExpr(const CoawaitExpr &E, |
262 | | AggValueSlot aggSlot, |
263 | 278 | bool ignoreResult) { |
264 | 278 | return emitSuspendExpression(*this, *CurCoro.Data, E, |
265 | 278 | CurCoro.Data->CurrentAwaitKind, aggSlot, |
266 | 278 | ignoreResult, /*forLValue*/false).RV; |
267 | 278 | } |
268 | | RValue CodeGenFunction::EmitCoyieldExpr(const CoyieldExpr &E, |
269 | | AggValueSlot aggSlot, |
270 | 6 | bool ignoreResult) { |
271 | 6 | return emitSuspendExpression(*this, *CurCoro.Data, E, AwaitKind::Yield, |
272 | 6 | aggSlot, ignoreResult, /*forLValue*/false).RV; |
273 | 6 | } |
274 | | |
275 | 114 | void CodeGenFunction::EmitCoreturnStmt(CoreturnStmt const &S) { |
276 | 114 | ++CurCoro.Data->CoreturnCount; |
277 | 114 | const Expr *RV = S.getOperand(); |
278 | 114 | if (RV && RV->getType()->isVoidType()13 && !isa<InitListExpr>(RV)4 ) { |
279 | | // Make sure to evaluate the non initlist expression of a co_return |
280 | | // with a void expression for side effects. |
281 | 2 | RunCleanupsScope cleanupScope(*this); |
282 | 2 | EmitIgnoredExpr(RV); |
283 | 2 | } |
284 | 114 | EmitStmt(S.getPromiseCall()); |
285 | 114 | EmitBranchThroughCleanup(CurCoro.Data->FinalJD); |
286 | 114 | } |
287 | | |
288 | | |
289 | | #ifndef NDEBUG |
290 | | static QualType getCoroutineSuspendExprReturnType(const ASTContext &Ctx, |
291 | 6 | const CoroutineSuspendExpr *E) { |
292 | 6 | const auto *RE = E->getResumeExpr(); |
293 | | // Is it possible for RE to be a CXXBindTemporaryExpr wrapping |
294 | | // a MemberCallExpr? |
295 | 6 | assert(isa<CallExpr>(RE) && "unexpected suspend expression type"); |
296 | 0 | return cast<CallExpr>(RE)->getCallReturnType(Ctx); |
297 | 6 | } |
298 | | #endif |
299 | | |
300 | | LValue |
301 | 4 | CodeGenFunction::EmitCoawaitLValue(const CoawaitExpr *E) { |
302 | 4 | assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() && |
303 | 4 | "Can't have a scalar return unless the return type is a " |
304 | 4 | "reference type!"); |
305 | 0 | return emitSuspendExpression(*this, *CurCoro.Data, *E, |
306 | 4 | CurCoro.Data->CurrentAwaitKind, AggValueSlot::ignored(), |
307 | 4 | /*ignoreResult*/false, /*forLValue*/true).LV; |
308 | 4 | } |
309 | | |
310 | | LValue |
311 | 2 | CodeGenFunction::EmitCoyieldLValue(const CoyieldExpr *E) { |
312 | 2 | assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() && |
313 | 2 | "Can't have a scalar return unless the return type is a " |
314 | 2 | "reference type!"); |
315 | 0 | return emitSuspendExpression(*this, *CurCoro.Data, *E, |
316 | 2 | AwaitKind::Yield, AggValueSlot::ignored(), |
317 | 2 | /*ignoreResult*/false, /*forLValue*/true).LV; |
318 | 2 | } |
319 | | |
320 | | // Hunts for the parameter reference in the parameter copy/move declaration. |
321 | | namespace { |
322 | | struct GetParamRef : public StmtVisitor<GetParamRef> { |
323 | | public: |
324 | | DeclRefExpr *Expr = nullptr; |
325 | 71 | GetParamRef() {} |
326 | 71 | void VisitDeclRefExpr(DeclRefExpr *E) { |
327 | 71 | assert(Expr == nullptr && "multilple declref in param move"); |
328 | 0 | Expr = E; |
329 | 71 | } |
330 | 100 | void VisitStmt(Stmt *S) { |
331 | 100 | for (auto *C : S->children()) { |
332 | 100 | if (C) |
333 | 100 | Visit(C); |
334 | 100 | } |
335 | 100 | } |
336 | | }; |
337 | | } |
338 | | |
339 | | // This class replaces references to parameters to their copies by changing |
340 | | // the addresses in CGF.LocalDeclMap and restoring back the original values in |
341 | | // its destructor. |
342 | | |
343 | | namespace { |
344 | | struct ParamReferenceReplacerRAII { |
345 | | CodeGenFunction::DeclMapTy SavedLocals; |
346 | | CodeGenFunction::DeclMapTy& LocalDeclMap; |
347 | | |
348 | | ParamReferenceReplacerRAII(CodeGenFunction::DeclMapTy &LocalDeclMap) |
349 | 118 | : LocalDeclMap(LocalDeclMap) {} |
350 | | |
351 | 71 | void addCopy(DeclStmt const *PM) { |
352 | | // Figure out what param it refers to. |
353 | | |
354 | 71 | assert(PM->isSingleDecl()); |
355 | 0 | VarDecl const*VD = static_cast<VarDecl const*>(PM->getSingleDecl()); |
356 | 71 | Expr const *InitExpr = VD->getInit(); |
357 | 71 | GetParamRef Visitor; |
358 | 71 | Visitor.Visit(const_cast<Expr*>(InitExpr)); |
359 | 71 | assert(Visitor.Expr); |
360 | 0 | DeclRefExpr *DREOrig = Visitor.Expr; |
361 | 71 | auto *PD = DREOrig->getDecl(); |
362 | | |
363 | 71 | auto it = LocalDeclMap.find(PD); |
364 | 71 | assert(it != LocalDeclMap.end() && "parameter is not found"); |
365 | 0 | SavedLocals.insert({ PD, it->second }); |
366 | | |
367 | 71 | auto copyIt = LocalDeclMap.find(VD); |
368 | 71 | assert(copyIt != LocalDeclMap.end() && "parameter copy is not found"); |
369 | 0 | it->second = copyIt->getSecond(); |
370 | 71 | } |
371 | | |
372 | 118 | ~ParamReferenceReplacerRAII() { |
373 | 118 | for (auto&& SavedLocal : SavedLocals) { |
374 | 71 | LocalDeclMap.insert({SavedLocal.first, SavedLocal.second}); |
375 | 71 | } |
376 | 118 | } |
377 | | }; |
378 | | } |
379 | | |
380 | | // For WinEH exception representation backend needs to know what funclet coro.end |
381 | | // belongs to. That information is passed in a funclet bundle. |
382 | | static SmallVector<llvm::OperandBundleDef, 1> |
383 | 23 | getBundlesForCoroEnd(CodeGenFunction &CGF) { |
384 | 23 | SmallVector<llvm::OperandBundleDef, 1> BundleList; |
385 | | |
386 | 23 | if (llvm::Instruction *EHPad = CGF.CurrentFuncletPad) |
387 | 6 | BundleList.emplace_back("funclet", EHPad); |
388 | | |
389 | 23 | return BundleList; |
390 | 23 | } |
391 | | |
392 | | namespace { |
393 | | // We will insert coro.end to cut any of the destructors for objects that |
394 | | // do not need to be destroyed once the coroutine is resumed. |
395 | | // See llvm/docs/Coroutines.rst for more details about coro.end. |
396 | | struct CallCoroEnd final : public EHScopeStack::Cleanup { |
397 | 23 | void Emit(CodeGenFunction &CGF, Flags flags) override { |
398 | 23 | auto &CGM = CGF.CGM; |
399 | 23 | auto *NullPtr = llvm::ConstantPointerNull::get(CGF.Int8PtrTy); |
400 | 23 | llvm::Function *CoroEndFn = CGM.getIntrinsic(llvm::Intrinsic::coro_end); |
401 | | // See if we have a funclet bundle to associate coro.end with. (WinEH) |
402 | 23 | auto Bundles = getBundlesForCoroEnd(CGF); |
403 | 23 | auto *CoroEnd = CGF.Builder.CreateCall( |
404 | 23 | CoroEndFn, {NullPtr, CGF.Builder.getTrue()}, Bundles); |
405 | 23 | if (Bundles.empty()) { |
406 | | // Otherwise, (landingpad model), create a conditional branch that leads |
407 | | // either to a cleanup block or a block with EH resume instruction. |
408 | 17 | auto *ResumeBB = CGF.getEHResumeBlock(/*isCleanup=*/true); |
409 | 17 | auto *CleanupContBB = CGF.createBasicBlock("cleanup.cont"); |
410 | 17 | CGF.Builder.CreateCondBr(CoroEnd, ResumeBB, CleanupContBB); |
411 | 17 | CGF.EmitBlock(CleanupContBB); |
412 | 17 | } |
413 | 23 | } |
414 | | }; |
415 | | } |
416 | | |
417 | | namespace { |
418 | | // Make sure to call coro.delete on scope exit. |
419 | | struct CallCoroDelete final : public EHScopeStack::Cleanup { |
420 | | Stmt *Deallocate; |
421 | | |
422 | | // Emit "if (coro.free(CoroId, CoroBegin)) Deallocate;" |
423 | | |
424 | | // Note: That deallocation will be emitted twice: once for a normal exit and |
425 | | // once for exceptional exit. This usage is safe because Deallocate does not |
426 | | // contain any declarations. The SubStmtBuilder::makeNewAndDeleteExpr() |
427 | | // builds a single call to a deallocation function which is safe to emit |
428 | | // multiple times. |
429 | 147 | void Emit(CodeGenFunction &CGF, Flags) override { |
430 | | // Remember the current point, as we are going to emit deallocation code |
431 | | // first to get to coro.free instruction that is an argument to a delete |
432 | | // call. |
433 | 147 | BasicBlock *SaveInsertBlock = CGF.Builder.GetInsertBlock(); |
434 | | |
435 | 147 | auto *FreeBB = CGF.createBasicBlock("coro.free"); |
436 | 147 | CGF.EmitBlock(FreeBB); |
437 | 147 | CGF.EmitStmt(Deallocate); |
438 | | |
439 | 147 | auto *AfterFreeBB = CGF.createBasicBlock("after.coro.free"); |
440 | 147 | CGF.EmitBlock(AfterFreeBB); |
441 | | |
442 | | // We should have captured coro.free from the emission of deallocate. |
443 | 147 | auto *CoroFree = CGF.CurCoro.Data->LastCoroFree; |
444 | 147 | if (!CoroFree) { |
445 | 0 | CGF.CGM.Error(Deallocate->getBeginLoc(), |
446 | 0 | "Deallocation expressoin does not refer to coro.free"); |
447 | 0 | return; |
448 | 0 | } |
449 | | |
450 | | // Get back to the block we were originally and move coro.free there. |
451 | 147 | auto *InsertPt = SaveInsertBlock->getTerminator(); |
452 | 147 | CoroFree->moveBefore(InsertPt); |
453 | 147 | CGF.Builder.SetInsertPoint(InsertPt); |
454 | | |
455 | | // Add if (auto *mem = coro.free) Deallocate; |
456 | 147 | auto *NullPtr = llvm::ConstantPointerNull::get(CGF.Int8PtrTy); |
457 | 147 | auto *Cond = CGF.Builder.CreateICmpNE(CoroFree, NullPtr); |
458 | 147 | CGF.Builder.CreateCondBr(Cond, FreeBB, AfterFreeBB); |
459 | | |
460 | | // No longer need old terminator. |
461 | 147 | InsertPt->eraseFromParent(); |
462 | 147 | CGF.Builder.SetInsertPoint(AfterFreeBB); |
463 | 147 | } |
464 | 118 | explicit CallCoroDelete(Stmt *DeallocStmt) : Deallocate(DeallocStmt) {} |
465 | | }; |
466 | | } |
467 | | |
468 | | static void emitBodyAndFallthrough(CodeGenFunction &CGF, |
469 | 118 | const CoroutineBodyStmt &S, Stmt *Body) { |
470 | 118 | CGF.EmitStmt(Body); |
471 | 118 | const bool CanFallthrough = CGF.Builder.GetInsertBlock(); |
472 | 118 | if (CanFallthrough) |
473 | 31 | if (Stmt *OnFallthrough = S.getFallthroughHandler()) |
474 | 31 | CGF.EmitStmt(OnFallthrough); |
475 | 118 | } |
476 | | |
477 | 118 | void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) { |
478 | 118 | auto *NullPtr = llvm::ConstantPointerNull::get(Builder.getInt8PtrTy()); |
479 | 118 | auto &TI = CGM.getContext().getTargetInfo(); |
480 | 118 | unsigned NewAlign = TI.getNewAlign() / TI.getCharWidth(); |
481 | | |
482 | 118 | auto *EntryBB = Builder.GetInsertBlock(); |
483 | 118 | auto *AllocBB = createBasicBlock("coro.alloc"); |
484 | 118 | auto *InitBB = createBasicBlock("coro.init"); |
485 | 118 | auto *FinalBB = createBasicBlock("coro.final"); |
486 | 118 | auto *RetBB = createBasicBlock("coro.ret"); |
487 | | |
488 | 118 | auto *CoroId = Builder.CreateCall( |
489 | 118 | CGM.getIntrinsic(llvm::Intrinsic::coro_id), |
490 | 118 | {Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr}); |
491 | 118 | createCoroData(*this, CurCoro, CoroId); |
492 | 118 | CurCoro.Data->SuspendBB = RetBB; |
493 | 118 | assert(ShouldEmitLifetimeMarkers && |
494 | 118 | "Must emit lifetime intrinsics for coroutines"); |
495 | | |
496 | | // Backend is allowed to elide memory allocations, to help it, emit |
497 | | // auto mem = coro.alloc() ? 0 : ... allocation code ...; |
498 | 0 | auto *CoroAlloc = Builder.CreateCall( |
499 | 118 | CGM.getIntrinsic(llvm::Intrinsic::coro_alloc), {CoroId}); |
500 | | |
501 | 118 | Builder.CreateCondBr(CoroAlloc, AllocBB, InitBB); |
502 | | |
503 | 118 | EmitBlock(AllocBB); |
504 | 118 | auto *AllocateCall = EmitScalarExpr(S.getAllocate()); |
505 | 118 | auto *AllocOrInvokeContBB = Builder.GetInsertBlock(); |
506 | | |
507 | | // Handle allocation failure if 'ReturnStmtOnAllocFailure' was provided. |
508 | 118 | if (auto *RetOnAllocFailure = S.getReturnStmtOnAllocFailure()) { |
509 | 4 | auto *RetOnFailureBB = createBasicBlock("coro.ret.on.failure"); |
510 | | |
511 | | // See if allocation was successful. |
512 | 4 | auto *NullPtr = llvm::ConstantPointerNull::get(Int8PtrTy); |
513 | 4 | auto *Cond = Builder.CreateICmpNE(AllocateCall, NullPtr); |
514 | 4 | Builder.CreateCondBr(Cond, InitBB, RetOnFailureBB); |
515 | | |
516 | | // If not, return OnAllocFailure object. |
517 | 4 | EmitBlock(RetOnFailureBB); |
518 | 4 | EmitStmt(RetOnAllocFailure); |
519 | 4 | } |
520 | 114 | else { |
521 | 114 | Builder.CreateBr(InitBB); |
522 | 114 | } |
523 | | |
524 | 118 | EmitBlock(InitBB); |
525 | | |
526 | | // Pass the result of the allocation to coro.begin. |
527 | 118 | auto *Phi = Builder.CreatePHI(VoidPtrTy, 2); |
528 | 118 | Phi->addIncoming(NullPtr, EntryBB); |
529 | 118 | Phi->addIncoming(AllocateCall, AllocOrInvokeContBB); |
530 | 118 | auto *CoroBegin = Builder.CreateCall( |
531 | 118 | CGM.getIntrinsic(llvm::Intrinsic::coro_begin), {CoroId, Phi}); |
532 | 118 | CurCoro.Data->CoroBegin = CoroBegin; |
533 | | |
534 | 118 | CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB); |
535 | 118 | { |
536 | 118 | CGDebugInfo *DI = getDebugInfo(); |
537 | 118 | ParamReferenceReplacerRAII ParamReplacer(LocalDeclMap); |
538 | 118 | CodeGenFunction::RunCleanupsScope ResumeScope(*this); |
539 | 118 | EHStack.pushCleanup<CallCoroDelete>(NormalAndEHCleanup, S.getDeallocate()); |
540 | | |
541 | | // Create mapping between parameters and copy-params for coroutine function. |
542 | 118 | auto ParamMoves = S.getParamMoves(); |
543 | 118 | assert( |
544 | 118 | (ParamMoves.size() == 0 || (ParamMoves.size() == FnArgs.size())) && |
545 | 118 | "ParamMoves and FnArgs should be the same size for coroutine function"); |
546 | 118 | if (ParamMoves.size() == FnArgs.size() && DI) |
547 | 2 | for (const auto Pair : llvm::zip(FnArgs, ParamMoves)) |
548 | 6 | DI->getCoroutineParameterMappings().insert( |
549 | 6 | {std::get<0>(Pair), std::get<1>(Pair)}); |
550 | | |
551 | | // Create parameter copies. We do it before creating a promise, since an |
552 | | // evolution of coroutine TS may allow promise constructor to observe |
553 | | // parameter copies. |
554 | 118 | for (auto *PM : S.getParamMoves()) { |
555 | 71 | EmitStmt(PM); |
556 | 71 | ParamReplacer.addCopy(cast<DeclStmt>(PM)); |
557 | | // TODO: if(CoroParam(...)) need to surround ctor and dtor |
558 | | // for the copy, so that llvm can elide it if the copy is |
559 | | // not needed. |
560 | 71 | } |
561 | | |
562 | 118 | EmitStmt(S.getPromiseDeclStmt()); |
563 | | |
564 | 118 | Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl()); |
565 | 118 | auto *PromiseAddrVoidPtr = |
566 | 118 | new llvm::BitCastInst(PromiseAddr.getPointer(), VoidPtrTy, "", CoroId); |
567 | | // Update CoroId to refer to the promise. We could not do it earlier because |
568 | | // promise local variable was not emitted yet. |
569 | 118 | CoroId->setArgOperand(1, PromiseAddrVoidPtr); |
570 | | |
571 | | // ReturnValue should be valid as long as the coroutine's return type |
572 | | // is not void. The assertion could help us to reduce the check later. |
573 | 118 | assert(ReturnValue.isValid() == (bool)S.getReturnStmt()); |
574 | | // Now we have the promise, initialize the GRO. |
575 | | // We need to emit `get_return_object` first. According to: |
576 | | // [dcl.fct.def.coroutine]p7 |
577 | | // The call to get_return_object is sequenced before the call to |
578 | | // initial_suspend and is invoked at most once. |
579 | | // |
580 | | // So we couldn't emit return value when we emit return statment, |
581 | | // otherwise the call to get_return_object wouldn't be in front |
582 | | // of initial_suspend. |
583 | 118 | if (ReturnValue.isValid()) { |
584 | 60 | EmitAnyExprToMem(S.getReturnValue(), ReturnValue, |
585 | 60 | S.getReturnValue()->getType().getQualifiers(), |
586 | 60 | /*IsInit*/ true); |
587 | 60 | } |
588 | | |
589 | 118 | EHStack.pushCleanup<CallCoroEnd>(EHCleanup); |
590 | | |
591 | 118 | CurCoro.Data->CurrentAwaitKind = AwaitKind::Init; |
592 | 118 | CurCoro.Data->ExceptionHandler = S.getExceptionHandler(); |
593 | 118 | EmitStmt(S.getInitSuspendStmt()); |
594 | 118 | CurCoro.Data->FinalJD = getJumpDestInCurrentScope(FinalBB); |
595 | | |
596 | 118 | CurCoro.Data->CurrentAwaitKind = AwaitKind::Normal; |
597 | | |
598 | 118 | if (CurCoro.Data->ExceptionHandler) { |
599 | | // If we generated IR to record whether an exception was thrown from |
600 | | // 'await_resume', then use that IR to determine whether the coroutine |
601 | | // body should be skipped. |
602 | | // If we didn't generate the IR (perhaps because 'await_resume' was marked |
603 | | // as 'noexcept'), then we skip this check. |
604 | 22 | BasicBlock *ContBB = nullptr; |
605 | 22 | if (CurCoro.Data->ResumeEHVar) { |
606 | 3 | BasicBlock *BodyBB = createBasicBlock("coro.resumed.body"); |
607 | 3 | ContBB = createBasicBlock("coro.resumed.cont"); |
608 | 3 | Value *SkipBody = Builder.CreateFlagLoad(CurCoro.Data->ResumeEHVar, |
609 | 3 | "coro.resumed.eh"); |
610 | 3 | Builder.CreateCondBr(SkipBody, ContBB, BodyBB); |
611 | 3 | EmitBlock(BodyBB); |
612 | 3 | } |
613 | | |
614 | 22 | auto Loc = S.getBeginLoc(); |
615 | 22 | CXXCatchStmt Catch(Loc, /*exDecl=*/nullptr, |
616 | 22 | CurCoro.Data->ExceptionHandler); |
617 | 22 | auto *TryStmt = |
618 | 22 | CXXTryStmt::Create(getContext(), Loc, S.getBody(), &Catch); |
619 | | |
620 | 22 | EnterCXXTryStmt(*TryStmt); |
621 | 22 | emitBodyAndFallthrough(*this, S, TryStmt->getTryBlock()); |
622 | 22 | ExitCXXTryStmt(*TryStmt); |
623 | | |
624 | 22 | if (ContBB) |
625 | 3 | EmitBlock(ContBB); |
626 | 22 | } |
627 | 96 | else { |
628 | 96 | emitBodyAndFallthrough(*this, S, S.getBody()); |
629 | 96 | } |
630 | | |
631 | | // See if we need to generate final suspend. |
632 | 118 | const bool CanFallthrough = Builder.GetInsertBlock(); |
633 | 118 | const bool HasCoreturns = CurCoro.Data->CoreturnCount > 0; |
634 | 118 | if (CanFallthrough || HasCoreturns99 ) { |
635 | 113 | EmitBlock(FinalBB); |
636 | 113 | CurCoro.Data->CurrentAwaitKind = AwaitKind::Final; |
637 | 113 | EmitStmt(S.getFinalSuspendStmt()); |
638 | 113 | } else { |
639 | | // We don't need FinalBB. Emit it to make sure the block is deleted. |
640 | 5 | EmitBlock(FinalBB, /*IsFinished=*/true); |
641 | 5 | } |
642 | 118 | } |
643 | | |
644 | 0 | EmitBlock(RetBB); |
645 | | // Emit coro.end before getReturnStmt (and parameter destructors), since |
646 | | // resume and destroy parts of the coroutine should not include them. |
647 | 118 | llvm::Function *CoroEnd = CGM.getIntrinsic(llvm::Intrinsic::coro_end); |
648 | 118 | Builder.CreateCall(CoroEnd, {NullPtr, Builder.getFalse()}); |
649 | | |
650 | 118 | if (Stmt *Ret = S.getReturnStmt()) { |
651 | | // Since we already emitted the return value above, so we shouldn't |
652 | | // emit it again here. |
653 | 60 | cast<ReturnStmt>(Ret)->setRetValue(nullptr); |
654 | 60 | EmitStmt(Ret); |
655 | 60 | } |
656 | | |
657 | | // LLVM require the frontend to mark the coroutine. |
658 | 118 | CurFn->setPresplitCoroutine(); |
659 | 118 | } |
660 | | |
661 | | // Emit coroutine intrinsic and patch up arguments of the token type. |
662 | | RValue CodeGenFunction::EmitCoroutineIntrinsic(const CallExpr *E, |
663 | 646 | unsigned int IID) { |
664 | 646 | SmallVector<llvm::Value *, 8> Args; |
665 | 646 | switch (IID) { |
666 | 49 | default: |
667 | 49 | break; |
668 | | // The coro.frame builtin is replaced with an SSA value of the coro.begin |
669 | | // intrinsic. |
670 | 443 | case llvm::Intrinsic::coro_frame: { |
671 | 443 | if (CurCoro.Data && CurCoro.Data->CoroBegin) { |
672 | 443 | return RValue::get(CurCoro.Data->CoroBegin); |
673 | 443 | } |
674 | 0 | CGM.Error(E->getBeginLoc(), "this builtin expect that __builtin_coro_begin " |
675 | 0 | "has been used earlier in this function"); |
676 | 0 | auto NullPtr = llvm::ConstantPointerNull::get(Builder.getInt8PtrTy()); |
677 | 0 | return RValue::get(NullPtr); |
678 | 443 | } |
679 | | // The following three intrinsics take a token parameter referring to a token |
680 | | // returned by earlier call to @llvm.coro.id. Since we cannot represent it in |
681 | | // builtins, we patch it up here. |
682 | 2 | case llvm::Intrinsic::coro_alloc: |
683 | 4 | case llvm::Intrinsic::coro_begin: |
684 | 153 | case llvm::Intrinsic::coro_free: { |
685 | 153 | if (CurCoro.Data && CurCoro.Data->CoroId150 ) { |
686 | 150 | Args.push_back(CurCoro.Data->CoroId); |
687 | 150 | break; |
688 | 150 | } |
689 | 3 | CGM.Error(E->getBeginLoc(), "this builtin expect that __builtin_coro_id has" |
690 | 3 | " been used earlier in this function"); |
691 | | // Fallthrough to the next case to add TokenNone as the first argument. |
692 | 3 | LLVM_FALLTHROUGH; |
693 | 3 | } |
694 | | // @llvm.coro.suspend takes a token parameter. Add token 'none' as the first |
695 | | // argument. |
696 | 4 | case llvm::Intrinsic::coro_suspend: |
697 | 4 | Args.push_back(llvm::ConstantTokenNone::get(getLLVMContext())); |
698 | 4 | break; |
699 | 646 | } |
700 | 203 | for (const Expr *Arg : E->arguments()) |
701 | 235 | Args.push_back(EmitScalarExpr(Arg)); |
702 | | |
703 | 203 | llvm::Function *F = CGM.getIntrinsic(IID); |
704 | 203 | llvm::CallInst *Call = Builder.CreateCall(F, Args); |
705 | | |
706 | | // Note: The following code is to enable to emit coro.id and coro.begin by |
707 | | // hand to experiment with coroutines in C. |
708 | | // If we see @llvm.coro.id remember it in the CoroData. We will update |
709 | | // coro.alloc, coro.begin and coro.free intrinsics to refer to it. |
710 | 203 | if (IID == llvm::Intrinsic::coro_id) { |
711 | 3 | createCoroData(*this, CurCoro, Call, E); |
712 | 3 | } |
713 | 200 | else if (IID == llvm::Intrinsic::coro_begin) { |
714 | 2 | if (CurCoro.Data) |
715 | 1 | CurCoro.Data->CoroBegin = Call; |
716 | 2 | } |
717 | 198 | else if (IID == llvm::Intrinsic::coro_free) { |
718 | | // Remember the last coro_free as we need it to build the conditional |
719 | | // deletion of the coroutine frame. |
720 | 149 | if (CurCoro.Data) |
721 | 148 | CurCoro.Data->LastCoroFree = Call; |
722 | 149 | } |
723 | 203 | return RValue::get(Call); |
724 | 646 | } |