/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- CoroEarly.cpp - Coroutine Early Function Pass ----------------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // This pass lowers coroutine intrinsics that hide the details of the exact |
9 | | // calling convention for coroutine resume and destroy functions and details of |
10 | | // the structure of the coroutine frame. |
11 | | //===----------------------------------------------------------------------===// |
12 | | |
13 | | #include "CoroInternal.h" |
14 | | #include "llvm/IR/CallSite.h" |
15 | | #include "llvm/IR/IRBuilder.h" |
16 | | #include "llvm/IR/InstIterator.h" |
17 | | #include "llvm/IR/Module.h" |
18 | | #include "llvm/Pass.h" |
19 | | |
20 | | using namespace llvm; |
21 | | |
22 | | #define DEBUG_TYPE "coro-early" |
23 | | |
24 | | namespace { |
25 | | // Created on demand if CoroEarly pass has work to do. |
26 | | class Lowerer : public coro::LowererBase { |
27 | | IRBuilder<> Builder; |
28 | | PointerType *const AnyResumeFnPtrTy; |
29 | | Constant *NoopCoro = nullptr; |
30 | | |
31 | | void lowerResumeOrDestroy(CallSite CS, CoroSubFnInst::ResumeKind); |
32 | | void lowerCoroPromise(CoroPromiseInst *Intrin); |
33 | | void lowerCoroDone(IntrinsicInst *II); |
34 | | void lowerCoroNoop(IntrinsicInst *II); |
35 | | |
36 | | public: |
37 | | Lowerer(Module &M) |
38 | | : LowererBase(M), Builder(Context), |
39 | | AnyResumeFnPtrTy(FunctionType::get(Type::getVoidTy(Context), Int8Ptr, |
40 | | /*isVarArg=*/false) |
41 | 12 | ->getPointerTo()) {} |
42 | | bool lowerEarlyIntrinsics(Function &F); |
43 | | }; |
44 | | } |
45 | | |
46 | | // Replace a direct call to coro.resume or coro.destroy with an indirect call to |
47 | | // an address returned by coro.subfn.addr intrinsic. This is done so that |
48 | | // CGPassManager recognizes devirtualization when CoroElide pass replaces a call |
49 | | // to coro.subfn.addr with an appropriate function address. |
50 | | void Lowerer::lowerResumeOrDestroy(CallSite CS, |
51 | 27 | CoroSubFnInst::ResumeKind Index) { |
52 | 27 | Value *ResumeAddr = |
53 | 27 | makeSubFnCall(CS.getArgOperand(0), Index, CS.getInstruction()); |
54 | 27 | CS.setCalledFunction(ResumeAddr); |
55 | 27 | CS.setCallingConv(CallingConv::Fast); |
56 | 27 | } |
57 | | |
58 | | // Coroutine promise field is always at the fixed offset from the beginning of |
59 | | // the coroutine frame. i8* coro.promise(i8*, i1 from) intrinsic adds an offset |
60 | | // to a passed pointer to move from coroutine frame to coroutine promise and |
61 | | // vice versa. Since we don't know exactly which coroutine frame it is, we build |
62 | | // a coroutine frame mock up starting with two function pointers, followed by a |
63 | | // properly aligned coroutine promise field. |
64 | | // TODO: Handle the case when coroutine promise alloca has align override. |
65 | 1 | void Lowerer::lowerCoroPromise(CoroPromiseInst *Intrin) { |
66 | 1 | Value *Operand = Intrin->getArgOperand(0); |
67 | 1 | unsigned Alignement = Intrin->getAlignment(); |
68 | 1 | Type *Int8Ty = Builder.getInt8Ty(); |
69 | 1 | |
70 | 1 | auto *SampleStruct = |
71 | 1 | StructType::get(Context, {AnyResumeFnPtrTy, AnyResumeFnPtrTy, Int8Ty}); |
72 | 1 | const DataLayout &DL = TheModule.getDataLayout(); |
73 | 1 | int64_t Offset = alignTo( |
74 | 1 | DL.getStructLayout(SampleStruct)->getElementOffset(2), Alignement); |
75 | 1 | if (Intrin->isFromPromise()) |
76 | 0 | Offset = -Offset; |
77 | 1 | |
78 | 1 | Builder.SetInsertPoint(Intrin); |
79 | 1 | Value *Replacement = |
80 | 1 | Builder.CreateConstInBoundsGEP1_32(Int8Ty, Operand, Offset); |
81 | 1 | |
82 | 1 | Intrin->replaceAllUsesWith(Replacement); |
83 | 1 | Intrin->eraseFromParent(); |
84 | 1 | } |
85 | | |
86 | | // When a coroutine reaches final suspend point, it zeros out ResumeFnAddr in |
87 | | // the coroutine frame (it is UB to resume from a final suspend point). |
88 | | // The llvm.coro.done intrinsic is used to check whether a coroutine is |
89 | | // suspended at the final suspend point or not. |
90 | 2 | void Lowerer::lowerCoroDone(IntrinsicInst *II) { |
91 | 2 | Value *Operand = II->getArgOperand(0); |
92 | 2 | |
93 | 2 | // ResumeFnAddr is the first pointer sized element of the coroutine frame. |
94 | 2 | auto *FrameTy = Int8Ptr; |
95 | 2 | PointerType *FramePtrTy = FrameTy->getPointerTo(); |
96 | 2 | |
97 | 2 | Builder.SetInsertPoint(II); |
98 | 2 | auto *BCI = Builder.CreateBitCast(Operand, FramePtrTy); |
99 | 2 | auto *Gep = Builder.CreateConstInBoundsGEP1_32(FrameTy, BCI, 0); |
100 | 2 | auto *Load = Builder.CreateLoad(FrameTy, Gep); |
101 | 2 | auto *Cond = Builder.CreateICmpEQ(Load, NullPtr); |
102 | 2 | |
103 | 2 | II->replaceAllUsesWith(Cond); |
104 | 2 | II->eraseFromParent(); |
105 | 2 | } |
106 | | |
107 | 1 | void Lowerer::lowerCoroNoop(IntrinsicInst *II) { |
108 | 1 | if (!NoopCoro) { |
109 | 1 | LLVMContext &C = Builder.getContext(); |
110 | 1 | Module &M = *II->getModule(); |
111 | 1 | |
112 | 1 | // Create a noop.frame struct type. |
113 | 1 | StructType *FrameTy = StructType::create(C, "NoopCoro.Frame"); |
114 | 1 | auto *FramePtrTy = FrameTy->getPointerTo(); |
115 | 1 | auto *FnTy = FunctionType::get(Type::getVoidTy(C), FramePtrTy, |
116 | 1 | /*isVarArg=*/false); |
117 | 1 | auto *FnPtrTy = FnTy->getPointerTo(); |
118 | 1 | FrameTy->setBody({FnPtrTy, FnPtrTy}); |
119 | 1 | |
120 | 1 | // Create a Noop function that does nothing. |
121 | 1 | Function *NoopFn = |
122 | 1 | Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage, |
123 | 1 | "NoopCoro.ResumeDestroy", &M); |
124 | 1 | NoopFn->setCallingConv(CallingConv::Fast); |
125 | 1 | auto *Entry = BasicBlock::Create(C, "entry", NoopFn); |
126 | 1 | ReturnInst::Create(C, Entry); |
127 | 1 | |
128 | 1 | // Create a constant struct for the frame. |
129 | 1 | Constant* Values[] = {NoopFn, NoopFn}; |
130 | 1 | Constant* NoopCoroConst = ConstantStruct::get(FrameTy, Values); |
131 | 1 | NoopCoro = new GlobalVariable(M, NoopCoroConst->getType(), /*isConstant=*/true, |
132 | 1 | GlobalVariable::PrivateLinkage, NoopCoroConst, |
133 | 1 | "NoopCoro.Frame.Const"); |
134 | 1 | } |
135 | 1 | |
136 | 1 | Builder.SetInsertPoint(II); |
137 | 1 | auto *NoopCoroVoidPtr = Builder.CreateBitCast(NoopCoro, Int8Ptr); |
138 | 1 | II->replaceAllUsesWith(NoopCoroVoidPtr); |
139 | 1 | II->eraseFromParent(); |
140 | 1 | } |
141 | | |
142 | | // Prior to CoroSplit, calls to coro.begin needs to be marked as NoDuplicate, |
143 | | // as CoroSplit assumes there is exactly one coro.begin. After CoroSplit, |
144 | | // NoDuplicate attribute will be removed from coro.begin otherwise, it will |
145 | | // interfere with inlining. |
146 | 10 | static void setCannotDuplicate(CoroIdInst *CoroId) { |
147 | 10 | for (User *U : CoroId->users()) |
148 | 24 | if (auto *CB = dyn_cast<CoroBeginInst>(U)) |
149 | 10 | CB->setCannotDuplicate(); |
150 | 10 | } |
151 | | |
152 | 32 | bool Lowerer::lowerEarlyIntrinsics(Function &F) { |
153 | 32 | bool Changed = false; |
154 | 32 | CoroIdInst *CoroId = nullptr; |
155 | 32 | SmallVector<CoroFreeInst *, 4> CoroFrees; |
156 | 416 | for (auto IB = inst_begin(F), IE = inst_end(F); IB != IE;) { |
157 | 384 | Instruction &I = *IB++; |
158 | 384 | if (auto CS = CallSite(&I)) { |
159 | 169 | switch (CS.getIntrinsicID()) { |
160 | 169 | default: |
161 | 94 | continue; |
162 | 169 | case Intrinsic::coro_free: |
163 | 10 | CoroFrees.push_back(cast<CoroFreeInst>(&I)); |
164 | 10 | break; |
165 | 169 | case Intrinsic::coro_suspend: |
166 | 14 | // Make sure that final suspend point is not duplicated as CoroSplit |
167 | 14 | // pass expects that there is at most one final suspend point. |
168 | 14 | if (cast<CoroSuspendInst>(&I)->isFinal()) |
169 | 2 | CS.setCannotDuplicate(); |
170 | 14 | break; |
171 | 169 | case Intrinsic::coro_end: |
172 | 10 | // Make sure that fallthrough coro.end is not duplicated as CoroSplit |
173 | 10 | // pass expects that there is at most one fallthrough coro.end. |
174 | 10 | if (cast<CoroEndInst>(&I)->isFallthrough()) |
175 | 10 | CS.setCannotDuplicate(); |
176 | 10 | break; |
177 | 169 | case Intrinsic::coro_noop: |
178 | 1 | lowerCoroNoop(cast<IntrinsicInst>(&I)); |
179 | 1 | break; |
180 | 169 | case Intrinsic::coro_id: |
181 | 10 | // Mark a function that comes out of the frontend that has a coro.id |
182 | 10 | // with a coroutine attribute. |
183 | 10 | if (auto *CII = cast<CoroIdInst>(&I)) { |
184 | 10 | if (CII->getInfo().isPreSplit()) { |
185 | 10 | F.addFnAttr(CORO_PRESPLIT_ATTR, UNPREPARED_FOR_SPLIT); |
186 | 10 | setCannotDuplicate(CII); |
187 | 10 | CII->setCoroutineSelf(); |
188 | 10 | CoroId = cast<CoroIdInst>(&I); |
189 | 10 | } |
190 | 10 | } |
191 | 10 | break; |
192 | 169 | case Intrinsic::coro_resume: |
193 | 18 | lowerResumeOrDestroy(CS, CoroSubFnInst::ResumeIndex); |
194 | 18 | break; |
195 | 169 | case Intrinsic::coro_destroy: |
196 | 9 | lowerResumeOrDestroy(CS, CoroSubFnInst::DestroyIndex); |
197 | 9 | break; |
198 | 169 | case Intrinsic::coro_promise: |
199 | 1 | lowerCoroPromise(cast<CoroPromiseInst>(&I)); |
200 | 1 | break; |
201 | 169 | case Intrinsic::coro_done: |
202 | 2 | lowerCoroDone(cast<IntrinsicInst>(&I)); |
203 | 2 | break; |
204 | 75 | } |
205 | 75 | Changed = true; |
206 | 75 | } |
207 | 384 | } |
208 | 32 | // Make sure that all CoroFree reference the coro.id intrinsic. |
209 | 32 | // Token type is not exposed through coroutine C/C++ builtins to plain C, so |
210 | 32 | // we allow specifying none and fixing it up here. |
211 | 32 | if (CoroId) |
212 | 10 | for (CoroFreeInst *CF : CoroFrees) |
213 | 10 | CF->setArgOperand(0, CoroId); |
214 | 32 | return Changed; |
215 | 32 | } |
216 | | |
217 | | //===----------------------------------------------------------------------===// |
218 | | // Top Level Driver |
219 | | //===----------------------------------------------------------------------===// |
220 | | |
221 | | namespace { |
222 | | |
223 | | struct CoroEarly : public FunctionPass { |
224 | | static char ID; // Pass identification, replacement for typeid. |
225 | 38 | CoroEarly() : FunctionPass(ID) { |
226 | 38 | initializeCoroEarlyPass(*PassRegistry::getPassRegistry()); |
227 | 38 | } |
228 | | |
229 | | std::unique_ptr<Lowerer> L; |
230 | | |
231 | | // This pass has work to do only if we find intrinsics we are going to lower |
232 | | // in the module. |
233 | 38 | bool doInitialization(Module &M) override { |
234 | 38 | if (coro::declaresIntrinsics( |
235 | 38 | M, {"llvm.coro.id", "llvm.coro.destroy", "llvm.coro.done", |
236 | 38 | "llvm.coro.end", "llvm.coro.noop", "llvm.coro.free", |
237 | 38 | "llvm.coro.promise", "llvm.coro.resume", "llvm.coro.suspend"})) |
238 | 12 | L = llvm::make_unique<Lowerer>(M); |
239 | 38 | return false; |
240 | 38 | } |
241 | | |
242 | 171 | bool runOnFunction(Function &F) override { |
243 | 171 | if (!L) |
244 | 139 | return false; |
245 | 32 | |
246 | 32 | return L->lowerEarlyIntrinsics(F); |
247 | 32 | } |
248 | | |
249 | 38 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
250 | 38 | AU.setPreservesCFG(); |
251 | 38 | } |
252 | 171 | StringRef getPassName() const override { |
253 | 171 | return "Lower early coroutine intrinsics"; |
254 | 171 | } |
255 | | }; |
256 | | } |
257 | | |
258 | | char CoroEarly::ID = 0; |
259 | | INITIALIZE_PASS(CoroEarly, "coro-early", "Lower early coroutine intrinsics", |
260 | | false, false) |
261 | | |
262 | 36 | Pass *llvm::createCoroEarlyPass() { return new CoroEarly(); } |