Coverage Report

Created: 2021-01-26 06:56

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGStmt.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGStmt.cpp - Emit LLVM Code from Statements ----------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This contains code to emit Stmt nodes as LLVM code.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGDebugInfo.h"
14
#include "CGOpenMPRuntime.h"
15
#include "CodeGenFunction.h"
16
#include "CodeGenModule.h"
17
#include "TargetInfo.h"
18
#include "clang/AST/Attr.h"
19
#include "clang/AST/StmtVisitor.h"
20
#include "clang/Basic/Builtins.h"
21
#include "clang/Basic/DiagnosticSema.h"
22
#include "clang/Basic/PrettyStackTrace.h"
23
#include "clang/Basic/SourceManager.h"
24
#include "clang/Basic/TargetInfo.h"
25
#include "llvm/ADT/SmallSet.h"
26
#include "llvm/ADT/StringExtras.h"
27
#include "llvm/IR/DataLayout.h"
28
#include "llvm/IR/InlineAsm.h"
29
#include "llvm/IR/Intrinsics.h"
30
#include "llvm/IR/MDBuilder.h"
31
#include "llvm/Support/SaveAndRestore.h"
32
33
using namespace clang;
34
using namespace CodeGen;
35
36
//===----------------------------------------------------------------------===//
37
//                              Statement Emission
38
//===----------------------------------------------------------------------===//
39
40
684k
void CodeGenFunction::EmitStopPoint(const Stmt *S) {
41
684k
  if (CGDebugInfo *DI = getDebugInfo()) {
42
327k
    SourceLocation Loc;
43
327k
    Loc = S->getBeginLoc();
44
327k
    DI->EmitLocation(Builder, Loc);
45
46
327k
    LastStopPoint = Loc;
47
327k
  }
48
684k
}
49
50
756k
void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
51
756k
  assert(S && "Null statement?");
52
756k
  PGO.setCurrentStmt(S);
53
54
  // These statements have their own debug info handling.
55
756k
  if (EmitSimpleStmt(S, Attrs))
56
258k
    return;
57
58
  // Check if we are generating unreachable code.
59
497k
  if (!HaveInsertPoint()) {
60
    // If so, and the statement doesn't contain a label, then we do not need to
61
    // generate actual code. This is safe because (1) the current point is
62
    // unreachable, so we don't need to execute the code, and (2) we've already
63
    // handled the statements which update internal data structures (like the
64
    // local variable map) which could be used by subsequent statements.
65
136
    if (!ContainsLabel(S)) {
66
      // Verify that any decl statements were handled as simple, they may be in
67
      // scope of subsequent reachable statements.
68
131
      assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");
69
131
      return;
70
131
    }
71
72
    // Otherwise, make a new block to hold the code.
73
5
    EnsureInsertPoint();
74
5
  }
75
76
  // Generate a stoppoint if we are emitting debug info.
77
497k
  EmitStopPoint(S);
78
79
  // Ignore all OpenMP directives except for simd if OpenMP with Simd is
80
  // enabled.
81
497k
  if (getLangOpts().OpenMP && 
getLangOpts().OpenMPSimd92.4k
) {
82
47.0k
    if (const auto *D = dyn_cast<OMPExecutableDirective>(S)) {
83
14.0k
      EmitSimpleOMPExecutableDirective(*D);
84
14.0k
      return;
85
14.0k
    }
86
483k
  }
87
88
483k
  switch (S->getStmtClass()) {
89
0
  case Stmt::NoStmtClass:
90
0
  case Stmt::CXXCatchStmtClass:
91
0
  case Stmt::SEHExceptStmtClass:
92
0
  case Stmt::SEHFinallyStmtClass:
93
0
  case Stmt::MSDependentExistsStmtClass:
94
0
    llvm_unreachable("invalid statement class to emit generically");
95
0
  case Stmt::NullStmtClass:
96
0
  case Stmt::CompoundStmtClass:
97
0
  case Stmt::DeclStmtClass:
98
0
  case Stmt::LabelStmtClass:
99
0
  case Stmt::AttributedStmtClass:
100
0
  case Stmt::GotoStmtClass:
101
0
  case Stmt::BreakStmtClass:
102
0
  case Stmt::ContinueStmtClass:
103
0
  case Stmt::DefaultStmtClass:
104
0
  case Stmt::CaseStmtClass:
105
0
  case Stmt::SEHLeaveStmtClass:
106
0
    llvm_unreachable("should have emitted these statements as simple");
107
108
0
#define STMT(Type, Base)
109
0
#define ABSTRACT_STMT(Op)
110
0
#define EXPR(Type, Base) \
111
21.7M
  case Stmt::Type##Class:
112
239k
#include 
"clang/AST/StmtNodes.inc"0
113
239k
  {
114
    // Remember the block we came in on.
115
239k
    llvm::BasicBlock *incoming = Builder.GetInsertBlock();
116
239k
    assert(incoming && "expression emission must have an insertion point");
117
118
239k
    EmitIgnoredExpr(cast<Expr>(S));
119
120
239k
    llvm::BasicBlock *outgoing = Builder.GetInsertBlock();
121
239k
    assert(outgoing && "expression emission cleared block!");
122
123
    // The expression emitters assume (reasonably!) that the insertion
124
    // point is always set.  To maintain that, the call-emission code
125
    // for noreturn functions has to enter a new block with no
126
    // predecessors.  We want to kill that block and mark the current
127
    // insertion point unreachable in the common case of a call like
128
    // "exit();".  Since expression emission doesn't otherwise create
129
    // blocks with no predecessors, we can just test for that.
130
    // However, we must be careful not to do this to our incoming
131
    // block, because *statement* emission does sometimes create
132
    // reachable blocks which will have no predecessors until later in
133
    // the function.  This occurs with, e.g., labels that are not
134
    // reachable by fallthrough.
135
239k
    if (incoming != outgoing && 
outgoing->use_empty()20.4k
) {
136
2.32k
      outgoing->eraseFromParent();
137
2.32k
      Builder.ClearInsertionPoint();
138
2.32k
    }
139
239k
    break;
140
21.5M
  }
141
142
22
  case Stmt::IndirectGotoStmtClass:
143
22
    EmitIndirectGotoStmt(cast<IndirectGotoStmt>(*S)); break;
144
145
67.2k
  case Stmt::IfStmtClass:      EmitIfStmt(cast<IfStmt>(*S));              break;
146
1.68k
  case Stmt::WhileStmtClass:   EmitWhileStmt(cast<WhileStmt>(*S), Attrs); break;
147
633
  case Stmt::DoStmtClass:      EmitDoStmt(cast<DoStmt>(*S), Attrs);       break;
148
9.69k
  case Stmt::ForStmtClass:     EmitForStmt(cast<ForStmt>(*S), Attrs);     break;
149
150
145k
  case Stmt::ReturnStmtClass:  EmitReturnStmt(cast<ReturnStmt>(*S));      break;
151
152
486
  case Stmt::SwitchStmtClass:  EmitSwitchStmt(cast<SwitchStmt>(*S));      break;
153
1.45k
  case Stmt::GCCAsmStmtClass:  // Intentional fall-through.
154
1.60k
  case Stmt::MSAsmStmtClass:   EmitAsmStmt(cast<AsmStmt>(*S));            break;
155
53
  case Stmt::CoroutineBodyStmtClass:
156
53
    EmitCoroutineBody(cast<CoroutineBodyStmt>(*S));
157
53
    break;
158
52
  case Stmt::CoreturnStmtClass:
159
52
    EmitCoreturnStmt(cast<CoreturnStmt>(*S));
160
52
    break;
161
27
  case Stmt::CapturedStmtClass: {
162
27
    const CapturedStmt *CS = cast<CapturedStmt>(S);
163
27
    EmitCapturedStmt(*CS, CS->getCapturedRegionKind());
164
27
    }
165
27
    break;
166
219
  case Stmt::ObjCAtTryStmtClass:
167
219
    EmitObjCAtTryStmt(cast<ObjCAtTryStmt>(*S));
168
219
    break;
169
0
  case Stmt::ObjCAtCatchStmtClass:
170
0
    llvm_unreachable(
171
1.45k
                    "@catch statements should be handled by EmitObjCAtTryStmt");
172
0
  case Stmt::ObjCAtFinallyStmtClass:
173
0
    llvm_unreachable(
174
1.45k
                  "@finally statements should be handled by EmitObjCAtTryStmt");
175
50
  case Stmt::ObjCAtThrowStmtClass:
176
50
    EmitObjCAtThrowStmt(cast<ObjCAtThrowStmt>(*S));
177
50
    break;
178
13
  case Stmt::ObjCAtSynchronizedStmtClass:
179
13
    EmitObjCAtSynchronizedStmt(cast<ObjCAtSynchronizedStmt>(*S));
180
13
    break;
181
73
  case Stmt::ObjCForCollectionStmtClass:
182
73
    EmitObjCForCollectionStmt(cast<ObjCForCollectionStmt>(*S));
183
73
    break;
184
113
  case Stmt::ObjCAutoreleasePoolStmtClass:
185
113
    EmitObjCAutoreleasePoolStmt(cast<ObjCAutoreleasePoolStmt>(*S));
186
113
    break;
187
188
325
  case Stmt::CXXTryStmtClass:
189
325
    EmitCXXTryStmt(cast<CXXTryStmt>(*S));
190
325
    break;
191
123
  case Stmt::CXXForRangeStmtClass:
192
123
    EmitCXXForRangeStmt(cast<CXXForRangeStmt>(*S), Attrs);
193
123
    break;
194
128
  case Stmt::SEHTryStmtClass:
195
128
    EmitSEHTryStmt(cast<SEHTryStmt>(*S));
196
128
    break;
197
947
  case Stmt::OMPParallelDirectiveClass:
198
947
    EmitOMPParallelDirective(cast<OMPParallelDirective>(*S));
199
947
    break;
200
167
  case Stmt::OMPSimdDirectiveClass:
201
167
    EmitOMPSimdDirective(cast<OMPSimdDirective>(*S));
202
167
    break;
203
345
  case Stmt::OMPForDirectiveClass:
204
345
    EmitOMPForDirective(cast<OMPForDirective>(*S));
205
345
    break;
206
249
  case Stmt::OMPForSimdDirectiveClass:
207
249
    EmitOMPForSimdDirective(cast<OMPForSimdDirective>(*S));
208
249
    break;
209
62
  case Stmt::OMPSectionsDirectiveClass:
210
62
    EmitOMPSectionsDirective(cast<OMPSectionsDirective>(*S));
211
62
    break;
212
54
  case Stmt::OMPSectionDirectiveClass:
213
54
    EmitOMPSectionDirective(cast<OMPSectionDirective>(*S));
214
54
    break;
215
57
  case Stmt::OMPSingleDirectiveClass:
216
57
    EmitOMPSingleDirective(cast<OMPSingleDirective>(*S));
217
57
    break;
218
15
  case Stmt::OMPMasterDirectiveClass:
219
15
    EmitOMPMasterDirective(cast<OMPMasterDirective>(*S));
220
15
    break;
221
56
  case Stmt::OMPCriticalDirectiveClass:
222
56
    EmitOMPCriticalDirective(cast<OMPCriticalDirective>(*S));
223
56
    break;
224
207
  case Stmt::OMPParallelForDirectiveClass:
225
207
    EmitOMPParallelForDirective(cast<OMPParallelForDirective>(*S));
226
207
    break;
227
96
  case Stmt::OMPParallelForSimdDirectiveClass:
228
96
    EmitOMPParallelForSimdDirective(cast<OMPParallelForSimdDirective>(*S));
229
96
    break;
230
30
  case Stmt::OMPParallelMasterDirectiveClass:
231
30
    EmitOMPParallelMasterDirective(cast<OMPParallelMasterDirective>(*S));
232
30
    break;
233
26
  case Stmt::OMPParallelSectionsDirectiveClass:
234
26
    EmitOMPParallelSectionsDirective(cast<OMPParallelSectionsDirective>(*S));
235
26
    break;
236
181
  case Stmt::OMPTaskDirectiveClass:
237
181
    EmitOMPTaskDirective(cast<OMPTaskDirective>(*S));
238
181
    break;
239
16
  case Stmt::OMPTaskyieldDirectiveClass:
240
16
    EmitOMPTaskyieldDirective(cast<OMPTaskyieldDirective>(*S));
241
16
    break;
242
34
  case Stmt::OMPBarrierDirectiveClass:
243
34
    EmitOMPBarrierDirective(cast<OMPBarrierDirective>(*S));
244
34
    break;
245
12
  case Stmt::OMPTaskwaitDirectiveClass:
246
12
    EmitOMPTaskwaitDirective(cast<OMPTaskwaitDirective>(*S));
247
12
    break;
248
39
  case Stmt::OMPTaskgroupDirectiveClass:
249
39
    EmitOMPTaskgroupDirective(cast<OMPTaskgroupDirective>(*S));
250
39
    break;
251
40
  case Stmt::OMPFlushDirectiveClass:
252
40
    EmitOMPFlushDirective(cast<OMPFlushDirective>(*S));
253
40
    break;
254
14
  case Stmt::OMPDepobjDirectiveClass:
255
14
    EmitOMPDepobjDirective(cast<OMPDepobjDirective>(*S));
256
14
    break;
257
36
  case Stmt::OMPScanDirectiveClass:
258
36
    EmitOMPScanDirective(cast<OMPScanDirective>(*S));
259
36
    break;
260
40
  case Stmt::OMPOrderedDirectiveClass:
261
40
    EmitOMPOrderedDirective(cast<OMPOrderedDirective>(*S));
262
40
    break;
263
454
  case Stmt::OMPAtomicDirectiveClass:
264
454
    EmitOMPAtomicDirective(cast<OMPAtomicDirective>(*S));
265
454
    break;
266
4.89k
  case Stmt::OMPTargetDirectiveClass:
267
4.89k
    EmitOMPTargetDirective(cast<OMPTargetDirective>(*S));
268
4.89k
    break;
269
1.01k
  case Stmt::OMPTeamsDirectiveClass:
270
1.01k
    EmitOMPTeamsDirective(cast<OMPTeamsDirective>(*S));
271
1.01k
    break;
272
48
  case Stmt::OMPCancellationPointDirectiveClass:
273
48
    EmitOMPCancellationPointDirective(cast<OMPCancellationPointDirective>(*S));
274
48
    break;
275
163
  case Stmt::OMPCancelDirectiveClass:
276
163
    EmitOMPCancelDirective(cast<OMPCancelDirective>(*S));
277
163
    break;
278
143
  case Stmt::OMPTargetDataDirectiveClass:
279
143
    EmitOMPTargetDataDirective(cast<OMPTargetDataDirective>(*S));
280
143
    break;
281
103
  case Stmt::OMPTargetEnterDataDirectiveClass:
282
103
    EmitOMPTargetEnterDataDirective(cast<OMPTargetEnterDataDirective>(*S));
283
103
    break;
284
73
  case Stmt::OMPTargetExitDataDirectiveClass:
285
73
    EmitOMPTargetExitDataDirective(cast<OMPTargetExitDataDirective>(*S));
286
73
    break;
287
540
  case Stmt::OMPTargetParallelDirectiveClass:
288
540
    EmitOMPTargetParallelDirective(cast<OMPTargetParallelDirective>(*S));
289
540
    break;
290
343
  case Stmt::OMPTargetParallelForDirectiveClass:
291
343
    EmitOMPTargetParallelForDirective(cast<OMPTargetParallelForDirective>(*S));
292
343
    break;
293
39
  case Stmt::OMPTaskLoopDirectiveClass:
294
39
    EmitOMPTaskLoopDirective(cast<OMPTaskLoopDirective>(*S));
295
39
    break;
296
40
  case Stmt::OMPTaskLoopSimdDirectiveClass:
297
40
    EmitOMPTaskLoopSimdDirective(cast<OMPTaskLoopSimdDirective>(*S));
298
40
    break;
299
35
  case Stmt::OMPMasterTaskLoopDirectiveClass:
300
35
    EmitOMPMasterTaskLoopDirective(cast<OMPMasterTaskLoopDirective>(*S));
301
35
    break;
302
40
  case Stmt::OMPMasterTaskLoopSimdDirectiveClass:
303
40
    EmitOMPMasterTaskLoopSimdDirective(
304
40
        cast<OMPMasterTaskLoopSimdDirective>(*S));
305
40
    break;
306
33
  case Stmt::OMPParallelMasterTaskLoopDirectiveClass:
307
33
    EmitOMPParallelMasterTaskLoopDirective(
308
33
        cast<OMPParallelMasterTaskLoopDirective>(*S));
309
33
    break;
310
39
  case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:
311
39
    EmitOMPParallelMasterTaskLoopSimdDirective(
312
39
        cast<OMPParallelMasterTaskLoopSimdDirective>(*S));
313
39
    break;
314
110
  case Stmt::OMPDistributeDirectiveClass:
315
110
    EmitOMPDistributeDirective(cast<OMPDistributeDirective>(*S));
316
110
    break;
317
225
  case Stmt::OMPTargetUpdateDirectiveClass:
318
225
    EmitOMPTargetUpdateDirective(cast<OMPTargetUpdateDirective>(*S));
319
225
    break;
320
412
  case Stmt::OMPDistributeParallelForDirectiveClass:
321
412
    EmitOMPDistributeParallelForDirective(
322
412
        cast<OMPDistributeParallelForDirective>(*S));
323
412
    break;
324
308
  case Stmt::OMPDistributeParallelForSimdDirectiveClass:
325
308
    EmitOMPDistributeParallelForSimdDirective(
326
308
        cast<OMPDistributeParallelForSimdDirective>(*S));
327
308
    break;
328
150
  case Stmt::OMPDistributeSimdDirectiveClass:
329
150
    EmitOMPDistributeSimdDirective(cast<OMPDistributeSimdDirective>(*S));
330
150
    break;
331
310
  case Stmt::OMPTargetParallelForSimdDirectiveClass:
332
310
    EmitOMPTargetParallelForSimdDirective(
333
310
        cast<OMPTargetParallelForSimdDirective>(*S));
334
310
    break;
335
327
  case Stmt::OMPTargetSimdDirectiveClass:
336
327
    EmitOMPTargetSimdDirective(cast<OMPTargetSimdDirective>(*S));
337
327
    break;
338
114
  case Stmt::OMPTeamsDistributeDirectiveClass:
339
114
    EmitOMPTeamsDistributeDirective(cast<OMPTeamsDistributeDirective>(*S));
340
114
    break;
341
126
  case Stmt::OMPTeamsDistributeSimdDirectiveClass:
342
126
    EmitOMPTeamsDistributeSimdDirective(
343
126
        cast<OMPTeamsDistributeSimdDirective>(*S));
344
126
    break;
345
328
  case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:
346
328
    EmitOMPTeamsDistributeParallelForSimdDirective(
347
328
        cast<OMPTeamsDistributeParallelForSimdDirective>(*S));
348
328
    break;
349
340
  case Stmt::OMPTeamsDistributeParallelForDirectiveClass:
350
340
    EmitOMPTeamsDistributeParallelForDirective(
351
340
        cast<OMPTeamsDistributeParallelForDirective>(*S));
352
340
    break;
353
633
  case Stmt::OMPTargetTeamsDirectiveClass:
354
633
    EmitOMPTargetTeamsDirective(cast<OMPTargetTeamsDirective>(*S));
355
633
    break;
356
406
  case Stmt::OMPTargetTeamsDistributeDirectiveClass:
357
406
    EmitOMPTargetTeamsDistributeDirective(
358
406
        cast<OMPTargetTeamsDistributeDirective>(*S));
359
406
    break;
360
394
  case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass:
361
394
    EmitOMPTargetTeamsDistributeParallelForDirective(
362
394
        cast<OMPTargetTeamsDistributeParallelForDirective>(*S));
363
394
    break;
364
560
  case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
365
560
    EmitOMPTargetTeamsDistributeParallelForSimdDirective(
366
560
        cast<OMPTargetTeamsDistributeParallelForSimdDirective>(*S));
367
560
    break;
368
428
  case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
369
428
    EmitOMPTargetTeamsDistributeSimdDirective(
370
428
        cast<OMPTargetTeamsDistributeSimdDirective>(*S));
371
428
    break;
372
483k
  }
373
483k
}
374
375
bool CodeGenFunction::EmitSimpleStmt(const Stmt *S,
376
756k
                                     ArrayRef<const Attr *> Attrs) {
377
756k
  switch (S->getStmtClass()) {
378
497k
  default:
379
497k
    return false;
380
10.2k
  case Stmt::NullStmtClass:
381
10.2k
    break;
382
82.3k
  case Stmt::CompoundStmtClass:
383
82.3k
    EmitCompoundStmt(cast<CompoundStmt>(*S));
384
82.3k
    break;
385
153k
  case Stmt::DeclStmtClass:
386
153k
    EmitDeclStmt(cast<DeclStmt>(*S));
387
153k
    break;
388
200
  case Stmt::LabelStmtClass:
389
200
    EmitLabelStmt(cast<LabelStmt>(*S));
390
200
    break;
391
183
  case Stmt::AttributedStmtClass:
392
183
    EmitAttributedStmt(cast<AttributedStmt>(*S));
393
183
    break;
394
3.12k
  case Stmt::GotoStmtClass:
395
3.12k
    EmitGotoStmt(cast<GotoStmt>(*S));
396
3.12k
    break;
397
3.14k
  case Stmt::BreakStmtClass:
398
3.14k
    EmitBreakStmt(cast<BreakStmt>(*S));
399
3.14k
    break;
400
4.59k
  case Stmt::ContinueStmtClass:
401
4.59k
    EmitContinueStmt(cast<ContinueStmt>(*S));
402
4.59k
    break;
403
182
  case Stmt::DefaultStmtClass:
404
182
    EmitDefaultStmt(cast<DefaultStmt>(*S), Attrs);
405
182
    break;
406
972
  case Stmt::CaseStmtClass:
407
972
    EmitCaseStmt(cast<CaseStmt>(*S), Attrs);
408
972
    break;
409
10
  case Stmt::SEHLeaveStmtClass:
410
10
    EmitSEHLeaveStmt(cast<SEHLeaveStmt>(*S));
411
10
    break;
412
258k
  }
413
258k
  return true;
414
258k
}
415
416
/// EmitCompoundStmt - Emit a compound statement {..} node.  If GetLast is true,
417
/// this captures the expression result of the last sub-statement and returns it
418
/// (for use by the statement expression extension).
419
Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
420
86.5k
                                          AggValueSlot AggSlot) {
421
86.5k
  PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
422
86.5k
                             "LLVM IR generation of compound statement ('{}')");
423
424
  // Keep track of the current cleanup stack depth, including debug scopes.
425
86.5k
  LexicalScope Scope(*this, S.getSourceRange());
426
427
86.5k
  return EmitCompoundStmtWithoutScope(S, GetLast, AggSlot);
428
86.5k
}
429
430
Address
431
CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
432
                                              bool GetLast,
433
285k
                                              AggValueSlot AggSlot) {
434
435
285k
  const Stmt *ExprResult = S.getStmtExprResult();
436
285k
  assert((!GetLast || (GetLast && ExprResult)) &&
437
285k
         "If GetLast is true then the CompoundStmt must have a StmtExprResult");
438
439
285k
  Address RetAlloca = Address::invalid();
440
441
581k
  for (auto *CurStmt : S.body()) {
442
581k
    if (GetLast && 
ExprResult == CurStmt16.0k
) {
443
      // We have to special case labels here.  They are statements, but when put
444
      // at the end of a statement expression, they yield the value of their
445
      // subexpression.  Handle this by walking through all labels we encounter,
446
      // emitting them before we evaluate the subexpr.
447
      // Similar issues arise for attributed statements.
448
3.69k
      while (!isa<Expr>(ExprResult)) {
449
5
        if (const auto *LS = dyn_cast<LabelStmt>(ExprResult)) {
450
3
          EmitLabel(LS->getDecl());
451
3
          ExprResult = LS->getSubStmt();
452
2
        } else if (const auto *AS = dyn_cast<AttributedStmt>(ExprResult)) {
453
          // FIXME: Update this if we ever have attributes that affect the
454
          // semantics of an expression.
455
2
          ExprResult = AS->getSubStmt();
456
0
        } else {
457
0
          llvm_unreachable("unknown value statement");
458
0
        }
459
5
      }
460
461
3.68k
      EnsureInsertPoint();
462
463
3.68k
      const Expr *E = cast<Expr>(ExprResult);
464
3.68k
      QualType ExprTy = E->getType();
465
3.68k
      if (hasAggregateEvaluationKind(ExprTy)) {
466
645
        EmitAggExpr(E, AggSlot);
467
3.04k
      } else {
468
        // We can't return an RValue here because there might be cleanups at
469
        // the end of the StmtExpr.  Because of that, we have to emit the result
470
        // here into a temporary alloca.
471
3.04k
        RetAlloca = CreateMemTemp(ExprTy);
472
3.04k
        EmitAnyExprToMem(E, RetAlloca, Qualifiers(),
473
3.04k
                         /*IsInit*/ false);
474
3.04k
      }
475
578k
    } else {
476
578k
      EmitStmt(CurStmt);
477
578k
    }
478
581k
  }
479
480
285k
  return RetAlloca;
481
285k
}
482
483
771
void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {
484
771
  llvm::BranchInst *BI = dyn_cast<llvm::BranchInst>(BB->getTerminator());
485
486
  // If there is a cleanup stack, then we it isn't worth trying to
487
  // simplify this block (we would need to remove it from the scope map
488
  // and cleanup entry).
489
771
  if (!EHStack.empty())
490
59
    return;
491
492
  // Can only simplify direct branches.
493
712
  if (!BI || !BI->isUnconditional())
494
0
    return;
495
496
  // Can only simplify empty blocks.
497
712
  if (BI->getIterator() != BB->begin())
498
1
    return;
499
500
711
  BB->replaceAllUsesWith(BI->getSuccessor(0));
501
711
  BI->eraseFromParent();
502
711
  BB->eraseFromParent();
503
711
}
504
505
448k
void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) {
506
448k
  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
507
508
  // Fall out of the current block (if necessary).
509
448k
  EmitBranch(BB);
510
511
448k
  if (IsFinished && 
BB->use_empty()97.8k
) {
512
528
    delete BB;
513
528
    return;
514
528
  }
515
516
  // Place the block after the current block, if possible, or else at
517
  // the end of the function.
518
447k
  if (CurBB && 
CurBB->getParent()297k
)
519
297k
    CurFn->getBasicBlockList().insertAfter(CurBB->getIterator(), BB);
520
149k
  else
521
149k
    CurFn->getBasicBlockList().push_back(BB);
522
447k
  Builder.SetInsertPoint(BB);
523
447k
}
524
525
570k
void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
526
  // Emit a branch from the current block to the target one if this
527
  // was a real block.  If this was just a fall-through block after a
528
  // terminator, don't emit it.
529
570k
  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
530
531
570k
  if (!CurBB || 
CurBB->getTerminator()404k
) {
532
    // If there is no insert point or the previous block is already
533
    // terminated, don't touch it.
534
209k
  } else {
535
    // Otherwise, create a fall-through branch.
536
209k
    Builder.CreateBr(Target);
537
209k
  }
538
539
570k
  Builder.ClearInsertionPoint();
540
570k
}
541
542
817
void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
543
817
  bool inserted = false;
544
817
  for (llvm::User *u : block->users()) {
545
817
    if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(u)) {
546
817
      CurFn->getBasicBlockList().insertAfter(insn->getParent()->getIterator(),
547
817
                                             block);
548
817
      inserted = true;
549
817
      break;
550
817
    }
551
817
  }
552
553
817
  if (!inserted)
554
0
    CurFn->getBasicBlockList().push_back(block);
555
556
817
  Builder.SetInsertPoint(block);
557
817
}
558
559
CodeGenFunction::JumpDest
560
3.22k
CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
561
3.22k
  JumpDest &Dest = LabelMap[D];
562
3.22k
  if (Dest.isValid()) 
return Dest3.08k
;
563
564
  // Create, but don't insert, the new block.
565
140
  Dest = JumpDest(createBasicBlock(D->getName()),
566
140
                  EHScopeStack::stable_iterator::invalid(),
567
140
                  NextCleanupDestIndex++);
568
140
  return Dest;
569
140
}
570
571
203
void CodeGenFunction::EmitLabel(const LabelDecl *D) {
572
  // Add this label to the current lexical scope if we're within any
573
  // normal cleanups.  Jumps "in" to this label --- when permitted by
574
  // the language --- may need to be routed around such cleanups.
575
203
  if (EHStack.hasNormalCleanups() && 
CurLexicalScope10
)
576
0
    CurLexicalScope->addLabel(D);
577
578
203
  JumpDest &Dest = LabelMap[D];
579
580
  // If we didn't need a forward reference to this label, just go
581
  // ahead and create a destination at the current scope.
582
203
  if (!Dest.isValid()) {
583
63
    Dest = getJumpDestInCurrentScope(D->getName());
584
585
  // Otherwise, we need to give this label a target depth and remove
586
  // it from the branch-fixups list.
587
140
  } else {
588
140
    assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
589
140
    Dest.setScopeDepth(EHStack.stable_begin());
590
140
    ResolveBranchFixups(Dest.getBlock());
591
140
  }
592
593
203
  EmitBlock(Dest.getBlock());
594
595
  // Emit debug info for labels.
596
203
  if (CGDebugInfo *DI = getDebugInfo()) {
597
19
    if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
598
19
      DI->setLocation(D->getLocation());
599
19
      DI->EmitLabel(D, Builder);
600
19
    }
601
19
  }
602
603
203
  incrementProfileCounter(D->getStmt());
604
203
}
605
606
/// Change the cleanup scope of the labels in this lexical scope to
607
/// match the scope of the enclosing context.
608
0
void CodeGenFunction::LexicalScope::rescopeLabels() {
609
0
  assert(!Labels.empty());
610
0
  EHScopeStack::stable_iterator innermostScope
611
0
    = CGF.EHStack.getInnermostNormalCleanup();
612
613
  // Change the scope depth of all the labels.
614
0
  for (SmallVectorImpl<const LabelDecl*>::const_iterator
615
0
         i = Labels.begin(), e = Labels.end(); i != e; ++i) {
616
0
    assert(CGF.LabelMap.count(*i));
617
0
    JumpDest &dest = CGF.LabelMap.find(*i)->second;
618
0
    assert(dest.getScopeDepth().isValid());
619
0
    assert(innermostScope.encloses(dest.getScopeDepth()));
620
0
    dest.setScopeDepth(innermostScope);
621
0
  }
622
623
  // Reparent the labels if the new scope also has cleanups.
624
0
  if (innermostScope != EHScopeStack::stable_end() && ParentScope) {
625
0
    ParentScope->Labels.append(Labels.begin(), Labels.end());
626
0
  }
627
0
}
628
629
630
200
void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
631
200
  EmitLabel(S.getDecl());
632
200
  EmitStmt(S.getSubStmt());
633
200
}
634
635
183
void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
636
183
  bool nomerge = false;
637
183
  for (const auto *A : S.getAttrs())
638
251
    if (A->getKind() == attr::NoMerge) {
639
6
      nomerge = true;
640
6
      break;
641
6
    }
642
183
  SaveAndRestore<bool> save_nomerge(InNoMergeAttributedStmt, nomerge);
643
183
  EmitStmt(S.getSubStmt(), S.getAttrs());
644
183
}
645
646
3.12k
void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
647
  // If this code is reachable then emit a stop point (if generating
648
  // debug info). We have to do this ourselves because we are on the
649
  // "simple" statement path.
650
3.12k
  if (HaveInsertPoint())
651
3.11k
    EmitStopPoint(&S);
652
653
3.12k
  EmitBranchThroughCleanup(getJumpDestForLabel(S.getLabel()));
654
3.12k
}
655
656
657
22
void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
658
22
  if (const LabelDecl *Target = S.getConstantTarget()) {
659
0
    EmitBranchThroughCleanup(getJumpDestForLabel(Target));
660
0
    return;
661
0
  }
662
663
  // Ensure that we have an i8* for our PHI node.
664
22
  llvm::Value *V = Builder.CreateBitCast(EmitScalarExpr(S.getTarget()),
665
22
                                         Int8PtrTy, "addr");
666
22
  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
667
668
  // Get the basic block for the indirect goto.
669
22
  llvm::BasicBlock *IndGotoBB = GetIndirectGotoBlock();
670
671
  // The first instruction in the block has to be the PHI for the switch dest,
672
  // add an entry for this branch.
673
22
  cast<llvm::PHINode>(IndGotoBB->begin())->addIncoming(V, CurBB);
674
675
22
  EmitBranch(IndGotoBB);
676
22
}
677
678
67.2k
void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
679
  // C99 6.8.4.1: The first substatement is executed if the expression compares
680
  // unequal to 0.  The condition must be a scalar type.
681
67.2k
  LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
682
683
67.2k
  if (S.getInit())
684
14
    EmitStmt(S.getInit());
685
686
67.2k
  if (S.getConditionVariable())
687
27
    EmitDecl(*S.getConditionVariable());
688
689
  // If the condition constant folds and can be elided, try to avoid emitting
690
  // the condition and the dead arm of the if/else.
691
67.2k
  bool CondConstant;
692
67.2k
  if (ConstantFoldsToSimpleInteger(S.getCond(), CondConstant,
693
1.25k
                                   S.isConstexpr())) {
694
    // Figure out which block (then or else) is executed.
695
1.25k
    const Stmt *Executed = S.getThen();
696
1.25k
    const Stmt *Skipped  = S.getElse();
697
1.25k
    if (!CondConstant)  // Condition false?
698
1.15k
      std::swap(Executed, Skipped);
699
700
    // If the skipped block has no labels in it, just emit the executed block.
701
    // This avoids emitting dead code and simplifies the CFG substantially.
702
1.25k
    if (S.isConstexpr() || 
!ContainsLabel(Skipped)1.25k
) {
703
1.25k
      if (CondConstant)
704
105
        incrementProfileCounter(&S);
705
1.25k
      if (Executed) {
706
1.13k
        RunCleanupsScope ExecutedScope(*this);
707
1.13k
        EmitStmt(Executed);
708
1.13k
      }
709
1.25k
      return;
710
1.25k
    }
711
65.9k
  }
712
713
  // Otherwise, the condition did not fold, or we couldn't elide it.  Just emit
714
  // the conditional branch.
715
65.9k
  llvm::BasicBlock *ThenBlock = createBasicBlock("if.then");
716
65.9k
  llvm::BasicBlock *ContBlock = createBasicBlock("if.end");
717
65.9k
  llvm::BasicBlock *ElseBlock = ContBlock;
718
65.9k
  if (S.getElse())
719
10.1k
    ElseBlock = createBasicBlock("if.else");
720
721
  // Prefer the PGO based weights over the likelihood attribute.
722
  // When the build isn't optimized the metadata isn't used, so don't generate
723
  // it.
724
65.9k
  Stmt::Likelihood LH = Stmt::LH_None;
725
65.9k
  uint64_t Count = getProfileCount(S.getThen());
726
65.9k
  if (!Count && 
CGM.getCodeGenOpts().OptimizationLevel65.8k
)
727
3.53k
    LH = Stmt::getLikelihood(S.getThen(), S.getElse());
728
65.9k
  EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, Count, LH);
729
730
  // Emit the 'then' code.
731
65.9k
  EmitBlock(ThenBlock);
732
65.9k
  incrementProfileCounter(&S);
733
65.9k
  {
734
65.9k
    RunCleanupsScope ThenScope(*this);
735
65.9k
    EmitStmt(S.getThen());
736
65.9k
  }
737
65.9k
  EmitBranch(ContBlock);
738
739
  // Emit the 'else' code if present.
740
65.9k
  if (const Stmt *Else = S.getElse()) {
741
10.1k
    {
742
      // There is no need to emit line number for an unconditional branch.
743
10.1k
      auto NL = ApplyDebugLocation::CreateEmpty(*this);
744
10.1k
      EmitBlock(ElseBlock);
745
10.1k
    }
746
10.1k
    {
747
10.1k
      RunCleanupsScope ElseScope(*this);
748
10.1k
      EmitStmt(Else);
749
10.1k
    }
750
10.1k
    {
751
      // There is no need to emit line number for an unconditional branch.
752
10.1k
      auto NL = ApplyDebugLocation::CreateEmpty(*this);
753
10.1k
      EmitBranch(ContBlock);
754
10.1k
    }
755
10.1k
  }
756
757
  // Emit the continuation block for code after the if.
758
65.9k
  EmitBlock(ContBlock, true);
759
65.9k
}
760
761
void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
762
1.68k
                                    ArrayRef<const Attr *> WhileAttrs) {
763
  // Emit the header for the loop, which will also become
764
  // the continue target.
765
1.68k
  JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond");
766
1.68k
  EmitBlock(LoopHeader.getBlock());
767
768
  // Create an exit block for when the condition fails, which will
769
  // also become the break target.
770
1.68k
  JumpDest LoopExit = getJumpDestInCurrentScope("while.end");
771
772
  // Store the blocks to use for break and continue.
773
1.68k
  BreakContinueStack.push_back(BreakContinue(LoopExit, LoopHeader));
774
775
  // C++ [stmt.while]p2:
776
  //   When the condition of a while statement is a declaration, the
777
  //   scope of the variable that is declared extends from its point
778
  //   of declaration (3.3.2) to the end of the while statement.
779
  //   [...]
780
  //   The object created in a condition is destroyed and created
781
  //   with each iteration of the loop.
782
1.68k
  RunCleanupsScope ConditionScope(*this);
783
784
1.68k
  if (S.getConditionVariable())
785
5
    EmitDecl(*S.getConditionVariable());
786
787
  // Evaluate the conditional in the while header.  C99 6.8.5.1: The
788
  // evaluation of the controlling expression takes place before each
789
  // execution of the loop body.
790
1.68k
  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
791
792
  // while(1) is common, avoid extra exit blocks.  Be sure
793
  // to correctly handle break/continue though.
794
1.68k
  bool EmitBoolCondBranch = true;
795
1.68k
  bool LoopMustProgress = false;
796
1.68k
  if (llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal)) {
797
336
    if (C->isOne()) {
798
320
      EmitBoolCondBranch = false;
799
320
      FnIsMustProgress = false;
800
320
    }
801
1.35k
  } else if (LanguageRequiresProgress())
802
1.33k
    LoopMustProgress = true;
803
804
1.68k
  const SourceRange &R = S.getSourceRange();
805
1.68k
  LoopStack.push(LoopHeader.getBlock(), CGM.getContext(), CGM.getCodeGenOpts(),
806
1.68k
                 WhileAttrs, SourceLocToDebugLoc(R.getBegin()),
807
1.68k
                 SourceLocToDebugLoc(R.getEnd()), LoopMustProgress);
808
809
  // As long as the condition is true, go to the loop body.
810
1.68k
  llvm::BasicBlock *LoopBody = createBasicBlock("while.body");
811
1.68k
  if (EmitBoolCondBranch) {
812
1.36k
    llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
813
1.36k
    if (ConditionScope.requiresCleanups())
814
3
      ExitBlock = createBasicBlock("while.exit");
815
1.36k
    llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
816
1.36k
        S.getCond(), getProfileCount(S.getBody()), S.getBody());
817
1.36k
    Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, Weights);
818
819
1.36k
    if (ExitBlock != LoopExit.getBlock()) {
820
3
      EmitBlock(ExitBlock);
821
3
      EmitBranchThroughCleanup(LoopExit);
822
3
    }
823
320
  } else if (const Attr *A = Stmt::getLikelihoodAttr(S.getBody())) {
824
2
    CGM.getDiags().Report(A->getLocation(),
825
2
                          diag::warn_attribute_has_no_effect_on_infinite_loop)
826
2
        << A << A->getRange();
827
2
    CGM.getDiags().Report(
828
2
        S.getWhileLoc(),
829
2
        diag::note_attribute_has_no_effect_on_infinite_loop_here)
830
2
        << SourceRange(S.getWhileLoc(), S.getRParenLoc());
831
2
  }
832
833
  // Emit the loop body.  We have to emit this in a cleanup scope
834
  // because it might be a singleton DeclStmt.
835
1.68k
  {
836
1.68k
    RunCleanupsScope BodyScope(*this);
837
1.68k
    EmitBlock(LoopBody);
838
1.68k
    incrementProfileCounter(&S);
839
1.68k
    EmitStmt(S.getBody());
840
1.68k
  }
841
842
1.68k
  BreakContinueStack.pop_back();
843
844
  // Immediately force cleanup.
845
1.68k
  ConditionScope.ForceCleanup();
846
847
1.68k
  EmitStopPoint(&S);
848
  // Branch to the loop header again.
849
1.68k
  EmitBranch(LoopHeader.getBlock());
850
851
1.68k
  LoopStack.pop();
852
853
  // Emit the exit block.
854
1.68k
  EmitBlock(LoopExit.getBlock(), true);
855
856
  // The LoopHeader typically is just a branch if we skipped emitting
857
  // a branch, try to erase it.
858
1.68k
  if (!EmitBoolCondBranch)
859
320
    SimplifyForwardingBlocks(LoopHeader.getBlock());
860
1.68k
}
861
862
void CodeGenFunction::EmitDoStmt(const DoStmt &S,
863
633
                                 ArrayRef<const Attr *> DoAttrs) {
864
633
  JumpDest LoopExit = getJumpDestInCurrentScope("do.end");
865
633
  JumpDest LoopCond = getJumpDestInCurrentScope("do.cond");
866
867
633
  uint64_t ParentCount = getCurrentProfileCount();
868
869
  // Store the blocks to use for break and continue.
870
633
  BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond));
871
872
  // Emit the body of the loop.
873
633
  llvm::BasicBlock *LoopBody = createBasicBlock("do.body");
874
875
633
  EmitBlockWithFallThrough(LoopBody, &S);
876
633
  {
877
633
    RunCleanupsScope BodyScope(*this);
878
633
    EmitStmt(S.getBody());
879
633
  }
880
881
633
  EmitBlock(LoopCond.getBlock());
882
883
  // C99 6.8.5.2: "The evaluation of the controlling expression takes place
884
  // after each execution of the loop body."
885
886
  // Evaluate the conditional in the while header.
887
  // C99 6.8.5p2/p4: The first substatement is executed if the expression
888
  // compares unequal to 0.  The condition must be a scalar type.
889
633
  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
890
891
633
  BreakContinueStack.pop_back();
892
893
  // "do {} while (0)" is common in macros, avoid extra blocks.  Be sure
894
  // to correctly handle break/continue though.
895
633
  bool EmitBoolCondBranch = true;
896
633
  bool LoopMustProgress = false;
897
633
  if (llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal)) {
898
484
    if (C->isZero())
899
451
      EmitBoolCondBranch = false;
900
33
    else if (C->isOne())
901
33
      FnIsMustProgress = false;
902
149
  } else if (LanguageRequiresProgress())
903
137
    LoopMustProgress = true;
904
905
633
  const SourceRange &R = S.getSourceRange();
906
633
  LoopStack.push(LoopBody, CGM.getContext(), CGM.getCodeGenOpts(), DoAttrs,
907
633
                 SourceLocToDebugLoc(R.getBegin()),
908
633
                 SourceLocToDebugLoc(R.getEnd()), LoopMustProgress);
909
910
  // As long as the condition is true, iterate the loop.
911
633
  if (EmitBoolCondBranch) {
912
182
    uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount;
913
182
    Builder.CreateCondBr(
914
182
        BoolCondVal, LoopBody, LoopExit.getBlock(),
915
182
        createProfileWeightsForLoop(S.getCond(), BackedgeCount));
916
182
  }
917
918
633
  LoopStack.pop();
919
920
  // Emit the exit block.
921
633
  EmitBlock(LoopExit.getBlock());
922
923
  // The DoCond block typically is just a branch if we skipped
924
  // emitting a branch, try to erase it.
925
633
  if (!EmitBoolCondBranch)
926
451
    SimplifyForwardingBlocks(LoopCond.getBlock());
927
633
}
928
929
void CodeGenFunction::EmitForStmt(const ForStmt &S,
930
9.69k
                                  ArrayRef<const Attr *> ForAttrs) {
931
9.69k
  JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
932
933
9.69k
  LexicalScope ForScope(*this, S.getSourceRange());
934
935
  // Evaluate the first part before the loop.
936
9.69k
  if (S.getInit())
937
9.26k
    EmitStmt(S.getInit());
938
939
  // Start the loop with a block that tests the condition.
940
  // If there's an increment, the continue scope will be overwritten
941
  // later.
942
9.69k
  JumpDest Continue = getJumpDestInCurrentScope("for.cond");
943
9.69k
  llvm::BasicBlock *CondBlock = Continue.getBlock();
944
9.69k
  EmitBlock(CondBlock);
945
946
9.69k
  bool LoopMustProgress = false;
947
9.69k
  Expr::EvalResult Result;
948
9.69k
  if (LanguageRequiresProgress()) {
949
9.63k
    if (!S.getCond()) {
950
42
      FnIsMustProgress = false;
951
9.59k
    } else if (!S.getCond()->EvaluateAsInt(Result, getContext())) {
952
9.57k
      LoopMustProgress = true;
953
9.57k
    }
954
9.63k
  }
955
956
9.69k
  const SourceRange &R = S.getSourceRange();
957
9.69k
  LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs,
958
9.69k
                 SourceLocToDebugLoc(R.getBegin()),
959
9.69k
                 SourceLocToDebugLoc(R.getEnd()), LoopMustProgress);
960
961
  // If the for loop doesn't have an increment we can just use the
962
  // condition as the continue block.  Otherwise we'll need to create
963
  // a block for it (in the current scope, i.e. in the scope of the
964
  // condition), and that we will become our continue block.
965
9.69k
  if (S.getInc())
966
9.53k
    Continue = getJumpDestInCurrentScope("for.inc");
967
968
  // Store the blocks to use for break and continue.
969
9.69k
  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
970
971
  // Create a cleanup scope for the condition variable cleanups.
972
9.69k
  LexicalScope ConditionScope(*this, S.getSourceRange());
973
974
9.69k
  if (S.getCond()) {
975
    // If the for statement has a condition scope, emit the local variable
976
    // declaration.
977
9.65k
    if (S.getConditionVariable()) {
978
3
      EmitDecl(*S.getConditionVariable());
979
3
    }
980
981
9.65k
    llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
982
    // If there are any cleanups between here and the loop-exit scope,
983
    // create a block to stage a loop exit along.
984
9.65k
    if (ForScope.requiresCleanups())
985
133
      ExitBlock = createBasicBlock("for.cond.cleanup");
986
987
    // As long as the condition is true, iterate the loop.
988
9.65k
    llvm::BasicBlock *ForBody = createBasicBlock("for.body");
989
990
    // C99 6.8.5p2/p4: The first substatement is executed if the expression
991
    // compares unequal to 0.  The condition must be a scalar type.
992
9.65k
    llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
993
9.65k
    llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
994
9.65k
        S.getCond(), getProfileCount(S.getBody()), S.getBody());
995
996
9.65k
    if (llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal))
997
26
      if (C->isOne())
998
26
        FnIsMustProgress = false;
999
1000
9.65k
    Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
1001
1002
9.65k
    if (ExitBlock != LoopExit.getBlock()) {
1003
133
      EmitBlock(ExitBlock);
1004
133
      EmitBranchThroughCleanup(LoopExit);
1005
133
    }
1006
1007
9.65k
    EmitBlock(ForBody);
1008
42
  } else {
1009
    // Treat it as a non-zero constant.  Don't even create a new block for the
1010
    // body, just fall into it.
1011
42
  }
1012
9.69k
  incrementProfileCounter(&S);
1013
1014
9.69k
  {
1015
    // Create a separate cleanup scope for the body, in case it is not
1016
    // a compound statement.
1017
9.69k
    RunCleanupsScope BodyScope(*this);
1018
9.69k
    EmitStmt(S.getBody());
1019
9.69k
  }
1020
1021
  // If there is an increment, emit it next.
1022
9.69k
  if (S.getInc()) {
1023
9.53k
    EmitBlock(Continue.getBlock());
1024
9.53k
    EmitStmt(S.getInc());
1025
9.53k
  }
1026
1027
9.69k
  BreakContinueStack.pop_back();
1028
1029
9.69k
  ConditionScope.ForceCleanup();
1030
1031
9.69k
  EmitStopPoint(&S);
1032
9.69k
  EmitBranch(CondBlock);
1033
1034
9.69k
  ForScope.ForceCleanup();
1035
1036
9.69k
  LoopStack.pop();
1037
1038
  // Emit the fall-through block.
1039
9.69k
  EmitBlock(LoopExit.getBlock(), true);
1040
9.69k
}
1041
1042
void
1043
CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
1044
123
                                     ArrayRef<const Attr *> ForAttrs) {
1045
123
  JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
1046
1047
123
  LexicalScope ForScope(*this, S.getSourceRange());
1048
1049
  // Evaluate the first pieces before the loop.
1050
123
  if (S.getInit())
1051
2
    EmitStmt(S.getInit());
1052
123
  EmitStmt(S.getRangeStmt());
1053
123
  EmitStmt(S.getBeginStmt());
1054
123
  EmitStmt(S.getEndStmt());
1055
1056
  // Start the loop with a block that tests the condition.
1057
  // If there's an increment, the continue scope will be overwritten
1058
  // later.
1059
123
  llvm::BasicBlock *CondBlock = createBasicBlock("for.cond");
1060
123
  EmitBlock(CondBlock);
1061
1062
123
  const SourceRange &R = S.getSourceRange();
1063
123
  LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs,
1064
123
                 SourceLocToDebugLoc(R.getBegin()),
1065
123
                 SourceLocToDebugLoc(R.getEnd()));
1066
1067
  // If there are any cleanups between here and the loop-exit scope,
1068
  // create a block to stage a loop exit along.
1069
123
  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1070
123
  if (ForScope.requiresCleanups())
1071
18
    ExitBlock = createBasicBlock("for.cond.cleanup");
1072
1073
  // The loop body, consisting of the specified body and the loop variable.
1074
123
  llvm::BasicBlock *ForBody = createBasicBlock("for.body");
1075
1076
  // The body is executed if the expression, contextually converted
1077
  // to bool, is true.
1078
123
  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
1079
123
  llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
1080
123
      S.getCond(), getProfileCount(S.getBody()), S.getBody());
1081
123
  Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
1082
1083
123
  if (ExitBlock != LoopExit.getBlock()) {
1084
18
    EmitBlock(ExitBlock);
1085
18
    EmitBranchThroughCleanup(LoopExit);
1086
18
  }
1087
1088
123
  EmitBlock(ForBody);
1089
123
  incrementProfileCounter(&S);
1090
1091
  // Create a block for the increment. In case of a 'continue', we jump there.
1092
123
  JumpDest Continue = getJumpDestInCurrentScope("for.inc");
1093
1094
  // Store the blocks to use for break and continue.
1095
123
  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1096
1097
123
  {
1098
    // Create a separate cleanup scope for the loop variable and body.
1099
123
    LexicalScope BodyScope(*this, S.getSourceRange());
1100
123
    EmitStmt(S.getLoopVarStmt());
1101
123
    EmitStmt(S.getBody());
1102
123
  }
1103
1104
123
  EmitStopPoint(&S);
1105
  // If there is an increment, emit it next.
1106
123
  EmitBlock(Continue.getBlock());
1107
123
  EmitStmt(S.getInc());
1108
1109
123
  BreakContinueStack.pop_back();
1110
1111
123
  EmitBranch(CondBlock);
1112
1113
123
  ForScope.ForceCleanup();
1114
1115
123
  LoopStack.pop();
1116
1117
  // Emit the fall-through block.
1118
123
  EmitBlock(LoopExit.getBlock(), true);
1119
123
}
1120
1121
449
void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
1122
449
  if (RV.isScalar()) {
1123
449
    Builder.CreateStore(RV.getScalarVal(), ReturnValue);
1124
0
  } else if (RV.isAggregate()) {
1125
0
    LValue Dest = MakeAddrLValue(ReturnValue, Ty);
1126
0
    LValue Src = MakeAddrLValue(RV.getAggregateAddress(), Ty);
1127
0
    EmitAggregateCopy(Dest, Src, Ty, getOverlapForReturnValue());
1128
0
  } else {
1129
0
    EmitStoreOfComplex(RV.getComplexVal(), MakeAddrLValue(ReturnValue, Ty),
1130
0
                       /*init*/ true);
1131
0
  }
1132
449
  EmitBranchThroughCleanup(ReturnBlock);
1133
449
}
1134
1135
namespace {
1136
// RAII struct used to save and restore a return statment's result expression.
1137
struct SaveRetExprRAII {
1138
  SaveRetExprRAII(const Expr *RetExpr, CodeGenFunction &CGF)
1139
145k
      : OldRetExpr(CGF.RetExpr), CGF(CGF) {
1140
145k
    CGF.RetExpr = RetExpr;
1141
145k
  }
1142
145k
  ~SaveRetExprRAII() { CGF.RetExpr = OldRetExpr; }
1143
  const Expr *OldRetExpr;
1144
  CodeGenFunction &CGF;
1145
};
1146
} // namespace
1147
1148
/// EmitReturnStmt - Note that due to GCC extensions, this can have an operand
1149
/// if the function returns void, or may be missing one if the function returns
1150
/// non-void.  Fun stuff :).
1151
145k
void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
1152
145k
  if (requiresReturnValueCheck()) {
1153
15
    llvm::Constant *SLoc = EmitCheckSourceLocation(S.getBeginLoc());
1154
15
    auto *SLocPtr =
1155
15
        new llvm::GlobalVariable(CGM.getModule(), SLoc->getType(), false,
1156
15
                                 llvm::GlobalVariable::PrivateLinkage, SLoc);
1157
15
    SLocPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1158
15
    CGM.getSanitizerMetadata()->disableSanitizerForGlobal(SLocPtr);
1159
15
    assert(ReturnLocation.isValid() && "No valid return location");
1160
15
    Builder.CreateStore(Builder.CreateBitCast(SLocPtr, Int8PtrTy),
1161
15
                        ReturnLocation);
1162
15
  }
1163
1164
  // Returning from an outlined SEH helper is UB, and we already warn on it.
1165
145k
  if (IsOutlinedSEHHelper) {
1166
8
    Builder.CreateUnreachable();
1167
8
    Builder.ClearInsertionPoint();
1168
8
  }
1169
1170
  // Emit the result value, even if unused, to evaluate the side effects.
1171
145k
  const Expr *RV = S.getRetValue();
1172
1173
  // Record the result expression of the return statement. The recorded
1174
  // expression is used to determine whether a block capture's lifetime should
1175
  // end at the end of the full expression as opposed to the end of the scope
1176
  // enclosing the block expression.
1177
  //
1178
  // This permits a small, easily-implemented exception to our over-conservative
1179
  // rules about not jumping to statements following block literals with
1180
  // non-trivial cleanups.
1181
145k
  SaveRetExprRAII SaveRetExpr(RV, *this);
1182
1183
145k
  RunCleanupsScope cleanupScope(*this);
1184
145k
  if (const auto *EWC = dyn_cast_or_null<ExprWithCleanups>(RV))
1185
5.13k
    RV = EWC->getSubExpr();
1186
  // FIXME: Clean this up by using an LValue for ReturnTemp,
1187
  // EmitStoreThroughLValue, and EmitAnyExpr.
1188
  // Check if the NRVO candidate was not globalized in OpenMP mode.
1189
145k
  if (getLangOpts().ElideConstructors && 
S.getNRVOCandidate()145k
&&
1190
1.31k
      S.getNRVOCandidate()->isNRVOVariable() &&
1191
1.30k
      (!getLangOpts().OpenMP ||
1192
84
       !CGM.getOpenMPRuntime()
1193
84
            .getAddressOfLocalVariable(*this, S.getNRVOCandidate())
1194
1.29k
            .isValid())) {
1195
    // Apply the named return value optimization for this return statement,
1196
    // which means doing nothing: the appropriate result has already been
1197
    // constructed into the NRVO variable.
1198
1199
    // If there is an NRVO flag for this variable, set it to 1 into indicate
1200
    // that the cleanup code should not destroy the variable.
1201
1.29k
    if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
1202
201
      Builder.CreateFlagStore(Builder.getTrue(), NRVOFlag);
1203
144k
  } else if (!ReturnValue.isValid() || 
(140k
RV140k
&&
RV->getType()->isVoidType()140k
)) {
1204
    // Make sure not to return anything, but evaluate the expression
1205
    // for side effects.
1206
3.34k
    if (RV)
1207
2.50k
      EmitAnyExpr(RV);
1208
140k
  } else if (!RV) {
1209
    // Do nothing (return value is left uninitialized)
1210
140k
  } else if (FnRetTy->isReferenceType()) {
1211
    // If this function returns a reference, take the address of the expression
1212
    // rather than the value.
1213
22.0k
    RValue Result = EmitReferenceBindingToExpr(RV);
1214
22.0k
    Builder.CreateStore(Result.getScalarVal(), ReturnValue);
1215
118k
  } else {
1216
118k
    switch (getEvaluationKind(RV->getType())) {
1217
112k
    case TEK_Scalar:
1218
112k
      Builder.CreateStore(EmitScalarExpr(RV), ReturnValue);
1219
112k
      break;
1220
652
    case TEK_Complex:
1221
652
      EmitComplexExprIntoLValue(RV, MakeAddrLValue(ReturnValue, RV->getType()),
1222
652
                                /*isInit*/ true);
1223
652
      break;
1224
5.76k
    case TEK_Aggregate:
1225
5.76k
      EmitAggExpr(RV, AggValueSlot::forAddr(
1226
5.76k
                          ReturnValue, Qualifiers(),
1227
5.76k
                          AggValueSlot::IsDestructed,
1228
5.76k
                          AggValueSlot::DoesNotNeedGCBarriers,
1229
5.76k
                          AggValueSlot::IsNotAliased,
1230
5.76k
                          getOverlapForReturnValue()));
1231
5.76k
      break;
1232
145k
    }
1233
145k
  }
1234
1235
145k
  ++NumReturnExprs;
1236
145k
  if (!RV || 
RV->isEvaluatable(getContext())144k
)
1237
14.8k
    ++NumSimpleReturnExprs;
1238
1239
145k
  cleanupScope.ForceCleanup();
1240
145k
  EmitBranchThroughCleanup(ReturnBlock);
1241
145k
}
1242
1243
153k
void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
1244
  // As long as debug info is modeled with instructions, we have to ensure we
1245
  // have a place to insert here and write the stop point here.
1246
153k
  if (HaveInsertPoint())
1247
153k
    EmitStopPoint(&S);
1248
1249
153k
  for (const auto *I : S.decls())
1250
155k
    EmitDecl(*I);
1251
153k
}
1252
1253
3.14k
void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
1254
3.14k
  assert(!BreakContinueStack.empty() && "break stmt not in a loop or switch!");
1255
1256
  // If this code is reachable then emit a stop point (if generating
1257
  // debug info). We have to do this ourselves because we are on the
1258
  // "simple" statement path.
1259
3.14k
  if (HaveInsertPoint())
1260
3.14k
    EmitStopPoint(&S);
1261
1262
3.14k
  EmitBranchThroughCleanup(BreakContinueStack.back().BreakBlock);
1263
3.14k
}
1264
1265
4.59k
void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
1266
4.59k
  assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
1267
1268
  // If this code is reachable then emit a stop point (if generating
1269
  // debug info). We have to do this ourselves because we are on the
1270
  // "simple" statement path.
1271
4.59k
  if (HaveInsertPoint())
1272
4.58k
    EmitStopPoint(&S);
1273
1274
4.59k
  EmitBranchThroughCleanup(BreakContinueStack.back().ContinueBlock);
1275
4.59k
}
1276
1277
/// EmitCaseStmtRange - If case statement range is not too big then
1278
/// add multiple cases to switch instruction, one for each value within
1279
/// the range. If range is too big then emit "if" condition check.
1280
void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
1281
43
                                        ArrayRef<const Attr *> Attrs) {
1282
43
  assert(S.getRHS() && "Expected RHS value in CaseStmt");
1283
1284
43
  llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(getContext());
1285
43
  llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(getContext());
1286
1287
  // Emit the code for this case. We do this first to make sure it is
1288
  // properly chained from our predecessor before generating the
1289
  // switch machinery to enter this block.
1290
43
  llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
1291
43
  EmitBlockWithFallThrough(CaseDest, &S);
1292
43
  EmitStmt(S.getSubStmt());
1293
1294
  // If range is empty, do nothing.
1295
43
  if (LHS.isSigned() ? 
RHS.slt(LHS)39
:
RHS.ult(LHS)4
)
1296
4
    return;
1297
1298
39
  Stmt::Likelihood LH = Stmt::getLikelihood(Attrs);
1299
39
  llvm::APInt Range = RHS - LHS;
1300
  // FIXME: parameters such as this should not be hardcoded.
1301
39
  if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) {
1302
    // Range is small enough to add multiple switch instruction cases.
1303
22
    uint64_t Total = getProfileCount(&S);
1304
22
    unsigned NCases = Range.getZExtValue() + 1;
1305
    // We only have one region counter for the entire set of cases here, so we
1306
    // need to divide the weights evenly between the generated cases, ensuring
1307
    // that the total weight is preserved. E.g., a weight of 5 over three cases
1308
    // will be distributed as weights of 2, 2, and 1.
1309
22
    uint64_t Weight = Total / NCases, Rem = Total % NCases;
1310
103
    for (unsigned I = 0; I != NCases; 
++I81
) {
1311
81
      if (SwitchWeights)
1312
24
        SwitchWeights->push_back(Weight + (Rem ? 
14
:
020
));
1313
57
      else if (SwitchLikelihood)
1314
35
        SwitchLikelihood->push_back(LH);
1315
1316
81
      if (Rem)
1317
4
        Rem--;
1318
81
      SwitchInsn->addCase(Builder.getInt(LHS), CaseDest);
1319
81
      ++LHS;
1320
81
    }
1321
22
    return;
1322
22
  }
1323
1324
  // The range is too big. Emit "if" condition into a new block,
1325
  // making sure to save and restore the current insertion point.
1326
17
  llvm::BasicBlock *RestoreBB = Builder.GetInsertBlock();
1327
1328
  // Push this test onto the chain of range checks (which terminates
1329
  // in the default basic block). The switch's default will be changed
1330
  // to the top of this chain after switch emission is complete.
1331
17
  llvm::BasicBlock *FalseDest = CaseRangeBlock;
1332
17
  CaseRangeBlock = createBasicBlock("sw.caserange");
1333
1334
17
  CurFn->getBasicBlockList().push_back(CaseRangeBlock);
1335
17
  Builder.SetInsertPoint(CaseRangeBlock);
1336
1337
  // Emit range check.
1338
17
  llvm::Value *Diff =
1339
17
    Builder.CreateSub(SwitchInsn->getCondition(), Builder.getInt(LHS));
1340
17
  llvm::Value *Cond =
1341
17
    Builder.CreateICmpULE(Diff, Builder.getInt(Range), "inbounds");
1342
1343
17
  llvm::MDNode *Weights = nullptr;
1344
17
  if (SwitchWeights) {
1345
8
    uint64_t ThisCount = getProfileCount(&S);
1346
8
    uint64_t DefaultCount = (*SwitchWeights)[0];
1347
8
    Weights = createProfileWeights(ThisCount, DefaultCount);
1348
1349
    // Since we're chaining the switch default through each large case range, we
1350
    // need to update the weight for the default, ie, the first case, to include
1351
    // this case.
1352
8
    (*SwitchWeights)[0] += ThisCount;
1353
9
  } else if (SwitchLikelihood)
1354
7
    Weights = createBranchWeights(LH);
1355
1356
17
  Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights);
1357
1358
  // Restore the appropriate insertion point.
1359
17
  if (RestoreBB)
1360
13
    Builder.SetInsertPoint(RestoreBB);
1361
4
  else
1362
4
    Builder.ClearInsertionPoint();
1363
17
}
1364
1365
void CodeGenFunction::EmitCaseStmt(const CaseStmt &S,
1366
972
                                   ArrayRef<const Attr *> Attrs) {
1367
  // If there is no enclosing switch instance that we're aware of, then this
1368
  // case statement and its block can be elided.  This situation only happens
1369
  // when we've constant-folded the switch, are emitting the constant case,
1370
  // and part of the constant case includes another case statement.  For
1371
  // instance: switch (4) { case 4: do { case 5: } while (1); }
1372
972
  if (!SwitchInsn) {
1373
2
    EmitStmt(S.getSubStmt());
1374
2
    return;
1375
2
  }
1376
1377
  // Handle case ranges.
1378
970
  if (S.getRHS()) {
1379
43
    EmitCaseStmtRange(S, Attrs);
1380
43
    return;
1381
43
  }
1382
1383
927
  llvm::ConstantInt *CaseVal =
1384
927
    Builder.getInt(S.getLHS()->EvaluateKnownConstInt(getContext()));
1385
927
  if (SwitchLikelihood)
1386
114
    SwitchLikelihood->push_back(Stmt::getLikelihood(Attrs));
1387
1388
  // If the body of the case is just a 'break', try to not emit an empty block.
1389
  // If we're profiling or we're not optimizing, leave the block in for better
1390
  // debug and coverage analysis.
1391
927
  if (!CGM.getCodeGenOpts().hasProfileClangInstr() &&
1392
897
      CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1393
114
      isa<BreakStmt>(S.getSubStmt())) {
1394
15
    JumpDest Block = BreakContinueStack.back().BreakBlock;
1395
1396
    // Only do this optimization if there are no cleanups that need emitting.
1397
15
    if (isObviouslyBranchWithoutCleanups(Block)) {
1398
15
      if (SwitchWeights)
1399
0
        SwitchWeights->push_back(getProfileCount(&S));
1400
15
      SwitchInsn->addCase(CaseVal, Block.getBlock());
1401
1402
      // If there was a fallthrough into this case, make sure to redirect it to
1403
      // the end of the switch as well.
1404
15
      if (Builder.GetInsertBlock()) {
1405
0
        Builder.CreateBr(Block.getBlock());
1406
0
        Builder.ClearInsertionPoint();
1407
0
      }
1408
15
      return;
1409
15
    }
1410
912
  }
1411
1412
912
  llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
1413
912
  EmitBlockWithFallThrough(CaseDest, &S);
1414
912
  if (SwitchWeights)
1415
47
    SwitchWeights->push_back(getProfileCount(&S));
1416
912
  SwitchInsn->addCase(CaseVal, CaseDest);
1417
1418
  // Recursively emitting the statement is acceptable, but is not wonderful for
1419
  // code where we have many case statements nested together, i.e.:
1420
  //  case 1:
1421
  //    case 2:
1422
  //      case 3: etc.
1423
  // Handling this recursively will create a new block for each case statement
1424
  // that falls through to the next case which is IR intensive.  It also causes
1425
  // deep recursion which can run into stack depth limitations.  Handle
1426
  // sequential non-range case statements specially.
1427
  //
1428
  // TODO When the next case has a likelihood attribute the code returns to the
1429
  // recursive algorithm. Maybe improve this case if it becomes common practice
1430
  // to use a lot of attributes.
1431
912
  const CaseStmt *CurCase = &S;
1432
912
  const CaseStmt *NextCase = dyn_cast<CaseStmt>(S.getSubStmt());
1433
1434
  // Otherwise, iteratively add consecutive cases to this switch stmt.
1435
1.02k
  while (NextCase && 
NextCase->getRHS() == nullptr116
) {
1436
110
    CurCase = NextCase;
1437
110
    llvm::ConstantInt *CaseVal =
1438
110
      Builder.getInt(CurCase->getLHS()->EvaluateKnownConstInt(getContext()));
1439
1440
110
    if (SwitchWeights)
1441
0
      SwitchWeights->push_back(getProfileCount(NextCase));
1442
110
    if (CGM.getCodeGenOpts().hasProfileClangInstr()) {
1443
3
      CaseDest = createBasicBlock("sw.bb");
1444
3
      EmitBlockWithFallThrough(CaseDest, CurCase);
1445
3
    }
1446
    // Since this loop is only executed when the CaseStmt has no attributes
1447
    // use a hard-coded value.
1448
110
    if (SwitchLikelihood)
1449
14
      SwitchLikelihood->push_back(Stmt::LH_None);
1450
1451
110
    SwitchInsn->addCase(CaseVal, CaseDest);
1452
110
    NextCase = dyn_cast<CaseStmt>(CurCase->getSubStmt());
1453
110
  }
1454
1455
  // Normal default recursion for non-cases.
1456
912
  EmitStmt(CurCase->getSubStmt());
1457
912
}
1458
1459
void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S,
1460
182
                                      ArrayRef<const Attr *> Attrs) {
1461
  // If there is no enclosing switch instance that we're aware of, then this
1462
  // default statement can be elided. This situation only happens when we've
1463
  // constant-folded the switch.
1464
182
  if (!SwitchInsn) {
1465
1
    EmitStmt(S.getSubStmt());
1466
1
    return;
1467
1
  }
1468
1469
181
  llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();
1470
181
  assert(DefaultBlock->empty() &&
1471
181
         "EmitDefaultStmt: Default block already defined?");
1472
1473
181
  if (SwitchLikelihood)
1474
33
    SwitchLikelihood->front() = Stmt::getLikelihood(Attrs);
1475
1476
181
  EmitBlockWithFallThrough(DefaultBlock, &S);
1477
1478
181
  EmitStmt(S.getSubStmt());
1479
181
}
1480
1481
/// CollectStatementsForCase - Given the body of a 'switch' statement and a
1482
/// constant value that is being switched on, see if we can dead code eliminate
1483
/// the body of the switch to a simple series of statements to emit.  Basically,
1484
/// on a switch (5) we want to find these statements:
1485
///    case 5:
1486
///      printf(...);    <--
1487
///      ++i;            <--
1488
///      break;
1489
///
1490
/// and add them to the ResultStmts vector.  If it is unsafe to do this
1491
/// transformation (for example, one of the elided statements contains a label
1492
/// that might be jumped to), return CSFC_Failure.  If we handled it and 'S'
1493
/// should include statements after it (e.g. the printf() line is a substmt of
1494
/// the case) then return CSFC_FallThrough.  If we handled it and found a break
1495
/// statement, then return CSFC_Success.
1496
///
1497
/// If Case is non-null, then we are looking for the specified case, checking
1498
/// that nothing we jump over contains labels.  If Case is null, then we found
1499
/// the case and are looking for the break.
1500
///
1501
/// If the recursive walk actually finds our Case, then we set FoundCase to
1502
/// true.
1503
///
1504
enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };
1505
static CSFC_Result CollectStatementsForCase(const Stmt *S,
1506
                                            const SwitchCase *Case,
1507
                                            bool &FoundCase,
1508
497
                              SmallVectorImpl<const Stmt*> &ResultStmts) {
1509
  // If this is a null statement, just succeed.
1510
497
  if (!S)
1511
0
    return Case ? CSFC_Success : CSFC_FallThrough;
1512
1513
  // If this is the switchcase (case 4: or default) that we're looking for, then
1514
  // we're in business.  Just add the substatement.
1515
497
  if (const SwitchCase *SC = dyn_cast<SwitchCase>(S)) {
1516
146
    if (S == Case) {
1517
78
      FoundCase = true;
1518
78
      return CollectStatementsForCase(SC->getSubStmt(), nullptr, FoundCase,
1519
78
                                      ResultStmts);
1520
78
    }
1521
1522
    // Otherwise, this is some other case or default statement, just ignore it.
1523
68
    return CollectStatementsForCase(SC->getSubStmt(), Case, FoundCase,
1524
68
                                    ResultStmts);
1525
68
  }
1526
1527
  // If we are in the live part of the code and we found our break statement,
1528
  // return a success!
1529
351
  if (!Case && 
isa<BreakStmt>(S)146
)
1530
60
    return CSFC_Success;
1531
1532
  // If this is a switch statement, then it might contain the SwitchCase, the
1533
  // break, or neither.
1534
291
  if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(S)) {
1535
    // Handle this as two cases: we might be looking for the SwitchCase (if so
1536
    // the skipped statements must be skippable) or we might already have it.
1537
83
    CompoundStmt::const_body_iterator I = CS->body_begin(), E = CS->body_end();
1538
83
    bool StartedInLiveCode = FoundCase;
1539
83
    unsigned StartSize = ResultStmts.size();
1540
1541
    // If we've not found the case yet, scan through looking for it.
1542
83
    if (Case) {
1543
      // Keep track of whether we see a skipped declaration.  The code could be
1544
      // using the declaration even if it is skipped, so we can't optimize out
1545
      // the decl if the kept statements might refer to it.
1546
76
      bool HadSkippedDecl = false;
1547
1548
      // If we're looking for the case, just see if we can skip each of the
1549
      // substatements.
1550
268
      for (; Case && 
I != E206
;
++I192
) {
1551
205
        HadSkippedDecl |= CodeGenFunction::mightAddDeclToScope(*I);
1552
1553
205
        switch (CollectStatementsForCase(*I, Case, FoundCase, ResultStmts)) {
1554
2
        case CSFC_Failure: return CSFC_Failure;
1555
138
        case CSFC_Success:
1556
          // A successful result means that either 1) that the statement doesn't
1557
          // have the case and is skippable, or 2) does contain the case value
1558
          // and also contains the break to exit the switch.  In the later case,
1559
          // we just verify the rest of the statements are elidable.
1560
138
          if (FoundCase) {
1561
            // If we found the case and skipped declarations, we can't do the
1562
            // optimization.
1563
8
            if (HadSkippedDecl)
1564
0
              return CSFC_Failure;
1565
1566
18
            
for (++I; 8
I != E;
++I10
)
1567
10
              if (CodeGenFunction::ContainsLabel(*I, true))
1568
0
                return CSFC_Failure;
1569
8
            return CSFC_Success;
1570
130
          }
1571
130
          break;
1572
65
        case CSFC_FallThrough:
1573
          // If we have a fallthrough condition, then we must have found the
1574
          // case started to include statements.  Consider the rest of the
1575
          // statements in the compound statement as candidates for inclusion.
1576
65
          assert(FoundCase && "Didn't find case but returned fallthrough?");
1577
          // We recursively found Case, so we're not looking for it anymore.
1578
65
          Case = nullptr;
1579
1580
          // If we found the case and skipped declarations, we can't do the
1581
          // optimization.
1582
65
          if (HadSkippedDecl)
1583
3
            return CSFC_Failure;
1584
62
          break;
1585
205
        }
1586
205
      }
1587
1588
63
      if (!FoundCase)
1589
1
        return CSFC_Success;
1590
1591
62
      assert(!HadSkippedDecl && "fallthrough after skipping decl");
1592
62
    }
1593
1594
    // If we have statements in our range, then we know that the statements are
1595
    // live and need to be added to the set of statements we're tracking.
1596
69
    bool AnyDecls = false;
1597
84
    for (; I != E; 
++I15
) {
1598
68
      AnyDecls |= CodeGenFunction::mightAddDeclToScope(*I);
1599
1600
68
      switch (CollectStatementsForCase(*I, nullptr, FoundCase, ResultStmts)) {
1601
0
      case CSFC_Failure: return CSFC_Failure;
1602
15
      case CSFC_FallThrough:
1603
        // A fallthrough result means that the statement was simple and just
1604
        // included in ResultStmt, keep adding them afterwards.
1605
15
        break;
1606
53
      case CSFC_Success:
1607
        // A successful result means that we found the break statement and
1608
        // stopped statement inclusion.  We just ensure that any leftover stmts
1609
        // are skippable and return success ourselves.
1610
165
        for (++I; I != E; 
++I112
)
1611
112
          if (CodeGenFunction::ContainsLabel(*I, true))
1612
0
            return CSFC_Failure;
1613
53
        return CSFC_Success;
1614
68
      }
1615
68
    }
1616
1617
    // If we're about to fall out of a scope without hitting a 'break;', we
1618
    // can't perform the optimization if there were any decls in that scope
1619
    // (we'd lose their end-of-lifetime).
1620
16
    if (AnyDecls) {
1621
      // If the entire compound statement was live, there's one more thing we
1622
      // can try before giving up: emit the whole thing as a single statement.
1623
      // We can do that unless the statement contains a 'break;'.
1624
      // FIXME: Such a break must be at the end of a construct within this one.
1625
      // We could emit this by just ignoring the BreakStmts entirely.
1626
3
      if (StartedInLiveCode && !CodeGenFunction::containsBreak(S)) {
1627
3
        ResultStmts.resize(StartSize);
1628
3
        ResultStmts.push_back(S);
1629
0
      } else {
1630
0
        return CSFC_Failure;
1631
0
      }
1632
16
    }
1633
1634
16
    return CSFC_FallThrough;
1635
16
  }
1636
1637
  // Okay, this is some other statement that we don't handle explicitly, like a
1638
  // for statement or increment etc.  If we are skipping over this statement,
1639
  // just verify it doesn't have labels, which would make it invalid to elide.
1640
208
  if (Case) {
1641
129
    if (CodeGenFunction::ContainsLabel(S, true))
1642
0
      return CSFC_Failure;
1643
129
    return CSFC_Success;
1644
129
  }
1645
1646
  // Otherwise, we want to include this statement.  Everything is cool with that
1647
  // so long as it doesn't contain a break out of the switch we're in.
1648
79
  if (CodeGenFunction::containsBreak(S)) 
return CSFC_Failure1
;
1649
1650
  // Otherwise, everything is great.  Include the statement and tell the caller
1651
  // that we fall through and include the next statement as well.
1652
78
  ResultStmts.push_back(S);
1653
78
  return CSFC_FallThrough;
1654
78
}
1655
1656
/// FindCaseStatementsForValue - Find the case statement being jumped to and
1657
/// then invoke CollectStatementsForCase to find the list of statements to emit
1658
/// for a switch on constant.  See the comment above CollectStatementsForCase
1659
/// for more details.
1660
static bool FindCaseStatementsForValue(const SwitchStmt &S,
1661
                                       const llvm::APSInt &ConstantCondValue,
1662
                                SmallVectorImpl<const Stmt*> &ResultStmts,
1663
                                       ASTContext &C,
1664
94
                                       const SwitchCase *&ResultCase) {
1665
  // First step, find the switch case that is being branched to.  We can do this
1666
  // efficiently by scanning the SwitchCase list.
1667
94
  const SwitchCase *Case = S.getSwitchCaseList();
1668
94
  const DefaultStmt *DefaultCase = nullptr;
1669
1670
234
  for (; Case; 
Case = Case->getNextSwitchCase()140
) {
1671
    // It's either a default or case.  Just remember the default statement in
1672
    // case we're not jumping to any numbered cases.
1673
198
    if (const DefaultStmt *DS = dyn_cast<DefaultStmt>(Case)) {
1674
66
      DefaultCase = DS;
1675
66
      continue;
1676
66
    }
1677
1678
    // Check to see if this case is the one we're looking for.
1679
132
    const CaseStmt *CS = cast<CaseStmt>(Case);
1680
    // Don't handle case ranges yet.
1681
132
    if (CS->getRHS()) 
return false7
;
1682
1683
    // If we found our case, remember it as 'case'.
1684
125
    if (CS->getLHS()->EvaluateKnownConstInt(C) == ConstantCondValue)
1685
51
      break;
1686
125
  }
1687
1688
  // If we didn't find a matching case, we use a default if it exists, or we
1689
  // elide the whole switch body!
1690
87
  if (!Case) {
1691
    // It is safe to elide the body of the switch if it doesn't contain labels
1692
    // etc.  If it is safe, return successfully with an empty ResultStmts list.
1693
36
    if (!DefaultCase)
1694
9
      return !CodeGenFunction::ContainsLabel(&S);
1695
27
    Case = DefaultCase;
1696
27
  }
1697
1698
  // Ok, we know which case is being jumped to, try to collect all the
1699
  // statements that follow it.  This can fail for a variety of reasons.  Also,
1700
  // check to see that the recursive walk actually found our case statement.
1701
  // Insane cases like this can fail to find it in the recursive walk since we
1702
  // don't handle every stmt kind:
1703
  // switch (4) {
1704
  //   while (1) {
1705
  //     case 4: ...
1706
78
  bool FoundCase = false;
1707
78
  ResultCase = Case;
1708
78
  return CollectStatementsForCase(S.getBody(), Case, FoundCase,
1709
78
                                  ResultStmts) != CSFC_Failure &&
1710
74
         FoundCase;
1711
87
}
1712
1713
static Optional<SmallVector<uint64_t, 16>>
1714
61
getLikelihoodWeights(ArrayRef<Stmt::Likelihood> Likelihoods) {
1715
  // Are there enough branches to weight them?
1716
61
  if (Likelihoods.size() <= 1)
1717
9
    return None;
1718
1719
52
  uint64_t NumUnlikely = 0;
1720
52
  uint64_t NumNone = 0;
1721
52
  uint64_t NumLikely = 0;
1722
215
  for (const auto LH : Likelihoods) {
1723
215
    switch (LH) {
1724
8
    case Stmt::LH_Unlikely:
1725
8
      ++NumUnlikely;
1726
8
      break;
1727
196
    case Stmt::LH_None:
1728
196
      ++NumNone;
1729
196
      break;
1730
11
    case Stmt::LH_Likely:
1731
11
      ++NumLikely;
1732
11
      break;
1733
215
    }
1734
215
  }
1735
1736
  // Is there a likelihood attribute used?
1737
52
  if (NumUnlikely == 0 && 
NumLikely == 044
)
1738
36
    return None;
1739
1740
  // When multiple cases share the same code they can be combined during
1741
  // optimization. In that case the weights of the branch will be the sum of
1742
  // the individual weights. Make sure the combined sum of all neutral cases
1743
  // doesn't exceed the value of a single likely attribute.
1744
  // The additions both avoid divisions by 0 and make sure the weights of None
1745
  // don't exceed the weight of Likely.
1746
16
  const uint64_t Likely = INT32_MAX / (NumLikely + 2);
1747
16
  const uint64_t None = Likely / (NumNone + 1);
1748
16
  const uint64_t Unlikely = 0;
1749
1750
16
  SmallVector<uint64_t, 16> Result;
1751
16
  Result.reserve(Likelihoods.size());
1752
60
  for (const auto LH : Likelihoods) {
1753
60
    switch (LH) {
1754
8
    case Stmt::LH_Unlikely:
1755
8
      Result.push_back(Unlikely);
1756
8
      break;
1757
41
    case Stmt::LH_None:
1758
41
      Result.push_back(None);
1759
41
      break;
1760
11
    case Stmt::LH_Likely:
1761
11
      Result.push_back(Likely);
1762
11
      break;
1763
60
    }
1764
60
  }
1765
1766
16
  return Result;
1767
16
}
1768
1769
486
void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
1770
  // Handle nested switch statements.
1771
486
  llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;
1772
486
  SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;
1773
486
  SmallVector<Stmt::Likelihood, 16> *SavedSwitchLikelihood = SwitchLikelihood;
1774
486
  llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;
1775
1776
  // See if we can constant fold the condition of the switch and therefore only
1777
  // emit the live case statement (if any) of the switch.
1778
486
  llvm::APSInt ConstantCondValue;
1779
486
  if (ConstantFoldsToSimpleInteger(S.getCond(), ConstantCondValue)) {
1780
94
    SmallVector<const Stmt*, 4> CaseStmts;
1781
94
    const SwitchCase *Case = nullptr;
1782
94
    if (FindCaseStatementsForValue(S, ConstantCondValue, CaseStmts,
1783
83
                                   getContext(), Case)) {
1784
83
      if (Case)
1785
74
        incrementProfileCounter(Case);
1786
83
      RunCleanupsScope ExecutedScope(*this);
1787
1788
83
      if (S.getInit())
1789
0
        EmitStmt(S.getInit());
1790
1791
      // Emit the condition variable if needed inside the entire cleanup scope
1792
      // used by this special case for constant folded switches.
1793
83
      if (S.getConditionVariable())
1794
0
        EmitDecl(*S.getConditionVariable());
1795
1796
      // At this point, we are no longer "within" a switch instance, so
1797
      // we can temporarily enforce this to ensure that any embedded case
1798
      // statements are not emitted.
1799
83
      SwitchInsn = nullptr;
1800
1801
      // Okay, we can dead code eliminate everything except this case.  Emit the
1802
      // specified series of statements and we're good.
1803
155
      for (unsigned i = 0, e = CaseStmts.size(); i != e; 
++i72
)
1804
72
        EmitStmt(CaseStmts[i]);
1805
83
      incrementProfileCounter(&S);
1806
1807
      // Now we want to restore the saved switch instance so that nested
1808
      // switches continue to function properly
1809
83
      SwitchInsn = SavedSwitchInsn;
1810
1811
83
      return;
1812
83
    }
1813
403
  }
1814
1815
403
  JumpDest SwitchExit = getJumpDestInCurrentScope("sw.epilog");
1816
1817
403
  RunCleanupsScope ConditionScope(*this);
1818
1819
403
  if (S.getInit())
1820
7
    EmitStmt(S.getInit());
1821
1822
403
  if (S.getConditionVariable())
1823
4
    EmitDecl(*S.getConditionVariable());
1824
403
  llvm::Value *CondV = EmitScalarExpr(S.getCond());
1825
1826
  // Create basic block to hold stuff that comes after switch
1827
  // statement. We also need to create a default block now so that
1828
  // explicit case ranges tests can have a place to jump to on
1829
  // failure.
1830
403
  llvm::BasicBlock *DefaultBlock = createBasicBlock("sw.default");
1831
403
  SwitchInsn = Builder.CreateSwitch(CondV, DefaultBlock);
1832
403
  if (PGO.haveRegionCounts()) {
1833
    // Walk the SwitchCase list to find how many there are.
1834
23
    uint64_t DefaultCount = 0;
1835
23
    unsigned NumCases = 0;
1836
23
    for (const SwitchCase *Case = S.getSwitchCaseList();
1837
101
         Case;
1838
78
         Case = Case->getNextSwitchCase()) {
1839
78
      if (isa<DefaultStmt>(Case))
1840
15
        DefaultCount = getProfileCount(Case);
1841
78
      NumCases += 1;
1842
78
    }
1843
23
    SwitchWeights = new SmallVector<uint64_t, 16>();
1844
23
    SwitchWeights->reserve(NumCases);
1845
    // The default needs to be first. We store the edge count, so we already
1846
    // know the right weight.
1847
23
    SwitchWeights->push_back(DefaultCount);
1848
380
  } else if (CGM.getCodeGenOpts().OptimizationLevel) {
1849
61
    SwitchLikelihood = new SmallVector<Stmt::Likelihood, 16>();
1850
    // Initialize the default case.
1851
61
    SwitchLikelihood->push_back(Stmt::LH_None);
1852
61
  }
1853
1854
403
  CaseRangeBlock = DefaultBlock;
1855
1856
  // Clear the insertion point to indicate we are in unreachable code.
1857
403
  Builder.ClearInsertionPoint();
1858
1859
  // All break statements jump to NextBlock. If BreakContinueStack is non-empty
1860
  // then reuse last ContinueBlock.
1861
403
  JumpDest OuterContinue;
1862
403
  if (!BreakContinueStack.empty())
1863
44
    OuterContinue = BreakContinueStack.back().ContinueBlock;
1864
1865
403
  BreakContinueStack.push_back(BreakContinue(SwitchExit, OuterContinue));
1866
1867
  // Emit switch body.
1868
403
  EmitStmt(S.getBody());
1869
1870
403
  BreakContinueStack.pop_back();
1871
1872
  // Update the default block in case explicit case range tests have
1873
  // been chained on top.
1874
403
  SwitchInsn->setDefaultDest(CaseRangeBlock);
1875
1876
  // If a default was never emitted:
1877
403
  if (!DefaultBlock->getParent()) {
1878
    // If we have cleanups, emit the default block so that there's a
1879
    // place to jump through the cleanups from.
1880
222
    if (ConditionScope.requiresCleanups()) {
1881
0
      EmitBlock(DefaultBlock);
1882
1883
    // Otherwise, just forward the default block to the switch end.
1884
222
    } else {
1885
222
      DefaultBlock->replaceAllUsesWith(SwitchExit.getBlock());
1886
222
      delete DefaultBlock;
1887
222
    }
1888
222
  }
1889
1890
403
  ConditionScope.ForceCleanup();
1891
1892
  // Emit continuation.
1893
403
  EmitBlock(SwitchExit.getBlock(), true);
1894
403
  incrementProfileCounter(&S);
1895
1896
  // If the switch has a condition wrapped by __builtin_unpredictable,
1897
  // create metadata that specifies that the switch is unpredictable.
1898
  // Don't bother if not optimizing because that metadata would not be used.
1899
403
  auto *Call = dyn_cast<CallExpr>(S.getCond());
1900
403
  if (Call && 
CGM.getCodeGenOpts().OptimizationLevel != 041
) {
1901
8
    auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
1902
8
    if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
1903
2
      llvm::MDBuilder MDHelper(getLLVMContext());
1904
2
      SwitchInsn->setMetadata(llvm::LLVMContext::MD_unpredictable,
1905
2
                              MDHelper.createUnpredictable());
1906
2
    }
1907
8
  }
1908
1909
403
  if (SwitchWeights) {
1910
23
    assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
1911
23
           "switch weights do not match switch cases");
1912
    // If there's only one jump destination there's no sense weighting it.
1913
23
    if (SwitchWeights->size() > 1)
1914
19
      SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof,
1915
19
                              createProfileWeights(*SwitchWeights));
1916
23
    delete SwitchWeights;
1917
380
  } else if (SwitchLikelihood) {
1918
61
    assert(SwitchLikelihood->size() == 1 + SwitchInsn->getNumCases() &&
1919
61
           "switch likelihoods do not match switch cases");
1920
61
    Optional<SmallVector<uint64_t, 16>> LHW =
1921
61
        getLikelihoodWeights(*SwitchLikelihood);
1922
61
    if (LHW) {
1923
16
      llvm::MDBuilder MDHelper(CGM.getLLVMContext());
1924
16
      SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof,
1925
16
                              createProfileWeights(*LHW));
1926
16
    }
1927
61
    delete SwitchLikelihood;
1928
61
  }
1929
403
  SwitchInsn = SavedSwitchInsn;
1930
403
  SwitchWeights = SavedSwitchWeights;
1931
403
  SwitchLikelihood = SavedSwitchLikelihood;
1932
403
  CaseRangeBlock = SavedCRBlock;
1933
403
}
1934
1935
static std::string
1936
SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
1937
2.83k
                 SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=nullptr) {
1938
2.83k
  std::string Result;
1939
1940
7.04k
  while (*Constraint) {
1941
4.21k
    switch (*Constraint) {
1942
3.42k
    default:
1943
3.42k
      Result += Target.convertConstraint(Constraint);
1944
3.42k
      break;
1945
    // Ignore these
1946
1
    case '*':
1947
1
    case '?':
1948
1
    case '!':
1949
3
    case '=': // Will see this and the following in mult-alt constraints.
1950
3
    case '+':
1951
3
      break;
1952
1
    case '#': // Ignore the rest of the constraint alternative.
1953
3
      while (Constraint[1] && Constraint[1] != ',')
1954
2
        Constraint++;
1955
1
      break;
1956
15
    case '&':
1957
17
    case '%':
1958
17
      Result += *Constraint;
1959
19
      while (Constraint[1] && Constraint[1] == *Constraint)
1960
2
        Constraint++;
1961
17
      break;
1962
622
    case ',':
1963
622
      Result += "|";
1964
622
      break;
1965
143
    case 'g':
1966
143
      Result += "imr";
1967
143
      break;
1968
6
    case '[': {
1969
6
      assert(OutCons &&
1970
6
             "Must pass output names to constraints with a symbolic name");
1971
6
      unsigned Index;
1972
6
      bool result = Target.resolveSymbolicName(Constraint, *OutCons, Index);
1973
6
      assert(result && "Could not resolve symbolic name"); (void)result;
1974
6
      Result += llvm::utostr(Index);
1975
6
      break;
1976
4.21k
    }
1977
4.21k
    }
1978
1979
4.21k
    Constraint++;
1980
4.21k
  }
1981
1982
2.83k
  return Result;
1983
2.83k
}
1984
1985
/// AddVariableConstraints - Look at AsmExpr and if it is a variable declared
1986
/// as using a particular register add that as a constraint that will be used
1987
/// in this asm stmt.
1988
static std::string
1989
AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr,
1990
                       const TargetInfo &Target, CodeGenModule &CGM,
1991
                       const AsmStmt &Stmt, const bool EarlyClobber,
1992
2.83k
                       std::string *GCCReg = nullptr) {
1993
2.83k
  const DeclRefExpr *AsmDeclRef = dyn_cast<DeclRefExpr>(&AsmExpr);
1994
2.83k
  if (!AsmDeclRef)
1995
811
    return Constraint;
1996
2.02k
  const ValueDecl &Value = *AsmDeclRef->getDecl();
1997
2.02k
  const VarDecl *Variable = dyn_cast<VarDecl>(&Value);
1998
2.02k
  if (!Variable)
1999
11
    return Constraint;
2000
2.01k
  if (Variable->getStorageClass() != SC_Register)
2001
1.37k
    return Constraint;
2002
643
  AsmLabelAttr *Attr = Variable->getAttr<AsmLabelAttr>();
2003
643
  if (!Attr)
2004
576
    return Constraint;
2005
67
  StringRef Register = Attr->getLabel();
2006
67
  assert(Target.isValidGCCRegisterName(Register));
2007
  // We're using validateOutputConstraint here because we only care if
2008
  // this is a register constraint.
2009
67
  TargetInfo::ConstraintInfo Info(Constraint, "");
2010
67
  if (Target.validateOutputConstraint(Info) &&
2011
0
      !Info.allowsRegister()) {
2012
0
    CGM.ErrorUnsupported(&Stmt, "__asm__");
2013
0
    return Constraint;
2014
0
  }
2015
  // Canonicalize the register here before returning it.
2016
67
  Register = Target.getNormalizedGCCRegisterName(Register);
2017
67
  if (GCCReg != nullptr)
2018
14
    *GCCReg = Register.str();
2019
62
  return (EarlyClobber ? 
"&{"5
: "{") + Register.str() + "}";
2020
67
}
2021
2022
llvm::Value*
2023
CodeGenFunction::EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info,
2024
                                    LValue InputValue, QualType InputType,
2025
                                    std::string &ConstraintStr,
2026
295
                                    SourceLocation Loc) {
2027
295
  llvm::Value *Arg;
2028
295
  if (Info.allowsRegister() || 
!Info.allowsMemory()168
) {
2029
127
    if (CodeGenFunction::hasScalarEvaluationKind(InputType)) {
2030
124
      Arg = EmitLoadOfLValue(InputValue, Loc).getScalarVal();
2031
3
    } else {
2032
3
      llvm::Type *Ty = ConvertType(InputType);
2033
3
      uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);
2034
3
      if (Size <= 64 && llvm::isPowerOf2_64(Size)) {
2035
3
        Ty = llvm::IntegerType::get(getLLVMContext(), Size);
2036
3
        Ty = llvm::PointerType::getUnqual(Ty);
2037
2038
3
        Arg = Builder.CreateLoad(
2039
3
            Builder.CreateBitCast(InputValue.getAddress(*this), Ty));
2040
0
      } else {
2041
0
        Arg = InputValue.getPointer(*this);
2042
0
        ConstraintStr += '*';
2043
0
      }
2044
3
    }
2045
168
  } else {
2046
168
    Arg = InputValue.getPointer(*this);
2047
168
    ConstraintStr += '*';
2048
168
  }
2049
2050
295
  return Arg;
2051
295
}
2052
2053
llvm::Value* CodeGenFunction::EmitAsmInput(
2054
                                         const TargetInfo::ConstraintInfo &Info,
2055
                                           const Expr *InputExpr,
2056
1.67k
                                           std::string &ConstraintStr) {
2057
  // If this can't be a register or memory, i.e., has to be a constant
2058
  // (immediate or symbolic), try to emit it as such.
2059
1.67k
  if (!Info.allowsRegister() && 
!Info.allowsMemory()434
) {
2060
278
    if (Info.requiresImmediateConstant()) {
2061
68
      Expr::EvalResult EVResult;
2062
68
      InputExpr->EvaluateAsRValue(EVResult, getContext(), true);
2063
2064
68
      llvm::APSInt IntResult;
2065
68
      if (EVResult.Val.toIntegralConstant(IntResult, InputExpr->getType(),
2066
68
                                          getContext()))
2067
67
        return llvm::ConstantInt::get(getLLVMContext(), IntResult);
2068
211
    }
2069
2070
211
    Expr::EvalResult Result;
2071
211
    if (InputExpr->EvaluateAsInt(Result, getContext()))
2072
162
      return llvm::ConstantInt::get(getLLVMContext(), Result.Val.getInt());
2073
1.45k
  }
2074
2075
1.45k
  if (Info.allowsRegister() || 
!Info.allowsMemory()205
)
2076
1.29k
    if (CodeGenFunction::hasScalarEvaluationKind(InputExpr->getType()))
2077
1.29k
      return EmitScalarExpr(InputExpr);
2078
159
  if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)
2079
3
    return EmitScalarExpr(InputExpr);
2080
156
  InputExpr = InputExpr->IgnoreParenNoopCasts(getContext());
2081
156
  LValue Dest = EmitLValue(InputExpr);
2082
156
  return EmitAsmInputLValue(Info, Dest, InputExpr->getType(), ConstraintStr,
2083
156
                            InputExpr->getExprLoc());
2084
156
}
2085
2086
/// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
2087
/// asm call instruction.  The !srcloc MDNode contains a list of constant
2088
/// integers which are the source locations of the start of each line in the
2089
/// asm.
2090
static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
2091
1.45k
                                      CodeGenFunction &CGF) {
2092
1.45k
  SmallVector<llvm::Metadata *, 8> Locs;
2093
  // Add the location of the first line to the MDNode.
2094
1.45k
  Locs.push_back(llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
2095
1.45k
      CGF.Int32Ty, Str->getBeginLoc().getRawEncoding())));
2096
1.45k
  StringRef StrVal = Str->getString();
2097
1.45k
  if (!StrVal.empty()) {
2098
1.38k
    const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
2099
1.38k
    const LangOptions &LangOpts = CGF.CGM.getLangOpts();
2100
1.38k
    unsigned StartToken = 0;
2101
1.38k
    unsigned ByteOffset = 0;
2102
2103
    // Add the location of the start of each subsequent line of the asm to the
2104
    // MDNode.
2105
27.0k
    for (unsigned i = 0, e = StrVal.size() - 1; i != e; 
++i25.6k
) {
2106
25.6k
      if (StrVal[i] != '\n') 
continue25.0k
;
2107
626
      SourceLocation LineLoc = Str->getLocationOfByte(
2108
626
          i + 1, SM, LangOpts, CGF.getTarget(), &StartToken, &ByteOffset);
2109
626
      Locs.push_back(llvm::ConstantAsMetadata::get(
2110
626
          llvm::ConstantInt::get(CGF.Int32Ty, LineLoc.getRawEncoding())));
2111
626
    }
2112
1.38k
  }
2113
2114
1.45k
  return llvm::MDNode::get(CGF.getLLVMContext(), Locs);
2115
1.45k
}
2116
2117
static void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
2118
                              bool ReadOnly, bool ReadNone, bool NoMerge,
2119
                              const AsmStmt &S,
2120
                              const std::vector<llvm::Type *> &ResultRegTypes,
2121
                              CodeGenFunction &CGF,
2122
1.60k
                              std::vector<llvm::Value *> &RegResults) {
2123
1.60k
  Result.addAttribute(llvm::AttributeList::FunctionIndex,
2124
1.60k
                      llvm::Attribute::NoUnwind);
2125
1.60k
  if (NoMerge)
2126
1
    Result.addAttribute(llvm::AttributeList::FunctionIndex,
2127
1
                        llvm::Attribute::NoMerge);
2128
  // Attach readnone and readonly attributes.
2129
1.60k
  if (!HasSideEffect) {
2130
766
    if (ReadNone)
2131
306
      Result.addAttribute(llvm::AttributeList::FunctionIndex,
2132
306
                          llvm::Attribute::ReadNone);
2133
460
    else if (ReadOnly)
2134
319
      Result.addAttribute(llvm::AttributeList::FunctionIndex,
2135
319
                          llvm::Attribute::ReadOnly);
2136
766
  }
2137
2138
  // Slap the source location of the inline asm into a !srcloc metadata on the
2139
  // call.
2140
1.60k
  if (const auto *gccAsmStmt = dyn_cast<GCCAsmStmt>(&S))
2141
1.45k
    Result.setMetadata("srcloc",
2142
1.45k
                       getAsmSrcLocInfo(gccAsmStmt->getAsmString(), CGF));
2143
156
  else {
2144
    // At least put the line number on MS inline asm blobs.
2145
156
    llvm::Constant *Loc = llvm::ConstantInt::get(CGF.Int32Ty,
2146
156
                                        S.getAsmLoc().getRawEncoding());
2147
156
    Result.setMetadata("srcloc",
2148
156
                       llvm::MDNode::get(CGF.getLLVMContext(),
2149
156
                                         llvm::ConstantAsMetadata::get(Loc)));
2150
156
  }
2151
2152
1.60k
  if (CGF.getLangOpts().assumeFunctionsAreConvergent())
2153
    // Conservatively, mark all inline asm blocks in CUDA or OpenCL as
2154
    // convergent (meaning, they may call an intrinsically convergent op, such
2155
    // as bar.sync, and so can't have certain optimizations applied around
2156
    // them).
2157
13
    Result.addAttribute(llvm::AttributeList::FunctionIndex,
2158
13
                        llvm::Attribute::Convergent);
2159
  // Extract all of the register value results from the asm.
2160
1.60k
  if (ResultRegTypes.size() == 1) {
2161
794
    RegResults.push_back(&Result);
2162
812
  } else {
2163
1.01k
    for (unsigned i = 0, e = ResultRegTypes.size(); i != e; 
++i204
) {
2164
204
      llvm::Value *Tmp = CGF.Builder.CreateExtractValue(&Result, i, "asmresult");
2165
204
      RegResults.push_back(Tmp);
2166
204
    }
2167
812
  }
2168
1.60k
}
2169
2170
1.60k
void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
2171
  // Assemble the final asm string.
2172
1.60k
  std::string AsmString = S.generateAsmString(getContext());
2173
2174
  // Get all the output and input constraints together.
2175
1.60k
  SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
2176
1.60k
  SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
2177
2178
2.76k
  for (unsigned i = 0, e = S.getNumOutputs(); i != e; 
i++1.15k
) {
2179
1.15k
    StringRef Name;
2180
1.15k
    if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))
2181
1.13k
      Name = GAS->getOutputName(i);
2182
1.15k
    TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i), Name);
2183
1.15k
    bool IsValid = getTarget().validateOutputConstraint(Info); (void)IsValid;
2184
1.15k
    assert(IsValid && "Failed to parse output constraint");
2185
1.15k
    OutputConstraintInfos.push_back(Info);
2186
1.15k
  }
2187
2188
3.28k
  for (unsigned i = 0, e = S.getNumInputs(); i != e; 
i++1.67k
) {
2189
1.67k
    StringRef Name;
2190
1.67k
    if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))
2191
1.57k
      Name = GAS->getInputName(i);
2192
1.67k
    TargetInfo::ConstraintInfo Info(S.getInputConstraint(i), Name);
2193
1.67k
    bool IsValid =
2194
1.67k
      getTarget().validateInputConstraint(OutputConstraintInfos, Info);
2195
1.67k
    assert(IsValid && "Failed to parse input constraint"); (void)IsValid;
2196
1.67k
    InputConstraintInfos.push_back(Info);
2197
1.67k
  }
2198
2199
1.60k
  std::string Constraints;
2200
2201
1.60k
  std::vector<LValue> ResultRegDests;
2202
1.60k
  std::vector<QualType> ResultRegQualTys;
2203
1.60k
  std::vector<llvm::Type *> ResultRegTypes;
2204
1.60k
  std::vector<llvm::Type *> ResultTruncRegTypes;
2205
1.60k
  std::vector<llvm::Type *> ArgTypes;
2206
1.60k
  std::vector<llvm::Value*> Args;
2207
1.60k
  llvm::BitVector ResultTypeRequiresCast;
2208
2209
  // Keep track of inout constraints.
2210
1.60k
  std::string InOutConstraints;
2211
1.60k
  std::vector<llvm::Value*> InOutArgs;
2212
1.60k
  std::vector<llvm::Type*> InOutArgTypes;
2213
2214
  // Keep track of out constraints for tied input operand.
2215
1.60k
  std::vector<std::string> OutputConstraints;
2216
2217
  // Keep track of defined physregs.
2218
1.60k
  llvm::SmallSet<std::string, 8> PhysRegOutputs;
2219
2220
  // An inline asm can be marked readonly if it meets the following conditions:
2221
  //  - it doesn't have any sideeffects
2222
  //  - it doesn't clobber memory
2223
  //  - it doesn't return a value by-reference
2224
  // It can be marked readnone if it doesn't have any input memory constraints
2225
  // in addition to meeting the conditions listed above.
2226
1.60k
  bool ReadOnly = true, ReadNone = true;
2227
2228
2.76k
  for (unsigned i = 0, e = S.getNumOutputs(); i != e; 
i++1.15k
) {
2229
1.15k
    TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[i];
2230
2231
    // Simplify the output constraint.
2232
1.15k
    std::string OutputConstraint(S.getOutputConstraint(i));
2233
1.15k
    OutputConstraint = SimplifyConstraint(OutputConstraint.c_str() + 1,
2234
1.15k
                                          getTarget(), &OutputConstraintInfos);
2235
2236
1.15k
    const Expr *OutExpr = S.getOutputExpr(i);
2237
1.15k
    OutExpr = OutExpr->IgnoreParenNoopCasts(getContext());
2238
2239
1.15k
    std::string GCCReg;
2240
1.15k
    OutputConstraint = AddVariableConstraints(OutputConstraint, *OutExpr,
2241
1.15k
                                              getTarget(), CGM, S,
2242
1.15k
                                              Info.earlyClobber(),
2243
1.15k
                                              &GCCReg);
2244
    // Give an error on multiple outputs to same physreg.
2245
1.15k
    if (!GCCReg.empty() && 
!PhysRegOutputs.insert(GCCReg).second14
)
2246
1
      CGM.Error(S.getAsmLoc(), "multiple outputs to hard register: " + GCCReg);
2247
2248
1.15k
    OutputConstraints.push_back(OutputConstraint);
2249
1.15k
    LValue Dest = EmitLValue(OutExpr);
2250
1.15k
    if (!Constraints.empty())
2251
184
      Constraints += ',';
2252
2253
    // If this is a register output, then make the inline asm return it
2254
    // by-value.  If this is a memory result, return the value by-reference.
2255
1.15k
    bool isScalarizableAggregate =
2256
1.15k
        hasAggregateEvaluationKind(OutExpr->getType());
2257
1.15k
    if (!Info.allowsMemory() && 
(975
hasScalarEvaluationKind(OutExpr->getType())975
||
2258
975
                                 
isScalarizableAggregate16
)) {
2259
975
      Constraints += "=" + OutputConstraint;
2260
975
      ResultRegQualTys.push_back(OutExpr->getType());
2261
975
      ResultRegDests.push_back(Dest);
2262
975
      ResultTruncRegTypes.push_back(ConvertTypeForMem(OutExpr->getType()));
2263
975
      if (Info.allowsRegister() && isScalarizableAggregate) {
2264
16
        ResultTypeRequiresCast.push_back(true);
2265
16
        unsigned Size = getContext().getTypeSize(OutExpr->getType());
2266
16
        llvm::Type *ConvTy = llvm::IntegerType::get(getLLVMContext(), Size);
2267
16
        ResultRegTypes.push_back(ConvTy);
2268
959
      } else {
2269
959
        ResultTypeRequiresCast.push_back(false);
2270
959
        ResultRegTypes.push_back(ResultTruncRegTypes.back());
2271
959
      }
2272
      // If this output is tied to an input, and if the input is larger, then
2273
      // we need to set the actual result type of the inline asm node to be the
2274
      // same as the input type.
2275
975
      if (Info.hasMatchingInput()) {
2276
41
        unsigned InputNo;
2277
44
        for (InputNo = 0; InputNo != S.getNumInputs(); 
++InputNo3
) {
2278
44
          TargetInfo::ConstraintInfo &Input = InputConstraintInfos[InputNo];
2279
44
          if (Input.hasTiedOperand() && 
Input.getTiedOperand() == i43
)
2280
41
            break;
2281
44
        }
2282
41
        assert(InputNo != S.getNumInputs() && "Didn't find matching input!");
2283
2284
41
        QualType InputTy = S.getInputExpr(InputNo)->getType();
2285
41
        QualType OutputType = OutExpr->getType();
2286
2287
41
        uint64_t InputSize = getContext().getTypeSize(InputTy);
2288
41
        if (getContext().getTypeSize(OutputType) < InputSize) {
2289
          // Form the asm to return the value as a larger integer or fp type.
2290
4
          ResultRegTypes.back() = ConvertType(InputTy);
2291
4
        }
2292
41
      }
2293
975
      if (llvm::Type* AdjTy =
2294
975
            getTargetHooks().adjustInlineAsmType(*this, OutputConstraint,
2295
975
                                                 ResultRegTypes.back()))
2296
975
        ResultRegTypes.back() = AdjTy;
2297
0
      else {
2298
0
        CGM.getDiags().Report(S.getAsmLoc(),
2299
0
                              diag::err_asm_invalid_type_in_input)
2300
0
            << OutExpr->getType() << OutputConstraint;
2301
0
      }
2302
2303
      // Update largest vector width for any vector types.
2304
975
      if (auto *VT = dyn_cast<llvm::VectorType>(ResultRegTypes.back()))
2305
131
        LargestVectorWidth =
2306
131
            std::max((uint64_t)LargestVectorWidth,
2307
131
                     VT->getPrimitiveSizeInBits().getKnownMinSize());
2308
181
    } else {
2309
181
      llvm::Type *DestAddrTy = Dest.getAddress(*this).getType();
2310
181
      llvm::Value *DestPtr = Dest.getPointer(*this);
2311
      // Matrix types in memory are represented by arrays, but accessed through
2312
      // vector pointers, with the alignment specified on the access operation.
2313
      // For inline assembly, update pointer arguments to use vector pointers.
2314
      // Otherwise there will be a mis-match if the matrix is also an
2315
      // input-argument which is represented as vector.
2316
181
      if (isa<MatrixType>(OutExpr->getType().getCanonicalType())) {
2317
1
        DestAddrTy = llvm::PointerType::get(
2318
1
            ConvertType(OutExpr->getType()),
2319
1
            cast<llvm::PointerType>(DestAddrTy)->getAddressSpace());
2320
1
        DestPtr = Builder.CreateBitCast(DestPtr, DestAddrTy);
2321
1
      }
2322
181
      ArgTypes.push_back(DestAddrTy);
2323
181
      Args.push_back(DestPtr);
2324
181
      Constraints += "=*";
2325
181
      Constraints += OutputConstraint;
2326
181
      ReadOnly = ReadNone = false;
2327
181
    }
2328
2329
1.15k
    if (Info.isReadWrite()) {
2330
139
      InOutConstraints += ',';
2331
2332
139
      const Expr *InputExpr = S.getOutputExpr(i);
2333
139
      llvm::Value *Arg = EmitAsmInputLValue(Info, Dest, InputExpr->getType(),
2334
139
                                            InOutConstraints,
2335
139
                                            InputExpr->getExprLoc());
2336
2337
139
      if (llvm::Type* AdjTy =
2338
139
          getTargetHooks().adjustInlineAsmType(*this, OutputConstraint,
2339
139
                                               Arg->getType()))
2340
139
        Arg = Builder.CreateBitCast(Arg, AdjTy);
2341
2342
      // Update largest vector width for any vector types.
2343
139
      if (auto *VT = dyn_cast<llvm::VectorType>(Arg->getType()))
2344
6
        LargestVectorWidth =
2345
6
            std::max((uint64_t)LargestVectorWidth,
2346
6
                     VT->getPrimitiveSizeInBits().getKnownMinSize());
2347
      // Only tie earlyclobber physregs.
2348
139
      if (Info.allowsRegister() && 
(124
GCCReg.empty()124
||
Info.earlyClobber()4
))
2349
121
        InOutConstraints += llvm::utostr(i);
2350
18
      else
2351
18
        InOutConstraints += OutputConstraint;
2352
2353
139
      InOutArgTypes.push_back(Arg->getType());
2354
139
      InOutArgs.push_back(Arg);
2355
139
    }
2356
1.15k
  }
2357
2358
  // If this is a Microsoft-style asm blob, store the return registers (EAX:EDX)
2359
  // to the return value slot. Only do this when returning in registers.
2360
1.60k
  if (isa<MSAsmStmt>(&S)) {
2361
156
    const ABIArgInfo &RetAI = CurFnInfo->getReturnInfo();
2362
156
    if (RetAI.isDirect() || 
RetAI.isExtend()133
) {
2363
      // Make a fake lvalue for the return value slot.
2364
26
      LValue ReturnSlot = MakeAddrLValue(ReturnValue, FnRetTy);
2365
26
      CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(
2366
26
          *this, ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,
2367
26
          ResultRegDests, AsmString, S.getNumOutputs());
2368
26
      SawAsmBlock = true;
2369
26
    }
2370
156
  }
2371
2372
3.28k
  for (unsigned i = 0, e = S.getNumInputs(); i != e; 
i++1.67k
) {
2373
1.67k
    const Expr *InputExpr = S.getInputExpr(i);
2374
2375
1.67k
    TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i];
2376
2377
1.67k
    if (Info.allowsMemory())
2378
591
      ReadNone = false;
2379
2380
1.67k
    if (!Constraints.empty())
2381
1.40k
      Constraints += ',';
2382
2383
    // Simplify the input constraint.
2384
1.67k
    std::string InputConstraint(S.getInputConstraint(i));
2385
1.67k
    InputConstraint = SimplifyConstraint(InputConstraint.c_str(), getTarget(),
2386
1.67k
                                         &OutputConstraintInfos);
2387
2388
1.67k
    InputConstraint = AddVariableConstraints(
2389
1.67k
        InputConstraint, *InputExpr->IgnoreParenNoopCasts(getContext()),
2390
1.67k
        getTarget(), CGM, S, false /* No EarlyClobber */);
2391
2392
1.67k
    std::string ReplaceConstraint (InputConstraint);
2393
1.67k
    llvm::Value *Arg = EmitAsmInput(Info, InputExpr, Constraints);
2394
2395
    // If this input argument is tied to a larger output result, extend the
2396
    // input to be the same size as the output.  The LLVM backend wants to see
2397
    // the input and output of a matching constraint be the same size.  Note
2398
    // that GCC does not define what the top bits are here.  We use zext because
2399
    // that is usually cheaper, but LLVM IR should really get an anyext someday.
2400
1.67k
    if (Info.hasTiedOperand()) {
2401
41
      unsigned Output = Info.getTiedOperand();
2402
41
      QualType OutputType = S.getOutputExpr(Output)->getType();
2403
41
      QualType InputTy = InputExpr->getType();
2404
2405
41
      if (getContext().getTypeSize(OutputType) >
2406
7
          getContext().getTypeSize(InputTy)) {
2407
        // Use ptrtoint as appropriate so that we can do our extension.
2408
7
        if (isa<llvm::PointerType>(Arg->getType()))
2409
0
          Arg = Builder.CreatePtrToInt(Arg, IntPtrTy);
2410
7
        llvm::Type *OutputTy = ConvertType(OutputType);
2411
7
        if (isa<llvm::IntegerType>(OutputTy))
2412
3
          Arg = Builder.CreateZExt(Arg, OutputTy);
2413
4
        else if (isa<llvm::PointerType>(OutputTy))
2414
1
          Arg = Builder.CreateZExt(Arg, IntPtrTy);
2415
3
        else {
2416
3
          assert(OutputTy->isFloatingPointTy() && "Unexpected output type");
2417
3
          Arg = Builder.CreateFPExt(Arg, OutputTy);
2418
3
        }
2419
7
      }
2420
      // Deal with the tied operands' constraint code in adjustInlineAsmType.
2421
41
      ReplaceConstraint = OutputConstraints[Output];
2422
41
    }
2423
1.67k
    if (llvm::Type* AdjTy =
2424
1.67k
          getTargetHooks().adjustInlineAsmType(*this, ReplaceConstraint,
2425
1.67k
                                                   Arg->getType()))
2426
1.67k
      Arg = Builder.CreateBitCast(Arg, AdjTy);
2427
0
    else
2428
0
      CGM.getDiags().Report(S.getAsmLoc(), diag::err_asm_invalid_type_in_input)
2429
0
          << InputExpr->getType() << InputConstraint;
2430
2431
    // Update largest vector width for any vector types.
2432
1.67k
    if (auto *VT = dyn_cast<llvm::VectorType>(Arg->getType()))
2433
139
      LargestVectorWidth =
2434
139
          std::max((uint64_t)LargestVectorWidth,
2435
139
                   VT->getPrimitiveSizeInBits().getKnownMinSize());
2436
2437
1.67k
    ArgTypes.push_back(Arg->getType());
2438
1.67k
    Args.push_back(Arg);
2439
1.67k
    Constraints += InputConstraint;
2440
1.67k
  }
2441
2442
  // Labels
2443
1.60k
  SmallVector<llvm::BasicBlock *, 16> Transfer;
2444
1.60k
  llvm::BasicBlock *Fallthrough = nullptr;
2445
1.60k
  bool IsGCCAsmGoto = false;
2446
1.60k
  if (const auto *GS =  dyn_cast<GCCAsmStmt>(&S)) {
2447
1.45k
    IsGCCAsmGoto = GS->isAsmGoto();
2448
1.45k
    if (IsGCCAsmGoto) {
2449
46
      for (const auto *E : GS->labels()) {
2450
46
        JumpDest Dest = getJumpDestForLabel(E->getLabel());
2451
46
        Transfer.push_back(Dest.getBlock());
2452
46
        llvm::BlockAddress *BA =
2453
46
            llvm::BlockAddress::get(CurFn, Dest.getBlock());
2454
46
        Args.push_back(BA);
2455
46
        ArgTypes.push_back(BA->getType());
2456
46
        if (!Constraints.empty())
2457
41
          Constraints += ',';
2458
46
        Constraints += 'X';
2459
46
      }
2460
27
      Fallthrough = createBasicBlock("asm.fallthrough");
2461
27
    }
2462
1.45k
  }
2463
2464
  // Append the "input" part of inout constraints last.
2465
1.74k
  for (unsigned i = 0, e = InOutArgs.size(); i != e; 
i++139
) {
2466
139
    ArgTypes.push_back(InOutArgTypes[i]);
2467
139
    Args.push_back(InOutArgs[i]);
2468
139
  }
2469
1.60k
  Constraints += InOutConstraints;
2470
2471
  // Clobbers
2472
2.75k
  for (unsigned i = 0, e = S.getNumClobbers(); i != e; 
i++1.14k
) {
2473
1.14k
    StringRef Clobber = S.getClobber(i);
2474
2475
1.14k
    if (Clobber == "memory")
2476
138
      ReadOnly = ReadNone = false;
2477
1.01k
    else if (Clobber != "cc") {
2478
969
      Clobber = getTarget().getNormalizedGCCRegisterName(Clobber);
2479
969
      if (CGM.getCodeGenOpts().StackClashProtector &&
2480
3
          getTarget().isSPRegName(Clobber)) {
2481
3
        CGM.getDiags().Report(S.getAsmLoc(),
2482
3
                              diag::warn_stack_clash_protection_inline_asm);
2483
3
      }
2484
969
    }
2485
2486
1.14k
    if (!Constraints.empty())
2487
939
      Constraints += ',';
2488
2489
1.14k
    Constraints += "~{";
2490
1.14k
    Constraints += Clobber;
2491
1.14k
    Constraints += '}';
2492
1.14k
  }
2493
2494
  // Add machine specific clobbers
2495
1.60k
  std::string MachineClobbers = getTarget().getClobbers();
2496
1.60k
  if (!MachineClobbers.empty()) {
2497
1.01k
    if (!Constraints.empty())
2498
898
      Constraints += ',';
2499
1.01k
    Constraints += MachineClobbers;
2500
1.01k
  }
2501
2502
1.60k
  llvm::Type *ResultType;
2503
1.60k
  if (ResultRegTypes.empty())
2504
742
    ResultType = VoidTy;
2505
864
  else if (ResultRegTypes.size() == 1)
2506
794
    ResultType = ResultRegTypes[0];
2507
70
  else
2508
70
    ResultType = llvm::StructType::get(getLLVMContext(), ResultRegTypes);
2509
2510
1.60k
  llvm::FunctionType *FTy =
2511
1.60k
    llvm::FunctionType::get(ResultType, ArgTypes, false);
2512
2513
1.60k
  bool HasSideEffect = S.isVolatile() || 
S.getNumOutputs() == 0910
;
2514
1.60k
  llvm::InlineAsm::AsmDialect AsmDialect = isa<MSAsmStmt>(&S) ?
2515
1.45k
    
llvm::InlineAsm::AD_Intel156
: llvm::InlineAsm::AD_ATT;
2516
1.60k
  llvm::InlineAsm *IA =
2517
1.60k
    llvm::InlineAsm::get(FTy, AsmString, Constraints, HasSideEffect,
2518
1.60k
                         /* IsAlignStack */ false, AsmDialect);
2519
1.60k
  std::vector<llvm::Value*> RegResults;
2520
1.60k
  if (IsGCCAsmGoto) {
2521
27
    llvm::CallBrInst *Result =
2522
27
        Builder.CreateCallBr(IA, Fallthrough, Transfer, Args);
2523
27
    EmitBlock(Fallthrough);
2524
27
    UpdateAsmCallInst(cast<llvm::CallBase>(*Result), HasSideEffect, ReadOnly,
2525
27
                      ReadNone, InNoMergeAttributedStmt, S, ResultRegTypes,
2526
27
                      *this, RegResults);
2527
1.57k
  } else {
2528
1.57k
    llvm::CallInst *Result =
2529
1.57k
        Builder.CreateCall(IA, Args, getBundlesForFunclet(IA));
2530
1.57k
    UpdateAsmCallInst(cast<llvm::CallBase>(*Result), HasSideEffect, ReadOnly,
2531
1.57k
                      ReadNone, InNoMergeAttributedStmt, S, ResultRegTypes,
2532
1.57k
                      *this, RegResults);
2533
1.57k
  }
2534
2535
1.60k
  assert(RegResults.size() == ResultRegTypes.size());
2536
1.60k
  assert(RegResults.size() == ResultTruncRegTypes.size());
2537
1.60k
  assert(RegResults.size() == ResultRegDests.size());
2538
  // ResultRegDests can be also populated by addReturnRegisterOutputs() above,
2539
  // in which case its size may grow.
2540
1.60k
  assert(ResultTypeRequiresCast.size() <= ResultRegDests.size());
2541
2.60k
  for (unsigned i = 0, e = RegResults.size(); i != e; 
++i995
) {
2542
998
    llvm::Value *Tmp = RegResults[i];
2543
2544
    // If the result type of the LLVM IR asm doesn't match the result type of
2545
    // the expression, do the conversion.
2546
998
    if (ResultRegTypes[i] != ResultTruncRegTypes[i]) {
2547
34
      llvm::Type *TruncTy = ResultTruncRegTypes[i];
2548
2549
      // Truncate the integer result to the right size, note that TruncTy can be
2550
      // a pointer.
2551
34
      if (TruncTy->isFloatingPointTy())
2552
1
        Tmp = Builder.CreateFPTrunc(Tmp, TruncTy);
2553
33
      else if (TruncTy->isPointerTy() && 
Tmp->getType()->isIntegerTy()0
) {
2554
0
        uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(TruncTy);
2555
0
        Tmp = Builder.CreateTrunc(Tmp,
2556
0
                   llvm::IntegerType::get(getLLVMContext(), (unsigned)ResSize));
2557
0
        Tmp = Builder.CreateIntToPtr(Tmp, TruncTy);
2558
33
      } else if (Tmp->getType()->isPointerTy() && 
TruncTy->isIntegerTy()0
) {
2559
0
        uint64_t TmpSize =CGM.getDataLayout().getTypeSizeInBits(Tmp->getType());
2560
0
        Tmp = Builder.CreatePtrToInt(Tmp,
2561
0
                   llvm::IntegerType::get(getLLVMContext(), (unsigned)TmpSize));
2562
0
        Tmp = Builder.CreateTrunc(Tmp, TruncTy);
2563
33
      } else if (TruncTy->isIntegerTy()) {
2564
7
        Tmp = Builder.CreateZExtOrTrunc(Tmp, TruncTy);
2565
26
      } else if (TruncTy->isVectorTy()) {
2566
10
        Tmp = Builder.CreateBitCast(Tmp, TruncTy);
2567
10
      }
2568
34
    }
2569
2570
998
    LValue Dest = ResultRegDests[i];
2571
    // ResultTypeRequiresCast elements correspond to the first
2572
    // ResultTypeRequiresCast.size() elements of RegResults.
2573
998
    if ((i < ResultTypeRequiresCast.size()) && 
ResultTypeRequiresCast[i]975
) {
2574
16
      unsigned Size = getContext().getTypeSize(ResultRegQualTys[i]);
2575
16
      Address A = Builder.CreateBitCast(Dest.getAddress(*this),
2576
16
                                        ResultRegTypes[i]->getPointerTo());
2577
16
      QualType Ty = getContext().getIntTypeForBitwidth(Size, /*Signed*/ false);
2578
16
      if (Ty.isNull()) {
2579
3
        const Expr *OutExpr = S.getOutputExpr(i);
2580
3
        CGM.Error(
2581
3
            OutExpr->getExprLoc(),
2582
3
            "impossible constraint in asm: can't store value into a register");
2583
3
        return;
2584
3
      }
2585
13
      Dest = MakeAddrLValue(A, Ty);
2586
13
    }
2587
995
    EmitStoreThroughLValue(RValue::get(Tmp), Dest);
2588
995
  }
2589
1.60k
}
2590
2591
882
LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {
2592
882
  const RecordDecl *RD = S.getCapturedRecordDecl();
2593
882
  QualType RecordTy = getContext().getRecordType(RD);
2594
2595
  // Initialize the captured struct.
2596
882
  LValue SlotLV =
2597
882
    MakeAddrLValue(CreateMemTemp(RecordTy, "agg.captured"), RecordTy);
2598
2599
882
  RecordDecl::field_iterator CurField = RD->field_begin();
2600
882
  for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
2601
882
                                                 E = S.capture_init_end();
2602
2.16k
       I != E; 
++I, ++CurField1.28k
) {
2603
1.28k
    LValue LV = EmitLValueForFieldInitialization(SlotLV, *CurField);
2604
1.28k
    if (CurField->hasCapturedVLAType()) {
2605
47
      EmitLambdaVLACapture(CurField->getCapturedVLAType(), LV);
2606
1.23k
    } else {
2607
1.23k
      EmitInitializerForField(*CurField, LV, *I);
2608
1.23k
    }
2609
1.28k
  }
2610
2611
882
  return SlotLV;
2612
882
}
2613
2614
/// Generate an outlined function for the body of a CapturedStmt, store any
2615
/// captured variables into the captured struct, and call the outlined function.
2616
llvm::Function *
2617
27
CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
2618
27
  LValue CapStruct = InitCapturedStruct(S);
2619
2620
  // Emit the CapturedDecl
2621
27
  CodeGenFunction CGF(CGM, true);
2622
27
  CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));
2623
27
  llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
2624
27
  delete CGF.CapturedStmtInfo;
2625
2626
  // Emit call to the helper function.
2627
27
  EmitCallOrInvoke(F, CapStruct.getPointer(*this));
2628
2629
27
  return F;
2630
27
}
2631
2632
855
Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
2633
855
  LValue CapStruct = InitCapturedStruct(S);
2634
855
  return CapStruct.getAddress(*this);
2635
855
}
2636
2637
/// Creates the outlined function for a CapturedStmt.
2638
llvm::Function *
2639
882
CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
2640
882
  assert(CapturedStmtInfo &&
2641
882
    "CapturedStmtInfo should be set when generating the captured function");
2642
882
  const CapturedDecl *CD = S.getCapturedDecl();
2643
882
  const RecordDecl *RD = S.getCapturedRecordDecl();
2644
882
  SourceLocation Loc = S.getBeginLoc();
2645
882
  assert(CD->hasBody() && "missing CapturedDecl body");
2646
2647
  // Build the argument list.
2648
882
  ASTContext &Ctx = CGM.getContext();
2649
882
  FunctionArgList Args;
2650
882
  Args.append(CD->param_begin(), CD->param_end());
2651
2652
  // Create the function declaration.
2653
882
  const CGFunctionInfo &FuncInfo =
2654
882
    CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
2655
882
  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
2656
2657
882
  llvm::Function *F =
2658
882
    llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
2659
882
                           CapturedStmtInfo->getHelperName(), &CGM.getModule());
2660
882
  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
2661
882
  if (CD->isNothrow())
2662
701
    F->addFnAttr(llvm::Attribute::NoUnwind);
2663
2664
  // Generate the function.
2665
882
  StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(),
2666
882
                CD->getBody()->getBeginLoc());
2667
  // Set the context parameter in CapturedStmtInfo.
2668
882
  Address DeclPtr = GetAddrOfLocalVar(CD->getContextParam());
2669
882
  CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr));
2670
2671
  // Initialize variable-length arrays.
2672
882
  LValue Base = MakeNaturalAlignAddrLValue(CapturedStmtInfo->getContextValue(),
2673
882
                                           Ctx.getTagDeclType(RD));
2674
1.28k
  for (auto *FD : RD->fields()) {
2675
1.28k
    if (FD->hasCapturedVLAType()) {
2676
47
      auto *ExprArg =
2677
47
          EmitLoadOfLValue(EmitLValueForField(Base, FD), S.getBeginLoc())
2678
47
              .getScalarVal();
2679
47
      auto VAT = FD->getCapturedVLAType();
2680
47
      VLASizeMap[VAT->getSizeExpr()] = ExprArg;
2681
47
    }
2682
1.28k
  }
2683
2684
  // If 'this' is captured, load it into CXXThisValue.
2685
882
  if (CapturedStmtInfo->isCXXThisExprCaptured()) {
2686
24
    FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();
2687
24
    LValue ThisLValue = EmitLValueForField(Base, FD);
2688
24
    CXXThisValue = EmitLoadOfLValue(ThisLValue, Loc).getScalarVal();
2689
24
  }
2690
2691
882
  PGO.assignRegionCounters(GlobalDecl(CD), F);
2692
882
  CapturedStmtInfo->EmitBody(*this, CD->getBody());
2693
882
  FinishFunction(CD->getBodyRBrace());
2694
2695
882
  return F;
2696
882
}