Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/tools/clang/lib/CodeGen/CGStmtOpenMP.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This contains code to emit OpenMP nodes as LLVM code.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGCleanup.h"
14
#include "CGOpenMPRuntime.h"
15
#include "CodeGenFunction.h"
16
#include "CodeGenModule.h"
17
#include "TargetInfo.h"
18
#include "clang/AST/Stmt.h"
19
#include "clang/AST/StmtOpenMP.h"
20
#include "clang/AST/DeclOpenMP.h"
21
using namespace clang;
22
using namespace CodeGen;
23
24
namespace {
25
/// Lexical scope for OpenMP executable constructs, that handles correct codegen
26
/// for captured expressions.
27
class OMPLexicalScope : public CodeGenFunction::LexicalScope {
28
9.12k
  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
29
10.4k
    for (const auto *C : S.clauses()) {
30
10.4k
      if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
31
6.57k
        if (const auto *PreInit =
32
641
                cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
33
699
          for (const auto *I : PreInit->decls()) {
34
699
            if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
35
686
              CGF.EmitVarDecl(cast<VarDecl>(*I));
36
686
            } else {
37
13
              CodeGenFunction::AutoVarEmission Emission =
38
13
                  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
39
13
              CGF.EmitAutoVarCleanups(Emission);
40
13
            }
41
699
          }
42
641
        }
43
6.57k
      }
44
10.4k
    }
45
9.12k
  }
46
  CodeGenFunction::OMPPrivateScope InlinedShareds;
47
48
10.4k
  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
49
10.4k
    return CGF.LambdaCaptureFields.lookup(VD) ||
50
10.4k
           
(10.0k
CGF.CapturedStmtInfo10.0k
&&
CGF.CapturedStmtInfo->lookup(VD)2.62k
) ||
51
10.4k
           
(7.72k
CGF.CurCodeDecl7.72k
&&
isa<BlockDecl>(CGF.CurCodeDecl)7.72k
);
52
10.4k
  }
53
54
public:
55
  OMPLexicalScope(
56
      CodeGenFunction &CGF, const OMPExecutableDirective &S,
57
      const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
58
      const bool EmitPreInitStmt = true)
59
      : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
60
14.5k
        InlinedShareds(CGF) {
61
14.5k
    if (EmitPreInitStmt)
62
9.12k
      emitPreInitStmt(CGF, S);
63
14.5k
    if (!CapturedRegion.hasValue())
64
7.58k
      return;
65
6.93k
    assert(S.hasAssociatedStmt() &&
66
6.93k
           "Expected associated statement for inlined directive.");
67
6.93k
    const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
68
11.7k
    for (const auto &C : CS->captures()) {
69
11.7k
      if (C.capturesVariable() || 
C.capturesVariableByCopy()6.95k
) {
70
10.4k
        auto *VD = C.getCapturedVar();
71
10.4k
        assert(VD == VD->getCanonicalDecl() &&
72
10.4k
               "Canonical decl must be captured.");
73
10.4k
        DeclRefExpr DRE(
74
10.4k
            CGF.getContext(), const_cast<VarDecl *>(VD),
75
10.4k
            isCapturedVar(CGF, VD) || 
(7.72k
CGF.CapturedStmtInfo7.72k
&&
76
7.72k
                                       
InlinedShareds.isGlobalVarCaptured(VD)259
),
77
10.4k
            VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
78
10.4k
        InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
79
10.4k
          return CGF.EmitLValue(&DRE).getAddress();
80
10.4k
        });
81
10.4k
      }
82
11.7k
    }
83
6.93k
    (void)InlinedShareds.Privatize();
84
6.93k
  }
85
};
86
87
/// Lexical scope for OpenMP parallel construct, that handles correct codegen
88
/// for captured expressions.
89
class OMPParallelScope final : public OMPLexicalScope {
90
3.78k
  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
91
3.78k
    OpenMPDirectiveKind Kind = S.getDirectiveKind();
92
3.78k
    return !(isOpenMPTargetExecutionDirective(Kind) ||
93
3.78k
             
isOpenMPLoopBoundSharingDirective(Kind)1.79k
) &&
94
3.78k
           
isOpenMPParallelDirective(Kind)836
;
95
3.78k
  }
96
97
public:
98
  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
99
      : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
100
3.78k
                        EmitPreInitStmt(S)) {}
101
};
102
103
/// Lexical scope for OpenMP teams construct, that handles correct codegen
104
/// for captured expressions.
105
class OMPTeamsScope final : public OMPLexicalScope {
106
3.55k
  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
107
3.55k
    OpenMPDirectiveKind Kind = S.getDirectiveKind();
108
3.55k
    return !isOpenMPTargetExecutionDirective(Kind) &&
109
3.55k
           
isOpenMPTeamsDirective(Kind)1.39k
;
110
3.55k
  }
111
112
public:
113
  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
114
      : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
115
3.55k
                        EmitPreInitStmt(S)) {}
116
};
117
118
/// Private scope for OpenMP loop-based directives, that supports capturing
119
/// of used expression from loop statement.
120
class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
121
10.9k
  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
122
10.9k
    CodeGenFunction::OMPMapVars PreCondVars;
123
11.3k
    for (const auto *E : S.counters()) {
124
11.3k
      const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
125
11.3k
      (void)PreCondVars.setVarAddr(
126
11.3k
          CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
127
11.3k
    }
128
10.9k
    (void)PreCondVars.apply(CGF);
129
10.9k
    if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
130
2.34k
      for (const auto *I : PreInits->decls())
131
4.91k
        CGF.EmitVarDecl(cast<VarDecl>(*I));
132
2.34k
    }
133
10.9k
    PreCondVars.restore(CGF);
134
10.9k
  }
135
136
public:
137
  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
138
10.9k
      : CodeGenFunction::RunCleanupsScope(CGF) {
139
10.9k
    emitPreInitStmt(CGF, S);
140
10.9k
  }
141
};
142
143
class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
144
  CodeGenFunction::OMPPrivateScope InlinedShareds;
145
146
29.6k
  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
147
29.6k
    return CGF.LambdaCaptureFields.lookup(VD) ||
148
29.6k
           
(28.7k
CGF.CapturedStmtInfo28.7k
&&
CGF.CapturedStmtInfo->lookup(VD)5.79k
) ||
149
29.6k
           
(28.7k
CGF.CurCodeDecl28.7k
&&
isa<BlockDecl>(CGF.CurCodeDecl)28.7k
&&
150
28.7k
            
cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)68
);
151
29.6k
  }
152
153
public:
154
  OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
155
      : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
156
9.29k
        InlinedShareds(CGF) {
157
11.7k
    for (const auto *C : S.clauses()) {
158
11.7k
      if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
159
7.52k
        if (const auto *PreInit =
160
849
                cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
161
907
          for (const auto *I : PreInit->decls()) {
162
907
            if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
163
894
              CGF.EmitVarDecl(cast<VarDecl>(*I));
164
894
            } else {
165
13
              CodeGenFunction::AutoVarEmission Emission =
166
13
                  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
167
13
              CGF.EmitAutoVarCleanups(Emission);
168
13
            }
169
907
          }
170
849
        }
171
7.52k
      } else 
if (const auto *4.26k
UDP4.26k
= dyn_cast<OMPUseDevicePtrClause>(C)) {
172
80
        for (const Expr *E : UDP->varlists()) {
173
80
          const Decl *D = cast<DeclRefExpr>(E)->getDecl();
174
80
          if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
175
20
            CGF.EmitVarDecl(*OED);
176
80
        }
177
72
      }
178
11.7k
    }
179
9.29k
    if (!isOpenMPSimdDirective(S.getDirectiveKind()))
180
7.14k
      CGF.EmitOMPPrivateClause(S, InlinedShareds);
181
9.29k
    if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
182
29
      if (const Expr *E = TG->getReductionRef())
183
18
        CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
184
29
    }
185
9.29k
    const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
186
27.8k
    while (CS) {
187
34.1k
      for (auto &C : CS->captures()) {
188
34.1k
        if (C.capturesVariable() || 
C.capturesVariableByCopy()20.9k
) {
189
29.6k
          auto *VD = C.getCapturedVar();
190
29.6k
          assert(VD == VD->getCanonicalDecl() &&
191
29.6k
                 "Canonical decl must be captured.");
192
29.6k
          DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
193
29.6k
                          isCapturedVar(CGF, VD) ||
194
29.6k
                              
(28.6k
CGF.CapturedStmtInfo28.6k
&&
195
28.6k
                               
InlinedShareds.isGlobalVarCaptured(VD)5.77k
),
196
29.6k
                          VD->getType().getNonReferenceType(), VK_LValue,
197
29.6k
                          C.getLocation());
198
29.6k
          InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
199
29.6k
            return CGF.EmitLValue(&DRE).getAddress();
200
29.6k
          });
201
29.6k
        }
202
34.1k
      }
203
18.5k
      CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
204
18.5k
    }
205
9.29k
    (void)InlinedShareds.Privatize();
206
9.29k
  }
207
};
208
209
} // namespace
210
211
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
212
                                         const OMPExecutableDirective &S,
213
                                         const RegionCodeGenTy &CodeGen);
214
215
5.34k
LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
216
5.34k
  if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
217
4.39k
    if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
218
4.39k
      OrigVD = OrigVD->getCanonicalDecl();
219
4.39k
      bool IsCaptured =
220
4.39k
          LambdaCaptureFields.lookup(OrigVD) ||
221
4.39k
          
(4.35k
CapturedStmtInfo4.35k
&&
CapturedStmtInfo->lookup(OrigVD)529
) ||
222
4.39k
          
(3.96k
CurCodeDecl3.96k
&&
isa<BlockDecl>(CurCodeDecl)3.96k
);
223
4.39k
      DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
224
4.39k
                      OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
225
4.39k
      return EmitLValue(&DRE);
226
4.39k
    }
227
953
  }
228
953
  return EmitLValue(E);
229
953
}
230
231
9.99k
llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
232
9.99k
  ASTContext &C = getContext();
233
9.99k
  llvm::Value *Size = nullptr;
234
9.99k
  auto SizeInChars = C.getTypeSizeInChars(Ty);
235
9.99k
  if (SizeInChars.isZero()) {
236
724
    // getTypeSizeInChars() returns 0 for a VLA.
237
1.44k
    while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
238
724
      VlaSizePair VlaSize = getVLASize(VAT);
239
724
      Ty = VlaSize.Type;
240
724
      Size = Size ? 
Builder.CreateNUWMul(Size, VlaSize.NumElts)0
241
724
                  : VlaSize.NumElts;
242
724
    }
243
724
    SizeInChars = C.getTypeSizeInChars(Ty);
244
724
    if (SizeInChars.isZero())
245
0
      return llvm::ConstantInt::get(SizeTy, /*V=*/0);
246
724
    return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
247
724
  }
248
9.26k
  return CGM.getSize(SizeInChars);
249
9.26k
}
250
251
void CodeGenFunction::GenerateOpenMPCapturedVars(
252
12.1k
    const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
253
12.1k
  const RecordDecl *RD = S.getCapturedRecordDecl();
254
12.1k
  auto CurField = RD->field_begin();
255
12.1k
  auto CurCap = S.captures().begin();
256
12.1k
  for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
257
12.1k
                                                 E = S.capture_init_end();
258
32.8k
       I != E; 
++I, ++CurField, ++CurCap20.6k
) {
259
20.6k
    if (CurField->hasCapturedVLAType()) {
260
1.55k
      const VariableArrayType *VAT = CurField->getCapturedVLAType();
261
1.55k
      llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
262
1.55k
      CapturedVars.push_back(Val);
263
19.1k
    } else if (CurCap->capturesThis()) {
264
1.04k
      CapturedVars.push_back(CXXThisValue);
265
18.0k
    } else if (CurCap->capturesVariableByCopy()) {
266
9.09k
      llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
267
9.09k
268
9.09k
      // If the field is not a pointer, we need to save the actual value
269
9.09k
      // and load it as a void pointer.
270
9.09k
      if (!CurField->getType()->isAnyPointerType()) {
271
7.85k
        ASTContext &Ctx = getContext();
272
7.85k
        Address DstAddr = CreateMemTemp(
273
7.85k
            Ctx.getUIntPtrType(),
274
7.85k
            Twine(CurCap->getCapturedVar()->getName(), ".casted"));
275
7.85k
        LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
276
7.85k
277
7.85k
        llvm::Value *SrcAddrVal = EmitScalarConversion(
278
7.85k
            DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
279
7.85k
            Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
280
7.85k
        LValue SrcLV =
281
7.85k
            MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
282
7.85k
283
7.85k
        // Store the value using the source type pointer.
284
7.85k
        EmitStoreThroughLValue(RValue::get(CV), SrcLV);
285
7.85k
286
7.85k
        // Load the value using the destination type pointer.
287
7.85k
        CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
288
7.85k
      }
289
9.09k
      CapturedVars.push_back(CV);
290
9.09k
    } else {
291
8.98k
      assert(CurCap->capturesVariable() && "Expected capture by reference.");
292
8.98k
      CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
293
8.98k
    }
294
20.6k
  }
295
12.1k
}
296
297
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
298
                                    QualType DstType, StringRef Name,
299
10.8k
                                    LValue AddrLV) {
300
10.8k
  ASTContext &Ctx = CGF.getContext();
301
10.8k
302
10.8k
  llvm::Value *CastedPtr = CGF.EmitScalarConversion(
303
10.8k
      AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
304
10.8k
      Ctx.getPointerType(DstType), Loc);
305
10.8k
  Address TmpAddr =
306
10.8k
      CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
307
10.8k
          .getAddress();
308
10.8k
  return TmpAddr;
309
10.8k
}
310
311
4.31k
static QualType getCanonicalParamType(ASTContext &C, QualType T) {
312
4.31k
  if (T->isLValueReferenceType())
313
1.27k
    return C.getLValueReferenceType(
314
1.27k
        getCanonicalParamType(C, T.getNonReferenceType()),
315
1.27k
        /*SpelledAsLValue=*/false);
316
3.04k
  if (T->isPointerType())
317
23
    return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
318
3.02k
  if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
319
1.75k
    if (const auto *VLA = dyn_cast<VariableArrayType>(A))
320
1.73k
      return getCanonicalParamType(C, VLA->getElementType());
321
20
    if (!A->isVariablyModifiedType())
322
20
      return C.getCanonicalType(T);
323
1.26k
  }
324
1.26k
  return C.getCanonicalParamType(T);
325
1.26k
}
326
327
namespace {
328
  /// Contains required data for proper outlined function codegen.
329
  struct FunctionOptions {
330
    /// Captured statement for which the function is generated.
331
    const CapturedStmt *S = nullptr;
332
    /// true if cast to/from  UIntPtr is required for variables captured by
333
    /// value.
334
    const bool UIntPtrCastRequired = true;
335
    /// true if only casted arguments must be registered as local args or VLA
336
    /// sizes.
337
    const bool RegisterCastedArgsOnly = false;
338
    /// Name of the generated function.
339
    const StringRef FunctionName;
340
    explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
341
                             bool RegisterCastedArgsOnly,
342
                             StringRef FunctionName)
343
        : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
344
          RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
345
13.7k
          FunctionName(FunctionName) {}
346
  };
347
}
348
349
static llvm::Function *emitOutlinedFunctionPrologue(
350
    CodeGenFunction &CGF, FunctionArgList &Args,
351
    llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
352
        &LocalAddrs,
353
    llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
354
        &VLASizes,
355
13.7k
    llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
356
13.7k
  const CapturedDecl *CD = FO.S->getCapturedDecl();
357
13.7k
  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
358
13.7k
  assert(CD->hasBody() && "missing CapturedDecl body");
359
13.7k
360
13.7k
  CXXThisValue = nullptr;
361
13.7k
  // Build the argument list.
362
13.7k
  CodeGenModule &CGM = CGF.CGM;
363
13.7k
  ASTContext &Ctx = CGM.getContext();
364
13.7k
  FunctionArgList TargetArgs;
365
13.7k
  Args.append(CD->param_begin(),
366
13.7k
              std::next(CD->param_begin(), CD->getContextParamPosition()));
367
13.7k
  TargetArgs.append(
368
13.7k
      CD->param_begin(),
369
13.7k
      std::next(CD->param_begin(), CD->getContextParamPosition()));
370
13.7k
  auto I = FO.S->captures().begin();
371
13.7k
  FunctionDecl *DebugFunctionDecl = nullptr;
372
13.7k
  if (!FO.UIntPtrCastRequired) {
373
46
    FunctionProtoType::ExtProtoInfo EPI;
374
46
    QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
375
46
    DebugFunctionDecl = FunctionDecl::Create(
376
46
        Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
377
46
        SourceLocation(), DeclarationName(), FunctionTy,
378
46
        Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
379
46
        /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
380
46
  }
381
23.1k
  for (const FieldDecl *FD : RD->fields()) {
382
23.1k
    QualType ArgType = FD->getType();
383
23.1k
    IdentifierInfo *II = nullptr;
384
23.1k
    VarDecl *CapVar = nullptr;
385
23.1k
386
23.1k
    // If this is a capture by copy and the type is not a pointer, the outlined
387
23.1k
    // function argument type should be uintptr and the value properly casted to
388
23.1k
    // uintptr. This is necessary given that the runtime library is only able to
389
23.1k
    // deal with pointers. We can pass in the same way the VLA type sizes to the
390
23.1k
    // outlined function.
391
23.1k
    if (FO.UIntPtrCastRequired &&
392
23.1k
        
(22.9k
(22.9k
I->capturesVariableByCopy()22.9k
&&
!ArgType->isAnyPointerType()10.4k
) ||
393
22.9k
         
I->capturesVariableArrayType()13.8k
))
394
10.8k
      ArgType = Ctx.getUIntPtrType();
395
23.1k
396
23.1k
    if (I->capturesVariable() || 
I->capturesVariableByCopy()13.3k
) {
397
20.1k
      CapVar = I->getCapturedVar();
398
20.1k
      II = CapVar->getIdentifier();
399
20.1k
    } else 
if (2.93k
I->capturesThis()2.93k
) {
400
1.14k
      II = &Ctx.Idents.get("this");
401
1.78k
    } else {
402
1.78k
      assert(I->capturesVariableArrayType());
403
1.78k
      II = &Ctx.Idents.get("vla");
404
1.78k
    }
405
23.1k
    if (ArgType->isVariablyModifiedType())
406
1.28k
      ArgType = getCanonicalParamType(Ctx, ArgType);
407
23.1k
    VarDecl *Arg;
408
23.1k
    if (DebugFunctionDecl && 
(122
CapVar122
||
I->capturesThis()7
)) {
409
119
      Arg = ParmVarDecl::Create(
410
119
          Ctx, DebugFunctionDecl,
411
119
          CapVar ? 
CapVar->getBeginLoc()115
:
FD->getBeginLoc()4
,
412
119
          CapVar ? 
CapVar->getLocation()115
:
FD->getLocation()4
, II, ArgType,
413
119
          /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
414
22.9k
    } else {
415
22.9k
      Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
416
22.9k
                                      II, ArgType, ImplicitParamDecl::Other);
417
22.9k
    }
418
23.1k
    Args.emplace_back(Arg);
419
23.1k
    // Do not cast arguments if we emit function with non-original types.
420
23.1k
    TargetArgs.emplace_back(
421
23.1k
        FO.UIntPtrCastRequired
422
23.1k
            ? 
Arg22.9k
423
23.1k
            : 
CGM.getOpenMPRuntime().translateParameter(FD, Arg)122
);
424
23.1k
    ++I;
425
23.1k
  }
426
13.7k
  Args.append(
427
13.7k
      std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
428
13.7k
      CD->param_end());
429
13.7k
  TargetArgs.append(
430
13.7k
      std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
431
13.7k
      CD->param_end());
432
13.7k
433
13.7k
  // Create the function declaration.
434
13.7k
  const CGFunctionInfo &FuncInfo =
435
13.7k
      CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
436
13.7k
  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
437
13.7k
438
13.7k
  auto *F =
439
13.7k
      llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
440
13.7k
                             FO.FunctionName, &CGM.getModule());
441
13.7k
  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
442
13.7k
  if (CD->isNothrow())
443
13.6k
    F->setDoesNotThrow();
444
13.7k
  F->setDoesNotRecurse();
445
13.7k
446
13.7k
  // Generate the function.
447
13.7k
  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
448
13.7k
                    FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
449
13.7k
  unsigned Cnt = CD->getContextParamPosition();
450
13.7k
  I = FO.S->captures().begin();
451
23.1k
  for (const FieldDecl *FD : RD->fields()) {
452
23.1k
    // Do not map arguments if we emit function with non-original types.
453
23.1k
    Address LocalAddr(Address::invalid());
454
23.1k
    if (!FO.UIntPtrCastRequired && 
Args[Cnt] != TargetArgs[Cnt]122
) {
455
54
      LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
456
54
                                                             TargetArgs[Cnt]);
457
23.0k
    } else {
458
23.0k
      LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
459
23.0k
    }
460
23.1k
    // If we are capturing a pointer by copy we don't need to do anything, just
461
23.1k
    // use the value that we get from the arguments.
462
23.1k
    if (I->capturesVariableByCopy() && 
FD->getType()->isAnyPointerType()10.4k
) {
463
1.30k
      const VarDecl *CurVD = I->getCapturedVar();
464
1.30k
      if (!FO.RegisterCastedArgsOnly)
465
1.30k
        LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
466
1.30k
      ++Cnt;
467
1.30k
      ++I;
468
1.30k
      continue;
469
1.30k
    }
470
21.7k
471
21.7k
    LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
472
21.7k
                                        AlignmentSource::Decl);
473
21.7k
    if (FD->hasCapturedVLAType()) {
474
1.78k
      if (FO.UIntPtrCastRequired) {
475
1.78k
        ArgLVal = CGF.MakeAddrLValue(
476
1.78k
            castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
477
1.78k
                                 Args[Cnt]->getName(), ArgLVal),
478
1.78k
            FD->getType(), AlignmentSource::Decl);
479
1.78k
      }
480
1.78k
      llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
481
1.78k
      const VariableArrayType *VAT = FD->getCapturedVLAType();
482
1.78k
      VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
483
20.0k
    } else if (I->capturesVariable()) {
484
9.74k
      const VarDecl *Var = I->getCapturedVar();
485
9.74k
      QualType VarTy = Var->getType();
486
9.74k
      Address ArgAddr = ArgLVal.getAddress();
487
9.74k
      if (ArgLVal.getType()->isLValueReferenceType()) {
488
9.74k
        ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
489
9.74k
      } else 
if (0
!VarTy->isVariablyModifiedType()0
||
!VarTy->isPointerType()0
) {
490
0
        assert(ArgLVal.getType()->isPointerType());
491
0
        ArgAddr = CGF.EmitLoadOfPointer(
492
0
            ArgAddr, ArgLVal.getType()->castAs<PointerType>());
493
0
      }
494
9.74k
      if (!FO.RegisterCastedArgsOnly) {
495
9.65k
        LocalAddrs.insert(
496
9.65k
            {Args[Cnt],
497
9.65k
             {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
498
9.65k
      }
499
10.2k
    } else if (I->capturesVariableByCopy()) {
500
9.11k
      assert(!FD->getType()->isAnyPointerType() &&
501
9.11k
             "Not expecting a captured pointer.");
502
9.11k
      const VarDecl *Var = I->getCapturedVar();
503
9.11k
      LocalAddrs.insert({Args[Cnt],
504
9.11k
                         {Var, FO.UIntPtrCastRequired
505
9.11k
                                   ? castValueFromUintptr(
506
9.09k
                                         CGF, I->getLocation(), FD->getType(),
507
9.09k
                                         Args[Cnt]->getName(), ArgLVal)
508
9.11k
                                   : 
ArgLVal.getAddress()21
}});
509
9.11k
    } else {
510
1.14k
      // If 'this' is captured, load it into CXXThisValue.
511
1.14k
      assert(I->capturesThis());
512
1.14k
      CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
513
1.14k
      LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
514
1.14k
    }
515
21.7k
    ++Cnt;
516
21.7k
    ++I;
517
21.7k
  }
518
13.7k
519
13.7k
  return F;
520
13.7k
}
521
522
llvm::Function *
523
13.6k
CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) {
524
13.6k
  assert(
525
13.6k
      CapturedStmtInfo &&
526
13.6k
      "CapturedStmtInfo should be set when generating the captured function");
527
13.6k
  const CapturedDecl *CD = S.getCapturedDecl();
528
13.6k
  // Build the argument list.
529
13.6k
  bool NeedWrapperFunction =
530
13.6k
      getDebugInfo() &&
531
13.6k
      
CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo86
;
532
13.6k
  FunctionArgList Args;
533
13.6k
  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
534
13.6k
  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
535
13.6k
  SmallString<256> Buffer;
536
13.6k
  llvm::raw_svector_ostream Out(Buffer);
537
13.6k
  Out << CapturedStmtInfo->getHelperName();
538
13.6k
  if (NeedWrapperFunction)
539
46
    Out << "_debug__";
540
13.6k
  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
541
13.6k
                     Out.str());
542
13.6k
  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
543
13.6k
                                                   VLASizes, CXXThisValue, FO);
544
13.6k
  CodeGenFunction::OMPPrivateScope LocalScope(*this);
545
21.1k
  for (const auto &LocalAddrPair : LocalAddrs) {
546
21.1k
    if (LocalAddrPair.second.first) {
547
20.0k
      LocalScope.addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() {
548
20.0k
        return LocalAddrPair.second.second;
549
20.0k
      });
550
20.0k
    }
551
21.1k
  }
552
13.6k
  (void)LocalScope.Privatize();
553
13.6k
  for (const auto &VLASizePair : VLASizes)
554
1.78k
    VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
555
13.6k
  PGO.assignRegionCounters(GlobalDecl(CD), F);
556
13.6k
  CapturedStmtInfo->EmitBody(*this, CD->getBody());
557
13.6k
  (void)LocalScope.ForceCleanup();
558
13.6k
  FinishFunction(CD->getBodyRBrace());
559
13.6k
  if (!NeedWrapperFunction)
560
13.6k
    return F;
561
46
562
46
  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
563
46
                            /*RegisterCastedArgsOnly=*/true,
564
46
                            CapturedStmtInfo->getHelperName());
565
46
  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
566
46
  WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
567
46
  Args.clear();
568
46
  LocalAddrs.clear();
569
46
  VLASizes.clear();
570
46
  llvm::Function *WrapperF =
571
46
      emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
572
46
                                   WrapperCGF.CXXThisValue, WrapperFO);
573
46
  llvm::SmallVector<llvm::Value *, 4> CallArgs;
574
178
  for (const auto *Arg : Args) {
575
178
    llvm::Value *CallArg;
576
178
    auto I = LocalAddrs.find(Arg);
577
178
    if (I != LocalAddrs.end()) {
578
25
      LValue LV = WrapperCGF.MakeAddrLValue(
579
25
          I->second.second,
580
25
          I->second.first ? 
I->second.first->getType()21
:
Arg->getType()4
,
581
25
          AlignmentSource::Decl);
582
25
      CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
583
153
    } else {
584
153
      auto EI = VLASizes.find(Arg);
585
153
      if (EI != VLASizes.end()) {
586
3
        CallArg = EI->second.second;
587
150
      } else {
588
150
        LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
589
150
                                              Arg->getType(),
590
150
                                              AlignmentSource::Decl);
591
150
        CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
592
150
      }
593
153
    }
594
178
    CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
595
178
  }
596
46
  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
597
46
                                                  F, CallArgs);
598
46
  WrapperCGF.FinishFunction();
599
46
  return WrapperF;
600
46
}
601
602
//===----------------------------------------------------------------------===//
603
//                              OpenMP Directive Emission
604
//===----------------------------------------------------------------------===//
605
void CodeGenFunction::EmitOMPAggregateAssign(
606
    Address DestAddr, Address SrcAddr, QualType OriginalType,
607
465
    const llvm::function_ref<void(Address, Address)> CopyGen) {
608
465
  // Perform element-by-element initialization.
609
465
  QualType ElementTy;
610
465
611
465
  // Drill down to the base element type on both arrays.
612
465
  const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
613
465
  llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
614
465
  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
615
465
616
465
  llvm::Value *SrcBegin = SrcAddr.getPointer();
617
465
  llvm::Value *DestBegin = DestAddr.getPointer();
618
465
  // Cast from pointer to array type to pointer to single element.
619
465
  llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
620
465
  // The basic structure here is a while-do loop.
621
465
  llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
622
465
  llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
623
465
  llvm::Value *IsEmpty =
624
465
      Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
625
465
  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
626
465
627
465
  // Enter the loop body, making that address the current address.
628
465
  llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
629
465
  EmitBlock(BodyBB);
630
465
631
465
  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
632
465
633
465
  llvm::PHINode *SrcElementPHI =
634
465
    Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
635
465
  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
636
465
  Address SrcElementCurrent =
637
465
      Address(SrcElementPHI,
638
465
              SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
639
465
640
465
  llvm::PHINode *DestElementPHI =
641
465
    Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
642
465
  DestElementPHI->addIncoming(DestBegin, EntryBB);
643
465
  Address DestElementCurrent =
644
465
    Address(DestElementPHI,
645
465
            DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
646
465
647
465
  // Emit copy.
648
465
  CopyGen(DestElementCurrent, SrcElementCurrent);
649
465
650
465
  // Shift the address forward by one element.
651
465
  llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
652
465
      DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
653
465
  llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
654
465
      SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
655
465
  // Check whether we've reached the end.
656
465
  llvm::Value *Done =
657
465
      Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
658
465
  Builder.CreateCondBr(Done, DoneBB, BodyBB);
659
465
  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
660
465
  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
661
465
662
465
  // Done.
663
465
  EmitBlock(DoneBB, /*IsFinished=*/true);
664
465
}
665
666
void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
667
                                  Address SrcAddr, const VarDecl *DestVD,
668
1.57k
                                  const VarDecl *SrcVD, const Expr *Copy) {
669
1.57k
  if (OriginalType->isArrayType()) {
670
460
    const auto *BO = dyn_cast<BinaryOperator>(Copy);
671
460
    if (BO && 
BO->getOpcode() == BO_Assign235
) {
672
235
      // Perform simple memcpy for simple copying.
673
235
      LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
674
235
      LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
675
235
      EmitAggregateAssign(Dest, Src, OriginalType);
676
235
    } else {
677
225
      // For arrays with complex element types perform element by element
678
225
      // copying.
679
225
      EmitOMPAggregateAssign(
680
225
          DestAddr, SrcAddr, OriginalType,
681
225
          [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
682
225
            // Working with the single array element, so have to remap
683
225
            // destination and source variables to corresponding array
684
225
            // elements.
685
225
            CodeGenFunction::OMPPrivateScope Remap(*this);
686
225
            Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
687
225
            Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
688
225
            (void)Remap.Privatize();
689
225
            EmitIgnoredExpr(Copy);
690
225
          });
691
225
    }
692
1.11k
  } else {
693
1.11k
    // Remap pseudo source variable to private copy.
694
1.11k
    CodeGenFunction::OMPPrivateScope Remap(*this);
695
1.11k
    Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
696
1.11k
    Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
697
1.11k
    (void)Remap.Privatize();
698
1.11k
    // Emit copying of the whole variable.
699
1.11k
    EmitIgnoredExpr(Copy);
700
1.11k
  }
701
1.57k
}
702
703
bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
704
11.3k
                                                OMPPrivateScope &PrivateScope) {
705
11.3k
  if (!HaveInsertPoint())
706
0
    return false;
707
11.3k
  bool DeviceConstTarget =
708
11.3k
      getLangOpts().OpenMPIsDevice &&
709
11.3k
      
isOpenMPTargetExecutionDirective(D.getDirectiveKind())2.52k
;
710
11.3k
  bool FirstprivateIsLastprivate = false;
711
11.3k
  llvm::DenseSet<const VarDecl *> Lastprivates;
712
11.3k
  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
713
315
    for (const auto *D : C->varlists())
714
1.37k
      Lastprivates.insert(
715
1.37k
          cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
716
315
  }
717
11.3k
  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
718
11.3k
  llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
719
11.3k
  getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
720
11.3k
  // Force emission of the firstprivate copy if the directive does not emit
721
11.3k
  // outlined function, like omp for, omp simd, omp distribute etc.
722
11.3k
  bool MustEmitFirstprivateCopy =
723
11.3k
      CaptureRegions.size() == 1 && 
CaptureRegions.back() == OMPD_unknown3.56k
;
724
11.3k
  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
725
4.19k
    auto IRef = C->varlist_begin();
726
4.19k
    auto InitsRef = C->inits().begin();
727
7.27k
    for (const Expr *IInit : C->private_copies()) {
728
7.27k
      const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
729
7.27k
      bool ThisFirstprivateIsLastprivate =
730
7.27k
          Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
731
7.27k
      const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
732
7.27k
      const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
733
7.27k
      if (!MustEmitFirstprivateCopy && 
!ThisFirstprivateIsLastprivate7.06k
&&
FD7.06k
&&
734
7.27k
          
!FD->getType()->isReferenceType()7.05k
&&
735
7.27k
          
(6.01k
!VD6.01k
||
!VD->hasAttr<OMPAllocateDeclAttr>()6.01k
)) {
736
6.00k
        EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
737
6.00k
        ++IRef;
738
6.00k
        ++InitsRef;
739
6.00k
        continue;
740
6.00k
      }
741
1.26k
      // Do not emit copy for firstprivate constant variables in target regions,
742
1.26k
      // captured by reference.
743
1.26k
      if (DeviceConstTarget && 
OrigVD->getType().isConstant(getContext())185
&&
744
1.26k
          
FD6
&&
FD->getType()->isReferenceType()6
&&
745
1.26k
          
(6
!VD6
||
!VD->hasAttr<OMPAllocateDeclAttr>()6
)) {
746
6
        (void)CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(*this,
747
6
                                                                    OrigVD);
748
6
        ++IRef;
749
6
        ++InitsRef;
750
6
        continue;
751
6
      }
752
1.25k
      FirstprivateIsLastprivate =
753
1.25k
          FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
754
1.25k
      if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
755
1.15k
        const auto *VDInit =
756
1.15k
            cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
757
1.15k
        bool IsRegistered;
758
1.15k
        DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
759
1.15k
                        /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
760
1.15k
                        (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
761
1.15k
        LValue OriginalLVal;
762
1.15k
        if (!FD) {
763
46
          // Check if the firstprivate variable is just a constant value.
764
46
          ConstantEmission CE = tryEmitAsConstant(&DRE);
765
46
          if (CE && 
!CE.isReference()6
) {
766
4
            // Constant value, no need to create a copy.
767
4
            ++IRef;
768
4
            ++InitsRef;
769
4
            continue;
770
4
          }
771
42
          if (CE && 
CE.isReference()2
) {
772
2
            OriginalLVal = CE.getReferenceLValue(*this, &DRE);
773
40
          } else {
774
40
            assert(!CE && "Expected non-constant firstprivate.");
775
40
            OriginalLVal = EmitLValue(&DRE);
776
40
          }
777
1.11k
        } else {
778
1.11k
          OriginalLVal = EmitLValue(&DRE);
779
1.11k
        }
780
1.15k
        QualType Type = VD->getType();
781
1.15k
        if (Type->isArrayType()) {
782
563
          // Emit VarDecl with copy init for arrays.
783
563
          // Get the address of the original variable captured in current
784
563
          // captured region.
785
563
          IsRegistered = PrivateScope.addPrivate(
786
563
              OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
787
563
                AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
788
563
                const Expr *Init = VD->getInit();
789
563
                if (!isa<CXXConstructExpr>(Init) ||
790
563
                    
isTrivialInitializer(Init)220
) {
791
343
                  // Perform simple memcpy.
792
343
                  LValue Dest =
793
343
                      MakeAddrLValue(Emission.getAllocatedAddress(), Type);
794
343
                  EmitAggregateAssign(Dest, OriginalLVal, Type);
795
343
                } else {
796
220
                  EmitOMPAggregateAssign(
797
220
                      Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
798
220
                      Type,
799
220
                      [this, VDInit, Init](Address DestElement,
800
220
                                           Address SrcElement) {
801
220
                        // Clean up any temporaries needed by the
802
220
                        // initialization.
803
220
                        RunCleanupsScope InitScope(*this);
804
220
                        // Emit initialization for single element.
805
220
                        setAddrOfLocalVar(VDInit, SrcElement);
806
220
                        EmitAnyExprToMem(Init, DestElement,
807
220
                                         Init->getType().getQualifiers(),
808
220
                                         /*IsInitializer*/ false);
809
220
                        LocalDeclMap.erase(VDInit);
810
220
                      });
811
220
                }
812
563
                EmitAutoVarCleanups(Emission);
813
563
                return Emission.getAllocatedAddress();
814
563
              });
815
592
        } else {
816
592
          Address OriginalAddr = OriginalLVal.getAddress();
817
592
          IsRegistered = PrivateScope.addPrivate(
818
592
              OrigVD, [this, VDInit, OriginalAddr, VD]() {
819
592
                // Emit private VarDecl with copy init.
820
592
                // Remap temp VDInit variable to the address of the original
821
592
                // variable (for proper handling of captured global variables).
822
592
                setAddrOfLocalVar(VDInit, OriginalAddr);
823
592
                EmitDecl(*VD);
824
592
                LocalDeclMap.erase(VDInit);
825
592
                return GetAddrOfLocalVar(VD);
826
592
              });
827
592
        }
828
1.15k
        assert(IsRegistered &&
829
1.15k
               "firstprivate var already registered as private");
830
1.15k
        // Silence the warning about unused variable.
831
1.15k
        (void)IsRegistered;
832
1.15k
      }
833
1.25k
      ++IRef;
834
1.25k
      ++InitsRef;
835
1.25k
    }
836
4.19k
  }
837
11.3k
  return FirstprivateIsLastprivate && 
!EmittedAsFirstprivate.empty()4
;
838
11.3k
}
839
840
void CodeGenFunction::EmitOMPPrivateClause(
841
    const OMPExecutableDirective &D,
842
20.9k
    CodeGenFunction::OMPPrivateScope &PrivateScope) {
843
20.9k
  if (!HaveInsertPoint())
844
0
    return;
845
20.9k
  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
846
20.9k
  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
847
744
    auto IRef = C->varlist_begin();
848
2.77k
    for (const Expr *IInit : C->private_copies()) {
849
2.77k
      const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
850
2.77k
      if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
851
2.54k
        const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
852
2.54k
        bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
853
2.54k
          // Emit private VarDecl with copy init.
854
2.54k
          EmitDecl(*VD);
855
2.54k
          return GetAddrOfLocalVar(VD);
856
2.54k
        });
857
2.54k
        assert(IsRegistered && "private var already registered as private");
858
2.54k
        // Silence the warning about unused variable.
859
2.54k
        (void)IsRegistered;
860
2.54k
      }
861
2.77k
      ++IRef;
862
2.77k
    }
863
744
  }
864
20.9k
}
865
866
648
bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
867
648
  if (!HaveInsertPoint())
868
0
    return false;
869
648
  // threadprivate_var1 = master_threadprivate_var1;
870
648
  // operator=(threadprivate_var2, master_threadprivate_var2);
871
648
  // ...
872
648
  // __kmpc_barrier(&loc, global_tid);
873
648
  llvm::DenseSet<const VarDecl *> CopiedVars;
874
648
  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
875
648
  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
876
23
    auto IRef = C->varlist_begin();
877
23
    auto ISrcRef = C->source_exprs().begin();
878
23
    auto IDestRef = C->destination_exprs().begin();
879
49
    for (const Expr *AssignOp : C->assignment_ops()) {
880
49
      const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
881
49
      QualType Type = VD->getType();
882
49
      if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
883
49
        // Get the address of the master variable. If we are emitting code with
884
49
        // TLS support, the address is passed from the master as field in the
885
49
        // captured declaration.
886
49
        Address MasterAddr = Address::invalid();
887
49
        if (getLangOpts().OpenMPUseTLS &&
888
49
            
getContext().getTargetInfo().isTLSSupported()25
) {
889
25
          assert(CapturedStmtInfo->lookup(VD) &&
890
25
                 "Copyin threadprivates should have been captured!");
891
25
          DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
892
25
                          (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
893
25
          MasterAddr = EmitLValue(&DRE).getAddress();
894
25
          LocalDeclMap.erase(VD);
895
25
        } else {
896
24
          MasterAddr =
897
24
            Address(VD->isStaticLocal() ? 
CGM.getStaticLocalDeclAddress(VD)22
898
24
                                        : 
CGM.GetAddrOfGlobal(VD)2
,
899
24
                    getContext().getDeclAlign(VD));
900
24
        }
901
49
        // Get the address of the threadprivate variable.
902
49
        Address PrivateAddr = EmitLValue(*IRef).getAddress();
903
49
        if (CopiedVars.size() == 1) {
904
23
          // At first check if current thread is a master thread. If it is, no
905
23
          // need to copy data.
906
23
          CopyBegin = createBasicBlock("copyin.not.master");
907
23
          CopyEnd = createBasicBlock("copyin.not.master.end");
908
23
          Builder.CreateCondBr(
909
23
              Builder.CreateICmpNE(
910
23
                  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
911
23
                  Builder.CreatePtrToInt(PrivateAddr.getPointer(),
912
23
                                         CGM.IntPtrTy)),
913
23
              CopyBegin, CopyEnd);
914
23
          EmitBlock(CopyBegin);
915
23
        }
916
49
        const auto *SrcVD =
917
49
            cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
918
49
        const auto *DestVD =
919
49
            cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
920
49
        EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
921
49
      }
922
49
      ++IRef;
923
49
      ++ISrcRef;
924
49
      ++IDestRef;
925
49
    }
926
23
  }
927
648
  if (CopyEnd) {
928
23
    // Exit out of copying procedure for non-master thread.
929
23
    EmitBlock(CopyEnd, /*IsFinished=*/true);
930
23
    return true;
931
23
  }
932
625
  return false;
933
625
}
934
935
bool CodeGenFunction::EmitOMPLastprivateClauseInit(
936
8.59k
    const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
937
8.59k
  if (!HaveInsertPoint())
938
0
    return false;
939
8.59k
  bool HasAtLeastOneLastprivate = false;
940
8.59k
  llvm::DenseSet<const VarDecl *> SIMDLCVs;
941
8.59k
  if (isOpenMPSimdDirective(D.getDirectiveKind())) {
942
5.46k
    const auto *LoopDirective = cast<OMPLoopDirective>(&D);
943
5.71k
    for (const Expr *C : LoopDirective->counters()) {
944
5.71k
      SIMDLCVs.insert(
945
5.71k
          cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
946
5.71k
    }
947
5.46k
  }
948
8.59k
  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
949
8.59k
  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
950
429
    HasAtLeastOneLastprivate = true;
951
429
    if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
952
429
        
!getLangOpts().OpenMPSimd22
)
953
15
      break;
954
414
    auto IRef = C->varlist_begin();
955
414
    auto IDestRef = C->destination_exprs().begin();
956
1.83k
    for (const Expr *IInit : C->private_copies()) {
957
1.83k
      // Keep the address of the original variable for future update at the end
958
1.83k
      // of the loop.
959
1.83k
      const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
960
1.83k
      // Taskloops do not require additional initialization, it is done in
961
1.83k
      // runtime support library.
962
1.83k
      if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
963
1.43k
        const auto *DestVD =
964
1.43k
            cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
965
1.43k
        PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
966
1.43k
          DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
967
1.43k
                          /*RefersToEnclosingVariableOrCapture=*/
968
1.43k
                              CapturedStmtInfo->lookup(OrigVD) != nullptr,
969
1.43k
                          (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
970
1.43k
          return EmitLValue(&DRE).getAddress();
971
1.43k
        });
972
1.43k
        // Check if the variable is also a firstprivate: in this case IInit is
973
1.43k
        // not generated. Initialization of this variable will happen in codegen
974
1.43k
        // for 'firstprivate' clause.
975
1.43k
        if (IInit && 
!SIMDLCVs.count(OrigVD->getCanonicalDecl())1.41k
) {
976
1.40k
          const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
977
1.40k
          bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
978
1.40k
            // Emit private VarDecl with copy init.
979
1.40k
            EmitDecl(*VD);
980
1.40k
            return GetAddrOfLocalVar(VD);
981
1.40k
          });
982
1.40k
          assert(IsRegistered &&
983
1.40k
                 "lastprivate var already registered as private");
984
1.40k
          (void)IsRegistered;
985
1.40k
        }
986
1.43k
      }
987
1.83k
      ++IRef;
988
1.83k
      ++IDestRef;
989
1.83k
    }
990
414
  }
991
8.59k
  return HasAtLeastOneLastprivate;
992
8.59k
}
993
994
void CodeGenFunction::EmitOMPLastprivateClauseFinal(
995
    const OMPExecutableDirective &D, bool NoFinals,
996
427
    llvm::Value *IsLastIterCond) {
997
427
  if (!HaveInsertPoint())
998
0
    return;
999
427
  // Emit following code:
1000
427
  // if (<IsLastIterCond>) {
1001
427
  //   orig_var1 = private_orig_var1;
1002
427
  //   ...
1003
427
  //   orig_varn = private_orig_varn;
1004
427
  // }
1005
427
  llvm::BasicBlock *ThenBB = nullptr;
1006
427
  llvm::BasicBlock *DoneBB = nullptr;
1007
427
  if (IsLastIterCond) {
1008
328
    ThenBB = createBasicBlock(".omp.lastprivate.then");
1009
328
    DoneBB = createBasicBlock(".omp.lastprivate.done");
1010
328
    Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1011
328
    EmitBlock(ThenBB);
1012
328
  }
1013
427
  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1014
427
  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1015
427
  if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1016
419
    auto IC = LoopDirective->counters().begin();
1017
431
    for (const Expr *F : LoopDirective->finals()) {
1018
431
      const auto *D =
1019
431
          cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1020
431
      if (NoFinals)
1021
161
        AlreadyEmittedVars.insert(D);
1022
270
      else
1023
270
        LoopCountersAndUpdates[D] = F;
1024
431
      ++IC;
1025
431
    }
1026
419
  }
1027
429
  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1028
429
    auto IRef = C->varlist_begin();
1029
429
    auto ISrcRef = C->source_exprs().begin();
1030
429
    auto IDestRef = C->destination_exprs().begin();
1031
1.89k
    for (const Expr *AssignOp : C->assignment_ops()) {
1032
1.89k
      const auto *PrivateVD =
1033
1.89k
          cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1034
1.89k
      QualType Type = PrivateVD->getType();
1035
1.89k
      const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1036
1.89k
      if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1037
1.47k
        // If lastprivate variable is a loop control variable for loop-based
1038
1.47k
        // directive, update its value before copyin back to original
1039
1.47k
        // variable.
1040
1.47k
        if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1041
8
          EmitIgnoredExpr(FinalExpr);
1042
1.47k
        const auto *SrcVD =
1043
1.47k
            cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1044
1.47k
        const auto *DestVD =
1045
1.47k
            cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1046
1.47k
        // Get the address of the original variable.
1047
1.47k
        Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1048
1.47k
        // Get the address of the private variable.
1049
1.47k
        Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1050
1.47k
        if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1051
308
          PrivateAddr =
1052
308
              Address(Builder.CreateLoad(PrivateAddr),
1053
308
                      getNaturalTypeAlignment(RefTy->getPointeeType()));
1054
1.47k
        EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1055
1.47k
      }
1056
1.89k
      ++IRef;
1057
1.89k
      ++ISrcRef;
1058
1.89k
      ++IDestRef;
1059
1.89k
    }
1060
429
    if (const Expr *PostUpdate = C->getPostUpdateExpr())
1061
5
      EmitIgnoredExpr(PostUpdate);
1062
429
  }
1063
427
  if (IsLastIterCond)
1064
328
    EmitBlock(DoneBB, /*IsFinished=*/true);
1065
427
}
1066
1067
void CodeGenFunction::EmitOMPReductionClauseInit(
1068
    const OMPExecutableDirective &D,
1069
10.3k
    CodeGenFunction::OMPPrivateScope &PrivateScope) {
1070
10.3k
  if (!HaveInsertPoint())
1071
0
    return;
1072
10.3k
  SmallVector<const Expr *, 4> Shareds;
1073
10.3k
  SmallVector<const Expr *, 4> Privates;
1074
10.3k
  SmallVector<const Expr *, 4> ReductionOps;
1075
10.3k
  SmallVector<const Expr *, 4> LHSs;
1076
10.3k
  SmallVector<const Expr *, 4> RHSs;
1077
10.3k
  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1078
494
    auto IPriv = C->privates().begin();
1079
494
    auto IRed = C->reduction_ops().begin();
1080
494
    auto ILHS = C->lhs_exprs().begin();
1081
494
    auto IRHS = C->rhs_exprs().begin();
1082
511
    for (const Expr *Ref : C->varlists()) {
1083
511
      Shareds.emplace_back(Ref);
1084
511
      Privates.emplace_back(*IPriv);
1085
511
      ReductionOps.emplace_back(*IRed);
1086
511
      LHSs.emplace_back(*ILHS);
1087
511
      RHSs.emplace_back(*IRHS);
1088
511
      std::advance(IPriv, 1);
1089
511
      std::advance(IRed, 1);
1090
511
      std::advance(ILHS, 1);
1091
511
      std::advance(IRHS, 1);
1092
511
    }
1093
494
  }
1094
10.3k
  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1095
10.3k
  unsigned Count = 0;
1096
10.3k
  auto ILHS = LHSs.begin();
1097
10.3k
  auto IRHS = RHSs.begin();
1098
10.3k
  auto IPriv = Privates.begin();
1099
10.3k
  for (const Expr *IRef : Shareds) {
1100
511
    const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1101
511
    // Emit private VarDecl with reduction init.
1102
511
    RedCG.emitSharedLValue(*this, Count);
1103
511
    RedCG.emitAggregateType(*this, Count);
1104
511
    AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1105
511
    RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1106
511
                             RedCG.getSharedLValue(Count),
1107
511
                             [&Emission](CodeGenFunction &CGF) {
1108
372
                               CGF.EmitAutoVarInit(Emission);
1109
372
                               return true;
1110
372
                             });
1111
511
    EmitAutoVarCleanups(Emission);
1112
511
    Address BaseAddr = RedCG.adjustPrivateAddress(
1113
511
        *this, Count, Emission.getAllocatedAddress());
1114
511
    bool IsRegistered = PrivateScope.addPrivate(
1115
511
        RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1116
511
    assert(IsRegistered && "private var already registered as private");
1117
511
    // Silence the warning about unused variable.
1118
511
    (void)IsRegistered;
1119
511
1120
511
    const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1121
511
    const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1122
511
    QualType Type = PrivateVD->getType();
1123
511
    bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1124
511
    if (isaOMPArraySectionExpr && 
Type->isVariablyModifiedType()83
) {
1125
40
      // Store the address of the original variable associated with the LHS
1126
40
      // implicit variable.
1127
40
      PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1128
40
        return RedCG.getSharedLValue(Count).getAddress();
1129
40
      });
1130
40
      PrivateScope.addPrivate(
1131
40
          RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1132
471
    } else if ((isaOMPArraySectionExpr && 
Type->isScalarType()43
) ||
1133
471
               isa<ArraySubscriptExpr>(IRef)) {
1134
0
      // Store the address of the original variable associated with the LHS
1135
0
      // implicit variable.
1136
0
      PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1137
0
        return RedCG.getSharedLValue(Count).getAddress();
1138
0
      });
1139
0
      PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1140
0
        return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1141
0
                                            ConvertTypeForMem(RHSVD->getType()),
1142
0
                                            "rhs.begin");
1143
0
      });
1144
471
    } else {
1145
471
      QualType Type = PrivateVD->getType();
1146
471
      bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1147
471
      Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1148
471
      // Store the address of the original variable associated with the LHS
1149
471
      // implicit variable.
1150
471
      if (IsArray) {
1151
77
        OriginalAddr = Builder.CreateElementBitCast(
1152
77
            OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1153
77
      }
1154
471
      PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1155
471
      PrivateScope.addPrivate(
1156
471
          RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1157
471
            return IsArray
1158
471
                       ? Builder.CreateElementBitCast(
1159
77
                             GetAddrOfLocalVar(PrivateVD),
1160
77
                             ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1161
471
                       : 
GetAddrOfLocalVar(PrivateVD)394
;
1162
471
          });
1163
471
    }
1164
511
    ++ILHS;
1165
511
    ++IRHS;
1166
511
    ++IPriv;
1167
511
    ++Count;
1168
511
  }
1169
10.3k
}
1170
1171
void CodeGenFunction::EmitOMPReductionClauseFinal(
1172
10.3k
    const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1173
10.3k
  if (!HaveInsertPoint())
1174
2
    return;
1175
10.3k
  llvm::SmallVector<const Expr *, 8> Privates;
1176
10.3k
  llvm::SmallVector<const Expr *, 8> LHSExprs;
1177
10.3k
  llvm::SmallVector<const Expr *, 8> RHSExprs;
1178
10.3k
  llvm::SmallVector<const Expr *, 8> ReductionOps;
1179
10.3k
  bool HasAtLeastOneReduction = false;
1180
10.3k
  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1181
486
    HasAtLeastOneReduction = true;
1182
486
    Privates.append(C->privates().begin(), C->privates().end());
1183
486
    LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1184
486
    RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1185
486
    ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1186
486
  }
1187
10.3k
  if (HasAtLeastOneReduction) {
1188
405
    bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1189
405
                      
isOpenMPParallelDirective(D.getDirectiveKind())399
||
1190
405
                      
ReductionKind == OMPD_simd200
;
1191
405
    bool SimpleReduction = ReductionKind == OMPD_simd;
1192
405
    // Emit nowait reduction if nowait clause is present or directive is a
1193
405
    // parallel directive (it always has implicit barrier).
1194
405
    CGM.getOpenMPRuntime().emitReduction(
1195
405
        *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1196
405
        {WithNowait, SimpleReduction, ReductionKind});
1197
405
  }
1198
10.3k
}
1199
1200
static void emitPostUpdateForReductionClause(
1201
    CodeGenFunction &CGF, const OMPExecutableDirective &D,
1202
10.3k
    const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1203
10.3k
  if (!CGF.HaveInsertPoint())
1204
0
    return;
1205
10.3k
  llvm::BasicBlock *DoneBB = nullptr;
1206
10.3k
  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1207
494
    if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1208
4
      if (!DoneBB) {
1209
4
        if (llvm::Value *Cond = CondGen(CGF)) {
1210
0
          // If the first post-update expression is found, emit conditional
1211
0
          // block if it was requested.
1212
0
          llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1213
0
          DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1214
0
          CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1215
0
          CGF.EmitBlock(ThenBB);
1216
0
        }
1217
4
      }
1218
4
      CGF.EmitIgnoredExpr(PostUpdate);
1219
4
    }
1220
494
  }
1221
10.3k
  if (DoneBB)
1222
0
    CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1223
10.3k
}
1224
1225
namespace {
1226
/// Codegen lambda for appending distribute lower and upper bounds to outlined
1227
/// parallel function. This is necessary for combined constructs such as
1228
/// 'distribute parallel for'
1229
typedef llvm::function_ref<void(CodeGenFunction &,
1230
                                const OMPExecutableDirective &,
1231
                                llvm::SmallVectorImpl<llvm::Value *> &)>
1232
    CodeGenBoundParametersTy;
1233
} // anonymous namespace
1234
1235
static void emitCommonOMPParallelDirective(
1236
    CodeGenFunction &CGF, const OMPExecutableDirective &S,
1237
    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1238
3.78k
    const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1239
3.78k
  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1240
3.78k
  llvm::Function *OutlinedFn =
1241
3.78k
      CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1242
3.78k
          S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1243
3.78k
  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1244
136
    CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1245
136
    llvm::Value *NumThreads =
1246
136
        CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1247
136
                           /*IgnoreResultAssign=*/true);
1248
136
    CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1249
136
        CGF, NumThreads, NumThreadsClause->getBeginLoc());
1250
136
  }
1251
3.78k
  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1252
68
    CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1253
68
    CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1254
68
        CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1255
68
  }
1256
3.78k
  const Expr *IfCond = nullptr;
1257
3.78k
  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1258
555
    if (C->getNameModifier() == OMPD_unknown ||
1259
555
        
C->getNameModifier() == OMPD_parallel394
) {
1260
229
      IfCond = C->getCondition();
1261
229
      break;
1262
229
    }
1263
555
  }
1264
3.78k
1265
3.78k
  OMPParallelScope Scope(CGF, S);
1266
3.78k
  llvm::SmallVector<llvm::Value *, 16> CapturedVars;
1267
3.78k
  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1268
3.78k
  // lower and upper bounds with the pragma 'for' chunking mechanism.
1269
3.78k
  // The following lambda takes care of appending the lower and upper bound
1270
3.78k
  // parameters when necessary
1271
3.78k
  CodeGenBoundParameters(CGF, S, CapturedVars);
1272
3.78k
  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1273
3.78k
  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1274
3.78k
                                              CapturedVars, IfCond);
1275
3.78k
}
1276
1277
static void emitEmptyBoundParameters(CodeGenFunction &,
1278
                                     const OMPExecutableDirective &,
1279
1.96k
                                     llvm::SmallVectorImpl<llvm::Value *> &) {}
1280
1281
648
void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
1282
648
  // Emit parallel region as a standalone region.
1283
648
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1284
648
    Action.Enter(CGF);
1285
648
    OMPPrivateScope PrivateScope(CGF);
1286
648
    bool Copyins = CGF.EmitOMPCopyinClause(S);
1287
648
    (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1288
648
    if (Copyins) {
1289
23
      // Emit implicit barrier to synchronize threads and avoid data races on
1290
23
      // propagation master's thread values of threadprivate variables to local
1291
23
      // instances of that variables of all other implicit threads.
1292
23
      CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1293
23
          CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1294
23
          /*ForceSimpleCall=*/true);
1295
23
    }
1296
648
    CGF.EmitOMPPrivateClause(S, PrivateScope);
1297
648
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1298
648
    (void)PrivateScope.Privatize();
1299
648
    CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1300
648
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1301
648
  };
1302
648
  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1303
648
                                 emitEmptyBoundParameters);
1304
648
  emitPostUpdateForReductionClause(*this, S,
1305
648
                                   [](CodeGenFunction &) 
{ return nullptr; }4
);
1306
648
}
1307
1308
void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
1309
6.72k
                                      JumpDest LoopExit) {
1310
6.72k
  RunCleanupsScope BodyScope(*this);
1311
6.72k
  // Update counters values on current iteration.
1312
6.72k
  for (const Expr *UE : D.updates())
1313
7.02k
    EmitIgnoredExpr(UE);
1314
6.72k
  // Update the linear variables.
1315
6.72k
  // In distribute directives only loop counters may be marked as linear, no
1316
6.72k
  // need to generate the code for them.
1317
6.72k
  if (!isOpenMPDistributeDirective(D.getDirectiveKind())) {
1318
2.40k
    for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1319
254
      for (const Expr *UE : C->updates())
1320
324
        EmitIgnoredExpr(UE);
1321
254
    }
1322
2.40k
  }
1323
6.72k
1324
6.72k
  // On a continue in the body, jump to the end.
1325
6.72k
  JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1326
6.72k
  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1327
6.72k
  // Emit loop body.
1328
6.72k
  EmitStmt(D.getBody());
1329
6.72k
  // The end (updates/cleanups).
1330
6.72k
  EmitBlock(Continue.getBlock());
1331
6.72k
  BreakContinueStack.pop_back();
1332
6.72k
}
1333
1334
void CodeGenFunction::EmitOMPInnerLoop(
1335
    const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1336
    const Expr *IncExpr,
1337
    const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1338
8.59k
    const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1339
8.59k
  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1340
8.59k
1341
8.59k
  // Start the loop with a block that tests the condition.
1342
8.59k
  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1343
8.59k
  EmitBlock(CondBlock);
1344
8.59k
  const SourceRange R = S.getSourceRange();
1345
8.59k
  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1346
8.59k
                 SourceLocToDebugLoc(R.getEnd()));
1347
8.59k
1348
8.59k
  // If there are any cleanups between here and the loop-exit scope,
1349
8.59k
  // create a block to stage a loop exit along.
1350
8.59k
  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1351
8.59k
  if (RequiresCleanup)
1352
757
    ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1353
8.59k
1354
8.59k
  llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1355
8.59k
1356
8.59k
  // Emit condition.
1357
8.59k
  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1358
8.59k
  if (ExitBlock != LoopExit.getBlock()) {
1359
757
    EmitBlock(ExitBlock);
1360
757
    EmitBranchThroughCleanup(LoopExit);
1361
757
  }
1362
8.59k
1363
8.59k
  EmitBlock(LoopBody);
1364
8.59k
  incrementProfileCounter(&S);
1365
8.59k
1366
8.59k
  // Create a block for the increment.
1367
8.59k
  JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1368
8.59k
  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1369
8.59k
1370
8.59k
  BodyGen(*this);
1371
8.59k
1372
8.59k
  // Emit "IV = IV + 1" and a back-edge to the condition block.
1373
8.59k
  EmitBlock(Continue.getBlock());
1374
8.59k
  EmitIgnoredExpr(IncExpr);
1375
8.59k
  PostIncGen(*this);
1376
8.59k
  BreakContinueStack.pop_back();
1377
8.59k
  EmitBranch(CondBlock);
1378
8.59k
  LoopStack.pop();
1379
8.59k
  // Emit the fall-through block.
1380
8.59k
  EmitBlock(LoopExit.getBlock());
1381
8.59k
}
1382
1383
5.51k
bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
1384
5.51k
  if (!HaveInsertPoint())
1385
0
    return false;
1386
5.51k
  // Emit inits for the linear variables.
1387
5.51k
  bool HasLinears = false;
1388
5.51k
  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1389
400
    for (const Expr *Init : C->inits()) {
1390
400
      HasLinears = true;
1391
400
      const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1392
400
      if (const auto *Ref =
1393
400
              dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1394
400
        AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1395
400
        const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1396
400
        DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1397
400
                        CapturedStmtInfo->lookup(OrigVD) != nullptr,
1398
400
                        VD->getInit()->getType(), VK_LValue,
1399
400
                        VD->getInit()->getExprLoc());
1400
400
        EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1401
400
                                                VD->getType()),
1402
400
                       /*capturedByInit=*/false);
1403
400
        EmitAutoVarCleanups(Emission);
1404
400
      } else {
1405
0
        EmitVarDecl(*VD);
1406
0
      }
1407
400
    }
1408
330
    // Emit the linear steps for the linear clauses.
1409
330
    // If a step is not constant, it is pre-calculated before the loop.
1410
330
    if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1411
88
      if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1412
88
        EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1413
88
        // Emit calculation of the linear step.
1414
88
        EmitIgnoredExpr(CS);
1415
88
      }
1416
330
  }
1417
5.51k
  return HasLinears;
1418
5.51k
}
1419
1420
void CodeGenFunction::EmitOMPLinearClauseFinal(
1421
    const OMPLoopDirective &D,
1422
5.51k
    const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1423
5.51k
  if (!HaveInsertPoint())
1424
0
    return;
1425
5.51k
  llvm::BasicBlock *DoneBB = nullptr;
1426
5.51k
  // Emit the final values of the linear variables.
1427
5.51k
  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1428
330
    auto IC = C->varlist_begin();
1429
400
    for (const Expr *F : C->finals()) {
1430
400
      if (!DoneBB) {
1431
352
        if (llvm::Value *Cond = CondGen(*this)) {
1432
90
          // If the first post-update expression is found, emit conditional
1433
90
          // block if it was requested.
1434
90
          llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1435
90
          DoneBB = createBasicBlock(".omp.linear.pu.done");
1436
90
          Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1437
90
          EmitBlock(ThenBB);
1438
90
        }
1439
352
      }
1440
400
      const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1441
400
      DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1442
400
                      CapturedStmtInfo->lookup(OrigVD) != nullptr,
1443
400
                      (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1444
400
      Address OrigAddr = EmitLValue(&DRE).getAddress();
1445
400
      CodeGenFunction::OMPPrivateScope VarScope(*this);
1446
400
      VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1447
400
      (void)VarScope.Privatize();
1448
400
      EmitIgnoredExpr(F);
1449
400
      ++IC;
1450
400
    }
1451
330
    if (const Expr *PostUpdate = C->getPostUpdateExpr())
1452
4
      EmitIgnoredExpr(PostUpdate);
1453
330
  }
1454
5.51k
  if (DoneBB)
1455
90
    EmitBlock(DoneBB, /*IsFinished=*/true);
1456
5.51k
}
1457
1458
static void emitAlignedClause(CodeGenFunction &CGF,
1459
8.47k
                              const OMPExecutableDirective &D) {
1460
8.47k
  if (!CGF.HaveInsertPoint())
1461
0
    return;
1462
8.47k
  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1463
250
    unsigned ClauseAlignment = 0;
1464
250
    if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1465
94
      auto *AlignmentCI =
1466
94
          cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1467
94
      ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1468
94
    }
1469
298
    for (const Expr *E : Clause->varlists()) {
1470
298
      unsigned Alignment = ClauseAlignment;
1471
298
      if (Alignment == 0) {
1472
204
        // OpenMP [2.8.1, Description]
1473
204
        // If no optional parameter is specified, implementation-defined default
1474
204
        // alignments for SIMD instructions on the target platforms are assumed.
1475
204
        Alignment =
1476
204
            CGF.getContext()
1477
204
                .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
1478
204
                    E->getType()->getPointeeType()))
1479
204
                .getQuantity();
1480
204
      }
1481
298
      assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1482
298
             "alignment is not power of 2");
1483
298
      if (Alignment != 0) {
1484
298
        llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1485
298
        CGF.EmitAlignmentAssumption(
1486
298
            PtrValue, E, /*No second loc needed*/ SourceLocation(), Alignment);
1487
298
      }
1488
298
    }
1489
250
  }
1490
8.47k
}
1491
1492
void CodeGenFunction::EmitOMPPrivateLoopCounters(
1493
10.3k
    const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1494
10.3k
  if (!HaveInsertPoint())
1495
0
    return;
1496
10.3k
  auto I = S.private_counters().begin();
1497
10.8k
  for (const Expr *E : S.counters()) {
1498
10.8k
    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1499
10.8k
    const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1500
10.8k
    // Emit var without initialization.
1501
10.8k
    AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1502
10.8k
    EmitAutoVarCleanups(VarEmission);
1503
10.8k
    LocalDeclMap.erase(PrivateVD);
1504
10.8k
    (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1505
10.8k
      return VarEmission.getAllocatedAddress();
1506
10.8k
    });
1507
10.8k
    if (LocalDeclMap.count(VD) || 
CapturedStmtInfo->lookup(VD)10.4k
||
1508
10.8k
        
VD->hasGlobalStorage()10.3k
) {
1509
458
      (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1510
458
        DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
1511
458
                        LocalDeclMap.count(VD) || 
CapturedStmtInfo->lookup(VD)32
,
1512
458
                        E->getType(), VK_LValue, E->getExprLoc());
1513
458
        return EmitLValue(&DRE).getAddress();
1514
458
      });
1515
10.3k
    } else {
1516
10.3k
      (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1517
10.3k
        return VarEmission.getAllocatedAddress();
1518
10.3k
      });
1519
10.3k
    }
1520
10.8k
    ++I;
1521
10.8k
  }
1522
10.3k
  // Privatize extra loop counters used in loops for ordered(n) clauses.
1523
10.3k
  for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1524
58
    if (!C->getNumForLoops())
1525
40
      continue;
1526
18
    for (unsigned I = S.getCollapsedNumber(),
1527
18
                  E = C->getLoopNumIterations().size();
1528
24
         I < E; 
++I6
) {
1529
6
      const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1530
6
      const auto *VD = cast<VarDecl>(DRE->getDecl());
1531
6
      // Override only those variables that can be captured to avoid re-emission
1532
6
      // of the variables declared within the loops.
1533
6
      if (DRE->refersToEnclosingVariableOrCapture()) {
1534
4
        (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1535
4
          return CreateMemTemp(DRE->getType(), VD->getName());
1536
4
        });
1537
4
      }
1538
6
    }
1539
18
  }
1540
10.3k
}
1541
1542
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1543
                        const Expr *Cond, llvm::BasicBlock *TrueBlock,
1544
1.78k
                        llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1545
1.78k
  if (!CGF.HaveInsertPoint())
1546
0
    return;
1547
1.78k
  {
1548
1.78k
    CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1549
1.78k
    CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1550
1.78k
    (void)PreCondScope.Privatize();
1551
1.78k
    // Get initial values of real counters.
1552
1.92k
    for (const Expr *I : S.inits()) {
1553
1.92k
      CGF.EmitIgnoredExpr(I);
1554
1.92k
    }
1555
1.78k
  }
1556
1.78k
  // Check that loop is executed at least one time.
1557
1.78k
  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1558
1.78k
}
1559
1560
void CodeGenFunction::EmitOMPLinearClause(
1561
5.51k
    const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1562
5.51k
  if (!HaveInsertPoint())
1563
0
    return;
1564
5.51k
  llvm::DenseSet<const VarDecl *> SIMDLCVs;
1565
5.51k
  if (isOpenMPSimdDirective(D.getDirectiveKind())) {
1566
3.92k
    const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1567
4.10k
    for (const Expr *C : LoopDirective->counters()) {
1568
4.10k
      SIMDLCVs.insert(
1569
4.10k
          cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1570
4.10k
    }
1571
3.92k
  }
1572
5.51k
  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1573
330
    auto CurPrivate = C->privates().begin();
1574
400
    for (const Expr *E : C->varlists()) {
1575
400
      const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1576
400
      const auto *PrivateVD =
1577
400
          cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1578
400
      if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1579
316
        bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1580
316
          // Emit private VarDecl with copy init.
1581
316
          EmitVarDecl(*PrivateVD);
1582
316
          return GetAddrOfLocalVar(PrivateVD);
1583
316
        });
1584
316
        assert(IsRegistered && "linear var already registered as private");
1585
316
        // Silence the warning about unused variable.
1586
316
        (void)IsRegistered;
1587
316
      } else {
1588
84
        EmitVarDecl(*PrivateVD);
1589
84
      }
1590
400
      ++CurPrivate;
1591
400
    }
1592
330
  }
1593
5.51k
}
1594
1595
static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
1596
                                     const OMPExecutableDirective &D,
1597
5.46k
                                     bool IsMonotonic) {
1598
5.46k
  if (!CGF.HaveInsertPoint())
1599
0
    return;
1600
5.46k
  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1601
172
    RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1602
172
                                 /*ignoreResult=*/true);
1603
172
    auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1604
172
    CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1605
172
    // In presence of finite 'safelen', it may be unsafe to mark all
1606
172
    // the memory instructions parallel, because loop-carried
1607
172
    // dependences of 'safelen' iterations are possible.
1608
172
    if (!IsMonotonic)
1609
92
      CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
1610
5.29k
  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1611
112
    RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1612
112
                                 /*ignoreResult=*/true);
1613
112
    auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1614
112
    CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1615
112
    // In presence of finite 'safelen', it may be unsafe to mark all
1616
112
    // the memory instructions parallel, because loop-carried
1617
112
    // dependences of 'safelen' iterations are possible.
1618
112
    CGF.LoopStack.setParallel(/*Enable=*/false);
1619
112
  }
1620
5.46k
}
1621
1622
void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
1623
5.46k
                                      bool IsMonotonic) {
1624
5.46k
  // Walk clauses and process safelen/lastprivate.
1625
5.46k
  LoopStack.setParallel(!IsMonotonic);
1626
5.46k
  LoopStack.setVectorizeEnable();
1627
5.46k
  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1628
5.46k
}
1629
1630
void CodeGenFunction::EmitOMPSimdFinal(
1631
    const OMPLoopDirective &D,
1632
5.43k
    const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1633
5.43k
  if (!HaveInsertPoint())
1634
0
    return;
1635
5.43k
  llvm::BasicBlock *DoneBB = nullptr;
1636
5.43k
  auto IC = D.counters().begin();
1637
5.43k
  auto IPC = D.private_counters().begin();
1638
5.67k
  for (const Expr *F : D.finals()) {
1639
5.67k
    const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1640
5.67k
    const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1641
5.67k
    const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1642
5.67k
    if (LocalDeclMap.count(OrigVD) || 
CapturedStmtInfo->lookup(OrigVD)0
||
1643
5.67k
        
OrigVD->hasGlobalStorage()0
||
CED0
) {
1644
5.67k
      if (!DoneBB) {
1645
5.56k
        if (llvm::Value *Cond = CondGen(*this)) {
1646
2.92k
          // If the first post-update expression is found, emit conditional
1647
2.92k
          // block if it was requested.
1648
2.92k
          llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1649
2.92k
          DoneBB = createBasicBlock(".omp.final.done");
1650
2.92k
          Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1651
2.92k
          EmitBlock(ThenBB);
1652
2.92k
        }
1653
5.56k
      }
1654
5.67k
      Address OrigAddr = Address::invalid();
1655
5.67k
      if (CED) {
1656
18
        OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1657
5.65k
      } else {
1658
5.65k
        DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
1659
5.65k
                        /*RefersToEnclosingVariableOrCapture=*/false,
1660
5.65k
                        (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1661
5.65k
        OrigAddr = EmitLValue(&DRE).getAddress();
1662
5.65k
      }
1663
5.67k
      OMPPrivateScope VarScope(*this);
1664
5.67k
      VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1665
5.67k
      (void)VarScope.Privatize();
1666
5.67k
      EmitIgnoredExpr(F);
1667
5.67k
    }
1668
5.67k
    ++IC;
1669
5.67k
    ++IPC;
1670
5.67k
  }
1671
5.43k
  if (DoneBB)
1672
2.92k
    EmitBlock(DoneBB, /*IsFinished=*/true);
1673
5.43k
}
1674
1675
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
1676
                                         const OMPLoopDirective &S,
1677
1.93k
                                         CodeGenFunction::JumpDest LoopExit) {
1678
1.93k
  CGF.EmitOMPLoopBody(S, LoopExit);
1679
1.93k
  CGF.EmitStopPoint(&S);
1680
1.93k
}
1681
1682
/// Emit a helper variable and return corresponding lvalue.
1683
static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
1684
27.3k
                               const DeclRefExpr *Helper) {
1685
27.3k
  auto VDecl = cast<VarDecl>(Helper->getDecl());
1686
27.3k
  CGF.EmitVarDecl(*VDecl);
1687
27.3k
  return CGF.EmitLValue(Helper);
1688
27.3k
}
1689
1690
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
1691
2.55k
                              PrePostActionTy &Action) {
1692
2.55k
  Action.Enter(CGF);
1693
2.55k
  assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
1694
2.55k
         "Expected simd directive");
1695
2.55k
  OMPLoopScope PreInitScope(CGF, S);
1696
2.55k
  // if (PreCond) {
1697
2.55k
  //   for (IV in 0..LastIteration) BODY;
1698
2.55k
  //   <Final counter/linear vars updates>;
1699
2.55k
  // }
1700
2.55k
  //
1701
2.55k
  if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
1702
2.55k
      
isOpenMPWorksharingDirective(S.getDirectiveKind())1.20k
||
1703
2.55k
      
isOpenMPTaskLoopDirective(S.getDirectiveKind())822
) {
1704
1.76k
    (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1705
1.76k
    (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1706
1.76k
  }
1707
2.55k
1708
2.55k
  // Emit: if (PreCond) - begin.
1709
2.55k
  // If the condition constant folds and can be elided, avoid emitting the
1710
2.55k
  // whole loop.
1711
2.55k
  bool CondConstant;
1712
2.55k
  llvm::BasicBlock *ContBlock = nullptr;
1713
2.55k
  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1714
2.01k
    if (!CondConstant)
1715
42
      return;
1716
537
  } else {
1717
537
    llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1718
537
    ContBlock = CGF.createBasicBlock("simd.if.end");
1719
537
    emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1720
537
                CGF.getProfileCount(&S));
1721
537
    CGF.EmitBlock(ThenBlock);
1722
537
    CGF.incrementProfileCounter(&S);
1723
537
  }
1724
2.55k
1725
2.55k
  // Emit the loop iteration variable.
1726
2.55k
  const Expr *IVExpr = S.getIterationVariable();
1727
2.50k
  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1728
2.50k
  CGF.EmitVarDecl(*IVDecl);
1729
2.50k
  CGF.EmitIgnoredExpr(S.getInit());
1730
2.50k
1731
2.50k
  // Emit the iterations count variable.
1732
2.50k
  // If it is not a variable, Sema decided to calculate iterations count on
1733
2.50k
  // each iteration (e.g., it is foldable into a constant).
1734
2.50k
  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1735
0
    CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1736
0
    // Emit calculation of the iterations count.
1737
0
    CGF.EmitIgnoredExpr(S.getCalcLastIteration());
1738
0
  }
1739
2.50k
1740
2.50k
  CGF.EmitOMPSimdInit(S);
1741
2.50k
1742
2.50k
  emitAlignedClause(CGF, S);
1743
2.50k
  (void)CGF.EmitOMPLinearClauseInit(S);
1744
2.50k
  {
1745
2.50k
    CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1746
2.50k
    CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1747
2.50k
    CGF.EmitOMPLinearClause(S, LoopScope);
1748
2.50k
    CGF.EmitOMPPrivateClause(S, LoopScope);
1749
2.50k
    CGF.EmitOMPReductionClauseInit(S, LoopScope);
1750
2.50k
    bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1751
2.50k
    (void)LoopScope.Privatize();
1752
2.50k
    if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
1753
1.57k
      CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
1754
2.50k
    CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1755
2.50k
                         S.getInc(),
1756
2.50k
                         [&S](CodeGenFunction &CGF) {
1757
2.50k
                           CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest());
1758
2.50k
                           CGF.EmitStopPoint(&S);
1759
2.50k
                         },
1760
2.50k
                         [](CodeGenFunction &) {});
1761
2.63k
    CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1762
2.50k
    // Emit final copy of the lastprivate variables at the end of loops.
1763
2.50k
    if (HasLastprivateClause)
1764
99
      CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1765
2.50k
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1766
2.50k
    emitPostUpdateForReductionClause(CGF, S,
1767
2.50k
                                     [](CodeGenFunction &) 
{ return nullptr; }0
);
1768
2.50k
  }
1769
2.50k
  CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) 
{ return nullptr; }262
);
1770
2.50k
  // Emit: if (PreCond) - end.
1771
2.50k
  if (ContBlock) {
1772
537
    CGF.EmitBranch(ContBlock);
1773
537
    CGF.EmitBlock(ContBlock, true);
1774
537
  }
1775
2.50k
}
1776
1777
118
void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
1778
118
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1779
118
    emitOMPSimdRegion(CGF, S, Action);
1780
118
  };
1781
118
  OMPLexicalScope Scope(*this, S, OMPD_unknown);
1782
118
  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1783
118
}
1784
1785
void CodeGenFunction::EmitOMPOuterLoop(
1786
    bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1787
    CodeGenFunction::OMPPrivateScope &LoopScope,
1788
    const CodeGenFunction::OMPLoopArguments &LoopArgs,
1789
    const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1790
905
    const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1791
905
  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1792
905
1793
905
  const Expr *IVExpr = S.getIterationVariable();
1794
905
  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1795
905
  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1796
905
1797
905
  JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1798
905
1799
905
  // Start the loop with a block that tests the condition.
1800
905
  llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1801
905
  EmitBlock(CondBlock);
1802
905
  const SourceRange R = S.getSourceRange();
1803
905
  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1804
905
                 SourceLocToDebugLoc(R.getEnd()));
1805
905
1806
905
  llvm::Value *BoolCondVal = nullptr;
1807
905
  if (!DynamicOrOrdered) {
1808
303
    // UB = min(UB, GlobalUB) or
1809
303
    // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1810
303
    // 'distribute parallel for')
1811
303
    EmitIgnoredExpr(LoopArgs.EUB);
1812
303
    // IV = LB
1813
303
    EmitIgnoredExpr(LoopArgs.Init);
1814
303
    // IV < UB
1815
303
    BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1816
602
  } else {
1817
602
    BoolCondVal =
1818
602
        RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
1819
602
                       LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1820
602
  }
1821
905
1822
905
  // If there are any cleanups between here and the loop-exit scope,
1823
905
  // create a block to stage a loop exit along.
1824
905
  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1825
905
  if (LoopScope.requiresCleanups())
1826
17
    ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1827
905
1828
905
  llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1829
905
  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1830
905
  if (ExitBlock != LoopExit.getBlock()) {
1831
17
    EmitBlock(ExitBlock);
1832
17
    EmitBranchThroughCleanup(LoopExit);
1833
17
  }
1834
905
  EmitBlock(LoopBody);
1835
905
1836
905
  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1837
905
  // LB for loop condition and emitted it above).
1838
905
  if (DynamicOrOrdered)
1839
602
    EmitIgnoredExpr(LoopArgs.Init);
1840
905
1841
905
  // Create a block for the increment.
1842
905
  JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1843
905
  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1844
905
1845
905
  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1846
905
  // with dynamic/guided scheduling and without ordered clause.
1847
905
  if (!isOpenMPSimdDirective(S.getDirectiveKind()))
1848
501
    LoopStack.setParallel(!IsMonotonic);
1849
404
  else
1850
404
    EmitOMPSimdInit(S, IsMonotonic);
1851
905
1852
905
  SourceLocation Loc = S.getBeginLoc();
1853
905
1854
905
  // when 'distribute' is not combined with a 'for':
1855
905
  // while (idx <= UB) { BODY; ++idx; }
1856
905
  // when 'distribute' is combined with a 'for'
1857
905
  // (e.g. 'distribute parallel for')
1858
905
  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1859
905
  EmitOMPInnerLoop(
1860
905
      S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1861
905
      [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1862
905
        CodeGenLoop(CGF, S, LoopExit);
1863
905
      },
1864
905
      [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1865
905
        CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1866
905
      });
1867
905
1868
905
  EmitBlock(Continue.getBlock());
1869
905
  BreakContinueStack.pop_back();
1870
905
  if (!DynamicOrOrdered) {
1871
303
    // Emit "LB = LB + Stride", "UB = UB + Stride".
1872
303
    EmitIgnoredExpr(LoopArgs.NextLB);
1873
303
    EmitIgnoredExpr(LoopArgs.NextUB);
1874
303
  }
1875
905
1876
905
  EmitBranch(CondBlock);
1877
905
  LoopStack.pop();
1878
905
  // Emit the fall-through block.
1879
905
  EmitBlock(LoopExit.getBlock());
1880
905
1881
905
  // Tell the runtime we are done.
1882
905
  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1883
905
    if (!DynamicOrOrdered)
1884
303
      CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
1885
303
                                                     S.getDirectiveKind());
1886
905
  };
1887
905
  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1888
905
}
1889
1890
void CodeGenFunction::EmitOMPForOuterLoop(
1891
    const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1892
    const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1893
    const OMPLoopArguments &LoopArgs,
1894
793
    const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1895
793
  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1896
793
1897
793
  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1898
793
  const bool DynamicOrOrdered =
1899
793
      Ordered || 
RT.isDynamic(ScheduleKind.Schedule)766
;
1900
793
1901
793
  assert((Ordered ||
1902
793
          !RT.isStaticNonchunked(ScheduleKind.Schedule,
1903
793
                                 LoopArgs.Chunk != nullptr)) &&
1904
793
         "static non-chunked schedule does not need outer loop");
1905
793
1906
793
  // Emit outer loop.
1907
793
  //
1908
793
  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1909
793
  // When schedule(dynamic,chunk_size) is specified, the iterations are
1910
793
  // distributed to threads in the team in chunks as the threads request them.
1911
793
  // Each thread executes a chunk of iterations, then requests another chunk,
1912
793
  // until no chunks remain to be distributed. Each chunk contains chunk_size
1913
793
  // iterations, except for the last chunk to be distributed, which may have
1914
793
  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1915
793
  //
1916
793
  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1917
793
  // to threads in the team in chunks as the executing threads request them.
1918
793
  // Each thread executes a chunk of iterations, then requests another chunk,
1919
793
  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1920
793
  // each chunk is proportional to the number of unassigned iterations divided
1921
793
  // by the number of threads in the team, decreasing to 1. For a chunk_size
1922
793
  // with value k (greater than 1), the size of each chunk is determined in the
1923
793
  // same way, with the restriction that the chunks do not contain fewer than k
1924
793
  // iterations (except for the last chunk to be assigned, which may have fewer
1925
793
  // than k iterations).
1926
793
  //
1927
793
  // When schedule(auto) is specified, the decision regarding scheduling is
1928
793
  // delegated to the compiler and/or runtime system. The programmer gives the
1929
793
  // implementation the freedom to choose any possible mapping of iterations to
1930
793
  // threads in the team.
1931
793
  //
1932
793
  // When schedule(runtime) is specified, the decision regarding scheduling is
1933
793
  // deferred until run time, and the schedule and chunk size are taken from the
1934
793
  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1935
793
  // implementation defined
1936
793
  //
1937
793
  // while(__kmpc_dispatch_next(&LB, &UB)) {
1938
793
  //   idx = LB;
1939
793
  //   while (idx <= UB) { BODY; ++idx;
1940
793
  //   __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1941
793
  //   } // inner loop
1942
793
  // }
1943
793
  //
1944
793
  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1945
793
  // When schedule(static, chunk_size) is specified, iterations are divided into
1946
793
  // chunks of size chunk_size, and the chunks are assigned to the threads in
1947
793
  // the team in a round-robin fashion in the order of the thread number.
1948
793
  //
1949
793
  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1950
793
  //   while (idx <= UB) { BODY; ++idx; } // inner loop
1951
793
  //   LB = LB + ST;
1952
793
  //   UB = UB + ST;
1953
793
  // }
1954
793
  //
1955
793
1956
793
  const Expr *IVExpr = S.getIterationVariable();
1957
793
  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1958
793
  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1959
793
1960
793
  if (DynamicOrOrdered) {
1961
602
    const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1962
602
        CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1963
602
    llvm::Value *LBVal = DispatchBounds.first;
1964
602
    llvm::Value *UBVal = DispatchBounds.second;
1965
602
    CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1966
602
                                                             LoopArgs.Chunk};
1967
602
    RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
1968
602
                           IVSigned, Ordered, DipatchRTInputValues);
1969
602
  } else {
1970
191
    CGOpenMPRuntime::StaticRTInput StaticInit(
1971
191
        IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1972
191
        LoopArgs.ST, LoopArgs.Chunk);
1973
191
    RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
1974
191
                         ScheduleKind, StaticInit);
1975
191
  }
1976
793
1977
793
  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1978
793
                                    const unsigned IVSize,
1979
793
                                    const bool IVSigned) {
1980
793
    if (Ordered) {
1981
27
      CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1982
27
                                                            IVSigned);
1983
27
    }
1984
793
  };
1985
793
1986
793
  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1987
793
                                 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1988
793
  OuterLoopArgs.IncExpr = S.getInc();
1989
793
  OuterLoopArgs.Init = S.getInit();
1990
793
  OuterLoopArgs.Cond = S.getCond();
1991
793
  OuterLoopArgs.NextLB = S.getNextLowerBound();
1992
793
  OuterLoopArgs.NextUB = S.getNextUpperBound();
1993
793
  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1994
793
                   emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1995
793
}
1996
1997
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
1998
112
                             const unsigned IVSize, const bool IVSigned) {}
1999
2000
void CodeGenFunction::EmitOMPDistributeOuterLoop(
2001
    OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
2002
    OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
2003
112
    const CodeGenLoopTy &CodeGenLoopContent) {
2004
112
2005
112
  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2006
112
2007
112
  // Emit outer loop.
2008
112
  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
2009
112
  // dynamic
2010
112
  //
2011
112
2012
112
  const Expr *IVExpr = S.getIterationVariable();
2013
112
  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2014
112
  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2015
112
2016
112
  CGOpenMPRuntime::StaticRTInput StaticInit(
2017
112
      IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2018
112
      LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2019
112
  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
2020
112
2021
112
  // for combined 'distribute' and 'for' the increment expression of distribute
2022
112
  // is stored in DistInc. For 'distribute' alone, it is in Inc.
2023
112
  Expr *IncExpr;
2024
112
  if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
2025
0
    IncExpr = S.getDistInc();
2026
112
  else
2027
112
    IncExpr = S.getInc();
2028
112
2029
112
  // this routine is shared by 'omp distribute parallel for' and
2030
112
  // 'omp distribute': select the right EUB expression depending on the
2031
112
  // directive
2032
112
  OMPLoopArguments OuterLoopArgs;
2033
112
  OuterLoopArgs.LB = LoopArgs.LB;
2034
112
  OuterLoopArgs.UB = LoopArgs.UB;
2035
112
  OuterLoopArgs.ST = LoopArgs.ST;
2036
112
  OuterLoopArgs.IL = LoopArgs.IL;
2037
112
  OuterLoopArgs.Chunk = LoopArgs.Chunk;
2038
112
  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2039
112
                          ? 
S.getCombinedEnsureUpperBound()0
2040
112
                          : S.getEnsureUpperBound();
2041
112
  OuterLoopArgs.IncExpr = IncExpr;
2042
112
  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2043
112
                           ? 
S.getCombinedInit()0
2044
112
                           : S.getInit();
2045
112
  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2046
112
                           ? 
S.getCombinedCond()0
2047
112
                           : S.getCond();
2048
112
  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2049
112
                             ? 
S.getCombinedNextLowerBound()0
2050
112
                             : S.getNextLowerBound();
2051
112
  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2052
112
                             ? 
S.getCombinedNextUpperBound()0
2053
112
                             : S.getNextUpperBound();
2054
112
2055
112
  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2056
112
                   LoopScope, OuterLoopArgs, CodeGenLoopContent,
2057
112
                   emitEmptyOrdered);
2058
112
}
2059
2060
static std::pair<LValue, LValue>
2061
emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
2062
1.81k
                                     const OMPExecutableDirective &S) {
2063
1.81k
  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2064
1.81k
  LValue LB =
2065
1.81k
      EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2066
1.81k
  LValue UB =
2067
1.81k
      EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2068
1.81k
2069
1.81k
  // When composing 'distribute' with 'for' (e.g. as in 'distribute
2070
1.81k
  // parallel for') we need to use the 'distribute'
2071
1.81k
  // chunk lower and upper bounds rather than the whole loop iteration
2072
1.81k
  // space. These are parameters to the outlined function for 'parallel'
2073
1.81k
  // and we copy the bounds of the previous schedule into the
2074
1.81k
  // the current ones.
2075
1.81k
  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2076
1.81k
  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2077
1.81k
  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2078
1.81k
      PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2079
1.81k
  PrevLBVal = CGF.EmitScalarConversion(
2080
1.81k
      PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2081
1.81k
      LS.getIterationVariable()->getType(),
2082
1.81k
      LS.getPrevLowerBoundVariable()->getExprLoc());
2083
1.81k
  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2084
1.81k
      PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2085
1.81k
  PrevUBVal = CGF.EmitScalarConversion(
2086
1.81k
      PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2087
1.81k
      LS.getIterationVariable()->getType(),
2088
1.81k
      LS.getPrevUpperBoundVariable()->getExprLoc());
2089
1.81k
2090
1.81k
  CGF.EmitStoreOfScalar(PrevLBVal, LB);
2091
1.81k
  CGF.EmitStoreOfScalar(PrevUBVal, UB);
2092
1.81k
2093
1.81k
  return {LB, UB};
2094
1.81k
}
2095
2096
/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2097
/// we need to use the LB and UB expressions generated by the worksharing
2098
/// code generation support, whereas in non combined situations we would
2099
/// just emit 0 and the LastIteration expression
2100
/// This function is necessary due to the difference of the LB and UB
2101
/// types for the RT emission routines for 'for_static_init' and
2102
/// 'for_dispatch_init'
2103
static std::pair<llvm::Value *, llvm::Value *>
2104
emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
2105
                                        const OMPExecutableDirective &S,
2106
344
                                        Address LB, Address UB) {
2107
344
  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2108
344
  const Expr *IVExpr = LS.getIterationVariable();
2109
344
  // when implementing a dynamic schedule for a 'for' combined with a
2110
344
  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2111
344
  // is not normalized as each team only executes its own assigned
2112
344
  // distribute chunk
2113
344
  QualType IteratorTy = IVExpr->getType();
2114
344
  llvm::Value *LBVal =
2115
344
      CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2116
344
  llvm::Value *UBVal =
2117
344
      CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2118
344
  return {LBVal, UBVal};
2119
344
}
2120
2121
static void emitDistributeParallelForDistributeInnerBoundParams(
2122
    CodeGenFunction &CGF, const OMPExecutableDirective &S,
2123
1.81k
    llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2124
1.81k
  const auto &Dir = cast<OMPLoopDirective>(S);
2125
1.81k
  LValue LB =
2126
1.81k
      CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2127
1.81k
  llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2128
1.81k
      CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2129
1.81k
  CapturedVars.push_back(LBCast);
2130
1.81k
  LValue UB =
2131
1.81k
      CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2132
1.81k
2133
1.81k
  llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2134
1.81k
      CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2135
1.81k
  CapturedVars.push_back(UBCast);
2136
1.81k
}
2137
2138
static void
2139
emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
2140
                                 const OMPLoopDirective &S,
2141
1.81k
                                 CodeGenFunction::JumpDest LoopExit) {
2142
1.81k
  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2143
1.81k
                                         PrePostActionTy &Action) {
2144
1.81k
    Action.Enter(CGF);
2145
1.81k
    bool HasCancel = false;
2146
1.81k
    if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
2147
892
      if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2148
218
        HasCancel = D->hasCancel();
2149
674
      else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2150
307
        HasCancel = D->hasCancel();
2151
367
      else if (const auto *D =
2152
367
                   dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2153
367
        HasCancel = D->hasCancel();
2154
892
    }
2155
1.81k
    CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
2156
1.81k
                                                     HasCancel);
2157
1.81k
    CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
2158
1.81k
                               emitDistributeParallelForInnerBounds,
2159
1.81k
                               emitDistributeParallelForDispatchBounds);
2160
1.81k
  };
2161
1.81k
2162
1.81k
  emitCommonOMPParallelDirective(
2163
1.81k
      CGF, S,
2164
1.81k
      isOpenMPSimdDirective(S.getDirectiveKind()) ? 
OMPD_for_simd927
:
OMPD_for892
,
2165
1.81k
      CGInlinedWorksharingLoop,
2166
1.81k
      emitDistributeParallelForDistributeInnerBoundParams);
2167
1.81k
}
2168
2169
void CodeGenFunction::EmitOMPDistributeParallelForDirective(
2170
307
    const OMPDistributeParallelForDirective &S) {
2171
307
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2172
307
    CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2173
307
                              S.getDistInc());
2174
307
  };
2175
307
  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2176
307
  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2177
307
}
2178
2179
void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
2180
224
    const OMPDistributeParallelForSimdDirective &S) {
2181
224
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2182
224
    CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2183
224
                              S.getDistInc());
2184
224
  };
2185
224
  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2186
224
  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2187
224
}
2188
2189
void CodeGenFunction::EmitOMPDistributeSimdDirective(
2190
100
    const OMPDistributeSimdDirective &S) {
2191
100
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2192
100
    CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
2193
100
  };
2194
100
  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2195
100
  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2196
100
}
2197
2198
void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
2199
104
    CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2200
104
  // Emit SPMD target parallel for region as a standalone region.
2201
104
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2202
104
    emitOMPSimdRegion(CGF, S, Action);
2203
104
  };
2204
104
  llvm::Function *Fn;
2205
104
  llvm::Constant *Addr;
2206
104
  // Emit target region as a standalone region.
2207
104
  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2208
104
      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2209
104
  assert(Fn && Addr && "Target device function emission failed.");
2210
104
}
2211
2212
void CodeGenFunction::EmitOMPTargetSimdDirective(
2213
177
    const OMPTargetSimdDirective &S) {
2214
177
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2215
177
    emitOMPSimdRegion(CGF, S, Action);
2216
177
  };
2217
177
  emitCommonOMPTargetDirective(*this, S, CodeGen);
2218
177
}
2219
2220
namespace {
2221
  struct ScheduleKindModifiersTy {
2222
    OpenMPScheduleClauseKind Kind;
2223
    OpenMPScheduleClauseModifier M1;
2224
    OpenMPScheduleClauseModifier M2;
2225
    ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2226
                            OpenMPScheduleClauseModifier M1,
2227
                            OpenMPScheduleClauseModifier M2)
2228
0
        : Kind(Kind), M1(M1), M2(M2) {}
2229
  };
2230
} // namespace
2231
2232
bool CodeGenFunction::EmitOMPWorksharingLoop(
2233
    const OMPLoopDirective &S, Expr *EUB,
2234
    const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2235
3.02k
    const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2236
3.02k
  // Emit the loop iteration variable.
2237
3.02k
  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2238
3.02k
  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2239
3.02k
  EmitVarDecl(*IVDecl);
2240
3.02k
2241
3.02k
  // Emit the iterations count variable.
2242
3.02k
  // If it is not a variable, Sema decided to calculate iterations count on each
2243
3.02k
  // iteration (e.g., it is foldable into a constant).
2244
3.02k
  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2245
0
    EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2246
0
    // Emit calculation of the iterations count.
2247
0
    EmitIgnoredExpr(S.getCalcLastIteration());
2248
0
  }
2249
3.02k
2250
3.02k
  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2251
3.02k
2252
3.02k
  bool HasLastprivateClause;
2253
3.02k
  // Check pre-condition.
2254
3.02k
  {
2255
3.02k
    OMPLoopScope PreInitScope(*this, S);
2256
3.02k
    // Skip the entire loop if we don't meet the precondition.
2257
3.02k
    // If the condition constant folds and can be elided, avoid emitting the
2258
3.02k
    // whole loop.
2259
3.02k
    bool CondConstant;
2260
3.02k
    llvm::BasicBlock *ContBlock = nullptr;
2261
3.02k
    if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2262
2.43k
      if (!CondConstant)
2263
26
        return false;
2264
589
    } else {
2265
589
      llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2266
589
      ContBlock = createBasicBlock("omp.precond.end");
2267
589
      emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2268
589
                  getProfileCount(&S));
2269
589
      EmitBlock(ThenBlock);
2270
589
      incrementProfileCounter(&S);
2271
589
    }
2272
3.02k
2273
3.02k
    RunCleanupsScope DoacrossCleanupScope(*this);
2274
3.00k
    bool Ordered = false;
2275
3.00k
    if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2276
39
      if (OrderedClause->getNumForLoops())
2277
12
        RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2278
27
      else
2279
27
        Ordered = true;
2280
39
    }
2281
3.00k
2282
3.00k
    llvm::DenseSet<const Expr *> EmittedFinals;
2283
3.00k
    emitAlignedClause(*this, S);
2284
3.00k
    bool HasLinears = EmitOMPLinearClauseInit(S);
2285
3.00k
    // Emit helper vars inits.
2286
3.00k
2287
3.00k
    std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2288
3.00k
    LValue LB = Bounds.first;
2289
3.00k
    LValue UB = Bounds.second;
2290
3.00k
    LValue ST =
2291
3.00k
        EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2292
3.00k
    LValue IL =
2293
3.00k
        EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2294
3.00k
2295
3.00k
    // Emit 'then' code.
2296
3.00k
    {
2297
3.00k
      OMPPrivateScope LoopScope(*this);
2298
3.00k
      if (EmitOMPFirstprivateClause(S, LoopScope) || 
HasLinears2.99k
) {
2299
94
        // Emit implicit barrier to synchronize threads and avoid data races on
2300
94
        // initialization of firstprivate variables and post-update of
2301
94
        // lastprivate variables.
2302
94
        CGM.getOpenMPRuntime().emitBarrierCall(
2303
94
            *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2304
94
            /*ForceSimpleCall=*/true);
2305
94
      }
2306
3.00k
      EmitOMPPrivateClause(S, LoopScope);
2307
3.00k
      HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2308
3.00k
      EmitOMPReductionClauseInit(S, LoopScope);
2309
3.00k
      EmitOMPPrivateLoopCounters(S, LoopScope);
2310
3.00k
      EmitOMPLinearClause(S, LoopScope);
2311
3.00k
      (void)LoopScope.Privatize();
2312
3.00k
      if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2313
1.44k
        CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
2314
3.00k
2315
3.00k
      // Detect the loop schedule kind and chunk.
2316
3.00k
      const Expr *ChunkExpr = nullptr;
2317
3.00k
      OpenMPScheduleTy ScheduleKind;
2318
3.00k
      if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2319
958
        ScheduleKind.Schedule = C->getScheduleKind();
2320
958
        ScheduleKind.M1 = C->getFirstScheduleModifier();
2321
958
        ScheduleKind.M2 = C->getSecondScheduleModifier();
2322
958
        ChunkExpr = C->getChunkSize();
2323
2.04k
      } else {
2324
2.04k
        // Default behaviour for schedule clause.
2325
2.04k
        CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
2326
2.04k
            *this, S, ScheduleKind.Schedule, ChunkExpr);
2327
2.04k
      }
2328
3.00k
      bool HasChunkSizeOne = false;
2329
3.00k
      llvm::Value *Chunk = nullptr;
2330
3.00k
      if (ChunkExpr) {
2331
390
        Chunk = EmitScalarExpr(ChunkExpr);
2332
390
        Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
2333
390
                                     S.getIterationVariable()->getType(),
2334
390
                                     S.getBeginLoc());
2335
390
        Expr::EvalResult Result;
2336
390
        if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
2337
262
          llvm::APSInt EvaluatedChunk = Result.Val.getInt();
2338
262
          HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
2339
262
        }
2340
390
      }
2341
3.00k
      const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2342
3.00k
      const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2343
3.00k
      // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2344
3.00k
      // If the static schedule kind is specified or if the ordered clause is
2345
3.00k
      // specified, and if no monotonic modifier is specified, the effect will
2346
3.00k
      // be as if the monotonic modifier was specified.
2347
3.00k
      bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
2348
3.00k
          /* Chunked */ Chunk != nullptr) && 
HasChunkSizeOne310
&&
2349
3.00k
          
isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())188
;
2350
3.00k
      if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
2351
3.00k
                                 /* Chunked */ Chunk != nullptr) ||
2352
3.00k
           
StaticChunkedOne895
) &&
2353
3.00k
          
!Ordered2.21k
) {
2354
2.20k
        if (isOpenMPSimdDirective(S.getDirectiveKind()))
2355
1.06k
          EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2356
2.20k
        // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2357
2.20k
        // When no chunk_size is specified, the iteration space is divided into
2358
2.20k
        // chunks that are approximately equal in size, and at most one chunk is
2359
2.20k
        // distributed to each thread. Note that the size of the chunks is
2360
2.20k
        // unspecified in this case.
2361
2.20k
        CGOpenMPRuntime::StaticRTInput StaticInit(
2362
2.20k
            IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2363
2.20k
            UB.getAddress(), ST.getAddress(),
2364
2.20k
            StaticChunkedOne ? 
Chunk112
:
nullptr2.09k
);
2365
2.20k
        RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2366
2.20k
                             ScheduleKind, StaticInit);
2367
2.20k
        JumpDest LoopExit =
2368
2.20k
            getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2369
2.20k
        // UB = min(UB, GlobalUB);
2370
2.20k
        if (!StaticChunkedOne)
2371
2.09k
          EmitIgnoredExpr(S.getEnsureUpperBound());
2372
2.20k
        // IV = LB;
2373
2.20k
        EmitIgnoredExpr(S.getInit());
2374
2.20k
        // For unchunked static schedule generate:
2375
2.20k
        //
2376
2.20k
        // while (idx <= UB) {
2377
2.20k
        //   BODY;
2378
2.20k
        //   ++idx;
2379
2.20k
        // }
2380
2.20k
        //
2381
2.20k
        // For static schedule with chunk one:
2382
2.20k
        //
2383
2.20k
        // while (IV <= PrevUB) {
2384
2.20k
        //   BODY;
2385
2.20k
        //   IV += ST;
2386
2.20k
        // }
2387
2.20k
        EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
2388
2.20k
            StaticChunkedOne ? 
S.getCombinedParForInDistCond()112
:
S.getCond()2.09k
,
2389
2.20k
            StaticChunkedOne ? 
S.getDistInc()112
:
S.getInc()2.09k
,
2390
2.20k
            [&S, LoopExit](CodeGenFunction &CGF) {
2391
2.20k
             CGF.EmitOMPLoopBody(S, LoopExit);
2392
2.20k
             CGF.EmitStopPoint(&S);
2393
2.20k
            },
2394
2.20k
            [](CodeGenFunction &) {});
2395
2.20k
        EmitBlock(LoopExit.getBlock());
2396
2.20k
        // Tell the runtime we are done.
2397
2.24k
        auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2398
2.24k
          CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2399
2.24k
                                                         S.getDirectiveKind());
2400
2.24k
        };
2401
2.20k
        OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2402
2.20k
      } else {
2403
793
        const bool IsMonotonic =
2404
793
            Ordered || 
ScheduleKind.Schedule == OMPC_SCHEDULE_static766
||
2405
793
            
ScheduleKind.Schedule == OMPC_SCHEDULE_unknown575
||
2406
793
            
ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic575
||
2407
793
            
ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic567
;
2408
793
        // Emit the outer loop, which requests its work chunk [LB..UB] from
2409
793
        // runtime and runs the inner loop to process it.
2410
793
        const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2411
793
                                             ST.getAddress(), IL.getAddress(),
2412
793
                                             Chunk, EUB);
2413
793
        EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2414
793
                            LoopArguments, CGDispatchBounds);
2415
793
      }
2416
3.00k
      if (isOpenMPSimdDirective(S.getDirectiveKind())) {
2417
1.41k
        EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2418
1.41k
          return CGF.Builder.CreateIsNotNull(
2419
1.41k
              CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2420
1.41k
        });
2421
1.41k
      }
2422
3.00k
      EmitOMPReductionClauseFinal(
2423
3.00k
          S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2424
3.00k
                 ? /*Parallel and Simd*/ 
OMPD_parallel_for_simd1.41k
2425
3.00k
                 : /*Parallel only*/ 
OMPD_parallel1.58k
);
2426
3.00k
      // Emit post-update of the reduction variables if IsLastIter != 0.
2427
3.00k
      emitPostUpdateForReductionClause(
2428
3.00k
          *this, S, [IL, &S](CodeGenFunction &CGF) {
2429
0
            return CGF.Builder.CreateIsNotNull(
2430
0
                CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2431
0
          });
2432
3.00k
      // Emit final copy of the lastprivate variables if IsLastIter != 0.
2433
3.00k
      if (HasLastprivateClause)
2434
130
        EmitOMPLastprivateClauseFinal(
2435
130
            S, isOpenMPSimdDirective(S.getDirectiveKind()),
2436
130
            Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
2437
3.00k
    }
2438
3.00k
    EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2439
90
      return CGF.Builder.CreateIsNotNull(
2440
90
          CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2441
90
    });
2442
3.00k
    DoacrossCleanupScope.ForceCleanup();
2443
3.00k
    // We're now done with the loop, so jump to the continuation block.
2444
3.00k
    if (ContBlock) {
2445
589
      EmitBranch(ContBlock);
2446
589
      EmitBlock(ContBlock, /*IsFinished=*/true);
2447
589
    }
2448
3.00k
  }
2449
3.00k
  return HasLastprivateClause;
2450
3.02k
}
2451
2452
/// The following two functions generate expressions for the loop lower
2453
/// and upper bounds in case of static and dynamic (dispatch) schedule
2454
/// of the associated 'for' or 'distribute' loop.
2455
static std::pair<LValue, LValue>
2456
1.18k
emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
2457
1.18k
  const auto &LS = cast<OMPLoopDirective>(S);
2458
1.18k
  LValue LB =
2459
1.18k
      EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2460
1.18k
  LValue UB =
2461
1.18k
      EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2462
1.18k
  return {LB, UB};
2463
1.18k
}
2464
2465
/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2466
/// consider the lower and upper bound expressions generated by the
2467
/// worksharing loop support, but we use 0 and the iteration space size as
2468
/// constants
2469
static std::pair<llvm::Value *, llvm::Value *>
2470
emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
2471
258
                          Address LB, Address UB) {
2472
258
  const auto &LS = cast<OMPLoopDirective>(S);
2473
258
  const Expr *IVExpr = LS.getIterationVariable();
2474
258
  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2475
258
  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2476
258
  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2477
258
  return {LBVal, UBVal};
2478
258
}
2479
2480
233
void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
2481
233
  bool HasLastprivates = false;
2482
233
  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2483
233
                                          PrePostActionTy &) {
2484
233
    OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2485
233
    HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2486
233
                                                 emitForLoopBounds,
2487
233
                                                 emitDispatchForLoopBounds);
2488
233
  };
2489
233
  {
2490
233
    OMPLexicalScope Scope(*this, S, OMPD_unknown);
2491
233
    CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2492
233
                                                S.hasCancel());
2493
233
  }
2494
233
2495
233
  // Emit an implicit barrier at the end.
2496
233
  if (!S.getSingleClause<OMPNowaitClause>() || 
HasLastprivates11
)
2497
222
    CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2498
233
}
2499
2500
202
void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
2501
202
  bool HasLastprivates = false;
2502
202
  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2503
202
                                          PrePostActionTy &) {
2504
202
    HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2505
202
                                                 emitForLoopBounds,
2506
202
                                                 emitDispatchForLoopBounds);
2507
202
  };
2508
202
  {
2509
202
    OMPLexicalScope Scope(*this, S, OMPD_unknown);
2510
202
    CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2511
202
  }
2512
202
2513
202
  // Emit an implicit barrier at the end.
2514
202
  if (!S.getSingleClause<OMPNowaitClause>() || 
HasLastprivates0
)
2515
202
    CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2516
202
}
2517
2518
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
2519
                                const Twine &Name,
2520
260
                                llvm::Value *Init = nullptr) {
2521
260
  LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2522
260
  if (Init)
2523
208
    CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2524
260
  return LVal;
2525
260
}
2526
2527
52
void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2528
52
  const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
2529
52
  const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2530
52
  bool HasLastprivates = false;
2531
52
  auto &&CodeGen = [&S, CapturedStmt, CS,
2532
52
                    &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2533
52
    ASTContext &C = CGF.getContext();
2534
52
    QualType KmpInt32Ty =
2535
52
        C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2536
52
    // Emit helper vars inits.
2537
52
    LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2538
52
                                  CGF.Builder.getInt32(0));
2539
52
    llvm::ConstantInt *GlobalUBVal = CS != nullptr
2540
52
                                         ? CGF.Builder.getInt32(CS->size() - 1)
2541
52
                                         : 
CGF.Builder.getInt32(0)0
;
2542
52
    LValue UB =
2543
52
        createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2544
52
    LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2545
52
                                  CGF.Builder.getInt32(1));
2546
52
    LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2547
52
                                  CGF.Builder.getInt32(0));
2548
52
    // Loop counter.
2549
52
    LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2550
52
    OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2551
52
    CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2552
52
    OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2553
52
    CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2554
52
    // Generate condition for loop.
2555
52
    BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2556
52
                        OK_Ordinary, S.getBeginLoc(), FPOptions());
2557
52
    // Increment for loop counter.
2558
52
    UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2559
52
                      S.getBeginLoc(), true);
2560
52
    auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2561
52
      // Iterate through all sections and emit a switch construct:
2562
52
      // switch (IV) {
2563
52
      //   case 0:
2564
52
      //     <SectionStmt[0]>;
2565
52
      //     break;
2566
52
      // ...
2567
52
      //   case <NumSection> - 1:
2568
52
      //     <SectionStmt[<NumSection> - 1]>;
2569
52
      //     break;
2570
52
      // }
2571
52
      // .omp.sections.exit:
2572
52
      llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2573
52
      llvm::SwitchInst *SwitchStmt =
2574
52
          CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
2575
52
                                   ExitBB, CS == nullptr ? 
10
: CS->size());
2576
52
      if (CS) {
2577
52
        unsigned CaseNumber = 0;
2578
84
        for (const Stmt *SubStmt : CS->children()) {
2579
84
          auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2580
84
          CGF.EmitBlock(CaseBB);
2581
84
          SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2582
84
          CGF.EmitStmt(SubStmt);
2583
84
          CGF.EmitBranch(ExitBB);
2584
84
          ++CaseNumber;
2585
84
        }
2586
52
      } else {
2587
0
        llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2588
0
        CGF.EmitBlock(CaseBB);
2589
0
        SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2590
0
        CGF.EmitStmt(CapturedStmt);
2591
0
        CGF.EmitBranch(ExitBB);
2592
0
      }
2593
52
      CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2594
52
    };
2595
52
2596
52
    CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2597
52
    if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2598
0
      // Emit implicit barrier to synchronize threads and avoid data races on
2599
0
      // initialization of firstprivate variables and post-update of lastprivate
2600
0
      // variables.
2601
0
      CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2602
0
          CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2603
0
          /*ForceSimpleCall=*/true);
2604
0
    }
2605
52
    CGF.EmitOMPPrivateClause(S, LoopScope);
2606
52
    HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2607
52
    CGF.EmitOMPReductionClauseInit(S, LoopScope);
2608
52
    (void)LoopScope.Privatize();
2609
52
    if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2610
0
      CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2611
52
2612
52
    // Emit static non-chunked loop.
2613
52
    OpenMPScheduleTy ScheduleKind;
2614
52
    ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2615
52
    CGOpenMPRuntime::StaticRTInput StaticInit(
2616
52
        /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2617
52
        LB.getAddress(), UB.getAddress(), ST.getAddress());
2618
52
    CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2619
52
        CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2620
52
    // UB = min(UB, GlobalUB);
2621
52
    llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
2622
52
    llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2623
52
        CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2624
52
    CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2625
52
    // IV = LB;
2626
52
    CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
2627
52
    // while (idx <= UB) { BODY; ++idx; }
2628
52
    CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2629
52
                         [](CodeGenFunction &) {});
2630
52
    // Tell the runtime we are done.
2631
68
    auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2632
68
      CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2633
68
                                                     S.getDirectiveKind());
2634
68
    };
2635
52
    CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2636
52
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2637
52
    // Emit post-update of the reduction variables if IsLastIter != 0.
2638
52
    emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2639
0
      return CGF.Builder.CreateIsNotNull(
2640
0
          CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2641
0
    });
2642
52
2643
52
    // Emit final copy of the lastprivate variables if IsLastIter != 0.
2644
52
    if (HasLastprivates)
2645
8
      CGF.EmitOMPLastprivateClauseFinal(
2646
8
          S, /*NoFinals=*/false,
2647
8
          CGF.Builder.CreateIsNotNull(
2648
8
              CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
2649
52
  };
2650
52
2651
52
  bool HasCancel = false;
2652
52
  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2653
40
    HasCancel = OSD->hasCancel();
2654
12
  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2655
12
    HasCancel = OPSD->hasCancel();
2656
52
  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2657
52
  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2658
52
                                              HasCancel);
2659
52
  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2660
52
  // clause. Otherwise the barrier will be generated by the codegen for the
2661
52
  // directive.
2662
52
  if (HasLastprivates && 
S.getSingleClause<OMPNowaitClause>()8
) {
2663
0
    // Emit implicit barrier to synchronize threads and avoid data races on
2664
0
    // initialization of firstprivate variables.
2665
0
    CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2666
0
                                           OMPD_unknown);
2667
0
  }
2668
52
}
2669
2670
40
void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
2671
40
  {
2672
40
    OMPLexicalScope Scope(*this, S, OMPD_unknown);
2673
40
    EmitSections(S);
2674
40
  }
2675
40
  // Emit an implicit barrier at the end.
2676
40
  if (!S.getSingleClause<OMPNowaitClause>()) {
2677
34
    CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2678
34
                                           OMPD_sections);
2679
34
  }
2680
40
}
2681
2682
36
void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
2683
36
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2684
36
    CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2685
36
  };
2686
36
  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2687
36
  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2688
36
                                              S.hasCancel());
2689
36
}
2690
2691
39
void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
2692
39
  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2693
39
  llvm::SmallVector<const Expr *, 8> DestExprs;
2694
39
  llvm::SmallVector<const Expr *, 8> SrcExprs;
2695
39
  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2696
39
  // Check if there are any 'copyprivate' clauses associated with this
2697
39
  // 'single' construct.
2698
39
  // Build a list of copyprivate variables along with helper expressions
2699
39
  // (<source>, <destination>, <destination>=<source> expressions)
2700
39
  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2701
18
    CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2702
18
    DestExprs.append(C->destination_exprs().begin(),
2703
18
                     C->destination_exprs().end());
2704
18
    SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2705
18
    AssignmentOps.append(C->assignment_ops().begin(),
2706
18
                         C->assignment_ops().end());
2707
18
  }
2708
39
  // Emit code for 'single' region along with 'copyprivate' clauses
2709
39
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2710
39
    Action.Enter(CGF);
2711
39
    OMPPrivateScope SingleScope(CGF);
2712
39
    (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2713
39
    CGF.EmitOMPPrivateClause(S, SingleScope);
2714
39
    (void)SingleScope.Privatize();
2715
39
    CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2716
39
  };
2717
39
  {
2718
39
    OMPLexicalScope Scope(*this, S, OMPD_unknown);
2719
39
    CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
2720
39
                                            CopyprivateVars, DestExprs,
2721
39
                                            SrcExprs, AssignmentOps);
2722
39
  }
2723
39
  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2724
39
  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2725
39
  if (!S.getSingleClause<OMPNowaitClause>() && 
CopyprivateVars.empty()34
) {
2726
16
    CGM.getOpenMPRuntime().emitBarrierCall(
2727
16
        *this, S.getBeginLoc(),
2728
16
        S.getSingleClause<OMPNowaitClause>() ? 
OMPD_unknown0
: OMPD_single);
2729
16
  }
2730
39
}
2731
2732
9
void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
2733
9
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2734
9
    Action.Enter(CGF);
2735
9
    CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2736
9
  };
2737
9
  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2738
9
  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
2739
9
}
2740
2741
26
void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
2742
26
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2743
26
    Action.Enter(CGF);
2744
26
    CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2745
26
  };
2746
26
  const Expr *Hint = nullptr;
2747
26
  if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2748
3
    Hint = HintClause->getHint();
2749
26
  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2750
26
  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2751
26
                                            S.getDirectiveName().getAsString(),
2752
26
                                            CodeGen, S.getBeginLoc(), Hint);
2753
26
}
2754
2755
void CodeGenFunction::EmitOMPParallelForDirective(
2756
137
    const OMPParallelForDirective &S) {
2757
137
  // Emit directive as a combined directive that consists of two implicit
2758
137
  // directives: 'parallel' with 'for' directive.
2759
137
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2760
137
    Action.Enter(CGF);
2761
137
    OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2762
137
    CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2763
137
                               emitDispatchForLoopBounds);
2764
137
  };
2765
137
  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2766
137
                                 emitEmptyBoundParameters);
2767
137
}
2768
2769
void CodeGenFunction::EmitOMPParallelForSimdDirective(
2770
39
    const OMPParallelForSimdDirective &S) {
2771
39
  // Emit directive as a combined directive that consists of two implicit
2772
39
  // directives: 'parallel' with 'for' directive.
2773
39
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2774
39
    Action.Enter(CGF);
2775
39
    CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2776
39
                               emitDispatchForLoopBounds);
2777
39
  };
2778
39
  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2779
39
                                 emitEmptyBoundParameters);
2780
39
}
2781
2782
void CodeGenFunction::EmitOMPParallelSectionsDirective(
2783
12
    const OMPParallelSectionsDirective &S) {
2784
12
  // Emit directive as a combined directive that consists of two implicit
2785
12
  // directives: 'parallel' with 'sections' directive.
2786
12
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2787
12
    Action.Enter(CGF);
2788
12
    CGF.EmitSections(S);
2789
12
  };
2790
12
  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2791
12
                                 emitEmptyBoundParameters);
2792
12
}
2793
2794
void CodeGenFunction::EmitOMPTaskBasedDirective(
2795
    const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2796
    const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2797
150
    OMPTaskDataTy &Data) {
2798
150
  // Emit outlined function for task construct.
2799
150
  const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2800
150
  auto I = CS->getCapturedDecl()->param_begin();
2801
150
  auto PartId = std::next(I);
2802
150
  auto TaskT = std::next(I, 4);
2803
150
  // Check if the task is final
2804
150
  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2805
8
    // If the condition constant folds and can be elided, try to avoid emitting
2806
8
    // the condition and the dead arm of the if/else.
2807
8
    const Expr *Cond = Clause->getCondition();
2808
8
    bool CondConstant;
2809
8
    if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2810
6
      Data.Final.setInt(CondConstant);
2811
2
    else
2812
2
      Data.Final.setPointer(EvaluateExprAsBool(Cond));
2813
142
  } else {
2814
142
    // By default the task is not final.
2815
142
    Data.Final.setInt(/*IntVal=*/false);
2816
142
  }
2817
150
  // Check if the task has 'priority' clause.
2818
150
  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2819
6
    const Expr *Prio = Clause->getPriority();
2820
6
    Data.Priority.setInt(/*IntVal=*/true);
2821
6
    Data.Priority.setPointer(EmitScalarConversion(
2822
6
        EmitScalarExpr(Prio), Prio->getType(),
2823
6
        getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2824
6
        Prio->getExprLoc()));
2825
6
  }
2826
150
  // The first function argument for tasks is a thread id, the second one is a
2827
150
  // part id (0 for tied tasks, >=0 for untied task).
2828
150
  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2829
150
  // Get list of private variables.
2830
150
  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2831
22
    auto IRef = C->varlist_begin();
2832
98
    for (const Expr *IInit : C->private_copies()) {
2833
98
      const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2834
98
      if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2835
74
        Data.PrivateVars.push_back(*IRef);
2836
74
        Data.PrivateCopies.push_back(IInit);
2837
74
      }
2838
98
      ++IRef;
2839
98
    }
2840
22
  }
2841
150
  EmittedAsPrivate.clear();
2842
150
  // Get list of firstprivate variables.
2843
150
  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2844
35
    auto IRef = C->varlist_begin();
2845
35
    auto IElemInitRef = C->inits().begin();
2846
115
    for (const Expr *IInit : C->private_copies()) {
2847
115
      const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2848
115
      if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2849
95
        Data.FirstprivateVars.push_back(*IRef);
2850
95
        Data.FirstprivateCopies.push_back(IInit);
2851
95
        Data.FirstprivateInits.push_back(*IElemInitRef);
2852
95
      }
2853
115
      ++IRef;
2854
115
      ++IElemInitRef;
2855
115
    }
2856
35
  }
2857
150
  // Get list of lastprivate variables (for taskloops).
2858
150
  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2859
150
  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2860
15
    auto IRef = C->varlist_begin();
2861
15
    auto ID = C->destination_exprs().begin();
2862
65
    for (const Expr *IInit : C->private_copies()) {
2863
65
      const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2864
65
      if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2865
49
        Data.LastprivateVars.push_back(*IRef);
2866
49
        Data.LastprivateCopies.push_back(IInit);
2867
49
      }
2868
65
      LastprivateDstsOrigs.insert(
2869
65
          {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2870
65
           cast<DeclRefExpr>(*IRef)});
2871
65
      ++IRef;
2872
65
      ++ID;
2873
65
    }
2874
15
  }
2875
150
  SmallVector<const Expr *, 4> LHSs;
2876
150
  SmallVector<const Expr *, 4> RHSs;
2877
150
  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2878
2
    auto IPriv = C->privates().begin();
2879
2
    auto IRed = C->reduction_ops().begin();
2880
2
    auto ILHS = C->lhs_exprs().begin();
2881
2
    auto IRHS = C->rhs_exprs().begin();
2882
8
    for (const Expr *Ref : C->varlists()) {
2883
8
      Data.ReductionVars.emplace_back(Ref);
2884
8
      Data.ReductionCopies.emplace_back(*IPriv);
2885
8
      Data.ReductionOps.emplace_back(*IRed);
2886
8
      LHSs.emplace_back(*ILHS);
2887
8
      RHSs.emplace_back(*IRHS);
2888
8
      std::advance(IPriv, 1);
2889
8
      std::advance(IRed, 1);
2890
8
      std::advance(ILHS, 1);
2891
8
      std::advance(IRHS, 1);
2892
8
    }
2893
2
  }
2894
150
  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2895
150
      *this, S.getBeginLoc(), LHSs, RHSs, Data);
2896
150
  // Build list of dependences.
2897
150
  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2898
16
    for (const Expr *IRef : C->varlists())
2899
30
      Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
2900
150
  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2901
150
                    CapturedRegion](CodeGenFunction &CGF,
2902
150
                                    PrePostActionTy &Action) {
2903
150
    // Set proper addresses for generated private copies.
2904
150
    OMPPrivateScope Scope(CGF);
2905
150
    if (!Data.PrivateVars.empty() || 
!Data.FirstprivateVars.empty()128
||
2906
150
        
!Data.LastprivateVars.empty()95
) {
2907
70
      llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
2908
70
          CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
2909
70
      enum { PrivatesParam = 2, CopyFnParam = 3 };
2910
70
      llvm::Value *CopyFn = CGF.Builder.CreateLoad(
2911
70
          CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2912
70
      llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2913
70
          CS->getCapturedDecl()->getParam(PrivatesParam)));
2914
70
      // Map privates.
2915
70
      llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
2916
70
      llvm::SmallVector<llvm::Value *, 16> CallArgs;
2917
70
      CallArgs.push_back(PrivatesPtr);
2918
74
      for (const Expr *E : Data.PrivateVars) {
2919
74
        const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2920
74
        Address PrivatePtr = CGF.CreateMemTemp(
2921
74
            CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2922
74
        PrivatePtrs.emplace_back(VD, PrivatePtr);
2923
74
        CallArgs.push_back(PrivatePtr.getPointer());
2924
74
      }
2925
95
      for (const Expr *E : Data.FirstprivateVars) {
2926
95
        const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2927
95
        Address PrivatePtr =
2928
95
            CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2929
95
                              ".firstpriv.ptr.addr");
2930
95
        PrivatePtrs.emplace_back(VD, PrivatePtr);
2931
95
        CallArgs.push_back(PrivatePtr.getPointer());
2932
95
      }
2933
70
      for (const Expr *E : Data.LastprivateVars) {
2934
49
        const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2935
49
        Address PrivatePtr =
2936
49
            CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2937
49
                              ".lastpriv.ptr.addr");
2938
49
        PrivatePtrs.emplace_back(VD, PrivatePtr);
2939
49
        CallArgs.push_back(PrivatePtr.getPointer());
2940
49
      }
2941
70
      CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
2942
70
          CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
2943
70
      for (const auto &Pair : LastprivateDstsOrigs) {
2944
65
        const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2945
65
        DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
2946
65
                        /*RefersToEnclosingVariableOrCapture=*/
2947
65
                            CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
2948
65
                        Pair.second->getType(), VK_LValue,
2949
65
                        Pair.second->getExprLoc());
2950
65
        Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2951
65
          return CGF.EmitLValue(&DRE).getAddress();
2952
65
        });
2953
65
      }
2954
218
      for (const auto &Pair : PrivatePtrs) {
2955
218
        Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2956
218
                            CGF.getContext().getDeclAlign(Pair.first));
2957
218
        Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2958
218
      }
2959
70
    }
2960
150
    if (Data.Reductions) {
2961
2
      OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2962
2
      ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies,
2963
2
                             Data.ReductionOps);
2964
2
      llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2965
2
          CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2966
10
      for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; 
++Cnt8
) {
2967
8
        RedCG.emitSharedLValue(CGF, Cnt);
2968
8
        RedCG.emitAggregateType(CGF, Cnt);
2969
8
        // FIXME: This must removed once the runtime library is fixed.
2970
8
        // Emit required threadprivate variables for
2971
8
        // initializer/combiner/finalizer.
2972
8
        CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
2973
8
                                                           RedCG, Cnt);
2974
8
        Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2975
8
            CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2976
8
        Replacement =
2977
8
            Address(CGF.EmitScalarConversion(
2978
8
                        Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2979
8
                        CGF.getContext().getPointerType(
2980
8
                            Data.ReductionCopies[Cnt]->getType()),
2981
8
                        Data.ReductionCopies[Cnt]->getExprLoc()),
2982
8
                    Replacement.getAlignment());
2983
8
        Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2984
8
        Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2985
8
                         [Replacement]() { return Replacement; });
2986
8
      }
2987
2
    }
2988
150
    // Privatize all private variables except for in_reduction items.
2989
150
    (void)Scope.Privatize();
2990
150
    SmallVector<const Expr *, 4> InRedVars;
2991
150
    SmallVector<const Expr *, 4> InRedPrivs;
2992
150
    SmallVector<const Expr *, 4> InRedOps;
2993
150
    SmallVector<const Expr *, 4> TaskgroupDescriptors;
2994
150
    for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2995
12
      auto IPriv = C->privates().begin();
2996
12
      auto IRed = C->reduction_ops().begin();
2997
12
      auto ITD = C->taskgroup_descriptors().begin();
2998
12
      for (const Expr *Ref : C->varlists()) {
2999
12
        InRedVars.emplace_back(Ref);
3000
12
        InRedPrivs.emplace_back(*IPriv);
3001
12
        InRedOps.emplace_back(*IRed);
3002
12
        TaskgroupDescriptors.emplace_back(*ITD);
3003
12
        std::advance(IPriv, 1);
3004
12
        std::advance(IRed, 1);
3005
12
        std::advance(ITD, 1);
3006
12
      }
3007
12
    }
3008
150
    // Privatize in_reduction items here, because taskgroup descriptors must be
3009
150
    // privatized earlier.
3010
150
    OMPPrivateScope InRedScope(CGF);
3011
150
    if (!InRedVars.empty()) {
3012
6
      ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
3013
18
      for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; 
++Cnt12
) {
3014
12
        RedCG.emitSharedLValue(CGF, Cnt);
3015
12
        RedCG.emitAggregateType(CGF, Cnt);
3016
12
        // The taskgroup descriptor variable is always implicit firstprivate and
3017
12
        // privatized already during processing of the firstprivates.
3018
12
        // FIXME: This must removed once the runtime library is fixed.
3019
12
        // Emit required threadprivate variables for
3020
12
        // initializer/combiner/finalizer.
3021
12
        CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3022
12
                                                           RedCG, Cnt);
3023
12
        llvm::Value *ReductionsPtr =
3024
12
            CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3025
12
                                 TaskgroupDescriptors[Cnt]->getExprLoc());
3026
12
        Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3027
12
            CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3028
12
        Replacement = Address(
3029
12
            CGF.EmitScalarConversion(
3030
12
                Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3031
12
                CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
3032
12
                InRedPrivs[Cnt]->getExprLoc()),
3033
12
            Replacement.getAlignment());
3034
12
        Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3035
12
        InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
3036
12
                              [Replacement]() { return Replacement; });
3037
12
      }
3038
6
    }
3039
150
    (void)InRedScope.Privatize();
3040
150
3041
150
    Action.Enter(CGF);
3042
150
    BodyGen(CGF);
3043
150
  };
3044
150
  llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3045
150
      S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
3046
150
      Data.NumberOfParts);
3047
150
  OMPLexicalScope Scope(*this, S);
3048
150
  TaskGen(*this, OutlinedFn, Data);
3049
150
}
3050
3051
static ImplicitParamDecl *
3052
createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
3053
                                  QualType Ty, CapturedDecl *CD,
3054
324
                                  SourceLocation Loc) {
3055
324
  auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3056
324
                                           ImplicitParamDecl::Other);
3057
324
  auto *OrigRef = DeclRefExpr::Create(
3058
324
      C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3059
324
      /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3060
324
  auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3061
324
                                              ImplicitParamDecl::Other);
3062
324
  auto *PrivateRef = DeclRefExpr::Create(
3063
324
      C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3064
324
      /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3065
324
  QualType ElemType = C.getBaseElementType(Ty);
3066
324
  auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3067
324
                                           ImplicitParamDecl::Other);
3068
324
  auto *InitRef = DeclRefExpr::Create(
3069
324
      C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3070
324
      /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3071
324
  PrivateVD->setInitStyle(VarDecl::CInit);
3072
324
  PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3073
324
                                              InitRef, /*BasePath=*/nullptr,
3074
324
                                              VK_RValue));
3075
324
  Data.FirstprivateVars.emplace_back(OrigRef);
3076
324
  Data.FirstprivateCopies.emplace_back(PrivateRef);
3077
324
  Data.FirstprivateInits.emplace_back(InitRef);
3078
324
  return OrigVD;
3079
324
}
3080
3081
void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
3082
    const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3083
292
    OMPTargetDataInfo &InputInfo) {
3084
292
  // Emit outlined function for task construct.
3085
292
  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3086
292
  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3087
292
  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3088
292
  auto I = CS->getCapturedDecl()->param_begin();
3089
292
  auto PartId = std::next(I);
3090
292
  auto TaskT = std::next(I, 4);
3091
292
  OMPTaskDataTy Data;
3092
292
  // The task is not final.
3093
292
  Data.Final.setInt(/*IntVal=*/false);
3094
292
  // Get list of firstprivate variables.
3095
292
  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3096
180
    auto IRef = C->varlist_begin();
3097
180
    auto IElemInitRef = C->inits().begin();
3098
300
    for (auto *IInit : C->private_copies()) {
3099
300
      Data.FirstprivateVars.push_back(*IRef);
3100
300
      Data.FirstprivateCopies.push_back(IInit);
3101
300
      Data.FirstprivateInits.push_back(*IElemInitRef);
3102
300
      ++IRef;
3103
300
      ++IElemInitRef;
3104
300
    }
3105
180
  }
3106
292
  OMPPrivateScope TargetScope(*this);
3107
292
  VarDecl *BPVD = nullptr;
3108
292
  VarDecl *PVD = nullptr;
3109
292
  VarDecl *SVD = nullptr;
3110
292
  if (InputInfo.NumberOfTargetItems > 0) {
3111
108
    auto *CD = CapturedDecl::Create(
3112
108
        getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3113
108
    llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3114
108
    QualType BaseAndPointersType = getContext().getConstantArrayType(
3115
108
        getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3116
108
        /*IndexTypeQuals=*/0);
3117
108
    BPVD = createImplicitFirstprivateForType(
3118
108
        getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3119
108
    PVD = createImplicitFirstprivateForType(
3120
108
        getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3121
108
    QualType SizesType = getContext().getConstantArrayType(
3122
108
        getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
3123
108
        ArrSize, ArrayType::Normal,
3124
108
        /*IndexTypeQuals=*/0);
3125
108
    SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3126
108
                                            S.getBeginLoc());
3127
108
    TargetScope.addPrivate(
3128
108
        BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3129
108
    TargetScope.addPrivate(PVD,
3130
108
                           [&InputInfo]() { return InputInfo.PointersArray; });
3131
108
    TargetScope.addPrivate(SVD,
3132
108
                           [&InputInfo]() { return InputInfo.SizesArray; });
3133
108
  }
3134
292
  (void)TargetScope.Privatize();
3135
292
  // Build list of dependences.
3136
292
  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3137
356
    for (const Expr *IRef : C->varlists())
3138
832
      Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3139
292
  auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3140
292
                    &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3141
292
    // Set proper addresses for generated private copies.
3142
292
    OMPPrivateScope Scope(CGF);
3143
292
    if (!Data.FirstprivateVars.empty()) {
3144
228
      llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3145
228
          CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
3146
228
      enum { PrivatesParam = 2, CopyFnParam = 3 };
3147
228
      llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3148
228
          CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3149
228
      llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3150
228
          CS->getCapturedDecl()->getParam(PrivatesParam)));
3151
228
      // Map privates.
3152
228
      llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
3153
228
      llvm::SmallVector<llvm::Value *, 16> CallArgs;
3154
228
      CallArgs.push_back(PrivatesPtr);
3155
624
      for (const Expr *E : Data.FirstprivateVars) {
3156
624
        const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3157
624
        Address PrivatePtr =
3158
624
            CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3159
624
                              ".firstpriv.ptr.addr");
3160
624
        PrivatePtrs.emplace_back(VD, PrivatePtr);
3161
624
        CallArgs.push_back(PrivatePtr.getPointer());
3162
624
      }
3163
228
      CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3164
228
          CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
3165
624
      for (const auto &Pair : PrivatePtrs) {
3166
624
        Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3167
624
                            CGF.getContext().getDeclAlign(Pair.first));
3168
624
        Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3169
624
      }
3170
228
    }
3171
292
    // Privatize all private variables except for in_reduction items.
3172
292
    (void)Scope.Privatize();
3173
292
    if (InputInfo.NumberOfTargetItems > 0) {
3174
108
      InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3175
108
          CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
3176
108
      InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3177
108
          CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
3178
108
      InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3179
108
          CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
3180
108
    }
3181
292
3182
292
    Action.Enter(CGF);
3183
292
    OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3184
292
    BodyGen(CGF);
3185
292
  };
3186
292
  llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3187
292
      S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3188
292
      Data.NumberOfParts);
3189
292
  llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 
1136
:
0156
);
3190
292
  IntegerLiteral IfCond(getContext(), TrueOrFalse,
3191
292
                        getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3192
292
                        SourceLocation());
3193
292
3194
292
  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
3195
292
                                      SharedsTy, CapturedStruct, &IfCond, Data);
3196
292
}
3197
3198
85
void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
3199
85
  // Emit outlined function for task construct.
3200
85
  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3201
85
  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3202
85
  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3203
85
  const Expr *IfCond = nullptr;
3204
85
  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3205
22
    if (C->getNameModifier() == OMPD_unknown ||
3206
22
        
C->getNameModifier() == OMPD_task6
) {
3207
22
      IfCond = C->getCondition();
3208
22
      break;
3209
22
    }
3210
22
  }
3211
85
3212
85
  OMPTaskDataTy Data;
3213
85
  // Check if we should emit tied or untied task.
3214
85
  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3215
85
  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3216
85
    CGF.EmitStmt(CS->getCapturedStmt());
3217
85
  };
3218
85
  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3219
85
                    IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
3220
85
                            const OMPTaskDataTy &Data) {
3221
85
    CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
3222
85
                                            SharedsTy, CapturedStruct, IfCond,
3223
85
                                            Data);
3224
85
  };
3225
85
  EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3226
85
}
3227
3228
void CodeGenFunction::EmitOMPTaskyieldDirective(
3229
8
    const OMPTaskyieldDirective &S) {
3230
8
  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
3231
8
}
3232
3233
13
void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
3234
13
  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
3235
13
}
3236
3237
6
void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
3238
6
  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
3239
6
}
3240
3241
void CodeGenFunction::EmitOMPTaskgroupDirective(
3242
31
    const OMPTaskgroupDirective &S) {
3243
31
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3244
31
    Action.Enter(CGF);
3245
31
    if (const Expr *E = S.getReductionRef()) {
3246
18
      SmallVector<const Expr *, 4> LHSs;
3247
18
      SmallVector<const Expr *, 4> RHSs;
3248
18
      OMPTaskDataTy Data;
3249
18
      for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3250
18
        auto IPriv = C->privates().begin();
3251
18
        auto IRed = C->reduction_ops().begin();
3252
18
        auto ILHS = C->lhs_exprs().begin();
3253
18
        auto IRHS = C->rhs_exprs().begin();
3254
45
        for (const Expr *Ref : C->varlists()) {
3255
45
          Data.ReductionVars.emplace_back(Ref);
3256
45
          Data.ReductionCopies.emplace_back(*IPriv);
3257
45
          Data.ReductionOps.emplace_back(*IRed);
3258
45
          LHSs.emplace_back(*ILHS);
3259
45
          RHSs.emplace_back(*IRHS);
3260
45
          std::advance(IPriv, 1);
3261
45
          std::advance(IRed, 1);
3262
45
          std::advance(ILHS, 1);
3263
45
          std::advance(IRHS, 1);
3264
45
        }
3265
18
      }
3266
18
      llvm::Value *ReductionDesc =
3267
18
          CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
3268
18
                                                           LHSs, RHSs, Data);
3269
18
      const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3270
18
      CGF.EmitVarDecl(*VD);
3271
18
      CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3272
18
                            /*Volatile=*/false, E->getType());
3273
18
    }
3274
31
    CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
3275
31
  };
3276
31
  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3277
31
  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
3278
31
}
3279
3280
8
void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
3281
8
  CGM.getOpenMPRuntime().emitFlush(
3282
8
      *this,
3283
8
      [&S]() -> ArrayRef<const Expr *> {
3284
8
        if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3285
4
          return llvm::makeArrayRef(FlushClause->varlist_begin(),
3286
4
                                    FlushClause->varlist_end());
3287
4
        return llvm::None;
3288
4
      }(),
3289
8
      S.getBeginLoc());
3290
8
}
3291
3292
void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
3293
                                            const CodeGenLoopTy &CodeGenLoop,
3294
2.96k
                                            Expr *IncExpr) {
3295
2.96k
  // Emit the loop iteration variable.
3296
2.96k
  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3297
2.96k
  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3298
2.96k
  EmitVarDecl(*IVDecl);
3299
2.96k
3300
2.96k
  // Emit the iterations count variable.
3301
2.96k
  // If it is not a variable, Sema decided to calculate iterations count on each
3302
2.96k
  // iteration (e.g., it is foldable into a constant).
3303
2.96k
  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3304
0
    EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3305
0
    // Emit calculation of the iterations count.
3306
0
    EmitIgnoredExpr(S.getCalcLastIteration());
3307
0
  }
3308
2.96k
3309
2.96k
  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3310
2.96k
3311
2.96k
  bool HasLastprivateClause = false;
3312
2.96k
  // Check pre-condition.
3313
2.96k
  {
3314
2.96k
    OMPLoopScope PreInitScope(*this, S);
3315
2.96k
    // Skip the entire loop if we don't meet the precondition.
3316
2.96k
    // If the condition constant folds and can be elided, avoid emitting the
3317
2.96k
    // whole loop.
3318
2.96k
    bool CondConstant;
3319
2.96k
    llvm::BasicBlock *ContBlock = nullptr;
3320
2.96k
    if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3321
2.31k
      if (!CondConstant)
3322
0
        return;
3323
652
    } else {
3324
652
      llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3325
652
      ContBlock = createBasicBlock("omp.precond.end");
3326
652
      emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3327
652
                  getProfileCount(&S));
3328
652
      EmitBlock(ThenBlock);
3329
652
      incrementProfileCounter(&S);
3330
652
    }
3331
2.96k
3332
2.96k
    emitAlignedClause(*this, S);
3333
2.96k
    // Emit 'then' code.
3334
2.96k
    {
3335
2.96k
      // Emit helper vars inits.
3336
2.96k
3337
2.96k
      LValue LB = EmitOMPHelperVar(
3338
2.96k
          *this, cast<DeclRefExpr>(
3339
2.96k
                     (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3340
2.96k
                          ? 
S.getCombinedLowerBoundVariable()1.81k
3341
2.96k
                          : 
S.getLowerBoundVariable()1.14k
)));
3342
2.96k
      LValue UB = EmitOMPHelperVar(
3343
2.96k
          *this, cast<DeclRefExpr>(
3344
2.96k
                     (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3345
2.96k
                          ? 
S.getCombinedUpperBoundVariable()1.81k
3346
2.96k
                          : 
S.getUpperBoundVariable()1.14k
)));
3347
2.96k
      LValue ST =
3348
2.96k
          EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3349
2.96k
      LValue IL =
3350
2.96k
          EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3351
2.96k
3352
2.96k
      OMPPrivateScope LoopScope(*this);
3353
2.96k
      if (EmitOMPFirstprivateClause(S, LoopScope)) {
3354
0
        // Emit implicit barrier to synchronize threads and avoid data races
3355
0
        // on initialization of firstprivate variables and post-update of
3356
0
        // lastprivate variables.
3357
0
        CGM.getOpenMPRuntime().emitBarrierCall(
3358
0
            *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3359
0
            /*ForceSimpleCall=*/true);
3360
0
      }
3361
2.96k
      EmitOMPPrivateClause(S, LoopScope);
3362
2.96k
      if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
3363
2.96k
          
!isOpenMPParallelDirective(S.getDirectiveKind())1.51k
&&
3364
2.96k
          
!isOpenMPTeamsDirective(S.getDirectiveKind())585
)
3365
100
        EmitOMPReductionClauseInit(S, LoopScope);
3366
2.96k
      HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3367
2.96k
      EmitOMPPrivateLoopCounters(S, LoopScope);
3368
2.96k
      (void)LoopScope.Privatize();
3369
2.96k
      if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
3370
1.58k
        CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3371
2.96k
3372
2.96k
      // Detect the distribute schedule kind and chunk.
3373
2.96k
      llvm::Value *Chunk = nullptr;
3374
2.96k
      OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
3375
2.96k
      if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3376
358
        ScheduleKind = C->getDistScheduleKind();
3377
358
        if (const Expr *Ch = C->getChunkSize()) {
3378
202
          Chunk = EmitScalarExpr(Ch);
3379
202
          Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3380
202
                                       S.getIterationVariable()->getType(),
3381
202
                                       S.getBeginLoc());
3382
202
        }
3383
2.60k
      } else {
3384
2.60k
        // Default behaviour for dist_schedule clause.
3385
2.60k
        CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3386
2.60k
            *this, S, ScheduleKind, Chunk);
3387
2.60k
      }
3388
2.96k
      const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3389
2.96k
      const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3390
2.96k
3391
2.96k
      // OpenMP [2.10.8, distribute Construct, Description]
3392
2.96k
      // If dist_schedule is specified, kind must be static. If specified,
3393
2.96k
      // iterations are divided into chunks of size chunk_size, chunks are
3394
2.96k
      // assigned to the teams of the league in a round-robin fashion in the
3395
2.96k
      // order of the team number. When no chunk_size is specified, the
3396
2.96k
      // iteration space is divided into chunks that are approximately equal
3397
2.96k
      // in size, and at most one chunk is distributed to each team of the
3398
2.96k
      // league. The size of the chunks is unspecified in this case.
3399
2.96k
      bool StaticChunked = RT.isStaticChunked(
3400
2.96k
          ScheduleKind, /* Chunked */ Chunk != nullptr) &&
3401
2.96k
          
isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())456
;
3402
2.96k
      if (RT.isStaticNonchunked(ScheduleKind,
3403
2.96k
                                /* Chunked */ Chunk != nullptr) ||
3404
2.96k
          
StaticChunked456
) {
3405
2.85k
        if (isOpenMPSimdDirective(S.getDirectiveKind()))
3406
1.45k
          EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3407
2.85k
        CGOpenMPRuntime::StaticRTInput StaticInit(
3408
2.85k
            IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3409
2.85k
            LB.getAddress(), UB.getAddress(), ST.getAddress(),
3410
2.85k
            StaticChunked ? 
Chunk344
:
nullptr2.50k
);
3411
2.85k
        RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
3412
2.85k
                                    StaticInit);
3413
2.85k
        JumpDest LoopExit =
3414
2.85k
            getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3415
2.85k
        // UB = min(UB, GlobalUB);
3416
2.85k
        EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3417
2.85k
                            ? 
S.getCombinedEnsureUpperBound()1.81k
3418
2.85k
                            : 
S.getEnsureUpperBound()1.03k
);
3419
2.85k
        // IV = LB;
3420
2.85k
        EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3421
2.85k
                            ? 
S.getCombinedInit()1.81k
3422
2.85k
                            : 
S.getInit()1.03k
);
3423
2.85k
3424
2.85k
        const Expr *Cond =
3425
2.85k
            isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3426
2.85k
                ? 
S.getCombinedCond()1.81k
3427
2.85k
                : 
S.getCond()1.03k
;
3428
2.85k
3429
2.85k
        if (StaticChunked)
3430
344
          Cond = S.getCombinedDistCond();
3431
2.85k
3432
2.85k
        // For static unchunked schedules generate:
3433
2.85k
        //
3434
2.85k
        //  1. For distribute alone, codegen
3435
2.85k
        //    while (idx <= UB) {
3436
2.85k
        //      BODY;
3437
2.85k
        //      ++idx;
3438
2.85k
        //    }
3439
2.85k
        //
3440
2.85k
        //  2. When combined with 'for' (e.g. as in 'distribute parallel for')
3441
2.85k
        //    while (idx <= UB) {
3442
2.85k
        //      <CodeGen rest of pragma>(LB, UB);
3443
2.85k
        //      idx += ST;
3444
2.85k
        //    }
3445
2.85k
        //
3446
2.85k
        // For static chunk one schedule generate:
3447
2.85k
        //
3448
2.85k
        // while (IV <= GlobalUB) {
3449
2.85k
        //   <CodeGen rest of pragma>(LB, UB);
3450
2.85k
        //   LB += ST;
3451
2.85k
        //   UB += ST;
3452
2.85k
        //   UB = min(UB, GlobalUB);
3453
2.85k
        //   IV = LB;
3454
2.85k
        // }
3455
2.85k
        //
3456
2.85k
        EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3457
2.85k
                         [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3458
2.85k
                           CodeGenLoop(CGF, S, LoopExit);
3459
2.85k
                         },
3460
2.85k
                         [&S, StaticChunked](CodeGenFunction &CGF) {
3461
2.85k
                           if (StaticChunked) {
3462
344
                             CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3463
344
                             CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3464
344
                             CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3465
344
                             CGF.EmitIgnoredExpr(S.getCombinedInit());
3466
344
                           }
3467
2.85k
                         });
3468
2.85k
        EmitBlock(LoopExit.getBlock());
3469
2.85k
        // Tell the runtime we are done.
3470
2.85k
        RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
3471
2.85k
      } else {
3472
112
        // Emit the outer loop, which requests its work chunk [LB..UB] from
3473
112
        // runtime and runs the inner loop to process it.
3474
112
        const OMPLoopArguments LoopArguments = {
3475
112
            LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3476
112
            Chunk};
3477
112
        EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3478
112
                                   CodeGenLoop);
3479
112
      }
3480
2.96k
      if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3481
1.51k
        EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3482
1.51k
          return CGF.Builder.CreateIsNotNull(
3483
1.51k
              CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3484
1.51k
        });
3485
1.51k
      }
3486
2.96k
      if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
3487
2.96k
          
!isOpenMPParallelDirective(S.getDirectiveKind())1.51k
&&
3488
2.96k
          
!isOpenMPTeamsDirective(S.getDirectiveKind())585
) {
3489
100
        EmitOMPReductionClauseFinal(S, OMPD_simd);
3490
100
        // Emit post-update of the reduction variables if IsLastIter != 0.
3491
100
        emitPostUpdateForReductionClause(
3492
100
            *this, S, [IL, &S](CodeGenFunction &CGF) {
3493
0
              return CGF.Builder.CreateIsNotNull(
3494
0
                  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3495
0
            });
3496
100
      }
3497
2.96k
      // Emit final copy of the lastprivate variables if IsLastIter != 0.
3498
2.96k
      if (HasLastprivateClause) {
3499
175
        EmitOMPLastprivateClauseFinal(
3500
175
            S, /*NoFinals=*/false,
3501
175
            Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3502
175
      }
3503
2.96k
    }
3504
2.96k
3505
2.96k
    // We're now done with the loop, so jump to the continuation block.
3506
2.96k
    if (ContBlock) {
3507
652
      EmitBranch(ContBlock);
3508
652
      EmitBlock(ContBlock, true);
3509
652
    }
3510
2.96k
  }
3511
2.96k
}
3512
3513
void CodeGenFunction::EmitOMPDistributeDirective(
3514
90
    const OMPDistributeDirective &S) {
3515
90
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3516
90
    CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
3517
90
  };
3518
90
  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3519
90
  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3520
90
}
3521
3522
static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3523
4
                                                   const CapturedStmt *S) {
3524
4
  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3525
4
  CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
3526
4
  CGF.CapturedStmtInfo = &CapStmtInfo;
3527
4
  llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3528
4
  Fn->setDoesNotRecurse();
3529
4
  return Fn;
3530
4
}
3531
3532
24
void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
3533
24
  if (S.hasClausesOfKind<OMPDependClause>()) {
3534
12
    assert(!S.getAssociatedStmt() &&
3535
12
           "No associated statement must be in ordered depend construct.");
3536
12
    for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3537
14
      CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3538
12
    return;
3539
12
  }
3540
12
  const auto *C = S.getSingleClause<OMPSIMDClause>();
3541
12
  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3542
12
                                 PrePostActionTy &Action) {
3543
12
    const CapturedStmt *CS = S.getInnermostCapturedStmt();
3544
12
    if (C) {
3545
4
      llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3546
4
      CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3547
4
      llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3548
4
      CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3549
4
                                                      OutlinedFn, CapturedVars);
3550
8
    } else {
3551
8
      Action.Enter(CGF);
3552
8
      CGF.EmitStmt(CS->getCapturedStmt());
3553
8
    }
3554
12
  };
3555
12
  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3556
12
  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
3557
12
}
3558
3559
static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
3560
                                         QualType SrcType, QualType DestType,
3561
237
                                         SourceLocation Loc) {
3562
237
  assert(CGF.hasScalarEvaluationKind(DestType) &&
3563
237
         "DestType must have scalar evaluation kind.");
3564
237
  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3565
237
  return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3566
233
                                                   DestType, Loc)
3567
237
                        : CGF.EmitComplexToScalarConversion(
3568
4
                              Val.getComplexVal(), SrcType, DestType, Loc);
3569
237
}
3570
3571
static CodeGenFunction::ComplexPairTy
3572
convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
3573
24
                      QualType DestType, SourceLocation Loc) {
3574
24
  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3575
24
         "DestType must have complex evaluation kind.");
3576
24
  CodeGenFunction::ComplexPairTy ComplexVal;
3577
24
  if (Val.isScalar()) {
3578
6
    // Convert the input element to the element type of the complex.
3579
6
    QualType DestElementType =
3580
6
        DestType->castAs<ComplexType>()->getElementType();
3581
6
    llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3582
6
        Val.getScalarVal(), SrcType, DestElementType, Loc);
3583
6
    ComplexVal = CodeGenFunction::ComplexPairTy(
3584
6
        ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3585
18
  } else {
3586
18
    assert(Val.isComplex() && "Must be a scalar or complex.");
3587
18
    QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3588
18
    QualType DestElementType =
3589
18
        DestType->castAs<ComplexType>()->getElementType();
3590
18
    ComplexVal.first = CGF.EmitScalarConversion(
3591
18
        Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3592
18
    ComplexVal.second = CGF.EmitScalarConversion(
3593
18
        Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3594
18
  }
3595
24
  return ComplexVal;
3596
24
}
3597
3598
static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3599
104
                                  LValue LVal, RValue RVal) {
3600
104
  if (LVal.isGlobalReg()) {
3601
0
    CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3602
104
  } else {
3603
104
    CGF.EmitAtomicStore(RVal, LVal,
3604
104
                        IsSeqCst ? 
llvm::AtomicOrdering::SequentiallyConsistent8
3605
104
                                 : 
llvm::AtomicOrdering::Monotonic96
,
3606
104
                        LVal.isVolatile(), /*isInit=*/false);
3607
104
  }
3608
104
}
3609
3610
void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
3611
259
                                         QualType RValTy, SourceLocation Loc) {
3612
259
  switch (getEvaluationKind(LVal.getType())) {
3613
259
  case TEK_Scalar:
3614
235
    EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3615
235
                               *this, RVal, RValTy, LVal.getType(), Loc)),
3616
235
                           LVal);
3617
235
    break;
3618
259
  case TEK_Complex:
3619
24
    EmitStoreOfComplex(
3620
24
        convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3621
24
        /*isInit=*/false);
3622
24
    break;
3623
259
  case TEK_Aggregate:
3624
0
    llvm_unreachable("Must be a scalar or complex.");
3625
259
  }
3626
259
}
3627
3628
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3629
                                  const Expr *X, const Expr *V,
3630
102
                                  SourceLocation Loc) {
3631
102
  // v = x;
3632
102
  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3633
102
  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3634
102
  LValue XLValue = CGF.EmitLValue(X);
3635
102
  LValue VLValue = CGF.EmitLValue(V);
3636
102
  RValue Res = XLValue.isGlobalReg()
3637
102
                   ? 
CGF.EmitLoadOfLValue(XLValue, Loc)2
3638
102
                   : CGF.EmitAtomicLoad(
3639
100
                         XLValue, Loc,
3640
100
                         IsSeqCst ? 
llvm::AtomicOrdering::SequentiallyConsistent6
3641
100
                                  : 
llvm::AtomicOrdering::Monotonic94
,
3642
100
                         XLValue.isVolatile());
3643
102
  // OpenMP, 2.12.6, atomic Construct
3644
102
  // Any atomic construct with a seq_cst clause forces the atomically
3645
102
  // performed operation to include an implicit flush operation without a
3646
102
  // list.
3647
102
  if (IsSeqCst)
3648
8
    CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3649
102
  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3650
102
}
3651
3652
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3653
                                   const Expr *X, const Expr *E,
3654
104
                                   SourceLocation Loc) {
3655
104
  // x = expr;
3656
104
  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3657
104
  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3658
104
  // OpenMP, 2.12.6, atomic Construct
3659
104
  // Any atomic construct with a seq_cst clause forces the atomically
3660
104
  // performed operation to include an implicit flush operation without a
3661
104
  // list.
3662
104
  if (IsSeqCst)
3663
8
    CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3664
104
}
3665
3666
static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3667
                                                RValue Update,
3668
                                                BinaryOperatorKind BO,
3669
                                                llvm::AtomicOrdering AO,
3670
504
                                                bool IsXLHSInRHSPart) {
3671
504
  ASTContext &Context = CGF.getContext();
3672
504
  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3673
504
  // expression is simple and atomic is allowed for the given type for the
3674
504
  // target platform.
3675
504
  if (BO == BO_Comma || !Update.isScalar() ||
3676
504
      
!Update.getScalarVal()->getType()->isIntegerTy()478
||
3677
504
      
!X.isSimple()371
||
(367
!isa<llvm::ConstantInt>(Update.getScalarVal())367
&&
3678
367
                        (Update.getScalarVal()->getType() !=
3679
343
                         X.getAddress().getElementType())) ||
3680
504
      
!X.getAddress().getElementType()->isIntegerTy()335
||
3681
504
      !Context.getTargetInfo().hasBuiltinAtomic(
3682
333
          Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3683
171
    return std::make_pair(false, RValue::get(nullptr));
3684
333
3685
333
  llvm::AtomicRMWInst::BinOp RMWOp;
3686
333
  switch (BO) {
3687
333
  case BO_Add:
3688
244
    RMWOp = llvm::AtomicRMWInst::Add;
3689
244
    break;
3690
333
  case BO_Sub:
3691
12
    if (!IsXLHSInRHSPart)
3692
0
      return std::make_pair(false, RValue::get(nullptr));
3693
12
    RMWOp = llvm::AtomicRMWInst::Sub;
3694
12
    break;
3695
15
  case BO_And:
3696
15
    RMWOp = llvm::AtomicRMWInst::And;
3697
15
    break;
3698
14
  case BO_Or:
3699
14
    RMWOp = llvm::AtomicRMWInst::Or;
3700
14
    break;
3701
12
  case BO_Xor:
3702
4
    RMWOp = llvm::AtomicRMWInst::Xor;
3703
4
    break;
3704
12
  case BO_LT:
3705
6
    RMWOp = X.getType()->hasSignedIntegerRepresentation()
3706
6
                ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3707
6
                                   : 
llvm::AtomicRMWInst::Max0
)
3708
6
                : 
(IsXLHSInRHSPart 0
?
llvm::AtomicRMWInst::UMin0
3709
0
                                   : llvm::AtomicRMWInst::UMax);
3710
6
    break;
3711
12
  case BO_GT:
3712
0
    RMWOp = X.getType()->hasSignedIntegerRepresentation()
3713
0
                ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3714
0
                                   : llvm::AtomicRMWInst::Min)
3715
0
                : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3716
0
                                   : llvm::AtomicRMWInst::UMin);
3717
0
    break;
3718
12
  case BO_Assign:
3719
2
    RMWOp = llvm::AtomicRMWInst::Xchg;
3720
2
    break;
3721
36
  case BO_Mul:
3722
36
  case BO_Div:
3723
36
  case BO_Rem:
3724
36
  case BO_Shl:
3725
36
  case BO_Shr:
3726
36
  case BO_LAnd:
3727
36
  case BO_LOr:
3728
36
    return std::make_pair(false, RValue::get(nullptr));
3729
36
  case BO_PtrMemD:
3730
0
  case BO_PtrMemI:
3731
0
  case BO_LE:
3732
0
  case BO_GE:
3733
0
  case BO_EQ:
3734
0
  case BO_NE:
3735
0
  case BO_Cmp:
3736
0
  case BO_AddAssign:
3737
0
  case BO_SubAssign:
3738
0
  case BO_AndAssign:
3739
0
  case BO_OrAssign:
3740
0
  case BO_XorAssign:
3741
0
  case BO_MulAssign:
3742
0
  case BO_DivAssign:
3743
0
  case BO_RemAssign:
3744
0
  case BO_ShlAssign:
3745
0
  case BO_ShrAssign:
3746
0
  case BO_Comma:
3747
0
    llvm_unreachable("Unsupported atomic update operation");
3748
297
  }
3749
297
  llvm::Value *UpdateVal = Update.getScalarVal();
3750
297
  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3751
22
    UpdateVal = CGF.Builder.CreateIntCast(
3752
22
        IC, X.getAddress().getElementType(),
3753
22
        X.getType()->hasSignedIntegerRepresentation());
3754
22
  }
3755
297
  llvm::Value *Res =
3756
297
      CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3757
297
  return std::make_pair(true, RValue::get(Res));
3758
297
}
3759
3760
std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
3761
    LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3762
    llvm::AtomicOrdering AO, SourceLocation Loc,
3763
504
    const llvm::function_ref<RValue(RValue)> CommonGen) {
3764
504
  // Update expressions are allowed to have the following forms:
3765
504
  // x binop= expr; -> xrval + expr;
3766
504
  // x++, ++x -> xrval + 1;
3767
504
  // x--, --x -> xrval - 1;
3768
504
  // x = x binop expr; -> xrval binop expr
3769
504
  // x = expr Op x; - > expr binop xrval;
3770
504
  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3771
504
  if (!Res.first) {
3772
207
    if (X.isGlobalReg()) {
3773
4
      // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3774
4
      // 'xrval'.
3775
4
      EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3776
203
    } else {
3777
203
      // Perform compare-and-swap procedure.
3778
203
      EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3779
203
    }
3780
207
  }
3781
504
  return Res;
3782
504
}
3783
3784
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3785
                                    const Expr *X, const Expr *E,
3786
                                    const Expr *UE, bool IsXLHSInRHSPart,
3787
108
                                    SourceLocation Loc) {
3788
108
  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3789
108
         "Update expr in 'atomic update' must be a binary operator.");
3790
108
  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3791
108
  // Update expressions are allowed to have the following forms:
3792
108
  // x binop= expr; -> xrval + expr;
3793
108
  // x++, ++x -> xrval + 1;
3794
108
  // x--, --x -> xrval - 1;
3795
108
  // x = x binop expr; -> xrval binop expr
3796
108
  // x = expr Op x; - > expr binop xrval;
3797
108
  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3798
108
  LValue XLValue = CGF.EmitLValue(X);
3799
108
  RValue ExprRValue = CGF.EmitAnyExpr(E);
3800
108
  llvm::AtomicOrdering AO = IsSeqCst
3801
108
                                ? 
llvm::AtomicOrdering::SequentiallyConsistent8
3802
108
                                : 
llvm::AtomicOrdering::Monotonic100
;
3803
108
  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3804
108
  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3805
108
  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? 
LHS82
:
RHS26
;
3806
108
  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? 
RHS82
:
LHS26
;
3807
108
  auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3808
78
    CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3809
78
    CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3810
78
    return CGF.EmitAnyExpr(UE);
3811
78
  };
3812
108
  (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
3813
108
      XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3814
108
  // OpenMP, 2.12.6, atomic Construct
3815
108
  // Any atomic construct with a seq_cst clause forces the atomically
3816
108
  // performed operation to include an implicit flush operation without a
3817
108
  // list.
3818
108
  if (IsSeqCst)
3819
8
    CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3820
108
}
3821
3822
static RValue convertToType(CodeGenFunction &CGF, RValue Value,
3823
                            QualType SourceType, QualType ResType,
3824
2
                            SourceLocation Loc) {
3825
2
  switch (CGF.getEvaluationKind(ResType)) {
3826
2
  case TEK_Scalar:
3827
2
    return RValue::get(
3828
2
        convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3829
2
  case TEK_Complex: {
3830
0
    auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3831
0
    return RValue::getComplex(Res.first, Res.second);
3832
2
  }
3833
2
  case TEK_Aggregate:
3834
0
    break;
3835
0
  }
3836
0
  llvm_unreachable("Must be a scalar or complex.");
3837
0
}
3838
3839
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3840
                                     bool IsPostfixUpdate, const Expr *V,
3841
                                     const Expr *X, const Expr *E,
3842
                                     const Expr *UE, bool IsXLHSInRHSPart,
3843
104
                                     SourceLocation Loc) {
3844
104
  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3845
104
  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3846
104
  RValue NewVVal;
3847
104
  LValue VLValue = CGF.EmitLValue(V);
3848
104
  LValue XLValue = CGF.EmitLValue(X);
3849
104
  RValue ExprRValue = CGF.EmitAnyExpr(E);
3850
104
  llvm::AtomicOrdering AO = IsSeqCst
3851
104
                                ? 
llvm::AtomicOrdering::SequentiallyConsistent8
3852
104
                                : 
llvm::AtomicOrdering::Monotonic96
;
3853
104
  QualType NewVValType;
3854
104
  if (UE) {
3855
102
    // 'x' is updated with some additional value.
3856
102
    assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3857
102
           "Update expr in 'atomic capture' must be a binary operator.");
3858
102
    const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3859
102
    // Update expressions are allowed to have the following forms:
3860
102
    // x binop= expr; -> xrval + expr;
3861
102
    // x++, ++x -> xrval + 1;
3862
102
    // x--, --x -> xrval - 1;
3863
102
    // x = x binop expr; -> xrval binop expr
3864
102
    // x = expr Op x; - > expr binop xrval;
3865
102
    const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3866
102
    const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3867
102
    const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? 
LHS76
:
RHS26
;
3868
102
    NewVValType = XRValExpr->getType();
3869
102
    const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? 
RHS76
:
LHS26
;
3870
102
    auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3871
102
                  IsPostfixUpdate](RValue XRValue) {
3872
76
      CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3873
76
      CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3874
76
      RValue Res = CGF.EmitAnyExpr(UE);
3875
76
      NewVVal = IsPostfixUpdate ? 
XRValue24
:
Res52
;
3876
76
      return Res;
3877
76
    };
3878
102
    auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3879
102
        XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3880
102
    if (Res.first) {
3881
26
      // 'atomicrmw' instruction was generated.
3882
26
      if (IsPostfixUpdate) {
3883
10
        // Use old value from 'atomicrmw'.
3884
10
        NewVVal = Res.second;
3885
16
      } else {
3886
16
        // 'atomicrmw' does not provide new value, so evaluate it using old
3887
16
        // value of 'x'.
3888
16
        CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3889
16
        CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3890
16
        NewVVal = CGF.EmitAnyExpr(UE);
3891
16
      }
3892
26
    }
3893
102
  } else {
3894
2
    // 'x' is simply rewritten with some 'expr'.
3895
2
    NewVValType = X->getType().getNonReferenceType();
3896
2
    ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3897
2
                               X->getType().getNonReferenceType(), Loc);
3898
2
    auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
3899
0
      NewVVal = XRValue;
3900
0
      return ExprRValue;
3901
0
    };
3902
2
    // Try to perform atomicrmw xchg, otherwise simple exchange.
3903
2
    auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3904
2
        XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3905
2
        Loc, Gen);
3906
2
    if (Res.first) {
3907
2
      // 'atomicrmw' instruction was generated.
3908
2
      NewVVal = IsPostfixUpdate ? Res.second : 
ExprRValue0
;
3909
2
    }
3910
2
  }
3911
104
  // Emit post-update store to 'v' of old/new 'x' value.
3912
104
  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3913
104
  // OpenMP, 2.12.6, atomic Construct
3914
104
  // Any atomic construct with a seq_cst clause forces the atomically
3915
104
  // performed operation to include an implicit flush operation without a
3916
104
  // list.
3917
104
  if (IsSeqCst)
3918
8
    CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3919
104
}
3920
3921
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
3922
                              bool IsSeqCst, bool IsPostfixUpdate,
3923
                              const Expr *X, const Expr *V, const Expr *E,
3924
                              const Expr *UE, bool IsXLHSInRHSPart,
3925
418
                              SourceLocation Loc) {
3926
418
  switch (Kind) {
3927
418
  case OMPC_read:
3928
102
    emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3929
102
    break;
3930
418
  case OMPC_write:
3931
104
    emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3932
104
    break;
3933
418
  case OMPC_unknown:
3934
108
  case OMPC_update:
3935
108
    emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3936
108
    break;
3937
108
  case OMPC_capture:
3938
104
    emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3939
104
                             IsXLHSInRHSPart, Loc);
3940
104
    break;
3941
108
  case OMPC_if:
3942
0
  case OMPC_final:
3943
0
  case OMPC_num_threads:
3944
0
  case OMPC_private:
3945
0
  case OMPC_firstprivate:
3946
0
  case OMPC_lastprivate:
3947
0
  case OMPC_reduction:
3948
0
  case OMPC_task_reduction:
3949
0
  case OMPC_in_reduction:
3950
0
  case OMPC_safelen:
3951
0
  case OMPC_simdlen:
3952
0
  case OMPC_allocator:
3953
0
  case OMPC_allocate:
3954
0
  case OMPC_collapse:
3955
0
  case OMPC_default:
3956
0
  case OMPC_seq_cst:
3957
0
  case OMPC_shared:
3958
0
  case OMPC_linear:
3959
0
  case OMPC_aligned:
3960
0
  case OMPC_copyin:
3961
0
  case OMPC_copyprivate:
3962
0
  case OMPC_flush:
3963
0
  case OMPC_proc_bind:
3964
0
  case OMPC_schedule:
3965
0
  case OMPC_ordered:
3966
0
  case OMPC_nowait:
3967
0
  case OMPC_untied:
3968
0
  case OMPC_threadprivate:
3969
0
  case OMPC_depend:
3970
0
  case OMPC_mergeable:
3971
0
  case OMPC_device:
3972
0
  case OMPC_threads:
3973
0
  case OMPC_simd:
3974
0
  case OMPC_map:
3975
0
  case OMPC_num_teams:
3976
0
  case OMPC_thread_limit:
3977
0
  case OMPC_priority:
3978
0
  case OMPC_grainsize:
3979
0
  case OMPC_nogroup:
3980
0
  case OMPC_num_tasks:
3981
0
  case OMPC_hint:
3982
0
  case OMPC_dist_schedule:
3983
0
  case OMPC_defaultmap:
3984
0
  case OMPC_uniform:
3985
0
  case OMPC_to:
3986
0
  case OMPC_from:
3987
0
  case OMPC_use_device_ptr:
3988
0
  case OMPC_is_device_ptr:
3989
0
  case OMPC_unified_address:
3990
0
  case OMPC_unified_shared_memory:
3991
0
  case OMPC_reverse_offload:
3992
0
  case OMPC_dynamic_allocators:
3993
0
  case OMPC_atomic_default_mem_order:
3994
0
    llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3995
418
  }
3996
418
}
3997
3998
418
void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
3999
418
  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
4000
418
  OpenMPClauseKind Kind = OMPC_unknown;
4001
418
  for (const OMPClause *C : S.clauses()) {
4002
376
    // Find first clause (skip seq_cst clause, if it is first).
4003
376
    if (C->getClauseKind() != OMPC_seq_cst) {
4004
360
      Kind = C->getClauseKind();
4005
360
      break;
4006
360
    }
4007
376
  }
4008
418
4009
418
  const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
4010
418
  if (const auto *FE = dyn_cast<FullExpr>(CS))
4011
0
    enterFullExpression(FE);
4012
418
  // Processing for statements under 'atomic capture'.
4013
418
  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
4014
0
    for (const Stmt *C : Compound->body()) {
4015
0
      if (const auto *FE = dyn_cast<FullExpr>(C))
4016
0
        enterFullExpression(FE);
4017
0
    }
4018
0
  }
4019
418
4020
418
  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
4021
418
                                            PrePostActionTy &) {
4022
418
    CGF.EmitStopPoint(CS);
4023
418
    emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
4024
418
                      S.getV(), S.getExpr(), S.getUpdateExpr(),
4025
418
                      S.isXLHSInRHSPart(), S.getBeginLoc());
4026
418
  };
4027
418
  OMPLexicalScope Scope(*this, S, OMPD_unknown);
4028
418
  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
4029
418
}
4030
4031
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
4032
                                         const OMPExecutableDirective &S,
4033
4.59k
                                         const RegionCodeGenTy &CodeGen) {
4034
4.59k
  assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
4035
4.59k
  CodeGenModule &CGM = CGF.CGM;
4036
4.59k
4037
4.59k
  // On device emit this construct as inlined code.
4038
4.59k
  if (CGM.getLangOpts().OpenMPIsDevice) {
4039
3
    OMPLexicalScope Scope(CGF, S, OMPD_target);
4040
3
    CGM.getOpenMPRuntime().emitInlinedDirective(
4041
3
        CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4042
3
          CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4043
3
        });
4044
3
    return;
4045
3
  }
4046
4.59k
4047
4.59k
  llvm::Function *Fn = nullptr;
4048
4.59k
  llvm::Constant *FnID = nullptr;
4049
4.59k
4050
4.59k
  const Expr *IfCond = nullptr;
4051
4.59k
  // Check for the at most one if clause associated with the target region.
4052
4.59k
  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4053
639
    if (C->getNameModifier() == OMPD_unknown ||
4054
639
        
C->getNameModifier() == OMPD_target433
) {
4055
613
      IfCond = C->getCondition();
4056
613
      break;
4057
613
    }
4058
639
  }
4059
4.59k
4060
4.59k
  // Check if we have any device clause associated with the directive.
4061
4.59k
  const Expr *Device = nullptr;
4062
4.59k
  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4063
134
    Device = C->getDevice();
4064
4.59k
4065
4.59k
  // Check if we have an if clause whose conditional always evaluates to false
4066
4.59k
  // or if we do not have any targets specified. If so the target region is not
4067
4.59k
  // an offload entry point.
4068
4.59k
  bool IsOffloadEntry = true;
4069
4.59k
  if (IfCond) {
4070
613
    bool Val;
4071
613
    if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && 
!Val247
)
4072
187
      IsOffloadEntry = false;
4073
613
  }
4074
4.59k
  if (CGM.getLangOpts().OMPTargetTriples.empty())
4075
121
    IsOffloadEntry = false;
4076
4.59k
4077
4.59k
  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
4078
4.59k
  StringRef ParentName;
4079
4.59k
  // In case we have Ctors/Dtors we use the complete type variant to produce
4080
4.59k
  // the mangling of the device outlined kernel.
4081
4.59k
  if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
4082
280
    ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
4083
4.31k
  else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
4084
252
    ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
4085
4.06k
  else
4086
4.06k
    ParentName =
4087
4.06k
        CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4088
4.59k
4089
4.59k
  // Emit target region as a standalone region.
4090
4.59k
  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4091
4.59k
                                                    IsOffloadEntry, CodeGen);
4092
4.59k
  OMPLexicalScope Scope(CGF, S, OMPD_task);
4093
4.59k
  auto &&SizeEmitter = [](CodeGenFunction &CGF, const OMPLoopDirective &D) {
4094
2.24k
    OMPLoopScope(CGF, D);
4095
2.24k
    // Emit calculation of the iterations count.
4096
2.24k
    llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
4097
2.24k
    NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
4098
2.24k
                                              /*isSigned=*/false);
4099
2.24k
    return NumIterations;
4100
2.24k
  };
4101
4.59k
  if (IsOffloadEntry)
4102
4.29k
    CGM.getOpenMPRuntime().emitTargetNumIterationsCall(CGF, S, Device,
4103
4.29k
                                                       SizeEmitter);
4104
4.59k
  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
4105
4.59k
}
4106
4107
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
4108
2.76k
                             PrePostActionTy &Action) {
4109
2.76k
  Action.Enter(CGF);
4110
2.76k
  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4111
2.76k
  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4112
2.76k
  CGF.EmitOMPPrivateClause(S, PrivateScope);
4113
2.76k
  (void)PrivateScope.Privatize();
4114
2.76k
  if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4115
2.76k
    CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4116
2.76k
4117
2.76k
  CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4118
2.76k
}
4119
4120
void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
4121
                                                  StringRef ParentName,
4122
484
                                                  const OMPTargetDirective &S) {
4123
484
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4124
484
    emitTargetRegion(CGF, S, Action);
4125
484
  };
4126
484
  llvm::Function *Fn;
4127
484
  llvm::Constant *Addr;
4128
484
  // Emit target region as a standalone region.
4129
484
  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4130
484
      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4131
484
  assert(Fn && Addr && "Target device function emission failed.");
4132
484
}
4133
4134
2.28k
void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
4135
2.28k
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4136
2.28k
    emitTargetRegion(CGF, S, Action);
4137
2.28k
  };
4138
2.28k
  emitCommonOMPTargetDirective(*this, S, CodeGen);
4139
2.28k
}
4140
4141
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
4142
                                        const OMPExecutableDirective &S,
4143
                                        OpenMPDirectiveKind InnermostKind,
4144
3.55k
                                        const RegionCodeGenTy &CodeGen) {
4145
3.55k
  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4146
3.55k
  llvm::Function *OutlinedFn =
4147
3.55k
      CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4148
3.55k
          S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4149
3.55k
4150
3.55k
  const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4151
3.55k
  const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4152
3.55k
  if (NT || 
TL3.32k
) {
4153
271
    const Expr *NumTeams = NT ? 
NT->getNumTeams()227
:
nullptr44
;
4154
271
    const Expr *ThreadLimit = TL ? 
TL->getThreadLimit()207
:
nullptr64
;
4155
271
4156
271
    CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4157
271
                                                  S.getBeginLoc());
4158
271
  }
4159
3.55k
4160
3.55k
  OMPTeamsScope Scope(CGF, S);
4161
3.55k
  llvm::SmallVector<llvm::Value *, 16> CapturedVars;
4162
3.55k
  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4163
3.55k
  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
4164
3.55k
                                           CapturedVars);
4165
3.55k
}
4166
4167
745
void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
4168
745
  // Emit teams region as a standalone region.
4169
745
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4170
745
    Action.Enter(CGF);
4171
745
    OMPPrivateScope PrivateScope(CGF);
4172
745
    (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4173
745
    CGF.EmitOMPPrivateClause(S, PrivateScope);
4174
745
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4175
745
    (void)PrivateScope.Privatize();
4176
745
    CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4177
745
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4178
745
  };
4179
745
  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4180
745
  emitPostUpdateForReductionClause(*this, S,
4181
745
                                   [](CodeGenFunction &) 
{ return nullptr; }0
);
4182
745
}
4183
4184
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4185
565
                                  const OMPTargetTeamsDirective &S) {
4186
565
  auto *CS = S.getCapturedStmt(OMPD_teams);
4187
565
  Action.Enter(CGF);
4188
565
  // Emit teams region as a standalone region.
4189
565
  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4190
565
    Action.Enter(CGF);
4191
565
    CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4192
565
    (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4193
565
    CGF.EmitOMPPrivateClause(S, PrivateScope);
4194
565
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4195
565
    (void)PrivateScope.Privatize();
4196
565
    if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4197
565
      CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4198
565
    CGF.EmitStmt(CS->getCapturedStmt());
4199
565
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4200
565
  };
4201
565
  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4202
565
  emitPostUpdateForReductionClause(CGF, S,
4203
565
                                   [](CodeGenFunction &) 
{ return nullptr; }0
);
4204
565
}
4205
4206
void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
4207
    CodeGenModule &CGM, StringRef ParentName,
4208
244
    const OMPTargetTeamsDirective &S) {
4209
244
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4210
244
    emitTargetTeamsRegion(CGF, Action, S);
4211
244
  };
4212
244
  llvm::Function *Fn;
4213
244
  llvm::Constant *Addr;
4214
244
  // Emit target region as a standalone region.
4215
244
  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4216
244
      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4217
244
  assert(Fn && Addr && "Target device function emission failed.");
4218
244
}
4219
4220
void CodeGenFunction::EmitOMPTargetTeamsDirective(
4221
321
    const OMPTargetTeamsDirective &S) {
4222
321
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4223
321
    emitTargetTeamsRegion(CGF, Action, S);
4224
321
  };
4225
321
  emitCommonOMPTargetDirective(*this, S, CodeGen);
4226
321
}
4227
4228
static void
4229
emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4230
356
                                const OMPTargetTeamsDistributeDirective &S) {
4231
356
  Action.Enter(CGF);
4232
356
  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4233
356
    CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4234
356
  };
4235
356
4236
356
  // Emit teams region as a standalone region.
4237
356
  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4238
356
                                            PrePostActionTy &Action) {
4239
356
    Action.Enter(CGF);
4240
356
    CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4241
356
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4242
356
    (void)PrivateScope.Privatize();
4243
356
    CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4244
356
                                                    CodeGenDistribute);
4245
356
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4246
356
  };
4247
356
  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4248
356
  emitPostUpdateForReductionClause(CGF, S,
4249
356
                                   [](CodeGenFunction &) 
{ return nullptr; }0
);
4250
356
}
4251
4252
void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
4253
    CodeGenModule &CGM, StringRef ParentName,
4254
95
    const OMPTargetTeamsDistributeDirective &S) {
4255
95
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4256
95
    emitTargetTeamsDistributeRegion(CGF, Action, S);
4257
95
  };
4258
95
  llvm::Function *Fn;
4259
95
  llvm::Constant *Addr;
4260
95
  // Emit target region as a standalone region.
4261
95
  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4262
95
      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4263
95
  assert(Fn && Addr && "Target device function emission failed.");
4264
95
}
4265
4266
void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
4267
261
    const OMPTargetTeamsDistributeDirective &S) {
4268
261
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4269
261
    emitTargetTeamsDistributeRegion(CGF, Action, S);
4270
261
  };
4271
261
  emitCommonOMPTargetDirective(*this, S, CodeGen);
4272
261
}
4273
4274
static void emitTargetTeamsDistributeSimdRegion(
4275
    CodeGenFunction &CGF, PrePostActionTy &Action,
4276
371
    const OMPTargetTeamsDistributeSimdDirective &S) {
4277
371
  Action.Enter(CGF);
4278
371
  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4279
371
    CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4280
371
  };
4281
371
4282
371
  // Emit teams region as a standalone region.
4283
371
  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4284
371
                                            PrePostActionTy &Action) {
4285
371
    Action.Enter(CGF);
4286
371
    CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4287
371
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4288
371
    (void)PrivateScope.Privatize();
4289
371
    CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4290
371
                                                    CodeGenDistribute);
4291
371
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4292
371
  };
4293
371
  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4294
371
  emitPostUpdateForReductionClause(CGF, S,
4295
371
                                   [](CodeGenFunction &) 
{ return nullptr; }0
);
4296
371
}
4297
4298
void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4299
    CodeGenModule &CGM, StringRef ParentName,
4300
104
    const OMPTargetTeamsDistributeSimdDirective &S) {
4301
104
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4302
104
    emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4303
104
  };
4304
104
  llvm::Function *Fn;
4305
104
  llvm::Constant *Addr;
4306
104
  // Emit target region as a standalone region.
4307
104
  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4308
104
      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4309
104
  assert(Fn && Addr && "Target device function emission failed.");
4310
104
}
4311
4312
void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
4313
267
    const OMPTargetTeamsDistributeSimdDirective &S) {
4314
267
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4315
267
    emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4316
267
  };
4317
267
  emitCommonOMPTargetDirective(*this, S, CodeGen);
4318
267
}
4319
4320
void CodeGenFunction::EmitOMPTeamsDistributeDirective(
4321
114
    const OMPTeamsDistributeDirective &S) {
4322
114
4323
114
  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4324
114
    CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4325
114
  };
4326
114
4327
114
  // Emit teams region as a standalone region.
4328
114
  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4329
114
                                            PrePostActionTy &Action) {
4330
114
    Action.Enter(CGF);
4331
114
    OMPPrivateScope PrivateScope(CGF);
4332
114
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4333
114
    (void)PrivateScope.Privatize();
4334
114
    CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4335
114
                                                    CodeGenDistribute);
4336
114
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4337
114
  };
4338
114
  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4339
114
  emitPostUpdateForReductionClause(*this, S,
4340
114
                                   [](CodeGenFunction &) 
{ return nullptr; }0
);
4341
114
}
4342
4343
void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
4344
114
    const OMPTeamsDistributeSimdDirective &S) {
4345
114
  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4346
114
    CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4347
114
  };
4348
114
4349
114
  // Emit teams region as a standalone region.
4350
114
  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4351
114
                                            PrePostActionTy &Action) {
4352
114
    Action.Enter(CGF);
4353
114
    OMPPrivateScope PrivateScope(CGF);
4354
114
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4355
114
    (void)PrivateScope.Privatize();
4356
114
    CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4357
114
                                                    CodeGenDistribute);
4358
114
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4359
114
  };
4360
114
  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4361
114
  emitPostUpdateForReductionClause(*this, S,
4362
114
                                   [](CodeGenFunction &) 
{ return nullptr; }0
);
4363
114
}
4364
4365
void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
4366
218
    const OMPTeamsDistributeParallelForDirective &S) {
4367
218
  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4368
218
    CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4369
218
                              S.getDistInc());
4370
218
  };
4371
218
4372
218
  // Emit teams region as a standalone region.
4373
218
  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4374
218
                                            PrePostActionTy &Action) {
4375
218
    Action.Enter(CGF);
4376
218
    OMPPrivateScope PrivateScope(CGF);
4377
218
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4378
218
    (void)PrivateScope.Privatize();
4379
218
    CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4380
218
                                                    CodeGenDistribute);
4381
218
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4382
218
  };
4383
218
  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4384
218
  emitPostUpdateForReductionClause(*this, S,
4385
218
                                   [](CodeGenFunction &) 
{ return nullptr; }0
);
4386
218
}
4387
4388
void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
4389
208
    const OMPTeamsDistributeParallelForSimdDirective &S) {
4390
208
  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4391
208
    CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4392
208
                              S.getDistInc());
4393
208
  };
4394
208
4395
208
  // Emit teams region as a standalone region.
4396
208
  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4397
208
                                            PrePostActionTy &Action) {
4398
208
    Action.Enter(CGF);
4399
208
    OMPPrivateScope PrivateScope(CGF);
4400
208
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4401
208
    (void)PrivateScope.Privatize();
4402
208
    CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4403
208
        CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4404
208
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4405
208
  };
4406
208
  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4407
208
  emitPostUpdateForReductionClause(*this, S,
4408
208
                                   [](CodeGenFunction &) 
{ return nullptr; }0
);
4409
208
}
4410
4411
static void emitTargetTeamsDistributeParallelForRegion(
4412
    CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
4413
367
    PrePostActionTy &Action) {
4414
367
  Action.Enter(CGF);
4415
367
  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4416
367
    CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4417
367
                              S.getDistInc());
4418
367
  };
4419
367
4420
367
  // Emit teams region as a standalone region.
4421
367
  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4422
367
                                                 PrePostActionTy &Action) {
4423
367
    Action.Enter(CGF);
4424
367
    CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4425
367
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4426
367
    (void)PrivateScope.Privatize();
4427
367
    CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4428
367
        CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4429
367
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4430
367
  };
4431
367
4432
367
  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4433
367
                              CodeGenTeams);
4434
367
  emitPostUpdateForReductionClause(CGF, S,
4435
367
                                   [](CodeGenFunction &) 
{ return nullptr; }0
);
4436
367
}
4437
4438
void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
4439
    CodeGenModule &CGM, StringRef ParentName,
4440
103
    const OMPTargetTeamsDistributeParallelForDirective &S) {
4441
103
  // Emit SPMD target teams distribute parallel for region as a standalone
4442
103
  // region.
4443
103
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4444
103
    emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
4445
103
  };
4446
103
  llvm::Function *Fn;
4447
103
  llvm::Constant *Addr;
4448
103
  // Emit target region as a standalone region.
4449
103
  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4450
103
      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4451
103
  assert(Fn && Addr && "Target device function emission failed.");
4452
103
}
4453
4454
void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
4455
264
    const OMPTargetTeamsDistributeParallelForDirective &S) {
4456
264
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4457
264
    emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
4458
264
  };
4459
264
  emitCommonOMPTargetDirective(*this, S, CodeGen);
4460
264
}
4461
4462
static void emitTargetTeamsDistributeParallelForSimdRegion(
4463
    CodeGenFunction &CGF,
4464
    const OMPTargetTeamsDistributeParallelForSimdDirective &S,
4465
495
    PrePostActionTy &Action) {
4466
495
  Action.Enter(CGF);
4467
495
  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4468
495
    CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4469
495
                              S.getDistInc());
4470
495
  };
4471
495
4472
495
  // Emit teams region as a standalone region.
4473
495
  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4474
495
                                                 PrePostActionTy &Action) {
4475
495
    Action.Enter(CGF);
4476
495
    CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4477
495
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4478
495
    (void)PrivateScope.Privatize();
4479
495
    CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4480
495
        CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4481
495
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4482
495
  };
4483
495
4484
495
  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4485
495
                              CodeGenTeams);
4486
495
  emitPostUpdateForReductionClause(CGF, S,
4487
495
                                   [](CodeGenFunction &) 
{ return nullptr; }0
);
4488
495
}
4489
4490
void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
4491
    CodeGenModule &CGM, StringRef ParentName,
4492
144
    const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
4493
144
  // Emit SPMD target teams distribute parallel for simd region as a standalone
4494
144
  // region.
4495
144
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4496
144
    emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
4497
144
  };
4498
144
  llvm::Function *Fn;
4499
144
  llvm::Constant *Addr;
4500
144
  // Emit target region as a standalone region.
4501
144
  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4502
144
      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4503
144
  assert(Fn && Addr && "Target device function emission failed.");
4504
144
}
4505
4506
void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
4507
351
    const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
4508
351
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4509
351
    emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
4510
351
  };
4511
351
  emitCommonOMPTargetDirective(*this, S, CodeGen);
4512
351
}
4513
4514
void CodeGenFunction::EmitOMPCancellationPointDirective(
4515
32
    const OMPCancellationPointDirective &S) {
4516
32
  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
4517
32
                                                   S.getCancelRegion());
4518
32
}
4519
4520
66
void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
4521
66
  const Expr *IfCond = nullptr;
4522
66
  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4523
4
    if (C->getNameModifier() == OMPD_unknown ||
4524
4
        
C->getNameModifier() == OMPD_cancel2
) {
4525
4
      IfCond = C->getCondition();
4526
4
      break;
4527
4
    }
4528
4
  }
4529
66
  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
4530
66
                                        S.getCancelRegion());
4531
66
}
4532
4533
CodeGenFunction::JumpDest
4534
100
CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
4535
100
  if (Kind == OMPD_parallel || 
Kind == OMPD_task92
||
4536
100
      
Kind == OMPD_target_parallel84
)
4537
16
    return ReturnBlock;
4538
84
  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4539
84
         Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4540
84
         Kind == OMPD_distribute_parallel_for ||
4541
84
         Kind == OMPD_target_parallel_for ||
4542
84
         Kind == OMPD_teams_distribute_parallel_for ||
4543
84
         Kind == OMPD_target_teams_distribute_parallel_for);
4544
84
  return OMPCancelStack.getExitBlock();
4545
84
}
4546
4547
void CodeGenFunction::EmitOMPUseDevicePtrClause(
4548
    const OMPClause &NC, OMPPrivateScope &PrivateScope,
4549
68
    const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4550
68
  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4551
68
  auto OrigVarIt = C.varlist_begin();
4552
68
  auto InitIt = C.inits().begin();
4553
76
  for (const Expr *PvtVarIt : C.private_copies()) {
4554
76
    const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4555
76
    const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4556
76
    const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4557
76
4558
76
    // In order to identify the right initializer we need to match the
4559
76
    // declaration used by the mapping logic. In some cases we may get
4560
76
    // OMPCapturedExprDecl that refers to the original declaration.
4561
76
    const ValueDecl *MatchingVD = OrigVD;
4562
76
    if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4563
20
      // OMPCapturedExprDecl are used to privative fields of the current
4564
20
      // structure.
4565
20
      const auto *ME = cast<MemberExpr>(OED->getInit());
4566
20
      assert(isa<CXXThisExpr>(ME->getBase()) &&
4567
20
             "Base should be the current struct!");
4568
20
      MatchingVD = ME->getMemberDecl();
4569
20
    }
4570
76
4571
76
    // If we don't have information about the current list item, move on to
4572
76
    // the next one.
4573
76
    auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4574
76
    if (InitAddrIt == CaptureDeviceAddrMap.end())
4575
0
      continue;
4576
76
4577
76
    bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4578
76
                                                         InitAddrIt, InitVD,
4579
76
                                                         PvtVD]() {
4580
76
      // Initialize the temporary initialization variable with the address we
4581
76
      // get from the runtime library. We have to cast the source address
4582
76
      // because it is always a void *. References are materialized in the
4583
76
      // privatization scope, so the initialization here disregards the fact
4584
76
      // the original variable is a reference.
4585
76
      QualType AddrQTy =
4586
76
          getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4587
76
      llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4588
76
      Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4589
76
      setAddrOfLocalVar(InitVD, InitAddr);
4590
76
4591
76
      // Emit private declaration, it will be initialized by the value we
4592
76
      // declaration we just added to the local declarations map.
4593
76
      EmitDecl(*PvtVD);
4594
76
4595
76
      // The initialization variables reached its purpose in the emission
4596
76
      // of the previous declaration, so we don't need it anymore.
4597
76
      LocalDeclMap.erase(InitVD);
4598
76
4599
76
      // Return the address of the private variable.
4600
76
      return GetAddrOfLocalVar(PvtVD);
4601
76
    });
4602
76
    assert(IsRegistered && "firstprivate var already registered as private");
4603
76
    // Silence the warning about unused variable.
4604
76
    (void)IsRegistered;
4605
76
4606
76
    ++OrigVarIt;
4607
76
    ++InitIt;
4608
76
  }
4609
68
}
4610
4611
// Generate the instructions for '#pragma omp target data' directive.
4612
void CodeGenFunction::EmitOMPTargetDataDirective(
4613
95
    const OMPTargetDataDirective &S) {
4614
95
  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4615
95
4616
95
  // Create a pre/post action to signal the privatization of the device pointer.
4617
95
  // This action can be replaced by the OpenMP runtime code generation to
4618
95
  // deactivate privatization.
4619
95
  bool PrivatizeDevicePointers = false;
4620
95
  class DevicePointerPrivActionTy : public PrePostActionTy {
4621
95
    bool &PrivatizeDevicePointers;
4622
95
4623
95
  public:
4624
95
    explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4625
95
        : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4626
95
    void Enter(CodeGenFunction &CGF) override {
4627
64
      PrivatizeDevicePointers = true;
4628
64
    }
4629
95
  };
4630
95
  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4631
95
4632
95
  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4633
99
                       CodeGenFunction &CGF, PrePostActionTy &Action) {
4634
99
    auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4635
99
      CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4636
99
    };
4637
99
4638
99
    // Codegen that selects whether to generate the privatization code or not.
4639
99
    auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4640
99
                          &InnermostCodeGen](CodeGenFunction &CGF,
4641
99
                                             PrePostActionTy &Action) {
4642
99
      RegionCodeGenTy RCG(InnermostCodeGen);
4643
99
      PrivatizeDevicePointers = false;
4644
99
4645
99
      // Call the pre-action to change the status of PrivatizeDevicePointers if
4646
99
      // needed.
4647
99
      Action.Enter(CGF);
4648
99
4649
99
      if (PrivatizeDevicePointers) {
4650
64
        OMPPrivateScope PrivateScope(CGF);
4651
64
        // Emit all instances of the use_device_ptr clause.
4652
64
        for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4653
68
          CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4654
68
                                        Info.CaptureDeviceAddrMap);
4655
64
        (void)PrivateScope.Privatize();
4656
64
        RCG(CGF);
4657
64
      } else {
4658
35
        RCG(CGF);
4659
35
      }
4660
99
    };
4661
99
4662
99
    // Forward the provided action to the privatization codegen.
4663
99
    RegionCodeGenTy PrivRCG(PrivCodeGen);
4664
99
    PrivRCG.setAction(Action);
4665
99
4666
99
    // Notwithstanding the body of the region is emitted as inlined directive,
4667
99
    // we don't use an inline scope as changes in the references inside the
4668
99
    // region are expected to be visible outside, so we do not privative them.
4669
99
    OMPLexicalScope Scope(CGF, S);
4670
99
    CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4671
99
                                                    PrivRCG);
4672
99
  };
4673
95
4674
95
  RegionCodeGenTy RCG(CodeGen);
4675
95
4676
95
  // If we don't have target devices, don't bother emitting the data mapping
4677
95
  // code.
4678
95
  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4679
3
    RCG(*this);
4680
3
    return;
4681
3
  }
4682
92
4683
92
  // Check if we have any if clause associated with the directive.
4684
92
  const Expr *IfCond = nullptr;
4685
92
  if (const auto *C = S.getSingleClause<OMPIfClause>())
4686
28
    IfCond = C->getCondition();
4687
92
4688
92
  // Check if we have any device clause associated with the directive.
4689
92
  const Expr *Device = nullptr;
4690
92
  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4691
12
    Device = C->getDevice();
4692
92
4693
92
  // Set the action to signal privatization of device pointers.
4694
92
  RCG.setAction(PrivAction);
4695
92
4696
92
  // Emit region code.
4697
92
  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4698
92
                                             Info);
4699
92
}
4700
4701
void CodeGenFunction::EmitOMPTargetEnterDataDirective(
4702
53
    const OMPTargetEnterDataDirective &S) {
4703
53
  // If we don't have target devices, don't bother emitting the data mapping
4704
53
  // code.
4705
53
  if (CGM.getLangOpts().OMPTargetTriples.empty())
4706
3
    return;
4707
50
4708
50
  // Check if we have any if clause associated with the directive.
4709
50
  const Expr *IfCond = nullptr;
4710
50
  if (const auto *C = S.getSingleClause<OMPIfClause>())
4711
34
    IfCond = C->getCondition();
4712
50
4713
50
  // Check if we have any device clause associated with the directive.
4714
50
  const Expr *Device = nullptr;
4715
50
  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4716
26
    Device = C->getDevice();
4717
50
4718
50
  OMPLexicalScope Scope(*this, S, OMPD_task);
4719
50
  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4720
50
}
4721
4722
void CodeGenFunction::EmitOMPTargetExitDataDirective(
4723
47
    const OMPTargetExitDataDirective &S) {
4724
47
  // If we don't have target devices, don't bother emitting the data mapping
4725
47
  // code.
4726
47
  if (CGM.getLangOpts().OMPTargetTriples.empty())
4727
3
    return;
4728
44
4729
44
  // Check if we have any if clause associated with the directive.
4730
44
  const Expr *IfCond = nullptr;
4731
44
  if (const auto *C = S.getSingleClause<OMPIfClause>())
4732
28
    IfCond = C->getCondition();
4733
44
4734
44
  // Check if we have any device clause associated with the directive.
4735
44
  const Expr *Device = nullptr;
4736
44
  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4737
20
    Device = C->getDevice();
4738
44
4739
44
  OMPLexicalScope Scope(*this, S, OMPD_task);
4740
44
  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4741
44
}
4742
4743
static void emitTargetParallelRegion(CodeGenFunction &CGF,
4744
                                     const OMPTargetParallelDirective &S,
4745
529
                                     PrePostActionTy &Action) {
4746
529
  // Get the captured statement associated with the 'parallel' region.
4747
529
  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4748
529
  Action.Enter(CGF);
4749
529
  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4750
529
    Action.Enter(CGF);
4751
529
    CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4752
529
    (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4753
529
    CGF.EmitOMPPrivateClause(S, PrivateScope);
4754
529
    CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4755
529
    (void)PrivateScope.Privatize();
4756
529
    if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4757
529
      CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4758
529
    // TODO: Add support for clauses.
4759
529
    CGF.EmitStmt(CS->getCapturedStmt());
4760
529
    CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4761
529
  };
4762
529
  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4763
529
                                 emitEmptyBoundParameters);
4764
529
  emitPostUpdateForReductionClause(CGF, S,
4765
529
                                   [](CodeGenFunction &) 
{ return nullptr; }0
);
4766
529
}
4767
4768
void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
4769
    CodeGenModule &CGM, StringRef ParentName,
4770
227
    const OMPTargetParallelDirective &S) {
4771
227
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4772
227
    emitTargetParallelRegion(CGF, S, Action);
4773
227
  };
4774
227
  llvm::Function *Fn;
4775
227
  llvm::Constant *Addr;
4776
227
  // Emit target region as a standalone region.
4777
227
  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4778
227
      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4779
227
  assert(Fn && Addr && "Target device function emission failed.");
4780
227
}
4781
4782
void CodeGenFunction::EmitOMPTargetParallelDirective(
4783
304
    const OMPTargetParallelDirective &S) {
4784
304
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4785
302
    emitTargetParallelRegion(CGF, S, Action);
4786
302
  };
4787
304
  emitCommonOMPTargetDirective(*this, S, CodeGen);
4788
304
}
4789
4790
static void emitTargetParallelForRegion(CodeGenFunction &CGF,
4791
                                        const OMPTargetParallelForDirective &S,
4792
337
                                        PrePostActionTy &Action) {
4793
337
  Action.Enter(CGF);
4794
337
  // Emit directive as a combined directive that consists of two implicit
4795
337
  // directives: 'parallel' with 'for' directive.
4796
337
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4797
337
    Action.Enter(CGF);
4798
337
    CodeGenFunction::OMPCancelStackRAII CancelRegion(
4799
337
        CGF, OMPD_target_parallel_for, S.hasCancel());
4800
337
    CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
4801
337
                               emitDispatchForLoopBounds);
4802
337
  };
4803
337
  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4804
337
                                 emitEmptyBoundParameters);
4805
337
}
4806
4807
void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
4808
    CodeGenModule &CGM, StringRef ParentName,
4809
137
    const OMPTargetParallelForDirective &S) {
4810
137
  // Emit SPMD target parallel for region as a standalone region.
4811
137
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4812
137
    emitTargetParallelForRegion(CGF, S, Action);
4813
137
  };
4814
137
  llvm::Function *Fn;
4815
137
  llvm::Constant *Addr;
4816
137
  // Emit target region as a standalone region.
4817
137
  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4818
137
      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4819
137
  assert(Fn && Addr && "Target device function emission failed.");
4820
137
}
4821
4822
void CodeGenFunction::EmitOMPTargetParallelForDirective(
4823
200
    const OMPTargetParallelForDirective &S) {
4824
200
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4825
200
    emitTargetParallelForRegion(CGF, S, Action);
4826
200
  };
4827
200
  emitCommonOMPTargetDirective(*this, S, CodeGen);
4828
200
}
4829
4830
static void
4831
emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
4832
                                const OMPTargetParallelForSimdDirective &S,
4833
261
                                PrePostActionTy &Action) {
4834
261
  Action.Enter(CGF);
4835
261
  // Emit directive as a combined directive that consists of two implicit
4836
261
  // directives: 'parallel' with 'for' directive.
4837
261
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4838
261
    Action.Enter(CGF);
4839
261
    CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
4840
261
                               emitDispatchForLoopBounds);
4841
261
  };
4842
261
  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4843
261
                                 emitEmptyBoundParameters);
4844
261
}
4845
4846
void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
4847
    CodeGenModule &CGM, StringRef ParentName,
4848
92
    const OMPTargetParallelForSimdDirective &S) {
4849
92
  // Emit SPMD target parallel for region as a standalone region.
4850
92
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4851
92
    emitTargetParallelForSimdRegion(CGF, S, Action);
4852
92
  };
4853
92
  llvm::Function *Fn;
4854
92
  llvm::Constant *Addr;
4855
92
  // Emit target region as a standalone region.
4856
92
  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4857
92
      S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4858
92
  assert(Fn && Addr && "Target device function emission failed.");
4859
92
}
4860
4861
void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
4862
169
    const OMPTargetParallelForSimdDirective &S) {
4863
169
  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4864
169
    emitTargetParallelForSimdRegion(CGF, S, Action);
4865
169
  };
4866
169
  emitCommonOMPTargetDirective(*this, S, CodeGen);
4867
169
}
4868
4869
/// Emit a helper variable and return corresponding lvalue.
4870
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4871
                     const ImplicitParamDecl *PVD,
4872
260
                     CodeGenFunction::OMPPrivateScope &Privates) {
4873
260
  const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4874
260
  Privates.addPrivate(VDecl,
4875
260
                      [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4876
260
}
4877
4878
65
void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
4879
65
  assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
4880
65
  // Emit outlined function for task construct.
4881
65
  const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4882
65
  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4883
65
  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4884
65
  const Expr *IfCond = nullptr;
4885
65
  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4886
4
    if (C->getNameModifier() == OMPD_unknown ||
4887
4
        
C->getNameModifier() == OMPD_taskloop0
) {
4888
4
      IfCond = C->getCondition();
4889
4
      break;
4890
4
    }
4891
4
  }
4892
65
4893
65
  OMPTaskDataTy Data;
4894
65
  // Check if taskloop must be emitted without taskgroup.
4895
65
  Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
4896
65
  // TODO: Check if we should emit tied or untied task.
4897
65
  Data.Tied = true;
4898
65
  // Set scheduling for taskloop
4899
65
  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4900
4
    // grainsize clause
4901
4
    Data.Schedule.setInt(/*IntVal=*/false);
4902
4
    Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4903
61
  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4904
8
    // num_tasks clause
4905
8
    Data.Schedule.setInt(/*IntVal=*/true);
4906
8
    Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4907
8
  }
4908
65
4909
65
  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4910
65
    // if (PreCond) {
4911
65
    //   for (IV in 0..LastIteration) BODY;
4912
65
    //   <Final counter/linear vars updates>;
4913
65
    // }
4914
65
    //
4915
65
4916
65
    // Emit: if (PreCond) - begin.
4917
65
    // If the condition constant folds and can be elided, avoid emitting the
4918
65
    // whole loop.
4919
65
    bool CondConstant;
4920
65
    llvm::BasicBlock *ContBlock = nullptr;
4921
65
    OMPLoopScope PreInitScope(CGF, S);
4922
65
    if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4923
55
      if (!CondConstant)
4924
0
        return;
4925
10
    } else {
4926
10
      llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4927
10
      ContBlock = CGF.createBasicBlock("taskloop.if.end");
4928
10
      emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4929
10
                  CGF.getProfileCount(&S));
4930
10
      CGF.EmitBlock(ThenBlock);
4931
10
      CGF.incrementProfileCounter(&S);
4932
10
    }
4933
65
4934
65
    if (isOpenMPSimdDirective(S.getDirectiveKind()))
4935
32
      CGF.EmitOMPSimdInit(S);
4936
65
4937
65
    OMPPrivateScope LoopScope(CGF);
4938
65
    // Emit helper vars inits.
4939
65
    enum { LowerBound = 5, UpperBound, Stride, LastIter };
4940
65
    auto *I = CS->getCapturedDecl()->param_begin();
4941
65
    auto *LBP = std::next(I, LowerBound);
4942
65
    auto *UBP = std::next(I, UpperBound);
4943
65
    auto *STP = std::next(I, Stride);
4944
65
    auto *LIP = std::next(I, LastIter);
4945
65
    mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4946
65
             LoopScope);
4947
65
    mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4948
65
             LoopScope);
4949
65
    mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4950
65
    mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4951
65
             LoopScope);
4952
65
    CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4953
65
    bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4954
65
    (void)LoopScope.Privatize();
4955
65
    // Emit the loop iteration variable.
4956
65
    const Expr *IVExpr = S.getIterationVariable();
4957
65
    const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4958
65
    CGF.EmitVarDecl(*IVDecl);
4959
65
    CGF.EmitIgnoredExpr(S.getInit());
4960
65
4961
65
    // Emit the iterations count variable.
4962
65
    // If it is not a variable, Sema decided to calculate iterations count on
4963
65
    // each iteration (e.g., it is foldable into a constant).
4964
65
    if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4965
0
      CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4966
0
      // Emit calculation of the iterations co