Coverage Report

Created: 2020-02-15 09:57

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGOpenMPRuntime.cpp
Line
Count
Source (jump to first uncovered line)
1
//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This provides a class for OpenMP runtime code generation.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGOpenMPRuntime.h"
14
#include "CGCXXABI.h"
15
#include "CGCleanup.h"
16
#include "CGRecordLayout.h"
17
#include "CodeGenFunction.h"
18
#include "clang/AST/Attr.h"
19
#include "clang/AST/Decl.h"
20
#include "clang/AST/OpenMPClause.h"
21
#include "clang/AST/StmtOpenMP.h"
22
#include "clang/AST/StmtVisitor.h"
23
#include "clang/Basic/BitmaskEnum.h"
24
#include "clang/Basic/OpenMPKinds.h"
25
#include "clang/CodeGen/ConstantInitBuilder.h"
26
#include "llvm/ADT/ArrayRef.h"
27
#include "llvm/ADT/SetOperations.h"
28
#include "llvm/ADT/StringExtras.h"
29
#include "llvm/Bitcode/BitcodeReader.h"
30
#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
31
#include "llvm/IR/DerivedTypes.h"
32
#include "llvm/IR/GlobalValue.h"
33
#include "llvm/IR/Value.h"
34
#include "llvm/Support/AtomicOrdering.h"
35
#include "llvm/Support/Format.h"
36
#include "llvm/Support/raw_ostream.h"
37
#include <cassert>
38
39
using namespace clang;
40
using namespace CodeGen;
41
using namespace llvm::omp;
42
43
namespace {
44
/// Base class for handling code generation inside OpenMP regions.
45
class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
46
public:
47
  /// Kinds of OpenMP regions used in codegen.
48
  enum CGOpenMPRegionKind {
49
    /// Region with outlined function for standalone 'parallel'
50
    /// directive.
51
    ParallelOutlinedRegion,
52
    /// Region with outlined function for standalone 'task' directive.
53
    TaskOutlinedRegion,
54
    /// Region for constructs that do not require function outlining,
55
    /// like 'for', 'sections', 'atomic' etc. directives.
56
    InlinedRegion,
57
    /// Region with outlined function for standalone 'target' directive.
58
    TargetRegion,
59
  };
60
61
  CGOpenMPRegionInfo(const CapturedStmt &CS,
62
                     const CGOpenMPRegionKind RegionKind,
63
                     const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
64
                     bool HasCancel)
65
      : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
66
15.6k
        CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
67
68
  CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
69
                     const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
70
                     bool HasCancel)
71
      : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
72
29.0k
        Kind(Kind), HasCancel(HasCancel) {}
73
74
  /// Get a variable or parameter for storing global thread id
75
  /// inside OpenMP construct.
76
  virtual const VarDecl *getThreadIDVariable() const = 0;
77
78
  /// Emit the captured statement body.
79
  void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
80
81
  /// Get an LValue for the current ThreadID variable.
82
  /// \return LValue for thread id variable. This LValue always has type int32*.
83
  virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
84
85
4
  virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
86
87
28.8k
  CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
88
89
130
  OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
90
91
290
  bool hasCancel() const { return HasCancel; }
92
93
99.9k
  static bool classof(const CGCapturedStmtInfo *Info) {
94
99.9k
    return Info->getKind() == CR_OpenMP;
95
99.9k
  }
96
97
44.6k
  ~CGOpenMPRegionInfo() override = default;
98
99
protected:
100
  CGOpenMPRegionKind RegionKind;
101
  RegionCodeGenTy CodeGen;
102
  OpenMPDirectiveKind Kind;
103
  bool HasCancel;
104
};
105
106
/// API for captured statement code generation in OpenMP constructs.
107
class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
108
public:
109
  CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
110
                             const RegionCodeGenTy &CodeGen,
111
                             OpenMPDirectiveKind Kind, bool HasCancel,
112
                             StringRef HelperName)
113
      : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
114
                           HasCancel),
115
7.99k
        ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
116
7.99k
    assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
117
7.99k
  }
118
119
  /// Get a variable or parameter for storing global thread id
120
  /// inside OpenMP construct.
121
27.5k
  const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
122
123
  /// Get the name of the capture helper.
124
8.04k
  StringRef getHelperName() const override { return HelperName; }
125
126
0
  static bool classof(const CGCapturedStmtInfo *Info) {
127
0
    return CGOpenMPRegionInfo::classof(Info) &&
128
0
           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
129
0
               ParallelOutlinedRegion;
130
0
  }
131
132
private:
133
  /// A variable or parameter storing global thread id for OpenMP
134
  /// constructs.
135
  const VarDecl *ThreadIDVar;
136
  StringRef HelperName;
137
};
138
139
/// API for captured statement code generation in OpenMP constructs.
140
class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
141
public:
142
  class UntiedTaskActionTy final : public PrePostActionTy {
143
    bool Untied;
144
    const VarDecl *PartIDVar;
145
    const RegionCodeGenTy UntiedCodeGen;
146
    llvm::SwitchInst *UntiedSwitch = nullptr;
147
148
  public:
149
    UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
150
                       const RegionCodeGenTy &UntiedCodeGen)
151
635
        : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
152
635
    void Enter(CodeGenFunction &CGF) override {
153
635
      if (Untied) {
154
16
        // Emit task switching point.
155
16
        LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
156
16
            CGF.GetAddrOfLocalVar(PartIDVar),
157
16
            PartIDVar->getType()->castAs<PointerType>());
158
16
        llvm::Value *Res =
159
16
            CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
160
16
        llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
161
16
        UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
162
16
        CGF.EmitBlock(DoneBB);
163
16
        CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
164
16
        CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
165
16
        UntiedSwitch->addCase(CGF.Builder.getInt32(0),
166
16
                              CGF.Builder.GetInsertBlock());
167
16
        emitUntiedSwitch(CGF);
168
16
      }
169
635
    }
170
30
    void emitUntiedSwitch(CodeGenFunction &CGF) const {
171
30
      if (Untied) {
172
28
        LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
173
28
            CGF.GetAddrOfLocalVar(PartIDVar),
174
28
            PartIDVar->getType()->castAs<PointerType>());
175
28
        CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
176
28
                              PartIdLVal);
177
28
        UntiedCodeGen(CGF);
178
28
        CodeGenFunction::JumpDest CurPoint =
179
28
            CGF.getJumpDestInCurrentScope(".untied.next.");
180
28
        CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
181
28
        CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
182
28
        UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
183
28
                              CGF.Builder.GetInsertBlock());
184
28
        CGF.EmitBranchThroughCleanup(CurPoint);
185
28
        CGF.EmitBlock(CurPoint.getBlock());
186
28
      }
187
30
    }
188
16
    unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
189
  };
190
  CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
191
                                 const VarDecl *ThreadIDVar,
192
                                 const RegionCodeGenTy &CodeGen,
193
                                 OpenMPDirectiveKind Kind, bool HasCancel,
194
                                 const UntiedTaskActionTy &Action)
195
      : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
196
635
        ThreadIDVar(ThreadIDVar), Action(Action) {
197
635
    assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
198
635
  }
199
200
  /// Get a variable or parameter for storing global thread id
201
  /// inside OpenMP construct.
202
240
  const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
203
204
  /// Get an LValue for the current ThreadID variable.
205
  LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
206
207
  /// Get the name of the capture helper.
208
635
  StringRef getHelperName() const override { return ".omp_outlined."; }
209
210
14
  void emitUntiedSwitch(CodeGenFunction &CGF) override {
211
14
    Action.emitUntiedSwitch(CGF);
212
14
  }
213
214
0
  static bool classof(const CGCapturedStmtInfo *Info) {
215
0
    return CGOpenMPRegionInfo::classof(Info) &&
216
0
           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
217
0
               TaskOutlinedRegion;
218
0
  }
219
220
private:
221
  /// A variable or parameter storing global thread id for OpenMP
222
  /// constructs.
223
  const VarDecl *ThreadIDVar;
224
  /// Action for emitting code for untied tasks.
225
  const UntiedTaskActionTy &Action;
226
};
227
228
/// API for inlined captured statement code generation in OpenMP
229
/// constructs.
230
class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
231
public:
232
  CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
233
                            const RegionCodeGenTy &CodeGen,
234
                            OpenMPDirectiveKind Kind, bool HasCancel)
235
      : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
236
        OldCSI(OldCSI),
237
29.0k
        OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
238
239
  // Retrieve the value of the context parameter.
240
0
  llvm::Value *getContextValue() const override {
241
0
    if (OuterRegionInfo)
242
0
      return OuterRegionInfo->getContextValue();
243
0
    llvm_unreachable("No context value for inlined OpenMP region");
244
0
  }
245
246
0
  void setContextValue(llvm::Value *V) override {
247
0
    if (OuterRegionInfo) {
248
0
      OuterRegionInfo->setContextValue(V);
249
0
      return;
250
0
    }
251
0
    llvm_unreachable("No context value for inlined OpenMP region");
252
0
  }
253
254
  /// Lookup the captured field decl for a variable.
255
22.3k
  const FieldDecl *lookup(const VarDecl *VD) const override {
256
22.3k
    if (OuterRegionInfo)
257
10.8k
      return OuterRegionInfo->lookup(VD);
258
11.4k
    // If there is no outer outlined region,no need to lookup in a list of
259
11.4k
    // captured variables, we can use the original one.
260
11.4k
    return nullptr;
261
11.4k
  }
262
263
0
  FieldDecl *getThisFieldDecl() const override {
264
0
    if (OuterRegionInfo)
265
0
      return OuterRegionInfo->getThisFieldDecl();
266
0
    return nullptr;
267
0
  }
268
269
  /// Get a variable or parameter for storing global thread id
270
  /// inside OpenMP construct.
271
5.23k
  const VarDecl *getThreadIDVariable() const override {
272
5.23k
    if (OuterRegionInfo)
273
5.08k
      return OuterRegionInfo->getThreadIDVariable();
274
144
    return nullptr;
275
144
  }
276
277
  /// Get an LValue for the current ThreadID variable.
278
5.08k
  LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
279
5.08k
    if (OuterRegionInfo)
280
5.08k
      return OuterRegionInfo->getThreadIDVariableLValue(CGF);
281
0
    llvm_unreachable("No LValue for inlined OpenMP construct");
282
0
  }
283
284
  /// Get the name of the capture helper.
285
0
  StringRef getHelperName() const override {
286
0
    if (auto *OuterRegionInfo = getOldCSI())
287
0
      return OuterRegionInfo->getHelperName();
288
0
    llvm_unreachable("No helper name for inlined OpenMP construct");
289
0
  }
290
291
6
  void emitUntiedSwitch(CodeGenFunction &CGF) override {
292
6
    if (OuterRegionInfo)
293
4
      OuterRegionInfo->emitUntiedSwitch(CGF);
294
6
  }
295
296
28.8k
  CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
297
298
28.8k
  static bool classof(const CGCapturedStmtInfo *Info) {
299
28.8k
    return CGOpenMPRegionInfo::classof(Info) &&
300
28.8k
           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
301
28.8k
  }
302
303
29.0k
  ~CGOpenMPInlinedRegionInfo() override = default;
304
305
private:
306
  /// CodeGen info about outer OpenMP region.
307
  CodeGenFunction::CGCapturedStmtInfo *OldCSI;
308
  CGOpenMPRegionInfo *OuterRegionInfo;
309
};
310
311
/// API for captured statement code generation in OpenMP target
312
/// constructs. For this captures, implicit parameters are used instead of the
313
/// captured fields. The name of the target region has to be unique in a given
314
/// application so it is provided by the client, because only the client has
315
/// the information to generate that.
316
class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
317
public:
318
  CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
319
                           const RegionCodeGenTy &CodeGen, StringRef HelperName)
320
      : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
321
                           /*HasCancel=*/false),
322
6.97k
        HelperName(HelperName) {}
323
324
  /// This is unused for target regions because each starts executing
325
  /// with a single thread.
326
1.54k
  const VarDecl *getThreadIDVariable() const override { return nullptr; }
327
328
  /// Get the name of the capture helper.
329
6.99k
  StringRef getHelperName() const override { return HelperName; }
330
331
0
  static bool classof(const CGCapturedStmtInfo *Info) {
332
0
    return CGOpenMPRegionInfo::classof(Info) &&
333
0
           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
334
0
  }
335
336
private:
337
  StringRef HelperName;
338
};
339
340
0
static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
341
0
  llvm_unreachable("No codegen for expressions");
342
0
}
343
/// API for generation of expressions captured in a innermost OpenMP
344
/// region.
345
class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
346
public:
347
  CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
348
      : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
349
                                  OMPD_unknown,
350
                                  /*HasCancel=*/false),
351
186
        PrivScope(CGF) {
352
186
    // Make sure the globals captured in the provided statement are local by
353
186
    // using the privatization logic. We assume the same variable is not
354
186
    // captured more than once.
355
386
    for (const auto &C : CS.captures()) {
356
386
      if (!C.capturesVariable() && 
!C.capturesVariableByCopy()296
)
357
16
        continue;
358
370
359
370
      const VarDecl *VD = C.getCapturedVar();
360
370
      if (VD->isLocalVarDeclOrParm())
361
302
        continue;
362
68
363
68
      DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
364
68
                      /*RefersToEnclosingVariableOrCapture=*/false,
365
68
                      VD->getType().getNonReferenceType(), VK_LValue,
366
68
                      C.getLocation());
367
68
      PrivScope.addPrivate(
368
68
          VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); });
369
68
    }
370
186
    (void)PrivScope.Privatize();
371
186
  }
372
373
  /// Lookup the captured field decl for a variable.
374
0
  const FieldDecl *lookup(const VarDecl *VD) const override {
375
0
    if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
376
0
      return FD;
377
0
    return nullptr;
378
0
  }
379
380
  /// Emit the captured statement body.
381
0
  void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
382
0
    llvm_unreachable("No body for expressions");
383
0
  }
384
385
  /// Get a variable or parameter for storing global thread id
386
  /// inside OpenMP construct.
387
0
  const VarDecl *getThreadIDVariable() const override {
388
0
    llvm_unreachable("No thread id for expressions");
389
0
  }
390
391
  /// Get the name of the capture helper.
392
0
  StringRef getHelperName() const override {
393
0
    llvm_unreachable("No helper name for expressions");
394
0
  }
395
396
0
  static bool classof(const CGCapturedStmtInfo *Info) { return false; }
397
398
private:
399
  /// Private scope to capture global variables.
400
  CodeGenFunction::OMPPrivateScope PrivScope;
401
};
402
403
/// RAII for emitting code of OpenMP constructs.
404
class InlinedOpenMPRegionRAII {
405
  CodeGenFunction &CGF;
406
  llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
407
  FieldDecl *LambdaThisCaptureField = nullptr;
408
  const CodeGen::CGBlockInfo *BlockInfo = nullptr;
409
410
public:
411
  /// Constructs region for combined constructs.
412
  /// \param CodeGen Code generation sequence for combined directives. Includes
413
  /// a list of functions used for code generation of implicitly inlined
414
  /// regions.
415
  InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
416
                          OpenMPDirectiveKind Kind, bool HasCancel)
417
28.8k
      : CGF(CGF) {
418
28.8k
    // Start emission for the construct.
419
28.8k
    CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
420
28.8k
        CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
421
28.8k
    std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
422
28.8k
    LambdaThisCaptureField = CGF.LambdaThisCaptureField;
423
28.8k
    CGF.LambdaThisCaptureField = nullptr;
424
28.8k
    BlockInfo = CGF.BlockInfo;
425
28.8k
    CGF.BlockInfo = nullptr;
426
28.8k
  }
427
428
28.8k
  ~InlinedOpenMPRegionRAII() {
429
28.8k
    // Restore original CapturedStmtInfo only if we're done with code emission.
430
28.8k
    auto *OldCSI =
431
28.8k
        cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
432
28.8k
    delete CGF.CapturedStmtInfo;
433
28.8k
    CGF.CapturedStmtInfo = OldCSI;
434
28.8k
    std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
435
28.8k
    CGF.LambdaThisCaptureField = LambdaThisCaptureField;
436
28.8k
    CGF.BlockInfo = BlockInfo;
437
28.8k
  }
438
};
439
440
/// Values for bit flags used in the ident_t to describe the fields.
441
/// All enumeric elements are named and described in accordance with the code
442
/// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
443
enum OpenMPLocationFlags : unsigned {
444
  /// Use trampoline for internal microtask.
445
  OMP_IDENT_IMD = 0x01,
446
  /// Use c-style ident structure.
447
  OMP_IDENT_KMPC = 0x02,
448
  /// Atomic reduction option for kmpc_reduce.
449
  OMP_ATOMIC_REDUCE = 0x10,
450
  /// Explicit 'barrier' directive.
451
  OMP_IDENT_BARRIER_EXPL = 0x20,
452
  /// Implicit barrier in code.
453
  OMP_IDENT_BARRIER_IMPL = 0x40,
454
  /// Implicit barrier in 'for' directive.
455
  OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
456
  /// Implicit barrier in 'sections' directive.
457
  OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
458
  /// Implicit barrier in 'single' directive.
459
  OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
460
  /// Call of __kmp_for_static_init for static loop.
461
  OMP_IDENT_WORK_LOOP = 0x200,
462
  /// Call of __kmp_for_static_init for sections.
463
  OMP_IDENT_WORK_SECTIONS = 0x400,
464
  /// Call of __kmp_for_static_init for distribute.
465
  OMP_IDENT_WORK_DISTRIBUTE = 0x800,
466
  LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)
467
};
468
469
namespace {
470
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
471
/// Values for bit flags for marking which requires clauses have been used.
472
enum OpenMPOffloadingRequiresDirFlags : int64_t {
473
  /// flag undefined.
474
  OMP_REQ_UNDEFINED               = 0x000,
475
  /// no requires clause present.
476
  OMP_REQ_NONE                    = 0x001,
477
  /// reverse_offload clause.
478
  OMP_REQ_REVERSE_OFFLOAD         = 0x002,
479
  /// unified_address clause.
480
  OMP_REQ_UNIFIED_ADDRESS         = 0x004,
481
  /// unified_shared_memory clause.
482
  OMP_REQ_UNIFIED_SHARED_MEMORY   = 0x008,
483
  /// dynamic_allocators clause.
484
  OMP_REQ_DYNAMIC_ALLOCATORS      = 0x010,
485
  LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)
486
};
487
488
enum OpenMPOffloadingReservedDeviceIDs {
489
  /// Device ID if the device was not defined, runtime should get it
490
  /// from environment variables in the spec.
491
  OMP_DEVICEID_UNDEF = -1,
492
};
493
} // anonymous namespace
494
495
/// Describes ident structure that describes a source location.
496
/// All descriptions are taken from
497
/// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h
498
/// Original structure:
499
/// typedef struct ident {
500
///    kmp_int32 reserved_1;   /**<  might be used in Fortran;
501
///                                  see above  */
502
///    kmp_int32 flags;        /**<  also f.flags; KMP_IDENT_xxx flags;
503
///                                  KMP_IDENT_KMPC identifies this union
504
///                                  member  */
505
///    kmp_int32 reserved_2;   /**<  not really used in Fortran any more;
506
///                                  see above */
507
///#if USE_ITT_BUILD
508
///                            /*  but currently used for storing
509
///                                region-specific ITT */
510
///                            /*  contextual information. */
511
///#endif /* USE_ITT_BUILD */
512
///    kmp_int32 reserved_3;   /**< source[4] in Fortran, do not use for
513
///                                 C++  */
514
///    char const *psource;    /**< String describing the source location.
515
///                            The string is composed of semi-colon separated
516
//                             fields which describe the source file,
517
///                            the function and a pair of line numbers that
518
///                            delimit the construct.
519
///                             */
520
/// } ident_t;
521
enum IdentFieldIndex {
522
  /// might be used in Fortran
523
  IdentField_Reserved_1,
524
  /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
525
  IdentField_Flags,
526
  /// Not really used in Fortran any more
527
  IdentField_Reserved_2,
528
  /// Source[4] in Fortran, do not use for C++
529
  IdentField_Reserved_3,
530
  /// String describing the source location. The string is composed of
531
  /// semi-colon separated fields which describe the source file, the function
532
  /// and a pair of line numbers that delimit the construct.
533
  IdentField_PSource
534
};
535
536
/// Schedule types for 'omp for' loops (these enumerators are taken from
537
/// the enum sched_type in kmp.h).
538
enum OpenMPSchedType {
539
  /// Lower bound for default (unordered) versions.
540
  OMP_sch_lower = 32,
541
  OMP_sch_static_chunked = 33,
542
  OMP_sch_static = 34,
543
  OMP_sch_dynamic_chunked = 35,
544
  OMP_sch_guided_chunked = 36,
545
  OMP_sch_runtime = 37,
546
  OMP_sch_auto = 38,
547
  /// static with chunk adjustment (e.g., simd)
548
  OMP_sch_static_balanced_chunked = 45,
549
  /// Lower bound for 'ordered' versions.
550
  OMP_ord_lower = 64,
551
  OMP_ord_static_chunked = 65,
552
  OMP_ord_static = 66,
553
  OMP_ord_dynamic_chunked = 67,
554
  OMP_ord_guided_chunked = 68,
555
  OMP_ord_runtime = 69,
556
  OMP_ord_auto = 70,
557
  OMP_sch_default = OMP_sch_static,
558
  /// dist_schedule types
559
  OMP_dist_sch_static_chunked = 91,
560
  OMP_dist_sch_static = 92,
561
  /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
562
  /// Set if the monotonic schedule modifier was present.
563
  OMP_sch_modifier_monotonic = (1 << 29),
564
  /// Set if the nonmonotonic schedule modifier was present.
565
  OMP_sch_modifier_nonmonotonic = (1 << 30),
566
};
567
568
enum OpenMPRTLFunction {
569
  /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
570
  /// kmpc_micro microtask, ...);
571
  OMPRTL__kmpc_fork_call,
572
  /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
573
  /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
574
  OMPRTL__kmpc_threadprivate_cached,
575
  /// Call to void __kmpc_threadprivate_register( ident_t *,
576
  /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
577
  OMPRTL__kmpc_threadprivate_register,
578
  // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
579
  OMPRTL__kmpc_global_thread_num,
580
  // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
581
  // kmp_critical_name *crit);
582
  OMPRTL__kmpc_critical,
583
  // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
584
  // global_tid, kmp_critical_name *crit, uintptr_t hint);
585
  OMPRTL__kmpc_critical_with_hint,
586
  // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
587
  // kmp_critical_name *crit);
588
  OMPRTL__kmpc_end_critical,
589
  // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
590
  // global_tid);
591
  OMPRTL__kmpc_cancel_barrier,
592
  // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
593
  OMPRTL__kmpc_barrier,
594
  // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
595
  OMPRTL__kmpc_for_static_fini,
596
  // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
597
  // global_tid);
598
  OMPRTL__kmpc_serialized_parallel,
599
  // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
600
  // global_tid);
601
  OMPRTL__kmpc_end_serialized_parallel,
602
  // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
603
  // kmp_int32 num_threads);
604
  OMPRTL__kmpc_push_num_threads,
605
  // Call to void __kmpc_flush(ident_t *loc);
606
  OMPRTL__kmpc_flush,
607
  // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
608
  OMPRTL__kmpc_master,
609
  // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
610
  OMPRTL__kmpc_end_master,
611
  // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
612
  // int end_part);
613
  OMPRTL__kmpc_omp_taskyield,
614
  // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
615
  OMPRTL__kmpc_single,
616
  // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
617
  OMPRTL__kmpc_end_single,
618
  // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
619
  // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
620
  // kmp_routine_entry_t *task_entry);
621
  OMPRTL__kmpc_omp_task_alloc,
622
  // Call to kmp_task_t * __kmpc_omp_target_task_alloc(ident_t *,
623
  // kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t,
624
  // size_t sizeof_shareds, kmp_routine_entry_t *task_entry,
625
  // kmp_int64 device_id);
626
  OMPRTL__kmpc_omp_target_task_alloc,
627
  // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
628
  // new_task);
629
  OMPRTL__kmpc_omp_task,
630
  // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
631
  // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
632
  // kmp_int32 didit);
633
  OMPRTL__kmpc_copyprivate,
634
  // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
635
  // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
636
  // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
637
  OMPRTL__kmpc_reduce,
638
  // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
639
  // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
640
  // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
641
  // *lck);
642
  OMPRTL__kmpc_reduce_nowait,
643
  // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
644
  // kmp_critical_name *lck);
645
  OMPRTL__kmpc_end_reduce,
646
  // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
647
  // kmp_critical_name *lck);
648
  OMPRTL__kmpc_end_reduce_nowait,
649
  // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
650
  // kmp_task_t * new_task);
651
  OMPRTL__kmpc_omp_task_begin_if0,
652
  // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
653
  // kmp_task_t * new_task);
654
  OMPRTL__kmpc_omp_task_complete_if0,
655
  // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
656
  OMPRTL__kmpc_ordered,
657
  // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
658
  OMPRTL__kmpc_end_ordered,
659
  // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
660
  // global_tid);
661
  OMPRTL__kmpc_omp_taskwait,
662
  // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
663
  OMPRTL__kmpc_taskgroup,
664
  // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
665
  OMPRTL__kmpc_end_taskgroup,
666
  // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
667
  // int proc_bind);
668
  OMPRTL__kmpc_push_proc_bind,
669
  // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
670
  // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
671
  // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
672
  OMPRTL__kmpc_omp_task_with_deps,
673
  // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
674
  // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
675
  // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
676
  OMPRTL__kmpc_omp_wait_deps,
677
  // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
678
  // global_tid, kmp_int32 cncl_kind);
679
  OMPRTL__kmpc_cancellationpoint,
680
  // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
681
  // kmp_int32 cncl_kind);
682
  OMPRTL__kmpc_cancel,
683
  // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
684
  // kmp_int32 num_teams, kmp_int32 thread_limit);
685
  OMPRTL__kmpc_push_num_teams,
686
  // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
687
  // microtask, ...);
688
  OMPRTL__kmpc_fork_teams,
689
  // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
690
  // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
691
  // sched, kmp_uint64 grainsize, void *task_dup);
692
  OMPRTL__kmpc_taskloop,
693
  // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
694
  // num_dims, struct kmp_dim *dims);
695
  OMPRTL__kmpc_doacross_init,
696
  // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
697
  OMPRTL__kmpc_doacross_fini,
698
  // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
699
  // *vec);
700
  OMPRTL__kmpc_doacross_post,
701
  // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
702
  // *vec);
703
  OMPRTL__kmpc_doacross_wait,
704
  // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
705
  // *data);
706
  OMPRTL__kmpc_task_reduction_init,
707
  // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
708
  // *d);
709
  OMPRTL__kmpc_task_reduction_get_th_data,
710
  // Call to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);
711
  OMPRTL__kmpc_alloc,
712
  // Call to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);
713
  OMPRTL__kmpc_free,
714
715
  //
716
  // Offloading related calls
717
  //
718
  // Call to void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
719
  // size);
720
  OMPRTL__kmpc_push_target_tripcount,
721
  // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
722
  // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
723
  // *arg_types);
724
  OMPRTL__tgt_target,
725
  // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
726
  // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
727
  // *arg_types);
728
  OMPRTL__tgt_target_nowait,
729
  // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
730
  // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
731
  // *arg_types, int32_t num_teams, int32_t thread_limit);
732
  OMPRTL__tgt_target_teams,
733
  // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
734
  // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
735
  // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
736
  OMPRTL__tgt_target_teams_nowait,
737
  // Call to void __tgt_register_requires(int64_t flags);
738
  OMPRTL__tgt_register_requires,
739
  // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
740
  // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
741
  OMPRTL__tgt_target_data_begin,
742
  // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
743
  // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
744
  // *arg_types);
745
  OMPRTL__tgt_target_data_begin_nowait,
746
  // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
747
  // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
748
  OMPRTL__tgt_target_data_end,
749
  // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
750
  // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
751
  // *arg_types);
752
  OMPRTL__tgt_target_data_end_nowait,
753
  // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
754
  // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
755
  OMPRTL__tgt_target_data_update,
756
  // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
757
  // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
758
  // *arg_types);
759
  OMPRTL__tgt_target_data_update_nowait,
760
  // Call to int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
761
  OMPRTL__tgt_mapper_num_components,
762
  // Call to void __tgt_push_mapper_component(void *rt_mapper_handle, void
763
  // *base, void *begin, int64_t size, int64_t type);
764
  OMPRTL__tgt_push_mapper_component,
765
};
766
767
/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
768
/// region.
769
class CleanupTy final : public EHScopeStack::Cleanup {
770
  PrePostActionTy *Action;
771
772
public:
773
3.61k
  explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
774
3.68k
  void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
775
3.68k
    if (!CGF.HaveInsertPoint())
776
0
      return;
777
3.68k
    Action->Exit(CGF);
778
3.68k
  }
779
};
780
781
} // anonymous namespace
782
783
86.7k
void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
784
86.7k
  CodeGenFunction::RunCleanupsScope Scope(CGF);
785
86.7k
  if (PrePostAction) {
786
3.61k
    CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
787
3.61k
    Callback(CodeGen, CGF, *PrePostAction);
788
83.1k
  } else {
789
83.1k
    PrePostActionTy Action;
790
83.1k
    Callback(CodeGen, CGF, Action);
791
83.1k
  }
792
86.7k
}
793
794
/// Check if the combiner is a call to UDR combiner and if it is so return the
795
/// UDR decl used for reduction.
796
static const OMPDeclareReductionDecl *
797
883
getReductionInit(const Expr *ReductionOp) {
798
883
  if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
799
109
    if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
800
73
      if (const auto *DRE =
801
73
              dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
802
73
        if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
803
73
          return DRD;
804
810
  return nullptr;
805
810
}
806
807
static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
808
                                             const OMPDeclareReductionDecl *DRD,
809
                                             const Expr *InitOp,
810
                                             Address Private, Address Original,
811
55
                                             QualType Ty) {
812
55
  if (DRD->getInitializer()) {
813
49
    std::pair<llvm::Function *, llvm::Function *> Reduction =
814
49
        CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
815
49
    const auto *CE = cast<CallExpr>(InitOp);
816
49
    const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
817
49
    const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
818
49
    const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
819
49
    const auto *LHSDRE =
820
49
        cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
821
49
    const auto *RHSDRE =
822
49
        cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
823
49
    CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
824
49
    PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
825
49
                            [=]() { return Private; });
826
49
    PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
827
49
                            [=]() { return Original; });
828
49
    (void)PrivateScope.Privatize();
829
49
    RValue Func = RValue::get(Reduction.second);
830
49
    CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
831
49
    CGF.EmitIgnoredExpr(InitOp);
832
49
  } else {
833
6
    llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
834
6
    std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
835
6
    auto *GV = new llvm::GlobalVariable(
836
6
        CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
837
6
        llvm::GlobalValue::PrivateLinkage, Init, Name);
838
6
    LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
839
6
    RValue InitRVal;
840
6
    switch (CGF.getEvaluationKind(Ty)) {
841
6
    case TEK_Scalar:
842
6
      InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
843
6
      break;
844
0
    case TEK_Complex:
845
0
      InitRVal =
846
0
          RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
847
0
      break;
848
0
    case TEK_Aggregate:
849
0
      InitRVal = RValue::getAggregate(LV.getAddress(CGF));
850
0
      break;
851
6
    }
852
6
    OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
853
6
    CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
854
6
    CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
855
6
                         /*IsInitializer=*/false);
856
6
  }
857
55
}
858
859
/// Emit initialization of arrays of complex types.
860
/// \param DestAddr Address of the array.
861
/// \param Type Type of array.
862
/// \param Init Initial expression of array.
863
/// \param SrcAddr Address of the original array.
864
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
865
                                 QualType Type, bool EmitDeclareReductionInit,
866
                                 const Expr *Init,
867
                                 const OMPDeclareReductionDecl *DRD,
868
179
                                 Address SrcAddr = Address::invalid()) {
869
179
  // Perform element-by-element initialization.
870
179
  QualType ElementTy;
871
179
872
179
  // Drill down to the base element type on both arrays.
873
179
  const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
874
179
  llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
875
179
  DestAddr =
876
179
      CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
877
179
  if (DRD)
878
29
    SrcAddr =
879
29
        CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
880
179
881
179
  llvm::Value *SrcBegin = nullptr;
882
179
  if (DRD)
883
29
    SrcBegin = SrcAddr.getPointer();
884
179
  llvm::Value *DestBegin = DestAddr.getPointer();
885
179
  // Cast from pointer to array type to pointer to single element.
886
179
  llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
887
179
  // The basic structure here is a while-do loop.
888
179
  llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
889
179
  llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
890
179
  llvm::Value *IsEmpty =
891
179
      CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
892
179
  CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
893
179
894
179
  // Enter the loop body, making that address the current address.
895
179
  llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
896
179
  CGF.EmitBlock(BodyBB);
897
179
898
179
  CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
899
179
900
179
  llvm::PHINode *SrcElementPHI = nullptr;
901
179
  Address SrcElementCurrent = Address::invalid();
902
179
  if (DRD) {
903
29
    SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
904
29
                                          "omp.arraycpy.srcElementPast");
905
29
    SrcElementPHI->addIncoming(SrcBegin, EntryBB);
906
29
    SrcElementCurrent =
907
29
        Address(SrcElementPHI,
908
29
                SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
909
29
  }
910
179
  llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
911
179
      DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
912
179
  DestElementPHI->addIncoming(DestBegin, EntryBB);
913
179
  Address DestElementCurrent =
914
179
      Address(DestElementPHI,
915
179
              DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
916
179
917
179
  // Emit copy.
918
179
  {
919
179
    CodeGenFunction::RunCleanupsScope InitScope(CGF);
920
179
    if (EmitDeclareReductionInit) {
921
29
      emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
922
29
                                       SrcElementCurrent, ElementTy);
923
29
    } else
924
150
      CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
925
150
                           /*IsInitializer=*/false);
926
179
  }
927
179
928
179
  if (DRD) {
929
29
    // Shift the address forward by one element.
930
29
    llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
931
29
        SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
932
29
    SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
933
29
  }
934
179
935
179
  // Shift the address forward by one element.
936
179
  llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
937
179
      DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
938
179
  // Check whether we've reached the end.
939
179
  llvm::Value *Done =
940
179
      CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
941
179
  CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
942
179
  DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
943
179
944
179
  // Done.
945
179
  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
946
179
}
947
948
730
LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
949
730
  return CGF.EmitOMPSharedLValue(E);
950
730
}
951
952
LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
953
730
                                            const Expr *E) {
954
730
  if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
955
97
    return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
956
633
  return LValue();
957
633
}
958
959
void ReductionCodeGen::emitAggregateInitialization(
960
    CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
961
179
    const OMPDeclareReductionDecl *DRD) {
962
179
  // Emit VarDecl with copy init for arrays.
963
179
  // Get the address of the original variable captured in current
964
179
  // captured region.
965
179
  const auto *PrivateVD =
966
179
      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
967
179
  bool EmitDeclareReductionInit =
968
179
      DRD && 
(29
DRD->getInitializer()29
||
!PrivateVD->hasInit()2
);
969
179
  EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
970
179
                       EmitDeclareReductionInit,
971
179
                       EmitDeclareReductionInit ? 
ClausesData[N].ReductionOp29
972
179
                                                : 
PrivateVD->getInit()150
,
973
179
                       DRD, SharedLVal.getAddress(CGF));
974
179
}
975
976
ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
977
                                   ArrayRef<const Expr *> Privates,
978
11.8k
                                   ArrayRef<const Expr *> ReductionOps) {
979
11.8k
  ClausesData.reserve(Shareds.size());
980
11.8k
  SharedAddresses.reserve(Shareds.size());
981
11.8k
  Sizes.reserve(Shareds.size());
982
11.8k
  BaseDecls.reserve(Shareds.size());
983
11.8k
  auto IPriv = Privates.begin();
984
11.8k
  auto IRed = ReductionOps.begin();
985
11.8k
  for (const Expr *Ref : Shareds) {
986
730
    ClausesData.emplace_back(Ref, *IPriv, *IRed);
987
730
    std::advance(IPriv, 1);
988
730
    std::advance(IRed, 1);
989
730
  }
990
11.8k
}
991
992
730
void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
993
730
  assert(SharedAddresses.size() == N &&
994
730
         "Number of generated lvalues must be exactly N.");
995
730
  LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
996
730
  LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
997
730
  SharedAddresses.emplace_back(First, Second);
998
730
}
999
1000
730
void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
1001
730
  const auto *PrivateVD =
1002
730
      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1003
730
  QualType PrivateType = PrivateVD->getType();
1004
730
  bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
1005
730
  if (!PrivateType->isVariablyModifiedType()) {
1006
625
    Sizes.emplace_back(
1007
625
        CGF.getTypeSize(
1008
625
            SharedAddresses[N].first.getType().getNonReferenceType()),
1009
625
        nullptr);
1010
625
    return;
1011
625
  }
1012
105
  llvm::Value *Size;
1013
105
  llvm::Value *SizeInChars;
1014
105
  auto *ElemType = cast<llvm::PointerType>(
1015
105
                       SharedAddresses[N].first.getPointer(CGF)->getType())
1016
105
                       ->getElementType();
1017
105
  auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
1018
105
  if (AsArraySection) {
1019
54
    Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(CGF),
1020
54
                                     SharedAddresses[N].first.getPointer(CGF));
1021
54
    Size = CGF.Builder.CreateNUWAdd(
1022
54
        Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
1023
54
    SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
1024
54
  } else {
1025
51
    SizeInChars = CGF.getTypeSize(
1026
51
        SharedAddresses[N].first.getType().getNonReferenceType());
1027
51
    Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
1028
51
  }
1029
105
  Sizes.emplace_back(SizeInChars, Size);
1030
105
  CodeGenFunction::OpaqueValueMapping OpaqueMap(
1031
105
      CGF,
1032
105
      cast<OpaqueValueExpr>(
1033
105
          CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1034
105
      RValue::get(Size));
1035
105
  CGF.EmitVariablyModifiedType(PrivateType);
1036
105
}
1037
1038
void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
1039
197
                                         llvm::Value *Size) {
1040
197
  const auto *PrivateVD =
1041
197
      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1042
197
  QualType PrivateType = PrivateVD->getType();
1043
197
  if (!PrivateType->isVariablyModifiedType()) {
1044
141
    assert(!Size && !Sizes[N].second &&
1045
141
           "Size should be nullptr for non-variably modified reduction "
1046
141
           "items.");
1047
141
    return;
1048
141
  }
1049
56
  CodeGenFunction::OpaqueValueMapping OpaqueMap(
1050
56
      CGF,
1051
56
      cast<OpaqueValueExpr>(
1052
56
          CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1053
56
      RValue::get(Size));
1054
56
  CGF.EmitVariablyModifiedType(PrivateType);
1055
56
}
1056
1057
void ReductionCodeGen::emitInitialization(
1058
    CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1059
686
    llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1060
686
  assert(SharedAddresses.size() > N && "No variable was generated");
1061
686
  const auto *PrivateVD =
1062
686
      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1063
686
  const OMPDeclareReductionDecl *DRD =
1064
686
      getReductionInit(ClausesData[N].ReductionOp);
1065
686
  QualType PrivateType = PrivateVD->getType();
1066
686
  PrivateAddr = CGF.Builder.CreateElementBitCast(
1067
686
      PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1068
686
  QualType SharedType = SharedAddresses[N].first.getType();
1069
686
  SharedLVal = CGF.MakeAddrLValue(
1070
686
      CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF),
1071
686
                                       CGF.ConvertTypeForMem(SharedType)),
1072
686
      SharedType, SharedAddresses[N].first.getBaseInfo(),
1073
686
      CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1074
686
  if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1075
179
    emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1076
507
  } else if (DRD && 
(32
DRD->getInitializer()32
||
!PrivateVD->hasInit()10
)) {
1077
26
    emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1078
26
                                     PrivateAddr, SharedLVal.getAddress(CGF),
1079
26
                                     SharedLVal.getType());
1080
481
  } else if (!DefaultInit(CGF) && 
PrivateVD->hasInit()51
&&
1081
481
             
!CGF.isTrivialInitializer(PrivateVD->getInit())51
) {
1082
51
    CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1083
51
                         PrivateVD->getType().getQualifiers(),
1084
51
                         /*IsInitializer=*/false);
1085
51
  }
1086
686
}
1087
1088
108
bool ReductionCodeGen::needCleanups(unsigned N) {
1089
108
  const auto *PrivateVD =
1090
108
      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1091
108
  QualType PrivateType = PrivateVD->getType();
1092
108
  QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1093
108
  return DTorKind != QualType::DK_none;
1094
108
}
1095
1096
void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1097
19
                                    Address PrivateAddr) {
1098
19
  const auto *PrivateVD =
1099
19
      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1100
19
  QualType PrivateType = PrivateVD->getType();
1101
19
  QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1102
19
  if (needCleanups(N)) {
1103
19
    PrivateAddr = CGF.Builder.CreateElementBitCast(
1104
19
        PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1105
19
    CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1106
19
  }
1107
19
}
1108
1109
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1110
91
                          LValue BaseLV) {
1111
91
  BaseTy = BaseTy.getNonReferenceType();
1112
115
  while ((BaseTy->isPointerType() || 
BaseTy->isReferenceType()91
) &&
1113
115
         
!CGF.getContext().hasSameType(BaseTy, ElTy)24
) {
1114
24
    if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1115
24
      BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy);
1116
24
    } else {
1117
0
      LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy);
1118
0
      BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1119
0
    }
1120
24
    BaseTy = BaseTy->getPointeeType();
1121
24
  }
1122
91
  return CGF.MakeAddrLValue(
1123
91
      CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF),
1124
91
                                       CGF.ConvertTypeForMem(ElTy)),
1125
91
      BaseLV.getType(), BaseLV.getBaseInfo(),
1126
91
      CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1127
91
}
1128
1129
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1130
                          llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1131
91
                          llvm::Value *Addr) {
1132
91
  Address Tmp = Address::invalid();
1133
91
  Address TopTmp = Address::invalid();
1134
91
  Address MostTopTmp = Address::invalid();
1135
91
  BaseTy = BaseTy.getNonReferenceType();
1136
115
  while ((BaseTy->isPointerType() || 
BaseTy->isReferenceType()91
) &&
1137
115
         
!CGF.getContext().hasSameType(BaseTy, ElTy)24
) {
1138
24
    Tmp = CGF.CreateMemTemp(BaseTy);
1139
24
    if (TopTmp.isValid())
1140
10
      CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1141
14
    else
1142
14
      MostTopTmp = Tmp;
1143
24
    TopTmp = Tmp;
1144
24
    BaseTy = BaseTy->getPointeeType();
1145
24
  }
1146
91
  llvm::Type *Ty = BaseLVType;
1147
91
  if (Tmp.isValid())
1148
14
    Ty = Tmp.getElementType();
1149
91
  Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1150
91
  if (Tmp.isValid()) {
1151
14
    CGF.Builder.CreateStore(Addr, Tmp);
1152
14
    return MostTopTmp;
1153
14
  }
1154
77
  return Address(Addr, BaseLVAlignment);
1155
77
}
1156
1157
743
static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1158
743
  const VarDecl *OrigVD = nullptr;
1159
743
  if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1160
127
    const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1161
139
    while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1162
12
      Base = TempOASE->getBase()->IgnoreParenImpCasts();
1163
135
    while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1164
8
      Base = TempASE->getBase()->IgnoreParenImpCasts();
1165
127
    DE = cast<DeclRefExpr>(Base);
1166
127
    OrigVD = cast<VarDecl>(DE->getDecl());
1167
616
  } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1168
0
    const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1169
0
    while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1170
0
      Base = TempASE->getBase()->IgnoreParenImpCasts();
1171
0
    DE = cast<DeclRefExpr>(Base);
1172
0
    OrigVD = cast<VarDecl>(DE->getDecl());
1173
0
  }
1174
743
  return OrigVD;
1175
743
}
1176
1177
Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1178
641
                                               Address PrivateAddr) {
1179
641
  const DeclRefExpr *DE;
1180
641
  if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1181
91
    BaseDecls.emplace_back(OrigVD);
1182
91
    LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1183
91
    LValue BaseLValue =
1184
91
        loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1185
91
                    OriginalBaseLValue);
1186
91
    llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1187
91
        BaseLValue.getPointer(CGF), SharedAddresses[N].first.getPointer(CGF));
1188
91
    llvm::Value *PrivatePointer =
1189
91
        CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1190
91
            PrivateAddr.getPointer(),
1191
91
            SharedAddresses[N].first.getAddress(CGF).getType());
1192
91
    llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1193
91
    return castToBase(CGF, OrigVD->getType(),
1194
91
                      SharedAddresses[N].first.getType(),
1195
91
                      OriginalBaseLValue.getAddress(CGF).getType(),
1196
91
                      OriginalBaseLValue.getAlignment(), Ptr);
1197
91
  }
1198
550
  BaseDecls.emplace_back(
1199
550
      cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1200
550
  return PrivateAddr;
1201
550
}
1202
1203
197
bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1204
197
  const OMPDeclareReductionDecl *DRD =
1205
197
      getReductionInit(ClausesData[N].ReductionOp);
1206
197
  return DRD && 
DRD->getInitializer()12
;
1207
197
}
1208
1209
9.17k
LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1210
9.17k
  return CGF.EmitLoadOfPointerLValue(
1211
9.17k
      CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1212
9.17k
      getThreadIDVariable()->getType()->castAs<PointerType>());
1213
9.17k
}
1214
1215
44.4k
void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1216
44.4k
  if (!CGF.HaveInsertPoint())
1217
0
    return;
1218
44.4k
  // 1.2.2 OpenMP Language Terminology
1219
44.4k
  // Structured block - An executable statement with a single entry at the
1220
44.4k
  // top and a single exit at the bottom.
1221
44.4k
  // The point of exit cannot be a branch out of the structured block.
1222
44.4k
  // longjmp() and throw() must not violate the entry/exit criteria.
1223
44.4k
  CGF.EHStack.pushTerminate();
1224
44.4k
  CodeGen(CGF);
1225
44.4k
  CGF.EHStack.popTerminate();
1226
44.4k
}
1227
1228
LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1229
80
    CodeGenFunction &CGF) {
1230
80
  return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1231
80
                            getThreadIDVariable()->getType(),
1232
80
                            AlignmentSource::Decl);
1233
80
}
1234
1235
static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1236
33.8k
                                       QualType FieldTy) {
1237
33.8k
  auto *Field = FieldDecl::Create(
1238
33.8k
      C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1239
33.8k
      C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1240
33.8k
      /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1241
33.8k
  Field->setAccess(AS_public);
1242
33.8k
  DC->addDecl(Field);
1243
33.8k
  return Field;
1244
33.8k
}
1245
1246
CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1247
                                 StringRef Separator)
1248
    : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1249
4.21k
      OffloadEntriesInfoManager(CGM) {
1250
4.21k
  ASTContext &C = CGM.getContext();
1251
4.21k
  RecordDecl *RD = C.buildImplicitRecord("ident_t");
1252
4.21k
  QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1253
4.21k
  RD->startDefinition();
1254
4.21k
  // reserved_1
1255
4.21k
  addFieldToRecordDecl(C, RD, KmpInt32Ty);
1256
4.21k
  // flags
1257
4.21k
  addFieldToRecordDecl(C, RD, KmpInt32Ty);
1258
4.21k
  // reserved_2
1259
4.21k
  addFieldToRecordDecl(C, RD, KmpInt32Ty);
1260
4.21k
  // reserved_3
1261
4.21k
  addFieldToRecordDecl(C, RD, KmpInt32Ty);
1262
4.21k
  // psource
1263
4.21k
  addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1264
4.21k
  RD->completeDefinition();
1265
4.21k
  IdentQTy = C.getRecordType(RD);
1266
4.21k
  IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1267
4.21k
  KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1268
4.21k
1269
4.21k
  loadOffloadInfoMetadata();
1270
4.21k
}
1271
1272
bool CGOpenMPRuntime::tryEmitDeclareVariant(const GlobalDecl &NewGD,
1273
                                            const GlobalDecl &OldGD,
1274
                                            llvm::GlobalValue *OrigAddr,
1275
691
                                            bool IsForDefinition) {
1276
691
  // Emit at least a definition for the aliasee if the the address of the
1277
691
  // original function is requested.
1278
691
  if (IsForDefinition || 
OrigAddr303
)
1279
520
    (void)CGM.GetAddrOfGlobal(NewGD);
1280
691
  StringRef NewMangledName = CGM.getMangledName(NewGD);
1281
691
  llvm::GlobalValue *Addr = CGM.GetGlobalValue(NewMangledName);
1282
691
  if (Addr && 
!Addr->isDeclaration()577
) {
1283
299
    const auto *D = cast<FunctionDecl>(OldGD.getDecl());
1284
299
    const CGFunctionInfo &FI = CGM.getTypes().arrangeGlobalDeclaration(NewGD);
1285
299
    llvm::Type *DeclTy = CGM.getTypes().GetFunctionType(FI);
1286
299
1287
299
    // Create a reference to the named value.  This ensures that it is emitted
1288
299
    // if a deferred decl.
1289
299
    llvm::GlobalValue::LinkageTypes LT = CGM.getFunctionLinkage(OldGD);
1290
299
1291
299
    // Create the new alias itself, but don't set a name yet.
1292
299
    auto *GA =
1293
299
        llvm::GlobalAlias::create(DeclTy, 0, LT, "", Addr, &CGM.getModule());
1294
299
1295
299
    if (OrigAddr) {
1296
90
      assert(OrigAddr->isDeclaration() && "Expected declaration");
1297
90
1298
90
      GA->takeName(OrigAddr);
1299
90
      OrigAddr->replaceAllUsesWith(
1300
90
          llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType()));
1301
90
      OrigAddr->eraseFromParent();
1302
209
    } else {
1303
209
      GA->setName(CGM.getMangledName(OldGD));
1304
209
    }
1305
299
1306
299
    // Set attributes which are particular to an alias; this is a
1307
299
    // specialization of the attributes which may be set on a global function.
1308
299
    if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
1309
299
        D->isWeakImported())
1310
0
      GA->setLinkage(llvm::Function::WeakAnyLinkage);
1311
299
1312
299
    CGM.SetCommonAttributes(OldGD, GA);
1313
299
    return true;
1314
299
  }
1315
392
  return false;
1316
392
}
1317
1318
4.21k
void CGOpenMPRuntime::clear() {
1319
4.21k
  InternalVars.clear();
1320
4.21k
  // Clean non-target variable declarations possibly used only in debug info.
1321
4.21k
  for (const auto &Data : EmittedNonTargetVariables) {
1322
12
    if (!Data.getValue().pointsToAliveValue())
1323
0
      continue;
1324
12
    auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1325
12
    if (!GV)
1326
0
      continue;
1327
12
    if (!GV->isDeclaration() || GV->getNumUses() > 0)
1328
11
      continue;
1329
1
    GV->eraseFromParent();
1330
1
  }
1331
4.21k
  // Emit aliases for the deferred aliasees.
1332
4.21k
  for (const auto &Pair : DeferredVariantFunction) {
1333
163
    StringRef MangledName = CGM.getMangledName(Pair.second.second);
1334
163
    llvm::GlobalValue *Addr = CGM.GetGlobalValue(MangledName);
1335
163
    // If not able to emit alias, just emit original declaration.
1336
163
    (void)tryEmitDeclareVariant(Pair.second.first, Pair.second.second, Addr,
1337
163
                                /*IsForDefinition=*/false);
1338
163
  }
1339
4.21k
}
1340
1341
29.7k
std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1342
29.7k
  SmallString<128> Buffer;
1343
29.7k
  llvm::raw_svector_ostream OS(Buffer);
1344
29.7k
  StringRef Sep = FirstSeparator;
1345
58.0k
  for (StringRef Part : Parts) {
1346
58.0k
    OS << Sep << Part;
1347
58.0k
    Sep = Separator;
1348
58.0k
  }
1349
29.7k
  return std::string(OS.str());
1350
29.7k
}
1351
1352
static llvm::Function *
1353
emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1354
                          const Expr *CombinerInitializer, const VarDecl *In,
1355
174
                          const VarDecl *Out, bool IsCombiner) {
1356
174
  // void .omp_combiner.(Ty *in, Ty *out);
1357
174
  ASTContext &C = CGM.getContext();
1358
174
  QualType PtrTy = C.getPointerType(Ty).withRestrict();
1359
174
  FunctionArgList Args;
1360
174
  ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1361
174
                               /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1362
174
  ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1363
174
                              /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1364
174
  Args.push_back(&OmpOutParm);
1365
174
  Args.push_back(&OmpInParm);
1366
174
  const CGFunctionInfo &FnInfo =
1367
174
      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1368
174
  llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1369
174
  std::string Name = CGM.getOpenMPRuntime().getName(
1370
174
      {IsCombiner ? 
"omp_combiner"109
:
"omp_initializer"65
, ""});
1371
174
  auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1372
174
                                    Name, &CGM.getModule());
1373
174
  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1374
174
  if (CGM.getLangOpts().Optimize) {
1375
0
    Fn->removeFnAttr(llvm::Attribute::NoInline);
1376
0
    Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1377
0
    Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1378
0
  }
1379
174
  CodeGenFunction CGF(CGM);
1380
174
  // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1381
174
  // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1382
174
  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1383
174
                    Out->getLocation());
1384
174
  CodeGenFunction::OMPPrivateScope Scope(CGF);
1385
174
  Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1386
174
  Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1387
174
    return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1388
174
        .getAddress(CGF);
1389
174
  });
1390
174
  Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1391
174
  Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1392
174
    return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1393
174
        .getAddress(CGF);
1394
174
  });
1395
174
  (void)Scope.Privatize();
1396
174
  if (!IsCombiner && 
Out->hasInit()65
&&
1397
174
      
!CGF.isTrivialInitializer(Out->getInit())41
) {
1398
41
    CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1399
41
                         Out->getType().getQualifiers(),
1400
41
                         /*IsInitializer=*/true);
1401
41
  }
1402
174
  if (CombinerInitializer)
1403
133
    CGF.EmitIgnoredExpr(CombinerInitializer);
1404
174
  Scope.ForceCleanup();
1405
174
  CGF.FinishFunction();
1406
174
  return Fn;
1407
174
}
1408
1409
void CGOpenMPRuntime::emitUserDefinedReduction(
1410
110
    CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1411
110
  if (UDRMap.count(D) > 0)
1412
1
    return;
1413
109
  llvm::Function *Combiner = emitCombinerOrInitializer(
1414
109
      CGM, D->getType(), D->getCombiner(),
1415
109
      cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()),
1416
109
      cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()),
1417
109
      /*IsCombiner=*/true);
1418
109
  llvm::Function *Initializer = nullptr;
1419
109
  if (const Expr *Init = D->getInitializer()) {
1420
65
    Initializer = emitCombinerOrInitializer(
1421
65
        CGM, D->getType(),
1422
65
        D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? 
Init24
1423
65
                                                                     : 
nullptr41
,
1424
65
        cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()),
1425
65
        cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()),
1426
65
        /*IsCombiner=*/false);
1427
65
  }
1428
109
  UDRMap.try_emplace(D, Combiner, Initializer);
1429
109
  if (CGF) {
1430
28
    auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1431
28
    Decls.second.push_back(D);
1432
28
  }
1433
109
}
1434
1435
std::pair<llvm::Function *, llvm::Function *>
1436
206
CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1437
206
  auto I = UDRMap.find(D);
1438
206
  if (I != UDRMap.end())
1439
172
    return I->second;
1440
34
  emitUserDefinedReduction(/*CGF=*/nullptr, D);
1441
34
  return UDRMap.lookup(D);
1442
34
}
1443
1444
namespace {
1445
// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1446
// Builder if one is present.
1447
struct PushAndPopStackRAII {
1448
  PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1449
                      bool HasCancel)
1450
7.99k
      : OMPBuilder(OMPBuilder) {
1451
7.99k
    if (!OMPBuilder)
1452
7.99k
      return;
1453
6
1454
6
    // The following callback is the crucial part of clangs cleanup process.
1455
6
    //
1456
6
    // NOTE:
1457
6
    // Once the OpenMPIRBuilder is used to create parallel regions (and
1458
6
    // similar), the cancellation destination (Dest below) is determined via
1459
6
    // IP. That means if we have variables to finalize we split the block at IP,
1460
6
    // use the new block (=BB) as destination to build a JumpDest (via
1461
6
    // getJumpDestInCurrentScope(BB)) which then is fed to
1462
6
    // EmitBranchThroughCleanup. Furthermore, there will not be the need
1463
6
    // to push & pop an FinalizationInfo object.
1464
6
    // The FiniCB will still be needed but at the point where the
1465
6
    // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1466
6
    auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1467
0
      assert(IP.getBlock()->end() == IP.getPoint() &&
1468
0
             "Clang CG should cause non-terminated block!");
1469
0
      CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1470
0
      CGF.Builder.restoreIP(IP);
1471
0
      CodeGenFunction::JumpDest Dest =
1472
0
          CGF.getOMPCancelDestination(OMPD_parallel);
1473
0
      CGF.EmitBranchThroughCleanup(Dest);
1474
0
    };
1475
6
1476
6
    // TODO: Remove this once we emit parallel regions through the
1477
6
    //       OpenMPIRBuilder as it can do this setup internally.
1478
6
    llvm::OpenMPIRBuilder::FinalizationInfo FI(
1479
6
        {FiniCB, OMPD_parallel, HasCancel});
1480
6
    OMPBuilder->pushFinalizationCB(std::move(FI));
1481
6
  }
1482
7.99k
  ~PushAndPopStackRAII() {
1483
7.99k
    if (OMPBuilder)
1484
6
      OMPBuilder->popFinalizationCB();
1485
7.99k
  }
1486
  llvm::OpenMPIRBuilder *OMPBuilder;
1487
};
1488
} // namespace
1489
1490
static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1491
    CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1492
    const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1493
7.99k
    const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1494
7.99k
  assert(ThreadIDVar->getType()->isPointerType() &&
1495
7.99k
         "thread id variable must be of type kmp_int32 *");
1496
7.99k
  CodeGenFunction CGF(CGM, true);
1497
7.99k
  bool HasCancel = false;
1498
7.99k
  if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1499
706
    HasCancel = OPD->hasCancel();
1500
7.29k
  else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1501
16
    HasCancel = OPSD->hasCancel();
1502
7.27k
  else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1503
147
    HasCancel = OPFD->hasCancel();
1504
7.12k
  else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1505
337
    HasCancel = OPFD->hasCancel();
1506
6.79k
  else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1507
307
    HasCancel = OPFD->hasCancel();
1508
6.48k
  else if (const auto *OPFD =
1509
436
               dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1510
436
    HasCancel = OPFD->hasCancel();
1511
6.04k
  else if (const auto *OPFD =
1512
738
               dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1513
738
    HasCancel = OPFD->hasCancel();
1514
7.99k
1515
7.99k
  // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1516
7.99k
  //       parallel region to make cancellation barriers work properly.
1517
7.99k
  llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder();
1518
7.99k
  PushAndPopStackRAII PSR(OMPBuilder, CGF, HasCancel);
1519
7.99k
  CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1520
7.99k
                                    HasCancel, OutlinedHelperName);
1521
7.99k
  CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1522
7.99k
  return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D.getBeginLoc());
1523
7.99k
}
1524
1525
llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1526
    const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1527
4.13k
    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1528
4.13k
  const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1529
4.13k
  return emitParallelOrTeamsOutlinedFunction(
1530
4.13k
      CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1531
4.13k
}
1532
1533
llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1534
    const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1535
3.86k
    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1536
3.86k
  const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1537
3.86k
  return emitParallelOrTeamsOutlinedFunction(
1538
3.86k
      CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1539
3.86k
}
1540
1541
llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1542
    const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1543
    const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1544
    OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1545
635
    bool Tied, unsigned &NumberOfParts) {
1546
635
  auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1547
635
                                              PrePostActionTy &) {
1548
28
    llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc());
1549
28
    llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc());
1550
28
    llvm::Value *TaskArgs[] = {
1551
28
        UpLoc, ThreadID,
1552
28
        CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1553
28
                                    TaskTVar->getType()->castAs<PointerType>())
1554
28
            .getPointer(CGF)};
1555
28
    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1556
28
  };
1557
635
  CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1558
635
                                                            UntiedCodeGen);
1559
635
  CodeGen.setAction(Action);
1560
635
  assert(!ThreadIDVar->getType()->isPointerType() &&
1561
635
         "thread id variable must be of type kmp_int32 for tasks");
1562
635
  const OpenMPDirectiveKind Region =
1563
635
      isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? 
OMPD_taskloop222
1564
635
                                                      : 
OMPD_task413
;
1565
635
  const CapturedStmt *CS = D.getCapturedStmt(Region);
1566
635
  bool HasCancel = false;
1567
635
  if (const auto *TD = dyn_cast<OMPTaskDirective>(&D))
1568
121
    HasCancel = TD->hasCancel();
1569
514
  else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D))
1570
35
    HasCancel = TD->hasCancel();
1571
479
  else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D))
1572
35
    HasCancel = TD->hasCancel();
1573
444
  else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D))
1574
33
    HasCancel = TD->hasCancel();
1575
635
1576
635
  CodeGenFunction CGF(CGM, true);
1577
635
  CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1578
635
                                        InnermostKind, HasCancel, Action);
1579
635
  CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1580
635
  llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1581
635
  if (!Tied)
1582
16
    NumberOfParts = Action.getNumberOfParts();
1583
635
  return Res;
1584
635
}
1585
1586
static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1587
                             const RecordDecl *RD, const CGRecordLayout &RL,
1588
9.85k
                             ArrayRef<llvm::Constant *> Data) {
1589
9.85k
  llvm::StructType *StructTy = RL.getLLVMType();
1590
9.85k
  unsigned PrevIdx = 0;
1591
9.85k
  ConstantInitBuilder CIBuilder(CGM);
1592
9.85k
  auto DI = Data.begin();
1593
49.2k
  for (const FieldDecl *FD : RD->fields()) {
1594
49.2k
    unsigned Idx = RL.getLLVMFieldNo(FD);
1595
49.2k
    // Fill the alignment.
1596
49.2k
    for (unsigned I = PrevIdx; I < Idx; 
++I0
)
1597
0
      Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1598
49.2k
    PrevIdx = Idx + 1;
1599
49.2k
    Fields.add(*DI);
1600
49.2k
    ++DI;
1601
49.2k
  }
1602
9.85k
}
1603
1604
template <class... As>
1605
static llvm::GlobalVariable *
1606
createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant,
1607
                   ArrayRef<llvm::Constant *> Data, const Twine &Name,
1608
9.85k
                   As &&... Args) {
1609
9.85k
  const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1610
9.85k
  const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1611
9.85k
  ConstantInitBuilder CIBuilder(CGM);
1612
9.85k
  ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1613
9.85k
  buildStructValue(Fields, CGM, RD, RL, Data);
1614
9.85k
  return Fields.finishAndCreateGlobal(
1615
9.85k
      Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant,
1616
9.85k
      std::forward<As>(Args)...);
1617
9.85k
}
1618
1619
template <typename T>
1620
static void
1621
createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1622
                                         ArrayRef<llvm::Constant *> Data,
1623
                                         T &Parent) {
1624
  const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1625
  const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1626
  ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1627
  buildStructValue(Fields, CGM, RD, RL, Data);
1628
  Fields.finishAndAddTo(Parent);
1629
}
1630
1631
25.8k
Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1632
25.8k
  CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1633
25.8k
  unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1634
25.8k
  FlagsTy FlagsKey(Flags, Reserved2Flags);
1635
25.8k
  llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1636
25.8k
  if (!Entry) {
1637
3.56k
    if (!DefaultOpenMPPSource) {
1638
1.67k
      // Initialize default location for psource field of ident_t structure of
1639
1.67k
      // all ident_t objects. Format is ";file;function;line;column;;".
1640
1.67k
      // Taken from
1641
1.67k
      // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp_str.cpp
1642
1.67k
      DefaultOpenMPPSource =
1643
1.67k
          CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1644
1.67k
      DefaultOpenMPPSource =
1645
1.67k
          llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1646
1.67k
    }
1647
3.56k
1648
3.56k
    llvm::Constant *Data[] = {
1649
3.56k
        llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1650
3.56k
        llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1651
3.56k
        llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags),
1652
3.56k
        llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource};
1653
3.56k
    llvm::GlobalValue *DefaultOpenMPLocation =
1654
3.56k
        createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "",
1655
3.56k
                           llvm::GlobalValue::PrivateLinkage);
1656
3.56k
    DefaultOpenMPLocation->setUnnamedAddr(
1657
3.56k
        llvm::GlobalValue::UnnamedAddr::Global);
1658
3.56k
1659
3.56k
    OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1660
3.56k
  }
1661
25.8k
  return Address(Entry, Align);
1662
25.8k
}
1663
1664
void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1665
1.84k
                                             bool AtCurrentPoint) {
1666
1.84k
  auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1667
1.84k
  assert(!Elem.second.ServiceInsertPt && "Insert point is set already.");
1668
1.84k
1669
1.84k
  llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty);
1670
1.84k
  if (AtCurrentPoint) {
1671
699
    Elem.second.ServiceInsertPt = new llvm::BitCastInst(
1672
699
        Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock());
1673
1.14k
  } else {
1674
1.14k
    Elem.second.ServiceInsertPt =
1675
1.14k
        new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1676
1.14k
    Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt);
1677
1.14k
  }
1678
1.84k
}
1679
1680
7.13k
void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1681
7.13k
  auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1682
7.13k
  if (Elem.second.ServiceInsertPt) {
1683
1.84k
    llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1684
1.84k
    Elem.second.ServiceInsertPt = nullptr;
1685
1.84k
    Ptr->eraseFromParent();
1686
1.84k
  }
1687
7.13k
}
1688
1689
llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1690
                                                 SourceLocation Loc,
1691
26.2k
                                                 unsigned Flags) {
1692
26.2k
  Flags |= OMP_IDENT_KMPC;
1693
26.2k
  // If no debug info is generated - return global default location.
1694
26.2k
  if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1695
26.2k
      
Loc.isInvalid()698
)
1696
25.5k
    return getOrCreateDefaultLocation(Flags).getPointer();
1697
624
1698
624
  assert(CGF.CurFn && "No function in current CodeGenFunction.");
1699
624
1700
624
  CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1701
624
  Address LocValue = Address::invalid();
1702
624
  auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1703
624
  if (I != OpenMPLocThreadIDMap.end())
1704
415
    LocValue = Address(I->second.DebugLoc, Align);
1705
624
1706
624
  // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1707
624
  // GetOpenMPThreadID was called before this routine.
1708
624
  if (!LocValue.isValid()) {
1709
227
    // Generate "ident_t .kmpc_loc.addr;"
1710
227
    Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1711
227
    auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1712
227
    Elem.second.DebugLoc = AI.getPointer();
1713
227
    LocValue = AI;
1714
227
1715
227
    if (!Elem.second.ServiceInsertPt)
1716
209
      setLocThreadIdInsertPt(CGF);
1717
227
    CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1718
227
    CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1719
227
    CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1720
227
                             CGF.getTypeSize(IdentQTy));
1721
227
  }
1722
624
1723
624
  // char **psource = &.kmpc_loc_<flags>.addr.psource;
1724
624
  LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1725
624
  auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1726
624
  LValue PSource =
1727
624
      CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1728
624
1729
624
  llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1730
624
  if (OMPDebugLoc == nullptr) {
1731
368
    SmallString<128> Buffer2;
1732
368
    llvm::raw_svector_ostream OS2(Buffer2);
1733
368
    // Build debug location
1734
368
    PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1735
368
    OS2 << ";" << PLoc.getFilename() << ";";
1736
368
    if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1737
350
      OS2 << FD->getQualifiedNameAsString();
1738
368
    OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1739
368
    OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1740
368
    OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1741
368
  }
1742
624
  // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1743
624
  CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1744
624
1745
624
  // Our callers always pass this to a runtime function, so for
1746
624
  // convenience, go ahead and return a naked pointer.
1747
624
  return LocValue.getPointer();
1748
624
}
1749
1750
llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1751
18.1k
                                          SourceLocation Loc) {
1752
18.1k
  assert(CGF.CurFn && "No function in current CodeGenFunction.");
1753
18.1k
1754
18.1k
  llvm::Value *ThreadID = nullptr;
1755
18.1k
  // Check whether we've already cached a load of the thread id in this
1756
18.1k
  // function.
1757
18.1k
  auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1758
18.1k
  if (I != OpenMPLocThreadIDMap.end()) {
1759
8.45k
    ThreadID = I->second.ThreadID;
1760
8.45k
    if (ThreadID != nullptr)
1761
7.67k
      return ThreadID;
1762
10.4k
  }
1763
10.4k
  // If exceptions are enabled, do not use parameter to avoid possible crash.
1764
10.4k
  if (auto *OMPRegionInfo =
1765
9.97k
          dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1766
9.97k
    if (OMPRegionInfo->getThreadIDVariable()) {
1767
8.88k
      // Check if this an outlined function with thread id passed as argument.
1768
8.88k
      LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1769
8.88k
      llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1770
8.88k
      if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1771
8.88k
          
!CGF.getLangOpts().CXXExceptions361
||
1772
8.88k
          
CGF.Builder.GetInsertBlock() == TopBlock361
||
1773
8.88k
          
!isa<llvm::Instruction>(LVal.getPointer(CGF))106
||
1774
8.88k
          cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1775
106
              TopBlock ||
1776
8.88k
          cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() ==
1777
8.88k
              CGF.Builder.GetInsertBlock()) {
1778
8.88k
        ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1779
8.88k
        // If value loaded in entry block, cache it and use it everywhere in
1780
8.88k
        // function.
1781
8.88k
        if (CGF.Builder.GetInsertBlock() == TopBlock) {
1782
4.66k
          auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1783
4.66k
          Elem.second.ThreadID = ThreadID;
1784
4.66k
        }
1785
8.88k
        return ThreadID;
1786
8.88k
      }
1787
1.60k
    }
1788
9.97k
  }
1789
1.60k
1790
1.60k
  // This is not an outlined function region - need to call __kmpc_int32
1791
1.60k
  // kmpc_global_thread_num(ident_t *loc).
1792
1.60k
  // Generate thread id value and cache this value for use across the
1793
1.60k
  // function.
1794
1.60k
  auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1795
1.60k
  if (!Elem.second.ServiceInsertPt)
1796
934
    setLocThreadIdInsertPt(CGF);
1797
1.60k
  CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1798
1.60k
  CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1799
1.60k
  llvm::CallInst *Call = CGF.Builder.CreateCall(
1800
1.60k
      createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1801
1.60k
      emitUpdateLocation(CGF, Loc));
1802
1.60k
  Call->setCallingConv(CGF.getRuntimeCC());
1803
1.60k
  Elem.second.ThreadID = Call;
1804
1.60k
  return Call;
1805
1.60k
}
1806
1807
47.4k
void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1808
47.4k
  assert(CGF.CurFn && "No function in current CodeGenFunction.");
1809
47.4k
  if (OpenMPLocThreadIDMap.count(CGF.CurFn)) {
1810
6.43k
    clearLocThreadIdInsertPt(CGF);
1811
6.43k
    OpenMPLocThreadIDMap.erase(CGF.CurFn);
1812
6.43k
  }
1813
47.4k
  if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1814
20
    for(const auto *D : FunctionUDRMap[CGF.CurFn])
1815
28
      UDRMap.erase(D);
1816
20
    FunctionUDRMap.erase(CGF.CurFn);
1817
20
  }
1818
47.4k
  auto I = FunctionUDMMap.find(CGF.CurFn);
1819
47.4k
  if (I != FunctionUDMMap.end()) {
1820
0
    for(const auto *D : I->second)
1821
0
      UDMMap.erase(D);
1822
0
    FunctionUDMMap.erase(I);
1823
0
  }
1824
47.4k
  LastprivateConditionalToTypes.erase(CGF.CurFn);
1825
47.4k
}
1826
1827
28.5k
llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1828
28.5k
  return IdentTy->getPointerTo();
1829
28.5k
}
1830
1831
14.1k
llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1832
14.1k
  if (!Kmpc_MicroTy) {
1833
1.42k
    // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1834
1.42k
    llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1835
1.42k
                                 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1836
1.42k
    Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1837
1.42k
  }
1838
14.1k
  return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1839
14.1k
}
1840
1841
29.5k
llvm::FunctionCallee CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1842
29.5k
  llvm::FunctionCallee RTLFn = nullptr;
1843
29.5k
  switch (static_cast<OpenMPRTLFunction>(Function)) {
1844
3.52k
  case OMPRTL__kmpc_fork_call: {
1845
3.52k
    // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1846
3.52k
    // microtask, ...);
1847
3.52k
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1848
3.52k
                                getKmpc_MicroPointerTy()};
1849
3.52k
    auto *FnTy =
1850
3.52k
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1851
3.52k
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1852
3.52k
    if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1853
3.52k
      if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1854
935
        llvm::LLVMContext &Ctx = F->getContext();
1855
935
        llvm::MDBuilder MDB(Ctx);
1856
935
        // Annotate the callback behavior of the __kmpc_fork_call:
1857
935
        //  - The callback callee is argument number 2 (microtask).
1858
935
        //  - The first two arguments of the callback callee are unknown (-1).
1859
935
        //  - All variadic arguments to the __kmpc_fork_call are passed to the
1860
935
        //    callback callee.
1861
935
        F->addMetadata(
1862
935
            llvm::LLVMContext::MD_callback,
1863
935
            *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1864
935
                                        2, {-1, -1},
1865
935
                                        /* VarArgsArePassed */ true)}));
1866
935
      }
1867
3.52k
    }
1868
3.52k
    break;
1869
0
  }
1870
1.62k
  case OMPRTL__kmpc_global_thread_num: {
1871
1.62k
    // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1872
1.62k
    llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1873
1.62k
    auto *FnTy =
1874
1.62k
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1875
1.62k
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1876
1.62k
    break;
1877
0
  }
1878
181
  case OMPRTL__kmpc_threadprivate_cached: {
1879
181
    // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1880
181
    // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1881
181
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1882
181
                                CGM.VoidPtrTy, CGM.SizeTy,
1883
181
                                CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1884
181
    auto *FnTy =
1885
181
        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1886
181
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1887
181
    break;
1888
0
  }
1889
133
  case OMPRTL__kmpc_critical: {
1890
133
    // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1891
133
    // kmp_critical_name *crit);
1892
133
    llvm::Type *TypeParams[] = {
1893
133
        getIdentTyPointerTy(), CGM.Int32Ty,
1894
133
        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1895
133
    auto *FnTy =
1896
133
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1897
133
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1898
133
    break;
1899
0
  }
1900
3
  case OMPRTL__kmpc_critical_with_hint: {
1901
3
    // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1902
3
    // kmp_critical_name *crit, uintptr_t hint);
1903
3
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1904
3
                                llvm::PointerType::getUnqual(KmpCriticalNameTy),
1905
3
                                CGM.IntPtrTy};
1906
3
    auto *FnTy =
1907
3
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1908
3
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1909
3
    break;
1910
0
  }
1911
25
  case OMPRTL__kmpc_threadprivate_register: {
1912
25
    // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1913
25
    // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1914
25
    // typedef void *(*kmpc_ctor)(void *);
1915
25
    auto *KmpcCtorTy =
1916
25
        llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1917
25
                                /*isVarArg*/ false)->getPointerTo();
1918
25
    // typedef void *(*kmpc_cctor)(void *, void *);
1919
25
    llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1920
25
    auto *KmpcCopyCtorTy =
1921
25
        llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1922
25
                                /*isVarArg*/ false)
1923
25
            ->getPointerTo();
1924
25
    // typedef void (*kmpc_dtor)(void *);
1925
25
    auto *KmpcDtorTy =
1926
25
        llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1927
25
            ->getPointerTo();
1928
25
    llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1929
25
                              KmpcCopyCtorTy, KmpcDtorTy};
1930
25
    auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1931
25
                                        /*isVarArg*/ false);
1932
25
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1933
25
    break;
1934
0
  }
1935
136
  case OMPRTL__kmpc_end_critical: {
1936
136
    // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1937
136
    // kmp_critical_name *crit);
1938
136
    llvm::Type *TypeParams[] = {
1939
136
        getIdentTyPointerTy(), CGM.Int32Ty,
1940
136
        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1941
136
    auto *FnTy =
1942
136
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1943
136
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1944
136
    break;
1945
0
  }
1946
2
  case OMPRTL__kmpc_cancel_barrier: {
1947
2
    // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1948
2
    // global_tid);
1949
2
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1950
2
    auto *FnTy =
1951
2
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1952
2
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1953
2
    break;
1954
0
  }
1955
651
  case OMPRTL__kmpc_barrier: {
1956
651
    // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1957
651
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1958
651
    auto *FnTy =
1959
651
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1960
651
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1961
651
    break;
1962
0
  }
1963
5.94k
  case OMPRTL__kmpc_for_static_fini: {
1964
5.94k
    // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1965
5.94k
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1966
5.94k
    auto *FnTy =
1967
5.94k
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1968
5.94k
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1969
5.94k
    break;
1970
0
  }
1971
131
  case OMPRTL__kmpc_push_num_threads: {
1972
131
    // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1973
131
    // kmp_int32 num_threads)
1974
131
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1975
131
                                CGM.Int32Ty};
1976
131
    auto *FnTy =
1977
131
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1978
131
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1979
131
    break;
1980
0
  }
1981
227
  case OMPRTL__kmpc_serialized_parallel: {
1982
227
    // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1983
227
    // global_tid);
1984
227
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1985
227
    auto *FnTy =
1986
227
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1987
227
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1988
227
    break;
1989
0
  }
1990
227
  case OMPRTL__kmpc_end_serialized_parallel: {
1991
227
    // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1992
227
    // global_tid);
1993
227
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1994
227
    auto *FnTy =
1995
227
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1996
227
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1997
227
    break;
1998
0
  }
1999
84
  case OMPRTL__kmpc_flush: {
2000
84
    // Build void __kmpc_flush(ident_t *loc);
2001
84
    llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
2002
84
    auto *FnTy =
2003
84
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2004
84
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
2005
84
    break;
2006
0
  }
2007
180
  case OMPRTL__kmpc_master: {
2008
180
    // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
2009
180
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2010
180
    auto *FnTy =
2011
180
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2012
180
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
2013
180
    break;
2014
0
  }
2015
180
  case OMPRTL__kmpc_end_master: {
2016
180
    // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
2017
180
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2018
180
    auto *FnTy =
2019
180
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2020
180
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
2021
180
    break;
2022
0
  }
2023
8
  case OMPRTL__kmpc_omp_taskyield: {
2024
8
    // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
2025
8
    // int end_part);
2026
8
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2027
8
    auto *FnTy =
2028
8
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2029
8
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
2030
8
    break;
2031
0
  }
2032
39
  case OMPRTL__kmpc_single: {
2033
39
    // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
2034
39
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2035
39
    auto *FnTy =
2036
39
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2037
39
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
2038
39
    break;
2039
0
  }
2040
39
  case OMPRTL__kmpc_end_single: {
2041
39
    // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
2042
39
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2043
39
    auto *FnTy =
2044
39
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2045
39
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
2046
39
    break;
2047
0
  }
2048
499
  case OMPRTL__kmpc_omp_task_alloc: {
2049
499
    // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
2050
499
    // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2051
499
    // kmp_routine_entry_t *task_entry);
2052
499
    assert(KmpRoutineEntryPtrTy != nullptr &&
2053
499
           "Type kmp_routine_entry_t must be created.");
2054
499
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2055
499
                                CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
2056
499
    // Return void * and then cast to particular kmp_task_t type.
2057
499
    auto *FnTy =
2058
499
        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2059
499
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
2060
499
    break;
2061
0
  }
2062
136
  case OMPRTL__kmpc_omp_target_task_alloc: {
2063
136
    // Build kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *, kmp_int32 gtid,
2064
136
    // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2065
136
    // kmp_routine_entry_t *task_entry, kmp_int64 device_id);
2066
136
    assert(KmpRoutineEntryPtrTy != nullptr &&
2067
136
           "Type kmp_routine_entry_t must be created.");
2068
136
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2069
136
                                CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy,
2070
136
                                CGM.Int64Ty};
2071
136
    // Return void * and then cast to particular kmp_task_t type.
2072
136
    auto *FnTy =
2073
136
        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2074
136
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_target_task_alloc");
2075
136
    break;
2076
0
  }
2077
119
  case OMPRTL__kmpc_omp_task: {
2078
119
    // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2079
119
    // *new_task);
2080
119
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2081
119
                                CGM.VoidPtrTy};
2082
119
    auto *FnTy =
2083
119
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2084
119
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
2085
119
    break;
2086
0
  }
2087
18
  case OMPRTL__kmpc_copyprivate: {
2088
18
    // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
2089
18
    // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
2090
18
    // kmp_int32 didit);
2091
18
    llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2092
18
    auto *CpyFnTy =
2093
18
        llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
2094
18
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
2095
18
                                CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
2096
18
                                CGM.Int32Ty};
2097
18
    auto *FnTy =
2098
18
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2099
18
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
2100
18
    break;
2101
0
  }
2102
179
  case OMPRTL__kmpc_reduce: {
2103
179
    // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
2104
179
    // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
2105
179
    // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
2106
179
    llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2107
179
    auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2108
179
                                               /*isVarArg=*/false);
2109
179
    llvm::Type *TypeParams[] = {
2110
179
        getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2111
179
        CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2112
179
        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2113
179
    auto *FnTy =
2114
179
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2115
179
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
2116
179
    break;
2117
0
  }
2118
181
  case OMPRTL__kmpc_reduce_nowait: {
2119
181
    // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
2120
181
    // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
2121
181
    // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
2122
181
    // *lck);
2123
181
    llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2124
181
    auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
2125
181
                                               /*isVarArg=*/false);
2126
181
    llvm::Type *TypeParams[] = {
2127
181
        getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
2128
181
        CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
2129
181
        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2130
181
    auto *FnTy =
2131
181
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2132
181
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
2133
181
    break;
2134
0
  }
2135
358
  case OMPRTL__kmpc_end_reduce: {
2136
358
    // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
2137
358
    // kmp_critical_name *lck);
2138
358
    llvm::Type *TypeParams[] = {
2139
358
        getIdentTyPointerTy(), CGM.Int32Ty,
2140
358
        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2141
358
    auto *FnTy =
2142
358
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2143
358
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
2144
358
    break;
2145
0
  }
2146
181
  case OMPRTL__kmpc_end_reduce_nowait: {
2147
181
    // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
2148
181
    // kmp_critical_name *lck);
2149
181
    llvm::Type *TypeParams[] = {
2150
181
        getIdentTyPointerTy(), CGM.Int32Ty,
2151
181
        llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2152
181
    auto *FnTy =
2153
181
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2154
181
    RTLFn =
2155
181
        CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
2156
181
    break;
2157
0
  }
2158
174
  case OMPRTL__kmpc_omp_task_begin_if0: {
2159
174
    // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2160
174
    // *new_task);
2161
174
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2162
174
                                CGM.VoidPtrTy};
2163
174
    auto *FnTy =
2164
174
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2165
174
    RTLFn =
2166
174
        CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
2167
174
    break;
2168
0
  }
2169
174
  case OMPRTL__kmpc_omp_task_complete_if0: {
2170
174
    // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2171
174
    // *new_task);
2172
174
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2173
174
                                CGM.VoidPtrTy};
2174
174
    auto *FnTy =
2175
174
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2176
174
    RTLFn = CGM.CreateRuntimeFunction(FnTy,
2177
174
                                      /*Name=*/"__kmpc_omp_task_complete_if0");
2178
174
    break;
2179
0
  }
2180
8
  case OMPRTL__kmpc_ordered: {
2181
8
    // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
2182
8
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2183
8
    auto *FnTy =
2184
8
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2185
8
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
2186
8
    break;
2187
0
  }
2188
8
  case OMPRTL__kmpc_end_ordered: {
2189
8
    // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
2190
8
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2191
8
    auto *FnTy =
2192
8
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2193
8
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
2194
8
    break;
2195
0
  }
2196
6
  case OMPRTL__kmpc_omp_taskwait: {
2197
6
    // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
2198
6
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2199
6
    auto *FnTy =
2200
6
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2201
6
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
2202
6
    break;
2203
0
  }
2204
243
  case OMPRTL__kmpc_taskgroup: {
2205
243
    // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
2206
243
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2207
243
    auto *FnTy =
2208
243
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2209
243
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
2210
243
    break;
2211
0
  }
2212
243
  case OMPRTL__kmpc_end_taskgroup: {
2213
243
    // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
2214
243
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2215
243
    auto *FnTy =
2216
243
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2217
243
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
2218
243
    break;
2219
0
  }
2220
58
  case OMPRTL__kmpc_push_proc_bind: {
2221
58
    // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
2222
58
    // int proc_bind)
2223
58
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2224
58
    auto *FnTy =
2225
58
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2226
58
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2227
58
    break;
2228
0
  }
2229
160
  case OMPRTL__kmpc_omp_task_with_deps: {
2230
160
    // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2231
160
    // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2232
160
    // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2233
160
    llvm::Type *TypeParams[] = {
2234
160
        getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2235
160
        CGM.VoidPtrTy,         CGM.Int32Ty, CGM.VoidPtrTy};
2236
160
    auto *FnTy =
2237
160
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2238
160
    RTLFn =
2239
160
        CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2240
160
    break;
2241
0
  }
2242
164
  case OMPRTL__kmpc_omp_wait_deps: {
2243
164
    // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2244
164
    // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2245
164
    // kmp_depend_info_t *noalias_dep_list);
2246
164
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2247
164
                                CGM.Int32Ty,           CGM.VoidPtrTy,
2248
164
                                CGM.Int32Ty,           CGM.VoidPtrTy};
2249
164
    auto *FnTy =
2250
164
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2251
164
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2252
164
    break;
2253
0
  }
2254
38
  case OMPRTL__kmpc_cancellationpoint: {
2255
38
    // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2256
38
    // global_tid, kmp_int32 cncl_kind)
2257
38
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2258
38
    auto *FnTy =
2259
38
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2260
38
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2261
38
    break;
2262
0
  }
2263
90
  case OMPRTL__kmpc_cancel: {
2264
90
    // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2265
90
    // kmp_int32 cncl_kind)
2266
90
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2267
90
    auto *FnTy =
2268
90
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2269
90
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2270
90
    break;
2271
0
  }
2272
271
  case OMPRTL__kmpc_push_num_teams: {
2273
271
    // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2274
271
    // kmp_int32 num_teams, kmp_int32 num_threads)
2275
271
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2276
271
        CGM.Int32Ty};
2277
271
    auto *FnTy =
2278
271
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2279
271
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2280
271
    break;
2281
0
  }
2282
3.57k
  case OMPRTL__kmpc_fork_teams: {
2283
3.57k
    // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2284
3.57k
    // microtask, ...);
2285
3.57k
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2286
3.57k
                                getKmpc_MicroPointerTy()};
2287
3.57k
    auto *FnTy =
2288
3.57k
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2289
3.57k
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2290
3.57k
    if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2291
3.57k
      if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2292
984
        llvm::LLVMContext &Ctx = F->getContext();
2293
984
        llvm::MDBuilder MDB(Ctx);
2294
984
        // Annotate the callback behavior of the __kmpc_fork_teams:
2295
984
        //  - The callback callee is argument number 2 (microtask).
2296
984
        //  - The first two arguments of the callback callee are unknown (-1).
2297
984
        //  - All variadic arguments to the __kmpc_fork_teams are passed to the
2298
984
        //    callback callee.
2299
984
        F->addMetadata(
2300
984
            llvm::LLVMContext::MD_callback,
2301
984
            *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2302
984
                                        2, {-1, -1},
2303
984
                                        /* VarArgsArePassed */ true)}));
2304
984
      }
2305
3.57k
    }
2306
3.57k
    break;
2307
0
  }
2308
222
  case OMPRTL__kmpc_taskloop: {
2309
222
    // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2310
222
    // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2311
222
    // sched, kmp_uint64 grainsize, void *task_dup);
2312
222
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2313
222
                                CGM.IntTy,
2314
222
                                CGM.VoidPtrTy,
2315
222
                                CGM.IntTy,
2316
222
                                CGM.Int64Ty->getPointerTo(),
2317
222
                                CGM.Int64Ty->getPointerTo(),
2318
222
                                CGM.Int64Ty,
2319
222
                                CGM.IntTy,
2320
222
                                CGM.IntTy,
2321
222
                                CGM.Int64Ty,
2322
222
                                CGM.VoidPtrTy};
2323
222
    auto *FnTy =
2324
222
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2325
222
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2326
222
    break;
2327
0
  }
2328
12
  case OMPRTL__kmpc_doacross_init: {
2329
12
    // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2330
12
    // num_dims, struct kmp_dim *dims);
2331
12
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2332
12
                                CGM.Int32Ty,
2333
12
                                CGM.Int32Ty,
2334
12
                                CGM.VoidPtrTy};
2335
12
    auto *FnTy =
2336
12
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2337
12
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2338
12
    break;
2339
0
  }
2340
12
  case OMPRTL__kmpc_doacross_fini: {
2341
12
    // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2342
12
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2343
12
    auto *FnTy =
2344
12
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2345
12
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2346
12
    break;
2347
0
  }
2348
6
  case OMPRTL__kmpc_doacross_post: {
2349
6
    // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2350
6
    // *vec);
2351
6
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2352
6
                                CGM.Int64Ty->getPointerTo()};
2353
6
    auto *FnTy =
2354
6
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2355
6
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2356
6
    break;
2357
0
  }
2358
8
  case OMPRTL__kmpc_doacross_wait: {
2359
8
    // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2360
8
    // *vec);
2361
8
    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2362
8
                                CGM.Int64Ty->getPointerTo()};
2363
8
    auto *FnTy =
2364
8
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2365
8
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2366
8
    break;
2367
0
  }
2368
32
  case OMPRTL__kmpc_task_reduction_init: {
2369
32
    // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2370
32
    // *data);
2371
32
    llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2372
32
    auto *FnTy =
2373
32
        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2374
32
    RTLFn =
2375
32
        CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2376
32
    break;
2377
0
  }
2378
44
  case OMPRTL__kmpc_task_reduction_get_th_data: {
2379
44
    // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2380
44
    // *d);
2381
44
    llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2382
44
    auto *FnTy =
2383
44
        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2384
44
    RTLFn = CGM.CreateRuntimeFunction(
2385
44
        FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2386
44
    break;
2387
0
  }
2388
40
  case OMPRTL__kmpc_alloc: {
2389
40
    // Build to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t
2390
40
    // al); omp_allocator_handle_t type is void *.
2391
40
    llvm::Type *TypeParams[] = {CGM.IntTy, CGM.SizeTy, CGM.VoidPtrTy};
2392
40
    auto *FnTy =
2393
40
        llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2394
40
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_alloc");
2395
40
    break;
2396
0
  }
2397
40
  case OMPRTL__kmpc_free: {
2398
40
    // Build to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t
2399
40
    // al); omp_allocator_handle_t type is void *.
2400
40
    llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2401
40
    auto *FnTy =
2402
40
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2403
40
    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_free");
2404
40
    break;
2405
0
  }
2406
2.39k
  case OMPRTL__kmpc_push_target_tripcount: {
2407
2.39k
    // Build void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64
2408
2.39k
    // size);
2409
2.39k
    llvm::Type *TypeParams[] = {CGM.Int64Ty, CGM.Int64Ty};
2410
2.39k
    llvm::FunctionType *FnTy =
2411
2.39k
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2412
2.39k
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_target_tripcount");
2413
2.39k
    break;
2414
0
  }
2415
1.02k
  case OMPRTL__tgt_target: {
2416
1.02k
    // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2417
1.02k
    // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2418
1.02k
    // *arg_types);
2419
1.02k
    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2420
1.02k
                                CGM.VoidPtrTy,
2421
1.02k
                                CGM.Int32Ty,
2422
1.02k
                                CGM.VoidPtrPtrTy,
2423
1.02k
                                CGM.VoidPtrPtrTy,
2424
1.02k
                                CGM.Int64Ty->getPointerTo(),
2425
1.02k
                                CGM.Int64Ty->getPointerTo()};
2426
1.02k
    auto *FnTy =
2427
1.02k
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2428
1.02k
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2429
1.02k
    break;
2430
0
  }
2431
16
  case OMPRTL__tgt_target_nowait: {
2432
16
    // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2433
16
    // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2434
16
    // int64_t *arg_types);
2435
16
    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2436
16
                                CGM.VoidPtrTy,
2437
16
                                CGM.Int32Ty,
2438
16
                                CGM.VoidPtrPtrTy,
2439
16
                                CGM.VoidPtrPtrTy,
2440
16
                                CGM.Int64Ty->getPointerTo(),
2441
16
                                CGM.Int64Ty->getPointerTo()};
2442
16
    auto *FnTy =
2443
16
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2444
16
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2445
16
    break;
2446
0
  }
2447
3.60k
  case OMPRTL__tgt_target_teams: {
2448
3.60k
    // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2449
3.60k
    // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes,
2450
3.60k
    // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2451
3.60k
    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2452
3.60k
                                CGM.VoidPtrTy,
2453
3.60k
                                CGM.Int32Ty,
2454
3.60k
                                CGM.VoidPtrPtrTy,
2455
3.60k
                                CGM.VoidPtrPtrTy,
2456
3.60k
                                CGM.Int64Ty->getPointerTo(),
2457
3.60k
                                CGM.Int64Ty->getPointerTo(),
2458
3.60k
                                CGM.Int32Ty,
2459
3.60k
                                CGM.Int32Ty};
2460
3.60k
    auto *FnTy =
2461
3.60k
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2462
3.60k
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2463
3.60k
    break;
2464
0
  }
2465
114
  case OMPRTL__tgt_target_teams_nowait: {
2466
114
    // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2467
114
    // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t
2468
114
    // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2469
114
    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2470
114
                                CGM.VoidPtrTy,
2471
114
                                CGM.Int32Ty,
2472
114
                                CGM.VoidPtrPtrTy,
2473
114
                                CGM.VoidPtrPtrTy,
2474
114
                                CGM.Int64Ty->getPointerTo(),
2475
114
                                CGM.Int64Ty->getPointerTo(),
2476
114
                                CGM.Int32Ty,
2477
114
                                CGM.Int32Ty};
2478
114
    auto *FnTy =
2479
114
        llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2480
114
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2481
114
    break;
2482
0
  }
2483
1.30k
  case OMPRTL__tgt_register_requires: {
2484
1.30k
    // Build void __tgt_register_requires(int64_t flags);
2485
1.30k
    llvm::Type *TypeParams[] = {CGM.Int64Ty};
2486
1.30k
    auto *FnTy =
2487
1.30k
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2488
1.30k
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_requires");
2489
1.30k
    break;
2490
0
  }
2491
150
  case OMPRTL__tgt_target_data_begin: {
2492
150
    // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2493
150
    // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2494
150
    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2495
150
                                CGM.Int32Ty,
2496
150
                                CGM.VoidPtrPtrTy,
2497
150
                                CGM.VoidPtrPtrTy,
2498
150
                                CGM.Int64Ty->getPointerTo(),
2499
150
                                CGM.Int64Ty->getPointerTo()};
2500
150
    auto *FnTy =
2501
150
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2502
150
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2503
150
    break;
2504
0
  }
2505
8
  case OMPRTL__tgt_target_data_begin_nowait: {
2506
8
    // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2507
8
    // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2508
8
    // *arg_types);
2509
8
    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2510
8
                                CGM.Int32Ty,
2511
8
                                CGM.VoidPtrPtrTy,
2512
8
                                CGM.VoidPtrPtrTy,
2513
8
                                CGM.Int64Ty->getPointerTo(),
2514
8
                                CGM.Int64Ty->getPointerTo()};
2515
8
    auto *FnTy =
2516
8
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2517
8
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2518
8
    break;
2519
0
  }
2520
144
  case OMPRTL__tgt_target_data_end: {
2521
144
    // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2522
144
    // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2523
144
    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2524
144
                                CGM.Int32Ty,
2525
144
                                CGM.VoidPtrPtrTy,
2526
144
                                CGM.VoidPtrPtrTy,
2527
144
                                CGM.Int64Ty->getPointerTo(),
2528
144
                                CGM.Int64Ty->getPointerTo()};
2529
144
    auto *FnTy =
2530
144
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2531
144
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2532
144
    break;
2533
0
  }
2534
8
  case OMPRTL__tgt_target_data_end_nowait: {
2535
8
    // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2536
8
    // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2537
8
    // *arg_types);
2538
8
    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2539
8
                                CGM.Int32Ty,
2540
8
                                CGM.VoidPtrPtrTy,
2541
8
                                CGM.VoidPtrPtrTy,
2542
8
                                CGM.Int64Ty->getPointerTo(),
2543
8
                                CGM.Int64Ty->getPointerTo()};
2544
8
    auto *FnTy =
2545
8
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2546
8
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2547
8
    break;
2548
0
  }
2549
42
  case OMPRTL__tgt_target_data_update: {
2550
42
    // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2551
42
    // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
2552
42
    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2553
42
                                CGM.Int32Ty,
2554
42
                                CGM.VoidPtrPtrTy,
2555
42
                                CGM.VoidPtrPtrTy,
2556
42
                                CGM.Int64Ty->getPointerTo(),
2557
42
                                CGM.Int64Ty->getPointerTo()};
2558
42
    auto *FnTy =
2559
42
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2560
42
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2561
42
    break;
2562
0
  }
2563
8
  case OMPRTL__tgt_target_data_update_nowait: {
2564
8
    // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2565
8
    // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t
2566
8
    // *arg_types);
2567
8
    llvm::Type *TypeParams[] = {CGM.Int64Ty,
2568
8
                                CGM.Int32Ty,
2569
8
                                CGM.VoidPtrPtrTy,
2570
8
                                CGM.VoidPtrPtrTy,
2571
8
                                CGM.Int64Ty->getPointerTo(),
2572
8
                                CGM.Int64Ty->getPointerTo()};
2573
8
    auto *FnTy =
2574
8
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2575
8
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2576
8
    break;
2577
0
  }
2578
8
  case OMPRTL__tgt_mapper_num_components: {
2579
8
    // Build int64_t __tgt_mapper_num_components(void *rt_mapper_handle);
2580
8
    llvm::Type *TypeParams[] = {CGM.VoidPtrTy};
2581
8
    auto *FnTy =
2582
8
        llvm::FunctionType::get(CGM.Int64Ty, TypeParams, /*isVarArg*/ false);
2583
8
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_mapper_num_components");
2584
8
    break;
2585
0
  }
2586
36
  case OMPRTL__tgt_push_mapper_component: {
2587
36
    // Build void __tgt_push_mapper_component(void *rt_mapper_handle, void
2588
36
    // *base, void *begin, int64_t size, int64_t type);
2589
36
    llvm::Type *TypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy, CGM.VoidPtrTy,
2590
36
                                CGM.Int64Ty, CGM.Int64Ty};
2591
36
    auto *FnTy =
2592
36
        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2593
36
    RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_push_mapper_component");
2594
36
    break;
2595
29.5k
  }
2596
29.5k
  }
2597
29.5k
  assert(RTLFn && "Unable to find OpenMP runtime function");
2598
29.5k
  return RTLFn;
2599
29.5k
}
2600
2601
llvm::FunctionCallee
2602
5.90k
CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) {
2603
5.90k
  assert((IVSize == 32 || IVSize == 64) &&
2604
5.90k
         "IV size is not compatible with the omp runtime");
2605
5.90k
  StringRef Name = IVSize == 32 ? 
(IVSigned 5.65k
?
"__kmpc_for_static_init_4"5.57k
2606
5.65k
                                            : 
"__kmpc_for_static_init_4u"74
)
2607
5.90k
                                : 
(IVSigned 257
?
"__kmpc_for_static_init_8"171
2608
257
                                            : 
"__kmpc_for_static_init_8u"86
);
2609
5.90k
  llvm::Type *ITy = IVSize == 32 ? 
CGM.Int32Ty5.65k
:
CGM.Int64Ty257
;
2610
5.90k
  auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2611
5.90k
  llvm::Type *TypeParams[] = {
2612
5.90k
    getIdentTyPointerTy(),                     // loc
2613
5.90k
    CGM.Int32Ty,                               // tid
2614
5.90k
    CGM.Int32Ty,                               // schedtype
2615
5.90k
    llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2616
5.90k
    PtrTy,                                     // p_lower
2617
5.90k
    PtrTy,                                     // p_upper
2618
5.90k
    PtrTy,                                     // p_stride
2619
5.90k
    ITy,                                       // incr
2620
5.90k
    ITy                                        // chunk
2621
5.90k
  };
2622
5.90k
  auto *FnTy =
2623
5.90k
      llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2624
5.90k
  return CGM.CreateRuntimeFunction(FnTy, Name);
2625
5.90k
}
2626
2627
llvm::FunctionCallee
2628
618
CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) {
2629
618
  assert((IVSize == 32 || IVSize == 64) &&
2630
618
         "IV size is not compatible with the omp runtime");
2631
618
  StringRef Name =
2632
618
      IVSize == 32
2633
618
          ? 
(IVSigned 590
?
"__kmpc_dispatch_init_4"590
:
"__kmpc_dispatch_init_4u"0
)
2634
618
          : 
(IVSigned 28
?
"__kmpc_dispatch_init_8"10
:
"__kmpc_dispatch_init_8u"18
);
2635
618
  llvm::Type *ITy = IVSize == 32 ? 
CGM.Int32Ty590
:
CGM.Int64Ty28
;
2636
618
  llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2637
618
                               CGM.Int32Ty,           // tid
2638
618
                               CGM.Int32Ty,           // schedtype
2639
618
                               ITy,                   // lower
2640
618
                               ITy,                   // upper
2641
618
                               ITy,                   // stride
2642
618
                               ITy                    // chunk
2643
618
  };
2644
618
  auto *FnTy =
2645
618
      llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2646
618
  return CGM.CreateRuntimeFunction(FnTy, Name);
2647
618
}
2648
2649
llvm::FunctionCallee
2650
27
CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) {
2651
27
  assert((IVSize == 32 || IVSize == 64) &&
2652
27
         "IV size is not compatible with the omp runtime");
2653
27
  StringRef Name =
2654
27
      IVSize == 32
2655
27
          ? 
(IVSigned 23
?
"__kmpc_dispatch_fini_4"23
:
"__kmpc_dispatch_fini_4u"0
)
2656
27
          : 
(IVSigned 4
?
"__kmpc_dispatch_fini_8"2
:
"__kmpc_dispatch_fini_8u"2
);
2657
27
  llvm::Type *TypeParams[] = {
2658
27
      getIdentTyPointerTy(), // loc
2659
27
      CGM.Int32Ty,           // tid
2660
27
  };
2661
27
  auto *FnTy =
2662
27
      llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2663
27
  return CGM.CreateRuntimeFunction(FnTy, Name);
2664
27
}
2665
2666
llvm::FunctionCallee
2667
618
CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) {
2668
618
  assert((IVSize == 32 || IVSize == 64) &&
2669
618
         "IV size is not compatible with the omp runtime");
2670
618
  StringRef Name =
2671
618
      IVSize == 32
2672
618
          ? 
(IVSigned 590
?
"__kmpc_dispatch_next_4"590
:
"__kmpc_dispatch_next_4u"0
)
2673
618
          : 
(IVSigned 28
?
"__kmpc_dispatch_next_8"10
:
"__kmpc_dispatch_next_8u"18
);
2674
618
  llvm::Type *ITy = IVSize == 32 ? 
CGM.Int32Ty590
:
CGM.Int64Ty28
;
2675
618
  auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2676
618
  llvm::Type *TypeParams[] = {
2677
618
    getIdentTyPointerTy(),                     // loc
2678
618
    CGM.Int32Ty,                               // tid
2679
618
    llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2680
618
    PtrTy,                                     // p_lower
2681
618
    PtrTy,                                     // p_upper
2682
618
    PtrTy                                      // p_stride
2683
618
  };
2684
618
  auto *FnTy =
2685
618
      llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2686
618
  return CGM.CreateRuntimeFunction(FnTy, Name);
2687
618
}
2688
2689
/// Obtain information that uniquely identifies a target entry. This
2690
/// consists of the file and device IDs as well as line number associated with
2691
/// the relevant entry source location.
2692
static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2693
                                     unsigned &DeviceID, unsigned &FileID,
2694
9.61k
                                     unsigned &LineNum) {
2695
9.61k
  SourceManager &SM = C.getSourceManager();
2696
9.61k
2697
9.61k
  // The loc should be always valid and have a file ID (the user cannot use
2698
9.61k
  // #pragma directives in macros)
2699
9.61k
2700
9.61k
  assert(Loc.isValid() && "Source location is expected to be always valid.");
2701
9.61k
2702
9.61k
  PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2703
9.61k
  assert(PLoc.isValid() && "Source location is expected to be always valid.");
2704
9.61k
2705
9.61k
  llvm::sys::fs::UniqueID ID;
2706
9.61k
  if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2707
0
    SM.getDiagnostics().Report(diag::err_cannot_open_file)
2708
0
        << PLoc.getFilename() << EC.message();
2709
9.61k
2710
9.61k
  DeviceID = ID.getDevice();
2711
9.61k
  FileID = ID.getFile();
2712
9.61k
  LineNum = PLoc.getLine();
2713
9.61k
}
2714
2715
458
Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
2716
458
  if (CGM.getLangOpts().OpenMPSimd)
2717
20
    return Address::invalid();
2718
438
  llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2719
438
      OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2720
438
  if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
2721
438
              
(22
*Res == OMPDeclareTargetDeclAttr::MT_To22
&&
2722
438
               
HasRequiresUnifiedSharedMemory22
))) {
2723
438
    SmallString<64> PtrName;
2724
438
    {
2725
438
      llvm::raw_svector_ostream OS(PtrName);
2726
438
      OS << CGM.getMangledName(GlobalDecl(VD));
2727
438
      if (!VD->isExternallyVisible()) {
2728
34
        unsigned DeviceID, FileID, Line;
2729
34
        getTargetEntryUniqueInfo(CGM.getContext(),
2730
34
                                 VD->getCanonicalDecl()->getBeginLoc(),
2731
34
                                 DeviceID, FileID, Line);
2732
34
        OS << llvm::format("_%x", FileID);
2733
34
      }
2734
438
      OS << "_decl_tgt_ref_ptr";
2735
438
    }
2736
438
    llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2737
438
    if (!Ptr) {
2738
52
      QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2739
52
      Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2740
52
                                        PtrName);
2741
52
2742
52
      auto *GV = cast<llvm::GlobalVariable>(Ptr);
2743
52
      GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
2744
52
2745
52
      if (!CGM.getLangOpts().OpenMPIsDevice)
2746
34
        GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2747
52
      registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2748
52
    }
2749
438
    return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2750
438
  }
2751
0
  return Address::invalid();
2752
0
}
2753
2754
llvm::Constant *
2755
100
CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2756
100
  assert(!CGM.getLangOpts().OpenMPUseTLS ||
2757
100
         !CGM.getContext().getTargetInfo().isTLSSupported());
2758
100
  // Lookup the entry, lazily creating it if necessary.
2759
100
  std::string Suffix = getName({"cache", ""});
2760
100
  return getOrCreateInternalVariable(
2761
100
      CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2762
100
}
2763
2764
Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2765
                                                const VarDecl *VD,
2766
                                                Address VDAddr,
2767
206
                                                SourceLocation Loc) {
2768
206
  if (CGM.getLangOpts().OpenMPUseTLS &&
2769
206
      
CGM.getContext().getTargetInfo().isTLSSupported()106
)
2770
106
    return VDAddr;
2771
100
2772
100
  llvm::Type *VarTy = VDAddr.getElementType();
2773
100
  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2774
100
                         CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2775
100
                                                       CGM.Int8PtrTy),
2776
100
                         CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2777
100
                         getOrCreateThreadPrivateCache(VD)};
2778
100
  return Address(CGF.EmitRuntimeCall(
2779
100
      createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2780
100
                 VDAddr.getAlignment());
2781
100
}
2782
2783
void CGOpenMPRuntime::emitThreadPrivateVarInit(
2784
    CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2785
25
    llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2786
25
  // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2787
25
  // library.
2788
25
  llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2789
25
  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2790
25
                      OMPLoc);
2791
25
  // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2792
25
  // to register constructor/destructor for variable.
2793
25
  llvm::Value *Args[] = {
2794
25
      OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2795
25
      Ctor, CopyCtor, Dtor};
2796
25
  CGF.EmitRuntimeCall(
2797
25
      createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2798
25
}
2799
2800
llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2801
    const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2802
101
    bool PerformInit, CodeGenFunction *CGF) {
2803
101
  if (CGM.getLangOpts().OpenMPUseTLS &&
2804
101
      
CGM.getContext().getTargetInfo().isTLSSupported()52
)
2805
52
    return nullptr;
2806
49
2807
49
  VD = VD->getDefinition(CGM.getContext());
2808
49
  if (VD && 
ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second43
) {
2809
36
    QualType ASTTy = VD->getType();
2810
36
2811
36
    llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2812
36
    const Expr *Init = VD->getAnyInitializer();
2813
36
    if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2814
25
      // Generate function that re-emits the declaration's initializer into the
2815
25
      // threadprivate copy of the variable VD
2816
25
      CodeGenFunction CtorCGF(CGM);
2817
25
      FunctionArgList Args;
2818
25
      ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2819
25
                            /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2820
25
                            ImplicitParamDecl::Other);
2821
25
      Args.push_back(&Dst);
2822
25
2823
25
      const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2824
25
          CGM.getContext().VoidPtrTy, Args);
2825
25
      llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2826
25
      std::string Name = getName({"__kmpc_global_ctor_", ""});
2827
25
      llvm::Function *Fn =
2828
25
          CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2829
25
      CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2830
25
                            Args, Loc, Loc);
2831
25
      llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2832
25
          CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2833
25
          CGM.getContext().VoidPtrTy, Dst.getLocation());
2834
25
      Address Arg = Address(ArgVal, VDAddr.getAlignment());
2835
25
      Arg = CtorCGF.Builder.CreateElementBitCast(
2836
25
          Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2837
25
      CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2838
25
                               /*IsInitializer=*/true);
2839
25
      ArgVal = CtorCGF.EmitLoadOfScalar(
2840
25
          CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2841
25
          CGM.getContext().VoidPtrTy, Dst.getLocation());
2842
25
      CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2843
25
      CtorCGF.FinishFunction();
2844
25
      Ctor = Fn;
2845
25
    }
2846
36
    if (VD->getType().isDestructedType() != QualType::DK_none) {
2847
23
      // Generate function that emits destructor call for the threadprivate copy
2848
23
      // of the variable VD
2849
23
      CodeGenFunction DtorCGF(CGM);
2850
23
      FunctionArgList Args;
2851
23
      ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2852
23
                            /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2853
23
                            ImplicitParamDecl::Other);
2854
23
      Args.push_back(&Dst);
2855
23
2856
23
      const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2857
23
          CGM.getContext().VoidTy, Args);
2858
23
      llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2859
23
      std::string Name = getName({"__kmpc_global_dtor_", ""});
2860
23
      llvm::Function *Fn =
2861
23
          CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2862
23
      auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2863
23
      DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2864
23
                            Loc, Loc);
2865
23
      // Create a scope with an artificial location for the body of this function.
2866
23
      auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2867
23
      llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2868
23
          DtorCGF.GetAddrOfLocalVar(&Dst),
2869
23
          /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2870
23
      DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2871
23
                          DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2872
23
                          DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2873
23
      DtorCGF.FinishFunction();
2874
23
      Dtor = Fn;
2875
23
    }
2876
36
    // Do not emit init function if it is not required.
2877
36
    if (!Ctor && 
!Dtor11
)
2878
11
      return nullptr;
2879
25
2880
25
    llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2881
25
    auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2882
25
                                               /*isVarArg=*/false)
2883
25
                           ->getPointerTo();
2884
25
    // Copying constructor for the threadprivate variable.
2885
25
    // Must be NULL - reserved by runtime, but currently it requires that this
2886
25
    // parameter is always NULL. Otherwise it fires assertion.
2887
25
    CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2888
25
    if (Ctor == nullptr) {
2889
0
      auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2890
0
                                             /*isVarArg=*/false)
2891
0
                         ->getPointerTo();
2892
0
      Ctor = llvm::Constant::getNullValue(CtorTy);
2893
0
    }
2894
25
    if (Dtor == nullptr) {
2895
2
      auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2896
2
                                             /*isVarArg=*/false)
2897
2
                         ->getPointerTo();
2898
2
      Dtor = llvm::Constant::getNullValue(DtorTy);
2899
2
    }
2900
25
    if (!CGF) {
2901
7
      auto *InitFunctionTy =
2902
7
          llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2903
7
      std::string Name = getName({"__omp_threadprivate_init_", ""});
2904
7
      llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2905
7
          InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2906
7
      CodeGenFunction InitCGF(CGM);
2907
7
      FunctionArgList ArgList;
2908
7
      InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2909
7
                            CGM.getTypes().arrangeNullaryFunction(), ArgList,
2910
7
                            Loc, Loc);
2911
7
      emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2912
7
      InitCGF.FinishFunction();
2913
7
      return InitFunction;
2914
7
    }
2915
18
    emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2916
18
  }
2917
49
  
return nullptr31
;
2918
49
}
2919
2920
bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2921
                                                     llvm::GlobalVariable *Addr,
2922
2.49k
                                                     bool PerformInit) {
2923
2.49k
  if (CGM.getLangOpts().OMPTargetTriples.empty() &&
2924
2.49k
      
!CGM.getLangOpts().OpenMPIsDevice423
)
2925
389
    return false;
2926
2.11k
  Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2927
2.11k
      OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2928
2.11k
  if (!Res || 
*Res == OMPDeclareTargetDeclAttr::MT_Link103
||
2929
2.11k
      
(103
*Res == OMPDeclareTargetDeclAttr::MT_To103
&&
2930
103
       HasRequiresUnifiedSharedMemory))
2931
2.00k
    return CGM.getLangOpts().OpenMPIsDevice;
2932
103
  VD = VD->getDefinition(CGM.getContext());
2933
103
  if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second)
2934
3
    return CGM.getLangOpts().OpenMPIsDevice;
2935
100
2936
100
  QualType ASTTy = VD->getType();
2937
100
2938
100
  SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc();
2939
100
  // Produce the unique prefix to identify the new target regions. We use
2940
100
  // the source location of the variable declaration which we know to not
2941
100
  // conflict with any target region.
2942
100
  unsigned DeviceID;
2943
100
  unsigned FileID;
2944
100
  unsigned Line;
2945
100
  getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2946
100
  SmallString<128> Buffer, Out;
2947
100
  {
2948
100
    llvm::raw_svector_ostream OS(Buffer);
2949
100
    OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2950
100
       << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2951
100
  }
2952
100
2953
100
  const Expr *Init = VD->getAnyInitializer();
2954
100
  if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2955
100
    llvm::Constant *Ctor;
2956
100
    llvm::Constant *ID;
2957
100
    if (CGM.getLangOpts().OpenMPIsDevice) {
2958
46
      // Generate function that re-emits the declaration's initializer into
2959
46
      // the threadprivate copy of the variable VD
2960
46
      CodeGenFunction CtorCGF(CGM);
2961
46
2962
46
      const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2963
46
      llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2964
46
      llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2965
46
          FTy, Twine(Buffer, "_ctor"), FI, Loc);
2966
46
      auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2967
46
      CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2968
46
                            FunctionArgList(), Loc, Loc);
2969
46
      auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2970
46
      CtorCGF.EmitAnyExprToMem(Init,
2971
46
                               Address(Addr, CGM.getContext().getDeclAlign(VD)),
2972
46
                               Init->getType().getQualifiers(),
2973
46
                               /*IsInitializer=*/true);
2974
46
      CtorCGF.FinishFunction();
2975
46
      Ctor = Fn;
2976
46
      ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2977
46
      CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2978
54
    } else {
2979
54
      Ctor = new llvm::GlobalVariable(
2980
54
          CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2981
54
          llvm::GlobalValue::PrivateLinkage,
2982
54
          llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2983
54
      ID = Ctor;
2984
54
    }
2985
100
2986
100
    // Register the information for the entry associated with the constructor.
2987
100
    Out.clear();
2988
100
    OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2989
100
        DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2990
100
        ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2991
100
  }
2992
100
  if (VD->getType().isDestructedType() != QualType::DK_none) {
2993
50
    llvm::Constant *Dtor;
2994
50
    llvm::Constant *ID;
2995
50
    if (CGM.getLangOpts().OpenMPIsDevice) {
2996
30
      // Generate function that emits destructor call for the threadprivate
2997
30
      // copy of the variable VD
2998
30
      CodeGenFunction DtorCGF(CGM);
2999
30
3000
30
      const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
3001
30
      llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
3002
30
      llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
3003
30
          FTy, Twine(Buffer, "_dtor"), FI, Loc);
3004
30
      auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
3005
30
      DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
3006
30
                            FunctionArgList(), Loc, Loc);
3007
30
      // Create a scope with an artificial location for the body of this
3008
30
      // function.
3009
30
      auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
3010
30
      DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
3011
30
                          ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
3012
30
                          DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
3013
30
      DtorCGF.FinishFunction();
3014
30
      Dtor = Fn;
3015
30
      ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
3016
30
      CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
3017
30
    } else {
3018
20
      Dtor = new llvm::GlobalVariable(
3019
20
          CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
3020
20
          llvm::GlobalValue::PrivateLinkage,
3021
20
          llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
3022
20
      ID = Dtor;
3023
20
    }
3024
50
    // Register the information for the entry associated with the destructor.
3025
50
    Out.clear();
3026
50
    OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
3027
50
        DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
3028
50
        ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
3029
50
  }
3030
100
  return CGM.getLangOpts().OpenMPIsDevice;
3031
100
}
3032
3033
Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
3034
                                                          QualType VarType,
3035
90
                                                          StringRef Name) {
3036
90
  std::string Suffix = getName({"artificial", ""});
3037
90
  llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
3038
90
  llvm::Value *GAddr =
3039
90
      getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
3040
90
  if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
3041
90
      
CGM.getTarget().isTLSSupported()45
) {
3042
9
    cast<llvm::GlobalVariable>(GAddr)->setThreadLocal(/*Val=*/true);
3043
9
    return Address(GAddr, CGM.getContext().getTypeAlignInChars(VarType));
3044
9
  }
3045
81
  std::string CacheSuffix = getName({"cache", ""});
3046
81
  llvm::Value *Args[] = {
3047
81
      emitUpdateLocation(CGF, SourceLocation()),
3048
81
      getThreadID(CGF, SourceLocation()),
3049
81
      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
3050
81
      CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
3051
81
                                /*isSigned=*/false),
3052
81
      getOrCreateInternalVariable(
3053
81
          CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
3054
81
  return Address(
3055
81
      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3056
81
          CGF.EmitRuntimeCall(
3057
81
              createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
3058
81
          VarLVType->getPointerTo(/*AddrSpace=*/0)),
3059
81
      CGM.getContext().getTypeAlignInChars(VarType));
3060
81
}
3061
3062
void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
3063
                                   const RegionCodeGenTy &ThenGen,
3064
1.60k
                                   const RegionCodeGenTy &ElseGen) {
3065
1.60k
  CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
3066
1.60k
3067
1.60k
  // If the condition constant folds and can be elided, try to avoid emitting
3068
1.60k
  // the condition and the dead arm of the if/else.
3069
1.60k
  bool CondConstant;
3070
1.60k
  if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
3071
772
    if (CondConstant)
3072
425
      ThenGen(CGF);
3073
347
    else
3074
347
      ElseGen(CGF);
3075
772
    return;
3076
772
  }
3077
836
3078
836
  // Otherwise, the condition did not fold, or we couldn't elide it.  Just
3079
836
  // emit the conditional branch.
3080
836
  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
3081
836
  llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
3082
836
  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
3083
836
  CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
3084
836
3085
836
  // Emit the 'then' code.
3086
836
  CGF.EmitBlock(ThenBlock);
3087
836
  ThenGen(CGF);
3088
836
  CGF.EmitBranch(ContBlock);
3089
836
  // Emit the 'else' code if present.
3090
836
  // There is no need to emit line number for unconditional branch.
3091
836
  (void)ApplyDebugLocation::CreateEmpty(CGF);
3092
836
  CGF.EmitBlock(ElseBlock);
3093
836
  ElseGen(CGF);
3094
836
  // There is no need to emit line number for unconditional branch.
3095
836
  (void)ApplyDebugLocation::CreateEmpty(CGF);
3096
836
  CGF.EmitBranch(ContBlock);
3097
836
  // Emit the continuation block for code after the if.
3098
836
  CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
3099
836
}
3100
3101
void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
3102
                                       llvm::Function *OutlinedFn,
3103
                                       ArrayRef<llvm::Value *> CapturedVars,
3104
3.62k
                                       const Expr *IfCond) {
3105
3.62k
  if (!CGF.HaveInsertPoint())
3106
0
    return;
3107
3.62k
  llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
3108
3.62k
  auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
3109
3.62k
                                                     PrePostActionTy &) {
3110
3.52k
    // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
3111
3.52k
    CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3112
3.52k
    llvm::Value *Args[] = {
3113
3.52k
        RTLoc,
3114
3.52k
        CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
3115
3.52k
        CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
3116
3.52k
    llvm::SmallVector<llvm::Value *, 16> RealArgs;
3117
3.52k
    RealArgs.append(std::begin(Args), std::end(Args));
3118
3.52k
    RealArgs.append(CapturedVars.begin(), CapturedVars.end());
3119
3.52k
3120
3.52k
    llvm::FunctionCallee RTLFn =
3121
3.52k
        RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
3122
3.52k
    CGF.EmitRuntimeCall(RTLFn, RealArgs);
3123
3.52k
  };
3124
3.62k
  auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
3125
3.62k
                                                          PrePostActionTy &) {
3126
227
    CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
3127
227
    llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
3128
227
    // Build calls:
3129
227
    // __kmpc_serialized_parallel(&Loc, GTid);
3130
227
    llvm::Value *Args[] = {RTLoc, ThreadID};
3131
227
    CGF.EmitRuntimeCall(
3132
227
        RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
3133
227
3134
227
    // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
3135
227
    Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
3136
227
    Address ZeroAddrBound =
3137
227
        CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
3138
227
                                         /*Name=*/".bound.zero.addr");
3139
227
    CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0));
3140
227
    llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
3141
227
    // ThreadId for serialized parallels is 0.
3142
227
    OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
3143
227
    OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
3144
227
    OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
3145
227
    RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
3146
227
3147
227
    // __kmpc_end_serialized_parallel(&Loc, GTid);
3148
227
    llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
3149
227
    CGF.EmitRuntimeCall(
3150
227
        RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
3151
227
        EndArgs);
3152
227
  };
3153
3.62k
  if (IfCond) {
3154
279
    emitIfClause(CGF, IfCond, ThenGen, ElseGen);
3155
3.34k
  } else {
3156
3.34k
    RegionCodeGenTy ThenRCG(ThenGen);
3157
3.34k
    ThenRCG(CGF);
3158
3.34k
  }
3159
3.62k
}
3160
3161
// If we're inside an (outlined) parallel region, use the region info's
3162
// thread-ID variable (it is passed in a first argument of the outlined function
3163
// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
3164
// regular serial code region, get thread ID by calling kmp_int32
3165
// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
3166
// return the address of that temp.
3167
Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
3168
995
                                             SourceLocation Loc) {
3169
995
  if (auto *OMPRegionInfo =
3170
971
          dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3171
971
    if (OMPRegionInfo->getThreadIDVariable())
3172
372
      return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
3173
623
3174
623
  llvm::Value *ThreadID = getThreadID(CGF, Loc);
3175
623
  QualType Int32Ty =
3176
623
      CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
3177
623
  Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
3178
623
  CGF.EmitStoreOfScalar(ThreadID,
3179
623
                        CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
3180
623
3181
623
  return ThreadIDTemp;
3182
623
}
3183
3184
llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable(
3185
870
    llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
3186
870
  SmallString<256> Buffer;
3187
870
  llvm::raw_svector_ostream Out(Buffer);
3188
870
  Out << Name;
3189
870
  StringRef RuntimeName = Out.str();
3190
870
  auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
3191
870
  if (Elem.second) {
3192
525
    assert(Elem.second->getType()->getPointerElementType() == Ty &&
3193
525
           "OMP internal variable has different type than requested");
3194
525
    return &*Elem.second;
3195
525
  }
3196
345
3197
345
  return Elem.second = new llvm::GlobalVariable(
3198
345
             CGM.getModule(), Ty, /*IsConstant*/ false,
3199
345
             llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
3200
345
             Elem.first(), /*InsertBefore=*/nullptr,
3201
345
             llvm::GlobalValue::NotThreadLocal, AddressSpace);
3202
345
}
3203
3204
496
llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
3205
496
  std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
3206
496
  std::string Name = getName({Prefix, "var"});
3207
496
  return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
3208
496
}
3209
3210
namespace {
3211
/// Common pre(post)-action for different OpenMP constructs.
3212
class CommonActionTy final : public PrePostActionTy {
3213
  llvm::FunctionCallee EnterCallee;
3214
  ArrayRef<llvm::Value *> EnterArgs;
3215
  llvm::FunctionCallee ExitCallee;
3216
  ArrayRef<llvm::Value *> ExitArgs;
3217
  bool Conditional;
3218
  llvm::BasicBlock *ContBlock = nullptr;
3219
3220
public:
3221
  CommonActionTy(llvm::FunctionCallee EnterCallee,
3222
                 ArrayRef<llvm::Value *> EnterArgs,
3223
                 llvm::FunctionCallee ExitCallee,
3224
                 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
3225
      : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3226
1.31k
        ExitArgs(ExitArgs), Conditional(Conditional) {}
3227
780
  void Enter(CodeGenFunction &CGF) override {
3228
780
    llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
3229
780
    if (Conditional) {
3230
219
      llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
3231
219
      auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
3232
219
      ContBlock = CGF.createBasicBlock("omp_if.end");
3233
219
      // Generate the branch (If-stmt)
3234
219
      CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3235
219
      CGF.EmitBlock(ThenBlock);
3236
219
    }
3237
780
  }
3238
219
  void Done(CodeGenFunction &CGF) {
3239
219
    // Emit the rest of blocks/branches
3240
219
    CGF.EmitBranch(ContBlock);
3241
219
    CGF.EmitBlock(ContBlock, true);
3242
219
  }
3243
1.38k
  void Exit(CodeGenFunction &CGF) override {
3244
1.38k
    CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
3245
1.38k
  }
3246
};
3247
} // anonymous namespace
3248
3249
void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
3250
                                         StringRef CriticalName,
3251
                                         const RegionCodeGenTy &CriticalOpGen,
3252
136
                                         SourceLocation Loc, const Expr *Hint) {
3253
136
  // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
3254
136
  // CriticalOpGen();
3255
136
  // __kmpc_end_critical(ident_t *, gtid, Lock);
3256
136
  // Prepare arguments and build a call to __kmpc_critical
3257
136
  if (!CGF.HaveInsertPoint())
3258
0
    return;
3259
136
  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3260
136
                         getCriticalRegionLock(CriticalName)};
3261
136
  llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
3262
136
                                                std::end(Args));
3263
136
  if (Hint) {
3264
3
    EnterArgs.push_back(CGF.Builder.CreateIntCast(
3265
3
        CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
3266
3
  }
3267
136
  CommonActionTy Action(
3268
136
      createRuntimeFunction(Hint ? 
OMPRTL__kmpc_critical_with_hint3
3269
136
                                 : 
OMPRTL__kmpc_critical133
),
3270
136
      EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
3271
136
  CriticalOpGen.setAction(Action);
3272
136
  emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
3273
136
}
3274
3275
void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
3276
                                       const RegionCodeGenTy &MasterOpGen,
3277
180
                                       SourceLocation Loc) {
3278
180
  if (!CGF.HaveInsertPoint())
3279
0
    return;
3280
180
  // if(__kmpc_master(ident_t *, gtid)) {
3281
180
  //   MasterOpGen();
3282
180
  //   __kmpc_end_master(ident_t *, gtid);
3283
180
  // }
3284
180
  // Prepare arguments and build a call to __kmpc_master
3285
180
  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3286
180
  CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
3287
180
                        createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
3288
180
                        /*Conditional=*/true);
3289
180
  MasterOpGen.setAction(Action);
3290
180
  emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
3291
180
  Action.Done(CGF);
3292
180
}
3293
3294
void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
3295
16
                                        SourceLocation Loc) {
3296
16
  if (!CGF.HaveInsertPoint())
3297
0
    return;
3298
16
  llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
3299
16
  if (OMPBuilder) {
3300
8
    OMPBuilder->CreateTaskyield(CGF.Builder);
3301
8
  } else {
3302
8
    // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3303
8
    llvm::Value *Args[] = {
3304
8
        emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3305
8
        llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3306
8
    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield),
3307
8
                        Args);
3308
8
  }
3309
16
3310
16
  if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3311
4
    Region->emitUntiedSwitch(CGF);
3312
16
}
3313
3314
void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3315
                                          const RegionCodeGenTy &TaskgroupOpGen,
3316
243
                                          SourceLocation Loc) {
3317
243
  if (!CGF.HaveInsertPoint())
3318
0
    return;
3319
243
  // __kmpc_taskgroup(ident_t *, gtid);
3320
243
  // TaskgroupOpGen();
3321
243
  // __kmpc_end_taskgroup(ident_t *, gtid);
3322
243
  // Prepare arguments and build a call to __kmpc_taskgroup
3323
243
  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3324
243
  CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3325
243
                        createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3326
243
                        Args);
3327
243
  TaskgroupOpGen.setAction(Action);
3328
243
  emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3329
243
}
3330
3331
/// Given an array of pointers to variables, project the address of a
3332
/// given variable.
3333
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3334
1.05k
                                      unsigned Index, const VarDecl *Var) {
3335
1.05k
  // Pull out the pointer to the variable.
3336
1.05k
  Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index);
3337
1.05k
  llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3338
1.05k
3339
1.05k
  Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3340
1.05k
  Addr = CGF.Builder.CreateElementBitCast(
3341
1.05k
      Addr, CGF.ConvertTypeForMem(Var->getType()));
3342
1.05k
  return Addr;
3343
1.05k
}
3344
3345
static llvm::Value *emitCopyprivateCopyFunction(
3346
    CodeGenModule &CGM, llvm::Type *ArgsType,
3347
    ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3348
    ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3349
18
    SourceLocation Loc) {
3350
18
  ASTContext &C = CGM.getContext();
3351
18
  // void copy_func(void *LHSArg, void *RHSArg);
3352
18
  FunctionArgList Args;
3353
18
  ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3354
18
                           ImplicitParamDecl::Other);
3355
18
  ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3356
18
                           ImplicitParamDecl::Other);
3357
18
  Args.push_back(&LHSArg);
3358
18
  Args.push_back(&RHSArg);
3359
18
  const auto &CGFI =
3360
18
      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3361
18
  std::string Name =
3362
18
      CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3363
18
  auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3364
18
                                    llvm::GlobalValue::InternalLinkage, Name,
3365
18
                                    &CGM.getModule());
3366
18
  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3367
18
  Fn->setDoesNotRecurse();
3368
18
  CodeGenFunction CGF(CGM);
3369
18
  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3370
18
  // Dest = (void*[n])(LHSArg);
3371
18
  // Src = (void*[n])(RHSArg);
3372
18
  Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3373
18
      CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3374
18
      ArgsType), CGF.getPointerAlign());
3375
18
  Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3376
18
      CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3377
18
      ArgsType), CGF.getPointerAlign());
3378
18
  // *(Type0*)Dst[0] = *(Type0*)Src[0];
3379
18
  // *(Type1*)Dst[1] = *(Type1*)Src[1];
3380
18
  // ...
3381
18
  // *(Typen*)Dst[n] = *(Typen*)Src[n];
3382
61
  for (unsigned I = 0, E = AssignmentOps.size(); I < E; 
++I43
) {
3383
43
    const auto *DestVar =
3384
43
        cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3385
43
    Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3386
43
3387
43
    const auto *SrcVar =
3388
43
        cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3389
43
    Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3390
43
3391
43
    const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3392
43
    QualType Type = VD->getType();
3393
43
    CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3394
43
  }
3395
18
  CGF.FinishFunction();
3396
18
  return Fn;
3397
18
}
3398
3399
void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3400
                                       const RegionCodeGenTy &SingleOpGen,
3401
                                       SourceLocation Loc,
3402
                                       ArrayRef<const Expr *> CopyprivateVars,
3403
                                       ArrayRef<const Expr *> SrcExprs,
3404
                                       ArrayRef<const Expr *> DstExprs,
3405
39
                                       ArrayRef<const Expr *> AssignmentOps) {
3406
39
  if (!CGF.HaveInsertPoint())
3407
0
    return;
3408
39
  assert(CopyprivateVars.size() == SrcExprs.size() &&
3409
39
         CopyprivateVars.size() == DstExprs.size() &&
3410
39
         CopyprivateVars.size() == AssignmentOps.size());
3411
39
  ASTContext &C = CGM.getContext();
3412
39
  // int32 did_it = 0;
3413
39
  // if(__kmpc_single(ident_t *, gtid)) {
3414
39
  //   SingleOpGen();
3415
39
  //   __kmpc_end_single(ident_t *, gtid);
3416
39
  //   did_it = 1;
3417
39
  // }
3418
39
  // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3419
39
  // <copy_func>, did_it);
3420
39
3421
39
  Address DidIt = Address::invalid();
3422
39
  if (!CopyprivateVars.empty()) {
3423
18
    // int32 did_it = 0;
3424
18
    QualType KmpInt32Ty =
3425
18
        C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3426
18
    DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3427
18
    CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3428
18
  }
3429
39
  // Prepare arguments and build a call to __kmpc_single
3430
39
  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3431
39
  CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3432
39
                        createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3433
39
                        /*Conditional=*/true);
3434
39
  SingleOpGen.setAction(Action);
3435
39
  emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3436
39
  if (DidIt.isValid()) {
3437
18
    // did_it = 1;
3438
18
    CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3439
18
  }
3440
39
  Action.Done(CGF);
3441
39
  // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3442
39
  // <copy_func>, did_it);
3443
39
  if (DidIt.isValid()) {
3444
18
    llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3445
18
    QualType CopyprivateArrayTy = C.getConstantArrayType(
3446
18
        C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal,
3447
18
        /*IndexTypeQuals=*/0);
3448
18
    // Create a list of all private variables for copyprivate.
3449
18
    Address CopyprivateList =
3450
18
        CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3451
61
    for (unsigned I = 0, E = CopyprivateVars.size(); I < E; 
++I43
) {
3452
43
      Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I);
3453
43
      CGF.Builder.CreateStore(
3454
43
          CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3455
43
              CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF),
3456
43
              CGF.VoidPtrTy),
3457
43
          Elem);
3458
43
    }
3459
18
    // Build function that copies private values from single region to all other
3460
18
    // threads in the corresponding parallel region.
3461
18
    llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3462
18
        CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3463
18
        CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3464
18
    llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3465
18
    Address CL =
3466
18
      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3467
18
                                                      CGF.VoidPtrTy);
3468
18
    llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3469
18
    llvm::Value *Args[] = {
3470
18
        emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3471
18
        getThreadID(CGF, Loc),        // i32 <gtid>
3472
18
        BufSize,                      // size_t <buf_size>
3473
18
        CL.getPointer(),              // void *<copyprivate list>
3474
18
        CpyFn,                        // void (*) (void *, void *) <copy_func>
3475
18
        DidItVal                      // i32 did_it
3476
18
    };
3477
18
    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3478
18
  }
3479
39
}
3480
3481
void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3482
                                        const RegionCodeGenTy &OrderedOpGen,
3483
12
                                        SourceLocation Loc, bool IsThreads) {
3484
12
  if (!CGF.HaveInsertPoint())
3485
0
    return;
3486
12
  // __kmpc_ordered(ident_t *, gtid);
3487
12
  // OrderedOpGen();
3488
12
  // __kmpc_end_ordered(ident_t *, gtid);
3489
12
  // Prepare arguments and build a call to __kmpc_ordered
3490
12
  if (IsThreads) {
3491
8
    llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3492
8
    CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3493
8
                          createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3494
8
                          Args);
3495
8
    OrderedOpGen.setAction(Action);
3496
8
    emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3497
8
    return;
3498
8
  }
3499
4
  emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3500
4
}
3501
3502
812
unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
3503
812
  unsigned Flags;
3504
812
  if (Kind == OMPD_for)
3505
510
    Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3506
302
  else if (Kind == OMPD_sections)
3507
42
    Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3508
260
  else if (Kind == OMPD_single)
3509
16
    Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3510
244
  else if (Kind == OMPD_barrier)
3511
13
    Flags = OMP_IDENT_BARRIER_EXPL;
3512
231
  else
3513
231
    Flags = OMP_IDENT_BARRIER_IMPL;
3514
812
  return Flags;
3515
812
}
3516
3517
void CGOpenMPRuntime::getDefaultScheduleAndChunk(
3518
    CodeGenFunction &CGF, const OMPLoopDirective &S,
3519
2.17k
    OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
3520
2.17k
  // Check if the loop directive is actually a doacross loop directive. In this
3521
2.17k
  // case choose static, 1 schedule.
3522
2.17k
  if (llvm::any_of(
3523
2.17k
          S.getClausesOfKind<OMPOrderedClause>(),
3524
2.17k
          [](const OMPOrderedClause *C) 
{ return C->getNumForLoops(); }14
)) {
3525
8
    ScheduleKind = OMPC_SCHEDULE_static;
3526
8
    // Chunk size is 1 in this case.
3527
8
    llvm::APInt ChunkSize(32, 1);
3528
8
    ChunkExpr = IntegerLiteral::Create(
3529
8
        CGF.getContext(), ChunkSize,
3530
8
        CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3531
8
        SourceLocation());
3532
8
  }
3533
2.17k
}
3534
3535
void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3536
                                      OpenMPDirectiveKind Kind, bool EmitChecks,
3537
673
                                      bool ForceSimpleCall) {
3538
673
  // Check if we should use the OMPBuilder
3539
673
  auto *OMPRegionInfo =
3540
673
      dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo);
3541
673
  llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
3542
673
  if (OMPBuilder) {
3543
20
    CGF.Builder.restoreIP(OMPBuilder->CreateBarrier(
3544
20
        CGF.Builder, Kind, ForceSimpleCall, EmitChecks));
3545
20
    return;
3546
20
  }
3547
653
3548
653
  if (!CGF.HaveInsertPoint())
3549
0
    return;
3550
653
  // Build call __kmpc_cancel_barrier(loc, thread_id);
3551
653
  // Build call __kmpc_barrier(loc, thread_id);
3552
653
  unsigned Flags = getDefaultFlagsForBarriers(Kind);
3553
653
  // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3554
653
  // thread_id);
3555
653
  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3556
653
                         getThreadID(CGF, Loc)};
3557
653
  if (OMPRegionInfo) {
3558
421
    if (!ForceSimpleCall && 
OMPRegionInfo->hasCancel()262
) {
3559
2
      llvm::Value *Result = CGF.EmitRuntimeCall(
3560
2
          createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3561
2
      if (EmitChecks) {
3562
2
        // if (__kmpc_cancel_barrier()) {
3563
2
        //   exit from construct;
3564
2
        // }
3565
2
        llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3566
2
        llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3567
2
        llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3568
2
        CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3569
2
        CGF.EmitBlock(ExitBB);
3570
2
        //   exit from construct;
3571
2
        CodeGenFunction::JumpDest CancelDestination =
3572
2
            CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3573
2
        CGF.EmitBranchThroughCleanup(CancelDestination);
3574
2
        CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3575
2
      }
3576
2
      return;
3577
2
    }
3578
651
  }
3579
651
  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3580
651
}
3581
3582
/// Map the OpenMP loop schedule to the runtime enumeration.
3583
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3584
11.4k
                                          bool Chunked, bool Ordered) {
3585
11.4k
  switch (ScheduleKind) {
3586
1.97k
  case OMPC_SCHEDULE_static:
3587
1.97k
    return Chunked ? 
(Ordered 1.20k
?
OMP_ord_static_chunked7
:
OMP_sch_static_chunked1.20k
)
3588
1.97k
                   : 
(Ordered 767
?
OMP_ord_static4
:
OMP_sch_static763
);
3589
1.41k
  case OMPC_SCHEDULE_dynamic:
3590
1.41k
    return Ordered ? 
OMP_ord_dynamic_chunked2
:
OMP_sch_dynamic_chunked1.40k
;
3591
528
  case OMPC_SCHEDULE_guided:
3592
528
    return Ordered ? 
OMP_ord_guided_chunked1
:
OMP_sch_guided_chunked527
;
3593
522
  case OMPC_SCHEDULE_runtime:
3594
522
    return Ordered ? 
OMP_ord_runtime4
:
OMP_sch_runtime518
;
3595
524
  case OMPC_SCHEDULE_auto:
3596
524
    return Ordered ? 
OMP_ord_auto3
:
OMP_sch_auto521
;
3597
6.52k
  case OMPC_SCHEDULE_unknown:
3598
6.52k
    assert(!Chunked && "chunk was specified but schedule kind not known");
3599
6.52k
    return Ordered ? 
OMP_ord_static6
:
OMP_sch_static6.51k
;
3600
0
  }
3601
0
  llvm_unreachable("Unexpected runtime schedule");
3602
0
}
3603
3604
/// Map the OpenMP distribute schedule to the runtime enumeration.
3605
static OpenMPSchedType
3606
9.51k
getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3607
9.51k
  // only static is allowed for dist_schedule
3608
9.51k
  return Chunked ? 
OMP_dist_sch_static_chunked1.45k
:
OMP_dist_sch_static8.06k
;
3609
9.51k
}
3610
3611
bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3612
4.06k
                                         bool Chunked) const {
3613
4.06k
  OpenMPSchedType Schedule =
3614
4.06k
      getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3615
4.06k
  return Schedule == OMP_sch_static;
3616
4.06k
}
3617
3618
bool CGOpenMPRuntime::isStaticNonchunked(
3619
3.17k
    OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3620
3.17k
  OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3621
3.17k
  return Schedule == OMP_dist_sch_static;
3622
3.17k
}
3623
3624
bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
3625
3.26k
                                      bool Chunked) const {
3626
3.26k
  OpenMPSchedType Schedule =
3627
3.26k
      getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3628
3.26k
  return Schedule == OMP_sch_static_chunked;
3629
3.26k
}
3630
3631
bool CGOpenMPRuntime::isStaticChunked(
3632
3.17k
    OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3633
3.17k
  OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3634
3.17k
  return Schedule == OMP_dist_sch_static_chunked;
3635
3.17k
}
3636
3637
804
bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3638
804
  OpenMPSchedType Schedule =
3639
804
      getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3640
804
  assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
3641
804
  return Schedule != OMP_sch_static;
3642
804
}
3643
3644
static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
3645
                                  OpenMPScheduleClauseModifier M1,
3646
6.52k
                                  OpenMPScheduleClauseModifier M2) {
3647
6.52k
  int Modifier = 0;
3648
6.52k
  switch (M1) {
3649
16
  case OMPC_SCHEDULE_MODIFIER_monotonic:
3650
16
    Modifier = OMP_sch_modifier_monotonic;
3651
16
    break;
3652
8
  case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3653
8
    Modifier = OMP_sch_modifier_nonmonotonic;
3654
8
    break;
3655
12
  case OMPC_SCHEDULE_MODIFIER_simd:
3656
12
    if (Schedule == OMP_sch_static_chunked)
3657
6
      Schedule = OMP_sch_static_balanced_chunked;
3658
12
    break;
3659
6.48k
  case OMPC_SCHEDULE_MODIFIER_last:
3660
6.48k
  case OMPC_SCHEDULE_MODIFIER_unknown:
3661
6.48k
    break;
3662
6.52k
  }
3663
6.52k
  switch (M2) {
3664
0
  case OMPC_SCHEDULE_MODIFIER_monotonic:
3665
0
    Modifier = OMP_sch_modifier_monotonic;
3666
0
    break;
3667
6
  case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3668
6
    Modifier = OMP_sch_modifier_nonmonotonic;
3669
6
    break;
3670
0
  case OMPC_SCHEDULE_MODIFIER_simd:
3671
0
    if (Schedule == OMP_sch_static_chunked)
3672
0
      Schedule = OMP_sch_static_balanced_chunked;
3673
0
    break;
3674
6.51k
  case OMPC_SCHEDULE_MODIFIER_last:
3675
6.51k
  case OMPC_SCHEDULE_MODIFIER_unknown:
3676
6.51k
    break;
3677
6.52k
  }
3678
6.52k
  // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
3679
6.52k
  // If the static schedule kind is specified or if the ordered clause is
3680
6.52k
  // specified, and if the nonmonotonic modifier is not specified, the effect is
3681
6.52k
  // as if the monotonic modifier is specified. Otherwise, unless the monotonic
3682
6.52k
  // modifier is specified, the effect is as if the nonmonotonic modifier is
3683
6.52k
  // specified.
3684
6.52k
  if (CGM.getLangOpts().OpenMP >= 50 && 
Modifier == 0496
) {
3685
491
    if (!(Schedule == OMP_sch_static_chunked || 
Schedule == OMP_sch_static476
||
3686
491
          
Schedule == OMP_sch_static_balanced_chunked223
||
3687
491
          
Schedule == OMP_ord_static_chunked220
||
Schedule == OMP_ord_static220
||
3688
491
          
Schedule == OMP_dist_sch_static_chunked220
||
3689
491
          
Schedule == OMP_dist_sch_static192
))
3690
12
      Modifier = OMP_sch_modifier_nonmonotonic;
3691
491
  }
3692
6.52k
  return Schedule | Modifier;
3693
6.52k
}
3694
3695
void CGOpenMPRuntime::emitForDispatchInit(
3696
    CodeGenFunction &CGF, SourceLocation Loc,
3697
    const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3698
618
    bool Ordered, const DispatchRTInput &DispatchValues) {
3699
618
  if (!CGF.HaveInsertPoint())
3700
0
    return;
3701
618
  OpenMPSchedType Schedule = getRuntimeSchedule(
3702
618
      ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3703
618
  assert(Ordered ||
3704
618
         (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
3705
618
          Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
3706
618
          Schedule != OMP_sch_static_balanced_chunked));
3707
618
  // Call __kmpc_dispatch_init(
3708
618
  //          ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3709
618
  //          kmp_int[32|64] lower, kmp_int[32|64] upper,
3710
618
  //          kmp_int[32|64] stride, kmp_int[32|64] chunk);
3711
618
3712
618
  // If the Chunk was not specified in the clause - use default value 1.
3713
618
  llvm::Value *Chunk = DispatchValues.Chunk ? 
DispatchValues.Chunk88
3714
618
                                            : 
CGF.Builder.getIntN(IVSize, 1)530
;
3715
618
  llvm::Value *Args[] = {
3716
618
      emitUpdateLocation(CGF, Loc),
3717
618
      getThreadID(CGF, Loc),
3718
618
      CGF.Builder.getInt32(addMonoNonMonoModifier(
3719
618
          CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3720
618
      DispatchValues.LB,                                     // Lower
3721
618
      DispatchValues.UB,                                     // Upper
3722
618
      CGF.Builder.getIntN(IVSize, 1),                        // Stride
3723
618
      Chunk                                                  // Chunk
3724
618
  };
3725
618
  CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3726
618
}
3727
3728
static void emitForStaticInitCall(
3729
    CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3730
    llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
3731
    OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3732
5.90k
    const CGOpenMPRuntime::StaticRTInput &Values) {
3733
5.90k
  if (!CGF.HaveInsertPoint())
3734
0
    return;
3735
5.90k
3736
5.90k
  assert(!Values.Ordered);
3737
5.90k
  assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
3738
5.90k
         Schedule == OMP_sch_static_balanced_chunked ||
3739
5.90k
         Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
3740
5.90k
         Schedule == OMP_dist_sch_static ||
3741
5.90k
         Schedule == OMP_dist_sch_static_chunked);
3742
5.90k
3743
5.90k
  // Call __kmpc_for_static_init(
3744
5.90k
  //          ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3745
5.90k
  //          kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3746
5.90k
  //          kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3747
5.90k
  //          kmp_int[32|64] incr, kmp_int[32|64] chunk);
3748
5.90k
  llvm::Value *Chunk = Values.Chunk;
3749
5.90k
  if (Chunk == nullptr) {
3750
5.09k
    assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
3751
5.09k
            Schedule == OMP_dist_sch_static) &&
3752
5.09k
           "expected static non-chunked schedule");
3753
5.09k
    // If the Chunk was not specified in the clause - use default value 1.
3754
5.09k
    Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3755
5.09k
  } else {
3756
809
    assert((Schedule == OMP_sch_static_chunked ||
3757
809
            Schedule == OMP_sch_static_balanced_chunked ||
3758
809
            Schedule == OMP_ord_static_chunked ||
3759
809
            Schedule == OMP_dist_sch_static_chunked) &&
3760
809
           "expected static chunked schedule");
3761
809
  }
3762
5.90k
  llvm::Value *Args[] = {
3763
5.90k
      UpdateLocation,
3764
5.90k
      ThreadId,
3765
5.90k
      CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
3766
5.90k
                                                  M2)), // Schedule type
3767
5.90k
      Values.IL.getPointer(),                           // &isLastIter
3768
5.90k
      Values.LB.getPointer(),                           // &LB
3769
5.90k
      Values.UB.getPointer(),                           // &UB
3770
5.90k
      Values.ST.getPointer(),                           // &Stride
3771
5.90k
      CGF.Builder.getIntN(Values.IVSize, 1),            // Incr
3772
5.90k
      Chunk                                             // Chunk
3773
5.90k
  };
3774
5.90k
  CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3775
5.90k
}
3776
3777
void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3778
                                        SourceLocation Loc,
3779
                                        OpenMPDirectiveKind DKind,
3780
                                        const OpenMPScheduleTy &ScheduleKind,
3781
2.73k
                                        const StaticRTInput &Values) {
3782
2.73k
  OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3783
2.73k
      ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3784
2.73k
  assert(isOpenMPWorksharingDirective(DKind) &&
3785
2.73k
         "Expected loop-based or sections-based directive.");
3786
2.73k
  llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3787
2.73k
                                             isOpenMPLoopDirective(DKind)
3788
2.73k
                                                 ? 
OMP_IDENT_WORK_LOOP2.66k
3789
2.73k
                                                 : 
OMP_IDENT_WORK_SECTIONS68
);
3790
2.73k
  llvm::Value *ThreadId = getThreadID(CGF, Loc);
3791
2.73k
  llvm::FunctionCallee StaticInitFunction =
3792
2.73k
      createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3793
2.73k
  auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
3794
2.73k
  emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3795
2.73k
                        ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3796
2.73k
}
3797
3798
void CGOpenMPRuntime::emitDistributeStaticInit(
3799
    CodeGenFunction &CGF, SourceLocation Loc,
3800
    OpenMPDistScheduleClauseKind SchedKind,
3801
3.17k
    const CGOpenMPRuntime::StaticRTInput &Values) {
3802
3.17k
  OpenMPSchedType ScheduleNum =
3803
3.17k
      getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3804
3.17k
  llvm::Value *UpdatedLocation =
3805
3.17k
      emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3806
3.17k
  llvm::Value *ThreadId = getThreadID(CGF, Loc);
3807
3.17k
  llvm::FunctionCallee StaticInitFunction =
3808
3.17k
      createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3809
3.17k
  emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3810
3.17k
                        ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3811
3.17k
                        OMPC_SCHEDULE_MODIFIER_unknown, Values);
3812
3.17k
}
3813
3814
void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3815
                                          SourceLocation Loc,
3816
5.94k
                                          OpenMPDirectiveKind DKind) {
3817
5.94k
  if (!CGF.HaveInsertPoint())
3818
0
    return;
3819
5.94k
  // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3820
5.94k
  llvm::Value *Args[] = {
3821
5.94k
      emitUpdateLocation(CGF, Loc,
3822
5.94k
                         isOpenMPDistributeDirective(DKind)
3823
5.94k
                             ? 
OMP_IDENT_WORK_DISTRIBUTE4.72k
3824
5.94k
                             : isOpenMPLoopDirective(DKind)
3825
1.22k
                                   ? 
OMP_IDENT_WORK_LOOP1.13k
3826
1.22k
                                   : 
OMP_IDENT_WORK_SECTIONS92
),
3827
5.94k
      getThreadID(CGF, Loc)};
3828
5.94k
  auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
3829
5.94k
  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3830
5.94k
                      Args);
3831
5.94k
}
3832
3833
void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3834
                                                 SourceLocation Loc,
3835
                                                 unsigned IVSize,
3836
27
                                                 bool IVSigned) {
3837
27
  if (!CGF.HaveInsertPoint())
3838
0
    return;
3839
27
  // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3840
27
  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3841
27
  CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3842
27
}
3843
3844
llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3845
                                          SourceLocation Loc, unsigned IVSize,
3846
                                          bool IVSigned, Address IL,
3847
                                          Address LB, Address UB,
3848
618
                                          Address ST) {
3849
618
  // Call __kmpc_dispatch_next(
3850
618
  //          ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3851
618
  //          kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3852
618
  //          kmp_int[32|64] *p_stride);
3853
618
  llvm::Value *Args[] = {
3854
618
      emitUpdateLocation(CGF, Loc),
3855
618
      getThreadID(CGF, Loc),
3856
618
      IL.getPointer(), // &isLastIter
3857
618
      LB.getPointer(), // &Lower
3858
618
      UB.getPointer(), // &Upper
3859
618
      ST.getPointer()  // &Stride
3860
618
  };
3861
618
  llvm::Value *Call =
3862
618
      CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3863
618
  return CGF.EmitScalarConversion(
3864
618
      Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3865
618
      CGF.getContext().BoolTy, Loc);
3866
618
}
3867
3868
void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3869
                                           llvm::Value *NumThreads,
3870
131
                                           SourceLocation Loc) {
3871
131
  if (!CGF.HaveInsertPoint())
3872
0
    return;
3873
131
  // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3874
131
  llvm::Value *Args[] = {
3875
131
      emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3876
131
      CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3877
131
  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3878
131
                      Args);
3879
131
}
3880
3881
void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3882
                                         ProcBindKind ProcBind,
3883
58
                                         SourceLocation Loc) {
3884
58
  if (!CGF.HaveInsertPoint())
3885
0
    return;
3886
58
  assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.");
3887
58
  // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3888
58
  llvm::Value *Args[] = {
3889
58
      emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3890
58
      llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)};
3891
58
  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3892
58
}
3893
3894
void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3895
104
                                SourceLocation Loc, llvm::AtomicOrdering AO) {
3896
104
  llvm::OpenMPIRBuilder *OMPBuilder = CGF.CGM.getOpenMPIRBuilder();
3897
104
  if (OMPBuilder) {
3898
20
    OMPBuilder->CreateFlush(CGF.Builder);
3899
84
  } else {
3900
84
    if (!CGF.HaveInsertPoint())
3901
0
      return;
3902
84
    // Build call void __kmpc_flush(ident_t *loc)
3903
84
    CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3904
84
                        emitUpdateLocation(CGF, Loc));
3905
84
  }
3906
104
}
3907
3908
namespace {
3909
/// Indexes of fields for type kmp_task_t.
3910
enum KmpTaskTFields {
3911
  /// List of shared variables.
3912
  KmpTaskTShareds,
3913
  /// Task routine.
3914
  KmpTaskTRoutine,
3915
  /// Partition id for the untied tasks.
3916
  KmpTaskTPartId,
3917
  /// Function with call of destructors for private variables.
3918
  Data1,
3919
  /// Task priority.
3920
  Data2,
3921
  /// (Taskloops only) Lower bound.
3922
  KmpTaskTLowerBound,
3923
  /// (Taskloops only) Upper bound.
3924
  KmpTaskTUpperBound,
3925
  /// (Taskloops only) Stride.
3926
  KmpTaskTStride,
3927
  /// (Taskloops only) Is last iteration flag.
3928
  KmpTaskTLastIter,
3929
  /// (Taskloops only) Reduction data.
3930
  KmpTaskTReductions,
3931
};
3932
} // anonymous namespace
3933
3934
3.63k
bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3935
3.63k
  return OffloadEntriesTargetRegion.empty() &&
3936
3.63k
         
OffloadEntriesDeviceGlobalVar.empty()675
;
3937
3.63k
}
3938
3939
/// Initialize target region entry.
3940
void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3941
    initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3942
                                    StringRef ParentName, unsigned LineNum,
3943
2.00k
                                    unsigned Order) {
3944
2.00k
  assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
3945
2.00k
                                             "only required for the device "
3946
2.00k
                                             "code generation.");
3947
2.00k
  OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3948
2.00k
      OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3949
2.00k
                                   OMPTargetRegionEntryTargetRegion);
3950
2.00k
  ++OffloadingEntriesNum;
3951
2.00k
}
3952
3953
void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3954
    registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3955
                                  StringRef ParentName, unsigned LineNum,
3956
                                  llvm::Constant *Addr, llvm::Constant *ID,
3957
6.79k
                                  OMPTargetRegionEntryKind Flags) {
3958
6.79k
  // If we are emitting code for a target, the entry is already initialized,
3959
6.79k
  // only has to be registered.
3960
6.79k
  if (CGM.getLangOpts().OpenMPIsDevice) {
3961
1.97k
    if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3962
0
      unsigned DiagID = CGM.getDiags().getCustomDiagID(
3963
0
          DiagnosticsEngine::Error,
3964
0
          "Unable to find target region on line '%0' in the device code.");
3965
0
      CGM.getDiags().Report(DiagID) << LineNum;
3966
0
      return;
3967
0
    }
3968
1.97k
    auto &Entry =
3969
1.97k
        OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3970
1.97k
    assert(Entry.isValid() && "Entry not initialized!");
3971
1.97k
    Entry.setAddress(Addr);
3972
1.97k
    Entry.setID(ID);
3973
1.97k
    Entry.setFlags(Flags);
3974
4.82k
  } else {
3975
4.82k
    OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3976
4.82k
    OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3977
4.82k
    ++OffloadingEntriesNum;
3978
4.82k
  }
3979
6.79k
}
3980
3981
bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3982
    unsigned DeviceID, unsigned FileID, StringRef ParentName,
3983
4.45k
    unsigned LineNum) const {
3984
4.45k
  auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3985
4.45k
  if (PerDevice == OffloadEntriesTargetRegion.end())
3986
0
    return false;
3987
4.45k
  auto PerFile = PerDevice->second.find(FileID);
3988
4.45k
  if (PerFile == PerDevice->second.end())
3989
0
    return false;
3990
4.45k
  auto PerParentName = PerFile->second.find(ParentName);
3991
4.45k
  if (PerParentName == PerFile->second.end())
3992
342
    return false;
3993
4.11k
  auto PerLine = PerParentName->second.find(LineNum);
3994
4.11k
  if (PerLine == PerParentName->second.end())
3995
99
    return false;
3996
4.01k
  // Fail if this entry is already registered.
3997
4.01k
  if (PerLine->second.getAddress() || 
PerLine->second.getID()3.86k
)
3998
149
    return false;
3999
3.86k
  return true;
4000
3.86k
}
4001
4002
void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
4003
1.65k
    const OffloadTargetRegionEntryInfoActTy &Action) {
4004
1.65k
  // Scan all target region entries and perform the provided action.
4005
1.65k
  for (const auto &D : OffloadEntriesTargetRegion)
4006
1.65k
    for (const auto &F : D.second)
4007
1.65k
      for (const auto &P : F.second)
4008
4.00k
        for (const auto &L : P.second)
4009
6.78k
          Action(D.first, F.first, P.first(), L.first, L.second);
4010
1.65k
}
4011
4012
void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4013
    initializeDeviceGlobalVarEntryInfo(StringRef Name,
4014
                                       OMPTargetGlobalVarEntryKind Flags,
4015
109
                                       unsigned Order) {
4016
109
  assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
4017
109
                                             "only required for the device "
4018
109
                                             "code generation.");
4019
109
  OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
4020
109
  ++OffloadingEntriesNum;
4021
109
}
4022
4023
void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4024
    registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
4025
                                     CharUnits VarSize,
4026
                                     OMPTargetGlobalVarEntryKind Flags,
4027
516
                                     llvm::GlobalValue::LinkageTypes Linkage) {
4028
516
  if (CGM.getLangOpts().OpenMPIsDevice) {
4029
173
    auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
4030
173
    assert(Entry.isValid() && Entry.getFlags() == Flags &&
4031
173
           "Entry not initialized!");
4032
173
    assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&
4033
173
           "Resetting with the new address.");
4034
173
    if (Entry.getAddress() && 
hasDeviceGlobalVarEntryInfo(VarName)67
) {
4035
67
      if (Entry.getVarSize().isZero()) {
4036
4
        Entry.setVarSize(VarSize);
4037
4
        Entry.setLinkage(Linkage);
4038
4
      }
4039
67
      return;
4040
67
    }
4041
106
    Entry.setVarSize(VarSize);
4042
106
    Entry.setLinkage(Linkage);
4043
106
    Entry.setAddress(Addr);
4044
343
  } else {
4045
343
    if (hasDeviceGlobalVarEntryInfo(VarName)) {
4046
205
      auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
4047
205
      assert(Entry.isValid() && Entry.getFlags() == Flags &&
4048
205
             "Entry not initialized!");
4049
205
      assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&
4050
205
             "Resetting with the new address.");
4051
205
      if (Entry.getVarSize().isZero()) {
4052
20
        Entry.setVarSize(VarSize);
4053
20
        Entry.setLinkage(Linkage);
4054
20
      }
4055
205
      return;
4056
205
    }
4057
138
    OffloadEntriesDeviceGlobalVar.try_emplace(
4058
138
        VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
4059
138
    ++OffloadingEntriesNum;
4060
138
  }
4061
516
}
4062
4063
void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4064
    actOnDeviceGlobalVarEntriesInfo(
4065
1.65k
        const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
4066
1.65k
  // Scan all target region entries and perform the provided action.
4067
1.65k
  for (const auto &E : OffloadEntriesDeviceGlobalVar)
4068
244
    Action(E.getKey(), E.getValue());
4069
1.65k
}
4070
4071
void CGOpenMPRuntime::createOffloadEntry(
4072
    llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
4073
6.29k
    llvm::GlobalValue::LinkageTypes Linkage) {
4074
6.29k
  StringRef Name = Addr->getName();
4075
6.29k
  llvm::Module &M = CGM.getModule();
4076
6.29k
  llvm::LLVMContext &C = M.getContext();
4077
6.29k
4078
6.29k
  // Create constant string with the name.
4079
6.29k
  llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4080
6.29k
4081
6.29k
  std::string StringName = getName({"omp_offloading", "entry_name"});
4082
6.29k
  auto *Str = new llvm::GlobalVariable(
4083
6.29k
      M, StrPtrInit->getType(), /*isConstant=*/true,
4084
6.29k
      llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
4085
6.29k
  Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4086
6.29k
4087
6.29k
  llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
4088
6.29k
                            llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
4089
6.29k
                            llvm::ConstantInt::get(CGM.SizeTy, Size),
4090
6.29k
                            llvm::ConstantInt::get(CGM.Int32Ty, Flags),
4091
6.29k
                            llvm::ConstantInt::get(CGM.Int32Ty, 0)};
4092
6.29k
  std::string EntryName = getName({"omp_offloading", "entry", ""});
4093
6.29k
  llvm::GlobalVariable *Entry = createGlobalStruct(
4094
6.29k
      CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data,
4095
6.29k
      Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4096
6.29k
4097
6.29k
  // The entry has to be created in the section the linker expects it to be.
4098
6.29k
  Entry->setSection("omp_offloading_entries");
4099
6.29k
}
4100
4101
4.19k
void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
4102
4.19k
  // Emit the offloading entries and metadata so that the device codegen side
4103
4.19k
  // can easily figure out what to emit. The produced metadata looks like
4104
4.19k
  // this:
4105
4.19k
  //
4106
4.19k
  // !omp_offload.info = !{!1, ...}
4107
4.19k
  //
4108
4.19k
  // Right now we only generate metadata for function that contain target
4109
4.19k
  // regions.
4110
4.19k
4111
4.19k
  // If we are in simd mode or there are no entries, we don't need to do
4112
4.19k
  // anything.
4113
4.19k
  if (CGM.getLangOpts().OpenMPSimd || 
OffloadEntriesInfoManager.empty()2.22k
)
4114
2.53k
    return;
4115
1.65k
4116
1.65k
  llvm::Module &M = CGM.getModule();
4117
1.65k
  llvm::LLVMContext &C = M.getContext();
4118
1.65k
  SmallVector<std::tuple<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *,
4119
1.65k
                         SourceLocation, StringRef>,
4120
1.65k
              16>
4121
1.65k
      OrderedEntries(OffloadEntriesInfoManager.size());
4122
1.65k
  llvm::SmallVector<StringRef, 16> ParentFunctions(
4123
1.65k
      OffloadEntriesInfoManager.size());
4124
1.65k
4125
1.65k
  // Auxiliary methods to create metadata values and strings.
4126
34.6k
  auto &&GetMDInt = [this](unsigned V) {
4127
34.6k
    return llvm::ConstantAsMetadata::get(
4128
34.6k
        llvm::ConstantInt::get(CGM.Int32Ty, V));
4129
34.6k
  };
4130
1.65k
4131
7.02k
  auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
4132
1.65k
4133
1.65k
  // Create the offloading info metadata node.
4134
1.65k
  llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
4135
1.65k
4136
1.65k
  // Create function that emits metadata for each target region entry;
4137
1.65k
  auto &&TargetRegionMetadataEmitter =
4138
1.65k
      [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
4139
1.65k
       &GetMDString](
4140
1.65k
          unsigned DeviceID, unsigned FileID, StringRef ParentName,
4141
1.65k
          unsigned Line,
4142
6.78k
          const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
4143
6.78k
        // Generate metadata for target regions. Each entry of this metadata
4144
6.78k
        // contains:
4145
6.78k
        // - Entry 0 -> Kind of this type of metadata (0).
4146
6.78k
        // - Entry 1 -> Device ID of the file where the entry was identified.
4147
6.78k
        // - Entry 2 -> File ID of the file where the entry was identified.
4148
6.78k
        // - Entry 3 -> Mangled name of the function where the entry was
4149
6.78k
        // identified.
4150
6.78k
        // - Entry 4 -> Line in the file where the entry was identified.
4151
6.78k
        // - Entry 5 -> Order the entry was created.
4152
6.78k
        // The first element of the metadata node is the kind.
4153
6.78k
        llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4154
6.78k
                                 GetMDInt(FileID),      GetMDString(ParentName),
4155
6.78k
                                 GetMDInt(Line),        GetMDInt(E.getOrder())};
4156
6.78k
4157
6.78k
        SourceLocation Loc;
4158
6.78k
        for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
4159
6.78k
                  E = CGM.getContext().getSourceManager().fileinfo_end();
4160
13.6k
             I != E; 
++I6.84k
) {
4161
6.84k
          if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
4162
6.84k
              I->getFirst()->getUniqueID().getFile() == FileID) {
4163
0
            Loc = CGM.getContext().getSourceManager().translateFileLineCol(
4164
0
                I->getFirst(), Line, 1);
4165
0
            break;
4166
0
          }
4167
6.84k
        }
4168
6.78k
        // Save this entry in the right position of the ordered entries array.
4169
6.78k
        OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
4170
6.78k
        ParentFunctions[E.getOrder()] = ParentName;
4171
6.78k
4172
6.78k
        // Add metadata to the named metadata node.
4173
6.78k
        MD->addOperand(llvm::MDNode::get(C, Ops));
4174
6.78k
      };
4175
1.65k
4176
1.65k
  OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
4177
1.65k
      TargetRegionMetadataEmitter);
4178
1.65k
4179
1.65k
  // Create function that emits metadata for each device global variable entry;
4180
1.65k
  auto &&DeviceGlobalVarMetadataEmitter =
4181
1.65k
      [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4182
1.65k
       MD](StringRef MangledName,
4183
1.65k
           const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
4184
1.65k
               &E) {
4185
244
        // Generate metadata for global variables. Each entry of this metadata
4186
244
        // contains:
4187
244
        // - Entry 0 -> Kind of this type of metadata (1).
4188
244
        // - Entry 1 -> Mangled name of the variable.
4189
244
        // - Entry 2 -> Declare target kind.
4190
244
        // - Entry 3 -> Order the entry was created.
4191
244
        // The first element of the metadata node is the kind.
4192
244
        llvm::Metadata *Ops[] = {
4193
244
            GetMDInt(E.getKind()), GetMDString(MangledName),
4194
244
            GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4195
244
4196
244
        // Save this entry in the right position of the ordered entries array.
4197
244
        OrderedEntries[E.getOrder()] =
4198
244
            std::make_tuple(&E, SourceLocation(), MangledName);
4199
244
4200
244
        // Add metadata to the named metadata node.
4201
244
        MD->addOperand(llvm::MDNode::get(C, Ops));
4202
244
      };
4203
1.65k
4204
1.65k
  OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
4205
1.65k
      DeviceGlobalVarMetadataEmitter);
4206
1.65k
4207
7.02k
  for (const auto &E : OrderedEntries) {
4208
7.02k
    assert(std::get<0>(E) && "All ordered entries must exist!");
4209
7.02k
    if (const auto *CE =
4210
6.78k
            dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4211
6.78k
                std::get<0>(E))) {
4212
6.78k
      if (!CE->getID() || 
!CE->getAddress()6.78k
) {
4213
2
        // Do not blame the entry if the parent funtion is not emitted.
4214
2
        StringRef FnName = ParentFunctions[CE->getOrder()];
4215
2
        if (!CGM.GetGlobalValue(FnName))
4216
1
          continue;
4217
1
        unsigned DiagID = CGM.getDiags().getCustomDiagID(
4218
1
            DiagnosticsEngine::Error,
4219
1
            "Offloading entry for target region in %0 is incorrect: either the "
4220
1
            "address or the ID is invalid.");
4221
1
        CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName;
4222
1
        continue;
4223
1
      }
4224
6.78k
      createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
4225
6.78k
                         CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4226
6.78k
    } else 
if (const auto *244
CE244
= dyn_cast<OffloadEntriesInfoManagerTy::
4227
244
                                             OffloadEntryInfoDeviceGlobalVar>(
4228
244
                   std::get<0>(E))) {
4229
244
      OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
4230
244
          static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4231
244
              CE->getFlags());
4232
244
      switch (Flags) {
4233
197
      case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
4234
197
        if (CGM.getLangOpts().OpenMPIsDevice &&
4235
197
            
CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()89
)
4236
1
          continue;
4237
196
        if (!CE->getAddress()) {
4238
0
          unsigned DiagID = CGM.getDiags().getCustomDiagID(
4239
0
              DiagnosticsEngine::Error, "Offloading entry for declare target "
4240
0
                                        "variable %0 is incorrect: the "
4241
0
                                        "address is invalid.");
4242
0
          CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E);
4243
0
          continue;
4244
0
        }
4245
196
        // The vaiable has no definition - no need to add the entry.
4246
196
        if (CE->getVarSize().isZero())
4247
46
          continue;
4248
150
        break;
4249
150
      }
4250
150
      case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
4251
47
        assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||
4252
47
                (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&
4253
47
               "Declaret target link address is set.");
4254
47
        if (CGM.getLangOpts().OpenMPIsDevice)
4255
17
          continue;
4256
30
        if (!CE->getAddress()) {
4257
0
          unsigned DiagID = CGM.getDiags().getCustomDiagID(
4258
0
              DiagnosticsEngine::Error,
4259
0
              "Offloading entry for declare target variable is incorrect: the "
4260
0
              "address is invalid.");
4261
0
          CGM.getDiags().Report(DiagID);
4262
0
          continue;
4263
0
        }
4264
30
        break;
4265
180
      }
4266
180
      createOffloadEntry(CE->getAddress(), CE->getAddress(),
4267
180
                         CE->getVarSize().getQuantity(), Flags,
4268
180
                         CE->getLinkage());
4269
180
    } else {
4270
0
      llvm_unreachable("Unsupported entry kind.");
4271
0
    }
4272
7.02k
  }
4273
1.65k
}
4274
4275
/// Loads all the offload entries information from the host IR
4276
/// metadata.
4277
4.21k
void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4278
4.21k
  // If we are in target mode, load the metadata from the host IR. This code has
4279
4.21k
  // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4280
4.21k
4281
4.21k
  if (!CGM.getLangOpts().OpenMPIsDevice)
4282
3.84k
    return;
4283
372
4284
372
  if (CGM.getLangOpts().OMPHostIRFile.empty())
4285
0
    return;
4286
372
4287
372
  auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4288
372
  if (auto EC = Buf.getError()) {
4289
0
    CGM.getDiags().Report(diag::err_cannot_open_file)
4290
0
        << CGM.getLangOpts().OMPHostIRFile << EC.message();
4291
0
    return;
4292
0
  }
4293
372
4294
372
  llvm::LLVMContext C;
4295
372
  auto ME = expectedToErrorOrAndEmitErrors(
4296
372
      C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4297
372
4298
372
  if (auto EC = ME.getError()) {
4299
0
    unsigned DiagID = CGM.getDiags().getCustomDiagID(
4300
0
        DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4301
0
    CGM.getDiags().Report(DiagID)
4302
0
        << CGM.getLangOpts().OMPHostIRFile << EC.message();
4303
0
    return;
4304
0
  }
4305
372
4306
372
  llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4307
372
  if (!MD)
4308
23
    return;
4309
349
4310
2.11k
  
for (llvm::MDNode *MN : MD->operands())349
{
4311
10.3k
    auto &&GetMDInt = [MN](unsigned Idx) {
4312
10.3k
      auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4313
10.3k
      return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4314
10.3k
    };
4315
2.11k
4316
2.11k
    auto &&GetMDString = [MN](unsigned Idx) {
4317
2.11k
      auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4318
2.11k
      return V->getString();
4319
2.11k
    };
4320
2.11k
4321
2.11k
    switch (GetMDInt(0)) {
4322
0
    default:
4323
0
      llvm_unreachable("Unexpected metadata!");
4324
0
      break;
4325
2.00k
    case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4326
2.00k
        OffloadingEntryInfoTargetRegion:
4327
2.00k
      OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4328
2.00k
          /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4329
2.00k
          /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4330
2.00k
          /*Order=*/GetMDInt(5));
4331
2.00k
      break;
4332
109
    case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4333
109
        OffloadingEntryInfoDeviceGlobalVar:
4334
109
      OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4335
109
          /*MangledName=*/GetMDString(1),
4336
109
          static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4337
109
              /*Flags=*/GetMDInt(2)),
4338
109
          /*Order=*/GetMDInt(3));
4339
109
      break;
4340
2.11k
    }
4341
2.11k
  }
4342
349
}
4343
4344
635
void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4345
635
  if (!KmpRoutineEntryPtrTy) {
4346
229
    // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4347
229
    ASTContext &C = CGM.getContext();
4348
229
    QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4349
229
    FunctionProtoType::ExtProtoInfo EPI;
4350
229
    KmpRoutineEntryPtrQTy = C.getPointerType(
4351
229
        C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4352
229
    KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4353
229
  }
4354
635
}
4355
4356
6.29k
QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4357
6.29k
  // Make sure the type of the entry is already created. This is the type we
4358
6.29k
  // have to create:
4359
6.29k
  // struct __tgt_offload_entry{
4360
6.29k
  //   void      *addr;       // Pointer to the offload entry info.
4361
6.29k
  //                          // (function or global)
4362
6.29k
  //   char      *name;       // Name of the function or global.
4363
6.29k
  //   size_t     size;       // Size of the entry info (0 if it a function).
4364
6.29k
  //   int32_t    flags;      // Flags associated with the entry, e.g. 'link'.
4365
6.29k
  //   int32_t    reserved;   // Reserved, to use by the runtime library.
4366
6.29k
  // };
4367
6.29k
  if (TgtOffloadEntryQTy.isNull()) {
4368
1.55k
    ASTContext &C = CGM.getContext();
4369
1.55k
    RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4370
1.55k
    RD->startDefinition();
4371
1.55k
    addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4372
1.55k
    addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4373
1.55k
    addFieldToRecordDecl(C, RD, C.getSizeType());
4374
1.55k
    addFieldToRecordDecl(
4375
1.55k
        C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4376
1.55k
    addFieldToRecordDecl(
4377
1.55k
        C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4378
1.55k
    RD->completeDefinition();
4379
1.55k
    RD->addAttr(PackedAttr::CreateImplicit(C));
4380
1.55k
    TgtOffloadEntryQTy = C.getRecordType(RD);
4381
1.55k
  }
4382
6.29k
  return TgtOffloadEntryQTy;
4383
6.29k
}
4384
4385
namespace {
4386
struct PrivateHelpersTy {
4387
  PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4388
                   const VarDecl *PrivateElemInit)
4389
      : Original(Original), PrivateCopy(PrivateCopy),
4390
1.15k
        PrivateElemInit(PrivateElemInit) {}
4391
  const VarDecl *Original;
4392
  const VarDecl *PrivateCopy;
4393
  const VarDecl *PrivateElemInit;
4394
};
4395
typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4396
} // anonymous namespace
4397
4398
static RecordDecl *
4399
635
createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4400
635
  if (!Privates.empty()) {
4401
396
    ASTContext &C = CGM.getContext();
4402
396
    // Build struct .kmp_privates_t. {
4403
396
    //         /*  private vars  */
4404
396
    //       };
4405
396
    RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4406
396
    RD->startDefinition();
4407
1.15k
    for (const auto &Pair : Privates) {
4408
1.15k
      const VarDecl *VD = Pair.second.Original;
4409
1.15k
      QualType Type = VD->getType().getNonReferenceType();
4410
1.15k
      FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4411
1.15k
      if (VD->hasAttrs()) {
4412
40
        for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4413
40
             E(VD->getAttrs().end());
4414
80
             I != E; 
++I40
)
4415
40
          FD->addAttr(*I);
4416
40
      }
4417
1.15k
    }
4418
396
    RD->completeDefinition();
4419
396
    return RD;
4420
396
  }
4421
239
  return nullptr;
4422
239
}
4423
4424
static RecordDecl *
4425
createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4426
                         QualType KmpInt32Ty,
4427
245
                         QualType KmpRoutineEntryPointerQTy) {
4428
245
  ASTContext &C = CGM.getContext();
4429
245
  // Build struct kmp_task_t {
4430
245
  //         void *              shareds;
4431
245
  //         kmp_routine_entry_t routine;
4432
245
  //         kmp_int32           part_id;
4433
245
  //         kmp_cmplrdata_t data1;
4434
245
  //         kmp_cmplrdata_t data2;
4435
245
  // For taskloops additional fields:
4436
245
  //         kmp_uint64          lb;
4437
245
  //         kmp_uint64          ub;
4438
245
  //         kmp_int64           st;
4439
245
  //         kmp_int32           liter;
4440
245
  //         void *              reductions;
4441
245
  //       };
4442
245
  RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4443
245
  UD->startDefinition();
4444
245
  addFieldToRecordDecl(C, UD, KmpInt32Ty);
4445
245
  addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4446
245
  UD->completeDefinition();
4447
245
  QualType KmpCmplrdataTy = C.getRecordType(UD);
4448
245
  RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4449
245
  RD->startDefinition();
4450
245
  addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4451
245
  addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4452
245
  addFieldToRecordDecl(C, RD, KmpInt32Ty);
4453
245
  addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4454
245
  addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4455
245
  if (isOpenMPTaskLoopDirective(Kind)) {
4456
126
    QualType KmpUInt64Ty =
4457
126
        CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4458
126
    QualType KmpInt64Ty =
4459
126
        CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4460
126
    addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4461
126
    addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4462
126
    addFieldToRecordDecl(C, RD, KmpInt64Ty);
4463
126
    addFieldToRecordDecl(C, RD, KmpInt32Ty);
4464
126
    addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4465
126
  }
4466
245
  RD->completeDefinition();
4467
245
  return RD;
4468
245
}
4469
4470
static RecordDecl *
4471
createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4472
635
                                     ArrayRef<PrivateDataTy> Privates) {
4473
635
  ASTContext &C = CGM.getContext();
4474
635
  // Build struct kmp_task_t_with_privates {
4475
635
  //         kmp_task_t task_data;
4476
635
  //         .kmp_privates_t. privates;
4477
635
  //       };
4478
635
  RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4479
635
  RD->startDefinition();
4480
635
  addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4481
635
  if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4482
396
    addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4483
635
  RD->completeDefinition();
4484
635
  return RD;
4485
635
}
4486
4487
/// Emit a proxy function which accepts kmp_task_t as the second
4488
/// argument.
4489
/// \code
4490
/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4491
///   TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4492
///   For taskloops:
4493
///   tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4494
///   tt->reductions, tt->shareds);
4495
///   return 0;
4496
/// }
4497
/// \endcode
4498
static llvm::Function *
4499
emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4500
                      OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4501
                      QualType KmpTaskTWithPrivatesPtrQTy,
4502
                      QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4503
                      QualType SharedsPtrTy, llvm::Function *TaskFunction,
4504
635
                      llvm::Value *TaskPrivatesMap) {
4505
635
  ASTContext &C = CGM.getContext();
4506
635
  FunctionArgList Args;
4507
635
  ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4508
635
                            ImplicitParamDecl::Other);
4509
635
  ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4510
635
                                KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4511
635
                                ImplicitParamDecl::Other);
4512
635
  Args.push_back(&GtidArg);
4513
635
  Args.push_back(&TaskTypeArg);
4514
635
  const auto &TaskEntryFnInfo =
4515
635
      CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4516
635
  llvm::FunctionType *TaskEntryTy =
4517
635
      CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4518
635
  std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4519
635
  auto *TaskEntry = llvm::Function::Create(
4520
635
      TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4521
635
  CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4522
635
  TaskEntry->setDoesNotRecurse();
4523
635
  CodeGenFunction CGF(CGM);
4524
635
  CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4525
635
                    Loc, Loc);
4526
635
4527
635
  // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4528
635
  // tt,
4529
635
  // For taskloops:
4530
635
  // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4531
635
  // tt->task_data.shareds);
4532
635
  llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4533
635
      CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4534
635
  LValue TDBase = CGF.EmitLoadOfPointerLValue(
4535
635
      CGF.GetAddrOfLocalVar(&TaskTypeArg),
4536
635
      KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4537
635
  const auto *KmpTaskTWithPrivatesQTyRD =
4538
635
      cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4539
635
  LValue Base =
4540
635
      CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4541
635
  const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4542
635
  auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4543
635
  LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4544
635
  llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
4545
635
4546
635
  auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4547
635
  LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4548
635
  llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4549
635
      CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4550
635
      CGF.ConvertTypeForMem(SharedsPtrTy));
4551
635
4552
635
  auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4553
635
  llvm::Value *PrivatesParam;
4554
635
  if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4555
396
    LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4556
396
    PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4557
396
        PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy);
4558
396
  } else {
4559
239
    PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4560
239
  }
4561
635
4562
635
  llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4563
635
                               TaskPrivatesMap,
4564
635
                               CGF.Builder
4565
635
                                   .CreatePointerBitCastOrAddrSpaceCast(
4566
635
                                       TDBase.getAddress(CGF), CGF.VoidPtrTy)
4567
635
                                   .getPointer()};
4568
635
  SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4569
635
                                          std::end(CommonArgs));
4570
635
  if (isOpenMPTaskLoopDirective(Kind)) {
4571
222
    auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4572
222
    LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4573
222
    llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4574
222
    auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4575
222
    LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4576
222
    llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4577
222
    auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4578
222
    LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4579
222
    llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4580
222
    auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4581
222
    LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4582
222
    llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4583
222
    auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4584
222
    LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4585
222
    llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4586
222
    CallArgs.push_back(LBParam);
4587
222
    CallArgs.push_back(UBParam);
4588
222
    CallArgs.push_back(StParam);
4589
222
    CallArgs.push_back(LIParam);
4590
222
    CallArgs.push_back(RParam);
4591
222
  }
4592
635
  CallArgs.push_back(SharedsParam);
4593
635
4594
635
  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4595
635
                                                  CallArgs);
4596
635
  CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4597
635
                             CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4598
635
  CGF.FinishFunction();
4599
635
  return TaskEntry;
4600
635
}
4601
4602
static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4603
                                            SourceLocation Loc,
4604
                                            QualType KmpInt32Ty,
4605
                                            QualType KmpTaskTWithPrivatesPtrQTy,
4606
81
                                            QualType KmpTaskTWithPrivatesQTy) {
4607
81
  ASTContext &C = CGM.getContext();
4608
81
  FunctionArgList Args;
4609
81
  ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4610
81
                            ImplicitParamDecl::Other);
4611
81
  ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4612
81
                                KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4613
81
                                ImplicitParamDecl::Other);
4614
81
  Args.push_back(&GtidArg);
4615
81
  Args.push_back(&TaskTypeArg);
4616
81
  const auto &DestructorFnInfo =
4617
81
      CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4618
81
  llvm::FunctionType *DestructorFnTy =
4619
81
      CGM.getTypes().GetFunctionType(DestructorFnInfo);
4620
81
  std::string Name =
4621
81
      CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4622
81
  auto *DestructorFn =
4623
81
      llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4624
81
                             Name, &CGM.getModule());
4625
81
  CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4626
81
                                    DestructorFnInfo);
4627
81
  DestructorFn->setDoesNotRecurse();
4628
81
  CodeGenFunction CGF(CGM);
4629
81
  CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4630
81
                    Args, Loc, Loc);
4631
81
4632
81
  LValue Base = CGF.EmitLoadOfPointerLValue(
4633
81
      CGF.GetAddrOfLocalVar(&TaskTypeArg),
4634
81
      KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4635
81
  const auto *KmpTaskTWithPrivatesQTyRD =
4636
81
      cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4637
81
  auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4638
81
  Base = CGF.EmitLValueForField(Base, *FI);
4639
81
  for (const auto *Field :
4640
362
       cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4641
362
    if (QualType::DestructionKind DtorKind =
4642
162
            Field->getType().isDestructedType()) {
4643
162
      LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4644
162
      CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType());
4645
162
    }
4646
362
  }
4647
81
  CGF.FinishFunction();
4648
81
  return DestructorFn;
4649
81
}
4650
4651
/// Emit a privates mapping function for correct handling of private and
4652
/// firstprivate variables.
4653
/// \code
4654
/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4655
/// **noalias priv1,...,  <tyn> **noalias privn) {
4656
///   *priv1 = &.privates.priv1;
4657
///   ...;
4658
///   *privn = &.privates.privn;
4659
/// }
4660
/// \endcode
4661
static llvm::Value *
4662
emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4663
                               ArrayRef<const Expr *> PrivateVars,
4664
                               ArrayRef<const Expr *> FirstprivateVars,
4665
                               ArrayRef<const Expr *> LastprivateVars,
4666
                               QualType PrivatesQTy,
4667
396
                               ArrayRef<PrivateDataTy> Privates) {
4668
396
  ASTContext &C = CGM.getContext();
4669
396
  FunctionArgList Args;
4670
396
  ImplicitParamDecl TaskPrivatesArg(
4671
396
      C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4672
396
      C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4673
396
      ImplicitParamDecl::Other);
4674
396
  Args.push_back(&TaskPrivatesArg);
4675
396
  llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4676
396
  unsigned Counter = 1;
4677
396
  for (const Expr *E : PrivateVars) {
4678
170
    Args.push_back(ImplicitParamDecl::Create(
4679
170
        C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4680
170
        C.getPointerType(C.getPointerType(E->getType()))
4681
170
            .withConst()
4682
170
            .withRestrict(),
4683
170
        ImplicitParamDecl::Other));
4684
170
    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4685
170
    PrivateVarsPos[VD] = Counter;
4686
170
    ++Counter;
4687
170
  }
4688
835
  for (const Expr *E : FirstprivateVars) {
4689
835
    Args.push_back(ImplicitParamDecl::Create(
4690
835
        C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4691
835
        C.getPointerType(C.getPointerType(E->getType()))
4692
835
            .withConst()
4693
835
            .withRestrict(),
4694
835
        ImplicitParamDecl::Other));
4695
835
    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4696
835
    PrivateVarsPos[VD] = Counter;
4697
835
    ++Counter;
4698
835
  }
4699
396
  for (const Expr *E : LastprivateVars) {
4700
151
    Args.push_back(ImplicitParamDecl::Create(
4701
151
        C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4702
151
        C.getPointerType(C.getPointerType(E->getType()))
4703
151
            .withConst()
4704
151
            .withRestrict(),
4705
151
        ImplicitParamDecl::Other));
4706
151
    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4707
151
    PrivateVarsPos[VD] = Counter;
4708
151
    ++Counter;
4709
151
  }
4710
396
  const auto &TaskPrivatesMapFnInfo =
4711
396
      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4712
396
  llvm::FunctionType *TaskPrivatesMapTy =
4713
396
      CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4714
396
  std::string Name =
4715
396
      CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4716
396
  auto *TaskPrivatesMap = llvm::Function::Create(
4717
396
      TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4718
396
      &CGM.getModule());
4719
396
  CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4720
396
                                    TaskPrivatesMapFnInfo);
4721
396
  if (CGM.getLangOpts().Optimize) {
4722
0
    TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4723
0
    TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4724
0
    TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4725
0
  }
4726
396
  CodeGenFunction CGF(CGM);
4727
396
  CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4728
396
                    TaskPrivatesMapFnInfo, Args, Loc, Loc);
4729
396
4730
396
  // *privi = &.privates.privi;
4731
396
  LValue Base = CGF.EmitLoadOfPointerLValue(
4732
396
      CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4733
396
      TaskPrivatesArg.getType()->castAs<PointerType>());
4734
396
  const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4735
396
  Counter = 0;
4736
1.15k
  for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4737
1.15k
    LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4738
1.15k
    const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4739
1.15k
    LValue RefLVal =
4740
1.15k
        CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4741
1.15k
    LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4742
1.15k
        RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>());
4743
1.15k
    CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal);
4744
1.15k
    ++Counter;
4745
1.15k
  }
4746
396
  CGF.FinishFunction();
4747
396
  return TaskPrivatesMap;
4748
396
}
4749
4750
/// Emit initialization for private variables in task-based directives.
4751
static void emitPrivatesInit(CodeGenFunction &CGF,
4752
                             const OMPExecutableDirective &D,
4753
                             Address KmpTaskSharedsPtr, LValue TDBase,
4754
                             const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4755
                             QualType SharedsTy, QualType SharedsPtrTy,
4756
                             const OMPTaskDataTy &Data,
4757
493
                             ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4758
493
  ASTContext &C = CGF.getContext();
4759
493
  auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4760
493
  LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4761
493
  OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4762
493
                                 ? 
OMPD_taskloop244
4763
493
                                 : 
OMPD_task249
;
4764
493
  const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4765
493
  CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4766
493
  LValue SrcBase;
4767
493
  bool IsTargetTask =
4768
493
      isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4769
493
      
isOpenMPTargetExecutionDirective(D.getDirectiveKind())445
;
4770
493
  // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4771
493
  // PointersArray and SizesArray. The original variables for these arrays are
4772
493
  // not captured and we get their addresses explicitly.
4773
493
  if ((!IsTargetTask && 
!Data.FirstprivateVars.empty()265
) ||
4774
493
      
(400
IsTargetTask400
&&
KmpTaskSharedsPtr.isValid()228
)) {
4775
297
    SrcBase = CGF.MakeAddrLValue(
4776
297
        CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4777
297
            KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4778
297
        SharedsTy);
4779
297
  }
4780
493
  FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4781
1.52k
  for (const PrivateDataTy &Pair : Privates) {
4782
1.52k
    const VarDecl *VD = Pair.second.PrivateCopy;
4783
1.52k
    const Expr *Init = VD->getAnyInitializer();
4784
1.52k
    if (Init && 
(1.14k
!ForDup1.14k
||
(204
isa<CXXConstructExpr>(Init)204
&&
4785
1.08k
                             
!CGF.isTrivialInitializer(Init)144
))) {
4786
1.08k
      LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4787
1.08k
      if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4788
883
        const VarDecl *OriginalVD = Pair.second.Original;
4789
883
        // Check if the variable is the target-based BasePointersArray,
4790
883
        // PointersArray or SizesArray.
4791
883
        LValue SharedRefLValue;
4792
883
        QualType Type = PrivateLValue.getType();
4793
883
        const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4794
883
        if (IsTargetTask && 
!SharedField624
) {
4795
324
          assert(isa<ImplicitParamDecl>(OriginalVD) &&
4796
324
                 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&
4797
324
                 cast<CapturedDecl>(OriginalVD->getDeclContext())
4798
324
                         ->getNumParams() == 0 &&
4799
324
                 isa<TranslationUnitDecl>(
4800
324
                     cast<CapturedDecl>(OriginalVD->getDeclContext())
4801
324
                         ->getDeclContext()) &&
4802
324
                 "Expected artificial target data variable.");
4803
324
          SharedRefLValue =
4804
324
              CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4805
559
        } else {
4806
559
          SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4807
559
          SharedRefLValue = CGF.MakeAddrLValue(
4808
559
              Address(SharedRefLValue.getPointer(CGF),
4809
559
                      C.getDeclAlign(OriginalVD)),
4810
559
              SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4811
559
              SharedRefLValue.getTBAAInfo());
4812
559
        }
4813
883
        if (Type->isArrayType()) {
4814
417
          // Initialize firstprivate array.
4815
417
          if (!isa<CXXConstructExpr>(Init) || 
CGF.isTrivialInitializer(Init)52
) {
4816
365
            // Perform simple memcpy.
4817
365
            CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4818
365
          } else {
4819
52
            // Initialize firstprivate array using element-by-element
4820
52
            // initialization.
4821
52
            CGF.EmitOMPAggregateAssign(
4822
52
                PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF),
4823
52
                Type,
4824
52
                [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4825
52
                                                  Address SrcElement) {
4826
52
                  // Clean up any temporaries needed by the initialization.
4827
52
                  CodeGenFunction::OMPPrivateScope InitScope(CGF);
4828
52
                  InitScope.addPrivate(
4829
52
                      Elem, [SrcElement]() -> Address { return SrcElement; });
4830
52
                  (void)InitScope.Privatize();
4831
52
                  // Emit initialization for single element.
4832
52
                  CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4833
52
                      CGF, &CapturesInfo);
4834
52
                  CGF.EmitAnyExprToMem(Init, DestElement,
4835
52
                                       Init->getType().getQualifiers(),
4836
52
                                       /*IsInitializer=*/false);
4837
52
                });
4838
52
          }
4839
466
        } else {
4840
466
          CodeGenFunction::OMPPrivateScope InitScope(CGF);
4841
466
          InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address {
4842
466
            return SharedRefLValue.getAddress(CGF);
4843
466
          });
4844
466
          (void)InitScope.Privatize();
4845
466
          CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4846
466
          CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4847
466
                             /*capturedByInit=*/false);
4848
466
        }
4849
883
      } else {
4850
202
        CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4851
202
      }
4852
1.08k
    }
4853
1.52k
    ++FI;
4854
1.52k
  }
4855
493
}
4856
4857
/// Check if duplication function is required for taskloops.
4858
static bool checkInitIsRequired(CodeGenFunction &CGF,
4859
98
                                ArrayRef<PrivateDataTy> Privates) {
4860
98
  bool InitRequired = false;
4861
202
  for (const PrivateDataTy &Pair : Privates) {
4862
202
    const VarDecl *VD = Pair.second.PrivateCopy;
4863
202
    const Expr *Init = VD->getAnyInitializer();
4864
202
    InitRequired = InitRequired || (Init && 
isa<CXXConstructExpr>(Init)142
&&
4865
202
                                    
!CGF.isTrivialInitializer(Init)48
);
4866
202
    if (InitRequired)
4867
48
      break;
4868
202
  }
4869
98
  return InitRequired;
4870
98
}
4871
4872
4873
/// Emit task_dup function (for initialization of
4874
/// private/firstprivate/lastprivate vars and last_iter flag)
4875
/// \code
4876
/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4877
/// lastpriv) {
4878
/// // setup lastprivate flag
4879
///    task_dst->last = lastpriv;
4880
/// // could be constructor calls here...
4881
/// }
4882
/// \endcode
4883
static llvm::Value *
4884
emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
4885
                    const OMPExecutableDirective &D,
4886
                    QualType KmpTaskTWithPrivatesPtrQTy,
4887
                    const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4888
                    const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
4889
                    QualType SharedsPtrTy, const OMPTaskDataTy &Data,
4890
97
                    ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
4891
97
  ASTContext &C = CGM.getContext();
4892
97
  FunctionArgList Args;
4893
97
  ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4894
97
                           KmpTaskTWithPrivatesPtrQTy,
4895
97
                           ImplicitParamDecl::Other);
4896
97
  ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4897
97
                           KmpTaskTWithPrivatesPtrQTy,
4898
97
                           ImplicitParamDecl::Other);
4899
97
  ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
4900
97
                                ImplicitParamDecl::Other);
4901
97
  Args.push_back(&DstArg);
4902
97
  Args.push_back(&SrcArg);
4903
97
  Args.push_back(&LastprivArg);
4904
97
  const auto &TaskDupFnInfo =
4905
97
      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4906
97
  llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
4907
97
  std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
4908
97
  auto *TaskDup = llvm::Function::Create(
4909
97
      TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4910
97
  CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
4911
97
  TaskDup->setDoesNotRecurse();
4912
97
  CodeGenFunction CGF(CGM);
4913
97
  CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
4914
97
                    Loc);
4915
97
4916
97
  LValue TDBase = CGF.EmitLoadOfPointerLValue(
4917
97
      CGF.GetAddrOfLocalVar(&DstArg),
4918
97
      KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4919
97
  // task_dst->liter = lastpriv;
4920
97
  if (WithLastIter) {
4921
49
    auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4922
49
    LValue Base = CGF.EmitLValueForField(
4923
49
        TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4924
49
    LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4925
49
    llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
4926
49
        CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
4927
49
    CGF.EmitStoreOfScalar(Lastpriv, LILVal);
4928
49
  }
4929
97
4930
97
  // Emit initial values for private copies (if any).
4931
97
  assert(!Privates.empty());
4932
97
  Address KmpTaskSharedsPtr = Address::invalid();
4933
97
  if (!Data.FirstprivateVars.empty()) {
4934
24
    LValue TDBase = CGF.EmitLoadOfPointerLValue(
4935
24
        CGF.GetAddrOfLocalVar(&SrcArg),
4936
24
        KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4937
24
    LValue Base = CGF.EmitLValueForField(
4938
24
        TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4939
24
    KmpTaskSharedsPtr = Address(
4940
24
        CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
4941
24
                                 Base, *std::next(KmpTaskTQTyRD->field_begin(),
4942
24
                                                  KmpTaskTShareds)),
4943
24
                             Loc),
4944
24
        CGF.getNaturalTypeAlignment(SharedsTy));
4945
24
  }
4946
97
  emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4947
97
                   SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
4948
97
  CGF.FinishFunction();
4949
97
  return TaskDup;
4950
97
}
4951
4952
/// Checks if destructor function is required to be generated.
4953
/// \return true if cleanups are required, false otherwise.
4954
static bool
4955
396
checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
4956
396
  bool NeedsCleanup = false;
4957
396
  auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4958
396
  const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4959
955
  for (const FieldDecl *FD : PrivateRD->fields()) {
4960
955
    NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4961
955
    if (NeedsCleanup)
4962
81
      break;
4963
955
  }
4964
396
  return NeedsCleanup;
4965
396
}
4966
4967
CGOpenMPRuntime::TaskResultTy
4968
CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4969
                              const OMPExecutableDirective &D,
4970
                              llvm::Function *TaskFunction, QualType SharedsTy,
4971
635
                              Address Shareds, const OMPTaskDataTy &Data) {
4972
635
  ASTContext &C = CGM.getContext();
4973
635
  llvm::SmallVector<PrivateDataTy, 4> Privates;
4974
635
  // Aggregate privates and sort them by the alignment.
4975
635
  auto I = Data.PrivateCopies.begin();
4976
635
  for (const Expr *E : Data.PrivateVars) {
4977
170
    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4978
170
    Privates.emplace_back(
4979
170
        C.getDeclAlign(VD),
4980
170
        PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4981
170
                         /*PrivateElemInit=*/nullptr));
4982
170
    ++I;
4983
170
  }
4984
635
  I = Data.FirstprivateCopies.begin();
4985
635
  auto IElemInitRef = Data.FirstprivateInits.begin();
4986
835
  for (const Expr *E : Data.FirstprivateVars) {
4987
835
    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4988
835
    Privates.emplace_back(
4989
835
        C.getDeclAlign(VD),
4990
835
        PrivateHelpersTy(
4991
835
            VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4992
835
            cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4993
835
    ++I;
4994
835
    ++IElemInitRef;
4995
835
  }
4996
635
  I = Data.LastprivateCopies.begin();
4997
635
  for (const Expr *E : Data.LastprivateVars) {
4998
151
    const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4999
151
    Privates.emplace_back(
5000
151
        C.getDeclAlign(VD),
5001
151
        PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
5002
151
                         /*PrivateElemInit=*/nullptr));
5003
151
    ++I;
5004
151
  }
5005
1.08k
  llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) {
5006
1.08k
    return L.first > R.first;
5007
1.08k
  });
5008
635
  QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
5009
635
  // Build type kmp_routine_entry_t (if not built yet).
5010
635
  emitKmpRoutineEntryT(KmpInt32Ty);
5011
635
  // Build type kmp_task_t (if not built yet).
5012
635
  if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
5013
222
    if (SavedKmpTaskloopTQTy.isNull()) {
5014
126
      SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5015
126
          CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5016
126
    }
5017
222
    KmpTaskTQTy = SavedKmpTaskloopTQTy;
5018
413
  } else {
5019
413
    assert((D.getDirectiveKind() == OMPD_task ||
5020
413
            isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||
5021
413
            isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
5022
413
           "Expected taskloop, task or target directive");
5023
413
    if (SavedKmpTaskTQTy.isNull()) {
5024
119
      SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
5025
119
          CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
5026
119
    }
5027
413
    KmpTaskTQTy = SavedKmpTaskTQTy;
5028
413
  }
5029
635
  const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
5030
635
  // Build particular struct kmp_task_t for the given task.
5031
635
  const RecordDecl *KmpTaskTWithPrivatesQTyRD =
5032
635
      createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
5033
635
  QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
5034
635
  QualType KmpTaskTWithPrivatesPtrQTy =
5035
635
      C.getPointerType(KmpTaskTWithPrivatesQTy);