Coverage Report

Created: 2021-08-24 07:12

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
Line
Count
Source (jump to first uncovered line)
1
//===------ CGOpenMPRuntimeGPU.h - Interface to OpenMP GPU Runtimes ------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This provides a generalized class for OpenMP runtime code generation
10
// specialized by GPU targets NVPTX and AMDGCN.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H
15
#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H
16
17
#include "CGOpenMPRuntime.h"
18
#include "CodeGenFunction.h"
19
#include "clang/AST/StmtOpenMP.h"
20
21
namespace clang {
22
namespace CodeGen {
23
24
class CGOpenMPRuntimeGPU : public CGOpenMPRuntime {
25
public:
26
  /// Defines the execution mode.
27
  enum ExecutionMode {
28
    /// SPMD execution mode (all threads are worker threads).
29
    EM_SPMD,
30
    /// Non-SPMD execution mode (1 master thread, others are workers).
31
    EM_NonSPMD,
32
    /// Unknown execution mode (orphaned directive).
33
    EM_Unknown,
34
  };
35
private:
36
  /// Parallel outlined function work for workers to execute.
37
  llvm::SmallVector<llvm::Function *, 16> Work;
38
39
  struct EntryFunctionState {
40
    SourceLocation Loc;
41
  };
42
43
  ExecutionMode getExecutionMode() const;
44
45
4.35k
  bool requiresFullRuntime() const { return RequiresFullRuntime; }
46
47
  /// Get barrier to synchronize all threads in a block.
48
  void syncCTAThreads(CodeGenFunction &CGF);
49
50
  /// Helper for target directive initialization.
51
  void emitKernelInit(CodeGenFunction &CGF, EntryFunctionState &EST,
52
                      bool IsSPMD);
53
54
  /// Helper for target directive finalization.
55
  void emitKernelDeinit(CodeGenFunction &CGF, EntryFunctionState &EST,
56
                        bool IsSPMD);
57
58
  /// Helper for generic variables globalization prolog.
59
  void emitGenericVarsProlog(CodeGenFunction &CGF, SourceLocation Loc,
60
                             bool WithSPMDCheck = false);
61
62
  /// Helper for generic variables globalization epilog.
63
  void emitGenericVarsEpilog(CodeGenFunction &CGF, bool WithSPMDCheck = false);
64
65
  //
66
  // Base class overrides.
67
  //
68
69
  /// Creates offloading entry for the provided entry ID \a ID,
70
  /// address \a Addr, size \a Size, and flags \a Flags.
71
  void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr,
72
                          uint64_t Size, int32_t Flags,
73
                          llvm::GlobalValue::LinkageTypes Linkage) override;
74
75
  /// Emit outlined function specialized for the Fork-Join
76
  /// programming model for applicable target directives on the NVPTX device.
77
  /// \param D Directive to emit.
78
  /// \param ParentName Name of the function that encloses the target region.
79
  /// \param OutlinedFn Outlined function value to be defined by this call.
80
  /// \param OutlinedFnID Outlined function ID value to be defined by this call.
81
  /// \param IsOffloadEntry True if the outlined function is an offload entry.
82
  /// An outlined function may not be an entry if, e.g. the if clause always
83
  /// evaluates to false.
84
  void emitNonSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
85
                         llvm::Function *&OutlinedFn,
86
                         llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
87
                         const RegionCodeGenTy &CodeGen);
88
89
  /// Emit outlined function specialized for the Single Program
90
  /// Multiple Data programming model for applicable target directives on the
91
  /// NVPTX device.
92
  /// \param D Directive to emit.
93
  /// \param ParentName Name of the function that encloses the target region.
94
  /// \param OutlinedFn Outlined function value to be defined by this call.
95
  /// \param OutlinedFnID Outlined function ID value to be defined by this call.
96
  /// \param IsOffloadEntry True if the outlined function is an offload entry.
97
  /// \param CodeGen Object containing the target statements.
98
  /// An outlined function may not be an entry if, e.g. the if clause always
99
  /// evaluates to false.
100
  void emitSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
101
                      llvm::Function *&OutlinedFn,
102
                      llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
103
                      const RegionCodeGenTy &CodeGen);
104
105
  /// Emit outlined function for 'target' directive on the NVPTX
106
  /// device.
107
  /// \param D Directive to emit.
108
  /// \param ParentName Name of the function that encloses the target region.
109
  /// \param OutlinedFn Outlined function value to be defined by this call.
110
  /// \param OutlinedFnID Outlined function ID value to be defined by this call.
111
  /// \param IsOffloadEntry True if the outlined function is an offload entry.
112
  /// An outlined function may not be an entry if, e.g. the if clause always
113
  /// evaluates to false.
114
  void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
115
                                  StringRef ParentName,
116
                                  llvm::Function *&OutlinedFn,
117
                                  llvm::Constant *&OutlinedFnID,
118
                                  bool IsOffloadEntry,
119
                                  const RegionCodeGenTy &CodeGen) override;
120
121
  /// Emits code for parallel or serial call of the \a OutlinedFn with
122
  /// variables captured in a record which address is stored in \a
123
  /// CapturedStruct.
124
  /// This call is for the Non-SPMD Execution Mode.
125
  /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
126
  /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
127
  /// \param CapturedVars A pointer to the record with the references to
128
  /// variables used in \a OutlinedFn function.
129
  /// \param IfCond Condition in the associated 'if' clause, if it was
130
  /// specified, nullptr otherwise.
131
  void emitNonSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
132
                               llvm::Value *OutlinedFn,
133
                               ArrayRef<llvm::Value *> CapturedVars,
134
                               const Expr *IfCond);
135
136
  /// Emits code for parallel or serial call of the \a OutlinedFn with
137
  /// variables captured in a record which address is stored in \a
138
  /// CapturedStruct.
139
  /// This call is for a parallel directive within an SPMD target directive.
140
  /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
141
  /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
142
  /// \param CapturedVars A pointer to the record with the references to
143
  /// variables used in \a OutlinedFn function.
144
  /// \param IfCond Condition in the associated 'if' clause, if it was
145
  /// specified, nullptr otherwise.
146
  ///
147
  void emitSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
148
                            llvm::Function *OutlinedFn,
149
                            ArrayRef<llvm::Value *> CapturedVars,
150
                            const Expr *IfCond);
151
152
protected:
153
  /// Get the function name of an outlined region.
154
  //  The name can be customized depending on the target.
155
  //
156
904
  StringRef getOutlinedHelperName() const override {
157
904
    return "__omp_outlined__";
158
904
  }
159
160
  /// Check if the default location must be constant.
161
  /// Constant for NVPTX for better optimization.
162
0
  bool isDefaultLocationConstant() const override { return true; }
163
164
  /// Returns additional flags that can be stored in reserved_2 field of the
165
  /// default location.
166
  /// For NVPTX target contains data about SPMD/Non-SPMD execution mode +
167
  /// Full/Lightweight runtime mode. Used for better optimization.
168
  unsigned getDefaultLocationReserved2Flags() const override;
169
170
public:
171
  explicit CGOpenMPRuntimeGPU(CodeGenModule &CGM);
172
  void clear() override;
173
174
  /// Declare generalized virtual functions which need to be defined
175
  /// by all specializations of OpenMPGPURuntime Targets like AMDGCN
176
  /// and NVPTX.
177
178
  /// Get the GPU warp size.
179
  virtual llvm::Value *getGPUWarpSize(CodeGenFunction &CGF) = 0;
180
181
  /// Get the id of the current thread on the GPU.
182
  virtual llvm::Value *getGPUThreadID(CodeGenFunction &CGF) = 0;
183
184
  /// Get the maximum number of threads in a block of the GPU.
185
  virtual llvm::Value *getGPUNumThreads(CodeGenFunction &CGF) = 0;
186
187
  /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
188
  /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
189
  virtual void emitProcBindClause(CodeGenFunction &CGF,
190
                                  llvm::omp::ProcBindKind ProcBind,
191
                                  SourceLocation Loc) override;
192
193
  /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
194
  /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
195
  /// clause.
196
  /// \param NumThreads An integer value of threads.
197
  virtual void emitNumThreadsClause(CodeGenFunction &CGF,
198
                                    llvm::Value *NumThreads,
199
                                    SourceLocation Loc) override;
200
201
  /// This function ought to emit, in the general case, a call to
202
  // the openmp runtime kmpc_push_num_teams. In NVPTX backend it is not needed
203
  // as these numbers are obtained through the PTX grid and block configuration.
204
  /// \param NumTeams An integer expression of teams.
205
  /// \param ThreadLimit An integer expression of threads.
206
  void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
207
                          const Expr *ThreadLimit, SourceLocation Loc) override;
208
209
  /// Emits inlined function for the specified OpenMP parallel
210
  //  directive.
211
  /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
212
  /// kmp_int32 BoundID, struct context_vars*).
213
  /// \param D OpenMP directive.
214
  /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
215
  /// \param InnermostKind Kind of innermost directive (for simple directives it
216
  /// is a directive itself, for combined - its innermost directive).
217
  /// \param CodeGen Code generation sequence for the \a D directive.
218
  llvm::Function *
219
  emitParallelOutlinedFunction(const OMPExecutableDirective &D,
220
                               const VarDecl *ThreadIDVar,
221
                               OpenMPDirectiveKind InnermostKind,
222
                               const RegionCodeGenTy &CodeGen) override;
223
224
  /// Emits inlined function for the specified OpenMP teams
225
  //  directive.
226
  /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
227
  /// kmp_int32 BoundID, struct context_vars*).
228
  /// \param D OpenMP directive.
229
  /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
230
  /// \param InnermostKind Kind of innermost directive (for simple directives it
231
  /// is a directive itself, for combined - its innermost directive).
232
  /// \param CodeGen Code generation sequence for the \a D directive.
233
  llvm::Function *
234
  emitTeamsOutlinedFunction(const OMPExecutableDirective &D,
235
                            const VarDecl *ThreadIDVar,
236
                            OpenMPDirectiveKind InnermostKind,
237
                            const RegionCodeGenTy &CodeGen) override;
238
239
  /// Emits code for teams call of the \a OutlinedFn with
240
  /// variables captured in a record which address is stored in \a
241
  /// CapturedStruct.
242
  /// \param OutlinedFn Outlined function to be run by team masters. Type of
243
  /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
244
  /// \param CapturedVars A pointer to the record with the references to
245
  /// variables used in \a OutlinedFn function.
246
  ///
247
  void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
248
                     SourceLocation Loc, llvm::Function *OutlinedFn,
249
                     ArrayRef<llvm::Value *> CapturedVars) override;
250
251
  /// Emits code for parallel or serial call of the \a OutlinedFn with
252
  /// variables captured in a record which address is stored in \a
253
  /// CapturedStruct.
254
  /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
255
  /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
256
  /// \param CapturedVars A pointer to the record with the references to
257
  /// variables used in \a OutlinedFn function.
258
  /// \param IfCond Condition in the associated 'if' clause, if it was
259
  /// specified, nullptr otherwise.
260
  void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
261
                        llvm::Function *OutlinedFn,
262
                        ArrayRef<llvm::Value *> CapturedVars,
263
                        const Expr *IfCond) override;
264
265
  /// Emit an implicit/explicit barrier for OpenMP threads.
266
  /// \param Kind Directive for which this implicit barrier call must be
267
  /// generated. Must be OMPD_barrier for explicit barrier generation.
268
  /// \param EmitChecks true if need to emit checks for cancellation barriers.
269
  /// \param ForceSimpleCall true simple barrier call must be emitted, false if
270
  /// runtime class decides which one to emit (simple or with cancellation
271
  /// checks).
272
  ///
273
  void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
274
                       OpenMPDirectiveKind Kind, bool EmitChecks = true,
275
                       bool ForceSimpleCall = false) override;
276
277
  /// Emits a critical region.
278
  /// \param CriticalName Name of the critical region.
279
  /// \param CriticalOpGen Generator for the statement associated with the given
280
  /// critical region.
281
  /// \param Hint Value of the 'hint' clause (optional).
282
  void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
283
                          const RegionCodeGenTy &CriticalOpGen,
284
                          SourceLocation Loc,
285
                          const Expr *Hint = nullptr) override;
286
287
  /// Emit a code for reduction clause.
288
  ///
289
  /// \param Privates List of private copies for original reduction arguments.
290
  /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
291
  /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
292
  /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
293
  /// or 'operator binop(LHS, RHS)'.
294
  /// \param Options List of options for reduction codegen:
295
  ///     WithNowait true if parent directive has also nowait clause, false
296
  ///     otherwise.
297
  ///     SimpleReduction Emit reduction operation only. Used for omp simd
298
  ///     directive on the host.
299
  ///     ReductionKind The kind of reduction to perform.
300
  virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
301
                             ArrayRef<const Expr *> Privates,
302
                             ArrayRef<const Expr *> LHSExprs,
303
                             ArrayRef<const Expr *> RHSExprs,
304
                             ArrayRef<const Expr *> ReductionOps,
305
                             ReductionOptionsTy Options) override;
306
307
  /// Returns specified OpenMP runtime function for the current OpenMP
308
  /// implementation.  Specialized for the NVPTX device.
309
  /// \param Function OpenMP runtime function.
310
  /// \return Specified function.
311
  llvm::FunctionCallee createNVPTXRuntimeFunction(unsigned Function);
312
313
  /// Translates the native parameter of outlined function if this is required
314
  /// for target.
315
  /// \param FD Field decl from captured record for the parameter.
316
  /// \param NativeParam Parameter itself.
317
  const VarDecl *translateParameter(const FieldDecl *FD,
318
                                    const VarDecl *NativeParam) const override;
319
320
  /// Gets the address of the native argument basing on the address of the
321
  /// target-specific parameter.
322
  /// \param NativeParam Parameter itself.
323
  /// \param TargetParam Corresponding target-specific parameter.
324
  Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
325
                              const VarDecl *TargetParam) const override;
326
327
  /// Emits call of the outlined function with the provided arguments,
328
  /// translating these arguments to correct target-specific arguments.
329
  void emitOutlinedFunctionCall(
330
      CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
331
      ArrayRef<llvm::Value *> Args = llvm::None) const override;
332
333
  /// Emits OpenMP-specific function prolog.
334
  /// Required for device constructs.
335
  void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) override;
336
337
  /// Gets the OpenMP-specific address of the local variable.
338
  Address getAddressOfLocalVariable(CodeGenFunction &CGF,
339
                                    const VarDecl *VD) override;
340
341
  /// Target codegen is specialized based on two data-sharing modes: CUDA, in
342
  /// which the local variables are actually global threadlocal, and Generic, in
343
  /// which the local variables are placed in global memory if they may escape
344
  /// their declaration context.
345
  enum DataSharingMode {
346
    /// CUDA data sharing mode.
347
    CUDA,
348
    /// Generic data-sharing mode.
349
    Generic,
350
  };
351
352
  /// Cleans up references to the objects in finished function.
353
  ///
354
  void functionFinished(CodeGenFunction &CGF) override;
355
356
  /// Choose a default value for the dist_schedule clause.
357
  void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
358
      const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
359
      llvm::Value *&Chunk) const override;
360
361
  /// Choose a default value for the schedule clause.
362
  void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
363
      const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
364
      const Expr *&ChunkExpr) const override;
365
366
  /// Adjust some parameters for the target-based directives, like addresses of
367
  /// the variables captured by reference in lambdas.
368
  void adjustTargetSpecificDataForLambdas(
369
      CodeGenFunction &CGF, const OMPExecutableDirective &D) const override;
370
371
  /// Perform check on requires decl to ensure that target architecture
372
  /// supports unified addressing
373
  void processRequiresDirective(const OMPRequiresDecl *D) override;
374
375
  /// Checks if the variable has associated OMPAllocateDeclAttr attribute with
376
  /// the predefined allocator and translates it into the corresponding address
377
  /// space.
378
  bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS) override;
379
380
private:
381
  /// Track the execution mode when codegening directives within a target
382
  /// region. The appropriate mode (SPMD/NON-SPMD) is set on entry to the
383
  /// target region and used by containing directives such as 'parallel'
384
  /// to emit optimized code.
385
  ExecutionMode CurrentExecutionMode = EM_Unknown;
386
387
  /// Check if the full runtime is required (default - yes).
388
  bool RequiresFullRuntime = true;
389
390
  /// true if we're emitting the code for the target region and next parallel
391
  /// region is L0 for sure.
392
  bool IsInTargetMasterThreadRegion = false;
393
  /// true if currently emitting code for target/teams/distribute region, false
394
  /// - otherwise.
395
  bool IsInTTDRegion = false;
396
  /// true if we're definitely in the parallel region.
397
  bool IsInParallelRegion = false;
398
399
  /// Map between an outlined function and its wrapper.
400
  llvm::DenseMap<llvm::Function *, llvm::Function *> WrapperFunctionsMap;
401
402
  /// Emit function which wraps the outline parallel region
403
  /// and controls the parameters which are passed to this function.
404
  /// The wrapper ensures that the outlined function is called
405
  /// with the correct arguments when data is shared.
406
  llvm::Function *createParallelDataSharingWrapper(
407
      llvm::Function *OutlinedParallelFn, const OMPExecutableDirective &D);
408
409
  /// The data for the single globalized variable.
410
  struct MappedVarData {
411
    /// Corresponding field in the global record.
412
    llvm::Value *GlobalizedVal = nullptr;
413
    /// Corresponding address.
414
    Address PrivateAddr = Address::invalid();
415
  };
416
  /// The map of local variables to their addresses in the global memory.
417
  using DeclToAddrMapTy = llvm::MapVector<const Decl *, MappedVarData>;
418
  /// Set of the parameters passed by value escaping OpenMP context.
419
  using EscapedParamsTy = llvm::SmallPtrSet<const Decl *, 4>;
420
  struct FunctionData {
421
    DeclToAddrMapTy LocalVarData;
422
    llvm::Optional<DeclToAddrMapTy> SecondaryLocalVarData = llvm::None;
423
    EscapedParamsTy EscapedParameters;
424
    llvm::SmallVector<const ValueDecl*, 4> EscapedVariableLengthDecls;
425
    llvm::SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4>
426
        EscapedVariableLengthDeclsAddrs;
427
    llvm::Value *IsInSPMDModeFlag = nullptr;
428
    std::unique_ptr<CodeGenFunction::OMPMapVars> MappedParams;
429
  };
430
  /// Maps the function to the list of the globalized variables with their
431
  /// addresses.
432
  llvm::SmallDenseMap<llvm::Function *, FunctionData> FunctionGlobalizedDecls;
433
  llvm::GlobalVariable *KernelTeamsReductionPtr = nullptr;
434
  /// List of the records with the list of fields for the reductions across the
435
  /// teams. Used to build the intermediate buffer for the fast teams
436
  /// reductions.
437
  /// All the records are gathered into a union `union.type` is created.
438
  llvm::SmallVector<const RecordDecl *, 4> TeamsReductions;
439
  /// Shared pointer for the global memory in the global memory buffer used for
440
  /// the given kernel.
441
  llvm::GlobalVariable *KernelStaticGlobalized = nullptr;
442
  /// Pair of the Non-SPMD team and all reductions variables in this team
443
  /// region.
444
  std::pair<const Decl *, llvm::SmallVector<const ValueDecl *, 4>>
445
      TeamAndReductions;
446
};
447
448
} // CodeGen namespace.
449
} // clang namespace.
450
451
#endif // LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H