Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- AMDGPULowerKernelAttributes.cpp ------------------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file This pass does attempts to make use of reqd_work_group_size metadata
10
/// to eliminate loads from the dispatch packet and to constant fold OpenCL
11
/// get_local_size-like functions.
12
//
13
//===----------------------------------------------------------------------===//
14
15
#include "AMDGPU.h"
16
#include "AMDGPUTargetMachine.h"
17
#include "llvm/Analysis/ValueTracking.h"
18
#include "llvm/CodeGen/Passes.h"
19
#include "llvm/CodeGen/TargetPassConfig.h"
20
#include "llvm/IR/Constants.h"
21
#include "llvm/IR/Function.h"
22
#include "llvm/IR/Instructions.h"
23
#include "llvm/IR/PatternMatch.h"
24
#include "llvm/Pass.h"
25
26
#define DEBUG_TYPE "amdgpu-lower-kernel-attributes"
27
28
using namespace llvm;
29
30
namespace {
31
32
// Field offsets in hsa_kernel_dispatch_packet_t.
33
enum DispatchPackedOffsets {
34
  WORKGROUP_SIZE_X = 4,
35
  WORKGROUP_SIZE_Y = 6,
36
  WORKGROUP_SIZE_Z = 8,
37
38
  GRID_SIZE_X = 12,
39
  GRID_SIZE_Y = 16,
40
  GRID_SIZE_Z = 20
41
};
42
43
class AMDGPULowerKernelAttributes : public ModulePass {
44
  Module *Mod = nullptr;
45
46
public:
47
  static char ID;
48
49
98
  AMDGPULowerKernelAttributes() : ModulePass(ID) {}
50
51
  bool processUse(CallInst *CI);
52
53
  bool doInitialization(Module &M) override;
54
  bool runOnModule(Module &M) override;
55
56
1
  StringRef getPassName() const override {
57
1
    return "AMDGPU Kernel Attributes";
58
1
  }
59
60
98
  void getAnalysisUsage(AnalysisUsage &AU) const override {
61
98
    AU.setPreservesAll();
62
98
 }
63
};
64
65
} // end anonymous namespace
66
67
98
bool AMDGPULowerKernelAttributes::doInitialization(Module &M) {
68
98
  Mod = &M;
69
98
  return false;
70
98
}
71
72
26
bool AMDGPULowerKernelAttributes::processUse(CallInst *CI) {
73
26
  Function *F = CI->getParent()->getParent();
74
26
75
26
  auto MD = F->getMetadata("reqd_work_group_size");
76
26
  const bool HasReqdWorkGroupSize = MD && 
MD->getNumOperands() == 322
;
77
26
78
26
  const bool HasUniformWorkGroupSize =
79
26
    F->getFnAttribute("uniform-work-group-size").getValueAsString() == "true";
80
26
81
26
  if (!HasReqdWorkGroupSize && 
!HasUniformWorkGroupSize5
)
82
3
    return false;
83
23
84
23
  Value *WorkGroupSizeX = nullptr;
85
23
  Value *WorkGroupSizeY = nullptr;
86
23
  Value *WorkGroupSizeZ = nullptr;
87
23
88
23
  Value *GridSizeX = nullptr;
89
23
  Value *GridSizeY = nullptr;
90
23
  Value *GridSizeZ = nullptr;
91
23
92
23
  const DataLayout &DL = Mod->getDataLayout();
93
23
94
23
  // We expect to see several GEP users, casted to the appropriate type and
95
23
  // loaded.
96
42
  for (User *U : CI->users()) {
97
42
    if (!U->hasOneUse())
98
0
      continue;
99
42
100
42
    int64_t Offset = 0;
101
42
    if (GetPointerBaseWithConstantOffset(U, Offset, DL) != CI)
102
0
      continue;
103
42
104
42
    auto *BCI = dyn_cast<BitCastInst>(*U->user_begin());
105
42
    if (!BCI || 
!BCI->hasOneUse()41
)
106
1
      continue;
107
41
108
41
    auto *Load = dyn_cast<LoadInst>(*BCI->user_begin());
109
41
    if (!Load || !Load->isSimple())
110
1
      continue;
111
40
112
40
    unsigned LoadSize = DL.getTypeStoreSize(Load->getType());
113
40
114
40
    // TODO: Handle merged loads.
115
40
    switch (Offset) {
116
40
    case WORKGROUP_SIZE_X:
117
16
      if (LoadSize == 2)
118
15
        WorkGroupSizeX = Load;
119
16
      break;
120
40
    case WORKGROUP_SIZE_Y:
121
5
      if (LoadSize == 2)
122
5
        WorkGroupSizeY = Load;
123
5
      break;
124
40
    case WORKGROUP_SIZE_Z:
125
4
      if (LoadSize == 2)
126
4
        WorkGroupSizeZ = Load;
127
4
      break;
128
40
    case GRID_SIZE_X:
129
8
      if (LoadSize == 4)
130
7
        GridSizeX = Load;
131
8
      break;
132
40
    case GRID_SIZE_Y:
133
4
      if (LoadSize == 4)
134
4
        GridSizeY = Load;
135
4
      break;
136
40
    case GRID_SIZE_Z:
137
3
      if (LoadSize == 4)
138
3
        GridSizeZ = Load;
139
3
      break;
140
40
    default:
141
0
      break;
142
40
    }
143
40
  }
144
23
145
23
  // Pattern match the code used to handle partial workgroup dispatches in the
146
23
  // library implementation of get_local_size, so the entire function can be
147
23
  // constant folded with a known group size.
148
23
  //
149
23
  // uint r = grid_size - group_id * group_size;
150
23
  // get_local_size = (r < group_size) ? r : group_size;
151
23
  //
152
23
  // If we have uniform-work-group-size (which is the default in OpenCL 1.2),
153
23
  // the grid_size is required to be a multiple of group_size). In this case:
154
23
  //
155
23
  // grid_size - (group_id * group_size) < group_size
156
23
  // ->
157
23
  // grid_size < group_size + (group_id * group_size)
158
23
  //
159
23
  // (grid_size / group_size) < 1 + group_id
160
23
  //
161
23
  // grid_size / group_size is at least 1, so we can conclude the select
162
23
  // condition is false (except for group_id == 0, where the select result is
163
23
  // the same).
164
23
165
23
  bool MadeChange = false;
166
23
  Value *WorkGroupSizes[3] = { WorkGroupSizeX, WorkGroupSizeY, WorkGroupSizeZ };
167
23
  Value *GridSizes[3] = { GridSizeX, GridSizeY, GridSizeZ };
168
23
169
89
  for (int I = 0; HasUniformWorkGroupSize && 
I < 388
;
++I66
) {
170
66
    Value *GroupSize = WorkGroupSizes[I];
171
66
    Value *GridSize = GridSizes[I];
172
66
    if (!GroupSize || 
!GridSize21
)
173
56
      continue;
174
10
175
10
    for (User *U : GroupSize->users()) {
176
10
      auto *ZextGroupSize = dyn_cast<ZExtInst>(U);
177
10
      if (!ZextGroupSize)
178
0
        continue;
179
10
180
30
      
for (User *ZextUser : ZextGroupSize->users())10
{
181
30
        auto *SI = dyn_cast<SelectInst>(ZextUser);
182
30
        if (!SI)
183
20
          continue;
184
10
185
10
        using namespace llvm::PatternMatch;
186
10
        auto GroupIDIntrin = I == 0 ?
187
6
          m_Intrinsic<Intrinsic::amdgcn_workgroup_id_x>() :
188
10
            
(I == 1 4
?
m_Intrinsic<Intrinsic::amdgcn_workgroup_id_y>()2
:
189
4
                      
m_Intrinsic<Intrinsic::amdgcn_workgroup_id_z>()2
);
190
10
191
10
        auto SubExpr = m_Sub(m_Specific(GridSize),
192
10
                             m_Mul(GroupIDIntrin, m_Specific(ZextGroupSize)));
193
10
194
10
        ICmpInst::Predicate Pred;
195
10
        if (match(SI,
196
10
                  m_Select(m_ICmp(Pred, SubExpr, m_Specific(ZextGroupSize)),
197
10
                           SubExpr,
198
10
                           m_Specific(ZextGroupSize))) &&
199
10
            
Pred == ICmpInst::ICMP_ULT8
) {
200
7
          if (HasReqdWorkGroupSize) {
201
6
            ConstantInt *KnownSize
202
6
              = mdconst::extract<ConstantInt>(MD->getOperand(I));
203
6
            SI->replaceAllUsesWith(ConstantExpr::getIntegerCast(KnownSize,
204
6
                                                                SI->getType(),
205
6
                                                                false));
206
6
          } else {
207
1
            SI->replaceAllUsesWith(ZextGroupSize);
208
1
          }
209
7
210
7
          MadeChange = true;
211
7
        }
212
10
      }
213
10
    }
214
10
  }
215
23
216
23
  if (!HasReqdWorkGroupSize)
217
2
    return MadeChange;
218
21
219
21
  // Eliminate any other loads we can from the dispatch packet.
220
84
  
for (int I = 0; 21
I < 3;
++I63
) {
221
63
    Value *GroupSize = WorkGroupSizes[I];
222
63
    if (!GroupSize)
223
41
      continue;
224
22
225
22
    ConstantInt *KnownSize = mdconst::extract<ConstantInt>(MD->getOperand(I));
226
22
    GroupSize->replaceAllUsesWith(
227
22
      ConstantExpr::getIntegerCast(KnownSize,
228
22
                                   GroupSize->getType(),
229
22
                                   false));
230
22
    MadeChange = true;
231
22
  }
232
21
233
21
  return MadeChange;
234
21
}
235
236
// TODO: Move makeLIDRangeMetadata usage into here. Seem to not get
237
// TargetPassConfig for subtarget.
238
98
bool AMDGPULowerKernelAttributes::runOnModule(Module &M) {
239
98
  StringRef DispatchPtrName
240
98
    = Intrinsic::getName(Intrinsic::amdgcn_dispatch_ptr);
241
98
242
98
  Function *DispatchPtr = Mod->getFunction(DispatchPtrName);
243
98
  if (!DispatchPtr) // Dispatch ptr not used.
244
96
    return false;
245
2
246
2
  bool MadeChange = false;
247
2
248
2
  SmallPtrSet<Instruction *, 4> HandledUses;
249
26
  for (auto *U : DispatchPtr->users()) {
250
26
    CallInst *CI = cast<CallInst>(U);
251
26
    if (HandledUses.insert(CI).second) {
252
26
      if (processUse(CI))
253
19
        MadeChange = true;
254
26
    }
255
26
  }
256
2
257
2
  return MadeChange;
258
2
}
259
260
101k
INITIALIZE_PASS_BEGIN(AMDGPULowerKernelAttributes, DEBUG_TYPE,
261
101k
                      "AMDGPU IR optimizations", false, false)
262
101k
INITIALIZE_PASS_END(AMDGPULowerKernelAttributes, DEBUG_TYPE, "AMDGPU IR optimizations",
263
                    false, false)
264
265
char AMDGPULowerKernelAttributes::ID = 0;
266
267
97
ModulePass *llvm::createAMDGPULowerKernelAttributesPass() {
268
97
  return new AMDGPULowerKernelAttributes();
269
97
}