Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- AMDGPUCodeGenPrepare.cpp ------------------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file
10
/// This pass does misc. AMDGPU optimizations on IR before instruction
11
/// selection.
12
//
13
//===----------------------------------------------------------------------===//
14
15
#include "AMDGPU.h"
16
#include "AMDGPUSubtarget.h"
17
#include "AMDGPUTargetMachine.h"
18
#include "llvm/ADT/StringRef.h"
19
#include "llvm/Analysis/AssumptionCache.h"
20
#include "llvm/Analysis/LegacyDivergenceAnalysis.h"
21
#include "llvm/Analysis/Loads.h"
22
#include "llvm/Analysis/ValueTracking.h"
23
#include "llvm/CodeGen/Passes.h"
24
#include "llvm/CodeGen/TargetPassConfig.h"
25
#include "llvm/IR/Attributes.h"
26
#include "llvm/IR/BasicBlock.h"
27
#include "llvm/IR/Constants.h"
28
#include "llvm/IR/DerivedTypes.h"
29
#include "llvm/IR/Function.h"
30
#include "llvm/IR/IRBuilder.h"
31
#include "llvm/IR/InstVisitor.h"
32
#include "llvm/IR/InstrTypes.h"
33
#include "llvm/IR/Instruction.h"
34
#include "llvm/IR/Instructions.h"
35
#include "llvm/IR/IntrinsicInst.h"
36
#include "llvm/IR/Intrinsics.h"
37
#include "llvm/IR/LLVMContext.h"
38
#include "llvm/IR/Operator.h"
39
#include "llvm/IR/Type.h"
40
#include "llvm/IR/Value.h"
41
#include "llvm/Pass.h"
42
#include "llvm/Support/Casting.h"
43
#include <cassert>
44
#include <iterator>
45
46
#define DEBUG_TYPE "amdgpu-codegenprepare"
47
48
using namespace llvm;
49
50
namespace {
51
52
static cl::opt<bool> WidenLoads(
53
  "amdgpu-codegenprepare-widen-constant-loads",
54
  cl::desc("Widen sub-dword constant address space loads in AMDGPUCodeGenPrepare"),
55
  cl::ReallyHidden,
56
  cl::init(true));
57
58
class AMDGPUCodeGenPrepare : public FunctionPass,
59
                             public InstVisitor<AMDGPUCodeGenPrepare, bool> {
60
  const GCNSubtarget *ST = nullptr;
61
  AssumptionCache *AC = nullptr;
62
  LegacyDivergenceAnalysis *DA = nullptr;
63
  Module *Mod = nullptr;
64
  const DataLayout *DL = nullptr;
65
  bool HasUnsafeFPMath = false;
66
67
  /// Copies exact/nsw/nuw flags (if any) from binary operation \p I to
68
  /// binary operation \p V.
69
  ///
70
  /// \returns Binary operation \p V.
71
  /// \returns \p T's base element bit width.
72
  unsigned getBaseElementBitWidth(const Type *T) const;
73
74
  /// \returns Equivalent 32 bit integer type for given type \p T. For example,
75
  /// if \p T is i7, then i32 is returned; if \p T is <3 x i12>, then <3 x i32>
76
  /// is returned.
77
  Type *getI32Ty(IRBuilder<> &B, const Type *T) const;
78
79
  /// \returns True if binary operation \p I is a signed binary operation, false
80
  /// otherwise.
81
  bool isSigned(const BinaryOperator &I) const;
82
83
  /// \returns True if the condition of 'select' operation \p I comes from a
84
  /// signed 'icmp' operation, false otherwise.
85
  bool isSigned(const SelectInst &I) const;
86
87
  /// \returns True if type \p T needs to be promoted to 32 bit integer type,
88
  /// false otherwise.
89
  bool needsPromotionToI32(const Type *T) const;
90
91
  /// Promotes uniform binary operation \p I to equivalent 32 bit binary
92
  /// operation.
93
  ///
94
  /// \details \p I's base element bit width must be greater than 1 and less
95
  /// than or equal 16. Promotion is done by sign or zero extending operands to
96
  /// 32 bits, replacing \p I with equivalent 32 bit binary operation, and
97
  /// truncating the result of 32 bit binary operation back to \p I's original
98
  /// type. Division operation is not promoted.
99
  ///
100
  /// \returns True if \p I is promoted to equivalent 32 bit binary operation,
101
  /// false otherwise.
102
  bool promoteUniformOpToI32(BinaryOperator &I) const;
103
104
  /// Promotes uniform 'icmp' operation \p I to 32 bit 'icmp' operation.
105
  ///
106
  /// \details \p I's base element bit width must be greater than 1 and less
107
  /// than or equal 16. Promotion is done by sign or zero extending operands to
108
  /// 32 bits, and replacing \p I with 32 bit 'icmp' operation.
109
  ///
110
  /// \returns True.
111
  bool promoteUniformOpToI32(ICmpInst &I) const;
112
113
  /// Promotes uniform 'select' operation \p I to 32 bit 'select'
114
  /// operation.
115
  ///
116
  /// \details \p I's base element bit width must be greater than 1 and less
117
  /// than or equal 16. Promotion is done by sign or zero extending operands to
118
  /// 32 bits, replacing \p I with 32 bit 'select' operation, and truncating the
119
  /// result of 32 bit 'select' operation back to \p I's original type.
120
  ///
121
  /// \returns True.
122
  bool promoteUniformOpToI32(SelectInst &I) const;
123
124
  /// Promotes uniform 'bitreverse' intrinsic \p I to 32 bit 'bitreverse'
125
  /// intrinsic.
126
  ///
127
  /// \details \p I's base element bit width must be greater than 1 and less
128
  /// than or equal 16. Promotion is done by zero extending the operand to 32
129
  /// bits, replacing \p I with 32 bit 'bitreverse' intrinsic, shifting the
130
  /// result of 32 bit 'bitreverse' intrinsic to the right with zero fill (the
131
  /// shift amount is 32 minus \p I's base element bit width), and truncating
132
  /// the result of the shift operation back to \p I's original type.
133
  ///
134
  /// \returns True.
135
  bool promoteUniformBitreverseToI32(IntrinsicInst &I) const;
136
137
138
  unsigned numBitsUnsigned(Value *Op, unsigned ScalarSize) const;
139
  unsigned numBitsSigned(Value *Op, unsigned ScalarSize) const;
140
  bool isI24(Value *V, unsigned ScalarSize) const;
141
  bool isU24(Value *V, unsigned ScalarSize) const;
142
143
  /// Replace mul instructions with llvm.amdgcn.mul.u24 or llvm.amdgcn.mul.s24.
144
  /// SelectionDAG has an issue where an and asserting the bits are known
145
  bool replaceMulWithMul24(BinaryOperator &I) const;
146
147
  /// Expands 24 bit div or rem.
148
  Value* expandDivRem24(IRBuilder<> &Builder, BinaryOperator &I,
149
                        Value *Num, Value *Den,
150
                        bool IsDiv, bool IsSigned) const;
151
152
  /// Expands 32 bit div or rem.
153
  Value* expandDivRem32(IRBuilder<> &Builder, BinaryOperator &I,
154
                        Value *Num, Value *Den) const;
155
156
  /// Widen a scalar load.
157
  ///
158
  /// \details \p Widen scalar load for uniform, small type loads from constant
159
  //  memory / to a full 32-bits and then truncate the input to allow a scalar
160
  //  load instead of a vector load.
161
  //
162
  /// \returns True.
163
164
  bool canWidenScalarExtLoad(LoadInst &I) const;
165
166
public:
167
  static char ID;
168
169
2.45k
  AMDGPUCodeGenPrepare() : FunctionPass(ID) {}
170
171
  bool visitFDiv(BinaryOperator &I);
172
173
96.7k
  bool visitInstruction(Instruction &I) { return false; }
174
  bool visitBinaryOperator(BinaryOperator &I);
175
  bool visitLoadInst(LoadInst &I);
176
  bool visitICmpInst(ICmpInst &I);
177
  bool visitSelectInst(SelectInst &I);
178
179
  bool visitIntrinsicInst(IntrinsicInst &I);
180
  bool visitBitreverseIntrinsicInst(IntrinsicInst &I);
181
182
  bool doInitialization(Module &M) override;
183
  bool runOnFunction(Function &F) override;
184
185
25.5k
  StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
186
187
2.43k
  void getAnalysisUsage(AnalysisUsage &AU) const override {
188
2.43k
    AU.addRequired<AssumptionCacheTracker>();
189
2.43k
    AU.addRequired<LegacyDivergenceAnalysis>();
190
2.43k
    AU.setPreservesAll();
191
2.43k
 }
192
};
193
194
} // end anonymous namespace
195
196
8
unsigned AMDGPUCodeGenPrepare::getBaseElementBitWidth(const Type *T) const {
197
8
  assert(needsPromotionToI32(T) && "T does not need promotion to i32");
198
8
199
8
  if (T->isIntegerTy())
200
6
    return T->getIntegerBitWidth();
201
2
  return cast<VectorType>(T)->getElementType()->getIntegerBitWidth();
202
2
}
203
204
1.56k
Type *AMDGPUCodeGenPrepare::getI32Ty(IRBuilder<> &B, const Type *T) const {
205
1.56k
  assert(needsPromotionToI32(T) && "T does not need promotion to i32");
206
1.56k
207
1.56k
  if (T->isIntegerTy())
208
1.35k
    return B.getInt32Ty();
209
211
  return VectorType::get(B.getInt32Ty(), cast<VectorType>(T)->getNumElements());
210
211
}
211
212
1.30k
bool AMDGPUCodeGenPrepare::isSigned(const BinaryOperator &I) const {
213
1.30k
  return I.getOpcode() == Instruction::AShr ||
214
1.30k
      
I.getOpcode() == Instruction::SDiv1.28k
||
I.getOpcode() == Instruction::SRem1.28k
;
215
1.30k
}
216
217
130
bool AMDGPUCodeGenPrepare::isSigned(const SelectInst &I) const {
218
130
  return isa<ICmpInst>(I.getOperand(0)) ?
219
126
      cast<ICmpInst>(I.getOperand(0))->isSigned() : 
false4
;
220
130
}
221
222
15.0k
bool AMDGPUCodeGenPrepare::needsPromotionToI32(const Type *T) const {
223
15.0k
  const IntegerType *IntTy = dyn_cast<IntegerType>(T);
224
15.0k
  if (IntTy && 
IntTy->getBitWidth() > 19.74k
&&
IntTy->getBitWidth() <= 169.58k
)
225
2.14k
    return true;
226
12.8k
227
12.8k
  if (const VectorType *VT = dyn_cast<VectorType>(T)) {
228
1.41k
    // TODO: The set of packed operations is more limited, so may want to
229
1.41k
    // promote some anyway.
230
1.41k
    if (ST->hasVOP3PInsts())
231
622
      return false;
232
789
233
789
    return needsPromotionToI32(VT->getElementType());
234
789
  }
235
11.4k
236
11.4k
  return false;
237
11.4k
}
238
239
// Return true if the op promoted to i32 should have nsw set.
240
1.30k
static bool promotedOpIsNSW(const Instruction &I) {
241
1.30k
  switch (I.getOpcode()) {
242
1.30k
  case Instruction::Shl:
243
776
  case Instruction::Add:
244
776
  case Instruction::Sub:
245
776
    return true;
246
776
  case Instruction::Mul:
247
460
    return I.hasNoUnsignedWrap();
248
776
  default:
249
68
    return false;
250
1.30k
  }
251
1.30k
}
252
253
// Return true if the op promoted to i32 should have nuw set.
254
1.30k
static bool promotedOpIsNUW(const Instruction &I) {
255
1.30k
  switch (I.getOpcode()) {
256
1.30k
  case Instruction::Shl:
257
1.21k
  case Instruction::Add:
258
1.21k
  case Instruction::Mul:
259
1.21k
    return true;
260
1.21k
  case Instruction::Sub:
261
26
    return I.hasNoUnsignedWrap();
262
1.21k
  default:
263
68
    return false;
264
1.30k
  }
265
1.30k
}
266
267
1.59k
bool AMDGPUCodeGenPrepare::canWidenScalarExtLoad(LoadInst &I) const {
268
1.59k
  Type *Ty = I.getType();
269
1.59k
  const DataLayout &DL = Mod->getDataLayout();
270
1.59k
  int TySize = DL.getTypeSizeInBits(Ty);
271
1.59k
  unsigned Align = I.getAlignment() ?
272
1.33k
                   
I.getAlignment()253
: DL.getABITypeAlignment(Ty);
273
1.59k
274
1.59k
  return I.isSimple() && 
TySize < 321.42k
&&
Align >= 4229
&&
DA->isUniform(&I)69
;
275
1.59k
}
276
277
1.32k
bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(BinaryOperator &I) const {
278
1.32k
  assert(needsPromotionToI32(I.getType()) &&
279
1.32k
         "I does not need promotion to i32");
280
1.32k
281
1.32k
  if (I.getOpcode() == Instruction::SDiv ||
282
1.32k
      
I.getOpcode() == Instruction::UDiv1.31k
||
283
1.32k
      
I.getOpcode() == Instruction::SRem1.31k
||
284
1.32k
      
I.getOpcode() == Instruction::URem1.31k
)
285
14
    return false;
286
1.30k
287
1.30k
  IRBuilder<> Builder(&I);
288
1.30k
  Builder.SetCurrentDebugLocation(I.getDebugLoc());
289
1.30k
290
1.30k
  Type *I32Ty = getI32Ty(Builder, I.getType());
291
1.30k
  Value *ExtOp0 = nullptr;
292
1.30k
  Value *ExtOp1 = nullptr;
293
1.30k
  Value *ExtRes = nullptr;
294
1.30k
  Value *TruncRes = nullptr;
295
1.30k
296
1.30k
  if (isSigned(I)) {
297
26
    ExtOp0 = Builder.CreateSExt(I.getOperand(0), I32Ty);
298
26
    ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty);
299
1.28k
  } else {
300
1.28k
    ExtOp0 = Builder.CreateZExt(I.getOperand(0), I32Ty);
301
1.28k
    ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty);
302
1.28k
  }
303
1.30k
304
1.30k
  ExtRes = Builder.CreateBinOp(I.getOpcode(), ExtOp0, ExtOp1);
305
1.30k
  if (Instruction *Inst = dyn_cast<Instruction>(ExtRes)) {
306
1.30k
    if (promotedOpIsNSW(cast<Instruction>(I)))
307
1.13k
      Inst->setHasNoSignedWrap();
308
1.30k
309
1.30k
    if (promotedOpIsNUW(cast<Instruction>(I)))
310
1.21k
      Inst->setHasNoUnsignedWrap();
311
1.30k
312
1.30k
    if (const auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
313
36
      Inst->setIsExact(ExactOp->isExact());
314
1.30k
  }
315
1.30k
316
1.30k
  TruncRes = Builder.CreateTrunc(ExtRes, I.getType());
317
1.30k
318
1.30k
  I.replaceAllUsesWith(TruncRes);
319
1.30k
  I.eraseFromParent();
320
1.30k
321
1.30k
  return true;
322
1.30k
}
323
324
116
bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(ICmpInst &I) const {
325
116
  assert(needsPromotionToI32(I.getOperand(0)->getType()) &&
326
116
         "I does not need promotion to i32");
327
116
328
116
  IRBuilder<> Builder(&I);
329
116
  Builder.SetCurrentDebugLocation(I.getDebugLoc());
330
116
331
116
  Type *I32Ty = getI32Ty(Builder, I.getOperand(0)->getType());
332
116
  Value *ExtOp0 = nullptr;
333
116
  Value *ExtOp1 = nullptr;
334
116
  Value *NewICmp  = nullptr;
335
116
336
116
  if (I.isSigned()) {
337
55
    ExtOp0 = Builder.CreateSExt(I.getOperand(0), I32Ty);
338
55
    ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty);
339
61
  } else {
340
61
    ExtOp0 = Builder.CreateZExt(I.getOperand(0), I32Ty);
341
61
    ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty);
342
61
  }
343
116
  NewICmp = Builder.CreateICmp(I.getPredicate(), ExtOp0, ExtOp1);
344
116
345
116
  I.replaceAllUsesWith(NewICmp);
346
116
  I.eraseFromParent();
347
116
348
116
  return true;
349
116
}
350
351
130
bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(SelectInst &I) const {
352
130
  assert(needsPromotionToI32(I.getType()) &&
353
130
         "I does not need promotion to i32");
354
130
355
130
  IRBuilder<> Builder(&I);
356
130
  Builder.SetCurrentDebugLocation(I.getDebugLoc());
357
130
358
130
  Type *I32Ty = getI32Ty(Builder, I.getType());
359
130
  Value *ExtOp1 = nullptr;
360
130
  Value *ExtOp2 = nullptr;
361
130
  Value *ExtRes = nullptr;
362
130
  Value *TruncRes = nullptr;
363
130
364
130
  if (isSigned(I)) {
365
59
    ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty);
366
59
    ExtOp2 = Builder.CreateSExt(I.getOperand(2), I32Ty);
367
71
  } else {
368
71
    ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty);
369
71
    ExtOp2 = Builder.CreateZExt(I.getOperand(2), I32Ty);
370
71
  }
371
130
  ExtRes = Builder.CreateSelect(I.getOperand(0), ExtOp1, ExtOp2);
372
130
  TruncRes = Builder.CreateTrunc(ExtRes, I.getType());
373
130
374
130
  I.replaceAllUsesWith(TruncRes);
375
130
  I.eraseFromParent();
376
130
377
130
  return true;
378
130
}
379
380
bool AMDGPUCodeGenPrepare::promoteUniformBitreverseToI32(
381
8
    IntrinsicInst &I) const {
382
8
  assert(I.getIntrinsicID() == Intrinsic::bitreverse &&
383
8
         "I must be bitreverse intrinsic");
384
8
  assert(needsPromotionToI32(I.getType()) &&
385
8
         "I does not need promotion to i32");
386
8
387
8
  IRBuilder<> Builder(&I);
388
8
  Builder.SetCurrentDebugLocation(I.getDebugLoc());
389
8
390
8
  Type *I32Ty = getI32Ty(Builder, I.getType());
391
8
  Function *I32 =
392
8
      Intrinsic::getDeclaration(Mod, Intrinsic::bitreverse, { I32Ty });
393
8
  Value *ExtOp = Builder.CreateZExt(I.getOperand(0), I32Ty);
394
8
  Value *ExtRes = Builder.CreateCall(I32, { ExtOp });
395
8
  Value *LShrOp =
396
8
      Builder.CreateLShr(ExtRes, 32 - getBaseElementBitWidth(I.getType()));
397
8
  Value *TruncRes =
398
8
      Builder.CreateTrunc(LShrOp, I.getType());
399
8
400
8
  I.replaceAllUsesWith(TruncRes);
401
8
  I.eraseFromParent();
402
8
403
8
  return true;
404
8
}
405
406
unsigned AMDGPUCodeGenPrepare::numBitsUnsigned(Value *Op,
407
283
                                               unsigned ScalarSize) const {
408
283
  KnownBits Known = computeKnownBits(Op, *DL, 0, AC);
409
283
  return ScalarSize - Known.countMinLeadingZeros();
410
283
}
411
412
unsigned AMDGPUCodeGenPrepare::numBitsSigned(Value *Op,
413
102
                                             unsigned ScalarSize) const {
414
102
  // In order for this to be a signed 24-bit value, bit 23, must
415
102
  // be a sign bit.
416
102
  return ScalarSize - ComputeNumSignBits(Op, *DL, 0, AC);
417
102
}
418
419
102
bool AMDGPUCodeGenPrepare::isI24(Value *V, unsigned ScalarSize) const {
420
102
  return ScalarSize >= 24 && // Types less than 24-bit should be treated
421
102
                                     // as unsigned 24-bit values.
422
102
    numBitsSigned(V, ScalarSize) < 24;
423
102
}
424
425
283
bool AMDGPUCodeGenPrepare::isU24(Value *V, unsigned ScalarSize) const {
426
283
  return numBitsUnsigned(V, ScalarSize) <= 24;
427
283
}
428
429
static void extractValues(IRBuilder<> &Builder,
430
226
                          SmallVectorImpl<Value *> &Values, Value *V) {
431
226
  VectorType *VT = dyn_cast<VectorType>(V->getType());
432
226
  if (!VT) {
433
200
    Values.push_back(V);
434
200
    return;
435
200
  }
436
26
437
84
  
for (int I = 0, E = VT->getNumElements(); 26
I != E;
++I58
)
438
58
    Values.push_back(Builder.CreateExtractElement(V, I));
439
26
}
440
441
static Value *insertValues(IRBuilder<> &Builder,
442
                           Type *Ty,
443
113
                           SmallVectorImpl<Value *> &Values) {
444
113
  if (Values.size() == 1)
445
100
    return Values[0];
446
13
447
13
  Value *NewVal = UndefValue::get(Ty);
448
42
  for (int I = 0, E = Values.size(); I != E; 
++I29
)
449
29
    NewVal = Builder.CreateInsertElement(NewVal, Values[I], I);
450
13
451
13
  return NewVal;
452
13
}
453
454
15.9k
bool AMDGPUCodeGenPrepare::replaceMulWithMul24(BinaryOperator &I) const {
455
15.9k
  if (I.getOpcode() != Instruction::Mul)
456
14.7k
    return false;
457
1.18k
458
1.18k
  Type *Ty = I.getType();
459
1.18k
  unsigned Size = Ty->getScalarSizeInBits();
460
1.18k
  if (Size <= 16 && 
ST->has16BitInsts()194
)
461
73
    return false;
462
1.10k
463
1.10k
  // Prefer scalar if this could be s_mul_i32
464
1.10k
  if (DA->isUniform(&I))
465
923
    return false;
466
185
467
185
  Value *LHS = I.getOperand(0);
468
185
  Value *RHS = I.getOperand(1);
469
185
  IRBuilder<> Builder(&I);
470
185
  Builder.SetCurrentDebugLocation(I.getDebugLoc());
471
185
472
185
  Intrinsic::ID IntrID = Intrinsic::not_intrinsic;
473
185
474
185
  // TODO: Should this try to match mulhi24?
475
185
  if (ST->hasMulU24() && isU24(LHS, Size) && 
isU24(RHS, Size)98
) {
476
98
    IntrID = Intrinsic::amdgcn_mul_u24;
477
98
  } else 
if (87
ST->hasMulI24()87
&&
isI24(LHS, Size)87
&&
isI24(RHS, Size)15
) {
478
15
    IntrID = Intrinsic::amdgcn_mul_i24;
479
15
  } else
480
72
    return false;
481
113
482
113
  SmallVector<Value *, 4> LHSVals;
483
113
  SmallVector<Value *, 4> RHSVals;
484
113
  SmallVector<Value *, 4> ResultVals;
485
113
  extractValues(Builder, LHSVals, LHS);
486
113
  extractValues(Builder, RHSVals, RHS);
487
113
488
113
489
113
  IntegerType *I32Ty = Builder.getInt32Ty();
490
113
  FunctionCallee Intrin = Intrinsic::getDeclaration(Mod, IntrID);
491
242
  for (int I = 0, E = LHSVals.size(); I != E; 
++I129
) {
492
129
    Value *LHS, *RHS;
493
129
    if (IntrID == Intrinsic::amdgcn_mul_u24) {
494
108
      LHS = Builder.CreateZExtOrTrunc(LHSVals[I], I32Ty);
495
108
      RHS = Builder.CreateZExtOrTrunc(RHSVals[I], I32Ty);
496
108
    } else {
497
21
      LHS = Builder.CreateSExtOrTrunc(LHSVals[I], I32Ty);
498
21
      RHS = Builder.CreateSExtOrTrunc(RHSVals[I], I32Ty);
499
21
    }
500
129
501
129
    Value *Result = Builder.CreateCall(Intrin, {LHS, RHS});
502
129
503
129
    if (IntrID == Intrinsic::amdgcn_mul_u24) {
504
108
      ResultVals.push_back(Builder.CreateZExtOrTrunc(Result,
505
108
                                                     LHSVals[I]->getType()));
506
108
    } else {
507
21
      ResultVals.push_back(Builder.CreateSExtOrTrunc(Result,
508
21
                                                     LHSVals[I]->getType()));
509
21
    }
510
129
  }
511
113
512
113
  I.replaceAllUsesWith(insertValues(Builder, Ty, ResultVals));
513
113
  I.eraseFromParent();
514
113
515
113
  return true;
516
113
}
517
518
74
static bool shouldKeepFDivF32(Value *Num, bool UnsafeDiv, bool HasDenormals) {
519
74
  const ConstantFP *CNum = dyn_cast<ConstantFP>(Num);
520
74
  if (!CNum)
521
15
    return HasDenormals;
522
59
523
59
  if (UnsafeDiv)
524
0
    return true;
525
59
526
59
  bool IsOne = CNum->isExactlyValue(+1.0) || 
CNum->isExactlyValue(-1.0)32
;
527
59
528
59
  // Reciprocal f32 is handled separately without denormals.
529
59
  return HasDenormals ^ IsOne;
530
59
}
531
532
// Insert an intrinsic for fast fdiv for safe math situations where we can
533
// reduce precision. Leave fdiv for situations where the generic node is
534
// expected to be optimized.
535
317
bool AMDGPUCodeGenPrepare::visitFDiv(BinaryOperator &FDiv) {
536
317
  Type *Ty = FDiv.getType();
537
317
538
317
  if (!Ty->getScalarType()->isFloatTy())
539
101
    return false;
540
216
541
216
  MDNode *FPMath = FDiv.getMetadata(LLVMContext::MD_fpmath);
542
216
  if (!FPMath)
543
147
    return false;
544
69
545
69
  const FPMathOperator *FPOp = cast<const FPMathOperator>(&FDiv);
546
69
  float ULP = FPOp->getFPAccuracy();
547
69
  if (ULP < 2.5f)
548
8
    return false;
549
61
550
61
  FastMathFlags FMF = FPOp->getFastMathFlags();
551
61
  bool UnsafeDiv = HasUnsafeFPMath || 
FMF.isFast()59
||
552
61
                                      
FMF.allowReciprocal()48
;
553
61
554
61
  // With UnsafeDiv node will be optimized to just rcp and mul.
555
61
  if (UnsafeDiv)
556
24
    return false;
557
37
558
37
  IRBuilder<> Builder(FDiv.getParent(), std::next(FDiv.getIterator()), FPMath);
559
37
  Builder.setFastMathFlags(FMF);
560
37
  Builder.SetCurrentDebugLocation(FDiv.getDebugLoc());
561
37
562
37
  Function *Decl = Intrinsic::getDeclaration(Mod, Intrinsic::amdgcn_fdiv_fast);
563
37
564
37
  Value *Num = FDiv.getOperand(0);
565
37
  Value *Den = FDiv.getOperand(1);
566
37
567
37
  Value *NewFDiv = nullptr;
568
37
569
37
  bool HasDenormals = ST->hasFP32Denormals();
570
37
  if (VectorType *VT = dyn_cast<VectorType>(Ty)) {
571
13
    NewFDiv = UndefValue::get(VT);
572
13
573
13
    // FIXME: Doesn't do the right thing for cases where the vector is partially
574
13
    // constant. This works when the scalarizer pass is run first.
575
63
    for (unsigned I = 0, E = VT->getNumElements(); I != E; 
++I50
) {
576
50
      Value *NumEltI = Builder.CreateExtractElement(Num, I);
577
50
      Value *DenEltI = Builder.CreateExtractElement(Den, I);
578
50
      Value *NewElt;
579
50
580
50
      if (shouldKeepFDivF32(NumEltI, UnsafeDiv, HasDenormals)) {
581
24
        NewElt = Builder.CreateFDiv(NumEltI, DenEltI);
582
26
      } else {
583
26
        NewElt = Builder.CreateCall(Decl, { NumEltI, DenEltI });
584
26
      }
585
50
586
50
      NewFDiv = Builder.CreateInsertElement(NewFDiv, NewElt, I);
587
50
    }
588
24
  } else {
589
24
    if (!shouldKeepFDivF32(Num, UnsafeDiv, HasDenormals))
590
11
      NewFDiv = Builder.CreateCall(Decl, { Num, Den });
591
24
  }
592
37
593
37
  if (NewFDiv) {
594
24
    FDiv.replaceAllUsesWith(NewFDiv);
595
24
    NewFDiv->takeName(&FDiv);
596
24
    FDiv.eraseFromParent();
597
24
  }
598
37
599
37
  return !!NewFDiv;
600
37
}
601
602
25.5k
static bool hasUnsafeFPMath(const Function &F) {
603
25.5k
  Attribute Attr = F.getFnAttribute("unsafe-fp-math");
604
25.5k
  return Attr.getValueAsString() == "true";
605
25.5k
}
606
607
static std::pair<Value*, Value*> getMul64(IRBuilder<> &Builder,
608
543
                                          Value *LHS, Value *RHS) {
609
543
  Type *I32Ty = Builder.getInt32Ty();
610
543
  Type *I64Ty = Builder.getInt64Ty();
611
543
612
543
  Value *LHS_EXT64 = Builder.CreateZExt(LHS, I64Ty);
613
543
  Value *RHS_EXT64 = Builder.CreateZExt(RHS, I64Ty);
614
543
  Value *MUL64 = Builder.CreateMul(LHS_EXT64, RHS_EXT64);
615
543
  Value *Lo = Builder.CreateTrunc(MUL64, I32Ty);
616
543
  Value *Hi = Builder.CreateLShr(MUL64, Builder.getInt64(32));
617
543
  Hi = Builder.CreateTrunc(Hi, I32Ty);
618
543
  return std::make_pair(Lo, Hi);
619
543
}
620
621
362
static Value* getMulHu(IRBuilder<> &Builder, Value *LHS, Value *RHS) {
622
362
  return getMul64(Builder, LHS, RHS).second;
623
362
}
624
625
// The fractional part of a float is enough to accurately represent up to
626
// a 24-bit signed integer.
627
Value* AMDGPUCodeGenPrepare::expandDivRem24(IRBuilder<> &Builder,
628
                                            BinaryOperator &I,
629
                                            Value *Num, Value *Den,
630
291
                                            bool IsDiv, bool IsSigned) const {
631
291
  assert(Num->getType()->isIntegerTy(32));
632
291
633
291
  const DataLayout &DL = Mod->getDataLayout();
634
291
  unsigned LHSSignBits = ComputeNumSignBits(Num, DL, 0, AC, &I);
635
291
  if (LHSSignBits < 9)
636
161
    return nullptr;
637
130
638
130
  unsigned RHSSignBits = ComputeNumSignBits(Den, DL, 0, AC, &I);
639
130
  if (RHSSignBits < 9)
640
20
    return nullptr;
641
110
642
110
643
110
  unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
644
110
  unsigned DivBits = 32 - SignBits;
645
110
  if (IsSigned)
646
58
    ++DivBits;
647
110
648
110
  Type *Ty = Num->getType();
649
110
  Type *I32Ty = Builder.getInt32Ty();
650
110
  Type *F32Ty = Builder.getFloatTy();
651
110
  ConstantInt *One = Builder.getInt32(1);
652
110
  Value *JQ = One;
653
110
654
110
  if (IsSigned) {
655
58
    // char|short jq = ia ^ ib;
656
58
    JQ = Builder.CreateXor(Num, Den);
657
58
658
58
    // jq = jq >> (bitsize - 2)
659
58
    JQ = Builder.CreateAShr(JQ, Builder.getInt32(30));
660
58
661
58
    // jq = jq | 0x1
662
58
    JQ = Builder.CreateOr(JQ, One);
663
58
  }
664
110
665
110
  // int ia = (int)LHS;
666
110
  Value *IA = Num;
667
110
668
110
  // int ib, (int)RHS;
669
110
  Value *IB = Den;
670
110
671
110
  // float fa = (float)ia;
672
110
  Value *FA = IsSigned ? 
Builder.CreateSIToFP(IA, F32Ty)58
673
110
                       : 
Builder.CreateUIToFP(IA, F32Ty)52
;
674
110
675
110
  // float fb = (float)ib;
676
110
  Value *FB = IsSigned ? 
Builder.CreateSIToFP(IB,F32Ty)58
677
110
                       : 
Builder.CreateUIToFP(IB,F32Ty)52
;
678
110
679
110
  Value *RCP = Builder.CreateFDiv(ConstantFP::get(F32Ty, 1.0), FB);
680
110
  Value *FQM = Builder.CreateFMul(FA, RCP);
681
110
682
110
  // fq = trunc(fqm);
683
110
  CallInst *FQ = Builder.CreateUnaryIntrinsic(Intrinsic::trunc, FQM);
684
110
  FQ->copyFastMathFlags(Builder.getFastMathFlags());
685
110
686
110
  // float fqneg = -fq;
687
110
  Value *FQNeg = Builder.CreateFNeg(FQ);
688
110
689
110
  // float fr = mad(fqneg, fb, fa);
690
110
  Value *FR = Builder.CreateIntrinsic(Intrinsic::amdgcn_fmad_ftz,
691
110
                                      {FQNeg->getType()}, {FQNeg, FB, FA}, FQ);
692
110
693
110
  // int iq = (int)fq;
694
110
  Value *IQ = IsSigned ? 
Builder.CreateFPToSI(FQ, I32Ty)58
695
110
                       : 
Builder.CreateFPToUI(FQ, I32Ty)52
;
696
110
697
110
  // fr = fabs(fr);
698
110
  FR = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, FR, FQ);
699
110
700
110
  // fb = fabs(fb);
701
110
  FB = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, FB, FQ);
702
110
703
110
  // int cv = fr >= fb;
704
110
  Value *CV = Builder.CreateFCmpOGE(FR, FB);
705
110
706
110
  // jq = (cv ? jq : 0);
707
110
  JQ = Builder.CreateSelect(CV, JQ, Builder.getInt32(0));
708
110
709
110
  // dst = iq + jq;
710
110
  Value *Div = Builder.CreateAdd(IQ, JQ);
711
110
712
110
  Value *Res = Div;
713
110
  if (!IsDiv) {
714
42
    // Rem needs compensation, it's easier to recompute it
715
42
    Value *Rem = Builder.CreateMul(Div, Den);
716
42
    Res = Builder.CreateSub(Num, Rem);
717
42
  }
718
110
719
110
  // Truncate to number of bits this divide really is.
720
110
  if (IsSigned) {
721
58
    Res = Builder.CreateTrunc(Res, Builder.getIntNTy(DivBits));
722
58
    Res = Builder.CreateSExt(Res, Ty);
723
58
  } else {
724
52
    ConstantInt *TruncMask = Builder.getInt32((UINT64_C(1) << DivBits) - 1);
725
52
    Res = Builder.CreateAnd(Res, TruncMask);
726
52
  }
727
110
728
110
  return Res;
729
110
}
730
731
Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
732
                                            BinaryOperator &I,
733
382
                                            Value *Num, Value *Den) const {
734
382
  Instruction::BinaryOps Opc = I.getOpcode();
735
382
  assert(Opc == Instruction::URem || Opc == Instruction::UDiv ||
736
382
         Opc == Instruction::SRem || Opc == Instruction::SDiv);
737
382
738
382
  FastMathFlags FMF;
739
382
  FMF.setFast();
740
382
  Builder.setFastMathFlags(FMF);
741
382
742
382
  if (isa<Constant>(Den))
743
91
    return nullptr; // Keep it for optimization
744
291
745
291
  bool IsDiv = Opc == Instruction::UDiv || 
Opc == Instruction::SDiv197
;
746
291
  bool IsSigned = Opc == Instruction::SRem || 
Opc == Instruction::SDiv239
;
747
291
748
291
  Type *Ty = Num->getType();
749
291
  Type *I32Ty = Builder.getInt32Ty();
750
291
  Type *F32Ty = Builder.getFloatTy();
751
291
752
291
  if (Ty->getScalarSizeInBits() < 32) {
753
96
    if (IsSigned) {
754
48
      Num = Builder.CreateSExt(Num, I32Ty);
755
48
      Den = Builder.CreateSExt(Den, I32Ty);
756
48
    } else {
757
48
      Num = Builder.CreateZExt(Num, I32Ty);
758
48
      Den = Builder.CreateZExt(Den, I32Ty);
759
48
    }
760
96
  }
761
291
762
291
  if (Value *Res = expandDivRem24(Builder, I, Num, Den, IsDiv, IsSigned)) {
763
110
    Res = Builder.CreateTrunc(Res, Ty);
764
110
    return Res;
765
110
  }
766
181
767
181
  ConstantInt *Zero = Builder.getInt32(0);
768
181
  ConstantInt *One = Builder.getInt32(1);
769
181
  ConstantInt *MinusOne = Builder.getInt32(~0);
770
181
771
181
  Value *Sign = nullptr;
772
181
  if (IsSigned) {
773
68
    ConstantInt *K31 = Builder.getInt32(31);
774
68
    Value *LHSign = Builder.CreateAShr(Num, K31);
775
68
    Value *RHSign = Builder.CreateAShr(Den, K31);
776
68
    // Remainder sign is the same as LHS
777
68
    Sign = IsDiv ? 
Builder.CreateXor(LHSign, RHSign)40
:
LHSign28
;
778
68
779
68
    Num = Builder.CreateAdd(Num, LHSign);
780
68
    Den = Builder.CreateAdd(Den, RHSign);
781
68
782
68
    Num = Builder.CreateXor(Num, LHSign);
783
68
    Den = Builder.CreateXor(Den, RHSign);
784
68
  }
785
181
786
181
  // RCP =  URECIP(Den) = 2^32 / Den + e
787
181
  // e is rounding error.
788
181
  Value *DEN_F32 = Builder.CreateUIToFP(Den, F32Ty);
789
181
  Value *RCP_F32 = Builder.CreateFDiv(ConstantFP::get(F32Ty, 1.0), DEN_F32);
790
181
  Constant *UINT_MAX_PLUS_1 = ConstantFP::get(F32Ty, BitsToFloat(0x4f800000));
791
181
  Value *RCP_SCALE = Builder.CreateFMul(RCP_F32, UINT_MAX_PLUS_1);
792
181
  Value *RCP = Builder.CreateFPToUI(RCP_SCALE, I32Ty);
793
181
794
181
  // RCP_LO, RCP_HI = mul(RCP, Den) */
795
181
  Value *RCP_LO, *RCP_HI;
796
181
  std::tie(RCP_LO, RCP_HI) = getMul64(Builder, RCP, Den);
797
181
798
181
  // NEG_RCP_LO = -RCP_LO
799
181
  Value *NEG_RCP_LO = Builder.CreateNeg(RCP_LO);
800
181
801
181
  // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
802
181
  Value *RCP_HI_0_CC = Builder.CreateICmpEQ(RCP_HI, Zero);
803
181
  Value *ABS_RCP_LO = Builder.CreateSelect(RCP_HI_0_CC, NEG_RCP_LO, RCP_LO);
804
181
805
181
  // Calculate the rounding error from the URECIP instruction
806
181
  // E = mulhu(ABS_RCP_LO, RCP)
807
181
  Value *E = getMulHu(Builder, ABS_RCP_LO, RCP);
808
181
809
181
  // RCP_A_E = RCP + E
810
181
  Value *RCP_A_E = Builder.CreateAdd(RCP, E);
811
181
812
181
  // RCP_S_E = RCP - E
813
181
  Value *RCP_S_E = Builder.CreateSub(RCP, E);
814
181
815
181
  // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
816
181
  Value *Tmp0 = Builder.CreateSelect(RCP_HI_0_CC, RCP_A_E, RCP_S_E);
817
181
818
181
  // Quotient = mulhu(Tmp0, Num)
819
181
  Value *Quotient = getMulHu(Builder, Tmp0, Num);
820
181
821
181
  // Num_S_Remainder = Quotient * Den
822
181
  Value *Num_S_Remainder = Builder.CreateMul(Quotient, Den);
823
181
824
181
  // Remainder = Num - Num_S_Remainder
825
181
  Value *Remainder = Builder.CreateSub(Num, Num_S_Remainder);
826
181
827
181
  // Remainder_GE_Den = (Remainder >= Den ? -1 : 0)
828
181
  Value *Rem_GE_Den_CC = Builder.CreateICmpUGE(Remainder, Den);
829
181
  Value *Remainder_GE_Den = Builder.CreateSelect(Rem_GE_Den_CC, MinusOne, Zero);
830
181
831
181
  // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0)
832
181
  Value *Num_GE_Num_S_Rem_CC = Builder.CreateICmpUGE(Num, Num_S_Remainder);
833
181
  Value *Remainder_GE_Zero = Builder.CreateSelect(Num_GE_Num_S_Rem_CC,
834
181
                                                  MinusOne, Zero);
835
181
836
181
  // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
837
181
  Value *Tmp1 = Builder.CreateAnd(Remainder_GE_Den, Remainder_GE_Zero);
838
181
  Value *Tmp1_0_CC = Builder.CreateICmpEQ(Tmp1, Zero);
839
181
840
181
  Value *Res;
841
181
  if (IsDiv) {
842
100
    // Quotient_A_One = Quotient + 1
843
100
    Value *Quotient_A_One = Builder.CreateAdd(Quotient, One);
844
100
845
100
    // Quotient_S_One = Quotient - 1
846
100
    Value *Quotient_S_One = Builder.CreateSub(Quotient, One);
847
100
848
100
    // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One)
849
100
    Value *Div = Builder.CreateSelect(Tmp1_0_CC, Quotient, Quotient_A_One);
850
100
851
100
    // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div)
852
100
    Res = Builder.CreateSelect(Num_GE_Num_S_Rem_CC, Div, Quotient_S_One);
853
100
  } else {
854
81
    // Remainder_S_Den = Remainder - Den
855
81
    Value *Remainder_S_Den = Builder.CreateSub(Remainder, Den);
856
81
857
81
    // Remainder_A_Den = Remainder + Den
858
81
    Value *Remainder_A_Den = Builder.CreateAdd(Remainder, Den);
859
81
860
81
    // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den)
861
81
    Value *Rem = Builder.CreateSelect(Tmp1_0_CC, Remainder, Remainder_S_Den);
862
81
863
81
    // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem)
864
81
    Res = Builder.CreateSelect(Num_GE_Num_S_Rem_CC, Rem, Remainder_A_Den);
865
81
  }
866
181
867
181
  if (IsSigned) {
868
68
    Res = Builder.CreateXor(Res, Sign);
869
68
    Res = Builder.CreateSub(Res, Sign);
870
68
  }
871
181
872
181
  Res = Builder.CreateTrunc(Res, Ty);
873
181
874
181
  return Res;
875
181
}
876
877
17.2k
bool AMDGPUCodeGenPrepare::visitBinaryOperator(BinaryOperator &I) {
878
17.2k
  if (ST->has16BitInsts() && 
needsPromotionToI32(I.getType())10.5k
&&
879
17.2k
      
DA->isUniform(&I)1.50k
&&
promoteUniformOpToI32(I)1.32k
)
880
1.30k
    return true;
881
15.9k
882
15.9k
  if (replaceMulWithMul24(I))
883
113
    return true;
884
15.8k
885
15.8k
  bool Changed = false;
886
15.8k
  Instruction::BinaryOps Opc = I.getOpcode();
887
15.8k
  Type *Ty = I.getType();
888
15.8k
  Value *NewDiv = nullptr;
889
15.8k
  if ((Opc == Instruction::URem || 
Opc == Instruction::UDiv15.7k
||
890
15.8k
       
Opc == Instruction::SRem15.6k
||
Opc == Instruction::SDiv15.6k
) &&
891
15.8k
      
Ty->getScalarSizeInBits() <= 32309
) {
892
248
    Value *Num = I.getOperand(0);
893
248
    Value *Den = I.getOperand(1);
894
248
    IRBuilder<> Builder(&I);
895
248
    Builder.SetCurrentDebugLocation(I.getDebugLoc());
896
248
897
248
    if (VectorType *VT = dyn_cast<VectorType>(Ty)) {
898
60
      NewDiv = UndefValue::get(VT);
899
60
900
254
      for (unsigned N = 0, E = VT->getNumElements(); N != E; 
++N194
) {
901
194
        Value *NumEltN = Builder.CreateExtractElement(Num, N);
902
194
        Value *DenEltN = Builder.CreateExtractElement(Den, N);
903
194
        Value *NewElt = expandDivRem32(Builder, I, NumEltN, DenEltN);
904
194
        if (!NewElt)
905
54
          NewElt = Builder.CreateBinOp(Opc, NumEltN, DenEltN);
906
194
        NewDiv = Builder.CreateInsertElement(NewDiv, NewElt, N);
907
194
      }
908
188
    } else {
909
188
      NewDiv = expandDivRem32(Builder, I, Num, Den);
910
188
    }
911
248
912
248
    if (NewDiv) {
913
211
      I.replaceAllUsesWith(NewDiv);
914
211
      I.eraseFromParent();
915
211
      Changed = true;
916
211
    }
917
248
  }
918
15.8k
919
15.8k
  return Changed;
920
15.8k
}
921
922
18.2k
bool AMDGPUCodeGenPrepare::visitLoadInst(LoadInst &I) {
923
18.2k
  if (!WidenLoads)
924
24
    return false;
925
18.2k
926
18.2k
  if ((I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
927
18.2k
       
I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT16.7k
) &&
928
18.2k
      
canWidenScalarExtLoad(I)1.59k
) {
929
69
    IRBuilder<> Builder(&I);
930
69
    Builder.SetCurrentDebugLocation(I.getDebugLoc());
931
69
932
69
    Type *I32Ty = Builder.getInt32Ty();
933
69
    Type *PT = PointerType::get(I32Ty, I.getPointerAddressSpace());
934
69
    Value *BitCast= Builder.CreateBitCast(I.getPointerOperand(), PT);
935
69
    LoadInst *WidenLoad = Builder.CreateLoad(I32Ty, BitCast);
936
69
    WidenLoad->copyMetadata(I);
937
69
938
69
    // If we have range metadata, we need to convert the type, and not make
939
69
    // assumptions about the high bits.
940
69
    if (auto *Range = WidenLoad->getMetadata(LLVMContext::MD_range)) {
941
6
      ConstantInt *Lower =
942
6
        mdconst::extract<ConstantInt>(Range->getOperand(0));
943
6
944
6
      if (Lower->getValue().isNullValue()) {
945
1
        WidenLoad->setMetadata(LLVMContext::MD_range, nullptr);
946
5
      } else {
947
5
        Metadata *LowAndHigh[] = {
948
5
          ConstantAsMetadata::get(ConstantInt::get(I32Ty, Lower->getValue().zext(32))),
949
5
          // Don't make assumptions about the high bits.
950
5
          ConstantAsMetadata::get(ConstantInt::get(I32Ty, 0))
951
5
        };
952
5
953
5
        WidenLoad->setMetadata(LLVMContext::MD_range,
954
5
                               MDNode::get(Mod->getContext(), LowAndHigh));
955
5
      }
956
6
    }
957
69
958
69
    int TySize = Mod->getDataLayout().getTypeSizeInBits(I.getType());
959
69
    Type *IntNTy = Builder.getIntNTy(TySize);
960
69
    Value *ValTrunc = Builder.CreateTrunc(WidenLoad, IntNTy);
961
69
    Value *ValOrig = Builder.CreateBitCast(ValTrunc, I.getType());
962
69
    I.replaceAllUsesWith(ValOrig);
963
69
    I.eraseFromParent();
964
69
    return true;
965
69
  }
966
18.1k
967
18.1k
  return false;
968
18.1k
}
969
970
3.40k
bool AMDGPUCodeGenPrepare::visitICmpInst(ICmpInst &I) {
971
3.40k
  bool Changed = false;
972
3.40k
973
3.40k
  if (ST->has16BitInsts() && 
needsPromotionToI32(I.getOperand(0)->getType())1.98k
&&
974
3.40k
      
DA->isUniform(&I)318
)
975
116
    Changed |= promoteUniformOpToI32(I);
976
3.40k
977
3.40k
  return Changed;
978
3.40k
}
979
980
2.58k
bool AMDGPUCodeGenPrepare::visitSelectInst(SelectInst &I) {
981
2.58k
  bool Changed = false;
982
2.58k
983
2.58k
  if (ST->has16BitInsts() && 
needsPromotionToI32(I.getType())1.65k
&&
984
2.58k
      
DA->isUniform(&I)312
)
985
130
    Changed |= promoteUniformOpToI32(I);
986
2.58k
987
2.58k
  return Changed;
988
2.58k
}
989
990
16.4k
bool AMDGPUCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
991
16.4k
  switch (I.getIntrinsicID()) {
992
16.4k
  case Intrinsic::bitreverse:
993
41
    return visitBitreverseIntrinsicInst(I);
994
16.4k
  default:
995
16.3k
    return false;
996
16.4k
  }
997
16.4k
}
998
999
41
bool AMDGPUCodeGenPrepare::visitBitreverseIntrinsicInst(IntrinsicInst &I) {
1000
41
  bool Changed = false;
1001
41
1002
41
  if (ST->has16BitInsts() && 
needsPromotionToI32(I.getType())26
&&
1003
41
      
DA->isUniform(&I)10
)
1004
8
    Changed |= promoteUniformBitreverseToI32(I);
1005
41
1006
41
  return Changed;
1007
41
}
1008
1009
2.43k
bool AMDGPUCodeGenPrepare::doInitialization(Module &M) {
1010
2.43k
  Mod = &M;
1011
2.43k
  DL = &Mod->getDataLayout();
1012
2.43k
  return false;
1013
2.43k
}
1014
1015
25.5k
bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
1016
25.5k
  if (skipFunction(F))
1017
13
    return false;
1018
25.5k
1019
25.5k
  auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
1020
25.5k
  if (!TPC)
1021
8
    return false;
1022
25.5k
1023
25.5k
  const AMDGPUTargetMachine &TM = TPC->getTM<AMDGPUTargetMachine>();
1024
25.5k
  ST = &TM.getSubtarget<GCNSubtarget>(F);
1025
25.5k
  AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
1026
25.5k
  DA = &getAnalysis<LegacyDivergenceAnalysis>();
1027
25.5k
  HasUnsafeFPMath = hasUnsafeFPMath(F);
1028
25.5k
1029
25.5k
  bool MadeChange = false;
1030
25.5k
1031
28.4k
  for (BasicBlock &BB : F) {
1032
28.4k
    BasicBlock::iterator Next;
1033
183k
    for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; 
I = Next155k
) {
1034
155k
      Next = std::next(I);
1035
155k
      MadeChange |= visit(*I);
1036
155k
    }
1037
28.4k
  }
1038
25.5k
1039
25.5k
  return MadeChange;
1040
25.5k
}
1041
1042
101k
INITIALIZE_PASS_BEGIN(AMDGPUCodeGenPrepare, DEBUG_TYPE,
1043
101k
                      "AMDGPU IR optimizations", false, false)
1044
101k
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
1045
101k
INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis)
1046
101k
INITIALIZE_PASS_END(AMDGPUCodeGenPrepare, DEBUG_TYPE, "AMDGPU IR optimizations",
1047
                    false, false)
1048
1049
char AMDGPUCodeGenPrepare::ID = 0;
1050
1051
2.44k
FunctionPass *llvm::createAMDGPUCodeGenPreparePass() {
1052
2.44k
  return new AMDGPUCodeGenPrepare();
1053
2.44k
}