Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- AMDGPULibCalls.cpp -------------------------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file
10
/// This file does AMD library function optimizations.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#define DEBUG_TYPE "amdgpu-simplifylib"
15
16
#include "AMDGPU.h"
17
#include "AMDGPULibFunc.h"
18
#include "AMDGPUSubtarget.h"
19
#include "llvm/Analysis/AliasAnalysis.h"
20
#include "llvm/Analysis/Loads.h"
21
#include "llvm/ADT/StringSet.h"
22
#include "llvm/ADT/StringRef.h"
23
#include "llvm/IR/Constants.h"
24
#include "llvm/IR/DerivedTypes.h"
25
#include "llvm/IR/Instructions.h"
26
#include "llvm/IR/Intrinsics.h"
27
#include "llvm/IR/IRBuilder.h"
28
#include "llvm/IR/Function.h"
29
#include "llvm/IR/LLVMContext.h"
30
#include "llvm/IR/Module.h"
31
#include "llvm/IR/ValueSymbolTable.h"
32
#include "llvm/Support/Debug.h"
33
#include "llvm/Support/raw_ostream.h"
34
#include "llvm/Target/TargetMachine.h"
35
#include "llvm/Target/TargetOptions.h"
36
#include <vector>
37
#include <cmath>
38
39
using namespace llvm;
40
41
static cl::opt<bool> EnablePreLink("amdgpu-prelink",
42
  cl::desc("Enable pre-link mode optimizations"),
43
  cl::init(false),
44
  cl::Hidden);
45
46
static cl::list<std::string> UseNative("amdgpu-use-native",
47
  cl::desc("Comma separated list of functions to replace with native, or all"),
48
  cl::CommaSeparated, cl::ValueOptional,
49
  cl::Hidden);
50
51
0
#define MATH_PI     3.14159265358979323846264338327950288419716939937511
52
#define MATH_E      2.71828182845904523536028747135266249775724709369996
53
#define MATH_SQRT2  1.41421356237309504880168872420969807856967187537695
54
55
#define MATH_LOG2E     1.4426950408889634073599246810018921374266459541529859
56
#define MATH_LOG10E    0.4342944819032518276511289189166050822943970058036665
57
// Value of log2(10)
58
#define MATH_LOG2_10   3.3219280948873623478703194294893901758648313930245806
59
// Value of 1 / log2(10)
60
#define MATH_RLOG2_10  0.3010299956639811952137388947244930267681898814621085
61
// Value of 1 / M_LOG2E_F = 1 / log2(e)
62
#define MATH_RLOG2_E   0.6931471805599453094172321214581765680755001343602552
63
64
namespace llvm {
65
66
class AMDGPULibCalls {
67
private:
68
69
  typedef llvm::AMDGPULibFunc FuncInfo;
70
71
  const TargetMachine *TM;
72
73
  // -fuse-native.
74
  bool AllNative = false;
75
76
  bool useNativeFunc(const StringRef F) const;
77
78
  // Return a pointer (pointer expr) to the function if function defintion with
79
  // "FuncName" exists. It may create a new function prototype in pre-link mode.
80
  FunctionCallee getFunction(Module *M, const FuncInfo &fInfo);
81
82
  // Replace a normal function with its native version.
83
  bool replaceWithNative(CallInst *CI, const FuncInfo &FInfo);
84
85
  bool parseFunctionName(const StringRef& FMangledName,
86
                         FuncInfo *FInfo=nullptr /*out*/);
87
88
  bool TDOFold(CallInst *CI, const FuncInfo &FInfo);
89
90
  /* Specialized optimizations */
91
92
  // recip (half or native)
93
  bool fold_recip(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
94
95
  // divide (half or native)
96
  bool fold_divide(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
97
98
  // pow/powr/pown
99
  bool fold_pow(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
100
101
  // rootn
102
  bool fold_rootn(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
103
104
  // fma/mad
105
  bool fold_fma_mad(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
106
107
  // -fuse-native for sincos
108
  bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo);
109
110
  // evaluate calls if calls' arguments are constants.
111
  bool evaluateScalarMathFunc(FuncInfo &FInfo, double& Res0,
112
    double& Res1, Constant *copr0, Constant *copr1, Constant *copr2);
113
  bool evaluateCall(CallInst *aCI, FuncInfo &FInfo);
114
115
  // exp
116
  bool fold_exp(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
117
118
  // exp2
119
  bool fold_exp2(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
120
121
  // exp10
122
  bool fold_exp10(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
123
124
  // log
125
  bool fold_log(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
126
127
  // log2
128
  bool fold_log2(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
129
130
  // log10
131
  bool fold_log10(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
132
133
  // sqrt
134
  bool fold_sqrt(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
135
136
  // sin/cos
137
  bool fold_sincos(CallInst * CI, IRBuilder<> &B, AliasAnalysis * AA);
138
139
  // __read_pipe/__write_pipe
140
  bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, FuncInfo &FInfo);
141
142
  // llvm.amdgcn.wavefrontsize
143
  bool fold_wavefrontsize(CallInst *CI, IRBuilder<> &B);
144
145
  // Get insertion point at entry.
146
  BasicBlock::iterator getEntryIns(CallInst * UI);
147
  // Insert an Alloc instruction.
148
  AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix);
149
  // Get a scalar native builtin signle argument FP function
150
  FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo);
151
152
protected:
153
  CallInst *CI;
154
155
  bool isUnsafeMath(const CallInst *CI) const;
156
157
92
  void replaceCall(Value *With) {
158
92
    CI->replaceAllUsesWith(With);
159
92
    CI->eraseFromParent();
160
92
  }
161
162
public:
163
263
  AMDGPULibCalls(const TargetMachine *TM_ = nullptr) : TM(TM_) {}
164
165
  bool fold(CallInst *CI, AliasAnalysis *AA = nullptr);
166
167
  void initNativeFuncs();
168
169
  // Replace a normal math function call with that native version
170
  bool useNative(CallInst *CI);
171
};
172
173
} // end llvm namespace
174
175
namespace {
176
177
  class AMDGPUSimplifyLibCalls : public FunctionPass {
178
179
  const TargetOptions Options;
180
181
  AMDGPULibCalls Simplifier;
182
183
  public:
184
    static char ID; // Pass identification
185
186
    AMDGPUSimplifyLibCalls(const TargetOptions &Opt = TargetOptions(),
187
                           const TargetMachine *TM = nullptr)
188
97
      : FunctionPass(ID), Options(Opt), Simplifier(TM) {
189
97
      initializeAMDGPUSimplifyLibCallsPass(*PassRegistry::getPassRegistry());
190
97
    }
191
192
97
    void getAnalysisUsage(AnalysisUsage &AU) const override {
193
97
      AU.addRequired<AAResultsWrapperPass>();
194
97
    }
195
196
    bool runOnFunction(Function &M) override;
197
  };
198
199
  class AMDGPUUseNativeCalls : public FunctionPass {
200
201
  AMDGPULibCalls Simplifier;
202
203
  public:
204
    static char ID; // Pass identification
205
206
166
    AMDGPUUseNativeCalls() : FunctionPass(ID) {
207
166
      initializeAMDGPUUseNativeCallsPass(*PassRegistry::getPassRegistry());
208
166
      Simplifier.initNativeFuncs();
209
166
    }
210
211
    bool runOnFunction(Function &F) override;
212
  };
213
214
} // end anonymous namespace.
215
216
char AMDGPUSimplifyLibCalls::ID = 0;
217
char AMDGPUUseNativeCalls::ID = 0;
218
219
101k
INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib",
220
101k
                      "Simplify well-known AMD library calls", false, false)
221
101k
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
222
101k
INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib",
223
                    "Simplify well-known AMD library calls", false, false)
224
225
INITIALIZE_PASS(AMDGPUUseNativeCalls, "amdgpu-usenative",
226
                "Replace builtin math calls with that native versions.",
227
                false, false)
228
229
template <typename IRB>
230
static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg,
231
31
                              const Twine &Name = "") {
232
31
  CallInst *R = B.CreateCall(Callee, Arg, Name);
233
31
  if (Function *F = dyn_cast<Function>(Callee.getCallee()))
234
31
    R->setCallingConv(F->getCallingConv());
235
31
  return R;
236
31
}
237
238
template <typename IRB>
239
static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1,
240
6
                               Value *Arg2, const Twine &Name = "") {
241
6
  CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name);
242
6
  if (Function *F = dyn_cast<Function>(Callee.getCallee()))
243
6
    R->setCallingConv(F->getCallingConv());
244
6
  return R;
245
6
}
246
247
//  Data structures for table-driven optimizations.
248
//  FuncTbl works for both f32 and f64 functions with 1 input argument
249
250
struct TableEntry {
251
  double   result;
252
  double   input;
253
};
254
255
/* a list of {result, input} */
256
static const TableEntry tbl_acos[] = {
257
  {MATH_PI/2.0, 0.0},
258
  {MATH_PI/2.0, -0.0},
259
  {0.0, 1.0},
260
  {MATH_PI, -1.0}
261
};
262
static const TableEntry tbl_acosh[] = {
263
  {0.0, 1.0}
264
};
265
static const TableEntry tbl_acospi[] = {
266
  {0.5, 0.0},
267
  {0.5, -0.0},
268
  {0.0, 1.0},
269
  {1.0, -1.0}
270
};
271
static const TableEntry tbl_asin[] = {
272
  {0.0, 0.0},
273
  {-0.0, -0.0},
274
  {MATH_PI/2.0, 1.0},
275
  {-MATH_PI/2.0, -1.0}
276
};
277
static const TableEntry tbl_asinh[] = {
278
  {0.0, 0.0},
279
  {-0.0, -0.0}
280
};
281
static const TableEntry tbl_asinpi[] = {
282
  {0.0, 0.0},
283
  {-0.0, -0.0},
284
  {0.5, 1.0},
285
  {-0.5, -1.0}
286
};
287
static const TableEntry tbl_atan[] = {
288
  {0.0, 0.0},
289
  {-0.0, -0.0},
290
  {MATH_PI/4.0, 1.0},
291
  {-MATH_PI/4.0, -1.0}
292
};
293
static const TableEntry tbl_atanh[] = {
294
  {0.0, 0.0},
295
  {-0.0, -0.0}
296
};
297
static const TableEntry tbl_atanpi[] = {
298
  {0.0, 0.0},
299
  {-0.0, -0.0},
300
  {0.25, 1.0},
301
  {-0.25, -1.0}
302
};
303
static const TableEntry tbl_cbrt[] = {
304
  {0.0, 0.0},
305
  {-0.0, -0.0},
306
  {1.0, 1.0},
307
  {-1.0, -1.0},
308
};
309
static const TableEntry tbl_cos[] = {
310
  {1.0, 0.0},
311
  {1.0, -0.0}
312
};
313
static const TableEntry tbl_cosh[] = {
314
  {1.0, 0.0},
315
  {1.0, -0.0}
316
};
317
static const TableEntry tbl_cospi[] = {
318
  {1.0, 0.0},
319
  {1.0, -0.0}
320
};
321
static const TableEntry tbl_erfc[] = {
322
  {1.0, 0.0},
323
  {1.0, -0.0}
324
};
325
static const TableEntry tbl_erf[] = {
326
  {0.0, 0.0},
327
  {-0.0, -0.0}
328
};
329
static const TableEntry tbl_exp[] = {
330
  {1.0, 0.0},
331
  {1.0, -0.0},
332
  {MATH_E, 1.0}
333
};
334
static const TableEntry tbl_exp2[] = {
335
  {1.0, 0.0},
336
  {1.0, -0.0},
337
  {2.0, 1.0}
338
};
339
static const TableEntry tbl_exp10[] = {
340
  {1.0, 0.0},
341
  {1.0, -0.0},
342
  {10.0, 1.0}
343
};
344
static const TableEntry tbl_expm1[] = {
345
  {0.0, 0.0},
346
  {-0.0, -0.0}
347
};
348
static const TableEntry tbl_log[] = {
349
  {0.0, 1.0},
350
  {1.0, MATH_E}
351
};
352
static const TableEntry tbl_log2[] = {
353
  {0.0, 1.0},
354
  {1.0, 2.0}
355
};
356
static const TableEntry tbl_log10[] = {
357
  {0.0, 1.0},
358
  {1.0, 10.0}
359
};
360
static const TableEntry tbl_rsqrt[] = {
361
  {1.0, 1.0},
362
  {1.0/MATH_SQRT2, 2.0}
363
};
364
static const TableEntry tbl_sin[] = {
365
  {0.0, 0.0},
366
  {-0.0, -0.0}
367
};
368
static const TableEntry tbl_sinh[] = {
369
  {0.0, 0.0},
370
  {-0.0, -0.0}
371
};
372
static const TableEntry tbl_sinpi[] = {
373
  {0.0, 0.0},
374
  {-0.0, -0.0}
375
};
376
static const TableEntry tbl_sqrt[] = {
377
  {0.0, 0.0},
378
  {1.0, 1.0},
379
  {MATH_SQRT2, 2.0}
380
};
381
static const TableEntry tbl_tan[] = {
382
  {0.0, 0.0},
383
  {-0.0, -0.0}
384
};
385
static const TableEntry tbl_tanh[] = {
386
  {0.0, 0.0},
387
  {-0.0, -0.0}
388
};
389
static const TableEntry tbl_tanpi[] = {
390
  {0.0, 0.0},
391
  {-0.0, -0.0}
392
};
393
static const TableEntry tbl_tgamma[] = {
394
  {1.0, 1.0},
395
  {1.0, 2.0},
396
  {2.0, 3.0},
397
  {6.0, 4.0}
398
};
399
400
53
static bool HasNative(AMDGPULibFunc::EFuncId id) {
401
53
  switch(id) {
402
53
  case AMDGPULibFunc::EI_DIVIDE:
403
27
  case AMDGPULibFunc::EI_COS:
404
27
  case AMDGPULibFunc::EI_EXP:
405
27
  case AMDGPULibFunc::EI_EXP2:
406
27
  case AMDGPULibFunc::EI_EXP10:
407
27
  case AMDGPULibFunc::EI_LOG:
408
27
  case AMDGPULibFunc::EI_LOG2:
409
27
  case AMDGPULibFunc::EI_LOG10:
410
27
  case AMDGPULibFunc::EI_POWR:
411
27
  case AMDGPULibFunc::EI_RECIP:
412
27
  case AMDGPULibFunc::EI_RSQRT:
413
27
  case AMDGPULibFunc::EI_SIN:
414
27
  case AMDGPULibFunc::EI_SINCOS:
415
27
  case AMDGPULibFunc::EI_SQRT:
416
27
  case AMDGPULibFunc::EI_TAN:
417
27
    return true;
418
27
  
default:;26
419
53
  }
420
53
  
return false26
;
421
53
}
422
423
struct TableRef {
424
  size_t size;
425
  const TableEntry *table; // variable size: from 0 to (size - 1)
426
427
140
  TableRef() : size(0), table(nullptr) {}
428
429
  template <size_t N>
430
70
  TableRef(const TableEntry (&tbl)[N]) : size(N), table(&tbl[0]) {}
Unexecuted instantiation: TableRef::TableRef<4ul>(TableEntry const (&) [4ul])
Unexecuted instantiation: TableRef::TableRef<1ul>(TableEntry const (&) [1ul])
TableRef::TableRef<2ul>(TableEntry const (&) [2ul])
Line
Count
Source
430
55
  TableRef(const TableEntry (&tbl)[N]) : size(N), table(&tbl[0]) {}
TableRef::TableRef<3ul>(TableEntry const (&) [3ul])
Line
Count
Source
430
15
  TableRef(const TableEntry (&tbl)[N]) : size(N), table(&tbl[0]) {}
431
};
432
433
210
static TableRef getOptTable(AMDGPULibFunc::EFuncId id) {
434
210
  switch(id) {
435
210
  
case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos)0
;
436
210
  
case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh)0
;
437
210
  
case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi)0
;
438
210
  
case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin)0
;
439
210
  
case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh)0
;
440
210
  
case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi)0
;
441
210
  
case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan)0
;
442
210
  
case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh)0
;
443
210
  
case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi)0
;
444
210
  
case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt)0
;
445
210
  case AMDGPULibFunc::EI_NCOS:
446
19
  case AMDGPULibFunc::EI_COS:     return TableRef(tbl_cos);
447
19
  
case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh)0
;
448
19
  
case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi)0
;
449
19
  
case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc)0
;
450
19
  
case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf)0
;
451
19
  
case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp)3
;
452
19
  case AMDGPULibFunc::EI_NEXP2:
453
3
  case AMDGPULibFunc::EI_EXP2:    return TableRef(tbl_exp2);
454
3
  case AMDGPULibFunc::EI_EXP10:   return TableRef(tbl_exp10);
455
3
  
case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1)0
;
456
3
  case AMDGPULibFunc::EI_LOG:     return TableRef(tbl_log);
457
3
  case AMDGPULibFunc::EI_NLOG2:
458
3
  case AMDGPULibFunc::EI_LOG2:    return TableRef(tbl_log2);
459
3
  case AMDGPULibFunc::EI_LOG10:   return TableRef(tbl_log10);
460
3
  case AMDGPULibFunc::EI_NRSQRT:
461
3
  case AMDGPULibFunc::EI_RSQRT:   return TableRef(tbl_rsqrt);
462
21
  case AMDGPULibFunc::EI_NSIN:
463
21
  case AMDGPULibFunc::EI_SIN:     return TableRef(tbl_sin);
464
21
  
case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh)0
;
465
21
  
case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi)0
;
466
21
  case AMDGPULibFunc::EI_NSQRT:
467
6
  case AMDGPULibFunc::EI_SQRT:    return TableRef(tbl_sqrt);
468
6
  
case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan)3
;
469
6
  
case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh)0
;
470
6
  
case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi)0
;
471
6
  
case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma)0
;
472
140
  default:;
473
210
  }
474
210
  
return TableRef()140
;
475
210
}
476
477
200
static inline int getVecSize(const AMDGPULibFunc& FInfo) {
478
200
  return FInfo.getLeads()[0].VectorSize;
479
200
}
480
481
125
static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) {
482
125
  return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType;
483
125
}
484
485
81
FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) {
486
81
  // If we are doing PreLinkOpt, the function is external. So it is safe to
487
81
  // use getOrInsertFunction() at this stage.
488
81
489
81
  return EnablePreLink ? 
AMDGPULibFunc::getOrInsertFunction(M, fInfo)63
490
81
                       : 
AMDGPULibFunc::getFunction(M, fInfo)18
;
491
81
}
492
493
bool AMDGPULibCalls::parseFunctionName(const StringRef& FMangledName,
494
1.04k
                                    FuncInfo *FInfo) {
495
1.04k
  return AMDGPULibFunc::parse(FMangledName, *FInfo);
496
1.04k
}
497
498
290
bool AMDGPULibCalls::isUnsafeMath(const CallInst *CI) const {
499
290
  if (auto Op = dyn_cast<FPMathOperator>(CI))
500
251
    if (Op->isFast())
501
247
      return true;
502
43
  const Function *F = CI->getParent()->getParent();
503
43
  Attribute Attr = F->getFnAttribute("unsafe-fp-math");
504
43
  return Attr.getValueAsString() == "true";
505
43
}
506
507
168
bool AMDGPULibCalls::useNativeFunc(const StringRef F) const {
508
168
  return AllNative ||
509
168
         
std::find(UseNative.begin(), UseNative.end(), F) != UseNative.end()166
;
510
168
}
511
512
166
void AMDGPULibCalls::initNativeFuncs() {
513
166
  AllNative = useNativeFunc("all") ||
514
166
              (UseNative.getNumOccurrences() && 
UseNative.size() == 11
&&
515
166
               
UseNative.begin()->empty()1
);
516
166
}
517
518
1
bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) {
519
1
  bool native_sin = useNativeFunc("sin");
520
1
  bool native_cos = useNativeFunc("cos");
521
1
522
1
  if (native_sin && native_cos) {
523
1
    Module *M = aCI->getModule();
524
1
    Value *opr0 = aCI->getArgOperand(0);
525
1
526
1
    AMDGPULibFunc nf;
527
1
    nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType;
528
1
    nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize;
529
1
530
1
    nf.setPrefix(AMDGPULibFunc::NATIVE);
531
1
    nf.setId(AMDGPULibFunc::EI_SIN);
532
1
    FunctionCallee sinExpr = getFunction(M, nf);
533
1
534
1
    nf.setPrefix(AMDGPULibFunc::NATIVE);
535
1
    nf.setId(AMDGPULibFunc::EI_COS);
536
1
    FunctionCallee cosExpr = getFunction(M, nf);
537
1
    if (sinExpr && cosExpr) {
538
1
      Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI);
539
1
      Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI);
540
1
      new StoreInst(cosval, aCI->getArgOperand(1), aCI);
541
1
542
1
      DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
543
1
                                          << " with native version of sin/cos");
544
1
545
1
      replaceCall(sinval);
546
1
      return true;
547
1
    }
548
0
  }
549
0
  return false;
550
0
}
551
552
73
bool AMDGPULibCalls::useNative(CallInst *aCI) {
553
73
  CI = aCI;
554
73
  Function *Callee = aCI->getCalledFunction();
555
73
556
73
  FuncInfo FInfo;
557
73
  if (!parseFunctionName(Callee->getName(), &FInfo) || 
!FInfo.isMangled()69
||
558
73
      
FInfo.getPrefix() != AMDGPULibFunc::NOPFX56
||
559
73
      
getArgType(FInfo) == AMDGPULibFunc::F6452
||
!HasNative(FInfo.getId())51
||
560
73
      
!(25
AllNative25
||
useNativeFunc(FInfo.getName())0
)) {
561
48
    return false;
562
48
  }
563
25
564
25
  if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS)
565
1
    return sincosUseNative(aCI, FInfo);
566
24
567
24
  FInfo.setPrefix(AMDGPULibFunc::NATIVE);
568
24
  FunctionCallee F = getFunction(aCI->getModule(), FInfo);
569
24
  if (!F)
570
0
    return false;
571
24
572
24
  aCI->setCalledFunction(F);
573
24
  DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
574
24
                                      << " with native version");
575
24
  return true;
576
24
}
577
578
// Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe
579
// builtin, with appended type size and alignment arguments, where 2 or 4
580
// indicates the original number of arguments. The library has optimized version
581
// of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same
582
// power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N
583
// for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ...,
584
// 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4.
585
bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
586
39
                                          FuncInfo &FInfo) {
587
39
  auto *Callee = CI->getCalledFunction();
588
39
  if (!Callee->isDeclaration())
589
0
    return false;
590
39
591
39
  assert(Callee->hasName() && "Invalid read_pipe/write_pipe function");
592
39
  auto *M = Callee->getParent();
593
39
  auto &Ctx = M->getContext();
594
39
  std::string Name = Callee->getName();
595
39
  auto NumArg = CI->getNumArgOperands();
596
39
  if (NumArg != 4 && 
NumArg != 66
)
597
0
    return false;
598
39
  auto *PacketSize = CI->getArgOperand(NumArg - 2);
599
39
  auto *PacketAlign = CI->getArgOperand(NumArg - 1);
600
39
  if (!isa<ConstantInt>(PacketSize) || !isa<ConstantInt>(PacketAlign))
601
0
    return false;
602
39
  unsigned Size = cast<ConstantInt>(PacketSize)->getZExtValue();
603
39
  unsigned Align = cast<ConstantInt>(PacketAlign)->getZExtValue();
604
39
  if (Size != Align || 
!isPowerOf2_32(Size)36
)
605
3
    return false;
606
36
607
36
  Type *PtrElemTy;
608
36
  if (Size <= 8)
609
24
    PtrElemTy = Type::getIntNTy(Ctx, Size * 8);
610
12
  else
611
12
    PtrElemTy = VectorType::get(Type::getInt64Ty(Ctx), Size / 8);
612
36
  unsigned PtrArgLoc = CI->getNumArgOperands() - 3;
613
36
  auto PtrArg = CI->getArgOperand(PtrArgLoc);
614
36
  unsigned PtrArgAS = PtrArg->getType()->getPointerAddressSpace();
615
36
  auto *PtrTy = llvm::PointerType::get(PtrElemTy, PtrArgAS);
616
36
617
36
  SmallVector<llvm::Type *, 6> ArgTys;
618
84
  for (unsigned I = 0; I != PtrArgLoc; 
++I48
)
619
48
    ArgTys.push_back(CI->getArgOperand(I)->getType());
620
36
  ArgTys.push_back(PtrTy);
621
36
622
36
  Name = Name + "_" + std::to_string(Size);
623
36
  auto *FTy = FunctionType::get(Callee->getReturnType(),
624
36
                                ArrayRef<Type *>(ArgTys), false);
625
36
  AMDGPULibFunc NewLibFunc(Name, FTy);
626
36
  FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, NewLibFunc);
627
36
  if (!F)
628
0
    return false;
629
36
630
36
  auto *BCast = B.CreatePointerCast(PtrArg, PtrTy);
631
36
  SmallVector<Value *, 6> Args;
632
84
  for (unsigned I = 0; I != PtrArgLoc; 
++I48
)
633
48
    Args.push_back(CI->getArgOperand(I));
634
36
  Args.push_back(BCast);
635
36
636
36
  auto *NCI = B.CreateCall(F, Args);
637
36
  NCI->setAttributes(CI->getAttributes());
638
36
  CI->replaceAllUsesWith(NCI);
639
36
  CI->dropAllReferences();
640
36
  CI->eraseFromParent();
641
36
642
36
  return true;
643
36
}
644
645
// This function returns false if no change; return true otherwise.
646
991
bool AMDGPULibCalls::fold(CallInst *CI, AliasAnalysis *AA) {
647
991
  this->CI = CI;
648
991
  Function *Callee = CI->getCalledFunction();
649
991
650
991
  // Ignore indirect calls.
651
991
  if (Callee == 0) 
return false0
;
652
991
653
991
  BasicBlock *BB = CI->getParent();
654
991
  LLVMContext &Context = CI->getParent()->getContext();
655
991
  IRBuilder<> B(Context);
656
991
657
991
  // Set the builder to the instruction after the call.
658
991
  B.SetInsertPoint(BB, CI->getIterator());
659
991
660
991
  // Copy fast flags from the original call.
661
991
  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(CI))
662
265
    B.setFastMathFlags(FPOp->getFastMathFlags());
663
991
664
991
  switch (Callee->getIntrinsicID()) {
665
991
  default:
666
973
    break;
667
991
  case Intrinsic::amdgcn_wavefrontsize:
668
18
    return !EnablePreLink && fold_wavefrontsize(CI, B);
669
973
  }
670
973
671
973
  FuncInfo FInfo;
672
973
  if (!parseFunctionName(Callee->getName(), &FInfo))
673
763
    return false;
674
210
675
210
  // Further check the number of arguments to see if they match.
676
210
  if (CI->getNumArgOperands() != FInfo.getNumArgs())
677
0
    return false;
678
210
679
210
  if (TDOFold(CI, FInfo))
680
0
    return true;
681
210
682
210
  // Under unsafe-math, evaluate calls if possible.
683
210
  // According to Brian Sumner, we can do this for all f32 function calls
684
210
  // using host's double function calls.
685
210
  if (isUnsafeMath(CI) && 
evaluateCall(CI, FInfo)167
)
686
6
    return true;
687
204
688
204
  // Specilized optimizations for each function call
689
204
  switch (FInfo.getId()) {
690
204
  case AMDGPULibFunc::EI_RECIP:
691
0
    // skip vector function
692
0
    assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE ||
693
0
             FInfo.getPrefix() == AMDGPULibFunc::HALF) &&
694
0
            "recip must be an either native or half function");
695
0
    return (getVecSize(FInfo) != 1) ? false : fold_recip(CI, B, FInfo);
696
204
697
204
  case AMDGPULibFunc::EI_DIVIDE:
698
6
    // skip vector function
699
6
    assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE ||
700
6
             FInfo.getPrefix() == AMDGPULibFunc::HALF) &&
701
6
            "divide must be an either native or half function");
702
6
    return (getVecSize(FInfo) != 1) ? 
false0
: fold_divide(CI, B, FInfo);
703
204
704
204
  case AMDGPULibFunc::EI_POW:
705
51
  case AMDGPULibFunc::EI_POWR:
706
51
  case AMDGPULibFunc::EI_POWN:
707
51
    return fold_pow(CI, B, FInfo);
708
51
709
51
  case AMDGPULibFunc::EI_ROOTN:
710
15
    // skip vector function
711
15
    return (getVecSize(FInfo) != 1) ? 
false0
: fold_rootn(CI, B, FInfo);
712
51
713
51
  case AMDGPULibFunc::EI_FMA:
714
21
  case AMDGPULibFunc::EI_MAD:
715
21
  case AMDGPULibFunc::EI_NFMA:
716
21
    // skip vector function
717
21
    return (getVecSize(FInfo) != 1) ? 
false0
: fold_fma_mad(CI, B, FInfo);
718
21
719
21
  case AMDGPULibFunc::EI_SQRT:
720
6
    return isUnsafeMath(CI) && fold_sqrt(CI, B, FInfo);
721
40
  case AMDGPULibFunc::EI_COS:
722
40
  case AMDGPULibFunc::EI_SIN:
723
40
    if ((getArgType(FInfo) == AMDGPULibFunc::F32 ||
724
40
         
getArgType(FInfo) == AMDGPULibFunc::F640
)
725
40
        && (FInfo.getPrefix() == AMDGPULibFunc::NOPFX))
726
26
      return fold_sincos(CI, B, AA);
727
14
728
14
    break;
729
39
  case AMDGPULibFunc::EI_READ_PIPE_2:
730
39
  case AMDGPULibFunc::EI_READ_PIPE_4:
731
39
  case AMDGPULibFunc::EI_WRITE_PIPE_2:
732
39
  case AMDGPULibFunc::EI_WRITE_PIPE_4:
733
39
    return fold_read_write_pipe(CI, B, FInfo);
734
39
735
39
  default:
736
26
    break;
737
40
  }
738
40
739
40
  return false;
740
40
}
741
742
210
bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) {
743
210
  // Table-Driven optimization
744
210
  const TableRef tr = getOptTable(FInfo.getId());
745
210
  if (tr.size==0)
746
140
    return false;
747
70
748
70
  int const sz = (int)tr.size;
749
70
  const TableEntry * const ftbl = tr.table;
750
70
  Value *opr0 = CI->getArgOperand(0);
751
70
752
70
  if (getVecSize(FInfo) > 1) {
753
30
    if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) {
754
0
      SmallVector<double, 0> DVal;
755
0
      for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) {
756
0
        ConstantFP *eltval = dyn_cast<ConstantFP>(
757
0
                               CV->getElementAsConstant((unsigned)eltNo));
758
0
        assert(eltval && "Non-FP arguments in math function!");
759
0
        bool found = false;
760
0
        for (int i=0; i < sz; ++i) {
761
0
          if (eltval->isExactlyValue(ftbl[i].input)) {
762
0
            DVal.push_back(ftbl[i].result);
763
0
            found = true;
764
0
            break;
765
0
          }
766
0
        }
767
0
        if (!found) {
768
0
          // This vector constants not handled yet.
769
0
          return false;
770
0
        }
771
0
      }
772
0
      LLVMContext &context = CI->getParent()->getParent()->getContext();
773
0
      Constant *nval;
774
0
      if (getArgType(FInfo) == AMDGPULibFunc::F32) {
775
0
        SmallVector<float, 0> FVal;
776
0
        for (unsigned i = 0; i < DVal.size(); ++i) {
777
0
          FVal.push_back((float)DVal[i]);
778
0
        }
779
0
        ArrayRef<float> tmp(FVal);
780
0
        nval = ConstantDataVector::get(context, tmp);
781
0
      } else { // F64
782
0
        ArrayRef<double> tmp(DVal);
783
0
        nval = ConstantDataVector::get(context, tmp);
784
0
      }
785
0
      LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
786
0
      replaceCall(nval);
787
0
      return true;
788
40
    }
789
40
  } else {
790
40
    // Scalar version
791
40
    if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) {
792
0
      for (int i = 0; i < sz; ++i) {
793
0
        if (CF->isExactlyValue(ftbl[i].input)) {
794
0
          Value *nval = ConstantFP::get(CF->getType(), ftbl[i].result);
795
0
          LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
796
0
          replaceCall(nval);
797
0
          return true;
798
0
        }
799
0
      }
800
0
    }
801
40
  }
802
70
803
70
  return false;
804
70
}
805
806
0
bool AMDGPULibCalls::replaceWithNative(CallInst *CI, const FuncInfo &FInfo) {
807
0
  Module *M = CI->getModule();
808
0
  if (getArgType(FInfo) != AMDGPULibFunc::F32 ||
809
0
      FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||
810
0
      !HasNative(FInfo.getId()))
811
0
    return false;
812
0
813
0
  AMDGPULibFunc nf = FInfo;
814
0
  nf.setPrefix(AMDGPULibFunc::NATIVE);
815
0
  if (FunctionCallee FPExpr = getFunction(M, nf)) {
816
0
    LLVM_DEBUG(dbgs() << "AMDIC: " << *CI << " ---> ");
817
0
818
0
    CI->setCalledFunction(FPExpr);
819
0
820
0
    LLVM_DEBUG(dbgs() << *CI << '\n');
821
0
822
0
    return true;
823
0
  }
824
0
  return false;
825
0
}
826
827
//  [native_]half_recip(c) ==> 1.0/c
828
bool AMDGPULibCalls::fold_recip(CallInst *CI, IRBuilder<> &B,
829
0
                                const FuncInfo &FInfo) {
830
0
  Value *opr0 = CI->getArgOperand(0);
831
0
  if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) {
832
0
    // Just create a normal div. Later, InstCombine will be able
833
0
    // to compute the divide into a constant (avoid check float infinity
834
0
    // or subnormal at this point).
835
0
    Value *nval = B.CreateFDiv(ConstantFP::get(CF->getType(), 1.0),
836
0
                               opr0,
837
0
                               "recip2div");
838
0
    LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
839
0
    replaceCall(nval);
840
0
    return true;
841
0
  }
842
0
  return false;
843
0
}
844
845
//  [native_]half_divide(x, c) ==> x/c
846
bool AMDGPULibCalls::fold_divide(CallInst *CI, IRBuilder<> &B,
847
6
                                 const FuncInfo &FInfo) {
848
6
  Value *opr0 = CI->getArgOperand(0);
849
6
  Value *opr1 = CI->getArgOperand(1);
850
6
  ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0);
851
6
  ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1);
852
6
853
6
  if ((CF0 && 
CF10
) || // both are constants
854
6
      (CF1 && (getArgType(FInfo) == AMDGPULibFunc::F32)))
855
6
      // CF1 is constant && f32 divide
856
6
  {
857
6
    Value *nval1 = B.CreateFDiv(ConstantFP::get(opr1->getType(), 1.0),
858
6
                                opr1, "__div2recip");
859
6
    Value *nval  = B.CreateFMul(opr0, nval1, "__div2mul");
860
6
    replaceCall(nval);
861
6
    return true;
862
6
  }
863
0
  return false;
864
0
}
865
866
namespace llvm {
867
0
static double log2(double V) {
868
#if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L
869
  return ::log2(V);
870
#else
871
  return log(V) / 0.693147180559945309417;
872
0
#endif
873
0
}
874
}
875
876
bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B,
877
51
                              const FuncInfo &FInfo) {
878
51
  assert((FInfo.getId() == AMDGPULibFunc::EI_POW ||
879
51
          FInfo.getId() == AMDGPULibFunc::EI_POWR ||
880
51
          FInfo.getId() == AMDGPULibFunc::EI_POWN) &&
881
51
         "fold_pow: encounter a wrong function call");
882
51
883
51
  Value *opr0, *opr1;
884
51
  ConstantFP *CF;
885
51
  ConstantInt *CINT;
886
51
  ConstantAggregateZero *CZero;
887
51
  Type *eltType;
888
51
889
51
  opr0 = CI->getArgOperand(0);
890
51
  opr1 = CI->getArgOperand(1);
891
51
  CZero = dyn_cast<ConstantAggregateZero>(opr1);
892
51
  if (getVecSize(FInfo) == 1) {
893
51
    eltType = opr0->getType();
894
51
    CF = dyn_cast<ConstantFP>(opr1);
895
51
    CINT = dyn_cast<ConstantInt>(opr1);
896
51
  } else {
897
0
    VectorType *VTy = dyn_cast<VectorType>(opr0->getType());
898
0
    assert(VTy && "Oprand of vector function should be of vectortype");
899
0
    eltType = VTy->getElementType();
900
0
    ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1);
901
0
902
0
    // Now, only Handle vector const whose elements have the same value.
903
0
    CF = CDV ? dyn_cast_or_null<ConstantFP>(CDV->getSplatValue()) : nullptr;
904
0
    CINT = CDV ? dyn_cast_or_null<ConstantInt>(CDV->getSplatValue()) : nullptr;
905
0
  }
906
51
907
51
  // No unsafe math , no constant argument, do nothing
908
51
  if (!isUnsafeMath(CI) && 
!CF0
&&
!CINT0
&&
!CZero0
)
909
0
    return false;
910
51
911
51
  // 0x1111111 means that we don't do anything for this call.
912
51
  int ci_opr1 = (CINT ? 
(int)CINT->getSExtValue()3
:
0x111111148
);
913
51
914
51
  if ((CF && 
CF->isZero()39
) ||
(45
CINT45
&&
ci_opr1 == 03
) ||
CZero45
) {
915
6
    //  pow/powr/pown(x, 0) == 1
916
6
    LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1\n");
917
6
    Constant *cnval = ConstantFP::get(eltType, 1.0);
918
6
    if (getVecSize(FInfo) > 1) {
919
0
      cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
920
0
    }
921
6
    replaceCall(cnval);
922
6
    return true;
923
6
  }
924
45
  if ((CF && 
CF->isExactlyValue(1.0)33
) ||
(39
CINT39
&&
ci_opr1 == 13
)) {
925
6
    // pow/powr/pown(x, 1.0) = x
926
6
    LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n");
927
6
    replaceCall(opr0);
928
6
    return true;
929
6
  }
930
39
  if ((CF && 
CF->isExactlyValue(2.0)27
) ||
(33
CINT33
&&
ci_opr1 == 23
)) {
931
6
    // pow/powr/pown(x, 2.0) = x*x
932
6
    LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " << *opr0
933
6
                      << "\n");
934
6
    Value *nval = B.CreateFMul(opr0, opr0, "__pow2");
935
6
    replaceCall(nval);
936
6
    return true;
937
6
  }
938
33
  if ((CF && 
CF->isExactlyValue(-1.0)21
) ||
(27
CINT27
&&
ci_opr1 == -13
)) {
939
6
    // pow/powr/pown(x, -1.0) = 1.0/x
940
6
    LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1 / " << *opr0 << "\n");
941
6
    Constant *cnval = ConstantFP::get(eltType, 1.0);
942
6
    if (getVecSize(FInfo) > 1) {
943
0
      cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
944
0
    }
945
6
    Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip");
946
6
    replaceCall(nval);
947
6
    return true;
948
6
  }
949
27
950
27
  Module *M = CI->getModule();
951
27
  if (CF && 
(15
CF->isExactlyValue(0.5)15
||
CF->isExactlyValue(-0.5)12
)) {
952
6
    // pow[r](x, [-]0.5) = sqrt(x)
953
6
    bool issqrt = CF->isExactlyValue(0.5);
954
6
    if (FunctionCallee FPExpr =
955
4
            getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT
956
4
                                                : AMDGPULibFunc::EI_RSQRT,
957
4
                                         FInfo))) {
958
4
      LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
959
4
                        << FInfo.getName().c_str() << "(" << *opr0 << ")\n");
960
4
      Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? 
"__pow2sqrt"2
961
4
                                                        : 
"__pow2rsqrt"2
);
962
4
      replaceCall(nval);
963
4
      return true;
964
4
    }
965
23
  }
966
23
967
23
  if (!isUnsafeMath(CI))
968
0
    return false;
969
23
970
23
  // Unsafe Math optimization
971
23
972
23
  // Remember that ci_opr1 is set if opr1 is integral
973
23
  if (CF) {
974
11
    double dval = (getArgType(FInfo) == AMDGPULibFunc::F32)
975
11
                    ? (double)CF->getValueAPF().convertToFloat()
976
11
                    : 
CF->getValueAPF().convertToDouble()0
;
977
11
    int ival = (int)dval;
978
11
    if ((double)ival == dval) {
979
9
      ci_opr1 = ival;
980
9
    } else
981
2
      ci_opr1 = 0x11111111;
982
11
  }
983
23
984
23
  // pow/powr/pown(x, c) = [1/](x*x*..x); where
985
23
  //   trunc(c) == c && the number of x == c && |c| <= 12
986
23
  unsigned abs_opr1 = (ci_opr1 < 0) ? 
-ci_opr10
: ci_opr1;
987
23
  if (abs_opr1 <= 12) {
988
9
    Constant *cnval;
989
9
    Value *nval;
990
9
    if (abs_opr1 == 0) {
991
0
      cnval = ConstantFP::get(eltType, 1.0);
992
0
      if (getVecSize(FInfo) > 1) {
993
0
        cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
994
0
      }
995
0
      nval = cnval;
996
9
    } else {
997
9
      Value *valx2 = nullptr;
998
9
      nval = nullptr;
999
45
      while (abs_opr1 > 0) {
1000
36
        valx2 = valx2 ? 
B.CreateFMul(valx2, valx2, "__powx2")27
:
opr09
;
1001
36
        if (abs_opr1 & 1) {
1002
27
          nval = nval ? 
B.CreateFMul(nval, valx2, "__powprod")18
:
valx29
;
1003
27
        }
1004
36
        abs_opr1 >>= 1;
1005
36
      }
1006
9
    }
1007
9
1008
9
    if (ci_opr1 < 0) {
1009
0
      cnval = ConstantFP::get(eltType, 1.0);
1010
0
      if (getVecSize(FInfo) > 1) {
1011
0
        cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
1012
0
      }
1013
0
      nval = B.CreateFDiv(cnval, nval, "__1powprod");
1014
0
    }
1015
9
    LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
1016
9
                      << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0
1017
9
                      << ")\n");
1018
9
    replaceCall(nval);
1019
9
    return true;
1020
9
  }
1021
14
1022
14
  // powr ---> exp2(y * log2(x))
1023
14
  // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31))
1024
14
  FunctionCallee ExpExpr =
1025
14
      getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo));
1026
14
  if (!ExpExpr)
1027
6
    return false;
1028
8
1029
8
  bool needlog = false;
1030
8
  bool needabs = false;
1031
8
  bool needcopysign = false;
1032
8
  Constant *cnval = nullptr;
1033
8
  if (getVecSize(FInfo) == 1) {
1034
8
    CF = dyn_cast<ConstantFP>(opr0);
1035
8
1036
8
    if (CF) {
1037
0
      double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
1038
0
                   ? (double)CF->getValueAPF().convertToFloat()
1039
0
                   : CF->getValueAPF().convertToDouble();
1040
0
1041
0
      V = log2(std::abs(V));
1042
0
      cnval = ConstantFP::get(eltType, V);
1043
0
      needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) &&
1044
0
                     CF->isNegative();
1045
8
    } else {
1046
8
      needlog = true;
1047
8
      needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1048
8
                               
(4
!CF4
||
CF->isNegative()0
);
1049
8
    }
1050
8
  } else {
1051
0
    ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr0);
1052
0
1053
0
    if (!CDV) {
1054
0
      needlog = true;
1055
0
      needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR;
1056
0
    } else {
1057
0
      assert ((int)CDV->getNumElements() == getVecSize(FInfo) &&
1058
0
              "Wrong vector size detected");
1059
0
1060
0
      SmallVector<double, 0> DVal;
1061
0
      for (int i=0; i < getVecSize(FInfo); ++i) {
1062
0
        double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
1063
0
                     ? (double)CDV->getElementAsFloat(i)
1064
0
                     : CDV->getElementAsDouble(i);
1065
0
        if (V < 0.0) needcopysign = true;
1066
0
        V = log2(std::abs(V));
1067
0
        DVal.push_back(V);
1068
0
      }
1069
0
      if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1070
0
        SmallVector<float, 0> FVal;
1071
0
        for (unsigned i=0; i < DVal.size(); ++i) {
1072
0
          FVal.push_back((float)DVal[i]);
1073
0
        }
1074
0
        ArrayRef<float> tmp(FVal);
1075
0
        cnval = ConstantDataVector::get(M->getContext(), tmp);
1076
0
      } else {
1077
0
        ArrayRef<double> tmp(DVal);
1078
0
        cnval = ConstantDataVector::get(M->getContext(), tmp);
1079
0
      }
1080
0
    }
1081
0
  }
1082
8
1083
8
  if (needcopysign && 
(FInfo.getId() == AMDGPULibFunc::EI_POW)4
) {
1084
2
    // We cannot handle corner cases for a general pow() function, give up
1085
2
    // unless y is a constant integral value. Then proceed as if it were pown.
1086
2
    if (getVecSize(FInfo) == 1) {
1087
2
      if (const ConstantFP *CF = dyn_cast<ConstantFP>(opr1)) {
1088
2
        double y = (getArgType(FInfo) == AMDGPULibFunc::F32)
1089
2
                   ? (double)CF->getValueAPF().convertToFloat()
1090
2
                   : 
CF->getValueAPF().convertToDouble()0
;
1091
2
        if (y != (double)(int64_t)y)
1092
0
          return false;
1093
0
      } else
1094
0
        return false;
1095
0
    } else {
1096
0
      if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1)) {
1097
0
        for (int i=0; i < getVecSize(FInfo); ++i) {
1098
0
          double y = (getArgType(FInfo) == AMDGPULibFunc::F32)
1099
0
                     ? (double)CDV->getElementAsFloat(i)
1100
0
                     : CDV->getElementAsDouble(i);
1101
0
          if (y != (double)(int64_t)y)
1102
0
            return false;
1103
0
        }
1104
0
      } else
1105
0
        return false;
1106
8
    }
1107
2
  }
1108
8
1109
8
  Value *nval;
1110
8
  if (needabs) {
1111
4
    FunctionCallee AbsExpr =
1112
4
        getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_FABS, FInfo));
1113
4
    if (!AbsExpr)
1114
0
      return false;
1115
4
    nval = CreateCallEx(B, AbsExpr, opr0, "__fabs");
1116
4
  } else {
1117
4
    nval = cnval ? 
cnval0
: opr0;
1118
4
  }
1119
8
  if (needlog) {
1120
8
    FunctionCallee LogExpr =
1121
8
        getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo));
1122
8
    if (!LogExpr)
1123
0
      return false;
1124
8
    nval = CreateCallEx(B,LogExpr, nval, "__log2");
1125
8
  }
1126
8
1127
8
  if (FInfo.getId() == AMDGPULibFunc::EI_POWN) {
1128
2
    // convert int(32) to fp(f32 or f64)
1129
2
    opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F");
1130
2
  }
1131
8
  nval = B.CreateFMul(opr1, nval, "__ylogx");
1132
8
  nval = CreateCallEx(B,ExpExpr, nval, "__exp2");
1133
8
1134
8
  if (needcopysign) {
1135
4
    Value *opr_n;
1136
4
    Type* rTy = opr0->getType();
1137
4
    Type* nTyS = eltType->isDoubleTy() ? 
B.getInt64Ty()0
: B.getInt32Ty();
1138
4
    Type *nTy = nTyS;
1139
4
    if (const VectorType *vTy = dyn_cast<VectorType>(rTy))
1140
0
      nTy = VectorType::get(nTyS, vTy->getNumElements());
1141
4
    unsigned size = nTy->getScalarSizeInBits();
1142
4
    opr_n = CI->getArgOperand(1);
1143
4
    if (opr_n->getType()->isIntegerTy())
1144
2
      opr_n = B.CreateZExtOrBitCast(opr_n, nTy, "__ytou");
1145
2
    else
1146
2
      opr_n = B.CreateFPToSI(opr1, nTy, "__ytou");
1147
4
1148
4
    Value *sign = B.CreateShl(opr_n, size-1, "__yeven");
1149
4
    sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign");
1150
4
    nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign);
1151
4
    nval = B.CreateBitCast(nval, opr0->getType());
1152
4
  }
1153
8
1154
8
  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
1155
8
                    << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n");
1156
8
  replaceCall(nval);
1157
8
1158
8
  return true;
1159
8
}
1160
1161
bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B,
1162
15
                                const FuncInfo &FInfo) {
1163
15
  Value *opr0 = CI->getArgOperand(0);
1164
15
  Value *opr1 = CI->getArgOperand(1);
1165
15
1166
15
  ConstantInt *CINT = dyn_cast<ConstantInt>(opr1);
1167
15
  if (!CINT) {
1168
0
    return false;
1169
0
  }
1170
15
  int ci_opr1 = (int)CINT->getSExtValue();
1171
15
  if (ci_opr1 == 1) {  // rootn(x, 1) = x
1172
3
    LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n");
1173
3
    replaceCall(opr0);
1174
3
    return true;
1175
3
  }
1176
12
  if (ci_opr1 == 2) {  // rootn(x, 2) = sqrt(x)
1177
3
    std::vector<const Type*> ParamsTys;
1178
3
    ParamsTys.push_back(opr0->getType());
1179
3
    Module *M = CI->getModule();
1180
3
    if (FunctionCallee FPExpr =
1181
2
            getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) {
1182
2
      LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> sqrt(" << *opr0 << ")\n");
1183
2
      Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt");
1184
2
      replaceCall(nval);
1185
2
      return true;
1186
2
    }
1187
9
  } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x)
1188
3
    Module *M = CI->getModule();
1189
3
    if (FunctionCallee FPExpr =
1190
2
            getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) {
1191
2
      LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> cbrt(" << *opr0 << ")\n");
1192
2
      Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt");
1193
2
      replaceCall(nval);
1194
2
      return true;
1195
2
    }
1196
6
  } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x
1197
3
    LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1.0 / " << *opr0 << "\n");
1198
3
    Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0),
1199
3
                               opr0,
1200
3
                               "__rootn2div");
1201
3
    replaceCall(nval);
1202
3
    return true;
1203
3
  } else if (ci_opr1 == -2) {  // rootn(x, -2) = rsqrt(x)
1204
3
    std::vector<const Type*> ParamsTys;
1205
3
    ParamsTys.push_back(opr0->getType());
1206
3
    Module *M = CI->getModule();
1207
3
    if (FunctionCallee FPExpr =
1208
2
            getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) {
1209
2
      LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> rsqrt(" << *opr0
1210
2
                        << ")\n");
1211
2
      Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt");
1212
2
      replaceCall(nval);
1213
2
      return true;
1214
2
    }
1215
3
  }
1216
3
  return false;
1217
3
}
1218
1219
bool AMDGPULibCalls::fold_fma_mad(CallInst *CI, IRBuilder<> &B,
1220
21
                                  const FuncInfo &FInfo) {
1221
21
  Value *opr0 = CI->getArgOperand(0);
1222
21
  Value *opr1 = CI->getArgOperand(1);
1223
21
  Value *opr2 = CI->getArgOperand(2);
1224
21
1225
21
  ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0);
1226
21
  ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1);
1227
21
  if ((CF0 && 
CF0->isZero()9
) ||
(15
CF115
&&
CF1->isZero()9
)) {
1228
12
    // fma/mad(a, b, c) = c if a=0 || b=0
1229
12
    LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr2 << "\n");
1230
12
    replaceCall(opr2);
1231
12
    return true;
1232
12
  }
1233
9
  if (CF0 && 
CF0->isExactlyValue(1.0f)3
) {
1234
3
    // fma/mad(a, b, c) = b+c if a=1
1235
3
    LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr1 << " + " << *opr2
1236
3
                      << "\n");
1237
3
    Value *nval = B.CreateFAdd(opr1, opr2, "fmaadd");
1238
3
    replaceCall(nval);
1239
3
    return true;
1240
3
  }
1241
6
  if (CF1 && 
CF1->isExactlyValue(1.0f)3
) {
1242
3
    // fma/mad(a, b, c) = a+c if b=1
1243
3
    LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " + " << *opr2
1244
3
                      << "\n");
1245
3
    Value *nval = B.CreateFAdd(opr0, opr2, "fmaadd");
1246
3
    replaceCall(nval);
1247
3
    return true;
1248
3
  }
1249
3
  if (ConstantFP *CF = dyn_cast<ConstantFP>(opr2)) {
1250
3
    if (CF->isZero()) {
1251
3
      // fma/mad(a, b, c) = a*b if c=0
1252
3
      LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * "
1253
3
                        << *opr1 << "\n");
1254
3
      Value *nval = B.CreateFMul(opr0, opr1, "fmamul");
1255
3
      replaceCall(nval);
1256
3
      return true;
1257
3
    }
1258
0
  }
1259
0
1260
0
  return false;
1261
0
}
1262
1263
// Get a scalar native builtin signle argument FP function
1264
FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M,
1265
2
                                                 const FuncInfo &FInfo) {
1266
2
  if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()))
1267
0
    return nullptr;
1268
2
  FuncInfo nf = FInfo;
1269
2
  nf.setPrefix(AMDGPULibFunc::NATIVE);
1270
2
  return getFunction(M, nf);
1271
2
}
1272
1273
// fold sqrt -> native_sqrt (x)
1274
bool AMDGPULibCalls::fold_sqrt(CallInst *CI, IRBuilder<> &B,
1275
6
                               const FuncInfo &FInfo) {
1276
6
  if (getArgType(FInfo) == AMDGPULibFunc::F32 && 
(getVecSize(FInfo) == 1)3
&&
1277
6
      
(FInfo.getPrefix() != AMDGPULibFunc::NATIVE)3
) {
1278
2
    if (FunctionCallee FPExpr = getNativeFunction(
1279
1
            CI->getModule(), AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) {
1280
1
      Value *opr0 = CI->getArgOperand(0);
1281
1
      LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
1282
1
                        << "sqrt(" << *opr0 << ")\n");
1283
1
      Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt");
1284
1
      replaceCall(nval);
1285
1
      return true;
1286
1
    }
1287
5
  }
1288
5
  return false;
1289
5
}
1290
1291
// fold sin, cos -> sincos.
1292
bool AMDGPULibCalls::fold_sincos(CallInst *CI, IRBuilder<> &B,
1293
26
                                 AliasAnalysis *AA) {
1294
26
  AMDGPULibFunc fInfo;
1295
26
  if (!AMDGPULibFunc::parse(CI->getCalledFunction()->getName(), fInfo))
1296
0
    return false;
1297
26
1298
26
  assert(fInfo.getId() == AMDGPULibFunc::EI_SIN ||
1299
26
         fInfo.getId() == AMDGPULibFunc::EI_COS);
1300
26
  bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN;
1301
26
1302
26
  Value *CArgVal = CI->getArgOperand(0);
1303
26
  BasicBlock * const CBB = CI->getParent();
1304
26
1305
26
  int const MaxScan = 30;
1306
26
1307
26
  { // fold in load value.
1308
26
    LoadInst *LI = dyn_cast<LoadInst>(CArgVal);
1309
26
    if (LI && 
LI->getParent() == CBB20
) {
1310
20
      BasicBlock::iterator BBI = LI->getIterator();
1311
20
      Value *AvailableVal = FindAvailableLoadedValue(LI, CBB, BBI, MaxScan, AA);
1312
20
      if (AvailableVal) {
1313
0
        CArgVal->replaceAllUsesWith(AvailableVal);
1314
0
        if (CArgVal->getNumUses() == 0)
1315
0
          LI->eraseFromParent();
1316
0
        CArgVal = CI->getArgOperand(0);
1317
0
      }
1318
20
    }
1319
26
  }
1320
26
1321
26
  Module *M = CI->getModule();
1322
26
  fInfo.setId(isSin ? 
AMDGPULibFunc::EI_COS14
:
AMDGPULibFunc::EI_SIN12
);
1323
26
  std::string const PairName = fInfo.mangle();
1324
26
1325
26
  CallInst *UI = nullptr;
1326
50
  for (User* U : CArgVal->users()) {
1327
50
    CallInst *XI = dyn_cast_or_null<CallInst>(U);
1328
50
    if (!XI || XI == CI || 
XI->getParent() != CBB24
)
1329
26
      continue;
1330
24
1331
24
    Function *UCallee = XI->getCalledFunction();
1332
24
    if (!UCallee || !UCallee->getName().equals(PairName))
1333
0
      continue;
1334
24
1335
24
    BasicBlock::iterator BBI = CI->getIterator();
1336
24
    if (BBI == CI->getParent()->begin())
1337
0
      break;
1338
24
    --BBI;
1339
42
    for (int I = MaxScan; I > 0 && BBI != CBB->begin(); 
--BBI, --I18
) {
1340
30
      if (cast<Instruction>(BBI) == XI) {
1341
12
        UI = XI;
1342
12
        break;
1343
12
      }
1344
30
    }
1345
24
    if (UI) 
break12
;
1346
24
  }
1347
26
1348
26
  if (!UI) 
return false14
;
1349
12
1350
12
  // Merge the sin and cos.
1351
12
1352
12
  // for OpenCL 2.0 we have only generic implementation of sincos
1353
12
  // function.
1354
12
  AMDGPULibFunc nf(AMDGPULibFunc::EI_SINCOS, fInfo);
1355
12
  nf.getLeads()[0].PtrKind = AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::FLAT_ADDRESS);
1356
12
  FunctionCallee Fsincos = getFunction(M, nf);
1357
12
  if (!Fsincos) 
return false6
;
1358
6
1359
6
  BasicBlock::iterator ItOld = B.GetInsertPoint();
1360
6
  AllocaInst *Alloc = insertAlloca(UI, B, "__sincos_");
1361
6
  B.SetInsertPoint(UI);
1362
6
1363
6
  Value *P = Alloc;
1364
6
  Type *PTy = Fsincos.getFunctionType()->getParamType(1);
1365
6
  // The allocaInst allocates the memory in private address space. This need
1366
6
  // to be bitcasted to point to the address space of cos pointer type.
1367
6
  // In OpenCL 2.0 this is generic, while in 1.2 that is private.
1368
6
  if (PTy->getPointerAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS)
1369
6
    P = B.CreateAddrSpaceCast(Alloc, PTy);
1370
6
  CallInst *Call = CreateCallEx2(B, Fsincos, UI->getArgOperand(0), P);
1371
6
1372
6
  LLVM_DEBUG(errs() << "AMDIC: fold_sincos (" << *CI << ", " << *UI << ") with "
1373
6
                    << *Call << "\n");
1374
6
1375
6
  if (!isSin) { // CI->cos, UI->sin
1376
6
    B.SetInsertPoint(&*ItOld);
1377
6
    UI->replaceAllUsesWith(&*Call);
1378
6
    Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc);
1379
6
    CI->replaceAllUsesWith(Reload);
1380
6
    UI->eraseFromParent();
1381
6
    CI->eraseFromParent();
1382
6
  } else { // CI->sin, UI->cos
1383
0
    Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc);
1384
0
    UI->replaceAllUsesWith(Reload);
1385
0
    CI->replaceAllUsesWith(Call);
1386
0
    UI->eraseFromParent();
1387
0
    CI->eraseFromParent();
1388
0
  }
1389
6
  return true;
1390
6
}
1391
1392
18
bool AMDGPULibCalls::fold_wavefrontsize(CallInst *CI, IRBuilder<> &B) {
1393
18
  if (!TM)
1394
0
    return false;
1395
18
1396
18
  StringRef CPU = TM->getTargetCPU();
1397
18
  StringRef Features = TM->getTargetFeatureString();
1398
18
  if ((CPU.empty() || CPU.equals_lower("generic")) &&
1399
18
      
(9
Features.empty()9
||
1400
9
       
Features.find_lower("wavefrontsize") == StringRef::npos6
))
1401
3
    return false;
1402
15
1403
15
  Function *F = CI->getParent()->getParent();
1404
15
  const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(*F);
1405
15
  unsigned N = ST.getWavefrontSize();
1406
15
1407
15
  LLVM_DEBUG(errs() << "AMDIC: fold_wavefrontsize (" << *CI << ") with "
1408
15
               << N << "\n");
1409
15
1410
15
  CI->replaceAllUsesWith(ConstantInt::get(B.getInt32Ty(), N));
1411
15
  CI->eraseFromParent();
1412
15
  return true;
1413
15
}
1414
1415
// Get insertion point at entry.
1416
6
BasicBlock::iterator AMDGPULibCalls::getEntryIns(CallInst * UI) {
1417
6
  Function * Func = UI->getParent()->getParent();
1418
6
  BasicBlock * BB = &Func->getEntryBlock();
1419
6
  assert(BB && "Entry block not found!");
1420
6
  BasicBlock::iterator ItNew = BB->begin();
1421
6
  return ItNew;
1422
6
}
1423
1424
// Insert a AllocsInst at the beginning of function entry block.
1425
AllocaInst* AMDGPULibCalls::insertAlloca(CallInst *UI, IRBuilder<> &B,
1426
6
                                         const char *prefix) {
1427
6
  BasicBlock::iterator ItNew = getEntryIns(UI);
1428
6
  Function *UCallee = UI->getCalledFunction();
1429
6
  Type *RetType = UCallee->getReturnType();
1430
6
  B.SetInsertPoint(&*ItNew);
1431
6
  AllocaInst *Alloc = B.CreateAlloca(RetType, 0,
1432
6
    std::string(prefix) + UI->getName());
1433
6
  Alloc->setAlignment(UCallee->getParent()->getDataLayout()
1434
6
                       .getTypeAllocSize(RetType));
1435
6
  return Alloc;
1436
6
}
1437
1438
bool AMDGPULibCalls::evaluateScalarMathFunc(FuncInfo &FInfo,
1439
                                            double& Res0, double& Res1,
1440
                                            Constant *copr0, Constant *copr1,
1441
6
                                            Constant *copr2) {
1442
6
  // By default, opr0/opr1/opr3 holds values of float/double type.
1443
6
  // If they are not float/double, each function has to its
1444
6
  // operand separately.
1445
6
  double opr0=0.0, opr1=0.0, opr2=0.0;
1446
6
  ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0);
1447
6
  ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1);
1448
6
  ConstantFP *fpopr2 = dyn_cast_or_null<ConstantFP>(copr2);
1449
6
  if (fpopr0) {
1450
6
    opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1451
6
             ? 
fpopr0->getValueAPF().convertToDouble()0
1452
6
             : (double)fpopr0->getValueAPF().convertToFloat();
1453
6
  }
1454
6
1455
6
  if (fpopr1) {
1456
0
    opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1457
0
             ? fpopr1->getValueAPF().convertToDouble()
1458
0
             : (double)fpopr1->getValueAPF().convertToFloat();
1459
0
  }
1460
6
1461
6
  if (fpopr2) {
1462
0
    opr2 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1463
0
             ? fpopr2->getValueAPF().convertToDouble()
1464
0
             : (double)fpopr2->getValueAPF().convertToFloat();
1465
0
  }
1466
6
1467
6
  switch (FInfo.getId()) {
1468
6
  
default : return false0
;
1469
6
1470
6
  case AMDGPULibFunc::EI_ACOS:
1471
0
    Res0 = acos(opr0);
1472
0
    return true;
1473
6
1474
6
  case AMDGPULibFunc::EI_ACOSH:
1475
0
    // acosh(x) == log(x + sqrt(x*x - 1))
1476
0
    Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0));
1477
0
    return true;
1478
6
1479
6
  case AMDGPULibFunc::EI_ACOSPI:
1480
0
    Res0 = acos(opr0) / MATH_PI;
1481
0
    return true;
1482
6
1483
6
  case AMDGPULibFunc::EI_ASIN:
1484
0
    Res0 = asin(opr0);
1485
0
    return true;
1486
6
1487
6
  case AMDGPULibFunc::EI_ASINH:
1488
0
    // asinh(x) == log(x + sqrt(x*x + 1))
1489
0
    Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0));
1490
0
    return true;
1491
6
1492
6
  case AMDGPULibFunc::EI_ASINPI:
1493
0
    Res0 = asin(opr0) / MATH_PI;
1494
0
    return true;
1495
6
1496
6
  case AMDGPULibFunc::EI_ATAN:
1497
0
    Res0 = atan(opr0);
1498
0
    return true;
1499
6
1500
6
  case AMDGPULibFunc::EI_ATANH:
1501
0
    // atanh(x) == (log(x+1) - log(x-1))/2;
1502
0
    Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0;
1503
0
    return true;
1504
6
1505
6
  case AMDGPULibFunc::EI_ATANPI:
1506
0
    Res0 = atan(opr0) / MATH_PI;
1507
0
    return true;
1508
6
1509
6
  case AMDGPULibFunc::EI_CBRT:
1510
0
    Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0);
1511
0
    return true;
1512
6
1513
6
  case AMDGPULibFunc::EI_COS:
1514
0
    Res0 = cos(opr0);
1515
0
    return true;
1516
6
1517
6
  case AMDGPULibFunc::EI_COSH:
1518
0
    Res0 = cosh(opr0);
1519
0
    return true;
1520
6
1521
6
  case AMDGPULibFunc::EI_COSPI:
1522
0
    Res0 = cos(MATH_PI * opr0);
1523
0
    return true;
1524
6
1525
6
  case AMDGPULibFunc::EI_EXP:
1526
0
    Res0 = exp(opr0);
1527
0
    return true;
1528
6
1529
6
  case AMDGPULibFunc::EI_EXP2:
1530
0
    Res0 = pow(2.0, opr0);
1531
0
    return true;
1532
6
1533
6
  case AMDGPULibFunc::EI_EXP10:
1534
0
    Res0 = pow(10.0, opr0);
1535
0
    return true;
1536
6
1537
6
  case AMDGPULibFunc::EI_EXPM1:
1538
0
    Res0 = exp(opr0) - 1.0;
1539
0
    return true;
1540
6
1541
6
  case AMDGPULibFunc::EI_LOG:
1542
0
    Res0 = log(opr0);
1543
0
    return true;
1544
6
1545
6
  case AMDGPULibFunc::EI_LOG2:
1546
0
    Res0 = log(opr0) / log(2.0);
1547
0
    return true;
1548
6
1549
6
  case AMDGPULibFunc::EI_LOG10:
1550
0
    Res0 = log(opr0) / log(10.0);
1551
0
    return true;
1552
6
1553
6
  case AMDGPULibFunc::EI_RSQRT:
1554
0
    Res0 = 1.0 / sqrt(opr0);
1555
0
    return true;
1556
6
1557
6
  case AMDGPULibFunc::EI_SIN:
1558
0
    Res0 = sin(opr0);
1559
0
    return true;
1560
6
1561
6
  case AMDGPULibFunc::EI_SINH:
1562
0
    Res0 = sinh(opr0);
1563
0
    return true;
1564
6
1565
6
  case AMDGPULibFunc::EI_SINPI:
1566
0
    Res0 = sin(MATH_PI * opr0);
1567
0
    return true;
1568
6
1569
6
  case AMDGPULibFunc::EI_SQRT:
1570
0
    Res0 = sqrt(opr0);
1571
0
    return true;
1572
6
1573
6
  case AMDGPULibFunc::EI_TAN:
1574
0
    Res0 = tan(opr0);
1575
0
    return true;
1576
6
1577
6
  case AMDGPULibFunc::EI_TANH:
1578
0
    Res0 = tanh(opr0);
1579
0
    return true;
1580
6
1581
6
  case AMDGPULibFunc::EI_TANPI:
1582
0
    Res0 = tan(MATH_PI * opr0);
1583
0
    return true;
1584
6
1585
6
  case AMDGPULibFunc::EI_RECIP:
1586
6
    Res0 = 1.0 / opr0;
1587
6
    return true;
1588
6
1589
6
  // two-arg functions
1590
6
  case AMDGPULibFunc::EI_DIVIDE:
1591
0
    Res0 = opr0 / opr1;
1592
0
    return true;
1593
6
1594
6
  case AMDGPULibFunc::EI_POW:
1595
0
  case AMDGPULibFunc::EI_POWR:
1596
0
    Res0 = pow(opr0, opr1);
1597
0
    return true;
1598
0
1599
0
  case AMDGPULibFunc::EI_POWN: {
1600
0
    if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {
1601
0
      double val = (double)iopr1->getSExtValue();
1602
0
      Res0 = pow(opr0, val);
1603
0
      return true;
1604
0
    }
1605
0
    return false;
1606
0
  }
1607
0
1608
0
  case AMDGPULibFunc::EI_ROOTN: {
1609
0
    if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {
1610
0
      double val = (double)iopr1->getSExtValue();
1611
0
      Res0 = pow(opr0, 1.0 / val);
1612
0
      return true;
1613
0
    }
1614
0
    return false;
1615
0
  }
1616
0
1617
0
  // with ptr arg
1618
0
  case AMDGPULibFunc::EI_SINCOS:
1619
0
    Res0 = sin(opr0);
1620
0
    Res1 = cos(opr0);
1621
0
    return true;
1622
0
1623
0
  // three-arg functions
1624
0
  case AMDGPULibFunc::EI_FMA:
1625
0
  case AMDGPULibFunc::EI_MAD:
1626
0
    Res0 = opr0 * opr1 + opr2;
1627
0
    return true;
1628
0
  }
1629
0
1630
0
  return false;
1631
0
}
1632
1633
167
bool AMDGPULibCalls::evaluateCall(CallInst *aCI, FuncInfo &FInfo) {
1634
167
  int numArgs = (int)aCI->getNumArgOperands();
1635
167
  if (numArgs > 3)
1636
0
    return false;
1637
167
1638
167
  Constant *copr0 = nullptr;
1639
167
  Constant *copr1 = nullptr;
1640
167
  Constant *copr2 = nullptr;
1641
167
  if (numArgs > 0) {
1642
167
    if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr)
1643
152
      return false;
1644
15
  }
1645
15
1646
15
  if (numArgs > 1) {
1647
9
    if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) {
1648
9
      if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS)
1649
9
        return false;
1650
6
    }
1651
9
  }
1652
6
1653
6
  if (numArgs > 2) {
1654
0
    if ((copr2 = dyn_cast<Constant>(aCI->getArgOperand(2))) == nullptr)
1655
0
      return false;
1656
6
  }
1657
6
1658
6
  // At this point, all arguments to aCI are constants.
1659
6
1660
6
  // max vector size is 16, and sincos will generate two results.
1661
6
  double DVal0[16], DVal1[16];
1662
6
  bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS);
1663
6
  if (getVecSize(FInfo) == 1) {
1664
6
    if (!evaluateScalarMathFunc(FInfo, DVal0[0],
1665
6
                                DVal1[0], copr0, copr1, copr2)) {
1666
0
      return false;
1667
0
    }
1668
0
  } else {
1669
0
    ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0);
1670
0
    ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1);
1671
0
    ConstantDataVector *CDV2 = dyn_cast_or_null<ConstantDataVector>(copr2);
1672
0
    for (int i=0; i < getVecSize(FInfo); ++i) {
1673
0
      Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr;
1674
0
      Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr;
1675
0
      Constant *celt2 = CDV2 ? CDV2->getElementAsConstant(i) : nullptr;
1676
0
      if (!evaluateScalarMathFunc(FInfo, DVal0[i],
1677
0
                                  DVal1[i], celt0, celt1, celt2)) {
1678
0
        return false;
1679
0
      }
1680
0
    }
1681
0
  }
1682
6
1683
6
  LLVMContext &context = CI->getParent()->getParent()->getContext();
1684
6
  Constant *nval0, *nval1;
1685
6
  if (getVecSize(FInfo) == 1) {
1686
6
    nval0 = ConstantFP::get(CI->getType(), DVal0[0]);
1687
6
    if (hasTwoResults)
1688
0
      nval1 = ConstantFP::get(CI->getType(), DVal1[0]);
1689
6
  } else {
1690
0
    if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1691
0
      SmallVector <float, 0> FVal0, FVal1;
1692
0
      for (int i=0; i < getVecSize(FInfo); ++i)
1693
0
        FVal0.push_back((float)DVal0[i]);
1694
0
      ArrayRef<float> tmp0(FVal0);
1695
0
      nval0 = ConstantDataVector::get(context, tmp0);
1696
0
      if (hasTwoResults) {
1697
0
        for (int i=0; i < getVecSize(FInfo); ++i)
1698
0
          FVal1.push_back((float)DVal1[i]);
1699
0
        ArrayRef<float> tmp1(FVal1);
1700
0
        nval1 = ConstantDataVector::get(context, tmp1);
1701
0
      }
1702
0
    } else {
1703
0
      ArrayRef<double> tmp0(DVal0);
1704
0
      nval0 = ConstantDataVector::get(context, tmp0);
1705
0
      if (hasTwoResults) {
1706
0
        ArrayRef<double> tmp1(DVal1);
1707
0
        nval1 = ConstantDataVector::get(context, tmp1);
1708
0
      }
1709
0
    }
1710
0
  }
1711
6
1712
6
  if (hasTwoResults) {
1713
0
    // sincos
1714
0
    assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS &&
1715
0
           "math function with ptr arg not supported yet");
1716
0
    new StoreInst(nval1, aCI->getArgOperand(1), aCI);
1717
0
  }
1718
6
1719
6
  replaceCall(nval0);
1720
6
  return true;
1721
6
}
1722
1723
// Public interface to the Simplify LibCalls pass.
1724
FunctionPass *llvm::createAMDGPUSimplifyLibCallsPass(const TargetOptions &Opt,
1725
97
                                                     const TargetMachine *TM) {
1726
97
  return new AMDGPUSimplifyLibCalls(Opt, TM);
1727
97
}
1728
1729
166
FunctionPass *llvm::createAMDGPUUseNativeCallsPass() {
1730
166
  return new AMDGPUUseNativeCalls();
1731
166
}
1732
1733
687
static bool setFastFlags(Function &F, const TargetOptions &Options) {
1734
687
  AttrBuilder B;
1735
687
1736
687
  if (Options.UnsafeFPMath || 
Options.NoInfsFPMath685
)
1737
4
    B.addAttribute("no-infs-fp-math", "true");
1738
687
  if (Options.UnsafeFPMath || 
Options.NoNaNsFPMath685
)
1739
4
    B.addAttribute("no-nans-fp-math", "true");
1740
687
  if (Options.UnsafeFPMath) {
1741
2
    B.addAttribute("less-precise-fpmad", "true");
1742
2
    B.addAttribute("unsafe-fp-math", "true");
1743
2
  }
1744
687
1745
687
  if (!B.hasAttributes())
1746
681
    return false;
1747
6
1748
6
  F.addAttributes(AttributeList::FunctionIndex, B);
1749
6
1750
6
  return true;
1751
6
}
1752
1753
793
bool AMDGPUSimplifyLibCalls::runOnFunction(Function &F) {
1754
793
  if (skipFunction(F))
1755
0
    return false;
1756
793
1757
793
  bool Changed = false;
1758
793
  auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
1759
793
1760
793
  LLVM_DEBUG(dbgs() << "AMDIC: process function ";
1761
793
             F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);
1762
793
1763
793
  if (!EnablePreLink)
1764
687
    Changed |= setFastFlags(F, Options);
1765
793
1766
978
  for (auto &BB : F) {
1767
7.53k
    for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
1768
6.55k
      // Ignore non-calls.
1769
6.55k
      CallInst *CI = dyn_cast<CallInst>(I);
1770
6.55k
      ++I;
1771
6.55k
      if (!CI) 
continue5.55k
;
1772
1.00k
1773
1.00k
      // Ignore indirect calls.
1774
1.00k
      Function *Callee = CI->getCalledFunction();
1775
1.00k
      if (Callee == 0) 
continue9
;
1776
991
1777
991
      LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << "\n";
1778
991
                 dbgs().flush());
1779
991
      if(Simplifier.fold(CI, AA))
1780
148
        Changed = true;
1781
991
    }
1782
978
  }
1783
793
  return Changed;
1784
793
}
1785
1786
1.33k
bool AMDGPUUseNativeCalls::runOnFunction(Function &F) {
1787
1.33k
  if (skipFunction(F) || 
UseNative.empty()816
)
1788
1.28k
    return false;
1789
53
1790
53
  bool Changed = false;
1791
53
  for (auto &BB : F) {
1792
336
    for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
1793
283
      // Ignore non-calls.
1794
283
      CallInst *CI = dyn_cast<CallInst>(I);
1795
283
      ++I;
1796
283
      if (!CI) 
continue210
;
1797
73
1798
73
      // Ignore indirect calls.
1799
73
      Function *Callee = CI->getCalledFunction();
1800
73
      if (Callee == 0) 
continue0
;
1801
73
1802
73
      if(Simplifier.useNative(CI))
1803
25
        Changed = true;
1804
73
    }
1805
53
  }
1806
53
  return Changed;
1807
53
}