Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/TargetLoweringBase.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This implements the TargetLoweringBase class.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "llvm/ADT/BitVector.h"
14
#include "llvm/ADT/STLExtras.h"
15
#include "llvm/ADT/SmallVector.h"
16
#include "llvm/ADT/StringExtras.h"
17
#include "llvm/ADT/StringRef.h"
18
#include "llvm/ADT/Triple.h"
19
#include "llvm/ADT/Twine.h"
20
#include "llvm/CodeGen/Analysis.h"
21
#include "llvm/CodeGen/ISDOpcodes.h"
22
#include "llvm/CodeGen/MachineBasicBlock.h"
23
#include "llvm/CodeGen/MachineFrameInfo.h"
24
#include "llvm/CodeGen/MachineFunction.h"
25
#include "llvm/CodeGen/MachineInstr.h"
26
#include "llvm/CodeGen/MachineInstrBuilder.h"
27
#include "llvm/CodeGen/MachineMemOperand.h"
28
#include "llvm/CodeGen/MachineOperand.h"
29
#include "llvm/CodeGen/MachineRegisterInfo.h"
30
#include "llvm/CodeGen/RuntimeLibcalls.h"
31
#include "llvm/CodeGen/StackMaps.h"
32
#include "llvm/CodeGen/TargetLowering.h"
33
#include "llvm/CodeGen/TargetOpcodes.h"
34
#include "llvm/CodeGen/TargetRegisterInfo.h"
35
#include "llvm/CodeGen/ValueTypes.h"
36
#include "llvm/IR/Attributes.h"
37
#include "llvm/IR/CallingConv.h"
38
#include "llvm/IR/DataLayout.h"
39
#include "llvm/IR/DerivedTypes.h"
40
#include "llvm/IR/Function.h"
41
#include "llvm/IR/GlobalValue.h"
42
#include "llvm/IR/GlobalVariable.h"
43
#include "llvm/IR/IRBuilder.h"
44
#include "llvm/IR/Module.h"
45
#include "llvm/IR/Type.h"
46
#include "llvm/Support/BranchProbability.h"
47
#include "llvm/Support/Casting.h"
48
#include "llvm/Support/CommandLine.h"
49
#include "llvm/Support/Compiler.h"
50
#include "llvm/Support/ErrorHandling.h"
51
#include "llvm/Support/MachineValueType.h"
52
#include "llvm/Support/MathExtras.h"
53
#include "llvm/Target/TargetMachine.h"
54
#include <algorithm>
55
#include <cassert>
56
#include <cstddef>
57
#include <cstdint>
58
#include <cstring>
59
#include <iterator>
60
#include <string>
61
#include <tuple>
62
#include <utility>
63
64
using namespace llvm;
65
66
static cl::opt<bool> JumpIsExpensiveOverride(
67
    "jump-is-expensive", cl::init(false),
68
    cl::desc("Do not create extra branches to split comparison logic."),
69
    cl::Hidden);
70
71
static cl::opt<unsigned> MinimumJumpTableEntries
72
  ("min-jump-table-entries", cl::init(4), cl::Hidden,
73
   cl::desc("Set minimum number of entries to use a jump table."));
74
75
static cl::opt<unsigned> MaximumJumpTableSize
76
  ("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden,
77
   cl::desc("Set maximum size of jump tables."));
78
79
/// Minimum jump table density for normal functions.
80
static cl::opt<unsigned>
81
    JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden,
82
                     cl::desc("Minimum density for building a jump table in "
83
                              "a normal function"));
84
85
/// Minimum jump table density for -Os or -Oz functions.
86
static cl::opt<unsigned> OptsizeJumpTableDensity(
87
    "optsize-jump-table-density", cl::init(40), cl::Hidden,
88
    cl::desc("Minimum density for building a jump table in "
89
             "an optsize function"));
90
91
26.3k
static bool darwinHasSinCos(const Triple &TT) {
92
26.3k
  assert(TT.isOSDarwin() && "should be called with darwin triple");
93
26.3k
  // Don't bother with 32 bit x86.
94
26.3k
  if (TT.getArch() == Triple::x86)
95
1.05k
    return false;
96
25.2k
  // Macos < 10.9 has no sincos_stret.
97
25.2k
  if (TT.isMacOSX())
98
16.0k
    return !TT.isMacOSXVersionLT(10, 9) && 
TT.isArch64Bit()13.1k
;
99
9.21k
  // iOS < 7.0 has no sincos_stret.
100
9.21k
  if (TT.isiOS())
101
9.19k
    return !TT.isOSVersionLT(7, 0);
102
21
  // Any other darwin such as WatchOS/TvOS is new enough.
103
21
  return true;
104
21
}
105
106
// Although this default value is arbitrary, it is not random. It is assumed
107
// that a condition that evaluates the same way by a higher percentage than this
108
// is best represented as control flow. Therefore, the default value N should be
109
// set such that the win from N% correct executions is greater than the loss
110
// from (100 - N)% mispredicted executions for the majority of intended targets.
111
static cl::opt<int> MinPercentageForPredictableBranch(
112
    "min-predictable-branch", cl::init(99),
113
    cl::desc("Minimum percentage (0-100) that a condition must be either true "
114
             "or false to assume that the condition is predictable"),
115
    cl::Hidden);
116
117
53.2k
void TargetLoweringBase::InitLibcalls(const Triple &TT) {
118
53.2k
#define HANDLE_LIBCALL(code, name) \
119
26.2M
  setLibcallName(RTLIB::code, name);
120
53.2k
#include "llvm/IR/RuntimeLibcalls.def"
121
53.2k
#undef HANDLE_LIBCALL
122
53.2k
  // Initialize calling conventions to their default.
123
26.2M
  for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; 
++LC26.1M
)
124
26.1M
    setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C);
125
53.2k
126
53.2k
  // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
127
53.2k
  if (TT.getArch() == Triple::ppc || 
TT.isPPC64()52.8k
) {
128
1.85k
    setLibcallName(RTLIB::ADD_F128, "__addkf3");
129
1.85k
    setLibcallName(RTLIB::SUB_F128, "__subkf3");
130
1.85k
    setLibcallName(RTLIB::MUL_F128, "__mulkf3");
131
1.85k
    setLibcallName(RTLIB::DIV_F128, "__divkf3");
132
1.85k
    setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2");
133
1.85k
    setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2");
134
1.85k
    setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2");
135
1.85k
    setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2");
136
1.85k
    setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi");
137
1.85k
    setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi");
138
1.85k
    setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi");
139
1.85k
    setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi");
140
1.85k
    setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf");
141
1.85k
    setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf");
142
1.85k
    setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf");
143
1.85k
    setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf");
144
1.85k
    setLibcallName(RTLIB::OEQ_F128, "__eqkf2");
145
1.85k
    setLibcallName(RTLIB::UNE_F128, "__nekf2");
146
1.85k
    setLibcallName(RTLIB::OGE_F128, "__gekf2");
147
1.85k
    setLibcallName(RTLIB::OLT_F128, "__ltkf2");
148
1.85k
    setLibcallName(RTLIB::OLE_F128, "__lekf2");
149
1.85k
    setLibcallName(RTLIB::OGT_F128, "__gtkf2");
150
1.85k
    setLibcallName(RTLIB::UO_F128, "__unordkf2");
151
1.85k
    setLibcallName(RTLIB::O_F128, "__unordkf2");
152
1.85k
  }
153
53.2k
154
53.2k
  // A few names are different on particular architectures or environments.
155
53.2k
  if (TT.isOSDarwin()) {
156
26.3k
    // For f16/f32 conversions, Darwin uses the standard naming scheme, instead
157
26.3k
    // of the gnueabi-style __gnu_*_ieee.
158
26.3k
    // FIXME: What about other targets?
159
26.3k
    setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
160
26.3k
    setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
161
26.3k
162
26.3k
    // Some darwins have an optimized __bzero/bzero function.
163
26.3k
    switch (TT.getArch()) {
164
26.3k
    case Triple::x86:
165
6.50k
    case Triple::x86_64:
166
6.50k
      if (TT.isMacOSX() && 
!TT.isMacOSXVersionLT(10, 6)5.39k
)
167
3.16k
        setLibcallName(RTLIB::BZERO, "__bzero");
168
6.50k
      break;
169
7.51k
    case Triple::aarch64:
170
7.51k
      setLibcallName(RTLIB::BZERO, "bzero");
171
7.51k
      break;
172
12.3k
    default:
173
12.3k
      break;
174
26.3k
    }
175
26.3k
176
26.3k
    if (darwinHasSinCos(TT)) {
177
14.7k
      setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret");
178
14.7k
      setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret");
179
14.7k
      if (TT.isWatchABI()) {
180
156
        setLibcallCallingConv(RTLIB::SINCOS_STRET_F32,
181
156
                              CallingConv::ARM_AAPCS_VFP);
182
156
        setLibcallCallingConv(RTLIB::SINCOS_STRET_F64,
183
156
                              CallingConv::ARM_AAPCS_VFP);
184
156
      }
185
14.7k
    }
186
26.9k
  } else {
187
26.9k
    setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee");
188
26.9k
    setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee");
189
26.9k
  }
190
53.2k
191
53.2k
  
if (53.2k
TT.isGNUEnvironment()53.2k
||
TT.isOSFuchsia()42.0k
||
192
53.2k
      
(42.0k
TT.isAndroid()42.0k
&&
!TT.isAndroidVersionLT(9)218
)) {
193
11.3k
    setLibcallName(RTLIB::SINCOS_F32, "sincosf");
194
11.3k
    setLibcallName(RTLIB::SINCOS_F64, "sincos");
195
11.3k
    setLibcallName(RTLIB::SINCOS_F80, "sincosl");
196
11.3k
    setLibcallName(RTLIB::SINCOS_F128, "sincosl");
197
11.3k
    setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl");
198
11.3k
  }
199
53.2k
200
53.2k
  if (TT.isOSOpenBSD()) {
201
7
    setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr);
202
7
  }
203
53.2k
}
204
205
/// getFPEXT - Return the FPEXT_*_* value for the given types, or
206
/// UNKNOWN_LIBCALL if there is none.
207
164
RTLIB::Libcall RTLIB::getFPEXT(EVT OpVT, EVT RetVT) {
208
164
  if (OpVT == MVT::f16) {
209
0
    if (RetVT == MVT::f32)
210
0
      return FPEXT_F16_F32;
211
164
  } else if (OpVT == MVT::f32) {
212
135
    if (RetVT == MVT::f64)
213
115
      return FPEXT_F32_F64;
214
20
    if (RetVT == MVT::f128)
215
20
      return FPEXT_F32_F128;
216
0
    if (RetVT == MVT::ppcf128)
217
0
      return FPEXT_F32_PPCF128;
218
29
  } else if (OpVT == MVT::f64) {
219
25
    if (RetVT == MVT::f128)
220
24
      return FPEXT_F64_F128;
221
1
    else if (RetVT == MVT::ppcf128)
222
1
      return FPEXT_F64_PPCF128;
223
4
  } else if (OpVT == MVT::f80) {
224
4
    if (RetVT == MVT::f128)
225
4
      return FPEXT_F80_F128;
226
0
  }
227
0
228
0
  return UNKNOWN_LIBCALL;
229
0
}
230
231
/// getFPROUND - Return the FPROUND_*_* value for the given types, or
232
/// UNKNOWN_LIBCALL if there is none.
233
786
RTLIB::Libcall RTLIB::getFPROUND(EVT OpVT, EVT RetVT) {
234
786
  if (RetVT == MVT::f16) {
235
715
    if (OpVT == MVT::f32)
236
414
      return FPROUND_F32_F16;
237
301
    if (OpVT == MVT::f64)
238
296
      return FPROUND_F64_F16;
239
5
    if (OpVT == MVT::f80)
240
3
      return FPROUND_F80_F16;
241
2
    if (OpVT == MVT::f128)
242
2
      return FPROUND_F128_F16;
243
0
    if (OpVT == MVT::ppcf128)
244
0
      return FPROUND_PPCF128_F16;
245
71
  } else if (RetVT == MVT::f32) {
246
42
    if (OpVT == MVT::f64)
247
24
      return FPROUND_F64_F32;
248
18
    if (OpVT == MVT::f80)
249
0
      return FPROUND_F80_F32;
250
18
    if (OpVT == MVT::f128)
251
17
      return FPROUND_F128_F32;
252
1
    if (OpVT == MVT::ppcf128)
253
1
      return FPROUND_PPCF128_F32;
254
29
  } else if (RetVT == MVT::f64) {
255
25
    if (OpVT == MVT::f80)
256
0
      return FPROUND_F80_F64;
257
25
    if (OpVT == MVT::f128)
258
24
      return FPROUND_F128_F64;
259
1
    if (OpVT == MVT::ppcf128)
260
1
      return FPROUND_PPCF128_F64;
261
4
  } else if (RetVT == MVT::f80) {
262
4
    if (OpVT == MVT::f128)
263
4
      return FPROUND_F128_F80;
264
0
  }
265
0
266
0
  return UNKNOWN_LIBCALL;
267
0
}
268
269
/// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
270
/// UNKNOWN_LIBCALL if there is none.
271
310
RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) {
272
310
  if (OpVT == MVT::f32) {
273
100
    if (RetVT == MVT::i32)
274
49
      return FPTOSINT_F32_I32;
275
51
    if (RetVT == MVT::i64)
276
42
      return FPTOSINT_F32_I64;
277
9
    if (RetVT == MVT::i128)
278
3
      return FPTOSINT_F32_I128;
279
210
  } else if (OpVT == MVT::f64) {
280
98
    if (RetVT == MVT::i32)
281
47
      return FPTOSINT_F64_I32;
282
51
    if (RetVT == MVT::i64)
283
51
      return FPTOSINT_F64_I64;
284
0
    if (RetVT == MVT::i128)
285
0
      return FPTOSINT_F64_I128;
286
112
  } else if (OpVT == MVT::f80) {
287
0
    if (RetVT == MVT::i32)
288
0
      return FPTOSINT_F80_I32;
289
0
    if (RetVT == MVT::i64)
290
0
      return FPTOSINT_F80_I64;
291
0
    if (RetVT == MVT::i128)
292
0
      return FPTOSINT_F80_I128;
293
112
  } else if (OpVT == MVT::f128) {
294
110
    if (RetVT == MVT::i32)
295
59
      return FPTOSINT_F128_I32;
296
51
    if (RetVT == MVT::i64)
297
48
      return FPTOSINT_F128_I64;
298
3
    if (RetVT == MVT::i128)
299
3
      return FPTOSINT_F128_I128;
300
2
  } else if (OpVT == MVT::ppcf128) {
301
2
    if (RetVT == MVT::i32)
302
0
      return FPTOSINT_PPCF128_I32;
303
2
    if (RetVT == MVT::i64)
304
2
      return FPTOSINT_PPCF128_I64;
305
0
    if (RetVT == MVT::i128)
306
0
      return FPTOSINT_PPCF128_I128;
307
6
  }
308
6
  return UNKNOWN_LIBCALL;
309
6
}
310
311
/// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
312
/// UNKNOWN_LIBCALL if there is none.
313
252
RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) {
314
252
  if (OpVT == MVT::f32) {
315
77
    if (RetVT == MVT::i32)
316
32
      return FPTOUINT_F32_I32;
317
45
    if (RetVT == MVT::i64)
318
39
      return FPTOUINT_F32_I64;
319
6
    if (RetVT == MVT::i128)
320
0
      return FPTOUINT_F32_I128;
321
175
  } else if (OpVT == MVT::f64) {
322
118
    if (RetVT == MVT::i32)
323
72
      return FPTOUINT_F64_I32;
324
46
    if (RetVT == MVT::i64)
325
44
      return FPTOUINT_F64_I64;
326
2
    if (RetVT == MVT::i128)
327
2
      return FPTOUINT_F64_I128;
328
57
  } else if (OpVT == MVT::f80) {
329
1
    if (RetVT == MVT::i32)
330
0
      return FPTOUINT_F80_I32;
331
1
    if (RetVT == MVT::i64)
332
0
      return FPTOUINT_F80_I64;
333
1
    if (RetVT == MVT::i128)
334
1
      return FPTOUINT_F80_I128;
335
56
  } else if (OpVT == MVT::f128) {
336
54
    if (RetVT == MVT::i32)
337
18
      return FPTOUINT_F128_I32;
338
36
    if (RetVT == MVT::i64)
339
33
      return FPTOUINT_F128_I64;
340
3
    if (RetVT == MVT::i128)
341
3
      return FPTOUINT_F128_I128;
342
2
  } else if (OpVT == MVT::ppcf128) {
343
2
    if (RetVT == MVT::i32)
344
0
      return FPTOUINT_PPCF128_I32;
345
2
    if (RetVT == MVT::i64)
346
1
      return FPTOUINT_PPCF128_I64;
347
1
    if (RetVT == MVT::i128)
348
1
      return FPTOUINT_PPCF128_I128;
349
6
  }
350
6
  return UNKNOWN_LIBCALL;
351
6
}
352
353
/// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
354
/// UNKNOWN_LIBCALL if there is none.
355
268
RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) {
356
268
  if (OpVT == MVT::i32) {
357
137
    if (RetVT == MVT::f32)
358
47
      return SINTTOFP_I32_F32;
359
90
    if (RetVT == MVT::f64)
360
63
      return SINTTOFP_I32_F64;
361
27
    if (RetVT == MVT::f80)
362
0
      return SINTTOFP_I32_F80;
363
27
    if (RetVT == MVT::f128)
364
26
      return SINTTOFP_I32_F128;
365
1
    if (RetVT == MVT::ppcf128)
366
1
      return SINTTOFP_I32_PPCF128;
367
131
  } else if (OpVT == MVT::i64) {
368
116
    if (RetVT == MVT::f32)
369
43
      return SINTTOFP_I64_F32;
370
73
    if (RetVT == MVT::f64)
371
52
      return SINTTOFP_I64_F64;
372
21
    if (RetVT == MVT::f80)
373
0
      return SINTTOFP_I64_F80;
374
21
    if (RetVT == MVT::f128)
375
21
      return SINTTOFP_I64_F128;
376
0
    if (RetVT == MVT::ppcf128)
377
0
      return SINTTOFP_I64_PPCF128;
378
15
  } else if (OpVT == MVT::i128) {
379
3
    if (RetVT == MVT::f32)
380
1
      return SINTTOFP_I128_F32;
381
2
    if (RetVT == MVT::f64)
382
2
      return SINTTOFP_I128_F64;
383
0
    if (RetVT == MVT::f80)
384
0
      return SINTTOFP_I128_F80;
385
0
    if (RetVT == MVT::f128)
386
0
      return SINTTOFP_I128_F128;
387
0
    if (RetVT == MVT::ppcf128)
388
0
      return SINTTOFP_I128_PPCF128;
389
12
  }
390
12
  return UNKNOWN_LIBCALL;
391
12
}
392
393
/// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
394
/// UNKNOWN_LIBCALL if there is none.
395
558
RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) {
396
558
  if (OpVT == MVT::i32) {
397
194
    if (RetVT == MVT::f32)
398
81
      return UINTTOFP_I32_F32;
399
113
    if (RetVT == MVT::f64)
400
91
      return UINTTOFP_I32_F64;
401
22
    if (RetVT == MVT::f80)
402
0
      return UINTTOFP_I32_F80;
403
22
    if (RetVT == MVT::f128)
404
21
      return UINTTOFP_I32_F128;
405
1
    if (RetVT == MVT::ppcf128)
406
1
      return UINTTOFP_I32_PPCF128;
407
364
  } else if (OpVT == MVT::i64) {
408
102
    if (RetVT == MVT::f32)
409
40
      return UINTTOFP_I64_F32;
410
62
    if (RetVT == MVT::f64)
411
43
      return UINTTOFP_I64_F64;
412
19
    if (RetVT == MVT::f80)
413
0
      return UINTTOFP_I64_F80;
414
19
    if (RetVT == MVT::f128)
415
19
      return UINTTOFP_I64_F128;
416
0
    if (RetVT == MVT::ppcf128)
417
0
      return UINTTOFP_I64_PPCF128;
418
262
  } else if (OpVT == MVT::i128) {
419
4
    if (RetVT == MVT::f32)
420
2
      return UINTTOFP_I128_F32;
421
2
    if (RetVT == MVT::f64)
422
0
      return UINTTOFP_I128_F64;
423
2
    if (RetVT == MVT::f80)
424
0
      return UINTTOFP_I128_F80;
425
2
    if (RetVT == MVT::f128)
426
2
      return UINTTOFP_I128_F128;
427
0
    if (RetVT == MVT::ppcf128)
428
0
      return UINTTOFP_I128_PPCF128;
429
258
  }
430
258
  return UNKNOWN_LIBCALL;
431
258
}
432
433
172
RTLIB::Libcall RTLIB::getSYNC(unsigned Opc, MVT VT) {
434
172
#define OP_TO_LIBCALL(Name, Enum)                                              \
435
172
  case Name:                                                                   \
436
172
    switch (VT.SimpleTy) {                                                     \
437
172
    default:                                                                   \
438
0
      return UNKNOWN_LIBCALL;                                                  \
439
172
    case MVT::i8:                                                              \
440
28
      return Enum##_1;                                                         \
441
172
    case MVT::i16:                                                             \
442
9
      return Enum##_2;                                                         \
443
172
    case MVT::i32:                                                             \
444
49
      return Enum##_4;                                                         \
445
172
    case MVT::i64:                                                             \
446
30
      return Enum##_8;                                                         \
447
172
    case MVT::i128:                                                            \
448
56
      return Enum##_16;                                                        \
449
172
    }
450
172
451
172
  switch (Opc) {
452
172
    
OP_TO_LIBCALL44
(ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET)
453
46
    OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP)
454
46
    
OP_TO_LIBCALL16
(ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD)
455
16
    
OP_TO_LIBCALL7
(ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB)
456
7
    
OP_TO_LIBCALL4
(ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND)
457
9
    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR)
458
9
    
OP_TO_LIBCALL3
(ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR)
459
3
    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND)
460
6
    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX)
461
14
    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX)
462
14
    
OP_TO_LIBCALL6
(ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN)
463
14
    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN)
464
172
  }
465
172
466
172
#undef OP_TO_LIBCALL
467
172
468
172
  
return UNKNOWN_LIBCALL0
;
469
172
}
470
471
15
RTLIB::Libcall RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
472
15
  switch (ElementSize) {
473
15
  case 1:
474
6
    return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1;
475
15
  case 2:
476
2
    return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2;
477
15
  case 4:
478
3
    return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4;
479
15
  case 8:
480
2
    return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8;
481
15
  case 16:
482
2
    return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16;
483
15
  default:
484
0
    return UNKNOWN_LIBCALL;
485
15
  }
486
15
}
487
488
13
RTLIB::Libcall RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
489
13
  switch (ElementSize) {
490
13
  case 1:
491
4
    return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1;
492
13
  case 2:
493
2
    return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2;
494
13
  case 4:
495
3
    return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4;
496
13
  case 8:
497
2
    return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8;
498
13
  case 16:
499
2
    return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16;
500
13
  default:
501
0
    return UNKNOWN_LIBCALL;
502
13
  }
503
13
}
504
505
23
RTLIB::Libcall RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
506
23
  switch (ElementSize) {
507
23
  case 1:
508
3
    return MEMSET_ELEMENT_UNORDERED_ATOMIC_1;
509
23
  case 2:
510
2
    return MEMSET_ELEMENT_UNORDERED_ATOMIC_2;
511
23
  case 4:
512
13
    return MEMSET_ELEMENT_UNORDERED_ATOMIC_4;
513
23
  case 8:
514
2
    return MEMSET_ELEMENT_UNORDERED_ATOMIC_8;
515
23
  case 16:
516
3
    return MEMSET_ELEMENT_UNORDERED_ATOMIC_16;
517
23
  default:
518
0
    return UNKNOWN_LIBCALL;
519
23
  }
520
23
}
521
522
/// InitCmpLibcallCCs - Set default comparison libcall CC.
523
53.2k
static void InitCmpLibcallCCs(ISD::CondCode *CCs) {
524
53.2k
  memset(CCs, ISD::SETCC_INVALID, sizeof(ISD::CondCode)*RTLIB::UNKNOWN_LIBCALL);
525
53.2k
  CCs[RTLIB::OEQ_F32] = ISD::SETEQ;
526
53.2k
  CCs[RTLIB::OEQ_F64] = ISD::SETEQ;
527
53.2k
  CCs[RTLIB::OEQ_F128] = ISD::SETEQ;
528
53.2k
  CCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ;
529
53.2k
  CCs[RTLIB::UNE_F32] = ISD::SETNE;
530
53.2k
  CCs[RTLIB::UNE_F64] = ISD::SETNE;
531
53.2k
  CCs[RTLIB::UNE_F128] = ISD::SETNE;
532
53.2k
  CCs[RTLIB::UNE_PPCF128] = ISD::SETNE;
533
53.2k
  CCs[RTLIB::OGE_F32] = ISD::SETGE;
534
53.2k
  CCs[RTLIB::OGE_F64] = ISD::SETGE;
535
53.2k
  CCs[RTLIB::OGE_F128] = ISD::SETGE;
536
53.2k
  CCs[RTLIB::OGE_PPCF128] = ISD::SETGE;
537
53.2k
  CCs[RTLIB::OLT_F32] = ISD::SETLT;
538
53.2k
  CCs[RTLIB::OLT_F64] = ISD::SETLT;
539
53.2k
  CCs[RTLIB::OLT_F128] = ISD::SETLT;
540
53.2k
  CCs[RTLIB::OLT_PPCF128] = ISD::SETLT;
541
53.2k
  CCs[RTLIB::OLE_F32] = ISD::SETLE;
542
53.2k
  CCs[RTLIB::OLE_F64] = ISD::SETLE;
543
53.2k
  CCs[RTLIB::OLE_F128] = ISD::SETLE;
544
53.2k
  CCs[RTLIB::OLE_PPCF128] = ISD::SETLE;
545
53.2k
  CCs[RTLIB::OGT_F32] = ISD::SETGT;
546
53.2k
  CCs[RTLIB::OGT_F64] = ISD::SETGT;
547
53.2k
  CCs[RTLIB::OGT_F128] = ISD::SETGT;
548
53.2k
  CCs[RTLIB::OGT_PPCF128] = ISD::SETGT;
549
53.2k
  CCs[RTLIB::UO_F32] = ISD::SETNE;
550
53.2k
  CCs[RTLIB::UO_F64] = ISD::SETNE;
551
53.2k
  CCs[RTLIB::UO_F128] = ISD::SETNE;
552
53.2k
  CCs[RTLIB::UO_PPCF128] = ISD::SETNE;
553
53.2k
  CCs[RTLIB::O_F32] = ISD::SETEQ;
554
53.2k
  CCs[RTLIB::O_F64] = ISD::SETEQ;
555
53.2k
  CCs[RTLIB::O_F128] = ISD::SETEQ;
556
53.2k
  CCs[RTLIB::O_PPCF128] = ISD::SETEQ;
557
53.2k
}
558
559
/// NOTE: The TargetMachine owns TLOF.
560
53.2k
TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
561
53.2k
  initActions();
562
53.2k
563
53.2k
  // Perform these initializations only once.
564
53.2k
  MaxStoresPerMemset = MaxStoresPerMemcpy = MaxStoresPerMemmove =
565
53.2k
      MaxLoadsPerMemcmp = 8;
566
53.2k
  MaxGluedStoresPerMemcpy = 0;
567
53.2k
  MaxStoresPerMemsetOptSize = MaxStoresPerMemcpyOptSize =
568
53.2k
      MaxStoresPerMemmoveOptSize = MaxLoadsPerMemcmpOptSize = 4;
569
53.2k
  UseUnderscoreSetJmp = false;
570
53.2k
  UseUnderscoreLongJmp = false;
571
53.2k
  HasMultipleConditionRegisters = false;
572
53.2k
  HasExtractBitsInsn = false;
573
53.2k
  JumpIsExpensive = JumpIsExpensiveOverride;
574
53.2k
  PredictableSelectIsExpensive = false;
575
53.2k
  EnableExtLdPromotion = false;
576
53.2k
  StackPointerRegisterToSaveRestore = 0;
577
53.2k
  BooleanContents = UndefinedBooleanContent;
578
53.2k
  BooleanFloatContents = UndefinedBooleanContent;
579
53.2k
  BooleanVectorContents = UndefinedBooleanContent;
580
53.2k
  SchedPreferenceInfo = Sched::ILP;
581
53.2k
  JumpBufSize = 0;
582
53.2k
  JumpBufAlignment = 0;
583
53.2k
  MinFunctionAlignment = 0;
584
53.2k
  PrefFunctionAlignment = 0;
585
53.2k
  PrefLoopAlignment = 0;
586
53.2k
  GatherAllAliasesMaxDepth = 18;
587
53.2k
  MinStackArgumentAlignment = 1;
588
53.2k
  // TODO: the default will be switched to 0 in the next commit, along
589
53.2k
  // with the Target-specific changes necessary.
590
53.2k
  MaxAtomicSizeInBitsSupported = 1024;
591
53.2k
592
53.2k
  MinCmpXchgSizeInBits = 0;
593
53.2k
  SupportsUnalignedAtomics = false;
594
53.2k
595
53.2k
  std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr);
596
53.2k
597
53.2k
  InitLibcalls(TM.getTargetTriple());
598
53.2k
  InitCmpLibcallCCs(CmpLibcallCCs);
599
53.2k
}
600
601
53.2k
void TargetLoweringBase::initActions() {
602
53.2k
  // All operations default to being supported.
603
53.2k
  memset(OpActions, 0, sizeof(OpActions));
604
53.2k
  memset(LoadExtActions, 0, sizeof(LoadExtActions));
605
53.2k
  memset(TruncStoreActions, 0, sizeof(TruncStoreActions));
606
53.2k
  memset(IndexedModeActions, 0, sizeof(IndexedModeActions));
607
53.2k
  memset(CondCodeActions, 0, sizeof(CondCodeActions));
608
53.2k
  std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr);
609
53.2k
  std::fill(std::begin(TargetDAGCombineArray),
610
53.2k
            std::end(TargetDAGCombineArray), 0);
611
53.2k
612
319k
  for (MVT VT : MVT::fp_valuetypes()) {
613
319k
    MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits());
614
319k
    if (IntVT.isValid()) {
615
266k
      setOperationAction(ISD::ATOMIC_SWAP, VT, Promote);
616
266k
      AddPromotedToType(ISD::ATOMIC_SWAP, VT, IntVT);
617
266k
    }
618
319k
  }
619
53.2k
620
53.2k
  // Set default actions for various operations.
621
6.87M
  for (MVT VT : MVT::all_valuetypes()) {
622
6.87M
    // Default all indexed load / store to expand.
623
6.87M
    for (unsigned IM = (unsigned)ISD::PRE_INC;
624
34.3M
         IM != (unsigned)ISD::LAST_INDEXED_MODE; 
++IM27.4M
) {
625
27.4M
      setIndexedLoadAction(IM, VT, Expand);
626
27.4M
      setIndexedStoreAction(IM, VT, Expand);
627
27.4M
    }
628
6.87M
629
6.87M
    // Most backends expect to see the node which just returns the value loaded.
630
6.87M
    setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Expand);
631
6.87M
632
6.87M
    // These operations default to expand.
633
6.87M
    setOperationAction(ISD::FGETSIGN, VT, Expand);
634
6.87M
    setOperationAction(ISD::CONCAT_VECTORS, VT, Expand);
635
6.87M
    setOperationAction(ISD::FMINNUM, VT, Expand);
636
6.87M
    setOperationAction(ISD::FMAXNUM, VT, Expand);
637
6.87M
    setOperationAction(ISD::FMINNUM_IEEE, VT, Expand);
638
6.87M
    setOperationAction(ISD::FMAXNUM_IEEE, VT, Expand);
639
6.87M
    setOperationAction(ISD::FMINIMUM, VT, Expand);
640
6.87M
    setOperationAction(ISD::FMAXIMUM, VT, Expand);
641
6.87M
    setOperationAction(ISD::FMAD, VT, Expand);
642
6.87M
    setOperationAction(ISD::SMIN, VT, Expand);
643
6.87M
    setOperationAction(ISD::SMAX, VT, Expand);
644
6.87M
    setOperationAction(ISD::UMIN, VT, Expand);
645
6.87M
    setOperationAction(ISD::UMAX, VT, Expand);
646
6.87M
    setOperationAction(ISD::ABS, VT, Expand);
647
6.87M
    setOperationAction(ISD::FSHL, VT, Expand);
648
6.87M
    setOperationAction(ISD::FSHR, VT, Expand);
649
6.87M
    setOperationAction(ISD::SADDSAT, VT, Expand);
650
6.87M
    setOperationAction(ISD::UADDSAT, VT, Expand);
651
6.87M
    setOperationAction(ISD::SSUBSAT, VT, Expand);
652
6.87M
    setOperationAction(ISD::USUBSAT, VT, Expand);
653
6.87M
    setOperationAction(ISD::SMULFIX, VT, Expand);
654
6.87M
    setOperationAction(ISD::SMULFIXSAT, VT, Expand);
655
6.87M
    setOperationAction(ISD::UMULFIX, VT, Expand);
656
6.87M
657
6.87M
    // Overflow operations default to expand
658
6.87M
    setOperationAction(ISD::SADDO, VT, Expand);
659
6.87M
    setOperationAction(ISD::SSUBO, VT, Expand);
660
6.87M
    setOperationAction(ISD::UADDO, VT, Expand);
661
6.87M
    setOperationAction(ISD::USUBO, VT, Expand);
662
6.87M
    setOperationAction(ISD::SMULO, VT, Expand);
663
6.87M
    setOperationAction(ISD::UMULO, VT, Expand);
664
6.87M
665
6.87M
    // ADDCARRY operations default to expand
666
6.87M
    setOperationAction(ISD::ADDCARRY, VT, Expand);
667
6.87M
    setOperationAction(ISD::SUBCARRY, VT, Expand);
668
6.87M
    setOperationAction(ISD::SETCCCARRY, VT, Expand);
669
6.87M
670
6.87M
    // ADDC/ADDE/SUBC/SUBE default to expand.
671
6.87M
    setOperationAction(ISD::ADDC, VT, Expand);
672
6.87M
    setOperationAction(ISD::ADDE, VT, Expand);
673
6.87M
    setOperationAction(ISD::SUBC, VT, Expand);
674
6.87M
    setOperationAction(ISD::SUBE, VT, Expand);
675
6.87M
676
6.87M
    // These default to Expand so they will be expanded to CTLZ/CTTZ by default.
677
6.87M
    setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand);
678
6.87M
    setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand);
679
6.87M
680
6.87M
    setOperationAction(ISD::BITREVERSE, VT, Expand);
681
6.87M
682
6.87M
    // These library functions default to expand.
683
6.87M
    setOperationAction(ISD::FROUND, VT, Expand);
684
6.87M
    setOperationAction(ISD::FPOWI, VT, Expand);
685
6.87M
686
6.87M
    // These operations default to expand for vector types.
687
6.87M
    if (VT.isVector()) {
688
5.91M
      setOperationAction(ISD::FCOPYSIGN, VT, Expand);
689
5.91M
      setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, VT, Expand);
690
5.91M
      setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Expand);
691
5.91M
      setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Expand);
692
5.91M
    }
693
6.87M
694
6.87M
    // Constrained floating-point operations default to expand.
695
6.87M
    setOperationAction(ISD::STRICT_FADD, VT, Expand);
696
6.87M
    setOperationAction(ISD::STRICT_FSUB, VT, Expand);
697
6.87M
    setOperationAction(ISD::STRICT_FMUL, VT, Expand);
698
6.87M
    setOperationAction(ISD::STRICT_FDIV, VT, Expand);
699
6.87M
    setOperationAction(ISD::STRICT_FREM, VT, Expand);
700
6.87M
    setOperationAction(ISD::STRICT_FMA, VT, Expand);
701
6.87M
    setOperationAction(ISD::STRICT_FSQRT, VT, Expand);
702
6.87M
    setOperationAction(ISD::STRICT_FPOW, VT, Expand);
703
6.87M
    setOperationAction(ISD::STRICT_FPOWI, VT, Expand);
704
6.87M
    setOperationAction(ISD::STRICT_FSIN, VT, Expand);
705
6.87M
    setOperationAction(ISD::STRICT_FCOS, VT, Expand);
706
6.87M
    setOperationAction(ISD::STRICT_FEXP, VT, Expand);
707
6.87M
    setOperationAction(ISD::STRICT_FEXP2, VT, Expand);
708
6.87M
    setOperationAction(ISD::STRICT_FLOG, VT, Expand);
709
6.87M
    setOperationAction(ISD::STRICT_FLOG10, VT, Expand);
710
6.87M
    setOperationAction(ISD::STRICT_FLOG2, VT, Expand);
711
6.87M
    setOperationAction(ISD::STRICT_FRINT, VT, Expand);
712
6.87M
    setOperationAction(ISD::STRICT_FNEARBYINT, VT, Expand);
713
6.87M
    setOperationAction(ISD::STRICT_FCEIL, VT, Expand);
714
6.87M
    setOperationAction(ISD::STRICT_FFLOOR, VT, Expand);
715
6.87M
    setOperationAction(ISD::STRICT_FROUND, VT, Expand);
716
6.87M
    setOperationAction(ISD::STRICT_FTRUNC, VT, Expand);
717
6.87M
    setOperationAction(ISD::STRICT_FMAXNUM, VT, Expand);
718
6.87M
    setOperationAction(ISD::STRICT_FMINNUM, VT, Expand);
719
6.87M
    setOperationAction(ISD::STRICT_FP_ROUND, VT, Expand);
720
6.87M
    setOperationAction(ISD::STRICT_FP_EXTEND, VT, Expand);
721
6.87M
722
6.87M
    // For most targets @llvm.get.dynamic.area.offset just returns 0.
723
6.87M
    setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, VT, Expand);
724
6.87M
725
6.87M
    // Vector reduction default to expand.
726
6.87M
    setOperationAction(ISD::VECREDUCE_FADD, VT, Expand);
727
6.87M
    setOperationAction(ISD::VECREDUCE_FMUL, VT, Expand);
728
6.87M
    setOperationAction(ISD::VECREDUCE_ADD, VT, Expand);
729
6.87M
    setOperationAction(ISD::VECREDUCE_MUL, VT, Expand);
730
6.87M
    setOperationAction(ISD::VECREDUCE_AND, VT, Expand);
731
6.87M
    setOperationAction(ISD::VECREDUCE_OR, VT, Expand);
732
6.87M
    setOperationAction(ISD::VECREDUCE_XOR, VT, Expand);
733
6.87M
    setOperationAction(ISD::VECREDUCE_SMAX, VT, Expand);
734
6.87M
    setOperationAction(ISD::VECREDUCE_SMIN, VT, Expand);
735
6.87M
    setOperationAction(ISD::VECREDUCE_UMAX, VT, Expand);
736
6.87M
    setOperationAction(ISD::VECREDUCE_UMIN, VT, Expand);
737
6.87M
    setOperationAction(ISD::VECREDUCE_FMAX, VT, Expand);
738
6.87M
    setOperationAction(ISD::VECREDUCE_FMIN, VT, Expand);
739
6.87M
  }
740
53.2k
741
53.2k
  // Most targets ignore the @llvm.prefetch intrinsic.
742
53.2k
  setOperationAction(ISD::PREFETCH, MVT::Other, Expand);
743
53.2k
744
53.2k
  // Most targets also ignore the @llvm.readcyclecounter intrinsic.
745
53.2k
  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Expand);
746
53.2k
747
53.2k
  // ConstantFP nodes default to expand.  Targets can either change this to
748
53.2k
  // Legal, in which case all fp constants are legal, or use isFPImmLegal()
749
53.2k
  // to optimize expansions for certain constants.
750
53.2k
  setOperationAction(ISD::ConstantFP, MVT::f16, Expand);
751
53.2k
  setOperationAction(ISD::ConstantFP, MVT::f32, Expand);
752
53.2k
  setOperationAction(ISD::ConstantFP, MVT::f64, Expand);
753
53.2k
  setOperationAction(ISD::ConstantFP, MVT::f80, Expand);
754
53.2k
  setOperationAction(ISD::ConstantFP, MVT::f128, Expand);
755
53.2k
756
53.2k
  // These library functions default to expand.
757
159k
  for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) {
758
159k
    setOperationAction(ISD::FCBRT,      VT, Expand);
759
159k
    setOperationAction(ISD::FLOG ,      VT, Expand);
760
159k
    setOperationAction(ISD::FLOG2,      VT, Expand);
761
159k
    setOperationAction(ISD::FLOG10,     VT, Expand);
762
159k
    setOperationAction(ISD::FEXP ,      VT, Expand);
763
159k
    setOperationAction(ISD::FEXP2,      VT, Expand);
764
159k
    setOperationAction(ISD::FFLOOR,     VT, Expand);
765
159k
    setOperationAction(ISD::FNEARBYINT, VT, Expand);
766
159k
    setOperationAction(ISD::FCEIL,      VT, Expand);
767
159k
    setOperationAction(ISD::FRINT,      VT, Expand);
768
159k
    setOperationAction(ISD::FTRUNC,     VT, Expand);
769
159k
    setOperationAction(ISD::FROUND,     VT, Expand);
770
159k
    setOperationAction(ISD::LROUND,     VT, Expand);
771
159k
    setOperationAction(ISD::LLROUND,    VT, Expand);
772
159k
    setOperationAction(ISD::LRINT,      VT, Expand);
773
159k
    setOperationAction(ISD::LLRINT,     VT, Expand);
774
159k
  }
775
53.2k
776
53.2k
  // Default ISD::TRAP to expand (which turns it into abort).
777
53.2k
  setOperationAction(ISD::TRAP, MVT::Other, Expand);
778
53.2k
779
53.2k
  // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand"
780
53.2k
  // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP.
781
53.2k
  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand);
782
53.2k
}
783
784
MVT TargetLoweringBase::getScalarShiftAmountTy(const DataLayout &DL,
785
159k
                                               EVT) const {
786
159k
  return MVT::getIntegerVT(DL.getPointerSizeInBits(0));
787
159k
}
788
789
EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy, const DataLayout &DL,
790
742k
                                         bool LegalTypes) const {
791
742k
  assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
792
742k
  if (LHSTy.isVector())
793
14.6k
    return LHSTy;
794
728k
  return LegalTypes ? 
getScalarShiftAmountTy(DL, LHSTy)699k
795
728k
                    : 
getPointerTy(DL)28.9k
;
796
728k
}
797
798
191
bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const {
799
191
  assert(isTypeLegal(VT));
800
191
  switch (Op) {
801
191
  default:
802
140
    return false;
803
191
  case ISD::SDIV:
804
51
  case ISD::UDIV:
805
51
  case ISD::SREM:
806
51
  case ISD::UREM:
807
51
    return true;
808
191
  }
809
191
}
810
811
6.03k
void TargetLoweringBase::setJumpIsExpensive(bool isExpensive) {
812
6.03k
  // If the command-line option was specified, ignore this request.
813
6.03k
  if (!JumpIsExpensiveOverride.getNumOccurrences())
814
6.03k
    JumpIsExpensive = isExpensive;
815
6.03k
}
816
817
TargetLoweringBase::LegalizeKind
818
86.9M
TargetLoweringBase::getTypeConversion(LLVMContext &Context, EVT VT) const {
819
86.9M
  // If this is a simple type, use the ComputeRegisterProp mechanism.
820
86.9M
  if (VT.isSimple()) {
821
86.2M
    MVT SVT = VT.getSimpleVT();
822
86.2M
    assert((unsigned)SVT.SimpleTy < array_lengthof(TransformToType));
823
86.2M
    MVT NVT = TransformToType[SVT.SimpleTy];
824
86.2M
    LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT);
825
86.2M
826
86.2M
    assert((LA == TypeLegal || LA == TypeSoftenFloat ||
827
86.2M
            ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger) &&
828
86.2M
           "Promote may not follow Expand or Promote");
829
86.2M
830
86.2M
    if (LA == TypeSplitVector)
831
1.37M
      return LegalizeKind(LA,
832
1.37M
                          EVT::getVectorVT(Context, SVT.getVectorElementType(),
833
1.37M
                                           SVT.getVectorNumElements() / 2));
834
84.8M
    if (LA == TypeScalarizeVector)
835
205k
      return LegalizeKind(LA, SVT.getVectorElementType());
836
84.6M
    return LegalizeKind(LA, NVT);
837
84.6M
  }
838
663k
839
663k
  // Handle Extended Scalar Types.
840
663k
  if (!VT.isVector()) {
841
531k
    assert(VT.isInteger() && "Float types must be simple");
842
531k
    unsigned BitSize = VT.getSizeInBits();
843
531k
    // First promote to a power-of-two size, then expand if necessary.
844
531k
    if (BitSize < 8 || 
!isPowerOf2_32(BitSize)491k
) {
845
311k
      EVT NVT = VT.getRoundIntegerType(Context);
846
311k
      assert(NVT != VT && "Unable to round integer VT");
847
311k
      LegalizeKind NextStep = getTypeConversion(Context, NVT);
848
311k
      // Avoid multi-step promotion.
849
311k
      if (NextStep.first == TypePromoteInteger)
850
28.4k
        return NextStep;
851
283k
      // Return rounded integer type.
852
283k
      return LegalizeKind(TypePromoteInteger, NVT);
853
283k
    }
854
219k
855
219k
    return LegalizeKind(TypeExpandInteger,
856
219k
                        EVT::getIntegerVT(Context, VT.getSizeInBits() / 2));
857
219k
  }
858
131k
859
131k
  // Handle vector types.
860
131k
  unsigned NumElts = VT.getVectorNumElements();
861
131k
  EVT EltVT = VT.getVectorElementType();
862
131k
863
131k
  // Vectors with only one element are always scalarized.
864
131k
  if (NumElts == 1)
865
13.0k
    return LegalizeKind(TypeScalarizeVector, EltVT);
866
118k
867
118k
  // Try to widen vector elements until the element type is a power of two and
868
118k
  // promote it to a legal type later on, for example:
869
118k
  // <3 x i8> -> <4 x i8> -> <4 x i32>
870
118k
  if (EltVT.isInteger()) {
871
59.3k
    // Vectors with a number of elements that is not a power of two are always
872
59.3k
    // widened, for example <3 x i8> -> <4 x i8>.
873
59.3k
    if (!VT.isPow2VectorType()) {
874
34.0k
      NumElts = (unsigned)NextPowerOf2(NumElts);
875
34.0k
      EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts);
876
34.0k
      return LegalizeKind(TypeWidenVector, NVT);
877
34.0k
    }
878
25.3k
879
25.3k
    // Examine the element type.
880
25.3k
    LegalizeKind LK = getTypeConversion(Context, EltVT);
881
25.3k
882
25.3k
    // If type is to be expanded, split the vector.
883
25.3k
    //  <4 x i140> -> <2 x i140>
884
25.3k
    if (LK.first == TypeExpandInteger)
885
14.6k
      return LegalizeKind(TypeSplitVector,
886
14.6k
                          EVT::getVectorVT(Context, EltVT, NumElts / 2));
887
10.6k
888
10.6k
    // Promote the integer element types until a legal vector type is found
889
10.6k
    // or until the element integer type is too big. If a legal type was not
890
10.6k
    // found, fallback to the usual mechanism of widening/splitting the
891
10.6k
    // vector.
892
10.6k
    EVT OldEltVT = EltVT;
893
24.2k
    while (true) {
894
24.2k
      // Increase the bitwidth of the element to the next pow-of-two
895
24.2k
      // (which is greater than 8 bits).
896
24.2k
      EltVT = EVT::getIntegerVT(Context, 1 + EltVT.getSizeInBits())
897
24.2k
                  .getRoundIntegerType(Context);
898
24.2k
899
24.2k
      // Stop trying when getting a non-simple element type.
900
24.2k
      // Note that vector elements may be greater than legal vector element
901
24.2k
      // types. Example: X86 XMM registers hold 64bit element on 32bit
902
24.2k
      // systems.
903
24.2k
      if (!EltVT.isSimple())
904
2.99k
        break;
905
21.2k
906
21.2k
      // Build a new vector type and check if it is legal.
907
21.2k
      MVT NVT = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
908
21.2k
      // Found a legal promoted vector type.
909
21.2k
      if (NVT != MVT() && 
ValueTypeActions.getTypeAction(NVT) == TypeLegal17.8k
)
910
7.70k
        return LegalizeKind(TypePromoteInteger,
911
7.70k
                            EVT::getVectorVT(Context, EltVT, NumElts));
912
21.2k
    }
913
10.6k
914
10.6k
    // Reset the type to the unexpanded type if we did not find a legal vector
915
10.6k
    // type with a promoted vector element type.
916
10.6k
    EltVT = OldEltVT;
917
2.99k
  }
918
118k
919
118k
  // Try to widen the vector until a legal type is found.
920
118k
  // If there is no wider legal type, split the vector.
921
118k
  
while (62.3k
true73.2k
) {
922
73.2k
    // Round up to the next power of 2.
923
73.2k
    NumElts = (unsigned)NextPowerOf2(NumElts);
924
73.2k
925
73.2k
    // If there is no simple vector type with this many elements then there
926
73.2k
    // cannot be a larger legal vector type.  Note that this assumes that
927
73.2k
    // there are no skipped intermediate vector types in the simple types.
928
73.2k
    if (!EltVT.isSimple())
929
363
      break;
930
72.8k
    MVT LargerVector = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
931
72.8k
    if (LargerVector == MVT())
932
61.0k
      break;
933
11.8k
934
11.8k
    // If this type is legal then widen the vector.
935
11.8k
    if (ValueTypeActions.getTypeAction(LargerVector) == TypeLegal)
936
1.01k
      return LegalizeKind(TypeWidenVector, LargerVector);
937
11.8k
  }
938
62.3k
939
62.3k
  // Widen odd vectors to next power of two.
940
62.3k
  
if (61.3k
!VT.isPow2VectorType()61.3k
) {
941
6.71k
    EVT NVT = VT.getPow2VectorType(Context);
942
6.71k
    return LegalizeKind(TypeWidenVector, NVT);
943
6.71k
  }
944
54.6k
945
54.6k
  // Vectors with illegal element types are expanded.
946
54.6k
  EVT NVT = EVT::getVectorVT(Context, EltVT, VT.getVectorNumElements() / 2);
947
54.6k
  return LegalizeKind(TypeSplitVector, NVT);
948
54.6k
}
949
950
static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
951
                                          unsigned &NumIntermediates,
952
                                          MVT &RegisterVT,
953
5.13M
                                          TargetLoweringBase *TLI) {
954
5.13M
  // Figure out the right, legal destination reg to copy into.
955
5.13M
  unsigned NumElts = VT.getVectorNumElements();
956
5.13M
  MVT EltTy = VT.getVectorElementType();
957
5.13M
958
5.13M
  unsigned NumVectorRegs = 1;
959
5.13M
960
5.13M
  // FIXME: We don't support non-power-of-2-sized vectors for now.  Ideally we
961
5.13M
  // could break down into LHS/RHS like LegalizeDAG does.
962
5.13M
  if (!isPowerOf2_32(NumElts)) {
963
130k
    NumVectorRegs = NumElts;
964
130k
    NumElts = 1;
965
130k
  }
966
5.13M
967
5.13M
  // Divide the input until we get to a supported size.  This will always
968
5.13M
  // end with a scalar if the target doesn't support vectors.
969
19.3M
  while (NumElts > 1 && 
!TLI->isTypeLegal(MVT::getVectorVT(EltTy, NumElts))15.9M
) {
970
14.2M
    NumElts >>= 1;
971
14.2M
    NumVectorRegs <<= 1;
972
14.2M
  }
973
5.13M
974
5.13M
  NumIntermediates = NumVectorRegs;
975
5.13M
976
5.13M
  MVT NewVT = MVT::getVectorVT(EltTy, NumElts);
977
5.13M
  if (!TLI->isTypeLegal(NewVT))
978
3.33M
    NewVT = EltTy;
979
5.13M
  IntermediateVT = NewVT;
980
5.13M
981
5.13M
  unsigned NewVTSize = NewVT.getSizeInBits();
982
5.13M
983
5.13M
  // Convert sizes such as i33 to i64.
984
5.13M
  if (!isPowerOf2_32(NewVTSize))
985
0
    NewVTSize = NextPowerOf2(NewVTSize);
986
5.13M
987
5.13M
  MVT DestVT = TLI->getRegisterType(NewVT);
988
5.13M
  RegisterVT = DestVT;
989
5.13M
  if (EVT(DestVT).bitsLT(NewVT))    // Value is expanded, e.g. i64 -> i16.
990
285k
    return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits());
991
4.85M
992
4.85M
  // Otherwise, promotion or legal types use the same number of registers as
993
4.85M
  // the vector decimated to the appropriate level.
994
4.85M
  return NumVectorRegs;
995
4.85M
}
996
997
/// isLegalRC - Return true if the value types that can be represented by the
998
/// specified register class are all legal.
999
bool TargetLoweringBase::isLegalRC(const TargetRegisterInfo &TRI,
1000
32.5M
                                   const TargetRegisterClass &RC) const {
1001
52.0M
  for (auto I = TRI.legalclasstypes_begin(RC); *I != MVT::Other; 
++I19.5M
)
1002
35.5M
    if (isTypeLegal(*I))
1003
16.0M
      return true;
1004
32.5M
  
return false16.5M
;
1005
32.5M
}
1006
1007
/// Replace/modify any TargetFrameIndex operands with a targte-dependent
1008
/// sequence of memory operands that is recognized by PrologEpilogInserter.
1009
MachineBasicBlock *
1010
TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI,
1011
439
                                   MachineBasicBlock *MBB) const {
1012
439
  MachineInstr *MI = &InitialMI;
1013
439
  MachineFunction &MF = *MI->getMF();
1014
439
  MachineFrameInfo &MFI = MF.getFrameInfo();
1015
439
1016
439
  // We're handling multiple types of operands here:
1017
439
  // PATCHPOINT MetaArgs - live-in, read only, direct
1018
439
  // STATEPOINT Deopt Spill - live-through, read only, indirect
1019
439
  // STATEPOINT Deopt Alloca - live-through, read only, direct
1020
439
  // (We're currently conservative and mark the deopt slots read/write in
1021
439
  // practice.)
1022
439
  // STATEPOINT GC Spill - live-through, read/write, indirect
1023
439
  // STATEPOINT GC Alloca - live-through, read/write, direct
1024
439
  // The live-in vs live-through is handled already (the live through ones are
1025
439
  // all stack slots), but we need to handle the different type of stackmap
1026
439
  // operands and memory effects here.
1027
439
1028
439
  // MI changes inside this loop as we grow operands.
1029
5.82k
  for(unsigned OperIdx = 0; OperIdx != MI->getNumOperands(); 
++OperIdx5.38k
) {
1030
5.38k
    MachineOperand &MO = MI->getOperand(OperIdx);
1031
5.38k
    if (!MO.isFI())
1032
5.20k
      continue;
1033
181
1034
181
    // foldMemoryOperand builds a new MI after replacing a single FI operand
1035
181
    // with the canonical set of five x86 addressing-mode operands.
1036
181
    int FI = MO.getIndex();
1037
181
    MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc());
1038
181
1039
181
    // Copy operands before the frame-index.
1040
3.34k
    for (unsigned i = 0; i < OperIdx; 
++i3.16k
)
1041
3.16k
      MIB.add(MI->getOperand(i));
1042
181
    // Add frame index operands recognized by stackmaps.cpp
1043
181
    if (MFI.isStatepointSpillSlotObjectIndex(FI)) {
1044
142
      // indirect-mem-ref tag, size, #FI, offset.
1045
142
      // Used for spills inserted by StatepointLowering.  This codepath is not
1046
142
      // used for patchpoints/stackmaps at all, for these spilling is done via
1047
142
      // foldMemoryOperand callback only.
1048
142
      assert(MI->getOpcode() == TargetOpcode::STATEPOINT && "sanity");
1049
142
      MIB.addImm(StackMaps::IndirectMemRefOp);
1050
142
      MIB.addImm(MFI.getObjectSize(FI));
1051
142
      MIB.add(MI->getOperand(OperIdx));
1052
142
      MIB.addImm(0);
1053
142
    } else {
1054
39
      // direct-mem-ref tag, #FI, offset.
1055
39
      // Used by patchpoint, and direct alloca arguments to statepoints
1056
39
      MIB.addImm(StackMaps::DirectMemRefOp);
1057
39
      MIB.add(MI->getOperand(OperIdx));
1058
39
      MIB.addImm(0);
1059
39
    }
1060
181
    // Copy the operands after the frame index.
1061
1.02k
    for (unsigned i = OperIdx + 1; i != MI->getNumOperands(); 
++i844
)
1062
844
      MIB.add(MI->getOperand(i));
1063
181
1064
181
    // Inherit previous memory operands.
1065
181
    MIB.cloneMemRefs(*MI);
1066
181
    assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!");
1067
181
1068
181
    // Add a new memory operand for this FI.
1069
181
    assert(MFI.getObjectOffset(FI) != -1);
1070
181
1071
181
    // Note: STATEPOINT MMOs are added during SelectionDAG.  STACKMAP, and
1072
181
    // PATCHPOINT should be updated to do the same. (TODO)
1073
181
    if (MI->getOpcode() != TargetOpcode::STATEPOINT) {
1074
29
      auto Flags = MachineMemOperand::MOLoad;
1075
29
      MachineMemOperand *MMO = MF.getMachineMemOperand(
1076
29
          MachinePointerInfo::getFixedStack(MF, FI), Flags,
1077
29
          MF.getDataLayout().getPointerSize(), MFI.getObjectAlignment(FI));
1078
29
      MIB->addMemOperand(MF, MMO);
1079
29
    }
1080
181
    
1081
181
    // Replace the instruction and update the operand index.
1082
181
    MBB->insert(MachineBasicBlock::iterator(MI), MIB);
1083
181
    OperIdx += (MIB->getNumOperands() - MI->getNumOperands()) - 1;
1084
181
    MI->eraseFromParent();
1085
181
    MI = MIB;
1086
181
  }
1087
439
  return MBB;
1088
439
}
1089
1090
MachineBasicBlock *
1091
TargetLoweringBase::emitXRayCustomEvent(MachineInstr &MI,
1092
2
                                        MachineBasicBlock *MBB) const {
1093
2
  assert(MI.getOpcode() == TargetOpcode::PATCHABLE_EVENT_CALL &&
1094
2
         "Called emitXRayCustomEvent on the wrong MI!");
1095
2
  auto &MF = *MI.getMF();
1096
2
  auto MIB = BuildMI(MF, MI.getDebugLoc(), MI.getDesc());
1097
6
  for (unsigned OpIdx = 0; OpIdx != MI.getNumOperands(); 
++OpIdx4
)
1098
4
    MIB.add(MI.getOperand(OpIdx));
1099
2
1100
2
  MBB->insert(MachineBasicBlock::iterator(MI), MIB);
1101
2
  MI.eraseFromParent();
1102
2
  return MBB;
1103
2
}
1104
1105
MachineBasicBlock *
1106
TargetLoweringBase::emitXRayTypedEvent(MachineInstr &MI,
1107
2
                                       MachineBasicBlock *MBB) const {
1108
2
  assert(MI.getOpcode() == TargetOpcode::PATCHABLE_TYPED_EVENT_CALL &&
1109
2
         "Called emitXRayTypedEvent on the wrong MI!");
1110
2
  auto &MF = *MI.getMF();
1111
2
  auto MIB = BuildMI(MF, MI.getDebugLoc(), MI.getDesc());
1112
8
  for (unsigned OpIdx = 0; OpIdx != MI.getNumOperands(); 
++OpIdx6
)
1113
6
    MIB.add(MI.getOperand(OpIdx));
1114
2
1115
2
  MBB->insert(MachineBasicBlock::iterator(MI), MIB);
1116
2
  MI.eraseFromParent();
1117
2
  return MBB;
1118
2
}
1119
1120
/// findRepresentativeClass - Return the largest legal super-reg register class
1121
/// of the register class for the specified type and its associated "cost".
1122
// This function is in TargetLowering because it uses RegClassForVT which would
1123
// need to be moved to TargetRegisterInfo and would necessitate moving
1124
// isTypeLegal over as well - a massive change that would just require
1125
// TargetLowering having a TargetRegisterInfo class member that it would use.
1126
std::pair<const TargetRegisterClass *, uint8_t>
1127
TargetLoweringBase::findRepresentativeClass(const TargetRegisterInfo *TRI,
1128
6.42M
                                            MVT VT) const {
1129
6.42M
  const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
1130
6.42M
  if (!RC)
1131
6.04M
    return std::make_pair(RC, 0);
1132
386k
1133
386k
  // Compute the set of all super-register classes.
1134
386k
  BitVector SuperRegRC(TRI->getNumRegClasses());
1135
3.87M
  for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); 
++RCI3.49M
)
1136
3.49M
    SuperRegRC.setBitsInMask(RCI.getMask());
1137
386k
1138
386k
  // Find the first legal register class with the largest spill size.
1139
386k
  const TargetRegisterClass *BestRC = RC;
1140
15.0M
  for (unsigned i : SuperRegRC.set_bits()) {
1141
15.0M
    const TargetRegisterClass *SuperRC = TRI->getRegClass(i);
1142
15.0M
    // We want the largest possible spill size.
1143
15.0M
    if (TRI->getSpillSize(*SuperRC) <= TRI->getSpillSize(*BestRC))
1144
2.53M
      continue;
1145
12.5M
    if (!isLegalRC(*TRI, *SuperRC))
1146
12.0M
      continue;
1147
460k
    BestRC = SuperRC;
1148
460k
  }
1149
386k
  return std::make_pair(BestRC, 1);
1150
386k
}
1151
1152
/// computeRegisterProperties - Once all of the register classes are added,
1153
/// this allows us to compute derived properties we expose.
1154
void TargetLoweringBase::computeRegisterProperties(
1155
53.2k
    const TargetRegisterInfo *TRI) {
1156
53.2k
  static_assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE,
1157
53.2k
                "Too many value types for ValueTypeActions to hold!");
1158
53.2k
1159
53.2k
  // Everything defaults to needing one register.
1160
6.97M
  for (unsigned i = 0; i != MVT::LAST_VALUETYPE; 
++i6.92M
) {
1161
6.92M
    NumRegistersForVT[i] = 1;
1162
6.92M
    RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i;
1163
6.92M
  }
1164
53.2k
  // ...except isVoid, which doesn't need any registers.
1165
53.2k
  NumRegistersForVT[MVT::isVoid] = 0;
1166
53.2k
1167
53.2k
  // Find the largest integer register class.
1168
53.2k
  unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE;
1169
126k
  for (; RegClassForVT[LargestIntReg] == nullptr; 
--LargestIntReg73.7k
)
1170
53.2k
    assert(LargestIntReg != MVT::i1 && "No integer registers defined!");
1171
53.2k
1172
53.2k
  // Every integer value type larger than this largest register takes twice as
1173
53.2k
  // many registers to represent as the previous ValueType.
1174
53.2k
  for (unsigned ExpandedReg = LargestIntReg + 1;
1175
126k
       ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; 
++ExpandedReg73.7k
) {
1176
73.7k
    NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1];
1177
73.7k
    RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg;
1178
73.7k
    TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1);
1179
73.7k
    ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg,
1180
73.7k
                                   TypeExpandInteger);
1181
73.7k
  }
1182
53.2k
1183
53.2k
  // Inspect all of the ValueType's smaller than the largest integer
1184
53.2k
  // register to see which ones need promotion.
1185
53.2k
  unsigned LegalIntReg = LargestIntReg;
1186
53.2k
  for (unsigned IntReg = LargestIntReg - 1;
1187
245k
       IntReg >= (unsigned)MVT::i1; 
--IntReg192k
) {
1188
192k
    MVT IVT = (MVT::SimpleValueType)IntReg;
1189
192k
    if (isTypeLegal(IVT)) {
1190
72.3k
      LegalIntReg = IntReg;
1191
120k
    } else {
1192
120k
      RegisterTypeForVT[IntReg] = TransformToType[IntReg] =
1193
120k
        (MVT::SimpleValueType)LegalIntReg;
1194
120k
      ValueTypeActions.setTypeAction(IVT, TypePromoteInteger);
1195
120k
    }
1196
192k
  }
1197
53.2k
1198
53.2k
  // ppcf128 type is really two f64's.
1199
53.2k
  if (
!isTypeLegal(MVT::ppcf128)53.2k
) {
1200
53.2k
    if (isTypeLegal(MVT::f64)) {
1201
46.1k
      NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64];
1202
46.1k
      RegisterTypeForVT[MVT::ppcf128] = MVT::f64;
1203
46.1k
      TransformToType[MVT::ppcf128] = MVT::f64;
1204
46.1k
      ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat);
1205
46.1k
    } else {
1206
7.06k
      NumRegistersForVT[MVT::ppcf128] = NumRegistersForVT[MVT::i128];
1207
7.06k
      RegisterTypeForVT[MVT::ppcf128] = RegisterTypeForVT[MVT::i128];
1208
7.06k
      TransformToType[MVT::ppcf128] = MVT::i128;
1209
7.06k
      ValueTypeActions.setTypeAction(MVT::ppcf128, TypeSoftenFloat);
1210
7.06k
    }
1211
53.2k
  }
1212
53.2k
1213
53.2k
  // Decide how to handle f128. If the target does not have native f128 support,
1214
53.2k
  // expand it to i128 and we will be generating soft float library calls.
1215
53.2k
  if (!isTypeLegal(MVT::f128)) {
1216
37.8k
    NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128];
1217
37.8k
    RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128];
1218
37.8k
    TransformToType[MVT::f128] = MVT::i128;
1219
37.8k
    ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat);
1220
37.8k
  }
1221
53.2k
1222
53.2k
  // Decide how to handle f64. If the target does not have native f64 support,
1223
53.2k
  // expand it to i64 and we will be generating soft float library calls.
1224
53.2k
  if (!isTypeLegal(MVT::f64)) {
1225
7.06k
    NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64];
1226
7.06k
    RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64];
1227
7.06k
    TransformToType[MVT::f64] = MVT::i64;
1228
7.06k
    ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat);
1229
7.06k
  }
1230
53.2k
1231
53.2k
  // Decide how to handle f32. If the target does not have native f32 support,
1232
53.2k
  // expand it to i32 and we will be generating soft float library calls.
1233
53.2k
  if (!isTypeLegal(MVT::f32)) {
1234
6.69k
    NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32];
1235
6.69k
    RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32];
1236
6.69k
    TransformToType[MVT::f32] = MVT::i32;
1237
6.69k
    ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat);
1238
6.69k
  }
1239
53.2k
1240
53.2k
  // Decide how to handle f16. If the target does not have native f16 support,
1241
53.2k
  // promote it to f32, because there are no f16 library calls (except for
1242
53.2k
  // conversions).
1243
53.2k
  if (!isTypeLegal(MVT::f16)) {
1244
40.7k
    NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32];
1245
40.7k
    RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32];
1246
40.7k
    TransformToType[MVT::f16] = MVT::f32;
1247
40.7k
    ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat);
1248
40.7k
  }
1249
53.2k
1250
53.2k
  // Loop over all of the vector value types to see which need transformations.
1251
53.2k
  for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE;
1252
5.96M
       i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; 
++i5.91M
) {
1253
5.91M
    MVT VT = (MVT::SimpleValueType) i;
1254
5.91M
    if (isTypeLegal(VT))
1255
383k
      continue;
1256
5.52M
1257
5.52M
    MVT EltVT = VT.getVectorElementType();
1258
5.52M
    unsigned NElts = VT.getVectorNumElements();
1259
5.52M
    bool IsLegalWiderType = false;
1260
5.52M
    LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT);
1261
5.52M
    switch (PreferredAction) {
1262
5.52M
    case TypePromoteInteger:
1263
4.06M
      // Try to promote the elements of integer vectors. If no legal
1264
4.06M
      // promotion was found, fall through to the widen-vector method.
1265
100M
      for (unsigned nVT = i + 1; nVT <= MVT::LAST_INTEGER_VECTOR_VALUETYPE; 
++nVT96.0M
) {
1266
96.3M
        MVT SVT = (MVT::SimpleValueType) nVT;
1267
96.3M
        // Promote vectors of integers to vectors with the same number
1268
96.3M
        // of elements, with a wider element type.
1269
96.3M
        if (SVT.getScalarSizeInBits() > EltVT.getSizeInBits() &&
1270
96.3M
            
SVT.getVectorNumElements() == NElts61.4M
&&
isTypeLegal(SVT)6.00M
) {
1271
238k
          TransformToType[i] = SVT;
1272
238k
          RegisterTypeForVT[i] = SVT;
1273
238k
          NumRegistersForVT[i] = 1;
1274
238k
          ValueTypeActions.setTypeAction(VT, TypePromoteInteger);
1275
238k
          IsLegalWiderType = true;
1276
238k
          break;
1277
238k
        }
1278
96.3M
      }
1279
4.06M
      if (IsLegalWiderType)
1280
238k
        break;
1281
3.82M
      LLVM_FALLTHROUGH;
1282
3.82M
1283
4.35M
    case TypeWidenVector:
1284
4.35M
      // Try to widen the vector.
1285
219M
      for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; 
++nVT215M
) {
1286
215M
        MVT SVT = (MVT::SimpleValueType) nVT;
1287
215M
        if (SVT.getVectorElementType() == EltVT
1288
215M
            && 
SVT.getVectorNumElements() > NElts25.2M
&&
isTypeLegal(SVT)13.9M
) {
1289
153k
          TransformToType[i] = SVT;
1290
153k
          RegisterTypeForVT[i] = SVT;
1291
153k
          NumRegistersForVT[i] = 1;
1292
153k
          ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1293
153k
          IsLegalWiderType = true;
1294
153k
          break;
1295
153k
        }
1296
215M
      }
1297
4.35M
      if (IsLegalWiderType)
1298
153k
        break;
1299
4.19M
      LLVM_FALLTHROUGH;
1300
4.19M
1301
5.13M
    case TypeSplitVector:
1302
5.13M
    case TypeScalarizeVector: {
1303
5.13M
      MVT IntermediateVT;
1304
5.13M
      MVT RegisterVT;
1305
5.13M
      unsigned NumIntermediates;
1306
5.13M
      NumRegistersForVT[i] = getVectorTypeBreakdownMVT(VT, IntermediateVT,
1307
5.13M
          NumIntermediates, RegisterVT, this);
1308
5.13M
      RegisterTypeForVT[i] = RegisterVT;
1309
5.13M
1310
5.13M
      MVT NVT = VT.getPow2VectorType();
1311
5.13M
      if (NVT == VT) {
1312
5.00M
        // Type is already a power of 2.  The default action is to split.
1313
5.00M
        TransformToType[i] = MVT::Other;
1314
5.00M
        if (PreferredAction == TypeScalarizeVector)
1315
700k
          ValueTypeActions.setTypeAction(VT, TypeScalarizeVector);
1316
4.30M
        else if (PreferredAction == TypeSplitVector)
1317
234k
          ValueTypeActions.setTypeAction(VT, TypeSplitVector);
1318
4.07M
        else
1319
4.07M
          // Set type action according to the number of elements.
1320
4.07M
          ValueTypeActions.setTypeAction(VT, NElts == 1 ? 
TypeScalarizeVector28.0k
1321
4.07M
                                                        : 
TypeSplitVector4.04M
);
1322
5.00M
      } else {
1323
130k
        TransformToType[i] = NVT;
1324
130k
        ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1325
130k
      }
1326
5.13M
      break;
1327
5.13M
    }
1328
5.13M
    default:
1329
0
      llvm_unreachable("Unknown vector legalization action!");
1330
5.52M
    }
1331
5.52M
  }
1332
53.2k
1333
53.2k
  // Determine the 'representative' register class for each value type.
1334
53.2k
  // An representative register class is the largest (meaning one which is
1335
53.2k
  // not a sub-register class / subreg register class) legal register class for
1336
53.2k
  // a group of value types. For example, on i386, i8, i16, and i32
1337
53.2k
  // representative would be GR32; while on x86_64 it's GR64.
1338
6.97M
  
for (unsigned i = 0; 53.2k
i != MVT::LAST_VALUETYPE;
++i6.92M
) {
1339
6.92M
    const TargetRegisterClass* RRC;
1340
6.92M
    uint8_t Cost;
1341
6.92M
    std::tie(RRC, Cost) = findRepresentativeClass(TRI, (MVT::SimpleValueType)i);
1342
6.92M
    RepRegClassForVT[i] = RRC;
1343
6.92M
    RepRegClassCostForVT[i] = Cost;
1344
6.92M
  }
1345
53.2k
}
1346
1347
EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &,
1348
3.85k
                                           EVT VT) const {
1349
3.85k
  assert(!VT.isVector() && "No default SetCC type for vectors!");
1350
3.85k
  return getPointerTy(DL).SimpleTy;
1351
3.85k
}
1352
1353
1.10k
MVT::SimpleValueType TargetLoweringBase::getCmpLibcallReturnType() const {
1354
1.10k
  return MVT::i32; // return the default value
1355
1.10k
}
1356
1357
/// getVectorTypeBreakdown - Vector types are broken down into some number of
1358
/// legal first class types.  For example, MVT::v8f32 maps to 2 MVT::v4f32
1359
/// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
1360
/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.
1361
///
1362
/// This method returns the number of registers needed, and the VT for each
1363
/// register.  It also returns the VT and quantity of the intermediate values
1364
/// before they are promoted/expanded.
1365
unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context, EVT VT,
1366
                                                EVT &IntermediateVT,
1367
                                                unsigned &NumIntermediates,
1368
50.5k
                                                MVT &RegisterVT) const {
1369
50.5k
  unsigned NumElts = VT.getVectorNumElements();
1370
50.5k
1371
50.5k
  // If there is a wider vector type with the same element type as this one,
1372
50.5k
  // or a promoted vector type that has the same number of elements which
1373
50.5k
  // are wider, then we should convert to that legal vector type.
1374
50.5k
  // This handles things like <2 x float> -> <4 x float> and
1375
50.5k
  // <4 x i1> -> <4 x i32>.
1376
50.5k
  LegalizeTypeAction TA = getTypeAction(Context, VT);
1377
50.5k
  if (NumElts != 1 && 
(49.9k
TA == TypeWidenVector49.9k
||
TA == TypePromoteInteger45.7k
)) {
1378
5.05k
    EVT RegisterEVT = getTypeToTransformTo(Context, VT);
1379
5.05k
    if (isTypeLegal(RegisterEVT)) {
1380
1.96k
      IntermediateVT = RegisterEVT;
1381
1.96k
      RegisterVT = RegisterEVT.getSimpleVT();
1382
1.96k
      NumIntermediates = 1;
1383
1.96k
      return 1;
1384
1.96k
    }
1385
48.6k
  }
1386
48.6k
1387
48.6k
  // Figure out the right, legal destination reg to copy into.
1388
48.6k
  EVT EltTy = VT.getVectorElementType();
1389
48.6k
1390
48.6k
  unsigned NumVectorRegs = 1;
1391
48.6k
1392
48.6k
  // FIXME: We don't support non-power-of-2-sized vectors for now.  Ideally we
1393
48.6k
  // could break down into LHS/RHS like LegalizeDAG does.
1394
48.6k
  if (!isPowerOf2_32(NumElts)) {
1395
3.09k
    NumVectorRegs = NumElts;
1396
3.09k
    NumElts = 1;
1397
3.09k
  }
1398
48.6k
1399
48.6k
  // Divide the input until we get to a supported size.  This will always
1400
48.6k
  // end with a scalar if the target doesn't support vectors.
1401
107k
  while (NumElts > 1 && !isTypeLegal(
1402
97.7k
                                   EVT::getVectorVT(Context, EltTy, NumElts))) {
1403
58.5k
    NumElts >>= 1;
1404
58.5k
    NumVectorRegs <<= 1;
1405
58.5k
  }
1406
48.6k
1407
48.6k
  NumIntermediates = NumVectorRegs;
1408
48.6k
1409
48.6k
  EVT NewVT = EVT::getVectorVT(Context, EltTy, NumElts);
1410
48.6k
  if (!isTypeLegal(NewVT))
1411
9.39k
    NewVT = EltTy;
1412
48.6k
  IntermediateVT = NewVT;
1413
48.6k
1414
48.6k
  MVT DestVT = getRegisterType(Context, NewVT);
1415
48.6k
  RegisterVT = DestVT;
1416
48.6k
  unsigned NewVTSize = NewVT.getSizeInBits();
1417
48.6k
1418
48.6k
  // Convert sizes such as i33 to i64.
1419
48.6k
  if (!isPowerOf2_32(NewVTSize))
1420
165
    NewVTSize = NextPowerOf2(NewVTSize);
1421
48.6k
1422
48.6k
  if (EVT(DestVT).bitsLT(NewVT))   // Value is expanded, e.g. i64 -> i16.
1423
2.46k
    return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits());
1424
46.1k
1425
46.1k
  // Otherwise, promotion or legal types use the same number of registers as
1426
46.1k
  // the vector decimated to the appropriate level.
1427
46.1k
  return NumVectorRegs;
1428
46.1k
}
1429
1430
/// Get the EVTs and ArgFlags collections that represent the legalized return
1431
/// type of the given function.  This does not require a DAG or a return value,
1432
/// and is suitable for use before any DAGs for the function are constructed.
1433
/// TODO: Move this out of TargetLowering.cpp.
1434
void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType,
1435
                         AttributeList attr,
1436
                         SmallVectorImpl<ISD::OutputArg> &Outs,
1437
750k
                         const TargetLowering &TLI, const DataLayout &DL) {
1438
750k
  SmallVector<EVT, 4> ValueVTs;
1439
750k
  ComputeValueVTs(TLI, DL, ReturnType, ValueVTs);
1440
750k
  unsigned NumValues = ValueVTs.size();
1441
750k
  if (NumValues == 0) 
return282k
;
1442
467k
1443
939k
  
for (unsigned j = 0, f = NumValues; 467k
j != f;
++j471k
) {
1444
471k
    EVT VT = ValueVTs[j];
1445
471k
    ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1446
471k
1447
471k
    if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
1448
5.90k
      ExtendKind = ISD::SIGN_EXTEND;
1449
465k
    else if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt))
1450
34.5k
      ExtendKind = ISD::ZERO_EXTEND;
1451
471k
1452
471k
    // FIXME: C calling convention requires the return type to be promoted to
1453
471k
    // at least 32-bit. But this is not necessary for non-C calling
1454
471k
    // conventions. The frontend should mark functions whose return values
1455
471k
    // require promoting with signext or zeroext attributes.
1456
471k
    if (ExtendKind != ISD::ANY_EXTEND && 
VT.isInteger()40.4k
) {
1457
35.3k
      MVT MinVT = TLI.getRegisterType(ReturnType->getContext(), MVT::i32);
1458
35.3k
      if (VT.bitsLT(MinVT))
1459
25.2k
        VT = MinVT;
1460
35.3k
    }
1461
471k
1462
471k
    unsigned NumParts =
1463
471k
        TLI.getNumRegistersForCallingConv(ReturnType->getContext(), CC, VT);
1464
471k
    MVT PartVT =
1465
471k
        TLI.getRegisterTypeForCallingConv(ReturnType->getContext(), CC, VT);
1466
471k
1467
471k
    // 'inreg' on function refers to return value
1468
471k
    ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1469
471k
    if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::InReg))
1470
474
      Flags.setInReg();
1471
471k
1472
471k
    // Propagate extension type if any
1473
471k
    if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
1474
5.90k
      Flags.setSExt();
1475
465k
    else if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt))
1476
34.5k
      Flags.setZExt();
1477
471k
1478
976k
    for (unsigned i = 0; i < NumParts; 
++i504k
)
1479
504k
      Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isfixed=*/true, 0, 0));
1480
471k
  }
1481
467k
}
1482
1483
/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
1484
/// function arguments in the caller parameter area.  This is the actual
1485
/// alignment, not its logarithm.
1486
unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty,
1487
407
                                                   const DataLayout &DL) const {
1488
407
  return DL.getABITypeAlignment(Ty);
1489
407
}
1490
1491
bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
1492
                                            const DataLayout &DL, EVT VT,
1493
                                            unsigned AddrSpace,
1494
                                            unsigned Alignment,
1495
                                            MachineMemOperand::Flags Flags,
1496
3.87M
                                            bool *Fast) const {
1497
3.87M
  // Check if the specified alignment is sufficient based on the data layout.
1498
3.87M
  // TODO: While using the data layout works in practice, a better solution
1499
3.87M
  // would be to implement this check directly (make this a virtual function).
1500
3.87M
  // For example, the ABI alignment may change based on software platform while
1501
3.87M
  // this function should only be affected by hardware implementation.
1502
3.87M
  Type *Ty = VT.getTypeForEVT(Context);
1503
3.87M
  if (Alignment >= DL.getABITypeAlignment(Ty)) {
1504
3.38M
    // Assume that an access that meets the ABI-specified alignment is fast.
1505
3.38M
    if (Fast != nullptr)
1506
230k
      *Fast = true;
1507
3.38M
    return true;
1508
3.38M
  }
1509
487k
1510
487k
  // This is a misaligned access.
1511
487k
  return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast);
1512
487k
}
1513
1514
bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
1515
                                            const DataLayout &DL, EVT VT,
1516
                                            const MachineMemOperand &MMO,
1517
3.85M
                                            bool *Fast) const {
1518
3.85M
  return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(),
1519
3.85M
                            MMO.getAlignment(), MMO.getFlags(), Fast);
1520
3.85M
}
1521
1522
14
BranchProbability TargetLoweringBase::getPredictableBranchThreshold() const {
1523
14
  return BranchProbability(MinPercentageForPredictableBranch, 100);
1524
14
}
1525
1526
//===----------------------------------------------------------------------===//
1527
//  TargetTransformInfo Helpers
1528
//===----------------------------------------------------------------------===//
1529
1530
2.31M
int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const {
1531
2.31M
  enum InstructionOpcodes {
1532
2.31M
#define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM,
1533
2.31M
#define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM
1534
2.31M
#include "llvm/IR/Instruction.def"
1535
2.31M
  };
1536
2.31M
  switch (static_cast<InstructionOpcodes>(Opcode)) {
1537
2.31M
  
case Ret: return 016
;
1538
2.31M
  
case Br: return 0127
;
1539
2.31M
  
case Switch: return 05
;
1540
2.31M
  
case IndirectBr: return 00
;
1541
2.31M
  
case Invoke: return 015
;
1542
2.31M
  
case CallBr: return 00
;
1543
2.31M
  
case Resume: return 00
;
1544
2.31M
  
case Unreachable: return 00
;
1545
2.31M
  
case CleanupRet: return 00
;
1546
2.31M
  
case CatchRet: return 00
;
1547
2.31M
  
case CatchPad: return 00
;
1548
2.31M
  
case CatchSwitch: return 00
;
1549
2.31M
  
case CleanupPad: return 00
;
1550
2.31M
  
case FNeg: return ISD::FNEG172
;
1551
2.31M
  
case Add: return ISD::ADD356k
;
1552
2.31M
  
case FAdd: return ISD::FADD148k
;
1553
2.31M
  
case Sub: return ISD::SUB161k
;
1554
2.31M
  
case FSub: return ISD::FSUB107k
;
1555
2.31M
  
case Mul: return ISD::MUL99.2k
;
1556
2.31M
  
case FMul: return ISD::FMUL224k
;
1557
2.31M
  
case UDiv: return ISD::UDIV11.6k
;
1558
2.31M
  
case SDiv: return ISD::SDIV18.2k
;
1559
2.31M
  
case FDiv: return ISD::FDIV50.5k
;
1560
2.31M
  
case URem: return ISD::UREM11.7k
;
1561
2.31M
  
case SRem: return ISD::SREM2.42k
;
1562
2.31M
  
case FRem: return ISD::FREM283
;
1563
2.31M
  
case Shl: return ISD::SHL99.6k
;
1564
2.31M
  
case LShr: return ISD::SRL89.0k
;
1565
2.31M
  
case AShr: return ISD::SRA35.2k
;
1566
2.31M
  
case And: return ISD::AND88.3k
;
1567
2.31M
  
case Or: return ISD::OR45.9k
;
1568
2.31M
  
case Xor: return ISD::XOR32.5k
;
1569
2.31M
  
case Alloca: return 00
;
1570
2.31M
  
case Load: return ISD::LOAD151
;
1571
2.31M
  
case Store: return ISD::STORE5.77k
;
1572
2.31M
  
case GetElementPtr: return 0273
;
1573
2.31M
  
case Fence: return 00
;
1574
2.31M
  
case AtomicCmpXchg: return 00
;
1575
2.31M
  
case AtomicRMW: return 00
;
1576
2.31M
  
case Trunc: return ISD::TRUNCATE100k
;
1577
2.31M
  
case ZExt: return ISD::ZERO_EXTEND49.3k
;
1578
2.31M
  
case SExt: return ISD::SIGN_EXTEND33.1k
;
1579
2.31M
  
case FPToUI: return ISD::FP_TO_UINT3.61k
;
1580
2.31M
  
case FPToSI: return ISD::FP_TO_SINT4.86k
;
1581
2.31M
  
case UIToFP: return ISD::UINT_TO_FP5.47k
;
1582
2.31M
  
case SIToFP: return ISD::SINT_TO_FP37.7k
;
1583
2.31M
  
case FPTrunc: return ISD::FP_ROUND2.78k
;
1584
2.31M
  
case FPExt: return ISD::FP_EXTEND2.91k
;
1585
2.31M
  
case PtrToInt: return ISD::BITCAST9.90k
;
1586
2.31M
  
case IntToPtr: return ISD::BITCAST3.04k
;
1587
2.31M
  
case BitCast: return ISD::BITCAST23.9k
;
1588
2.31M
  
case AddrSpaceCast: return ISD::ADDRSPACECAST15
;
1589
2.31M
  
case ICmp: return ISD::SETCC267k
;
1590
2.31M
  
case FCmp: return ISD::SETCC12.2k
;
1591
2.31M
  
case PHI: return 0403
;
1592
2.31M
  
case Call: return 0214
;
1593
2.31M
  
case Select: return ISD::SELECT152k
;
1594
2.31M
  
case UserOp1: return 00
;
1595
2.31M
  
case UserOp2: return 00
;
1596
2.31M
  
case VAArg: return 00
;
1597
2.31M
  
case ExtractElement: return ISD::EXTRACT_VECTOR_ELT7.28k
;
1598
2.31M
  
case InsertElement: return ISD::INSERT_VECTOR_ELT7.13k
;
1599
2.31M
  
case ShuffleVector: return ISD::VECTOR_SHUFFLE3
;
1600
2.31M
  
case ExtractValue: return ISD::MERGE_VALUES5
;
1601
2.31M
  
case InsertValue: return ISD::MERGE_VALUES1
;
1602
2.31M
  
case LandingPad: return 00
;
1603
0
  }
1604
0
1605
0
  llvm_unreachable("Unknown instruction type encountered!");
1606
0
}
1607
1608
std::pair<int, MVT>
1609
TargetLoweringBase::getTypeLegalizationCost(const DataLayout &DL,
1610
5.74M
                                            Type *Ty) const {
1611
5.74M
  LLVMContext &C = Ty->getContext();
1612
5.74M
  EVT MTy = getValueType(DL, Ty);
1613
5.74M
1614
5.74M
  int Cost = 1;
1615
5.74M
  // We keep legalizing the type until we find a legal kind. We assume that
1616
5.74M
  // the only operation that costs anything is the split. After splitting
1617
5.74M
  // we need to handle two types.
1618
6.64M
  while (true) {
1619
6.64M
    LegalizeKind LK = getTypeConversion(C, MTy);
1620
6.64M
1621
6.64M
    if (LK.first == TypeLegal)
1622
5.74M
      return std::make_pair(Cost, MTy.getSimpleVT());
1623
894k
1624
894k
    if (LK.first == TypeSplitVector || 
LK.first == TypeExpandInteger369k
)
1625
536k
      Cost *= 2;
1626
894k
1627
894k
    // Do not loop with f128 type.
1628
894k
    if (MTy == LK.second)
1629
0
      return std::make_pair(Cost, MTy.getSimpleVT());
1630
894k
1631
894k
    // Keep legalizing the type.
1632
894k
    MTy = LK.second;
1633
894k
  }
1634
5.74M
}
1635
1636
Value *TargetLoweringBase::getDefaultSafeStackPointerLocation(IRBuilder<> &IRB,
1637
141
                                                              bool UseTLS) const {
1638
141
  // compiler-rt provides a variable with a magic name.  Targets that do not
1639
141
  // link with compiler-rt may also provide such a variable.
1640
141
  Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1641
141
  const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr";
1642
141
  auto UnsafeStackPtr =
1643
141
      dyn_cast_or_null<GlobalVariable>(M->getNamedValue(UnsafeStackPtrVar));
1644
141
1645
141
  Type *StackPtrTy = Type::getInt8PtrTy(M->getContext());
1646
141
1647
141
  if (!UnsafeStackPtr) {
1648
80
    auto TLSModel = UseTLS ?
1649
79
        GlobalValue::InitialExecTLSModel :
1650
80
        
GlobalValue::NotThreadLocal1
;
1651
80
    // The global variable is not defined yet, define it ourselves.
1652
80
    // We use the initial-exec TLS model because we do not support the
1653
80
    // variable living anywhere other than in the main executable.
1654
80
    UnsafeStackPtr = new GlobalVariable(
1655
80
        *M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr,
1656
80
        UnsafeStackPtrVar, nullptr, TLSModel);
1657
80
  } else {
1658
61
    // The variable exists, check its type and attributes.
1659
61
    if (UnsafeStackPtr->getValueType() != StackPtrTy)
1660
0
      report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type");
1661
61
    if (UseTLS != UnsafeStackPtr->isThreadLocal())
1662
0
      report_fatal_error(Twine(UnsafeStackPtrVar) + " must " +
1663
0
                         (UseTLS ? "" : "not ") + "be thread-local");
1664
141
  }
1665
141
  return UnsafeStackPtr;
1666
141
}
1667
1668
139
Value *TargetLoweringBase::getSafeStackPointerLocation(IRBuilder<> &IRB) const {
1669
139
  if (!TM.getTargetTriple().isAndroid())
1670
137
    return getDefaultSafeStackPointerLocation(IRB, true);
1671
2
1672
2
  // Android provides a libc function to retrieve the address of the current
1673
2
  // thread's unsafe stack pointer.
1674
2
  Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1675
2
  Type *StackPtrTy = Type::getInt8PtrTy(M->getContext());
1676
2
  FunctionCallee Fn = M->getOrInsertFunction("__safestack_pointer_address",
1677
2
                                             StackPtrTy->getPointerTo(0));
1678
2
  return IRB.CreateCall(Fn);
1679
2
}
1680
1681
//===----------------------------------------------------------------------===//
1682
//  Loop Strength Reduction hooks
1683
//===----------------------------------------------------------------------===//
1684
1685
/// isLegalAddressingMode - Return true if the addressing mode represented
1686
/// by AM is legal for this target, for a load/store of the specified type.
1687
bool TargetLoweringBase::isLegalAddressingMode(const DataLayout &DL,
1688
                                               const AddrMode &AM, Type *Ty,
1689
529k
                                               unsigned AS, Instruction *I) const {
1690
529k
  // The default implementation of this implements a conservative RISCy, r+r and
1691
529k
  // r+i addr mode.
1692
529k
1693
529k
  // Allows a sign-extended 16-bit immediate field.
1694
529k
  if (AM.BaseOffs <= -(1LL << 16) || 
AM.BaseOffs >= (1LL << 16)-1529k
)
1695
6
    return false;
1696
529k
1697
529k
  // No global is ever allowed as a base.
1698
529k
  if (AM.BaseGV)
1699
1.39k
    return false;
1700
527k
1701
527k
  // Only support r+r,
1702
527k
  switch (AM.Scale) {
1703
527k
  case 0:  // "r+i" or just "i", depending on HasBaseReg.
1704
525k
    break;
1705
527k
  case 1:
1706
784
    if (AM.HasBaseReg && AM.BaseOffs)  // "r+r+i" is not allowed.
1707
14
      return false;
1708
770
    // Otherwise we have r+r or r+i.
1709
770
    break;
1710
770
  case 2:
1711
170
    if (AM.HasBaseReg || 
AM.BaseOffs10
) // 2*r+r or 2*r+i is not allowed.
1712
160
      return false;
1713
10
    // Allow 2*r as r+r.
1714
10
    break;
1715
1.88k
  default: // Don't allow n * r
1716
1.88k
    return false;
1717
525k
  }
1718
525k
1719
525k
  return true;
1720
525k
}
1721
1722
//===----------------------------------------------------------------------===//
1723
//  Stack Protector
1724
//===----------------------------------------------------------------------===//
1725
1726
// For OpenBSD return its special guard variable. Otherwise return nullptr,
1727
// so that SelectionDAG handle SSP.
1728
4.54k
Value *TargetLoweringBase::getIRStackGuard(IRBuilder<> &IRB) const {
1729
4.54k
  if (getTargetMachine().getTargetTriple().isOSOpenBSD()) {
1730
120
    Module &M = *IRB.GetInsertBlock()->getParent()->getParent();
1731
120
    PointerType *PtrTy = Type::getInt8PtrTy(M.getContext());
1732
120
    return M.getOrInsertGlobal("__guard_local", PtrTy);
1733
120
  }
1734
4.42k
  return nullptr;
1735
4.42k
}
1736
1737
// Currently only support "standard" __stack_chk_guard.
1738
// TODO: add LOAD_STACK_GUARD support.
1739
4.33k
void TargetLoweringBase::insertSSPDeclarations(Module &M) const {
1740
4.33k
  if (!M.getNamedValue("__stack_chk_guard"))
1741
872
    new GlobalVariable(M, Type::getInt8PtrTy(M.getContext()), false,
1742
872
                       GlobalVariable::ExternalLinkage,
1743
872
                       nullptr, "__stack_chk_guard");
1744
4.33k
}
1745
1746
// Currently only support "standard" __stack_chk_guard.
1747
// TODO: add LOAD_STACK_GUARD support.
1748
8.11k
Value *TargetLoweringBase::getSDagStackGuard(const Module &M) const {
1749
8.11k
  return M.getNamedValue("__stack_chk_guard");
1750
8.11k
}
1751
1752
3.24k
Function *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const {
1753
3.24k
  return nullptr;
1754
3.24k
}
1755
1756
50.7k
unsigned TargetLoweringBase::getMinimumJumpTableEntries() const {
1757
50.7k
  return MinimumJumpTableEntries;
1758
50.7k
}
1759
1760
1.95k
void TargetLoweringBase::setMinimumJumpTableEntries(unsigned Val) {
1761
1.95k
  MinimumJumpTableEntries = Val;
1762
1.95k
}
1763
1764
39.7k
unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize) const {
1765
39.7k
  return OptForSize ? 
OptsizeJumpTableDensity118
:
JumpTableDensity39.6k
;
1766
39.7k
}
1767
1768
39.8k
unsigned TargetLoweringBase::getMaximumJumpTableSize() const {
1769
39.8k
  return MaximumJumpTableSize;
1770
39.8k
}
1771
1772
46
void TargetLoweringBase::setMaximumJumpTableSize(unsigned Val) {
1773
46
  MaximumJumpTableSize = Val;
1774
46
}
1775
1776
//===----------------------------------------------------------------------===//
1777
//  Reciprocal Estimates
1778
//===----------------------------------------------------------------------===//
1779
1780
/// Get the reciprocal estimate attribute string for a function that will
1781
/// override the target defaults.
1782
2.46k
static StringRef getRecipEstimateForFunc(MachineFunction &MF) {
1783
2.46k
  const Function &F = MF.getFunction();
1784
2.46k
  return F.getFnAttribute("reciprocal-estimates").getValueAsString();
1785
2.46k
}
1786
1787
/// Construct a string for the given reciprocal operation of the given type.
1788
/// This string should match the corresponding option to the front-end's
1789
/// "-mrecip" flag assuming those strings have been passed through in an
1790
/// attribute string. For example, "vec-divf" for a division of a vXf32.
1791
1.03k
static std::string getReciprocalOpName(bool IsSqrt, EVT VT) {
1792
1.03k
  std::string Name = VT.isVector() ? 
"vec-"831
:
""208
;
1793
1.03k
1794
1.03k
  Name += IsSqrt ? 
"sqrt"143
:
"div"896
;
1795
1.03k
1796
1.03k
  // TODO: Handle "half" or other float types?
1797
1.03k
  if (VT.getScalarType() == MVT::f64) {
1798
32
    Name += "d";
1799
1.00k
  } else {
1800
1.00k
    assert(VT.getScalarType() == MVT::f32 &&
1801
1.00k
           "Unexpected FP type for reciprocal estimate");
1802
1.00k
    Name += "f";
1803
1.00k
  }
1804
1.03k
1805
1.03k
  return Name;
1806
1.03k
}
1807
1808
/// Return the character position and value (a single numeric character) of a
1809
/// customized refinement operation in the input string if it exists. Return
1810
/// false if there is no customized refinement step count.
1811
static bool parseRefinementStep(StringRef In, size_t &Position,
1812
2.02k
                                uint8_t &Value) {
1813
2.02k
  const char RefStepToken = ':';
1814
2.02k
  Position = In.find(RefStepToken);
1815
2.02k
  if (Position == StringRef::npos)
1816
1.17k
    return false;
1817
850
1818
850
  StringRef RefStepString = In.substr(Position + 1);
1819
850
  // Allow exactly one numeric character for the additional refinement
1820
850
  // step parameter.
1821
850
  if (RefStepString.size() == 1) {
1822
850
    char RefStepChar = RefStepString[0];
1823
850
    if (RefStepChar >= '0' && RefStepChar <= '9') {
1824
850
      Value = RefStepChar - '0';
1825
850
      return true;
1826
850
    }
1827
0
  }
1828
0
  report_fatal_error("Invalid refinement step for -recip.");
1829
0
}
1830
1831
/// For the input attribute string, return one of the ReciprocalEstimate enum
1832
/// status values (enabled, disabled, or not specified) for this operation on
1833
/// the specified data type.
1834
1.29k
static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) {
1835
1.29k
  if (Override.empty())
1836
713
    return TargetLoweringBase::ReciprocalEstimate::Unspecified;
1837
581
1838
581
  SmallVector<StringRef, 4> OverrideVector;
1839
581
  Override.split(OverrideVector, ',');
1840
581
  unsigned NumArgs = OverrideVector.size();
1841
581
1842
581
  // Check if "all", "none", or "default" was specified.
1843
581
  if (NumArgs == 1) {
1844
2
    // Look for an optional setting of the number of refinement steps needed
1845
2
    // for this type of reciprocal operation.
1846
2
    size_t RefPos;
1847
2
    uint8_t RefSteps;
1848
2
    if (parseRefinementStep(Override, RefPos, RefSteps)) {
1849
2
      // Split the string for further processing.
1850
2
      Override = Override.substr(0, RefPos);
1851
2
    }
1852
2
1853
2
    // All reciprocal types are enabled.
1854
2
    if (Override == "all")
1855
0
      return TargetLoweringBase::ReciprocalEstimate::Enabled;
1856
2
1857
2
    // All reciprocal types are disabled.
1858
2
    if (Override == "none")
1859
0
      return TargetLoweringBase::ReciprocalEstimate::Disabled;
1860
2
1861
2
    // Target defaults for enablement are used.
1862
2
    if (Override == "default")
1863
0
      return TargetLoweringBase::ReciprocalEstimate::Unspecified;
1864
581
  }
1865
581
1866
581
  // The attribute string may omit the size suffix ('f'/'d').
1867
581
  std::string VTName = getReciprocalOpName(IsSqrt, VT);
1868
581
  std::string VTNameNoSize = VTName;
1869
581
  VTNameNoSize.pop_back();
1870
581
  static const char DisabledPrefix = '!';
1871
581
1872
1.11k
  for (StringRef RecipType : OverrideVector) {
1873
1.11k
    size_t RefPos;
1874
1.11k
    uint8_t RefSteps;
1875
1.11k
    if (parseRefinementStep(RecipType, RefPos, RefSteps))
1876
423
      RecipType = RecipType.substr(0, RefPos);
1877
1.11k
1878
1.11k
    // Ignore the disablement token for string matching.
1879
1.11k
    bool IsDisabled = RecipType[0] == DisabledPrefix;
1880
1.11k
    if (IsDisabled)
1881
276
      RecipType = RecipType.substr(1);
1882
1.11k
1883
1.11k
    if (RecipType.equals(VTName) || 
RecipType.equals(VTNameNoSize)590
)
1884
575
      return IsDisabled ? 
TargetLoweringBase::ReciprocalEstimate::Disabled123
1885
575
                        : 
TargetLoweringBase::ReciprocalEstimate::Enabled452
;
1886
1.11k
  }
1887
581
1888
581
  
return TargetLoweringBase::ReciprocalEstimate::Unspecified6
;
1889
581
}
1890
1891
/// For the input attribute string, return the customized refinement step count
1892
/// for this operation on the specified data type. If the step count does not
1893
/// exist, return the ReciprocalEstimate enum value for unspecified.
1894
1.17k
static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) {
1895
1.17k
  if (Override.empty())
1896
713
    return TargetLoweringBase::ReciprocalEstimate::Unspecified;
1897
458
1898
458
  SmallVector<StringRef, 4> OverrideVector;
1899
458
  Override.split(OverrideVector, ',');
1900
458
  unsigned NumArgs = OverrideVector.size();
1901
458
1902
458
  // Check if "all", "default", or "none" was specified.
1903
458
  if (NumArgs == 1) {
1904
2
    // Look for an optional setting of the number of refinement steps needed
1905
2
    // for this type of reciprocal operation.
1906
2
    size_t RefPos;
1907
2
    uint8_t RefSteps;
1908
2
    if (!parseRefinementStep(Override, RefPos, RefSteps))
1909
0
      return TargetLoweringBase::ReciprocalEstimate::Unspecified;
1910
2
1911
2
    // Split the string for further processing.
1912
2
    Override = Override.substr(0, RefPos);
1913
2
    assert(Override != "none" &&
1914
2
           "Disabled reciprocals, but specifed refinement steps?");
1915
2
1916
2
    // If this is a general override, return the specified number of steps.
1917
2
    if (Override == "all" || Override == "default")
1918
0
      return RefSteps;
1919
458
  }
1920
458
1921
458
  // The attribute string may omit the size suffix ('f'/'d').
1922
458
  std::string VTName = getReciprocalOpName(IsSqrt, VT);
1923
458
  std::string VTNameNoSize = VTName;
1924
458
  VTNameNoSize.pop_back();
1925
458
1926
901
  for (StringRef RecipType : OverrideVector) {
1927
901
    size_t RefPos;
1928
901
    uint8_t RefSteps;
1929
901
    if (!parseRefinementStep(RecipType, RefPos, RefSteps))
1930
478
      continue;
1931
423
1932
423
    RecipType = RecipType.substr(0, RefPos);
1933
423
    if (RecipType.equals(VTName) || 
RecipType.equals(VTNameNoSize)203
)
1934
222
      return RefSteps;
1935
423
  }
1936
458
1937
458
  
return TargetLoweringBase::ReciprocalEstimate::Unspecified236
;
1938
458
}
1939
1940
int TargetLoweringBase::getRecipEstimateSqrtEnabled(EVT VT,
1941
267
                                                    MachineFunction &MF) const {
1942
267
  return getOpEnabled(true, VT, getRecipEstimateForFunc(MF));
1943
267
}
1944
1945
int TargetLoweringBase::getRecipEstimateDivEnabled(EVT VT,
1946
1.02k
                                                   MachineFunction &MF) const {
1947
1.02k
  return getOpEnabled(false, VT, getRecipEstimateForFunc(MF));
1948
1.02k
}
1949
1950
int TargetLoweringBase::getSqrtRefinementSteps(EVT VT,
1951
224
                                               MachineFunction &MF) const {
1952
224
  return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF));
1953
224
}
1954
1955
int TargetLoweringBase::getDivRefinementSteps(EVT VT,
1956
947
                                              MachineFunction &MF) const {
1957
947
  return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF));
1958
947
}
1959
1960
1.00M
void TargetLoweringBase::finalizeLowering(MachineFunction &MF) const {
1961
1.00M
  MF.getRegInfo().freezeReservedRegs(MF);
1962
1.00M
}