Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file This file contains the AArch64 implementation of the DAG scheduling
10
///  mutation to pair instructions back to back.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "AArch64Subtarget.h"
15
#include "llvm/CodeGen/MacroFusion.h"
16
#include "llvm/CodeGen/TargetInstrInfo.h"
17
18
using namespace llvm;
19
20
namespace {
21
22
/// CMN, CMP, TST followed by Bcc
23
static bool isArithmeticBccPair(const MachineInstr *FirstMI,
24
11.9M
                                const MachineInstr &SecondMI) {
25
11.9M
  if (SecondMI.getOpcode() != AArch64::Bcc)
26
11.1M
    return false;
27
781k
28
781k
  // Assume the 1st instr to be a wildcard if it is unspecified.
29
781k
  if (FirstMI == nullptr)
30
388k
    return true;
31
392k
32
392k
  switch (FirstMI->getOpcode()) {
33
392k
  case AArch64::ADDSWri:
34
372k
  case AArch64::ADDSWrr:
35
372k
  case AArch64::ADDSXri:
36
372k
  case AArch64::ADDSXrr:
37
372k
  case AArch64::ANDSWri:
38
372k
  case AArch64::ANDSWrr:
39
372k
  case AArch64::ANDSXri:
40
372k
  case AArch64::ANDSXrr:
41
372k
  case AArch64::SUBSWri:
42
372k
  case AArch64::SUBSWrr:
43
372k
  case AArch64::SUBSXri:
44
372k
  case AArch64::SUBSXrr:
45
372k
  case AArch64::BICSWrr:
46
372k
  case AArch64::BICSXrr:
47
372k
    return true;
48
372k
  case AArch64::ADDSWrs:
49
1.63k
  case AArch64::ADDSXrs:
50
1.63k
  case AArch64::ANDSWrs:
51
1.63k
  case AArch64::ANDSXrs:
52
1.63k
  case AArch64::SUBSWrs:
53
1.63k
  case AArch64::SUBSXrs:
54
1.63k
  case AArch64::BICSWrs:
55
1.63k
  case AArch64::BICSXrs:
56
1.63k
    // Shift value can be 0 making these behave like the "rr" variant...
57
1.63k
    return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
58
18.5k
  }
59
18.5k
60
18.5k
  return false;
61
18.5k
}
62
63
/// ALU operations followed by CBZ/CBNZ.
64
static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
65
11.1M
                                const MachineInstr &SecondMI) {
66
11.1M
  if (SecondMI.getOpcode() != AArch64::CBZW &&
67
11.1M
      
SecondMI.getOpcode() != AArch64::CBZX11.0M
&&
68
11.1M
      
SecondMI.getOpcode() != AArch64::CBNZW11.0M
&&
69
11.1M
      
SecondMI.getOpcode() != AArch64::CBNZX10.8M
)
70
10.7M
    return false;
71
399k
72
399k
  // Assume the 1st instr to be a wildcard if it is unspecified.
73
399k
  if (FirstMI == nullptr)
74
185k
    return true;
75
214k
76
214k
  switch (FirstMI->getOpcode()) {
77
214k
  case AArch64::ADDWri:
78
6.03k
  case AArch64::ADDWrr:
79
6.03k
  case AArch64::ADDXri:
80
6.03k
  case AArch64::ADDXrr:
81
6.03k
  case AArch64::ANDWri:
82
6.03k
  case AArch64::ANDWrr:
83
6.03k
  case AArch64::ANDXri:
84
6.03k
  case AArch64::ANDXrr:
85
6.03k
  case AArch64::EORWri:
86
6.03k
  case AArch64::EORWrr:
87
6.03k
  case AArch64::EORXri:
88
6.03k
  case AArch64::EORXrr:
89
6.03k
  case AArch64::ORRWri:
90
6.03k
  case AArch64::ORRWrr:
91
6.03k
  case AArch64::ORRXri:
92
6.03k
  case AArch64::ORRXrr:
93
6.03k
  case AArch64::SUBWri:
94
6.03k
  case AArch64::SUBWrr:
95
6.03k
  case AArch64::SUBXri:
96
6.03k
  case AArch64::SUBXrr:
97
6.03k
    return true;
98
6.03k
  case AArch64::ADDWrs:
99
15
  case AArch64::ADDXrs:
100
15
  case AArch64::ANDWrs:
101
15
  case AArch64::ANDXrs:
102
15
  case AArch64::SUBWrs:
103
15
  case AArch64::SUBXrs:
104
15
  case AArch64::BICWrs:
105
15
  case AArch64::BICXrs:
106
15
    // Shift value can be 0 making these behave like the "rr" variant...
107
15
    return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
108
208k
  }
109
208k
110
208k
  return false;
111
208k
}
112
113
/// AES crypto encoding or decoding.
114
static bool isAESPair(const MachineInstr *FirstMI,
115
11.1M
                      const MachineInstr &SecondMI) {
116
11.1M
  // Assume the 1st instr to be a wildcard if it is unspecified.
117
11.1M
  switch (SecondMI.getOpcode()) {
118
11.1M
  // AES encode.
119
11.1M
  case AArch64::AESMCrr:
120
408
  case AArch64::AESMCrrTied:
121
408
    return FirstMI == nullptr || 
FirstMI->getOpcode() == AArch64::AESErr204
;
122
408
  // AES decode.
123
408
  case AArch64::AESIMCrr:
124
328
  case AArch64::AESIMCrrTied:
125
328
    return FirstMI == nullptr || 
FirstMI->getOpcode() == AArch64::AESDrr164
;
126
11.1M
  }
127
11.1M
128
11.1M
  return false;
129
11.1M
}
130
131
/// AESE/AESD/PMULL + EOR.
132
static bool isCryptoEORPair(const MachineInstr *FirstMI,
133
10.9M
                            const MachineInstr &SecondMI) {
134
10.9M
  if (SecondMI.getOpcode() != AArch64::EORv16i8)
135
10.9M
    return false;
136
511
137
511
  // Assume the 1st instr to be a wildcard if it is unspecified.
138
511
  if (FirstMI == nullptr)
139
205
    return true;
140
306
141
306
  switch (FirstMI->getOpcode()) {
142
306
  case AArch64::AESErr:
143
0
  case AArch64::AESDrr:
144
0
  case AArch64::PMULLv16i8:
145
0
  case AArch64::PMULLv8i8:
146
0
  case AArch64::PMULLv1i64:
147
0
  case AArch64::PMULLv2i64:
148
0
    return true;
149
306
  }
150
306
151
306
  return false;
152
306
}
153
154
/// Literal generation.
155
static bool isLiteralsPair(const MachineInstr *FirstMI,
156
5.51k
                           const MachineInstr &SecondMI) {
157
5.51k
  // Assume the 1st instr to be a wildcard if it is unspecified.
158
5.51k
159
5.51k
  // PC relative address.
160
5.51k
  if ((FirstMI == nullptr || 
FirstMI->getOpcode() == AArch64::ADRP539
) &&
161
5.51k
      
SecondMI.getOpcode() == AArch64::ADDXri5.08k
)
162
56
    return true;
163
5.46k
164
5.46k
  // 32 bit immediate.
165
5.46k
  if ((FirstMI == nullptr || 
FirstMI->getOpcode() == AArch64::MOVZWi525
) &&
166
5.46k
      
(4.94k
SecondMI.getOpcode() == AArch64::MOVKWi4.94k
&&
167
4.94k
       
SecondMI.getOperand(3).getImm() == 1610
))
168
10
    return true;
169
5.45k
170
5.45k
  // Lower half of 64 bit immediate.
171
5.45k
  if((FirstMI == nullptr || 
FirstMI->getOpcode() == AArch64::MOVZXi520
) &&
172
5.45k
     
(4.94k
SecondMI.getOpcode() == AArch64::MOVKXi4.94k
&&
173
4.94k
      
SecondMI.getOperand(3).getImm() == 1640
))
174
20
    return true;
175
5.43k
176
5.43k
  // Upper half of 64 bit immediate.
177
5.43k
  if ((FirstMI == nullptr ||
178
5.43k
       
(510
FirstMI->getOpcode() == AArch64::MOVKXi510
&&
179
510
        
FirstMI->getOperand(3).getImm() == 3210
)) &&
180
5.43k
      
(4.93k
SecondMI.getOpcode() == AArch64::MOVKXi4.93k
&&
181
4.93k
       
SecondMI.getOperand(3).getImm() == 4830
))
182
20
    return true;
183
5.41k
184
5.41k
  return false;
185
5.41k
}
186
187
/// Fuse address generation and loads or stores.
188
static bool isAddressLdStPair(const MachineInstr *FirstMI,
189
3.91k
                              const MachineInstr &SecondMI) {
190
3.91k
  switch (SecondMI.getOpcode()) {
191
3.91k
  case AArch64::STRBBui:
192
741
  case AArch64::STRBui:
193
741
  case AArch64::STRDui:
194
741
  case AArch64::STRHHui:
195
741
  case AArch64::STRHui:
196
741
  case AArch64::STRQui:
197
741
  case AArch64::STRSui:
198
741
  case AArch64::STRWui:
199
741
  case AArch64::STRXui:
200
741
  case AArch64::LDRBBui:
201
741
  case AArch64::LDRBui:
202
741
  case AArch64::LDRDui:
203
741
  case AArch64::LDRHHui:
204
741
  case AArch64::LDRHui:
205
741
  case AArch64::LDRQui:
206
741
  case AArch64::LDRSui:
207
741
  case AArch64::LDRWui:
208
741
  case AArch64::LDRXui:
209
741
  case AArch64::LDRSBWui:
210
741
  case AArch64::LDRSBXui:
211
741
  case AArch64::LDRSHWui:
212
741
  case AArch64::LDRSHXui:
213
741
  case AArch64::LDRSWui:
214
741
    // Assume the 1st instr to be a wildcard if it is unspecified.
215
741
    if (FirstMI == nullptr)
216
256
      return true;
217
485
218
485
   switch (FirstMI->getOpcode()) {
219
485
    case AArch64::ADR:
220
0
      return SecondMI.getOperand(2).getImm() == 0;
221
485
    case AArch64::ADRP:
222
120
      return true;
223
3.53k
    }
224
3.53k
  }
225
3.53k
226
3.53k
  return false;
227
3.53k
}
228
229
/// Compare and conditional select.
230
static bool isCCSelectPair(const MachineInstr *FirstMI,
231
3.48k
                           const MachineInstr &SecondMI) {
232
3.48k
  // 32 bits
233
3.48k
  if (SecondMI.getOpcode() == AArch64::CSELWr) {
234
16
    // Assume the 1st instr to be a wildcard if it is unspecified.
235
16
    if (FirstMI == nullptr)
236
8
      return true;
237
8
238
8
    if (FirstMI->definesRegister(AArch64::WZR))
239
8
      switch (FirstMI->getOpcode()) {
240
8
      case AArch64::SUBSWrs:
241
0
        return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
242
8
      case AArch64::SUBSWrx:
243
0
        return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
244
8
      case AArch64::SUBSWrr:
245
8
      case AArch64::SUBSWri:
246
8
        return true;
247
3.47k
      }
248
8
  }
249
3.47k
250
3.47k
  // 64 bits
251
3.47k
  if (SecondMI.getOpcode() == AArch64::CSELXr) {
252
16
    // Assume the 1st instr to be a wildcard if it is unspecified.
253
16
    if (FirstMI == nullptr)
254
8
      return true;
255
8
256
8
    if (FirstMI->definesRegister(AArch64::XZR))
257
8
      switch (FirstMI->getOpcode()) {
258
8
      case AArch64::SUBSXrs:
259
0
        return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
260
8
      case AArch64::SUBSXrx:
261
0
      case AArch64::SUBSXrx64:
262
0
        return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
263
8
      case AArch64::SUBSXrr:
264
8
      case AArch64::SUBSXri:
265
8
        return true;
266
3.45k
      }
267
8
  }
268
3.45k
269
3.45k
  return false;
270
3.45k
}
271
272
// Arithmetic and logic.
273
static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
274
690
                                  const MachineInstr &SecondMI) {
275
690
  if (AArch64InstrInfo::hasShiftedReg(SecondMI))
276
0
    return false;
277
690
278
690
  switch (SecondMI.getOpcode()) {
279
690
  // Arithmetic
280
690
  case AArch64::ADDWrr:
281
44
  case AArch64::ADDXrr:
282
44
  case AArch64::SUBWrr:
283
44
  case AArch64::SUBXrr:
284
44
  case AArch64::ADDWrs:
285
44
  case AArch64::ADDXrs:
286
44
  case AArch64::SUBWrs:
287
44
  case AArch64::SUBXrs:
288
44
  // Logic
289
44
  case AArch64::ANDWrr:
290
44
  case AArch64::ANDXrr:
291
44
  case AArch64::BICWrr:
292
44
  case AArch64::BICXrr:
293
44
  case AArch64::EONWrr:
294
44
  case AArch64::EONXrr:
295
44
  case AArch64::EORWrr:
296
44
  case AArch64::EORXrr:
297
44
  case AArch64::ORNWrr:
298
44
  case AArch64::ORNXrr:
299
44
  case AArch64::ORRWrr:
300
44
  case AArch64::ORRXrr:
301
44
  case AArch64::ANDWrs:
302
44
  case AArch64::ANDXrs:
303
44
  case AArch64::BICWrs:
304
44
  case AArch64::BICXrs:
305
44
  case AArch64::EONWrs:
306
44
  case AArch64::EONXrs:
307
44
  case AArch64::EORWrs:
308
44
  case AArch64::EORXrs:
309
44
  case AArch64::ORNWrs:
310
44
  case AArch64::ORNXrs:
311
44
  case AArch64::ORRWrs:
312
44
  case AArch64::ORRXrs:
313
44
    // Assume the 1st instr to be a wildcard if it is unspecified.
314
44
    if (FirstMI == nullptr)
315
20
      return true;
316
24
317
24
    // Arithmetic
318
24
    switch (FirstMI->getOpcode()) {
319
24
    case AArch64::ADDWrr:
320
4
    case AArch64::ADDXrr:
321
4
    case AArch64::ADDSWrr:
322
4
    case AArch64::ADDSXrr:
323
4
    case AArch64::SUBWrr:
324
4
    case AArch64::SUBXrr:
325
4
    case AArch64::SUBSWrr:
326
4
    case AArch64::SUBSXrr:
327
4
      return true;
328
4
    case AArch64::ADDWrs:
329
4
    case AArch64::ADDXrs:
330
4
    case AArch64::ADDSWrs:
331
4
    case AArch64::ADDSXrs:
332
4
    case AArch64::SUBWrs:
333
4
    case AArch64::SUBXrs:
334
4
    case AArch64::SUBSWrs:
335
4
    case AArch64::SUBSXrs:
336
4
      return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
337
16
    }
338
16
    break;
339
16
340
16
  // Arithmetic, setting flags.
341
16
  case AArch64::ADDSWrr:
342
0
  case AArch64::ADDSXrr:
343
0
  case AArch64::SUBSWrr:
344
0
  case AArch64::SUBSXrr:
345
0
  case AArch64::ADDSWrs:
346
0
  case AArch64::ADDSXrs:
347
0
  case AArch64::SUBSWrs:
348
0
  case AArch64::SUBSXrs:
349
0
    // Assume the 1st instr to be a wildcard if it is unspecified.
350
0
    if (FirstMI == nullptr)
351
0
      return true;
352
0
353
0
    // Arithmetic, not setting flags.
354
0
    switch (FirstMI->getOpcode()) {
355
0
    case AArch64::ADDWrr:
356
0
    case AArch64::ADDXrr:
357
0
    case AArch64::SUBWrr:
358
0
    case AArch64::SUBXrr:
359
0
      return true;
360
0
    case AArch64::ADDWrs:
361
0
    case AArch64::ADDXrs:
362
0
    case AArch64::SUBWrs:
363
0
    case AArch64::SUBXrs:
364
0
      return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
365
0
    }
366
0
    break;
367
662
  }
368
662
369
662
  return false;
370
662
}
371
372
/// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
373
/// together. Given SecondMI, when FirstMI is unspecified, then check if
374
/// SecondMI may be part of a fused pair at all.
375
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
376
                                   const TargetSubtargetInfo &TSI,
377
                                   const MachineInstr *FirstMI,
378
12.0M
                                   const MachineInstr &SecondMI) {
379
12.0M
  const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
380
12.0M
381
12.0M
  // All checking functions assume that the 1st instr is a wildcard if it is
382
12.0M
  // unspecified.
383
12.0M
  if (ST.hasArithmeticBccFusion() && 
isArithmeticBccPair(FirstMI, SecondMI)11.9M
)
384
760k
    return true;
385
11.3M
  if (ST.hasArithmeticCbzFusion() && 
isArithmeticCbzPair(FirstMI, SecondMI)11.1M
)
386
191k
    return true;
387
11.1M
  if (ST.hasFuseAES() && 
isAESPair(FirstMI, SecondMI)11.1M
)
388
728
    return true;
389
11.1M
  if (ST.hasFuseCryptoEOR() && 
isCryptoEORPair(FirstMI, SecondMI)10.9M
)
390
205
    return true;
391
11.1M
  if (ST.hasFuseLiterals() && 
isLiteralsPair(FirstMI, SecondMI)5.51k
)
392
106
    return true;
393
11.1M
  if (ST.hasFuseAddress() && 
isAddressLdStPair(FirstMI, SecondMI)3.91k
)
394
376
    return true;
395
11.1M
  if (ST.hasFuseCCSelect() && 
isCCSelectPair(FirstMI, SecondMI)3.48k
)
396
32
    return true;
397
11.1M
  if (ST.hasFuseArithmeticLogic() && 
isArithmeticLogicPair(FirstMI, SecondMI)690
)
398
28
    return true;
399
11.1M
400
11.1M
  return false;
401
11.1M
}
402
403
} // end namespace
404
405
406
namespace llvm {
407
408
268k
std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () {
409
268k
  return createMacroFusionDAGMutation(shouldScheduleAdjacent);
410
268k
}
411
412
} // end namespace llvm