Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Target/PowerPC/PPCHazardRecognizers.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- PPCHazardRecognizers.cpp - PowerPC Hazard Recognizer Impls --------===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
// This file implements hazard recognizers for scheduling on PowerPC processors.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "PPCHazardRecognizers.h"
15
#include "PPC.h"
16
#include "PPCInstrInfo.h"
17
#include "PPCTargetMachine.h"
18
#include "llvm/CodeGen/ScheduleDAG.h"
19
#include "llvm/Support/Debug.h"
20
#include "llvm/Support/ErrorHandling.h"
21
#include "llvm/Support/raw_ostream.h"
22
using namespace llvm;
23
24
#define DEBUG_TYPE "pre-RA-sched"
25
26
65.4k
bool PPCDispatchGroupSBHazardRecognizer::isLoadAfterStore(SUnit *SU) {
27
65.4k
  // FIXME: Move this.
28
65.4k
  if (isBCTRAfterSet(SU))
29
0
    return true;
30
65.4k
31
65.4k
  const MCInstrDesc *MCID = DAG->getInstrDesc(SU);
32
65.4k
  if (!MCID)
33
0
    return false;
34
65.4k
35
65.4k
  
if (65.4k
!MCID->mayLoad()65.4k
)
36
52.0k
    return false;
37
13.3k
38
13.3k
  // SU is a load; for any predecessors in this dispatch group, that are stores,
39
13.3k
  // and with which we have an ordering dependency, return true.
40
28.2k
  
for (unsigned i = 0, ie = (unsigned) SU->Preds.size(); 13.3k
i != ie28.2k
;
++i14.9k
) {
41
15.3k
    const MCInstrDesc *PredMCID = DAG->getInstrDesc(SU->Preds[i].getSUnit());
42
15.3k
    if (
!PredMCID || 15.3k
!PredMCID->mayStore()15.3k
)
43
11.2k
      continue;
44
4.09k
45
4.09k
    
if (4.09k
!SU->Preds[i].isNormalMemory() && 4.09k
!SU->Preds[i].isBarrier()2.18k
)
46
1.75k
      continue;
47
2.34k
48
4.25k
    
for (unsigned j = 0, je = CurGroup.size(); 2.34k
j != je4.25k
;
++j1.91k
)
49
2.31k
      
if (2.31k
SU->Preds[i].getSUnit() == CurGroup[j]2.31k
)
50
401
        return true;
51
15.3k
  }
52
13.3k
53
12.9k
  return false; 
54
65.4k
}
55
56
65.4k
bool PPCDispatchGroupSBHazardRecognizer::isBCTRAfterSet(SUnit *SU) {
57
65.4k
  const MCInstrDesc *MCID = DAG->getInstrDesc(SU);
58
65.4k
  if (!MCID)
59
0
    return false;
60
65.4k
61
65.4k
  
if (65.4k
!MCID->isBranch()65.4k
)
62
65.3k
    return false;
63
36
64
36
  // SU is a branch; for any predecessors in this dispatch group, with which we
65
36
  // have a data dependence and set the counter register, return true.
66
108
  
for (unsigned i = 0, ie = (unsigned) SU->Preds.size(); 36
i != ie108
;
++i72
) {
67
72
    const MCInstrDesc *PredMCID = DAG->getInstrDesc(SU->Preds[i].getSUnit());
68
72
    if (
!PredMCID || 72
PredMCID->getSchedClass() != PPC::Sched::IIC_SprMTSPR72
)
69
72
      continue;
70
0
71
0
    
if (0
SU->Preds[i].isCtrl()0
)
72
0
      continue;
73
0
74
0
    
for (unsigned j = 0, je = CurGroup.size(); 0
j != je0
;
++j0
)
75
0
      
if (0
SU->Preds[i].getSUnit() == CurGroup[j]0
)
76
0
        return true;
77
72
  }
78
36
79
36
  return false; 
80
65.4k
}
81
82
// FIXME: Remove this when we don't need this:
83
namespace llvm { namespace PPC { extern int getNonRecordFormOpcode(uint16_t); } }
84
85
// FIXME: A lot of code in PPCDispatchGroupSBHazardRecognizer is P7 specific.
86
87
bool PPCDispatchGroupSBHazardRecognizer::mustComeFirst(const MCInstrDesc *MCID,
88
46.3k
                                                       unsigned &NSlots) {
89
46.3k
  // FIXME: Indirectly, this information is contained in the itinerary, and
90
46.3k
  // we should derive it from there instead of separately specifying it
91
46.3k
  // here.
92
46.3k
  unsigned IIC = MCID->getSchedClass();
93
46.3k
  switch (IIC) {
94
44.0k
  default:
95
44.0k
    NSlots = 1;
96
44.0k
    break;
97
143
  case PPC::Sched::IIC_IntDivW:
98
143
  case PPC::Sched::IIC_IntDivD:
99
143
  case PPC::Sched::IIC_LdStLoadUpd:
100
143
  case PPC::Sched::IIC_LdStLDU:
101
143
  case PPC::Sched::IIC_LdStLFDU:
102
143
  case PPC::Sched::IIC_LdStLFDUX:
103
143
  case PPC::Sched::IIC_LdStLHA:
104
143
  case PPC::Sched::IIC_LdStLHAU:
105
143
  case PPC::Sched::IIC_LdStLWA:
106
143
  case PPC::Sched::IIC_LdStSTDU:
107
143
  case PPC::Sched::IIC_LdStSTFDU:
108
143
    NSlots = 2;
109
143
    break;
110
2.17k
  case PPC::Sched::IIC_LdStLoadUpdX:
111
2.17k
  case PPC::Sched::IIC_LdStLDUX:
112
2.17k
  case PPC::Sched::IIC_LdStLHAUX:
113
2.17k
  case PPC::Sched::IIC_LdStLWARX:
114
2.17k
  case PPC::Sched::IIC_LdStLDARX:
115
2.17k
  case PPC::Sched::IIC_LdStSTDUX:
116
2.17k
  case PPC::Sched::IIC_LdStSTDCX:
117
2.17k
  case PPC::Sched::IIC_LdStSTWCX:
118
2.17k
  case PPC::Sched::IIC_BrMCRX: // mtcr
119
2.17k
  // FIXME: Add sync/isync (here and in the itinerary).
120
2.17k
    NSlots = 4;
121
2.17k
    break;
122
46.3k
  }
123
46.3k
124
46.3k
  // FIXME: record-form instructions need a different itinerary class.
125
46.3k
  
if (46.3k
NSlots == 1 && 46.3k
PPC::getNonRecordFormOpcode(MCID->getOpcode()) != -144.0k
)
126
128
    NSlots = 2;
127
46.3k
128
46.3k
  switch (IIC) {
129
45.1k
  default:
130
45.1k
    // All multi-slot instructions must come first.
131
45.1k
    return NSlots > 1;
132
1.24k
  case PPC::Sched::IIC_BrCR: // cr logicals
133
1.24k
  case PPC::Sched::IIC_SprMFCR:
134
1.24k
  case PPC::Sched::IIC_SprMFCRF:
135
1.24k
  case PPC::Sched::IIC_SprMTSPR:
136
1.24k
    return true;
137
0
  }
138
0
}
139
140
ScheduleHazardRecognizer::HazardType
141
41.2k
PPCDispatchGroupSBHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
142
41.2k
  if (
Stalls == 0 && 41.2k
isLoadAfterStore(SU)41.2k
)
143
401
    return NoopHazard;
144
40.8k
145
40.8k
  return ScoreboardHazardRecognizer::getHazardType(SU, Stalls);
146
40.8k
}
147
148
24.1k
bool PPCDispatchGroupSBHazardRecognizer::ShouldPreferAnother(SUnit *SU) {
149
24.1k
  const MCInstrDesc *MCID = DAG->getInstrDesc(SU);
150
24.1k
  unsigned NSlots;
151
24.1k
  if (
MCID && 24.1k
mustComeFirst(MCID, NSlots)24.1k
&&
CurSlots1.99k
)
152
959
    return true;
153
23.2k
154
23.2k
  return ScoreboardHazardRecognizer::ShouldPreferAnother(SU);
155
23.2k
}
156
157
24.1k
unsigned PPCDispatchGroupSBHazardRecognizer::PreEmitNoops(SUnit *SU) {
158
24.1k
  // We only need to fill out a maximum of 5 slots here: The 6th slot could
159
24.1k
  // only be a second branch, and otherwise the next instruction will start a
160
24.1k
  // new group.
161
24.1k
  if (
isLoadAfterStore(SU) && 24.1k
CurSlots < 60
) {
162
0
    unsigned Directive =
163
0
        DAG->MF.getSubtarget<PPCSubtarget>().getDarwinDirective();
164
0
    // If we're using a special group-terminating nop, then we need only one.
165
0
    // FIXME: the same for P9 as previous gen until POWER9 scheduling is ready
166
0
    if (
Directive == PPC::DIR_PWR6 || 0
Directive == PPC::DIR_PWR70
||
167
0
        
Directive == PPC::DIR_PWR80
||
Directive == PPC::DIR_PWR90
)
168
0
      return 1;
169
0
170
0
    return 5 - CurSlots;
171
0
  }
172
24.1k
173
24.1k
  return ScoreboardHazardRecognizer::PreEmitNoops(SU);
174
24.1k
}
175
176
24.1k
void PPCDispatchGroupSBHazardRecognizer::EmitInstruction(SUnit *SU) {
177
24.1k
  const MCInstrDesc *MCID = DAG->getInstrDesc(SU);
178
24.1k
  if (
MCID24.1k
) {
179
24.1k
    if (
CurSlots == 5 || 24.1k
(MCID->isBranch() && 22.2k
CurBranches == 117
)) {
180
1.95k
      CurGroup.clear();
181
1.95k
      CurSlots = CurBranches = 0;
182
24.1k
    } else {
183
22.2k
      DEBUG(dbgs() << "**** Adding to dispatch group: SU(" <<
184
22.2k
                      SU->NodeNum << "): ");
185
22.2k
      DEBUG(DAG->dumpNode(SU));
186
22.2k
187
22.2k
      unsigned NSlots;
188
22.2k
      bool MustBeFirst = mustComeFirst(MCID, NSlots);
189
22.2k
190
22.2k
      // If this instruction must come first, but does not, then it starts a
191
22.2k
      // new group.
192
22.2k
      if (
MustBeFirst && 22.2k
CurSlots1.69k
) {
193
659
        CurSlots = CurBranches = 0;
194
659
        CurGroup.clear();
195
659
      }
196
22.2k
197
22.2k
      CurSlots += NSlots;
198
22.2k
      CurGroup.push_back(SU);
199
22.2k
200
22.2k
      if (MCID->isBranch())
201
17
        ++CurBranches;
202
22.2k
    }
203
24.1k
  }
204
24.1k
205
24.1k
  return ScoreboardHazardRecognizer::EmitInstruction(SU);
206
24.1k
}
207
208
22.7k
void PPCDispatchGroupSBHazardRecognizer::AdvanceCycle() {
209
22.7k
  return ScoreboardHazardRecognizer::AdvanceCycle();
210
22.7k
}
211
212
0
void PPCDispatchGroupSBHazardRecognizer::RecedeCycle() {
213
0
  llvm_unreachable("Bottom-up scheduling not supported");
214
0
}
215
216
21.0k
void PPCDispatchGroupSBHazardRecognizer::Reset() {
217
21.0k
  CurGroup.clear();
218
21.0k
  CurSlots = CurBranches = 0;
219
21.0k
  return ScoreboardHazardRecognizer::Reset();
220
21.0k
}
221
222
288
void PPCDispatchGroupSBHazardRecognizer::EmitNoop() {
223
288
  unsigned Directive =
224
288
      DAG->MF.getSubtarget<PPCSubtarget>().getDarwinDirective();
225
288
  // If the group has now filled all of its slots, or if we're using a special
226
288
  // group-terminating nop, the group is complete.
227
288
  // FIXME: the same for P9 as previous gen until POWER9 scheduling is ready
228
288
  if (
Directive == PPC::DIR_PWR6 || 288
Directive == PPC::DIR_PWR7288
||
229
288
      
Directive == PPC::DIR_PWR8104
||
Directive == PPC::DIR_PWR90
||
230
288
      
CurSlots == 60
) {
231
288
    CurGroup.clear();
232
288
    CurSlots = CurBranches = 0;
233
288
  } else {
234
0
    CurGroup.push_back(nullptr);
235
0
    ++CurSlots;
236
0
  }
237
288
}
238
239
//===----------------------------------------------------------------------===//
240
// PowerPC 970 Hazard Recognizer
241
//
242
// This models the dispatch group formation of the PPC970 processor.  Dispatch
243
// groups are bundles of up to five instructions that can contain various mixes
244
// of instructions.  The PPC970 can dispatch a peak of 4 non-branch and one
245
// branch instruction per-cycle.
246
//
247
// There are a number of restrictions to dispatch group formation: some
248
// instructions can only be issued in the first slot of a dispatch group, & some
249
// instructions fill an entire dispatch group.  Additionally, only branches can
250
// issue in the 5th (last) slot.
251
//
252
// Finally, there are a number of "structural" hazards on the PPC970.  These
253
// conditions cause large performance penalties due to misprediction, recovery,
254
// and replay logic that has to happen.  These cases include setting a CTR and
255
// branching through it in the same dispatch group, and storing to an address,
256
// then loading from the same address within a dispatch group.  To avoid these
257
// conditions, we insert no-op instructions when appropriate.
258
//
259
// FIXME: This is missing some significant cases:
260
//   1. Modeling of microcoded instructions.
261
//   2. Handling of serialized operations.
262
//   3. Handling of the esoteric cases in "Resource-based Instruction Grouping".
263
//
264
265
PPCHazardRecognizer970::PPCHazardRecognizer970(const ScheduleDAG &DAG)
266
2.44k
    : DAG(DAG) {
267
2.44k
  EndDispatchGroup();
268
2.44k
}
269
270
19.0k
void PPCHazardRecognizer970::EndDispatchGroup() {
271
19.0k
  DEBUG(errs() << "=== Start of dispatch group\n");
272
19.0k
  NumIssued = 0;
273
19.0k
274
19.0k
  // Structural hazard info.
275
19.0k
  HasCTRSet = false;
276
19.0k
  NumStores = 0;
277
19.0k
}
278
279
280
PPCII::PPC970_Unit
281
PPCHazardRecognizer970::GetInstrType(unsigned Opcode,
282
                                     bool &isFirst, bool &isSingle,
283
                                     bool &isCracked,
284
43.5k
                                     bool &isLoad, bool &isStore) {
285
43.5k
  const MCInstrDesc &MCID = DAG.TII->get(Opcode);
286
43.5k
287
43.5k
  isLoad  = MCID.mayLoad();
288
43.5k
  isStore = MCID.mayStore();
289
43.5k
290
43.5k
  uint64_t TSFlags = MCID.TSFlags;
291
43.5k
292
43.5k
  isFirst   = TSFlags & PPCII::PPC970_First;
293
43.5k
  isSingle  = TSFlags & PPCII::PPC970_Single;
294
43.5k
  isCracked = TSFlags & PPCII::PPC970_Cracked;
295
43.5k
  return (PPCII::PPC970_Unit)(TSFlags & PPCII::PPC970_Mask);
296
43.5k
}
297
298
/// isLoadOfStoredAddress - If we have a load from the previously stored pointer
299
/// as indicated by StorePtr1/StorePtr2/StoreSize, return true.
300
bool PPCHazardRecognizer970::
301
isLoadOfStoredAddress(uint64_t LoadSize, int64_t LoadOffset,
302
880
  const Value *LoadValue) const {
303
970
  for (unsigned i = 0, e = NumStores; 
i != e970
;
++i90
) {
304
897
    // Handle exact and commuted addresses.
305
897
    if (
LoadValue == StoreValue[i] && 897
LoadOffset == StoreOffset[i]809
)
306
600
      return true;
307
297
308
297
    // Okay, we don't have an exact match, if this is an indexed offset, see if
309
297
    // we have overlap (which happens during fp->int conversion for example).
310
297
    
if (297
StoreValue[i] == LoadValue297
) {
311
209
      // Okay the base pointers match, so we have [c1+r] vs [c2+r].  Check
312
209
      // to see if the load and store actually overlap.
313
209
      if (
StoreOffset[i] < LoadOffset209
) {
314
108
        if (
int64_t(StoreOffset[i]+StoreSize[i]) > LoadOffset108
)
return true108
;
315
101
      } else {
316
101
        if (
int64_t(LoadOffset+LoadSize) > StoreOffset[i]101
)
return true99
;
317
101
      }
318
209
    }
319
897
  }
320
73
  return false;
321
880
}
322
323
/// getHazardType - We return hazard for any non-branch instruction that would
324
/// terminate the dispatch group.  We turn NoopHazard for any
325
/// instructions that wouldn't terminate the dispatch group that would cause a
326
/// pipeline flush.
327
ScheduleHazardRecognizer::HazardType PPCHazardRecognizer970::
328
26.4k
getHazardType(SUnit *SU, int Stalls) {
329
26.4k
  assert(Stalls == 0 && "PPC hazards don't support scoreboard lookahead");
330
26.4k
331
26.4k
  MachineInstr *MI = SU->getInstr();
332
26.4k
333
26.4k
  if (MI->isDebugValue())
334
0
    return NoHazard;
335
26.4k
336
26.4k
  unsigned Opcode = MI->getOpcode();
337
26.4k
  bool isFirst, isSingle, isCracked, isLoad, isStore;
338
26.4k
  PPCII::PPC970_Unit InstrType =
339
26.4k
    GetInstrType(Opcode, isFirst, isSingle, isCracked,
340
26.4k
                 isLoad, isStore);
341
26.4k
  if (
InstrType == PPCII::PPC970_Pseudo26.4k
)
return NoHazard3.74k
;
342
22.6k
343
22.6k
  // We can only issue a PPC970_First/PPC970_Single instruction (such as
344
22.6k
  // crand/mtspr/etc) if this is the first cycle of the dispatch group.
345
22.6k
  
if (22.6k
NumIssued != 0 && 22.6k
(isFirst || 16.7k
isSingle15.8k
))
346
923
    return Hazard;
347
21.7k
348
21.7k
  // If this instruction is cracked into two ops by the decoder, we know that
349
21.7k
  // it is not a branch and that it cannot issue if 3 other instructions are
350
21.7k
  // already in the dispatch group.
351
21.7k
  
if (21.7k
isCracked && 21.7k
NumIssued > 2304
)
352
96
    return Hazard;
353
21.6k
354
21.6k
  switch (InstrType) {
355
0
  
default: 0
llvm_unreachable0
("Unknown instruction type!");
356
21.5k
  case PPCII::PPC970_FXU:
357
21.5k
  case PPCII::PPC970_LSU:
358
21.5k
  case PPCII::PPC970_FPU:
359
21.5k
  case PPCII::PPC970_VALU:
360
21.5k
  case PPCII::PPC970_VPERM:
361
21.5k
    // We can only issue a branch as the last instruction in a group.
362
21.5k
    if (
NumIssued == 421.5k
)
return Hazard7.46k
;
363
14.0k
    break;
364
136
  case PPCII::PPC970_CRU:
365
136
    // We can only issue a CR instruction in the first two slots.
366
136
    if (
NumIssued >= 2136
)
return Hazard6
;
367
130
    break;
368
18
  case PPCII::PPC970_BRU:
369
18
    break;
370
14.2k
  }
371
14.2k
372
14.2k
  // Do not allow MTCTR and BCTRL to be in the same dispatch group.
373
14.2k
  
if (14.2k
HasCTRSet && 14.2k
Opcode == PPC::BCTRL0
)
374
0
    return NoopHazard;
375
14.2k
376
14.2k
  // If this is a load following a store, make sure it's not to the same or
377
14.2k
  // overlapping address.
378
14.2k
  
if (14.2k
isLoad && 14.2k
NumStores4.20k
&&
!MI->memoperands_empty()882
) {
379
880
    MachineMemOperand *MO = *MI->memoperands_begin();
380
880
    if (isLoadOfStoredAddress(MO->getSize(),
381
880
                              MO->getOffset(), MO->getValue()))
382
807
      return NoopHazard;
383
13.3k
  }
384
13.3k
385
13.3k
  return NoHazard;
386
13.3k
}
387
388
17.1k
void PPCHazardRecognizer970::EmitInstruction(SUnit *SU) {
389
17.1k
  MachineInstr *MI = SU->getInstr();
390
17.1k
391
17.1k
  if (MI->isDebugValue())
392
0
    return;
393
17.1k
394
17.1k
  unsigned Opcode = MI->getOpcode();
395
17.1k
  bool isFirst, isSingle, isCracked, isLoad, isStore;
396
17.1k
  PPCII::PPC970_Unit InstrType =
397
17.1k
    GetInstrType(Opcode, isFirst, isSingle, isCracked,
398
17.1k
                 isLoad, isStore);
399
17.1k
  if (
InstrType == PPCII::PPC970_Pseudo17.1k
)
return3.74k
;
400
13.3k
401
13.3k
  // Update structural hazard information.
402
13.3k
  
if (13.3k
Opcode == PPC::MTCTR || 13.3k
Opcode == PPC::MTCTR813.3k
)
HasCTRSet = true22
;
403
13.3k
404
13.3k
  // Track the address stored to.
405
13.3k
  if (
isStore && 13.3k
NumStores < 42.95k
&&
!MI->memoperands_empty()2.95k
) {
406
2.38k
    MachineMemOperand *MO = *MI->memoperands_begin();
407
2.38k
    StoreSize[NumStores] = MO->getSize();
408
2.38k
    StoreOffset[NumStores] = MO->getOffset();
409
2.38k
    StoreValue[NumStores] = MO->getValue();
410
2.38k
    ++NumStores;
411
2.38k
  }
412
13.3k
413
13.3k
  if (
InstrType == PPCII::PPC970_BRU || 13.3k
isSingle13.3k
)
414
63
    NumIssued = 4;  // Terminate a d-group.
415
13.3k
  ++NumIssued;
416
13.3k
417
13.3k
  // If this instruction is cracked into two ops by the decoder, remember that
418
13.3k
  // we issued two pieces.
419
13.3k
  if (isCracked)
420
208
    ++NumIssued;
421
13.3k
422
13.3k
  if (NumIssued == 5)
423
63
    EndDispatchGroup();
424
17.1k
}
425
426
14.9k
void PPCHazardRecognizer970::AdvanceCycle() {
427
14.9k
  assert(NumIssued < 5 && "Illegal dispatch group!");
428
14.9k
  ++NumIssued;
429
14.9k
  if (NumIssued == 5)
430
4.21k
    EndDispatchGroup();
431
14.9k
}
432
433
12.2k
void PPCHazardRecognizer970::Reset() {
434
12.2k
  EndDispatchGroup();
435
12.2k
}
436