Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/TargetInstrInfo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- TargetInstrInfo.cpp - Target Instruction Information --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements the TargetInstrInfo class.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "llvm/CodeGen/TargetInstrInfo.h"
14
#include "llvm/CodeGen/MachineFrameInfo.h"
15
#include "llvm/CodeGen/MachineInstrBuilder.h"
16
#include "llvm/CodeGen/MachineMemOperand.h"
17
#include "llvm/CodeGen/MachineRegisterInfo.h"
18
#include "llvm/CodeGen/PseudoSourceValue.h"
19
#include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
20
#include "llvm/CodeGen/StackMaps.h"
21
#include "llvm/CodeGen/TargetFrameLowering.h"
22
#include "llvm/CodeGen/TargetLowering.h"
23
#include "llvm/CodeGen/TargetRegisterInfo.h"
24
#include "llvm/CodeGen/TargetSchedule.h"
25
#include "llvm/IR/DataLayout.h"
26
#include "llvm/MC/MCAsmInfo.h"
27
#include "llvm/MC/MCInstrItineraries.h"
28
#include "llvm/Support/CommandLine.h"
29
#include "llvm/Support/ErrorHandling.h"
30
#include "llvm/Support/raw_ostream.h"
31
#include "llvm/Target/TargetMachine.h"
32
#include <cctype>
33
34
using namespace llvm;
35
36
static cl::opt<bool> DisableHazardRecognizer(
37
  "disable-sched-hazard", cl::Hidden, cl::init(false),
38
  cl::desc("Disable hazard detection during preRA scheduling"));
39
40
40.4k
TargetInstrInfo::~TargetInstrInfo() {
41
40.4k
}
42
43
const TargetRegisterClass*
44
TargetInstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum,
45
                             const TargetRegisterInfo *TRI,
46
44.8M
                             const MachineFunction &MF) const {
47
44.8M
  if (OpNum >= MCID.getNumOperands())
48
1.29M
    return nullptr;
49
43.5M
50
43.5M
  short RegClass = MCID.OpInfo[OpNum].RegClass;
51
43.5M
  if (MCID.OpInfo[OpNum].isLookupPtrRegClass())
52
2.09M
    return TRI->getPointerRegClass(MF, RegClass);
53
41.4M
54
41.4M
  // Instructions like INSERT_SUBREG do not have fixed register classes.
55
41.4M
  if (RegClass < 0)
56
5.82M
    return nullptr;
57
35.6M
58
35.6M
  // Otherwise just look it up normally.
59
35.6M
  return TRI->getRegClass(RegClass);
60
35.6M
}
61
62
/// insertNoop - Insert a noop into the instruction stream at the specified
63
/// point.
64
void TargetInstrInfo::insertNoop(MachineBasicBlock &MBB,
65
0
                                 MachineBasicBlock::iterator MI) const {
66
0
  llvm_unreachable("Target didn't implement insertNoop!");
67
0
}
68
69
82.8k
static bool isAsmComment(const char *Str, const MCAsmInfo &MAI) {
70
82.8k
  return strncmp(Str, MAI.getCommentString().data(),
71
82.8k
                 MAI.getCommentString().size()) == 0;
72
82.8k
}
73
74
/// Measure the specified inline asm to determine an approximation of its
75
/// length.
76
/// Comments (which run till the next SeparatorString or newline) do not
77
/// count as an instruction.
78
/// Any other non-whitespace text is considered an instruction, with
79
/// multiple instructions separated by SeparatorString or newlines.
80
/// Variable-length instructions are not handled here; this function
81
/// may be overloaded in the target code to do that.
82
/// We implement a special case of the .space directive which takes only a
83
/// single integer argument in base 10 that is the size in bytes. This is a
84
/// restricted form of the GAS directive in that we only interpret
85
/// simple--i.e. not a logical or arithmetic expression--size values without
86
/// the optional fill value. This is primarily used for creating arbitrary
87
/// sized inline asm blocks for testing purposes.
88
unsigned TargetInstrInfo::getInlineAsmLength(
89
  const char *Str,
90
174k
  const MCAsmInfo &MAI, const TargetSubtargetInfo *STI) const {
91
174k
  // Count the number of instructions in the asm.
92
174k
  bool AtInsnStart = true;
93
174k
  unsigned Length = 0;
94
174k
  const unsigned MaxInstLength = MAI.getMaxInstLength(STI);
95
261k
  for (; *Str; 
++Str87.0k
) {
96
87.0k
    if (*Str == '\n' || strncmp(Str, MAI.getSeparatorString(),
97
85.7k
                                strlen(MAI.getSeparatorString())) == 0) {
98
4.26k
      AtInsnStart = true;
99
82.8k
    } else if (isAsmComment(Str, MAI)) {
100
1.88k
      // Stop counting as an instruction after a comment until the next
101
1.88k
      // separator.
102
1.88k
      AtInsnStart = false;
103
1.88k
    }
104
87.0k
105
87.0k
    if (AtInsnStart && 
!std::isspace(static_cast<unsigned char>(*Str))10.9k
) {
106
7.68k
      unsigned AddLength = MaxInstLength;
107
7.68k
      if (strncmp(Str, ".space", 6) == 0) {
108
478
        char *EStr;
109
478
        int SpaceSize;
110
478
        SpaceSize = strtol(Str + 6, &EStr, 10);
111
478
        SpaceSize = SpaceSize < 0 ? 
00
: SpaceSize;
112
478
        while (*EStr != '\n' && std::isspace(static_cast<unsigned char>(*EStr)))
113
0
          ++EStr;
114
478
        if (*EStr == '\0' || 
*EStr == '\n'0
||
115
478
            
isAsmComment(EStr, MAI)0
) // Successfully parsed .space argument
116
478
          AddLength = SpaceSize;
117
478
      }
118
7.68k
      Length += AddLength;
119
7.68k
      AtInsnStart = false;
120
7.68k
    }
121
87.0k
  }
122
174k
123
174k
  return Length;
124
174k
}
125
126
/// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
127
/// after it, replacing it with an unconditional branch to NewDest.
128
void
129
TargetInstrInfo::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail,
130
198k
                                         MachineBasicBlock *NewDest) const {
131
198k
  MachineBasicBlock *MBB = Tail->getParent();
132
198k
133
198k
  // Remove all the old successors of MBB from the CFG.
134
310k
  while (!MBB->succ_empty())
135
111k
    MBB->removeSuccessor(MBB->succ_begin());
136
198k
137
198k
  // Save off the debug loc before erasing the instruction.
138
198k
  DebugLoc DL = Tail->getDebugLoc();
139
198k
140
198k
  // Update call site info and remove all the dead instructions
141
198k
  // from the end of MBB.
142
947k
  while (Tail != MBB->end()) {
143
748k
    auto MI = Tail++;
144
748k
    if (MI->isCall())
145
121k
      MBB->getParent()->updateCallSiteInfo(&*MI);
146
748k
    MBB->erase(MI);
147
748k
  }
148
198k
149
198k
  // If MBB isn't immediately before MBB, insert a branch to it.
150
198k
  if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest))
151
189k
    insertBranch(*MBB, NewDest, nullptr, SmallVector<MachineOperand, 0>(), DL);
152
198k
  MBB->addSuccessor(NewDest);
153
198k
}
154
155
MachineInstr *TargetInstrInfo::commuteInstructionImpl(MachineInstr &MI,
156
                                                      bool NewMI, unsigned Idx1,
157
789k
                                                      unsigned Idx2) const {
158
789k
  const MCInstrDesc &MCID = MI.getDesc();
159
789k
  bool HasDef = MCID.getNumDefs();
160
789k
  if (HasDef && 
!MI.getOperand(0).isReg()678k
)
161
0
    // No idea how to commute this instruction. Target should implement its own.
162
0
    return nullptr;
163
789k
164
789k
  unsigned CommutableOpIdx1 = Idx1; (void)CommutableOpIdx1;
165
789k
  unsigned CommutableOpIdx2 = Idx2; (void)CommutableOpIdx2;
166
789k
  assert(findCommutedOpIndices(MI, CommutableOpIdx1, CommutableOpIdx2) &&
167
789k
         CommutableOpIdx1 == Idx1 && CommutableOpIdx2 == Idx2 &&
168
789k
         "TargetInstrInfo::CommuteInstructionImpl(): not commutable operands.");
169
789k
  assert(MI.getOperand(Idx1).isReg() && MI.getOperand(Idx2).isReg() &&
170
789k
         "This only knows how to commute register operands so far");
171
789k
172
789k
  Register Reg0 = HasDef ? 
MI.getOperand(0).getReg()678k
:
Register()110k
;
173
789k
  Register Reg1 = MI.getOperand(Idx1).getReg();
174
789k
  Register Reg2 = MI.getOperand(Idx2).getReg();
175
789k
  unsigned SubReg0 = HasDef ? 
MI.getOperand(0).getSubReg()678k
:
0110k
;
176
789k
  unsigned SubReg1 = MI.getOperand(Idx1).getSubReg();
177
789k
  unsigned SubReg2 = MI.getOperand(Idx2).getSubReg();
178
789k
  bool Reg1IsKill = MI.getOperand(Idx1).isKill();
179
789k
  bool Reg2IsKill = MI.getOperand(Idx2).isKill();
180
789k
  bool Reg1IsUndef = MI.getOperand(Idx1).isUndef();
181
789k
  bool Reg2IsUndef = MI.getOperand(Idx2).isUndef();
182
789k
  bool Reg1IsInternal = MI.getOperand(Idx1).isInternalRead();
183
789k
  bool Reg2IsInternal = MI.getOperand(Idx2).isInternalRead();
184
789k
  // Avoid calling isRenamable for virtual registers since we assert that
185
789k
  // renamable property is only queried/set for physical registers.
186
789k
  bool Reg1IsRenamable = TargetRegisterInfo::isPhysicalRegister(Reg1)
187
789k
                             ? 
MI.getOperand(Idx1).isRenamable()16.4k
188
789k
                             : 
false772k
;
189
789k
  bool Reg2IsRenamable = TargetRegisterInfo::isPhysicalRegister(Reg2)
190
789k
                             ? 
MI.getOperand(Idx2).isRenamable()16.4k
191
789k
                             : 
false772k
;
192
789k
  // If destination is tied to either of the commuted source register, then
193
789k
  // it must be updated.
194
789k
  if (HasDef && 
Reg0 == Reg1678k
&&
195
789k
      
MI.getDesc().getOperandConstraint(Idx1, MCOI::TIED_TO) == 03.45k
) {
196
653
    Reg2IsKill = false;
197
653
    Reg0 = Reg2;
198
653
    SubReg0 = SubReg2;
199
788k
  } else if (HasDef && 
Reg0 == Reg2677k
&&
200
788k
             
MI.getDesc().getOperandConstraint(Idx2, MCOI::TIED_TO) == 05.43k
) {
201
0
    Reg1IsKill = false;
202
0
    Reg0 = Reg1;
203
0
    SubReg0 = SubReg1;
204
0
  }
205
789k
206
789k
  MachineInstr *CommutedMI = nullptr;
207
789k
  if (NewMI) {
208
0
    // Create a new instruction.
209
0
    MachineFunction &MF = *MI.getMF();
210
0
    CommutedMI = MF.CloneMachineInstr(&MI);
211
789k
  } else {
212
789k
    CommutedMI = &MI;
213
789k
  }
214
789k
215
789k
  if (HasDef) {
216
678k
    CommutedMI->getOperand(0).setReg(Reg0);
217
678k
    CommutedMI->getOperand(0).setSubReg(SubReg0);
218
678k
  }
219
789k
  CommutedMI->getOperand(Idx2).setReg(Reg1);
220
789k
  CommutedMI->getOperand(Idx1).setReg(Reg2);
221
789k
  CommutedMI->getOperand(Idx2).setSubReg(SubReg1);
222
789k
  CommutedMI->getOperand(Idx1).setSubReg(SubReg2);
223
789k
  CommutedMI->getOperand(Idx2).setIsKill(Reg1IsKill);
224
789k
  CommutedMI->getOperand(Idx1).setIsKill(Reg2IsKill);
225
789k
  CommutedMI->getOperand(Idx2).setIsUndef(Reg1IsUndef);
226
789k
  CommutedMI->getOperand(Idx1).setIsUndef(Reg2IsUndef);
227
789k
  CommutedMI->getOperand(Idx2).setIsInternalRead(Reg1IsInternal);
228
789k
  CommutedMI->getOperand(Idx1).setIsInternalRead(Reg2IsInternal);
229
789k
  // Avoid calling setIsRenamable for virtual registers since we assert that
230
789k
  // renamable property is only queried/set for physical registers.
231
789k
  if (TargetRegisterInfo::isPhysicalRegister(Reg1))
232
16.4k
    CommutedMI->getOperand(Idx2).setIsRenamable(Reg1IsRenamable);
233
789k
  if (TargetRegisterInfo::isPhysicalRegister(Reg2))
234
16.4k
    CommutedMI->getOperand(Idx1).setIsRenamable(Reg2IsRenamable);
235
789k
  return CommutedMI;
236
789k
}
237
238
MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr &MI, bool NewMI,
239
                                                  unsigned OpIdx1,
240
905k
                                                  unsigned OpIdx2) const {
241
905k
  // If OpIdx1 or OpIdx2 is not specified, then this method is free to choose
242
905k
  // any commutable operand, which is done in findCommutedOpIndices() method
243
905k
  // called below.
244
905k
  if ((OpIdx1 == CommuteAnyOperandIndex || 
OpIdx2 == CommuteAnyOperandIndex72.4k
) &&
245
905k
      
!findCommutedOpIndices(MI, OpIdx1, OpIdx2)833k
) {
246
2.37k
    assert(MI.isCommutable() &&
247
2.37k
           "Precondition violation: MI must be commutable.");
248
2.37k
    return nullptr;
249
2.37k
  }
250
903k
  return commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
251
903k
}
252
253
bool TargetInstrInfo::fixCommutedOpIndices(unsigned &ResultIdx1,
254
                                           unsigned &ResultIdx2,
255
                                           unsigned CommutableOpIdx1,
256
989k
                                           unsigned CommutableOpIdx2) {
257
989k
  if (ResultIdx1 == CommuteAnyOperandIndex &&
258
989k
      
ResultIdx2 == CommuteAnyOperandIndex852k
) {
259
852k
    ResultIdx1 = CommutableOpIdx1;
260
852k
    ResultIdx2 = CommutableOpIdx2;
261
852k
  } else 
if (137k
ResultIdx1 == CommuteAnyOperandIndex137k
) {
262
0
    if (ResultIdx2 == CommutableOpIdx1)
263
0
      ResultIdx1 = CommutableOpIdx2;
264
0
    else if (ResultIdx2 == CommutableOpIdx2)
265
0
      ResultIdx1 = CommutableOpIdx1;
266
0
    else
267
0
      return false;
268
137k
  } else if (ResultIdx2 == CommuteAnyOperandIndex) {
269
25.4k
    if (ResultIdx1 == CommutableOpIdx1)
270
23.1k
      ResultIdx2 = CommutableOpIdx2;
271
2.31k
    else if (ResultIdx1 == CommutableOpIdx2)
272
2.02k
      ResultIdx2 = CommutableOpIdx1;
273
296
    else
274
296
      return false;
275
112k
  } else
276
112k
    // Check that the result operand indices match the given commutable
277
112k
    // operand indices.
278
112k
    return (ResultIdx1 == CommutableOpIdx1 && 
ResultIdx2 == CommutableOpIdx2105k
) ||
279
112k
           
(11.8k
ResultIdx1 == CommutableOpIdx211.8k
&&
ResultIdx2 == CommutableOpIdx1371
);
280
877k
281
877k
  return true;
282
877k
}
283
284
bool TargetInstrInfo::findCommutedOpIndices(MachineInstr &MI,
285
                                            unsigned &SrcOpIdx1,
286
622k
                                            unsigned &SrcOpIdx2) const {
287
622k
  assert(!MI.isBundle() &&
288
622k
         "TargetInstrInfo::findCommutedOpIndices() can't handle bundles");
289
622k
290
622k
  const MCInstrDesc &MCID = MI.getDesc();
291
622k
  if (!MCID.isCommutable())
292
17.2k
    return false;
293
605k
294
605k
  // This assumes v0 = op v1, v2 and commuting would swap v1 and v2. If this
295
605k
  // is not true, then the target must implement this.
296
605k
  unsigned CommutableOpIdx1 = MCID.getNumDefs();
297
605k
  unsigned CommutableOpIdx2 = CommutableOpIdx1 + 1;
298
605k
  if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
299
605k
                            CommutableOpIdx1, CommutableOpIdx2))
300
5.59k
    return false;
301
600k
302
600k
  if (!MI.getOperand(SrcOpIdx1).isReg() || 
!MI.getOperand(SrcOpIdx2).isReg()599k
)
303
186
    // No idea.
304
186
    return false;
305
599k
  return true;
306
599k
}
307
308
101M
bool TargetInstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const {
309
101M
  if (!MI.isTerminator()) 
return false52.3M
;
310
48.7M
311
48.7M
  // Conditional branch is a special case.
312
48.7M
  if (MI.isBranch() && 
!MI.isBarrier()44.3M
)
313
32.4M
    return true;
314
16.3M
  if (!MI.isPredicable())
315
16.3M
    return true;
316
31.5k
  return !isPredicated(MI);
317
31.5k
}
318
319
bool TargetInstrInfo::PredicateInstruction(
320
0
    MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
321
0
  bool MadeChange = false;
322
0
323
0
  assert(!MI.isBundle() &&
324
0
         "TargetInstrInfo::PredicateInstruction() can't handle bundles");
325
0
326
0
  const MCInstrDesc &MCID = MI.getDesc();
327
0
  if (!MI.isPredicable())
328
0
    return false;
329
0
330
0
  for (unsigned j = 0, i = 0, e = MI.getNumOperands(); i != e; ++i) {
331
0
    if (MCID.OpInfo[i].isPredicate()) {
332
0
      MachineOperand &MO = MI.getOperand(i);
333
0
      if (MO.isReg()) {
334
0
        MO.setReg(Pred[j].getReg());
335
0
        MadeChange = true;
336
0
      } else if (MO.isImm()) {
337
0
        MO.setImm(Pred[j].getImm());
338
0
        MadeChange = true;
339
0
      } else if (MO.isMBB()) {
340
0
        MO.setMBB(Pred[j].getMBB());
341
0
        MadeChange = true;
342
0
      }
343
0
      ++j;
344
0
    }
345
0
  }
346
0
  return MadeChange;
347
0
}
348
349
bool TargetInstrInfo::hasLoadFromStackSlot(
350
    const MachineInstr &MI,
351
7.23M
    SmallVectorImpl<const MachineMemOperand *> &Accesses) const {
352
7.23M
  size_t StartSize = Accesses.size();
353
7.23M
  for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
354
7.23M
                                  oe = MI.memoperands_end();
355
8.71M
       o != oe; 
++o1.47M
) {
356
1.47M
    if ((*o)->isLoad() &&
357
1.47M
        
dyn_cast_or_null<FixedStackPseudoSourceValue>((*o)->getPseudoValue())917k
)
358
110k
      Accesses.push_back(*o);
359
1.47M
  }
360
7.23M
  return Accesses.size() != StartSize;
361
7.23M
}
362
363
bool TargetInstrInfo::hasStoreToStackSlot(
364
    const MachineInstr &MI,
365
7.09M
    SmallVectorImpl<const MachineMemOperand *> &Accesses) const {
366
7.09M
  size_t StartSize = Accesses.size();
367
7.09M
  for (MachineInstr::mmo_iterator o = MI.memoperands_begin(),
368
7.09M
                                  oe = MI.memoperands_end();
369
8.42M
       o != oe; 
++o1.33M
) {
370
1.33M
    if ((*o)->isStore() &&
371
1.33M
        
dyn_cast_or_null<FixedStackPseudoSourceValue>((*o)->getPseudoValue())578k
)
372
59.8k
      Accesses.push_back(*o);
373
1.33M
  }
374
7.09M
  return Accesses.size() != StartSize;
375
7.09M
}
376
377
bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC,
378
                                        unsigned SubIdx, unsigned &Size,
379
                                        unsigned &Offset,
380
252
                                        const MachineFunction &MF) const {
381
252
  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
382
252
  if (!SubIdx) {
383
248
    Size = TRI->getSpillSize(*RC);
384
248
    Offset = 0;
385
248
    return true;
386
248
  }
387
4
  unsigned BitSize = TRI->getSubRegIdxSize(SubIdx);
388
4
  // Convert bit size to byte size.
389
4
  if (BitSize % 8)
390
0
    return false;
391
4
392
4
  int BitOffset = TRI->getSubRegIdxOffset(SubIdx);
393
4
  if (BitOffset < 0 || BitOffset % 8)
394
0
    return false;
395
4
396
4
  Size = BitSize /= 8;
397
4
  Offset = (unsigned)BitOffset / 8;
398
4
399
4
  assert(TRI->getSpillSize(*RC) >= (Offset + Size) && "bad subregister range");
400
4
401
4
  if (!MF.getDataLayout().isLittleEndian()) {
402
3
    Offset = TRI->getSpillSize(*RC) - (Offset + Size);
403
3
  }
404
4
  return true;
405
4
}
406
407
void TargetInstrInfo::reMaterialize(MachineBasicBlock &MBB,
408
                                    MachineBasicBlock::iterator I,
409
                                    unsigned DestReg, unsigned SubIdx,
410
                                    const MachineInstr &Orig,
411
942k
                                    const TargetRegisterInfo &TRI) const {
412
942k
  MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig);
413
942k
  MI->substituteRegister(MI->getOperand(0).getReg(), DestReg, SubIdx, TRI);
414
942k
  MBB.insert(I, MI);
415
942k
}
416
417
bool TargetInstrInfo::produceSameValue(const MachineInstr &MI0,
418
                                       const MachineInstr &MI1,
419
466k
                                       const MachineRegisterInfo *MRI) const {
420
466k
  return MI0.isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs);
421
466k
}
422
423
MachineInstr &TargetInstrInfo::duplicate(MachineBasicBlock &MBB,
424
685k
    MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig) const {
425
685k
  assert(!Orig.isNotDuplicable() && "Instruction cannot be duplicated");
426
685k
  MachineFunction &MF = *MBB.getParent();
427
685k
  return MF.CloneMachineInstrBundle(MBB, InsertBefore, Orig);
428
685k
}
429
430
// If the COPY instruction in MI can be folded to a stack operation, return
431
// the register class to use.
432
static const TargetRegisterClass *canFoldCopy(const MachineInstr &MI,
433
51.5k
                                              unsigned FoldIdx) {
434
51.5k
  assert(MI.isCopy() && "MI must be a COPY instruction");
435
51.5k
  if (MI.getNumOperands() != 2)
436
0
    return nullptr;
437
51.5k
  assert(FoldIdx<2 && "FoldIdx refers no nonexistent operand");
438
51.5k
439
51.5k
  const MachineOperand &FoldOp = MI.getOperand(FoldIdx);
440
51.5k
  const MachineOperand &LiveOp = MI.getOperand(1 - FoldIdx);
441
51.5k
442
51.5k
  if (FoldOp.getSubReg() || 
LiveOp.getSubReg()48.6k
)
443
3.08k
    return nullptr;
444
48.4k
445
48.4k
  unsigned FoldReg = FoldOp.getReg();
446
48.4k
  unsigned LiveReg = LiveOp.getReg();
447
48.4k
448
48.4k
  assert(TargetRegisterInfo::isVirtualRegister(FoldReg) &&
449
48.4k
         "Cannot fold physregs");
450
48.4k
451
48.4k
  const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
452
48.4k
  const TargetRegisterClass *RC = MRI.getRegClass(FoldReg);
453
48.4k
454
48.4k
  if (TargetRegisterInfo::isPhysicalRegister(LiveOp.getReg()))
455
12.1k
    return RC->contains(LiveOp.getReg()) ? 
RC12.1k
:
nullptr30
;
456
36.2k
457
36.2k
  if (RC->hasSubClassEq(MRI.getRegClass(LiveReg)))
458
35.3k
    return RC;
459
922
460
922
  // FIXME: Allow folding when register classes are memory compatible.
461
922
  return nullptr;
462
922
}
463
464
0
void TargetInstrInfo::getNoop(MCInst &NopInst) const {
465
0
  llvm_unreachable("Not implemented");
466
0
}
467
468
static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI,
469
                                    ArrayRef<unsigned> Ops, int FrameIndex,
470
369
                                    const TargetInstrInfo &TII) {
471
369
  unsigned StartIdx = 0;
472
369
  switch (MI.getOpcode()) {
473
369
  case TargetOpcode::STACKMAP: {
474
38
    // StackMapLiveValues are foldable
475
38
    StartIdx = StackMapOpers(&MI).getVarIdx();
476
38
    break;
477
369
  }
478
369
  case TargetOpcode::PATCHPOINT: {
479
206
    // For PatchPoint, the call args are not foldable (even if reported in the
480
206
    // stackmap e.g. via anyregcc).
481
206
    StartIdx = PatchPointOpers(&MI).getVarIdx();
482
206
    break;
483
369
  }
484
369
  case TargetOpcode::STATEPOINT: {
485
125
    // For statepoints, fold deopt and gc arguments, but not call arguments.
486
125
    StartIdx = StatepointOpers(&MI).getVarIdx();
487
125
    break;
488
369
  }
489
369
  default:
490
0
    llvm_unreachable("unexpected stackmap opcode");
491
369
  }
492
369
493
369
  // Return false if any operands requested for folding are not foldable (not
494
369
  // part of the stackmap's live values).
495
369
  for (unsigned Op : Ops) {
496
369
    if (Op < StartIdx)
497
133
      return nullptr;
498
369
  }
499
369
500
369
  MachineInstr *NewMI =
501
236
      MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true);
502
236
  MachineInstrBuilder MIB(MF, NewMI);
503
236
504
236
  // No need to fold return, the meta data, and function arguments
505
1.57k
  for (unsigned i = 0; i < StartIdx; 
++i1.34k
)
506
1.34k
    MIB.add(MI.getOperand(i));
507
236
508
11.1k
  for (unsigned i = StartIdx; i < MI.getNumOperands(); 
++i10.8k
) {
509
10.8k
    MachineOperand &MO = MI.getOperand(i);
510
10.8k
    if (is_contained(Ops, i)) {
511
236
      unsigned SpillSize;
512
236
      unsigned SpillOffset;
513
236
      // Compute the spill slot size and offset.
514
236
      const TargetRegisterClass *RC =
515
236
        MF.getRegInfo().getRegClass(MO.getReg());
516
236
      bool Valid =
517
236
          TII.getStackSlotRange(RC, MO.getSubReg(), SpillSize, SpillOffset, MF);
518
236
      if (!Valid)
519
0
        report_fatal_error("cannot spill patchpoint subregister operand");
520
236
      MIB.addImm(StackMaps::IndirectMemRefOp);
521
236
      MIB.addImm(SpillSize);
522
236
      MIB.addFrameIndex(FrameIndex);
523
236
      MIB.addImm(SpillOffset);
524
236
    }
525
10.6k
    else
526
10.6k
      MIB.add(MO);
527
10.8k
  }
528
236
  return NewMI;
529
236
}
530
531
MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
532
                                                 ArrayRef<unsigned> Ops, int FI,
533
                                                 LiveIntervals *LIS,
534
333k
                                                 VirtRegMap *VRM) const {
535
333k
  auto Flags = MachineMemOperand::MONone;
536
333k
  for (unsigned OpIdx : Ops)
537
334k
    Flags |= MI.getOperand(OpIdx).isDef() ? 
MachineMemOperand::MOStore126k
538
334k
                                          : 
MachineMemOperand::MOLoad208k
;
539
333k
540
333k
  MachineBasicBlock *MBB = MI.getParent();
541
333k
  assert(MBB && "foldMemoryOperand needs an inserted instruction");
542
333k
  MachineFunction &MF = *MBB->getParent();
543
333k
544
333k
  // If we're not folding a load into a subreg, the size of the load is the
545
333k
  // size of the spill slot. But if we are, we need to figure out what the
546
333k
  // actual load size is.
547
333k
  int64_t MemSize = 0;
548
333k
  const MachineFrameInfo &MFI = MF.getFrameInfo();
549
333k
  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
550
333k
551
333k
  if (Flags & MachineMemOperand::MOStore) {
552
126k
    MemSize = MFI.getObjectSize(FI);
553
206k
  } else {
554
207k
    for (unsigned OpIdx : Ops) {
555
207k
      int64_t OpSize = MFI.getObjectSize(FI);
556
207k
557
207k
      if (auto SubReg = MI.getOperand(OpIdx).getSubReg()) {
558
12.8k
        unsigned SubRegSize = TRI->getSubRegIdxSize(SubReg);
559
12.8k
        if (SubRegSize > 0 && !(SubRegSize % 8))
560
12.8k
          OpSize = SubRegSize / 8;
561
12.8k
      }
562
207k
563
207k
      MemSize = std::max(MemSize, OpSize);
564
207k
    }
565
206k
  }
566
333k
567
333k
  assert(MemSize && "Did not expect a zero-sized stack slot");
568
333k
569
333k
  MachineInstr *NewMI = nullptr;
570
333k
571
333k
  if (MI.getOpcode() == TargetOpcode::STACKMAP ||
572
333k
      
MI.getOpcode() == TargetOpcode::PATCHPOINT333k
||
573
333k
      
MI.getOpcode() == TargetOpcode::STATEPOINT333k
) {
574
190
    // Fold stackmap/patchpoint.
575
190
    NewMI = foldPatchpoint(MF, MI, Ops, FI, *this);
576
190
    if (NewMI)
577
104
      MBB->insert(MI, NewMI);
578
333k
  } else {
579
333k
    // Ask the target to do the actual folding.
580
333k
    NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI, LIS, VRM);
581
333k
  }
582
333k
583
333k
  if (NewMI) {
584
146k
    NewMI->setMemRefs(MF, MI.memoperands());
585
146k
    // Add a memory operand, foldMemoryOperandImpl doesn't do that.
586
146k
    assert((!(Flags & MachineMemOperand::MOStore) ||
587
146k
            NewMI->mayStore()) &&
588
146k
           "Folded a def to a non-store!");
589
146k
    assert((!(Flags & MachineMemOperand::MOLoad) ||
590
146k
            NewMI->mayLoad()) &&
591
146k
           "Folded a use to a non-load!");
592
146k
    assert(MFI.getObjectOffset(FI) != -1);
593
146k
    MachineMemOperand *MMO = MF.getMachineMemOperand(
594
146k
        MachinePointerInfo::getFixedStack(MF, FI), Flags, MemSize,
595
146k
        MFI.getObjectAlignment(FI));
596
146k
    NewMI->addMemOperand(MF, MMO);
597
146k
598
146k
    return NewMI;
599
146k
  }
600
186k
601
186k
  // Straight COPY may fold as load/store.
602
186k
  if (!MI.isCopy() || 
Ops.size() != 151.5k
)
603
135k
    return nullptr;
604
51.5k
605
51.5k
  const TargetRegisterClass *RC = canFoldCopy(MI, Ops[0]);
606
51.5k
  if (!RC)
607
4.03k
    return nullptr;
608
47.4k
609
47.4k
  const MachineOperand &MO = MI.getOperand(1 - Ops[0]);
610
47.4k
  MachineBasicBlock::iterator Pos = MI;
611
47.4k
612
47.4k
  if (Flags == MachineMemOperand::MOStore)
613
19.3k
    storeRegToStackSlot(*MBB, Pos, MO.getReg(), MO.isKill(), FI, RC, TRI);
614
28.1k
  else
615
28.1k
    loadRegFromStackSlot(*MBB, Pos, MO.getReg(), FI, RC, TRI);
616
47.4k
  return &*--Pos;
617
47.4k
}
618
619
MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
620
                                                 ArrayRef<unsigned> Ops,
621
                                                 MachineInstr &LoadMI,
622
97.1k
                                                 LiveIntervals *LIS) const {
623
97.1k
  assert(LoadMI.canFoldAsLoad() && "LoadMI isn't foldable!");
624
#ifndef NDEBUG
625
  for (unsigned OpIdx : Ops)
626
    assert(MI.getOperand(OpIdx).isUse() && "Folding load into def!");
627
#endif
628
629
97.1k
  MachineBasicBlock &MBB = *MI.getParent();
630
97.1k
  MachineFunction &MF = *MBB.getParent();
631
97.1k
632
97.1k
  // Ask the target to do the actual folding.
633
97.1k
  MachineInstr *NewMI = nullptr;
634
97.1k
  int FrameIndex = 0;
635
97.1k
636
97.1k
  if ((MI.getOpcode() == TargetOpcode::STACKMAP ||
637
97.1k
       
MI.getOpcode() == TargetOpcode::PATCHPOINT97.1k
||
638
97.1k
       
MI.getOpcode() == TargetOpcode::STATEPOINT97.0k
) &&
639
97.1k
      
isLoadFromStackSlot(LoadMI, FrameIndex)236
) {
640
179
    // Fold stackmap/patchpoint.
641
179
    NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this);
642
179
    if (NewMI)
643
132
      NewMI = &*MBB.insert(MI, NewMI);
644
96.9k
  } else {
645
96.9k
    // Ask the target to do the actual folding.
646
96.9k
    NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, LoadMI, LIS);
647
96.9k
  }
648
97.1k
649
97.1k
  if (!NewMI)
650
88.3k
    return nullptr;
651
8.84k
652
8.84k
  // Copy the memoperands from the load to the folded instruction.
653
8.84k
  if (MI.memoperands_empty()) {
654
8.72k
    NewMI->setMemRefs(MF, LoadMI.memoperands());
655
8.72k
  } else {
656
120
    // Handle the rare case of folding multiple loads.
657
120
    NewMI->setMemRefs(MF, MI.memoperands());
658
120
    for (MachineInstr::mmo_iterator I = LoadMI.memoperands_begin(),
659
120
                                    E = LoadMI.memoperands_end();
660
241
         I != E; 
++I121
) {
661
121
      NewMI->addMemOperand(MF, *I);
662
121
    }
663
120
  }
664
8.84k
  return NewMI;
665
8.84k
}
666
667
bool TargetInstrInfo::hasReassociableOperands(
668
86.5k
    const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
669
86.5k
  const MachineOperand &Op1 = Inst.getOperand(1);
670
86.5k
  const MachineOperand &Op2 = Inst.getOperand(2);
671
86.5k
  const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
672
86.5k
673
86.5k
  // We need virtual register definitions for the operands that we will
674
86.5k
  // reassociate.
675
86.5k
  MachineInstr *MI1 = nullptr;
676
86.5k
  MachineInstr *MI2 = nullptr;
677
86.5k
  if (Op1.isReg() && TargetRegisterInfo::isVirtualRegister(Op1.getReg()))
678
86.5k
    MI1 = MRI.getUniqueVRegDef(Op1.getReg());
679
86.5k
  if (Op2.isReg() && TargetRegisterInfo::isVirtualRegister(Op2.getReg()))
680
86.5k
    MI2 = MRI.getUniqueVRegDef(Op2.getReg());
681
86.5k
682
86.5k
  // And they need to be in the trace (otherwise, they won't have a depth).
683
86.5k
  return MI1 && MI2 && MI1->getParent() == MBB && 
MI2->getParent() == MBB81.6k
;
684
86.5k
}
685
686
bool TargetInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
687
70.1k
                                             bool &Commuted) const {
688
70.1k
  const MachineBasicBlock *MBB = Inst.getParent();
689
70.1k
  const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
690
70.1k
  MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg());
691
70.1k
  MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());
692
70.1k
  unsigned AssocOpcode = Inst.getOpcode();
693
70.1k
694
70.1k
  // If only one operand has the same opcode and it's the second source operand,
695
70.1k
  // the operands must be commuted.
696
70.1k
  Commuted = MI1->getOpcode() != AssocOpcode && 
MI2->getOpcode() == AssocOpcode62.1k
;
697
70.1k
  if (Commuted)
698
1.12k
    std::swap(MI1, MI2);
699
70.1k
700
70.1k
  // 1. The previous instruction must be the same type as Inst.
701
70.1k
  // 2. The previous instruction must have virtual register definitions for its
702
70.1k
  //    operands in the same basic block as Inst.
703
70.1k
  // 3. The previous instruction's result must only be used by Inst.
704
70.1k
  return MI1->getOpcode() == AssocOpcode &&
705
70.1k
         
hasReassociableOperands(*MI1, MBB)9.08k
&&
706
70.1k
         
MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg())8.98k
;
707
70.1k
}
708
709
// 1. The operation must be associative and commutative.
710
// 2. The instruction must have virtual register definitions for its
711
//    operands in the same basic block.
712
// 3. The instruction must have a reassociable sibling.
713
bool TargetInstrInfo::isReassociationCandidate(const MachineInstr &Inst,
714
22.5M
                                               bool &Commuted) const {
715
22.5M
  return isAssociativeAndCommutative(Inst) &&
716
22.5M
         
hasReassociableOperands(Inst, Inst.getParent())78.6k
&&
717
22.5M
         
hasReassociableSibling(Inst, Commuted)70.1k
;
718
22.5M
}
719
720
// The concept of the reassociation pass is that these operations can benefit
721
// from this kind of transformation:
722
//
723
// A = ? op ?
724
// B = A op X (Prev)
725
// C = B op Y (Root)
726
// -->
727
// A = ? op ?
728
// B = X op Y
729
// C = A op B
730
//
731
// breaking the dependency between A and B, allowing them to be executed in
732
// parallel (or back-to-back in a pipeline) instead of depending on each other.
733
734
// FIXME: This has the potential to be expensive (compile time) while not
735
// improving the code at all. Some ways to limit the overhead:
736
// 1. Track successful transforms; bail out if hit rate gets too low.
737
// 2. Only enable at -O3 or some other non-default optimization level.
738
// 3. Pre-screen pattern candidates here: if an operand of the previous
739
//    instruction is known to not increase the critical path, then don't match
740
//    that pattern.
741
bool TargetInstrInfo::getMachineCombinerPatterns(
742
    MachineInstr &Root,
743
22.5M
    SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
744
22.5M
  bool Commute;
745
22.5M
  if (isReassociationCandidate(Root, Commute)) {
746
4.05k
    // We found a sequence of instructions that may be suitable for a
747
4.05k
    // reassociation of operands to increase ILP. Specify each commutation
748
4.05k
    // possibility for the Prev instruction in the sequence and let the
749
4.05k
    // machine combiner decide if changing the operands is worthwhile.
750
4.05k
    if (Commute) {
751
937
      Patterns.push_back(MachineCombinerPattern::REASSOC_AX_YB);
752
937
      Patterns.push_back(MachineCombinerPattern::REASSOC_XA_YB);
753
3.11k
    } else {
754
3.11k
      Patterns.push_back(MachineCombinerPattern::REASSOC_AX_BY);
755
3.11k
      Patterns.push_back(MachineCombinerPattern::REASSOC_XA_BY);
756
3.11k
    }
757
4.05k
    return true;
758
4.05k
  }
759
22.5M
760
22.5M
  return false;
761
22.5M
}
762
763
/// Return true when a code sequence can improve loop throughput.
764
bool
765
691
TargetInstrInfo::isThroughputPattern(MachineCombinerPattern Pattern) const {
766
691
  return false;
767
691
}
768
769
/// Attempt the reassociation transformation to reduce critical path length.
770
/// See the above comments before getMachineCombinerPatterns().
771
void TargetInstrInfo::reassociateOps(
772
    MachineInstr &Root, MachineInstr &Prev,
773
    MachineCombinerPattern Pattern,
774
    SmallVectorImpl<MachineInstr *> &InsInstrs,
775
    SmallVectorImpl<MachineInstr *> &DelInstrs,
776
6.17k
    DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
777
6.17k
  MachineFunction *MF = Root.getMF();
778
6.17k
  MachineRegisterInfo &MRI = MF->getRegInfo();
779
6.17k
  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
780
6.17k
  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
781
6.17k
  const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI);
782
6.17k
783
6.17k
  // This array encodes the operand index for each parameter because the
784
6.17k
  // operands may be commuted. Each row corresponds to a pattern value,
785
6.17k
  // and each column specifies the index of A, B, X, Y.
786
6.17k
  unsigned OpIdx[4][4] = {
787
6.17k
    { 1, 1, 2, 2 },
788
6.17k
    { 1, 2, 2, 1 },
789
6.17k
    { 2, 1, 1, 2 },
790
6.17k
    { 2, 2, 1, 1 }
791
6.17k
  };
792
6.17k
793
6.17k
  int Row;
794
6.17k
  switch (Pattern) {
795
6.17k
  
case MachineCombinerPattern::REASSOC_AX_BY: Row = 0; break3.17k
;
796
6.17k
  
case MachineCombinerPattern::REASSOC_AX_YB: Row = 1; break1.23k
;
797
6.17k
  
case MachineCombinerPattern::REASSOC_XA_BY: Row = 2; break842
;
798
6.17k
  
case MachineCombinerPattern::REASSOC_XA_YB: Row = 3; break921
;
799
6.17k
  
default: 0
llvm_unreachable0
("unexpected MachineCombinerPattern");
800
6.17k
  }
801
6.17k
802
6.17k
  MachineOperand &OpA = Prev.getOperand(OpIdx[Row][0]);
803
6.17k
  MachineOperand &OpB = Root.getOperand(OpIdx[Row][1]);
804
6.17k
  MachineOperand &OpX = Prev.getOperand(OpIdx[Row][2]);
805
6.17k
  MachineOperand &OpY = Root.getOperand(OpIdx[Row][3]);
806
6.17k
  MachineOperand &OpC = Root.getOperand(0);
807
6.17k
808
6.17k
  unsigned RegA = OpA.getReg();
809
6.17k
  unsigned RegB = OpB.getReg();
810
6.17k
  unsigned RegX = OpX.getReg();
811
6.17k
  unsigned RegY = OpY.getReg();
812
6.17k
  unsigned RegC = OpC.getReg();
813
6.17k
814
6.17k
  if (TargetRegisterInfo::isVirtualRegister(RegA))
815
6.17k
    MRI.constrainRegClass(RegA, RC);
816
6.17k
  if (TargetRegisterInfo::isVirtualRegister(RegB))
817
6.17k
    MRI.constrainRegClass(RegB, RC);
818
6.17k
  if (TargetRegisterInfo::isVirtualRegister(RegX))
819
6.17k
    MRI.constrainRegClass(RegX, RC);
820
6.17k
  if (TargetRegisterInfo::isVirtualRegister(RegY))
821
6.17k
    MRI.constrainRegClass(RegY, RC);
822
6.17k
  if (TargetRegisterInfo::isVirtualRegister(RegC))
823
6.17k
    MRI.constrainRegClass(RegC, RC);
824
6.17k
825
6.17k
  // Create a new virtual register for the result of (X op Y) instead of
826
6.17k
  // recycling RegB because the MachineCombiner's computation of the critical
827
6.17k
  // path requires a new register definition rather than an existing one.
828
6.17k
  unsigned NewVR = MRI.createVirtualRegister(RC);
829
6.17k
  InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
830
6.17k
831
6.17k
  unsigned Opcode = Root.getOpcode();
832
6.17k
  bool KillA = OpA.isKill();
833
6.17k
  bool KillX = OpX.isKill();
834
6.17k
  bool KillY = OpY.isKill();
835
6.17k
836
6.17k
  // Create new instructions for insertion.
837
6.17k
  MachineInstrBuilder MIB1 =
838
6.17k
      BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR)
839
6.17k
          .addReg(RegX, getKillRegState(KillX))
840
6.17k
          .addReg(RegY, getKillRegState(KillY));
841
6.17k
  MachineInstrBuilder MIB2 =
842
6.17k
      BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC)
843
6.17k
          .addReg(RegA, getKillRegState(KillA))
844
6.17k
          .addReg(NewVR, getKillRegState(true));
845
6.17k
846
6.17k
  setSpecialOperandAttr(Root, Prev, *MIB1, *MIB2);
847
6.17k
848
6.17k
  // Record new instructions for insertion and old instructions for deletion.
849
6.17k
  InsInstrs.push_back(MIB1);
850
6.17k
  InsInstrs.push_back(MIB2);
851
6.17k
  DelInstrs.push_back(&Prev);
852
6.17k
  DelInstrs.push_back(&Root);
853
6.17k
}
854
855
void TargetInstrInfo::genAlternativeCodeSequence(
856
    MachineInstr &Root, MachineCombinerPattern Pattern,
857
    SmallVectorImpl<MachineInstr *> &InsInstrs,
858
    SmallVectorImpl<MachineInstr *> &DelInstrs,
859
6.17k
    DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const {
860
6.17k
  MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
861
6.17k
862
6.17k
  // Select the previous instruction in the sequence based on the input pattern.
863
6.17k
  MachineInstr *Prev = nullptr;
864
6.17k
  switch (Pattern) {
865
6.17k
  case MachineCombinerPattern::REASSOC_AX_BY:
866
4.01k
  case MachineCombinerPattern::REASSOC_XA_BY:
867
4.01k
    Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
868
4.01k
    break;
869
4.01k
  case MachineCombinerPattern::REASSOC_AX_YB:
870
2.15k
  case MachineCombinerPattern::REASSOC_XA_YB:
871
2.15k
    Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
872
2.15k
    break;
873
2.15k
  default:
874
0
    break;
875
6.17k
  }
876
6.17k
877
6.17k
  assert(Prev && "Unknown pattern for machine combiner");
878
6.17k
879
6.17k
  reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg);
880
6.17k
}
881
882
bool TargetInstrInfo::isReallyTriviallyReMaterializableGeneric(
883
3.87M
    const MachineInstr &MI, AliasAnalysis *AA) const {
884
3.87M
  const MachineFunction &MF = *MI.getMF();
885
3.87M
  const MachineRegisterInfo &MRI = MF.getRegInfo();
886
3.87M
887
3.87M
  // Remat clients assume operand 0 is the defined register.
888
3.87M
  if (
!MI.getNumOperands()3.87M
|| !MI.getOperand(0).isReg())
889
0
    return false;
890
3.87M
  unsigned DefReg = MI.getOperand(0).getReg();
891
3.87M
892
3.87M
  // A sub-register definition can only be rematerialized if the instruction
893
3.87M
  // doesn't read the other parts of the register.  Otherwise it is really a
894
3.87M
  // read-modify-write operation on the full virtual register which cannot be
895
3.87M
  // moved safely.
896
3.87M
  if (TargetRegisterInfo::isVirtualRegister(DefReg) &&
897
3.87M
      MI.getOperand(0).getSubReg() && 
MI.readsVirtualRegister(DefReg)118k
)
898
6.50k
    return false;
899
3.86M
900
3.86M
  // A load from a fixed stack slot can be rematerialized. This may be
901
3.86M
  // redundant with subsequent checks, but it's target-independent,
902
3.86M
  // simple, and a common case.
903
3.86M
  int FrameIdx = 0;
904
3.86M
  if (isLoadFromStackSlot(MI, FrameIdx) &&
905
3.86M
      
MF.getFrameInfo().isImmutableObjectIndex(FrameIdx)50.4k
)
906
29.1k
    return true;
907
3.83M
908
3.83M
  // Avoid instructions obviously unsafe for remat.
909
3.83M
  
if (3.83M
MI.isNotDuplicable()3.83M
|| MI.mayStore() || MI.mayRaiseFPException() ||
910
3.83M
      MI.hasUnmodeledSideEffects())
911
32
    return false;
912
3.83M
913
3.83M
  // Don't remat inline asm. We have no idea how expensive it is
914
3.83M
  // even if it's side effect free.
915
3.83M
  if (MI.isInlineAsm())
916
0
    return false;
917
3.83M
918
3.83M
  // Avoid instructions which load from potentially varying memory.
919
3.83M
  if (MI.mayLoad() && 
!MI.isDereferenceableInvariantLoad(AA)134k
)
920
116k
    return false;
921
3.72M
922
3.72M
  // If any of the registers accessed are non-constant, conservatively assume
923
3.72M
  // the instruction is not rematerializable.
924
11.5M
  
for (unsigned i = 0, e = MI.getNumOperands(); 3.72M
i != e;
++i7.86M
) {
925
8.84M
    const MachineOperand &MO = MI.getOperand(i);
926
8.84M
    if (!MO.isReg()) 
continue3.85M
;
927
4.98M
    unsigned Reg = MO.getReg();
928
4.98M
    if (Reg == 0)
929
226k
      continue;
930
4.75M
931
4.75M
    // Check for a well-behaved physical register.
932
4.75M
    if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
933
60.8k
      if (MO.isUse()) {
934
60.8k
        // If the physreg has no defs anywhere, it's just an ambient register
935
60.8k
        // and we can freely move its uses. Alternatively, if it's allocatable,
936
60.8k
        // it could get allocated to something with a def during allocation.
937
60.8k
        if (!MRI.isConstantPhysReg(Reg))
938
4.26k
          return false;
939
0
      } else {
940
0
        // A physreg def. We can't remat it.
941
0
        return false;
942
0
      }
943
56.5k
      continue;
944
56.5k
    }
945
4.69M
946
4.69M
    // Only allow one virtual-register def.  There may be multiple defs of the
947
4.69M
    // same virtual register, though.
948
4.69M
    if (MO.isDef() && 
Reg != DefReg3.72M
)
949
0
      return false;
950
4.69M
951
4.69M
    // Don't allow any virtual-register uses. Rematting an instruction with
952
4.69M
    // virtual register uses would length the live ranges of the uses, which
953
4.69M
    // is not necessarily a good idea, certainly not "trivial".
954
4.69M
    if (MO.isUse())
955
974k
      return false;
956
4.69M
  }
957
3.72M
958
3.72M
  // Everything checked out.
959
3.72M
  
return true2.74M
;
960
3.72M
}
961
962
1.57k
int TargetInstrInfo::getSPAdjust(const MachineInstr &MI) const {
963
1.57k
  const MachineFunction *MF = MI.getMF();
964
1.57k
  const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
965
1.57k
  bool StackGrowsDown =
966
1.57k
    TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
967
1.57k
968
1.57k
  unsigned FrameSetupOpcode = getCallFrameSetupOpcode();
969
1.57k
  unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();
970
1.57k
971
1.57k
  if (!isFrameInstr(MI))
972
1.47k
    return 0;
973
102
974
102
  int SPAdj = TFI->alignSPAdjust(getFrameSize(MI));
975
102
976
102
  if ((!StackGrowsDown && 
MI.getOpcode() == FrameSetupOpcode0
) ||
977
102
      (StackGrowsDown && MI.getOpcode() == FrameDestroyOpcode))
978
51
    SPAdj = -SPAdj;
979
102
980
102
  return SPAdj;
981
102
}
982
983
/// isSchedulingBoundary - Test if the given instruction should be
984
/// considered a scheduling boundary. This primarily includes labels
985
/// and terminators.
986
bool TargetInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
987
                                           const MachineBasicBlock *MBB,
988
20.1M
                                           const MachineFunction &MF) const {
989
20.1M
  // Terminators and labels can't be scheduled around.
990
20.1M
  if (MI.isTerminator() || 
MI.isPosition()17.4M
)
991
2.77M
    return true;
992
17.3M
993
17.3M
  // Don't attempt to schedule around any instruction that defines
994
17.3M
  // a stack-oriented pointer, as it's unlikely to be profitable. This
995
17.3M
  // saves compile time, because it doesn't require every single
996
17.3M
  // stack slot reference to depend on the instruction that does the
997
17.3M
  // modification.
998
17.3M
  const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
999
17.3M
  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
1000
17.3M
  return MI.modifiesRegister(TLI.getStackPointerRegisterToSaveRestore(), TRI);
1001
17.3M
}
1002
1003
// Provide a global flag for disabling the PreRA hazard recognizer that targets
1004
// may choose to honor.
1005
64.5k
bool TargetInstrInfo::usePreRAHazardRecognizer() const {
1006
64.5k
  return !DisableHazardRecognizer;
1007
64.5k
}
1008
1009
// Default implementation of CreateTargetRAHazardRecognizer.
1010
ScheduleHazardRecognizer *TargetInstrInfo::
1011
CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
1012
20.1k
                             const ScheduleDAG *DAG) const {
1013
20.1k
  // Dummy hazard recognizer allows all instructions to issue.
1014
20.1k
  return new ScheduleHazardRecognizer();
1015
20.1k
}
1016
1017
// Default implementation of CreateTargetMIHazardRecognizer.
1018
ScheduleHazardRecognizer *TargetInstrInfo::
1019
CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
1020
857k
                               const ScheduleDAG *DAG) const {
1021
857k
  return (ScheduleHazardRecognizer *)
1022
857k
    new ScoreboardHazardRecognizer(II, DAG, "machine-scheduler");
1023
857k
}
1024
1025
// Default implementation of CreateTargetPostRAHazardRecognizer.
1026
ScheduleHazardRecognizer *TargetInstrInfo::
1027
CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1028
3.19k
                                   const ScheduleDAG *DAG) const {
1029
3.19k
  return (ScheduleHazardRecognizer *)
1030
3.19k
    new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched");
1031
3.19k
}
1032
1033
//===----------------------------------------------------------------------===//
1034
//  SelectionDAG latency interface.
1035
//===----------------------------------------------------------------------===//
1036
1037
int
1038
TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
1039
                                   SDNode *DefNode, unsigned DefIdx,
1040
127k
                                   SDNode *UseNode, unsigned UseIdx) const {
1041
127k
  if (!ItinData || 
ItinData->isEmpty()92.5k
)
1042
127k
    return -1;
1043
0
1044
0
  if (!DefNode->isMachineOpcode())
1045
0
    return -1;
1046
0
1047
0
  unsigned DefClass = get(DefNode->getMachineOpcode()).getSchedClass();
1048
0
  if (!UseNode->isMachineOpcode())
1049
0
    return ItinData->getOperandCycle(DefClass, DefIdx);
1050
0
  unsigned UseClass = get(UseNode->getMachineOpcode()).getSchedClass();
1051
0
  return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1052
0
}
1053
1054
int TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
1055
0
                                     SDNode *N) const {
1056
0
  if (!ItinData || ItinData->isEmpty())
1057
0
    return 1;
1058
0
1059
0
  if (!N->isMachineOpcode())
1060
0
    return 1;
1061
0
1062
0
  return ItinData->getStageLatency(get(N->getMachineOpcode()).getSchedClass());
1063
0
}
1064
1065
//===----------------------------------------------------------------------===//
1066
//  MachineInstr latency interface.
1067
//===----------------------------------------------------------------------===//
1068
1069
unsigned TargetInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
1070
0
                                         const MachineInstr &MI) const {
1071
0
  if (!ItinData || ItinData->isEmpty())
1072
0
    return 1;
1073
0
1074
0
  unsigned Class = MI.getDesc().getSchedClass();
1075
0
  int UOps = ItinData->Itineraries[Class].NumMicroOps;
1076
0
  if (UOps >= 0)
1077
0
    return UOps;
1078
0
1079
0
  // The # of u-ops is dynamically determined. The specific target should
1080
0
  // override this function to return the right number.
1081
0
  return 1;
1082
0
}
1083
1084
/// Return the default expected latency for a def based on it's opcode.
1085
unsigned TargetInstrInfo::defaultDefLatency(const MCSchedModel &SchedModel,
1086
4.01M
                                            const MachineInstr &DefMI) const {
1087
4.01M
  if (DefMI.isTransient())
1088
937k
    return 0;
1089
3.07M
  if (DefMI.mayLoad())
1090
392k
    return SchedModel.LoadLatency;
1091
2.68M
  if (isHighLatencyDef(DefMI.getOpcode()))
1092
2.02k
    return SchedModel.HighLatency;
1093
2.68M
  return 1;
1094
2.68M
}
1095
1096
37.4k
unsigned TargetInstrInfo::getPredicationCost(const MachineInstr &) const {
1097
37.4k
  return 0;
1098
37.4k
}
1099
1100
unsigned TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
1101
                                          const MachineInstr &MI,
1102
29.1k
                                          unsigned *PredCost) const {
1103
29.1k
  // Default to one cycle for no itinerary. However, an "empty" itinerary may
1104
29.1k
  // still have a MinLatency property, which getStageLatency checks.
1105
29.1k
  if (!ItinData)
1106
19.6k
    return MI.mayLoad() ? 
21.89k
:
117.7k
;
1107
9.57k
1108
9.57k
  return ItinData->getStageLatency(MI.getDesc().getSchedClass());
1109
9.57k
}
1110
1111
bool TargetInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
1112
                                       const MachineInstr &DefMI,
1113
68.0k
                                       unsigned DefIdx) const {
1114
68.0k
  const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
1115
68.0k
  if (!ItinData || 
ItinData->isEmpty()178
)
1116
67.8k
    return false;
1117
178
1118
178
  unsigned DefClass = DefMI.getDesc().getSchedClass();
1119
178
  int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
1120
178
  return (DefCycle != -1 && 
DefCycle <= 1143
);
1121
178
}
1122
1123
/// Both DefMI and UseMI must be valid.  By default, call directly to the
1124
/// itinerary. This may be overriden by the target.
1125
int TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
1126
                                       const MachineInstr &DefMI,
1127
                                       unsigned DefIdx,
1128
                                       const MachineInstr &UseMI,
1129
307k
                                       unsigned UseIdx) const {
1130
307k
  unsigned DefClass = DefMI.getDesc().getSchedClass();
1131
307k
  unsigned UseClass = UseMI.getDesc().getSchedClass();
1132
307k
  return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
1133
307k
}
1134
1135
/// If we can determine the operand latency from the def only, without itinerary
1136
/// lookup, do so. Otherwise return -1.
1137
int TargetInstrInfo::computeDefOperandLatency(
1138
0
    const InstrItineraryData *ItinData, const MachineInstr &DefMI) const {
1139
0
1140
0
  // Let the target hook getInstrLatency handle missing itineraries.
1141
0
  if (!ItinData)
1142
0
    return getInstrLatency(ItinData, DefMI);
1143
0
1144
0
  if(ItinData->isEmpty())
1145
0
    return defaultDefLatency(ItinData->SchedModel, DefMI);
1146
0
1147
0
  // ...operand lookup required
1148
0
  return -1;
1149
0
}
1150
1151
bool TargetInstrInfo::getRegSequenceInputs(
1152
    const MachineInstr &MI, unsigned DefIdx,
1153
278k
    SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
1154
278k
  assert((MI.isRegSequence() ||
1155
278k
          MI.isRegSequenceLike()) && "Instruction do not have the proper type");
1156
278k
1157
278k
  if (!MI.isRegSequence())
1158
2.70k
    return getRegSequenceLikeInputs(MI, DefIdx, InputRegs);
1159
275k
1160
275k
  // We are looking at:
1161
275k
  // Def = REG_SEQUENCE v0, sub0, v1, sub1, ...
1162
275k
  assert(DefIdx == 0 && "REG_SEQUENCE only has one def");
1163
1.12M
  for (unsigned OpIdx = 1, EndOpIdx = MI.getNumOperands(); OpIdx != EndOpIdx;
1164
848k
       OpIdx += 2) {
1165
848k
    const MachineOperand &MOReg = MI.getOperand(OpIdx);
1166
848k
    if (MOReg.isUndef())
1167
0
      continue;
1168
848k
    const MachineOperand &MOSubIdx = MI.getOperand(OpIdx + 1);
1169
848k
    assert(MOSubIdx.isImm() &&
1170
848k
           "One of the subindex of the reg_sequence is not an immediate");
1171
848k
    // Record Reg:SubReg, SubIdx.
1172
848k
    InputRegs.push_back(RegSubRegPairAndIdx(MOReg.getReg(), MOReg.getSubReg(),
1173
848k
                                            (unsigned)MOSubIdx.getImm()));
1174
848k
  }
1175
275k
  return true;
1176
275k
}
1177
1178
bool TargetInstrInfo::getExtractSubregInputs(
1179
    const MachineInstr &MI, unsigned DefIdx,
1180
3.94k
    RegSubRegPairAndIdx &InputReg) const {
1181
3.94k
  assert((MI.isExtractSubreg() ||
1182
3.94k
      MI.isExtractSubregLike()) && "Instruction do not have the proper type");
1183
3.94k
1184
3.94k
  if (!MI.isExtractSubreg())
1185
3.94k
    return getExtractSubregLikeInputs(MI, DefIdx, InputReg);
1186
0
1187
0
  // We are looking at:
1188
0
  // Def = EXTRACT_SUBREG v0.sub1, sub0.
1189
0
  assert(DefIdx == 0 && "EXTRACT_SUBREG only has one def");
1190
0
  const MachineOperand &MOReg = MI.getOperand(1);
1191
0
  if (MOReg.isUndef())
1192
0
    return false;
1193
0
  const MachineOperand &MOSubIdx = MI.getOperand(2);
1194
0
  assert(MOSubIdx.isImm() &&
1195
0
         "The subindex of the extract_subreg is not an immediate");
1196
0
1197
0
  InputReg.Reg = MOReg.getReg();
1198
0
  InputReg.SubReg = MOReg.getSubReg();
1199
0
  InputReg.SubIdx = (unsigned)MOSubIdx.getImm();
1200
0
  return true;
1201
0
}
1202
1203
bool TargetInstrInfo::getInsertSubregInputs(
1204
    const MachineInstr &MI, unsigned DefIdx,
1205
135k
    RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const {
1206
135k
  assert((MI.isInsertSubreg() ||
1207
135k
      MI.isInsertSubregLike()) && "Instruction do not have the proper type");
1208
135k
1209
135k
  if (!MI.isInsertSubreg())
1210
245
    return getInsertSubregLikeInputs(MI, DefIdx, BaseReg, InsertedReg);
1211
135k
1212
135k
  // We are looking at:
1213
135k
  // Def = INSERT_SEQUENCE v0, v1, sub0.
1214
135k
  assert(DefIdx == 0 && "INSERT_SUBREG only has one def");
1215
135k
  const MachineOperand &MOBaseReg = MI.getOperand(1);
1216
135k
  const MachineOperand &MOInsertedReg = MI.getOperand(2);
1217
135k
  if (MOInsertedReg.isUndef())
1218
0
    return false;
1219
135k
  const MachineOperand &MOSubIdx = MI.getOperand(3);
1220
135k
  assert(MOSubIdx.isImm() &&
1221
135k
         "One of the subindex of the reg_sequence is not an immediate");
1222
135k
  BaseReg.Reg = MOBaseReg.getReg();
1223
135k
  BaseReg.SubReg = MOBaseReg.getSubReg();
1224
135k
1225
135k
  InsertedReg.Reg = MOInsertedReg.getReg();
1226
135k
  InsertedReg.SubReg = MOInsertedReg.getSubReg();
1227
135k
  InsertedReg.SubIdx = (unsigned)MOSubIdx.getImm();
1228
135k
  return true;
1229
135k
}