Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/X86/X86CallFrameOptimization.cpp
Line
Count
Source (jump to first uncovered line)
1
//===----- X86CallFrameOptimization.cpp - Optimize x86 call sequences -----===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file defines a pass that optimizes call sequences on x86.
10
// Currently, it converts movs of function parameters onto the stack into
11
// pushes. This is beneficial for two main reasons:
12
// 1) The push instruction encoding is much smaller than a stack-ptr-based mov.
13
// 2) It is possible to push memory arguments directly. So, if the
14
//    the transformation is performed pre-reg-alloc, it can help relieve
15
//    register pressure.
16
//
17
//===----------------------------------------------------------------------===//
18
19
#include "MCTargetDesc/X86BaseInfo.h"
20
#include "X86FrameLowering.h"
21
#include "X86InstrInfo.h"
22
#include "X86MachineFunctionInfo.h"
23
#include "X86RegisterInfo.h"
24
#include "X86Subtarget.h"
25
#include "llvm/ADT/DenseSet.h"
26
#include "llvm/ADT/SmallVector.h"
27
#include "llvm/ADT/StringRef.h"
28
#include "llvm/CodeGen/MachineBasicBlock.h"
29
#include "llvm/CodeGen/MachineFrameInfo.h"
30
#include "llvm/CodeGen/MachineFunction.h"
31
#include "llvm/CodeGen/MachineFunctionPass.h"
32
#include "llvm/CodeGen/MachineInstr.h"
33
#include "llvm/CodeGen/MachineInstrBuilder.h"
34
#include "llvm/CodeGen/MachineOperand.h"
35
#include "llvm/CodeGen/MachineRegisterInfo.h"
36
#include "llvm/CodeGen/TargetInstrInfo.h"
37
#include "llvm/CodeGen/TargetRegisterInfo.h"
38
#include "llvm/IR/DebugLoc.h"
39
#include "llvm/IR/Function.h"
40
#include "llvm/MC/MCDwarf.h"
41
#include "llvm/Support/CommandLine.h"
42
#include "llvm/Support/ErrorHandling.h"
43
#include "llvm/Support/MathExtras.h"
44
#include <cassert>
45
#include <cstddef>
46
#include <cstdint>
47
#include <iterator>
48
49
using namespace llvm;
50
51
#define DEBUG_TYPE "x86-cf-opt"
52
53
static cl::opt<bool>
54
    NoX86CFOpt("no-x86-call-frame-opt",
55
               cl::desc("Avoid optimizing x86 call frames for size"),
56
               cl::init(false), cl::Hidden);
57
58
namespace {
59
60
class X86CallFrameOptimization : public MachineFunctionPass {
61
public:
62
11.4k
  X86CallFrameOptimization() : MachineFunctionPass(ID) { }
63
64
  bool runOnMachineFunction(MachineFunction &MF) override;
65
66
  static char ID;
67
68
private:
69
  // Information we know about a particular call site
70
  struct CallContext {
71
132k
    CallContext() : FrameSetup(nullptr), ArgStoreVector(4, nullptr) {}
72
73
    // Iterator referring to the frame setup instruction
74
    MachineBasicBlock::iterator FrameSetup;
75
76
    // Actual call instruction
77
    MachineInstr *Call = nullptr;
78
79
    // A copy of the stack pointer
80
    MachineInstr *SPCopy = nullptr;
81
82
    // The total displacement of all passed parameters
83
    int64_t ExpectedDist = 0;
84
85
    // The sequence of storing instructions used to pass the parameters
86
    SmallVector<MachineInstr *, 4> ArgStoreVector;
87
88
    // True if this call site has no stack parameters
89
    bool NoStackParams = false;
90
91
    // True if this call site can use push instructions
92
    bool UsePush = false;
93
  };
94
95
  typedef SmallVector<CallContext, 8> ContextVector;
96
97
  bool isLegal(MachineFunction &MF);
98
99
  bool isProfitable(MachineFunction &MF, ContextVector &CallSeqMap);
100
101
  void collectCallInfo(MachineFunction &MF, MachineBasicBlock &MBB,
102
                       MachineBasicBlock::iterator I, CallContext &Context);
103
104
  void adjustCallSequence(MachineFunction &MF, const CallContext &Context);
105
106
  MachineInstr *canFoldIntoRegPush(MachineBasicBlock::iterator FrameSetup,
107
                                   unsigned Reg);
108
109
  enum InstClassification { Convert, Skip, Exit };
110
111
  InstClassification classifyInstruction(MachineBasicBlock &MBB,
112
                                         MachineBasicBlock::iterator MI,
113
                                         const X86RegisterInfo &RegInfo,
114
                                         DenseSet<unsigned int> &UsedRegs);
115
116
146k
  StringRef getPassName() const override { return "X86 Optimize Call Frame"; }
117
118
  const X86InstrInfo *TII;
119
  const X86FrameLowering *TFL;
120
  const X86Subtarget *STI;
121
  MachineRegisterInfo *MRI;
122
  unsigned SlotSize;
123
  unsigned Log2SlotSize;
124
};
125
126
} // end anonymous namespace
127
char X86CallFrameOptimization::ID = 0;
128
INITIALIZE_PASS(X86CallFrameOptimization, DEBUG_TYPE,
129
                "X86 Call Frame Optimization", false, false)
130
131
// This checks whether the transformation is legal.
132
// Also returns false in cases where it's potentially legal, but
133
// we don't even want to try.
134
135k
bool X86CallFrameOptimization::isLegal(MachineFunction &MF) {
135
135k
  if (NoX86CFOpt.getValue())
136
56
    return false;
137
135k
138
135k
  // We can't encode multiple DW_CFA_GNU_args_size or DW_CFA_def_cfa_offset
139
135k
  // in the compact unwind encoding that Darwin uses. So, bail if there
140
135k
  // is a danger of that being generated.
141
135k
  if (STI->isTargetDarwin() &&
142
135k
      
(37.7k
!MF.getLandingPads().empty()37.7k
||
143
37.7k
       
(37.7k
MF.getFunction().needsUnwindTableEntry()37.7k
&&
!TFL->hasFP(MF)32.6k
)))
144
11.2k
    return false;
145
123k
146
123k
  // It is not valid to change the stack pointer outside the prolog/epilog
147
123k
  // on 64-bit Windows.
148
123k
  if (STI->isTargetWin64())
149
1.58k
    return false;
150
122k
151
122k
  // You would expect straight-line code between call-frame setup and
152
122k
  // call-frame destroy. You would be wrong. There are circumstances (e.g.
153
122k
  // CMOV_GR8 expansion of a select that feeds a function call!) where we can
154
122k
  // end up with the setup and the destroy in different basic blocks.
155
122k
  // This is bad, and breaks SP adjustment.
156
122k
  // So, check that all of the frames in the function are closed inside
157
122k
  // the same block, and, for good measure, that there are no nested frames.
158
122k
  unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode();
159
122k
  unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
160
372k
  for (MachineBasicBlock &BB : MF) {
161
372k
    bool InsideFrameSequence = false;
162
2.99M
    for (MachineInstr &MI : BB) {
163
2.99M
      if (MI.getOpcode() == FrameSetupOpcode) {
164
132k
        if (InsideFrameSequence)
165
0
          return false;
166
132k
        InsideFrameSequence = true;
167
2.86M
      } else if (MI.getOpcode() == FrameDestroyOpcode) {
168
132k
        if (!InsideFrameSequence)
169
0
          return false;
170
132k
        InsideFrameSequence = false;
171
132k
      }
172
2.99M
    }
173
372k
174
372k
    if (InsideFrameSequence)
175
0
      return false;
176
372k
  }
177
122k
178
122k
  return true;
179
122k
}
180
181
// Check whether this transformation is profitable for a particular
182
// function - in terms of code size.
183
bool X86CallFrameOptimization::isProfitable(MachineFunction &MF,
184
122k
                                            ContextVector &CallSeqVector) {
185
122k
  // This transformation is always a win when we do not expect to have
186
122k
  // a reserved call frame. Under other circumstances, it may be either
187
122k
  // a win or a loss, and requires a heuristic.
188
122k
  bool CannotReserveFrame = MF.getFrameInfo().hasVarSizedObjects();
189
122k
  if (CannotReserveFrame)
190
212
    return true;
191
122k
192
122k
  unsigned StackAlign = TFL->getStackAlignment();
193
122k
194
122k
  int64_t Advantage = 0;
195
131k
  for (auto CC : CallSeqVector) {
196
131k
    // Call sites where no parameters are passed on the stack
197
131k
    // do not affect the cost, since there needs to be no
198
131k
    // stack adjustment.
199
131k
    if (CC.NoStackParams)
200
114k
      continue;
201
16.5k
202
16.5k
    if (!CC.UsePush) {
203
797
      // If we don't use pushes for a particular call site,
204
797
      // we pay for not having a reserved call frame with an
205
797
      // additional sub/add esp pair. The cost is ~3 bytes per instruction,
206
797
      // depending on the size of the constant.
207
797
      // TODO: Callee-pop functions should have a smaller penalty, because
208
797
      // an add is needed even with a reserved call frame.
209
797
      Advantage -= 6;
210
15.8k
    } else {
211
15.8k
      // We can use pushes. First, account for the fixed costs.
212
15.8k
      // We'll need a add after the call.
213
15.8k
      Advantage -= 3;
214
15.8k
      // If we have to realign the stack, we'll also need a sub before
215
15.8k
      if (CC.ExpectedDist % StackAlign)
216
12.1k
        Advantage -= 3;
217
15.8k
      // Now, for each push, we save ~3 bytes. For small constants, we actually,
218
15.8k
      // save more (up to 5 bytes), but 3 should be a good approximation.
219
15.8k
      Advantage += (CC.ExpectedDist >> Log2SlotSize) * 3;
220
15.8k
    }
221
16.5k
  }
222
122k
223
122k
  return Advantage >= 0;
224
122k
}
225
226
135k
bool X86CallFrameOptimization::runOnMachineFunction(MachineFunction &MF) {
227
135k
  STI = &MF.getSubtarget<X86Subtarget>();
228
135k
  TII = STI->getInstrInfo();
229
135k
  TFL = STI->getFrameLowering();
230
135k
  MRI = &MF.getRegInfo();
231
135k
232
135k
  const X86RegisterInfo &RegInfo =
233
135k
      *static_cast<const X86RegisterInfo *>(STI->getRegisterInfo());
234
135k
  SlotSize = RegInfo.getSlotSize();
235
135k
  assert(isPowerOf2_32(SlotSize) && "Expect power of 2 stack slot size");
236
135k
  Log2SlotSize = Log2_32(SlotSize);
237
135k
238
135k
  if (skipFunction(MF.getFunction()) || 
!isLegal(MF)135k
)
239
13.0k
    return false;
240
122k
241
122k
  unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode();
242
122k
243
122k
  bool Changed = false;
244
122k
245
122k
  ContextVector CallSeqVector;
246
122k
247
122k
  for (auto &MBB : MF)
248
372k
    for (auto &MI : MBB)
249
2.99M
      if (MI.getOpcode() == FrameSetupOpcode) {
250
132k
        CallContext Context;
251
132k
        collectCallInfo(MF, MBB, MI, Context);
252
132k
        CallSeqVector.push_back(Context);
253
132k
      }
254
122k
255
122k
  if (!isProfitable(MF, CallSeqVector))
256
1.32k
    return false;
257
121k
258
126k
  
for (auto CC : CallSeqVector)121k
{
259
126k
    if (CC.UsePush) {
260
14.1k
      adjustCallSequence(MF, CC);
261
14.1k
      Changed = true;
262
14.1k
    }
263
126k
  }
264
121k
265
121k
  return Changed;
266
121k
}
267
268
X86CallFrameOptimization::InstClassification
269
X86CallFrameOptimization::classifyInstruction(
270
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
271
90.7k
    const X86RegisterInfo &RegInfo, DenseSet<unsigned int> &UsedRegs) {
272
90.7k
  if (MI == MBB.end())
273
0
    return Exit;
274
90.7k
275
90.7k
  // The instructions we actually care about are movs onto the stack or special
276
90.7k
  // cases of constant-stores to stack
277
90.7k
  switch (MI->getOpcode()) {
278
90.7k
    case X86::AND16mi8:
279
97
    case X86::AND32mi8:
280
97
    case X86::AND64mi8: {
281
97
      MachineOperand ImmOp = MI->getOperand(X86::AddrNumOperands);
282
97
      return ImmOp.getImm() == 0 ? Convert : 
Exit0
;
283
97
    }
284
97
    case X86::OR16mi8:
285
15
    case X86::OR32mi8:
286
15
    case X86::OR64mi8: {
287
15
      MachineOperand ImmOp = MI->getOperand(X86::AddrNumOperands);
288
15
      return ImmOp.getImm() == -1 ? Convert : 
Exit0
;
289
15
    }
290
49.6k
    case X86::MOV32mi:
291
49.6k
    case X86::MOV32mr:
292
49.6k
    case X86::MOV64mi32:
293
49.6k
    case X86::MOV64mr:
294
49.6k
      return Convert;
295
40.9k
  }
296
40.9k
297
40.9k
  // Not all calling conventions have only stack MOVs between the stack
298
40.9k
  // adjust and the call.
299
40.9k
300
40.9k
  // We want to tolerate other instructions, to cover more cases.
301
40.9k
  // In particular:
302
40.9k
  // a) PCrel calls, where we expect an additional COPY of the basereg.
303
40.9k
  // b) Passing frame-index addresses.
304
40.9k
  // c) Calling conventions that have inreg parameters. These generate
305
40.9k
  //    both copies and movs into registers.
306
40.9k
  // To avoid creating lots of special cases, allow any instruction
307
40.9k
  // that does not write into memory, does not def or use the stack
308
40.9k
  // pointer, and does not def any register that was used by a preceding
309
40.9k
  // push.
310
40.9k
  // (Reading from memory is allowed, even if referenced through a
311
40.9k
  // frame index, since these will get adjusted properly in PEI)
312
40.9k
313
40.9k
  // The reason for the last condition is that the pushes can't replace
314
40.9k
  // the movs in place, because the order must be reversed.
315
40.9k
  // So if we have a MOV32mr that uses EDX, then an instruction that defs
316
40.9k
  // EDX, and then the call, after the transformation the push will use
317
40.9k
  // the modified version of EDX, and not the original one.
318
40.9k
  // Since we are still in SSA form at this point, we only need to
319
40.9k
  // make sure we don't clobber any *physical* registers that were
320
40.9k
  // used by an earlier mov that will become a push.
321
40.9k
322
40.9k
  if (MI->isCall() || 
MI->mayStore()24.8k
)
323
16.8k
    return Exit;
324
24.1k
325
84.8k
  
for (const MachineOperand &MO : MI->operands())24.1k
{
326
84.8k
    if (!MO.isReg())
327
24.6k
      continue;
328
60.2k
    unsigned int Reg = MO.getReg();
329
60.2k
    if (!RegInfo.isPhysicalRegister(Reg))
330
46.8k
      continue;
331
13.3k
    if (RegInfo.regsOverlap(Reg, RegInfo.getStackRegister()))
332
9
      return Exit;
333
13.3k
    if (MO.isDef()) {
334
13.0k
      for (unsigned int U : UsedRegs)
335
38
        if (RegInfo.regsOverlap(Reg, U))
336
0
          return Exit;
337
13.0k
    }
338
13.3k
  }
339
24.1k
340
24.1k
  
return Skip24.1k
;
341
24.1k
}
342
343
void X86CallFrameOptimization::collectCallInfo(MachineFunction &MF,
344
                                               MachineBasicBlock &MBB,
345
                                               MachineBasicBlock::iterator I,
346
132k
                                               CallContext &Context) {
347
132k
  // Check that this particular call sequence is amenable to the
348
132k
  // transformation.
349
132k
  const X86RegisterInfo &RegInfo =
350
132k
      *static_cast<const X86RegisterInfo *>(STI->getRegisterInfo());
351
132k
352
132k
  // We expect to enter this at the beginning of a call sequence
353
132k
  assert(I->getOpcode() == TII->getCallFrameSetupOpcode());
354
132k
  MachineBasicBlock::iterator FrameSetup = I++;
355
132k
  Context.FrameSetup = FrameSetup;
356
132k
357
132k
  // How much do we adjust the stack? This puts an upper bound on
358
132k
  // the number of parameters actually passed on it.
359
132k
  unsigned int MaxAdjust = TII->getFrameSize(*FrameSetup) >> Log2SlotSize;
360
132k
361
132k
  // A zero adjustment means no stack parameters
362
132k
  if (!MaxAdjust) {
363
115k
    Context.NoStackParams = true;
364
115k
    return;
365
115k
  }
366
16.8k
367
16.8k
  // Skip over DEBUG_VALUE.
368
16.8k
  // For globals in PIC mode, we can have some LEAs here. Skip them as well.
369
16.8k
  // TODO: Extend this to something that covers more cases.
370
19.1k
  
while (16.8k
I->getOpcode() == X86::LEA32r ||
I->isDebugInstr()16.8k
)
371
2.30k
    ++I;
372
16.8k
373
16.8k
  unsigned StackPtr = RegInfo.getStackRegister();
374
16.8k
  auto StackPtrCopyInst = MBB.end();
375
16.8k
  // SelectionDAG (but not FastISel) inserts a copy of ESP into a virtual
376
16.8k
  // register.  If it's there, use that virtual register as stack pointer
377
16.8k
  // instead. Also, we need to locate this instruction so that we can later
378
16.8k
  // safely ignore it while doing the conservative processing of the call chain.
379
16.8k
  // The COPY can be located anywhere between the call-frame setup
380
16.8k
  // instruction and its first use. We use the call instruction as a boundary
381
16.8k
  // because it is usually cheaper to check if an instruction is a call than
382
16.8k
  // checking if an instruction uses a register.
383
17.9k
  for (auto J = I; !J->isCall(); 
++J1.08k
)
384
17.8k
    if (J->isCopy() && 
J->getOperand(0).isReg()16.9k
&&
J->getOperand(1).isReg()16.9k
&&
385
17.8k
        
J->getOperand(1).getReg() == StackPtr16.9k
) {
386
16.7k
      StackPtrCopyInst = J;
387
16.7k
      Context.SPCopy = &*J++;
388
16.7k
      StackPtr = Context.SPCopy->getOperand(0).getReg();
389
16.7k
      break;
390
16.7k
    }
391
16.8k
392
16.8k
  // Scan the call setup sequence for the pattern we're looking for.
393
16.8k
  // We only handle a simple case - a sequence of store instructions that
394
16.8k
  // push a sequence of stack-slot-aligned values onto the stack, with
395
16.8k
  // no gaps between them.
396
16.8k
  if (MaxAdjust > 4)
397
3.64k
    Context.ArgStoreVector.resize(MaxAdjust, nullptr);
398
16.8k
399
16.8k
  DenseSet<unsigned int> UsedRegs;
400
16.8k
401
124k
  for (InstClassification Classification = Skip; Classification != Exit; 
++I107k
) {
402
107k
    // If this is the COPY of the stack pointer, it's ok to ignore.
403
107k
    if (I == StackPtrCopyInst)
404
16.7k
      continue;
405
90.7k
    Classification = classifyInstruction(MBB, I, RegInfo, UsedRegs);
406
90.7k
    if (Classification != Convert)
407
40.9k
      continue;
408
49.7k
    // We know the instruction has a supported store opcode.
409
49.7k
    // We only want movs of the form:
410
49.7k
    // mov imm/reg, k(%StackPtr)
411
49.7k
    // If we run into something else, bail.
412
49.7k
    // Note that AddrBaseReg may, counter to its name, not be a register,
413
49.7k
    // but rather a frame index.
414
49.7k
    // TODO: Support the fi case. This should probably work now that we
415
49.7k
    // have the infrastructure to track the stack pointer within a call
416
49.7k
    // sequence.
417
49.7k
    if (!I->getOperand(X86::AddrBaseReg).isReg() ||
418
49.7k
        
(I->getOperand(X86::AddrBaseReg).getReg() != StackPtr)49.7k
||
419
49.7k
        
!I->getOperand(X86::AddrScaleAmt).isImm()49.7k
||
420
49.7k
        
(I->getOperand(X86::AddrScaleAmt).getImm() != 1)49.7k
||
421
49.7k
        
(I->getOperand(X86::AddrIndexReg).getReg() != X86::NoRegister)49.7k
||
422
49.7k
        
(I->getOperand(X86::AddrSegmentReg).getReg() != X86::NoRegister)49.7k
||
423
49.7k
        
!I->getOperand(X86::AddrDisp).isImm()49.7k
)
424
35
      return;
425
49.7k
426
49.7k
    int64_t StackDisp = I->getOperand(X86::AddrDisp).getImm();
427
49.7k
    assert(StackDisp >= 0 &&
428
49.7k
           "Negative stack displacement when passing parameters");
429
49.7k
430
49.7k
    // We really don't want to consider the unaligned case.
431
49.7k
    if (StackDisp & (SlotSize - 1))
432
2
      return;
433
49.7k
    StackDisp >>= Log2SlotSize;
434
49.7k
435
49.7k
    assert((size_t)StackDisp < Context.ArgStoreVector.size() &&
436
49.7k
           "Function call has more parameters than the stack is adjusted for.");
437
49.7k
438
49.7k
    // If the same stack slot is being filled twice, something's fishy.
439
49.7k
    if (Context.ArgStoreVector[StackDisp] != nullptr)
440
0
      return;
441
49.7k
    Context.ArgStoreVector[StackDisp] = &*I;
442
49.7k
443
298k
    for (const MachineOperand &MO : I->uses()) {
444
298k
      if (!MO.isReg())
445
116k
        continue;
446
181k
      unsigned int Reg = MO.getReg();
447
181k
      if (RegInfo.isPhysicalRegister(Reg))
448
124
        UsedRegs.insert(Reg);
449
181k
    }
450
49.7k
  }
451
16.8k
452
16.8k
  --I;
453
16.8k
454
16.8k
  // We now expect the end of the sequence. If we stopped early,
455
16.8k
  // or reached the end of the block without finding a call, bail.
456
16.8k
  if (I == MBB.end() || !I->isCall())
457
719
    return;
458
16.1k
459
16.1k
  Context.Call = &*I;
460
16.1k
  if ((++I)->getOpcode() != TII->getCallFrameDestroyOpcode())
461
1
    return;
462
16.1k
463
16.1k
  // Now, go through the vector, and see that we don't have any gaps,
464
16.1k
  // but only a series of storing instructions.
465
16.1k
  auto MMI = Context.ArgStoreVector.begin(), MME = Context.ArgStoreVector.end();
466
65.5k
  for (; MMI != MME; 
++MMI, Context.ExpectedDist += SlotSize49.4k
)
467
60.9k
    if (*MMI == nullptr)
468
11.5k
      break;
469
16.1k
470
16.1k
  // If the call had no parameters, do nothing
471
16.1k
  if (MMI == Context.ArgStoreVector.begin())
472
40
    return;
473
16.0k
474
16.0k
  // We are either at the last parameter, or a gap.
475
16.0k
  // Make sure it's not a gap
476
40.7k
  
for (; 16.0k
MMI != MME;
++MMI24.6k
)
477
24.6k
    if (*MMI != nullptr)
478
7
      return;
479
16.0k
480
16.0k
  Context.UsePush = true;
481
16.0k
}
482
483
void X86CallFrameOptimization::adjustCallSequence(MachineFunction &MF,
484
14.1k
                                                  const CallContext &Context) {
485
14.1k
  // Ok, we can in fact do the transformation for this call.
486
14.1k
  // Do not remove the FrameSetup instruction, but adjust the parameters.
487
14.1k
  // PEI will end up finalizing the handling of this.
488
14.1k
  MachineBasicBlock::iterator FrameSetup = Context.FrameSetup;
489
14.1k
  MachineBasicBlock &MBB = *(FrameSetup->getParent());
490
14.1k
  TII->setFrameAdjustment(*FrameSetup, Context.ExpectedDist);
491
14.1k
492
14.1k
  DebugLoc DL = FrameSetup->getDebugLoc();
493
14.1k
  bool Is64Bit = STI->is64Bit();
494
14.1k
  // Now, iterate through the vector in reverse order, and replace the store to
495
14.1k
  // stack with pushes. MOVmi/MOVmr doesn't have any defs, so no need to
496
14.1k
  // replace uses.
497
61.1k
  for (int Idx = (Context.ExpectedDist >> Log2SlotSize) - 1; Idx >= 0; 
--Idx46.9k
) {
498
46.9k
    MachineBasicBlock::iterator Store = *Context.ArgStoreVector[Idx];
499
46.9k
    MachineOperand PushOp = Store->getOperand(X86::AddrNumOperands);
500
46.9k
    MachineBasicBlock::iterator Push = nullptr;
501
46.9k
    unsigned PushOpcode;
502
46.9k
    switch (Store->getOpcode()) {
503
46.9k
    default:
504
0
      llvm_unreachable("Unexpected Opcode!");
505
46.9k
    case X86::AND16mi8:
506
16.4k
    case X86::AND32mi8:
507
16.4k
    case X86::AND64mi8:
508
16.4k
    case X86::OR16mi8:
509
16.4k
    case X86::OR32mi8:
510
16.4k
    case X86::OR64mi8:
511
16.4k
    case X86::MOV32mi:
512
16.4k
    case X86::MOV64mi32:
513
16.4k
      PushOpcode = Is64Bit ? 
X86::PUSH64i322.71k
:
X86::PUSHi3213.7k
;
514
16.4k
      // If the operand is a small (8-bit) immediate, we can use a
515
16.4k
      // PUSH instruction with a shorter encoding.
516
16.4k
      // Note that isImm() may fail even though this is a MOVmi, because
517
16.4k
      // the operand can also be a symbol.
518
16.4k
      if (PushOp.isImm()) {
519
16.1k
        int64_t Val = PushOp.getImm();
520
16.1k
        if (isInt<8>(Val))
521
13.6k
          PushOpcode = Is64Bit ? 
X86::PUSH64i82.69k
:
X86::PUSH32i810.9k
;
522
16.1k
      }
523
16.4k
      Push = BuildMI(MBB, Context.Call, DL, TII->get(PushOpcode)).add(PushOp);
524
16.4k
      break;
525
30.5k
    case X86::MOV32mr:
526
30.5k
    case X86::MOV64mr: {
527
30.5k
      unsigned int Reg = PushOp.getReg();
528
30.5k
529
30.5k
      // If storing a 32-bit vreg on 64-bit targets, extend to a 64-bit vreg
530
30.5k
      // in preparation for the PUSH64. The upper 32 bits can be undef.
531
30.5k
      if (Is64Bit && 
Store->getOpcode() == X86::MOV32mr1.70k
) {
532
418
        unsigned UndefReg = MRI->createVirtualRegister(&X86::GR64RegClass);
533
418
        Reg = MRI->createVirtualRegister(&X86::GR64RegClass);
534
418
        BuildMI(MBB, Context.Call, DL, TII->get(X86::IMPLICIT_DEF), UndefReg);
535
418
        BuildMI(MBB, Context.Call, DL, TII->get(X86::INSERT_SUBREG), Reg)
536
418
            .addReg(UndefReg)
537
418
            .add(PushOp)
538
418
            .addImm(X86::sub_32bit);
539
418
      }
540
30.5k
541
30.5k
      // If PUSHrmm is not slow on this target, try to fold the source of the
542
30.5k
      // push into the instruction.
543
30.5k
      bool SlowPUSHrmm = STI->isAtom() || 
STI->isSLM()30.4k
;
544
30.5k
545
30.5k
      // Check that this is legal to fold. Right now, we're extremely
546
30.5k
      // conservative about that.
547
30.5k
      MachineInstr *DefMov = nullptr;
548
30.5k
      if (!SlowPUSHrmm && 
(DefMov = canFoldIntoRegPush(FrameSetup, Reg))30.4k
) {
549
3.08k
        PushOpcode = Is64Bit ? 
X86::PUSH64rmm117
:
X86::PUSH32rmm2.96k
;
550
3.08k
        Push = BuildMI(MBB, Context.Call, DL, TII->get(PushOpcode));
551
3.08k
552
3.08k
        unsigned NumOps = DefMov->getDesc().getNumOperands();
553
18.5k
        for (unsigned i = NumOps - X86::AddrNumOperands; i != NumOps; 
++i15.4k
)
554
15.4k
          Push->addOperand(DefMov->getOperand(i));
555
3.08k
556
3.08k
        DefMov->eraseFromParent();
557
27.4k
      } else {
558
27.4k
        PushOpcode = Is64Bit ? 
X86::PUSH64r1.58k
:
X86::PUSH32r25.8k
;
559
27.4k
        Push = BuildMI(MBB, Context.Call, DL, TII->get(PushOpcode))
560
27.4k
                   .addReg(Reg)
561
27.4k
                   .getInstr();
562
27.4k
      }
563
30.5k
      break;
564
46.9k
    }
565
46.9k
    }
566
46.9k
567
46.9k
    // For debugging, when using SP-based CFA, we need to adjust the CFA
568
46.9k
    // offset after each push.
569
46.9k
    // TODO: This is needed only if we require precise CFA.
570
46.9k
    if (!TFL->hasFP(MF))
571
2.93k
      TFL->BuildCFI(
572
2.93k
          MBB, std::next(Push), DL,
573
2.93k
          MCCFIInstruction::createAdjustCfaOffset(nullptr, SlotSize));
574
46.9k
575
46.9k
    MBB.erase(Store);
576
46.9k
  }
577
14.1k
578
14.1k
  // The stack-pointer copy is no longer used in the call sequences.
579
14.1k
  // There should not be any other users, but we can't commit to that, so:
580
14.1k
  if (Context.SPCopy && MRI->use_empty(Context.SPCopy->getOperand(0).getReg()))
581
14.1k
    Context.SPCopy->eraseFromParent();
582
14.1k
583
14.1k
  // Once we've done this, we need to make sure PEI doesn't assume a reserved
584
14.1k
  // frame.
585
14.1k
  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
586
14.1k
  FuncInfo->setHasPushSequences(true);
587
14.1k
}
588
589
MachineInstr *X86CallFrameOptimization::canFoldIntoRegPush(
590
30.4k
    MachineBasicBlock::iterator FrameSetup, unsigned Reg) {
591
30.4k
  // Do an extremely restricted form of load folding.
592
30.4k
  // ISel will often create patterns like:
593
30.4k
  // movl    4(%edi), %eax
594
30.4k
  // movl    8(%edi), %ecx
595
30.4k
  // movl    12(%edi), %edx
596
30.4k
  // movl    %edx, 8(%esp)
597
30.4k
  // movl    %ecx, 4(%esp)
598
30.4k
  // movl    %eax, (%esp)
599
30.4k
  // call
600
30.4k
  // Get rid of those with prejudice.
601
30.4k
  if (!TargetRegisterInfo::isVirtualRegister(Reg))
602
0
    return nullptr;
603
30.4k
604
30.4k
  // Make sure this is the only use of Reg.
605
30.4k
  if (!MRI->hasOneNonDBGUse(Reg))
606
13.7k
    return nullptr;
607
16.7k
608
16.7k
  MachineInstr &DefMI = *MRI->getVRegDef(Reg);
609
16.7k
610
16.7k
  // Make sure the def is a MOV from memory.
611
16.7k
  // If the def is in another block, give up.
612
16.7k
  if ((DefMI.getOpcode() != X86::MOV32rm &&
613
16.7k
       
DefMI.getOpcode() != X86::MOV64rm13.0k
) ||
614
16.7k
      
DefMI.getParent() != FrameSetup->getParent()4.50k
)
615
12.4k
    return nullptr;
616
4.29k
617
4.29k
  // Make sure we don't have any instructions between DefMI and the
618
4.29k
  // push that make folding the load illegal.
619
10.7k
  
for (MachineBasicBlock::iterator I = DefMI; 4.29k
I != FrameSetup;
++I6.47k
)
620
7.68k
    if (I->isLoadFoldBarrier())
621
1.21k
      return nullptr;
622
4.29k
623
4.29k
  
return &DefMI3.08k
;
624
4.29k
}
625
626
11.3k
FunctionPass *llvm::createX86CallFrameOptimization() {
627
11.3k
  return new X86CallFrameOptimization();
628
11.3k
}