Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/X86/X86ExpandPseudo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===------- X86ExpandPseudo.cpp - Expand pseudo instructions -------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file contains a pass that expands pseudo instructions into target
10
// instructions to allow proper scheduling, if-conversion, other late
11
// optimizations, or simply the encoding of the instructions.
12
//
13
//===----------------------------------------------------------------------===//
14
15
#include "X86.h"
16
#include "X86FrameLowering.h"
17
#include "X86InstrBuilder.h"
18
#include "X86InstrInfo.h"
19
#include "X86MachineFunctionInfo.h"
20
#include "X86Subtarget.h"
21
#include "llvm/Analysis/EHPersonalities.h"
22
#include "llvm/CodeGen/MachineFunctionPass.h"
23
#include "llvm/CodeGen/MachineInstrBuilder.h"
24
#include "llvm/CodeGen/Passes.h" // For IDs of passes that are preserved.
25
#include "llvm/IR/GlobalValue.h"
26
using namespace llvm;
27
28
#define DEBUG_TYPE "x86-pseudo"
29
#define X86_EXPAND_PSEUDO_NAME "X86 pseudo instruction expansion pass"
30
31
namespace {
32
class X86ExpandPseudo : public MachineFunctionPass {
33
public:
34
  static char ID;
35
12.2k
  X86ExpandPseudo() : MachineFunctionPass(ID) {}
36
37
12.1k
  void getAnalysisUsage(AnalysisUsage &AU) const override {
38
12.1k
    AU.setPreservesCFG();
39
12.1k
    AU.addPreservedID(MachineLoopInfoID);
40
12.1k
    AU.addPreservedID(MachineDominatorsID);
41
12.1k
    MachineFunctionPass::getAnalysisUsage(AU);
42
12.1k
  }
43
44
  const X86Subtarget *STI;
45
  const X86InstrInfo *TII;
46
  const X86RegisterInfo *TRI;
47
  const X86MachineFunctionInfo *X86FI;
48
  const X86FrameLowering *X86FL;
49
50
  bool runOnMachineFunction(MachineFunction &Fn) override;
51
52
12.1k
  MachineFunctionProperties getRequiredProperties() const override {
53
12.1k
    return MachineFunctionProperties().set(
54
12.1k
        MachineFunctionProperties::Property::NoVRegs);
55
12.1k
  }
56
57
150k
  StringRef getPassName() const override {
58
150k
    return "X86 pseudo instruction expansion pass";
59
150k
  }
60
61
private:
62
  void ExpandICallBranchFunnel(MachineBasicBlock *MBB,
63
                               MachineBasicBlock::iterator MBBI);
64
65
  bool ExpandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
66
  bool ExpandMBB(MachineBasicBlock &MBB);
67
};
68
char X86ExpandPseudo::ID = 0;
69
70
} // End anonymous namespace.
71
72
INITIALIZE_PASS(X86ExpandPseudo, DEBUG_TYPE, X86_EXPAND_PSEUDO_NAME, false,
73
                false)
74
75
void X86ExpandPseudo::ExpandICallBranchFunnel(
76
13
    MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI) {
77
13
  MachineBasicBlock *JTMBB = MBB;
78
13
  MachineInstr *JTInst = &*MBBI;
79
13
  MachineFunction *MF = MBB->getParent();
80
13
  const BasicBlock *BB = MBB->getBasicBlock();
81
13
  auto InsPt = MachineFunction::iterator(MBB);
82
13
  ++InsPt;
83
13
84
13
  std::vector<std::pair<MachineBasicBlock *, unsigned>> TargetMBBs;
85
13
  DebugLoc DL = JTInst->getDebugLoc();
86
13
  MachineOperand Selector = JTInst->getOperand(0);
87
13
  const GlobalValue *CombinedGlobal = JTInst->getOperand(1).getGlobal();
88
13
89
19
  auto CmpTarget = [&](unsigned Target) {
90
19
    if (Selector.isReg())
91
19
      MBB->addLiveIn(Selector.getReg());
92
19
    BuildMI(*MBB, MBBI, DL, TII->get(X86::LEA64r), X86::R11)
93
19
        .addReg(X86::RIP)
94
19
        .addImm(1)
95
19
        .addReg(0)
96
19
        .addGlobalAddress(CombinedGlobal,
97
19
                          JTInst->getOperand(2 + 2 * Target).getImm())
98
19
        .addReg(0);
99
19
    BuildMI(*MBB, MBBI, DL, TII->get(X86::CMP64rr))
100
19
        .add(Selector)
101
19
        .addReg(X86::R11);
102
19
  };
103
13
104
54
  auto CreateMBB = [&]() {
105
54
    auto *NewMBB = MF->CreateMachineBasicBlock(BB);
106
54
    MBB->addSuccessor(NewMBB);
107
54
    if (!MBB->isLiveIn(X86::EFLAGS))
108
27
      MBB->addLiveIn(X86::EFLAGS);
109
54
    return NewMBB;
110
54
  };
111
13
112
27
  auto EmitCondJump = [&](unsigned CC, MachineBasicBlock *ThenMBB) {
113
27
    BuildMI(*MBB, MBBI, DL, TII->get(X86::JCC_1)).addMBB(ThenMBB).addImm(CC);
114
27
115
27
    auto *ElseMBB = CreateMBB();
116
27
    MF->insert(InsPt, ElseMBB);
117
27
    MBB = ElseMBB;
118
27
    MBBI = MBB->end();
119
27
  };
120
13
121
25
  auto EmitCondJumpTarget = [&](unsigned CC, unsigned Target) {
122
25
    auto *ThenMBB = CreateMBB();
123
25
    TargetMBBs.push_back({ThenMBB, Target});
124
25
    EmitCondJump(CC, ThenMBB);
125
25
  };
126
13
127
15
  auto EmitTailCall = [&](unsigned Target) {
128
15
    BuildMI(*MBB, MBBI, DL, TII->get(X86::TAILJMPd64))
129
15
        .add(JTInst->getOperand(3 + 2 * Target));
130
15
  };
131
13
132
13
  std::function<void(unsigned, unsigned)> EmitBranchFunnel =
133
23
      [&](unsigned FirstTarget, unsigned NumTargets) {
134
23
    if (NumTargets == 1) {
135
4
      EmitTailCall(FirstTarget);
136
4
      return;
137
4
    }
138
19
139
19
    if (NumTargets == 2) {
140
11
      CmpTarget(FirstTarget + 1);
141
11
      EmitCondJumpTarget(X86::COND_B, FirstTarget);
142
11
      EmitTailCall(FirstTarget + 1);
143
11
      return;
144
11
    }
145
8
146
8
    if (NumTargets < 6) {
147
6
      CmpTarget(FirstTarget + 1);
148
6
      EmitCondJumpTarget(X86::COND_B, FirstTarget);
149
6
      EmitCondJumpTarget(X86::COND_E, FirstTarget + 1);
150
6
      EmitBranchFunnel(FirstTarget + 2, NumTargets - 2);
151
6
      return;
152
6
    }
153
2
154
2
    auto *ThenMBB = CreateMBB();
155
2
    CmpTarget(FirstTarget + (NumTargets / 2));
156
2
    EmitCondJump(X86::COND_B, ThenMBB);
157
2
    EmitCondJumpTarget(X86::COND_E, FirstTarget + (NumTargets / 2));
158
2
    EmitBranchFunnel(FirstTarget + (NumTargets / 2) + 1,
159
2
                  NumTargets - (NumTargets / 2) - 1);
160
2
161
2
    MF->insert(InsPt, ThenMBB);
162
2
    MBB = ThenMBB;
163
2
    MBBI = MBB->end();
164
2
    EmitBranchFunnel(FirstTarget, NumTargets / 2);
165
2
  };
166
13
167
13
  EmitBranchFunnel(0, (JTInst->getNumOperands() - 2) / 2);
168
25
  for (auto P : TargetMBBs) {
169
25
    MF->insert(InsPt, P.first);
170
25
    BuildMI(P.first, DL, TII->get(X86::TAILJMPd64))
171
25
        .add(JTInst->getOperand(3 + 2 * P.second));
172
25
  }
173
13
  JTMBB->erase(JTInst);
174
13
}
175
176
/// If \p MBBI is a pseudo instruction, this method expands
177
/// it to the corresponding (sequence of) actual instruction(s).
178
/// \returns true if \p MBBI has been expanded.
179
bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
180
2.53M
                               MachineBasicBlock::iterator MBBI) {
181
2.53M
  MachineInstr &MI = *MBBI;
182
2.53M
  unsigned Opcode = MI.getOpcode();
183
2.53M
  DebugLoc DL = MBBI->getDebugLoc();
184
2.53M
  switch (Opcode) {
185
2.53M
  default:
186
2.40M
    return false;
187
2.53M
  case X86::TCRETURNdi:
188
7.38k
  case X86::TCRETURNdicc:
189
7.38k
  case X86::TCRETURNri:
190
7.38k
  case X86::TCRETURNmi:
191
7.38k
  case X86::TCRETURNdi64:
192
7.38k
  case X86::TCRETURNdi64cc:
193
7.38k
  case X86::TCRETURNri64:
194
7.38k
  case X86::TCRETURNmi64: {
195
7.38k
    bool isMem = Opcode == X86::TCRETURNmi || 
Opcode == X86::TCRETURNmi647.37k
;
196
7.38k
    MachineOperand &JumpTarget = MBBI->getOperand(0);
197
7.38k
    MachineOperand &StackAdjust = MBBI->getOperand(isMem ? 
5105
:
17.27k
);
198
7.38k
    assert(StackAdjust.isImm() && "Expecting immediate value.");
199
7.38k
200
7.38k
    // Adjust stack pointer.
201
7.38k
    int StackAdj = StackAdjust.getImm();
202
7.38k
    int MaxTCDelta = X86FI->getTCReturnAddrDelta();
203
7.38k
    int Offset = 0;
204
7.38k
    assert(MaxTCDelta <= 0 && "MaxTCDelta should never be positive");
205
7.38k
206
7.38k
    // Incoporate the retaddr area.
207
7.38k
    Offset = StackAdj - MaxTCDelta;
208
7.38k
    assert(Offset >= 0 && "Offset should never be negative");
209
7.38k
210
7.38k
    if (Opcode == X86::TCRETURNdicc || 
Opcode == X86::TCRETURNdi64cc7.37k
) {
211
28
      assert(Offset == 0 && "Conditional tail call cannot adjust the stack.");
212
28
    }
213
7.38k
214
7.38k
    if (Offset) {
215
1
      // Check for possible merge with preceding ADD instruction.
216
1
      Offset += X86FL->mergeSPUpdates(MBB, MBBI, true);
217
1
      X86FL->emitSPUpdate(MBB, MBBI, DL, Offset, /*InEpilogue=*/true);
218
1
    }
219
7.38k
220
7.38k
    // Jump to label or value in register.
221
7.38k
    bool IsWin64 = STI->isTargetWin64();
222
7.38k
    if (Opcode == X86::TCRETURNdi || 
Opcode == X86::TCRETURNdicc6.80k
||
223
7.38k
        
Opcode == X86::TCRETURNdi646.80k
||
Opcode == X86::TCRETURNdi64cc476
) {
224
6.92k
      unsigned Op;
225
6.92k
      switch (Opcode) {
226
6.92k
      case X86::TCRETURNdi:
227
576
        Op = X86::TAILJMPd;
228
576
        break;
229
6.92k
      case X86::TCRETURNdicc:
230
3
        Op = X86::TAILJMPd_CC;
231
3
        break;
232
6.92k
      case X86::TCRETURNdi64cc:
233
25
        assert(!MBB.getParent()->hasWinCFI() &&
234
25
               "Conditional tail calls confuse "
235
25
               "the Win64 unwinder.");
236
25
        Op = X86::TAILJMPd64_CC;
237
25
        break;
238
6.92k
      default:
239
6.32k
        // Note: Win64 uses REX prefixes indirect jumps out of functions, but
240
6.32k
        // not direct ones.
241
6.32k
        Op = X86::TAILJMPd64;
242
6.32k
        break;
243
6.92k
      }
244
6.92k
      MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));
245
6.92k
      if (JumpTarget.isGlobal()) {
246
6.75k
        MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
247
6.75k
                             JumpTarget.getTargetFlags());
248
6.75k
      } else {
249
178
        assert(JumpTarget.isSymbol());
250
178
        MIB.addExternalSymbol(JumpTarget.getSymbolName(),
251
178
                              JumpTarget.getTargetFlags());
252
178
      }
253
6.92k
      if (Op == X86::TAILJMPd_CC || 
Op == X86::TAILJMPd64_CC6.92k
) {
254
28
        MIB.addImm(MBBI->getOperand(2).getImm());
255
28
      }
256
6.92k
257
6.92k
    } else 
if (452
Opcode == X86::TCRETURNmi452
||
Opcode == X86::TCRETURNmi64442
) {
258
105
      unsigned Op = (Opcode == X86::TCRETURNmi)
259
105
                        ? 
X86::TAILJMPm9
260
105
                        : 
(IsWin64 96
?
X86::TAILJMPm64_REX4
:
X86::TAILJMPm6492
);
261
105
      MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));
262
635
      for (unsigned i = 0; i != 5; 
++i530
)
263
530
        MIB.add(MBBI->getOperand(i));
264
347
    } else if (Opcode == X86::TCRETURNri64) {
265
277
      JumpTarget.setIsKill();
266
277
      BuildMI(MBB, MBBI, DL,
267
277
              TII->get(IsWin64 ? 
X86::TAILJMPr64_REX6
:
X86::TAILJMPr64271
))
268
277
          .add(JumpTarget);
269
277
    } else {
270
70
      JumpTarget.setIsKill();
271
70
      BuildMI(MBB, MBBI, DL, TII->get(X86::TAILJMPr))
272
70
          .add(JumpTarget);
273
70
    }
274
7.38k
275
7.38k
    MachineInstr &NewMI = *std::prev(MBBI);
276
7.38k
    NewMI.copyImplicitOps(*MBBI->getParent()->getParent(), *MBBI);
277
7.38k
    MBB.getParent()->updateCallSiteInfo(&*MBBI, &NewMI);
278
7.38k
279
7.38k
    // Delete the pseudo instruction TCRETURN.
280
7.38k
    MBB.erase(MBBI);
281
7.38k
282
7.38k
    return true;
283
7.38k
  }
284
7.38k
  case X86::EH_RETURN:
285
6
  case X86::EH_RETURN64: {
286
6
    MachineOperand &DestAddr = MBBI->getOperand(0);
287
6
    assert(DestAddr.isReg() && "Offset should be in register!");
288
6
    const bool Uses64BitFramePtr =
289
6
        STI->isTarget64BitLP64() || 
STI->isTargetNaCl64()2
;
290
6
    unsigned StackPtr = TRI->getStackRegister();
291
6
    BuildMI(MBB, MBBI, DL,
292
6
            TII->get(Uses64BitFramePtr ? 
X86::MOV64rr4
:
X86::MOV32rr2
), StackPtr)
293
6
        .addReg(DestAddr.getReg());
294
6
    // The EH_RETURN pseudo is really removed during the MC Lowering.
295
6
    return true;
296
6
  }
297
39
  case X86::IRET: {
298
39
    // Adjust stack to erase error code
299
39
    int64_t StackAdj = MBBI->getOperand(0).getImm();
300
39
    X86FL->emitSPUpdate(MBB, MBBI, DL, StackAdj, true);
301
39
    // Replace pseudo with machine iret
302
39
    BuildMI(MBB, MBBI, DL,
303
39
            TII->get(STI->is64Bit() ? 
X86::IRET6421
:
X86::IRET3218
));
304
39
    MBB.erase(MBBI);
305
39
    return true;
306
6
  }
307
125k
  case X86::RET: {
308
125k
    // Adjust stack to erase error code
309
125k
    int64_t StackAdj = MBBI->getOperand(0).getImm();
310
125k
    MachineInstrBuilder MIB;
311
125k
    if (StackAdj == 0) {
312
125k
      MIB = BuildMI(MBB, MBBI, DL,
313
125k
                    TII->get(STI->is64Bit() ? 
X86::RETQ102k
:
X86::RETL22.6k
));
314
125k
    } else 
if (617
isUInt<16>(StackAdj)617
) {
315
616
      MIB = BuildMI(MBB, MBBI, DL,
316
616
                    TII->get(STI->is64Bit() ? 
X86::RETIQ1
:
X86::RETIL615
))
317
616
                .addImm(StackAdj);
318
616
    } else {
319
1
      assert(!STI->is64Bit() &&
320
1
             "shouldn't need to do this for x86_64 targets!");
321
1
      // A ret can only handle immediates as big as 2**16-1.  If we need to pop
322
1
      // off bytes before the return address, we must do it manually.
323
1
      BuildMI(MBB, MBBI, DL, TII->get(X86::POP32r)).addReg(X86::ECX, RegState::Define);
324
1
      X86FL->emitSPUpdate(MBB, MBBI, DL, StackAdj, /*InEpilogue=*/true);
325
1
      BuildMI(MBB, MBBI, DL, TII->get(X86::PUSH32r)).addReg(X86::ECX);
326
1
      MIB = BuildMI(MBB, MBBI, DL, TII->get(X86::RETL));
327
1
    }
328
236k
    for (unsigned I = 1, E = MBBI->getNumOperands(); I != E; 
++I111k
)
329
111k
      MIB.add(MBBI->getOperand(I));
330
125k
    MBB.erase(MBBI);
331
125k
    return true;
332
6
  }
333
37
  case X86::EH_RESTORE: {
334
37
    // Restore ESP and EBP, and optionally ESI if required.
335
37
    bool IsSEH = isAsynchronousEHPersonality(classifyEHPersonality(
336
37
        MBB.getParent()->getFunction().getPersonalityFn()));
337
37
    X86FL->restoreWin32EHStackPointers(MBB, MBBI, DL, /*RestoreSP=*/IsSEH);
338
37
    MBBI->eraseFromParent();
339
37
    return true;
340
6
  }
341
6
  case X86::LCMPXCHG8B_SAVE_EBX:
342
2
  case X86::LCMPXCHG16B_SAVE_RBX: {
343
2
    // Perform the following transformation.
344
2
    // SaveRbx = pseudocmpxchg Addr, <4 opds for the address>, InArg, SaveRbx
345
2
    // =>
346
2
    // [E|R]BX = InArg
347
2
    // actualcmpxchg Addr
348
2
    // [E|R]BX = SaveRbx
349
2
    const MachineOperand &InArg = MBBI->getOperand(6);
350
2
    unsigned SaveRbx = MBBI->getOperand(7).getReg();
351
2
352
2
    unsigned ActualInArg =
353
2
        Opcode == X86::LCMPXCHG8B_SAVE_EBX ? 
X86::EBX0
: X86::RBX;
354
2
    // Copy the input argument of the pseudo into the argument of the
355
2
    // actual instruction.
356
2
    TII->copyPhysReg(MBB, MBBI, DL, ActualInArg, InArg.getReg(),
357
2
                     InArg.isKill());
358
2
    // Create the actual instruction.
359
2
    unsigned ActualOpc =
360
2
        Opcode == X86::LCMPXCHG8B_SAVE_EBX ? 
X86::LCMPXCHG8B0
: X86::LCMPXCHG16B;
361
2
    MachineInstr *NewInstr = BuildMI(MBB, MBBI, DL, TII->get(ActualOpc));
362
2
    // Copy the operands related to the address.
363
12
    for (unsigned Idx = 1; Idx < 6; 
++Idx10
)
364
10
      NewInstr->addOperand(MBBI->getOperand(Idx));
365
2
    // Finally, restore the value of RBX.
366
2
    TII->copyPhysReg(MBB, MBBI, DL, ActualInArg, SaveRbx,
367
2
                     /*SrcIsKill*/ true);
368
2
369
2
    // Delete the pseudo.
370
2
    MBBI->eraseFromParent();
371
2
    return true;
372
2
  }
373
13
  case TargetOpcode::ICALL_BRANCH_FUNNEL:
374
13
    ExpandICallBranchFunnel(&MBB, MBBI);
375
13
    return true;
376
0
  }
377
0
  llvm_unreachable("Previous switch has a fallthrough?");
378
0
}
379
380
/// Expand all pseudo instructions contained in \p MBB.
381
/// \returns true if any expansion occurred for \p MBB.
382
396k
bool X86ExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) {
383
396k
  bool Modified = false;
384
396k
385
396k
  // MBBI may be invalidated by the expansion.
386
396k
  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
387
2.93M
  while (MBBI != E) {
388
2.53M
    MachineBasicBlock::iterator NMBBI = std::next(MBBI);
389
2.53M
    Modified |= ExpandMI(MBB, MBBI);
390
2.53M
    MBBI = NMBBI;
391
2.53M
  }
392
396k
393
396k
  return Modified;
394
396k
}
395
396
137k
bool X86ExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
397
137k
  STI = &static_cast<const X86Subtarget &>(MF.getSubtarget());
398
137k
  TII = STI->getInstrInfo();
399
137k
  TRI = STI->getRegisterInfo();
400
137k
  X86FI = MF.getInfo<X86MachineFunctionInfo>();
401
137k
  X86FL = STI->getFrameLowering();
402
137k
403
137k
  bool Modified = false;
404
137k
  for (MachineBasicBlock &MBB : MF)
405
396k
    Modified |= ExpandMBB(MBB);
406
137k
  return Modified;
407
137k
}
408
409
/// Returns an instance of the pseudo instruction expansion pass.
410
12.2k
FunctionPass *llvm::createX86ExpandPseudoPass() {
411
12.2k
  return new X86ExpandPseudo();
412
12.2k
}