Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/X86/X86VZeroUpper.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter ------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file defines the pass which inserts x86 AVX vzeroupper instructions
10
// before calls to SSE encoded functions. This avoids transition latency
11
// penalty when transferring control between AVX encoded instructions and old
12
// SSE encoding mode.
13
//
14
//===----------------------------------------------------------------------===//
15
16
#include "X86.h"
17
#include "X86InstrInfo.h"
18
#include "X86Subtarget.h"
19
#include "llvm/ADT/SmallVector.h"
20
#include "llvm/ADT/Statistic.h"
21
#include "llvm/CodeGen/MachineBasicBlock.h"
22
#include "llvm/CodeGen/MachineFunction.h"
23
#include "llvm/CodeGen/MachineFunctionPass.h"
24
#include "llvm/CodeGen/MachineInstr.h"
25
#include "llvm/CodeGen/MachineInstrBuilder.h"
26
#include "llvm/CodeGen/MachineOperand.h"
27
#include "llvm/CodeGen/MachineRegisterInfo.h"
28
#include "llvm/CodeGen/TargetInstrInfo.h"
29
#include "llvm/CodeGen/TargetRegisterInfo.h"
30
#include "llvm/IR/CallingConv.h"
31
#include "llvm/IR/DebugLoc.h"
32
#include "llvm/IR/Function.h"
33
#include "llvm/Support/Debug.h"
34
#include "llvm/Support/ErrorHandling.h"
35
#include "llvm/Support/raw_ostream.h"
36
#include <cassert>
37
38
using namespace llvm;
39
40
#define DEBUG_TYPE "x86-vzeroupper"
41
42
STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
43
44
namespace {
45
46
  class VZeroUpperInserter : public MachineFunctionPass {
47
  public:
48
12.2k
    VZeroUpperInserter() : MachineFunctionPass(ID) {}
49
50
    bool runOnMachineFunction(MachineFunction &MF) override;
51
52
12.1k
    MachineFunctionProperties getRequiredProperties() const override {
53
12.1k
      return MachineFunctionProperties().set(
54
12.1k
          MachineFunctionProperties::Property::NoVRegs);
55
12.1k
    }
56
57
150k
    StringRef getPassName() const override { return "X86 vzeroupper inserter"; }
58
59
  private:
60
    void processBasicBlock(MachineBasicBlock &MBB);
61
    void insertVZeroUpper(MachineBasicBlock::iterator I,
62
                          MachineBasicBlock &MBB);
63
    void addDirtySuccessor(MachineBasicBlock &MBB);
64
65
    using BlockExitState = enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
66
67
    static const char* getBlockExitStateName(BlockExitState ST);
68
69
    // Core algorithm state:
70
    // BlockState - Each block is either:
71
    //   - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
72
    //                   vzeroupper instructions in this block.
73
    //   - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
74
    //                  block that will ensure that YMM/ZMM is clean on exit.
75
    //   - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
76
    //                  subsequent vzeroupper in the block clears it.
77
    //
78
    // AddedToDirtySuccessors - This flag is raised when a block is added to the
79
    //                          DirtySuccessors list to ensure that it's not
80
    //                          added multiple times.
81
    //
82
    // FirstUnguardedCall - Records the location of the first unguarded call in
83
    //                      each basic block that may need to be guarded by a
84
    //                      vzeroupper. We won't know whether it actually needs
85
    //                      to be guarded until we discover a predecessor that
86
    //                      is DIRTY_OUT.
87
    struct BlockState {
88
      BlockExitState ExitState = PASS_THROUGH;
89
      bool AddedToDirtySuccessors = false;
90
      MachineBasicBlock::iterator FirstUnguardedCall;
91
92
54.2k
      BlockState() = default;
93
    };
94
95
    using BlockStateMap = SmallVector<BlockState, 8>;
96
    using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;
97
98
    BlockStateMap BlockStates;
99
    DirtySuccessorsWorkList DirtySuccessors;
100
    bool EverMadeChange;
101
    bool IsX86INTR;
102
    const TargetInstrInfo *TII;
103
104
    static char ID;
105
  };
106
107
} // end anonymous namespace
108
109
char VZeroUpperInserter::ID = 0;
110
111
12.2k
FunctionPass *llvm::createX86IssueVZeroUpperPass() {
112
12.2k
  return new VZeroUpperInserter();
113
12.2k
}
114
115
#ifndef NDEBUG
116
const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) {
117
  switch (ST) {
118
    case PASS_THROUGH: return "Pass-through";
119
    case EXITS_DIRTY: return "Exits-dirty";
120
    case EXITS_CLEAN: return "Exits-clean";
121
  }
122
  llvm_unreachable("Invalid block exit state.");
123
}
124
#endif
125
126
/// VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
127
/// Thus, there is no need to check for Y/ZMM16 and above.
128
471k
static bool isYmmOrZmmReg(unsigned Reg) {
129
471k
  return (Reg >= X86::YMM0 && 
Reg <= X86::YMM15102k
) ||
130
471k
         
(404k
Reg >= X86::ZMM0404k
&&
Reg <= X86::ZMM1535.0k
);
131
471k
}
132
133
75.4k
static bool checkFnHasLiveInYmmOrZmm(MachineRegisterInfo &MRI) {
134
75.4k
  for (std::pair<unsigned, unsigned> LI : MRI.liveins())
135
106k
    if (isYmmOrZmmReg(LI.first))
136
30.8k
      return true;
137
75.4k
138
75.4k
  
return false44.5k
;
139
75.4k
}
140
141
3.08k
static bool clobbersAllYmmAndZmmRegs(const MachineOperand &MO) {
142
52.2k
  for (unsigned reg = X86::YMM0; reg <= X86::YMM15; 
++reg49.2k
) {
143
49.2k
    if (!MO.clobbersPhysReg(reg))
144
18
      return false;
145
49.2k
  }
146
52.1k
  
for (unsigned reg = X86::ZMM0; 3.07k
reg <= X86::ZMM15;
++reg49.1k
) {
147
49.1k
    if (!MO.clobbersPhysReg(reg))
148
0
      return false;
149
49.1k
  }
150
3.07k
  return true;
151
3.07k
}
152
153
165k
static bool hasYmmOrZmmReg(MachineInstr &MI) {
154
469k
  for (const MachineOperand &MO : MI.operands()) {
155
469k
    if (MI.isCall() && 
MO.isRegMask()25.8k
&&
!clobbersAllYmmAndZmmRegs(MO)3.08k
)
156
18
      return true;
157
469k
    if (!MO.isReg())
158
104k
      continue;
159
365k
    if (MO.isDebug())
160
0
      continue;
161
365k
    if (isYmmOrZmmReg(MO.getReg()))
162
68.2k
      return true;
163
365k
  }
164
165k
  
return false96.7k
;
165
165k
}
166
167
/// Check if given call instruction has a RegMask operand.
168
3.02k
static bool callHasRegMask(MachineInstr &MI) {
169
3.02k
  assert(MI.isCall() && "Can only be called on call instructions.");
170
6.49k
  for (const MachineOperand &MO : MI.operands()) {
171
6.49k
    if (MO.isRegMask())
172
3.02k
      return true;
173
6.49k
  }
174
3.02k
  
return false3
;
175
3.02k
}
176
177
/// Insert a vzeroupper instruction before I.
178
void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I,
179
11.9k
                                          MachineBasicBlock &MBB) {
180
11.9k
  DebugLoc dl = I->getDebugLoc();
181
11.9k
  BuildMI(MBB, I, dl, TII->get(X86::VZEROUPPER));
182
11.9k
  ++NumVZU;
183
11.9k
  EverMadeChange = true;
184
11.9k
}
185
186
/// Add MBB to the DirtySuccessors list if it hasn't already been added.
187
44.9k
void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
188
44.9k
  if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
189
40.1k
    DirtySuccessors.push_back(&MBB);
190
40.1k
    BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
191
40.1k
  }
192
44.9k
}
193
194
/// Loop over all of the instructions in the basic block, inserting vzeroupper
195
/// instructions before function calls.
196
53.9k
void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
197
53.9k
  // Start by assuming that the block is PASS_THROUGH which implies no unguarded
198
53.9k
  // calls.
199
53.9k
  BlockExitState CurState = PASS_THROUGH;
200
53.9k
  BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
201
53.9k
202
307k
  for (MachineInstr &MI : MBB) {
203
307k
    bool IsCall = MI.isCall();
204
307k
    bool IsReturn = MI.isReturn();
205
307k
    bool IsControlFlow = IsCall || 
IsReturn304k
;
206
307k
207
307k
    // No need for vzeroupper before iret in interrupt handler function,
208
307k
    // epilogue will restore YMM/ZMM registers if needed.
209
307k
    if (IsX86INTR && 
IsReturn328
)
210
3
      continue;
211
307k
212
307k
    // An existing VZERO* instruction resets the state.
213
307k
    if (MI.getOpcode() == X86::VZEROALL || 
MI.getOpcode() == X86::VZEROUPPER307k
) {
214
20
      CurState = EXITS_CLEAN;
215
20
      continue;
216
20
    }
217
307k
218
307k
    // Shortcut: don't need to check regular instructions in dirty state.
219
307k
    if (!IsControlFlow && 
CurState == EXITS_DIRTY265k
)
220
142k
      continue;
221
165k
222
165k
    if (hasYmmOrZmmReg(MI)) {
223
68.2k
      // We found a ymm/zmm-using instruction; this could be an AVX/AVX512
224
68.2k
      // instruction, or it could be control flow.
225
68.2k
      CurState = EXITS_DIRTY;
226
68.2k
      continue;
227
68.2k
    }
228
96.7k
229
96.7k
    // Check for control-flow out of the current function (which might
230
96.7k
    // indirectly execute SSE instructions).
231
96.7k
    if (!IsControlFlow)
232
82.2k
      continue;
233
14.5k
234
14.5k
    // If the call has no RegMask, skip it as well. It usually happens on
235
14.5k
    // helper function calls (such as '_chkstk', '_ftol2') where standard
236
14.5k
    // calling convention is not used (RegMask is not used to mark register
237
14.5k
    // clobbered and register usage (def/implicit-def/use) is well-defined and
238
14.5k
    // explicitly specified.
239
14.5k
    if (IsCall && 
!callHasRegMask(MI)3.02k
)
240
3
      continue;
241
14.5k
242
14.5k
    // The VZEROUPPER instruction resets the upper 128 bits of YMM0-YMM15
243
14.5k
    // registers. In addition, the processor changes back to Clean state, after
244
14.5k
    // which execution of SSE instructions or AVX instructions has no transition
245
14.5k
    // penalty. Add the VZEROUPPER instruction before any function call/return
246
14.5k
    // that might execute SSE code.
247
14.5k
    // FIXME: In some cases, we may want to move the VZEROUPPER into a
248
14.5k
    // predecessor block.
249
14.5k
    if (CurState == EXITS_DIRTY) {
250
11.1k
      // After the inserted VZEROUPPER the state becomes clean again, but
251
11.1k
      // other YMM/ZMM may appear before other subsequent calls or even before
252
11.1k
      // the end of the BB.
253
11.1k
      insertVZeroUpper(MI, MBB);
254
11.1k
      CurState = EXITS_CLEAN;
255
11.1k
    } else 
if (3.31k
CurState == PASS_THROUGH3.31k
) {
256
2.37k
      // If this block is currently in pass-through state and we encounter a
257
2.37k
      // call then whether we need a vzeroupper or not depends on whether this
258
2.37k
      // block has successors that exit dirty. Record the location of the call,
259
2.37k
      // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet.
260
2.37k
      // It will be inserted later if necessary.
261
2.37k
      BlockStates[MBB.getNumber()].FirstUnguardedCall = MI;
262
2.37k
      CurState = EXITS_CLEAN;
263
2.37k
    }
264
14.5k
  }
265
53.9k
266
53.9k
  LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
267
53.9k
                    << getBlockExitStateName(CurState) << '\n');
268
53.9k
269
53.9k
  if (CurState == EXITS_DIRTY)
270
29.7k
    for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(),
271
29.7k
                                          SE = MBB.succ_end();
272
33.5k
         SI != SE; 
++SI3.73k
)
273
3.73k
      addDirtySuccessor(**SI);
274
53.9k
275
53.9k
  BlockStates[MBB.getNumber()].ExitState = CurState;
276
53.9k
}
277
278
/// Loop over all of the basic blocks, inserting vzeroupper instructions before
279
/// function calls.
280
137k
bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
281
137k
  const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
282
137k
  if (!ST.hasAVX() || 
ST.hasFastPartialYMMorZMMWrite()77.2k
)
283
62.3k
    return false;
284
75.4k
  TII = ST.getInstrInfo();
285
75.4k
  MachineRegisterInfo &MRI = MF.getRegInfo();
286
75.4k
  EverMadeChange = false;
287
75.4k
  IsX86INTR = MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
288
75.4k
289
75.4k
  bool FnHasLiveInYmmOrZmm = checkFnHasLiveInYmmOrZmm(MRI);
290
75.4k
291
75.4k
  // Fast check: if the function doesn't use any ymm/zmm registers, we don't
292
75.4k
  // need to insert any VZEROUPPER instructions.  This is constant-time, so it
293
75.4k
  // is cheap in the common case of no ymm/zmm use.
294
75.4k
  bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
295
75.4k
  const TargetRegisterClass *RCs[2] = {&X86::VR256RegClass, &X86::VR512RegClass};
296
150k
  for (auto *RC : RCs) {
297
150k
    if (!YmmOrZmmUsed) {
298
1.86M
      for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end(); i != e;
299
1.79M
           
i++1.78M
) {
300
1.79M
        if (!MRI.reg_nodbg_empty(*i)) {
301
8.16k
          YmmOrZmmUsed = true;
302
8.16k
          break;
303
8.16k
        }
304
1.79M
      }
305
83.1k
    }
306
150k
  }
307
75.4k
  if (!YmmOrZmmUsed) {
308
36.4k
    return false;
309
36.4k
  }
310
39.0k
311
39.0k
  assert(BlockStates.empty() && DirtySuccessors.empty() &&
312
39.0k
         "X86VZeroUpper state should be clear");
313
39.0k
  BlockStates.resize(MF.getNumBlockIDs());
314
39.0k
315
39.0k
  // Process all blocks. This will compute block exit states, record the first
316
39.0k
  // unguarded call in each block, and add successors of dirty blocks to the
317
39.0k
  // DirtySuccessors list.
318
39.0k
  for (MachineBasicBlock &MBB : MF)
319
53.9k
    processBasicBlock(MBB);
320
39.0k
321
39.0k
  // If any YMM/ZMM regs are live-in to this function, add the entry block to
322
39.0k
  // the DirtySuccessors list
323
39.0k
  if (FnHasLiveInYmmOrZmm)
324
30.8k
    addDirtySuccessor(MF.front());
325
39.0k
326
39.0k
  // Re-visit all blocks that are successors of EXITS_DIRTY blocks. Add
327
39.0k
  // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
328
39.0k
  // through PASS_THROUGH blocks.
329
79.1k
  while (!DirtySuccessors.empty()) {
330
40.1k
    MachineBasicBlock &MBB = *DirtySuccessors.back();
331
40.1k
    DirtySuccessors.pop_back();
332
40.1k
    BlockState &BBState = BlockStates[MBB.getNumber()];
333
40.1k
334
40.1k
    // MBB is a successor of a dirty block, so its first call needs to be
335
40.1k
    // guarded.
336
40.1k
    if (BBState.FirstUnguardedCall != MBB.end())
337
765
      insertVZeroUpper(BBState.FirstUnguardedCall, MBB);
338
40.1k
339
40.1k
    // If this successor was a pass-through block, then it is now dirty. Its
340
40.1k
    // successors need to be added to the worklist (if they haven't been
341
40.1k
    // already).
342
40.1k
    if (BBState.ExitState == PASS_THROUGH) {
343
6.74k
      LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber()
344
6.74k
                        << " was Pass-through, is now Dirty-out.\n");
345
6.74k
      for (MachineBasicBlock *Succ : MBB.successors())
346
10.4k
        addDirtySuccessor(*Succ);
347
6.74k
    }
348
40.1k
  }
349
39.0k
350
39.0k
  BlockStates.clear();
351
39.0k
  return EverMadeChange;
352
39.0k
}