Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/X86/X86InsertPrefetch.cpp
Line
Count
Source (jump to first uncovered line)
1
//===------- X86InsertPrefetch.cpp - Insert cache prefetch hints ----------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This pass applies cache prefetch instructions based on a profile. The pass
10
// assumes DiscriminateMemOps ran immediately before, to ensure debug info
11
// matches the one used at profile generation time. The profile is encoded in
12
// afdo format (text or binary). It contains prefetch hints recommendations.
13
// Each recommendation is made in terms of debug info locations, a type (i.e.
14
// nta, t{0|1|2}) and a delta. The debug info identifies an instruction with a
15
// memory operand (see X86DiscriminateMemOps). The prefetch will be made for
16
// a location at that memory operand + the delta specified in the
17
// recommendation.
18
//
19
//===----------------------------------------------------------------------===//
20
21
#include "X86.h"
22
#include "X86InstrBuilder.h"
23
#include "X86InstrInfo.h"
24
#include "X86MachineFunctionInfo.h"
25
#include "X86Subtarget.h"
26
#include "llvm/CodeGen/MachineModuleInfo.h"
27
#include "llvm/IR/DebugInfoMetadata.h"
28
#include "llvm/ProfileData/SampleProf.h"
29
#include "llvm/ProfileData/SampleProfReader.h"
30
#include "llvm/Transforms/IPO/SampleProfile.h"
31
using namespace llvm;
32
using namespace sampleprof;
33
34
static cl::opt<std::string>
35
    PrefetchHintsFile("prefetch-hints-file",
36
                      cl::desc("Path to the prefetch hints profile. See also "
37
                               "-x86-discriminate-memops"),
38
                      cl::Hidden);
39
namespace {
40
41
class X86InsertPrefetch : public MachineFunctionPass {
42
  void getAnalysisUsage(AnalysisUsage &AU) const override;
43
  bool doInitialization(Module &) override;
44
45
  bool runOnMachineFunction(MachineFunction &MF) override;
46
  struct PrefetchInfo {
47
    unsigned InstructionID;
48
    int64_t Delta;
49
  };
50
  typedef SmallVectorImpl<PrefetchInfo> Prefetches;
51
  bool findPrefetchInfo(const FunctionSamples *Samples, const MachineInstr &MI,
52
                        Prefetches &prefetches) const;
53
54
public:
55
  static char ID;
56
  X86InsertPrefetch(const std::string &PrefetchHintsFilename);
57
150k
  StringRef getPassName() const override {
58
150k
    return "X86 Insert Cache Prefetches";
59
150k
  }
60
61
private:
62
  std::string Filename;
63
  std::unique_ptr<SampleProfileReader> Reader;
64
};
65
66
using PrefetchHints = SampleRecord::CallTargetMap;
67
68
// Return any prefetching hints for the specified MachineInstruction. The hints
69
// are returned as pairs (name, delta).
70
ErrorOr<PrefetchHints> getPrefetchHints(const FunctionSamples *TopSamples,
71
7
                                        const MachineInstr &MI) {
72
7
  if (const auto &Loc = MI.getDebugLoc())
73
7
    if (const auto *Samples = TopSamples->findFunctionSamples(Loc))
74
7
      return Samples->findCallTargetMapAt(FunctionSamples::getOffset(Loc),
75
7
                                          Loc->getBaseDiscriminator());
76
0
  return std::error_code();
77
0
}
78
79
// The prefetch instruction can't take memory operands involving vector
80
// registers.
81
8
bool IsMemOpCompatibleWithPrefetch(const MachineInstr &MI, int Op) {
82
8
  unsigned BaseReg = MI.getOperand(Op + X86::AddrBaseReg).getReg();
83
8
  unsigned IndexReg = MI.getOperand(Op + X86::AddrIndexReg).getReg();
84
8
  return (BaseReg == 0 ||
85
8
          
X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg)7
||
86
8
          
X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg)0
) &&
87
8
         (IndexReg == 0 ||
88
8
          
X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg)5
||
89
8
          
X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg)1
);
90
8
}
91
92
} // end anonymous namespace
93
94
//===----------------------------------------------------------------------===//
95
//            Implementation
96
//===----------------------------------------------------------------------===//
97
98
char X86InsertPrefetch::ID = 0;
99
100
X86InsertPrefetch::X86InsertPrefetch(const std::string &PrefetchHintsFilename)
101
12.2k
    : MachineFunctionPass(ID), Filename(PrefetchHintsFilename) {}
102
103
/// Return true if the provided MachineInstruction has cache prefetch hints. In
104
/// that case, the prefetch hints are stored, in order, in the Prefetches
105
/// vector.
106
bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples,
107
                                         const MachineInstr &MI,
108
7
                                         Prefetches &Prefetches) const {
109
7
  assert(Prefetches.empty() &&
110
7
         "Expected caller passed empty PrefetchInfo vector.");
111
7
  static const std::pair<const StringRef, unsigned> HintTypes[] = {
112
7
      {"_nta_", X86::PREFETCHNTA},
113
7
      {"_t0_", X86::PREFETCHT0},
114
7
      {"_t1_", X86::PREFETCHT1},
115
7
      {"_t2_", X86::PREFETCHT2},
116
7
  };
117
7
  static const char *SerializedPrefetchPrefix = "__prefetch";
118
7
119
7
  const ErrorOr<PrefetchHints> T = getPrefetchHints(TopSamples, MI);
120
7
  if (!T)
121
1
    return false;
122
6
  int16_t max_index = -1;
123
6
  // Convert serialized prefetch hints into PrefetchInfo objects, and populate
124
6
  // the Prefetches vector.
125
9
  for (const auto &S_V : *T) {
126
9
    StringRef Name = S_V.getKey();
127
9
    if (Name.consume_front(SerializedPrefetchPrefix)) {
128
9
      int64_t D = static_cast<int64_t>(S_V.second);
129
9
      unsigned IID = 0;
130
15
      for (const auto &HintType : HintTypes) {
131
15
        if (Name.startswith(HintType.first)) {
132
9
          Name = Name.drop_front(HintType.first.size());
133
9
          IID = HintType.second;
134
9
          break;
135
9
        }
136
15
      }
137
9
      if (IID == 0)
138
0
        return false;
139
9
      uint8_t index = 0;
140
9
      Name.consumeInteger(10, index);
141
9
142
9
      if (index >= Prefetches.size())
143
8
        Prefetches.resize(index + 1);
144
9
      Prefetches[index] = {IID, D};
145
9
      max_index = std::max(max_index, static_cast<int16_t>(index));
146
9
    }
147
9
  }
148
6
  assert(max_index + 1 >= 0 &&
149
6
         "Possible overflow: max_index + 1 should be positive.");
150
6
  assert(static_cast<size_t>(max_index + 1) == Prefetches.size() &&
151
6
         "The number of prefetch hints received should match the number of "
152
6
         "PrefetchInfo objects returned");
153
6
  return !Prefetches.empty();
154
6
}
155
156
12.1k
bool X86InsertPrefetch::doInitialization(Module &M) {
157
12.1k
  if (Filename.empty())
158
12.1k
    return false;
159
3
160
3
  LLVMContext &Ctx = M.getContext();
161
3
  ErrorOr<std::unique_ptr<SampleProfileReader>> ReaderOrErr =
162
3
      SampleProfileReader::create(Filename, Ctx);
163
3
  if (std::error_code EC = ReaderOrErr.getError()) {
164
0
    std::string Msg = "Could not open profile: " + EC.message();
165
0
    Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg,
166
0
                                             DiagnosticSeverity::DS_Warning));
167
0
    return false;
168
0
  }
169
3
  Reader = std::move(ReaderOrErr.get());
170
3
  Reader->read();
171
3
  return true;
172
3
}
173
174
12.1k
void X86InsertPrefetch::getAnalysisUsage(AnalysisUsage &AU) const {
175
12.1k
  AU.setPreservesAll();
176
12.1k
  AU.addRequired<MachineModuleInfo>();
177
12.1k
}
178
179
137k
bool X86InsertPrefetch::runOnMachineFunction(MachineFunction &MF) {
180
137k
  if (!Reader)
181
137k
    return false;
182
7
  const FunctionSamples *Samples = Reader->getSamplesFor(MF.getFunction());
183
7
  if (!Samples)
184
1
    return false;
185
6
186
6
  bool Changed = false;
187
6
188
6
  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
189
6
  SmallVector<PrefetchInfo, 4> Prefetches;
190
6
  for (auto &MBB : MF) {
191
25
    for (auto MI = MBB.instr_begin(); MI != MBB.instr_end();) {
192
21
      auto Current = MI;
193
21
      ++MI;
194
21
195
21
      int Offset = X86II::getMemoryOperandNo(Current->getDesc().TSFlags);
196
21
      if (Offset < 0)
197
13
        continue;
198
8
      unsigned Bias = X86II::getOperandBias(Current->getDesc());
199
8
      int MemOpOffset = Offset + Bias;
200
8
      // FIXME(mtrofin): ORE message when the recommendation cannot be taken.
201
8
      if (!IsMemOpCompatibleWithPrefetch(*Current, MemOpOffset))
202
1
        continue;
203
7
      Prefetches.clear();
204
7
      if (!findPrefetchInfo(Samples, *Current, Prefetches))
205
1
        continue;
206
6
      assert(!Prefetches.empty() &&
207
6
             "The Prefetches vector should contain at least a value if "
208
6
             "findPrefetchInfo returned true.");
209
9
      for (auto &PrefInfo : Prefetches) {
210
9
        unsigned PFetchInstrID = PrefInfo.InstructionID;
211
9
        int64_t Delta = PrefInfo.Delta;
212
9
        const MCInstrDesc &Desc = TII->get(PFetchInstrID);
213
9
        MachineInstr *PFetch =
214
9
            MF.CreateMachineInstr(Desc, Current->getDebugLoc(), true);
215
9
        MachineInstrBuilder MIB(MF, PFetch);
216
9
217
9
        assert(X86::AddrBaseReg == 0 && X86::AddrScaleAmt == 1 &&
218
9
               X86::AddrIndexReg == 2 && X86::AddrDisp == 3 &&
219
9
               X86::AddrSegmentReg == 4 &&
220
9
               "Unexpected change in X86 operand offset order.");
221
9
222
9
        // This assumes X86::AddBaseReg = 0, {...}ScaleAmt = 1, etc.
223
9
        // FIXME(mtrofin): consider adding a:
224
9
        //     MachineInstrBuilder::set(unsigned offset, op).
225
9
        MIB.addReg(Current->getOperand(MemOpOffset + X86::AddrBaseReg).getReg())
226
9
            .addImm(
227
9
                Current->getOperand(MemOpOffset + X86::AddrScaleAmt).getImm())
228
9
            .addReg(
229
9
                Current->getOperand(MemOpOffset + X86::AddrIndexReg).getReg())
230
9
            .addImm(Current->getOperand(MemOpOffset + X86::AddrDisp).getImm() +
231
9
                    Delta)
232
9
            .addReg(Current->getOperand(MemOpOffset + X86::AddrSegmentReg)
233
9
                        .getReg());
234
9
235
9
        if (!Current->memoperands_empty()) {
236
9
          MachineMemOperand *CurrentOp = *(Current->memoperands_begin());
237
9
          MIB.addMemOperand(MF.getMachineMemOperand(
238
9
              CurrentOp, CurrentOp->getOffset() + Delta, CurrentOp->getSize()));
239
9
        }
240
9
241
9
        // Insert before Current. This is because Current may clobber some of
242
9
        // the registers used to describe the input memory operand.
243
9
        MBB.insert(Current, PFetch);
244
9
        Changed = true;
245
9
      }
246
6
    }
247
4
  }
248
6
  return Changed;
249
6
}
250
251
12.2k
FunctionPass *llvm::createX86InsertPrefetchPass() {
252
12.2k
  return new X86InsertPrefetch(PrefetchHintsFile);
253
12.2k
}