/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Target/AArch64/AArch64A53Fix835769.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===-- AArch64A53Fix835769.cpp -------------------------------------------===// |
2 | | // |
3 | | // The LLVM Compiler Infrastructure |
4 | | // |
5 | | // This file is distributed under the University of Illinois Open Source |
6 | | // License. See LICENSE.TXT for details. |
7 | | // |
8 | | //===----------------------------------------------------------------------===// |
9 | | // This pass changes code to work around Cortex-A53 erratum 835769. |
10 | | // It works around it by inserting a nop instruction in code sequences that |
11 | | // in some circumstances may trigger the erratum. |
12 | | // It inserts a nop instruction between a sequence of the following 2 classes |
13 | | // of instructions: |
14 | | // instr 1: mem-instr (including loads, stores and prefetches). |
15 | | // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers. |
16 | | //===----------------------------------------------------------------------===// |
17 | | |
18 | | #include "AArch64.h" |
19 | | #include "llvm/ADT/Statistic.h" |
20 | | #include "llvm/CodeGen/MachineFunction.h" |
21 | | #include "llvm/CodeGen/MachineFunctionPass.h" |
22 | | #include "llvm/CodeGen/MachineInstr.h" |
23 | | #include "llvm/CodeGen/MachineInstrBuilder.h" |
24 | | #include "llvm/CodeGen/MachineRegisterInfo.h" |
25 | | #include "llvm/Support/Debug.h" |
26 | | #include "llvm/Support/raw_ostream.h" |
27 | | #include "llvm/Target/TargetInstrInfo.h" |
28 | | |
29 | | using namespace llvm; |
30 | | |
31 | | #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769" |
32 | | |
33 | | STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769"); |
34 | | |
35 | | //===----------------------------------------------------------------------===// |
36 | | // Helper functions |
37 | | |
38 | | // Is the instruction a match for the instruction that comes first in the |
39 | | // sequence of instructions that can trigger the erratum? |
40 | 6 | static bool isFirstInstructionInSequence(MachineInstr *MI) { |
41 | 6 | // Must return true if this instruction is a load, a store or a prefetch. |
42 | 6 | switch (MI->getOpcode()) { |
43 | 0 | case AArch64::PRFMl: |
44 | 0 | case AArch64::PRFMroW: |
45 | 0 | case AArch64::PRFMroX: |
46 | 0 | case AArch64::PRFMui: |
47 | 0 | case AArch64::PRFUMi: |
48 | 0 | return true; |
49 | 6 | default: |
50 | 6 | return MI->mayLoadOrStore(); |
51 | 0 | } |
52 | 0 | } |
53 | | |
54 | | // Is the instruction a match for the instruction that comes second in the |
55 | | // sequence that can trigger the erratum? |
56 | 3 | static bool isSecondInstructionInSequence(MachineInstr *MI) { |
57 | 3 | // Must return true for non-SIMD integer multiply-accumulates, writing |
58 | 3 | // to a 64-bit register. |
59 | 3 | switch (MI->getOpcode()) { |
60 | 3 | // Erratum cannot be triggered when the destination register is 32 bits, |
61 | 3 | // therefore only include the following. |
62 | 3 | case AArch64::MSUBXrrr: |
63 | 3 | case AArch64::MADDXrrr: |
64 | 3 | case AArch64::SMADDLrrr: |
65 | 3 | case AArch64::SMSUBLrrr: |
66 | 3 | case AArch64::UMADDLrrr: |
67 | 3 | case AArch64::UMSUBLrrr: |
68 | 3 | // Erratum can only be triggered by multiply-adds, not by regular |
69 | 3 | // non-accumulating multiplies, i.e. when Ra=XZR='11111' |
70 | 3 | return MI->getOperand(3).getReg() != AArch64::XZR; |
71 | 0 | default: |
72 | 0 | return false; |
73 | 0 | } |
74 | 0 | } |
75 | | |
76 | | |
77 | | //===----------------------------------------------------------------------===// |
78 | | |
79 | | namespace { |
80 | | class AArch64A53Fix835769 : public MachineFunctionPass { |
81 | | const TargetInstrInfo *TII; |
82 | | |
83 | | public: |
84 | | static char ID; |
85 | 3 | explicit AArch64A53Fix835769() : MachineFunctionPass(ID) { |
86 | 3 | initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry()); |
87 | 3 | } |
88 | | |
89 | | bool runOnMachineFunction(MachineFunction &F) override; |
90 | | |
91 | 3 | MachineFunctionProperties getRequiredProperties() const override { |
92 | 3 | return MachineFunctionProperties().set( |
93 | 3 | MachineFunctionProperties::Property::NoVRegs); |
94 | 3 | } |
95 | | |
96 | 3 | StringRef getPassName() const override { |
97 | 3 | return "Workaround A53 erratum 835769 pass"; |
98 | 3 | } |
99 | | |
100 | 3 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
101 | 3 | AU.setPreservesCFG(); |
102 | 3 | MachineFunctionPass::getAnalysisUsage(AU); |
103 | 3 | } |
104 | | |
105 | | private: |
106 | | bool runOnBasicBlock(MachineBasicBlock &MBB); |
107 | | }; |
108 | | char AArch64A53Fix835769::ID = 0; |
109 | | |
110 | | } // end anonymous namespace |
111 | | |
112 | | INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass", |
113 | | "AArch64 fix for A53 erratum 835769", false, false) |
114 | | |
115 | | //===----------------------------------------------------------------------===// |
116 | | |
117 | | bool |
118 | 3 | AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) { |
119 | 3 | DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n"); |
120 | 3 | bool Changed = false; |
121 | 3 | TII = F.getSubtarget().getInstrInfo(); |
122 | 3 | |
123 | 3 | for (auto &MBB : F) { |
124 | 3 | Changed |= runOnBasicBlock(MBB); |
125 | 3 | } |
126 | 3 | return Changed; |
127 | 3 | } |
128 | | |
129 | | // Return the block that was fallen through to get to MBB, if any, |
130 | | // otherwise nullptr. |
131 | | static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB, |
132 | 3 | const TargetInstrInfo *TII) { |
133 | 3 | // Get the previous machine basic block in the function. |
134 | 3 | MachineFunction::iterator MBBI(MBB); |
135 | 3 | |
136 | 3 | // Can't go off top of function. |
137 | 3 | if (MBBI == MBB->getParent()->begin()) |
138 | 3 | return nullptr; |
139 | 0 |
|
140 | 0 | MachineBasicBlock *TBB = nullptr, *FBB = nullptr; |
141 | 0 | SmallVector<MachineOperand, 2> Cond; |
142 | 0 |
|
143 | 0 | MachineBasicBlock *PrevBB = &*std::prev(MBBI); |
144 | 0 | for (MachineBasicBlock *S : MBB->predecessors()) |
145 | 0 | if (0 S == PrevBB && 0 !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond)0 && !TBB0 && |
146 | 0 | !FBB) |
147 | 0 | return S; |
148 | 0 |
|
149 | 0 | return nullptr; |
150 | 0 | } |
151 | | |
152 | | // Iterate through fallen through blocks trying to find a previous non-pseudo if |
153 | | // there is one, otherwise return nullptr. Only look for instructions in |
154 | | // previous blocks, not the current block, since we only use this to look at |
155 | | // previous blocks. |
156 | | static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB, |
157 | 3 | const TargetInstrInfo *TII) { |
158 | 3 | MachineBasicBlock *FMBB = &MBB; |
159 | 3 | |
160 | 3 | // If there is no non-pseudo in the current block, loop back around and try |
161 | 3 | // the previous block (if there is one). |
162 | 3 | while ((FMBB = getBBFallenThrough(FMBB, TII))3 ) { |
163 | 0 | for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend())) |
164 | 0 | if (0 !I.isPseudo()0 ) |
165 | 0 | return &I; |
166 | 0 | } |
167 | 3 | |
168 | 3 | // There was no previous non-pseudo in the fallen through blocks |
169 | 3 | return nullptr; |
170 | 3 | } |
171 | | |
172 | | static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI, |
173 | 3 | const TargetInstrInfo *TII) { |
174 | 3 | // If we are the first instruction of the block, put the NOP at the end of |
175 | 3 | // the previous fallthrough block |
176 | 3 | if (MI == &MBB.front()3 ) { |
177 | 0 | MachineInstr *I = getLastNonPseudo(MBB, TII); |
178 | 0 | assert(I && "Expected instruction"); |
179 | 0 | DebugLoc DL = I->getDebugLoc(); |
180 | 0 | BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0); |
181 | 0 | } |
182 | 3 | else { |
183 | 3 | DebugLoc DL = MI->getDebugLoc(); |
184 | 3 | BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0); |
185 | 3 | } |
186 | 3 | |
187 | 3 | ++NumNopsAdded; |
188 | 3 | } |
189 | | |
190 | | bool |
191 | 3 | AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) { |
192 | 3 | bool Changed = false; |
193 | 3 | DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n"); |
194 | 3 | |
195 | 3 | // First, scan the basic block, looking for a sequence of 2 instructions |
196 | 3 | // that match the conditions under which the erratum may trigger. |
197 | 3 | |
198 | 3 | // List of terminating instructions in matching sequences |
199 | 3 | std::vector<MachineInstr*> Sequences; |
200 | 3 | unsigned Idx = 0; |
201 | 3 | MachineInstr *PrevInstr = nullptr; |
202 | 3 | |
203 | 3 | // Try and find the last non-pseudo instruction in any fallen through blocks, |
204 | 3 | // if there isn't one, then we use nullptr to represent that. |
205 | 3 | PrevInstr = getLastNonPseudo(MBB, TII); |
206 | 3 | |
207 | 9 | for (auto &MI : MBB) { |
208 | 9 | MachineInstr *CurrInstr = &MI; |
209 | 9 | DEBUG(dbgs() << " Examining: " << MI); |
210 | 9 | if (PrevInstr9 ) { |
211 | 6 | DEBUG(dbgs() << " PrevInstr: " << *PrevInstr |
212 | 6 | << " CurrInstr: " << *CurrInstr |
213 | 6 | << " isFirstInstructionInSequence(PrevInstr): " |
214 | 6 | << isFirstInstructionInSequence(PrevInstr) << "\n" |
215 | 6 | << " isSecondInstructionInSequence(CurrInstr): " |
216 | 6 | << isSecondInstructionInSequence(CurrInstr) << "\n"); |
217 | 6 | if (isFirstInstructionInSequence(PrevInstr) && |
218 | 6 | isSecondInstructionInSequence(CurrInstr)3 ) { |
219 | 3 | DEBUG(dbgs() << " ** pattern found at Idx " << Idx << "!\n"); |
220 | 3 | Sequences.push_back(CurrInstr); |
221 | 3 | } |
222 | 6 | } |
223 | 9 | if (!CurrInstr->isPseudo()) |
224 | 9 | PrevInstr = CurrInstr; |
225 | 9 | ++Idx; |
226 | 9 | } |
227 | 3 | |
228 | 3 | DEBUG(dbgs() << "Scan complete, " << Sequences.size() |
229 | 3 | << " occurrences of pattern found.\n"); |
230 | 3 | |
231 | 3 | // Then update the basic block, inserting nops between the detected sequences. |
232 | 3 | for (auto &MI : Sequences) { |
233 | 3 | Changed = true; |
234 | 3 | insertNopBeforeInstruction(MBB, MI, TII); |
235 | 3 | } |
236 | 3 | |
237 | 3 | return Changed; |
238 | 3 | } |
239 | | |
240 | | // Factory function used by AArch64TargetMachine to add the pass to |
241 | | // the passmanager. |
242 | 3 | FunctionPass *llvm::createAArch64A53Fix835769() { |
243 | 3 | return new AArch64A53Fix835769(); |
244 | 3 | } |