/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AArch64/AArch64A53Fix835769.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===-- AArch64A53Fix835769.cpp -------------------------------------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // This pass changes code to work around Cortex-A53 erratum 835769. |
9 | | // It works around it by inserting a nop instruction in code sequences that |
10 | | // in some circumstances may trigger the erratum. |
11 | | // It inserts a nop instruction between a sequence of the following 2 classes |
12 | | // of instructions: |
13 | | // instr 1: mem-instr (including loads, stores and prefetches). |
14 | | // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers. |
15 | | //===----------------------------------------------------------------------===// |
16 | | |
17 | | #include "AArch64.h" |
18 | | #include "llvm/ADT/Statistic.h" |
19 | | #include "llvm/CodeGen/MachineFunction.h" |
20 | | #include "llvm/CodeGen/MachineFunctionPass.h" |
21 | | #include "llvm/CodeGen/MachineInstr.h" |
22 | | #include "llvm/CodeGen/MachineInstrBuilder.h" |
23 | | #include "llvm/CodeGen/MachineRegisterInfo.h" |
24 | | #include "llvm/CodeGen/TargetInstrInfo.h" |
25 | | #include "llvm/Support/Debug.h" |
26 | | #include "llvm/Support/raw_ostream.h" |
27 | | |
28 | | using namespace llvm; |
29 | | |
30 | | #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769" |
31 | | |
32 | | STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769"); |
33 | | |
34 | | //===----------------------------------------------------------------------===// |
35 | | // Helper functions |
36 | | |
37 | | // Is the instruction a match for the instruction that comes first in the |
38 | | // sequence of instructions that can trigger the erratum? |
39 | 6 | static bool isFirstInstructionInSequence(MachineInstr *MI) { |
40 | 6 | // Must return true if this instruction is a load, a store or a prefetch. |
41 | 6 | switch (MI->getOpcode()) { |
42 | 6 | case AArch64::PRFMl: |
43 | 0 | case AArch64::PRFMroW: |
44 | 0 | case AArch64::PRFMroX: |
45 | 0 | case AArch64::PRFMui: |
46 | 0 | case AArch64::PRFUMi: |
47 | 0 | return true; |
48 | 6 | default: |
49 | 6 | return MI->mayLoadOrStore(); |
50 | 6 | } |
51 | 6 | } |
52 | | |
53 | | // Is the instruction a match for the instruction that comes second in the |
54 | | // sequence that can trigger the erratum? |
55 | 3 | static bool isSecondInstructionInSequence(MachineInstr *MI) { |
56 | 3 | // Must return true for non-SIMD integer multiply-accumulates, writing |
57 | 3 | // to a 64-bit register. |
58 | 3 | switch (MI->getOpcode()) { |
59 | 3 | // Erratum cannot be triggered when the destination register is 32 bits, |
60 | 3 | // therefore only include the following. |
61 | 3 | case AArch64::MSUBXrrr: |
62 | 3 | case AArch64::MADDXrrr: |
63 | 3 | case AArch64::SMADDLrrr: |
64 | 3 | case AArch64::SMSUBLrrr: |
65 | 3 | case AArch64::UMADDLrrr: |
66 | 3 | case AArch64::UMSUBLrrr: |
67 | 3 | // Erratum can only be triggered by multiply-adds, not by regular |
68 | 3 | // non-accumulating multiplies, i.e. when Ra=XZR='11111' |
69 | 3 | return MI->getOperand(3).getReg() != AArch64::XZR; |
70 | 3 | default: |
71 | 0 | return false; |
72 | 3 | } |
73 | 3 | } |
74 | | |
75 | | |
76 | | //===----------------------------------------------------------------------===// |
77 | | |
78 | | namespace { |
79 | | class AArch64A53Fix835769 : public MachineFunctionPass { |
80 | | const TargetInstrInfo *TII; |
81 | | |
82 | | public: |
83 | | static char ID; |
84 | 3 | explicit AArch64A53Fix835769() : MachineFunctionPass(ID) { |
85 | 3 | initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry()); |
86 | 3 | } |
87 | | |
88 | | bool runOnMachineFunction(MachineFunction &F) override; |
89 | | |
90 | 3 | MachineFunctionProperties getRequiredProperties() const override { |
91 | 3 | return MachineFunctionProperties().set( |
92 | 3 | MachineFunctionProperties::Property::NoVRegs); |
93 | 3 | } |
94 | | |
95 | 6 | StringRef getPassName() const override { |
96 | 6 | return "Workaround A53 erratum 835769 pass"; |
97 | 6 | } |
98 | | |
99 | 3 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
100 | 3 | AU.setPreservesCFG(); |
101 | 3 | MachineFunctionPass::getAnalysisUsage(AU); |
102 | 3 | } |
103 | | |
104 | | private: |
105 | | bool runOnBasicBlock(MachineBasicBlock &MBB); |
106 | | }; |
107 | | char AArch64A53Fix835769::ID = 0; |
108 | | |
109 | | } // end anonymous namespace |
110 | | |
111 | | INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass", |
112 | | "AArch64 fix for A53 erratum 835769", false, false) |
113 | | |
114 | | //===----------------------------------------------------------------------===// |
115 | | |
116 | | bool |
117 | 3 | AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) { |
118 | 3 | LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n"); |
119 | 3 | bool Changed = false; |
120 | 3 | TII = F.getSubtarget().getInstrInfo(); |
121 | 3 | |
122 | 3 | for (auto &MBB : F) { |
123 | 3 | Changed |= runOnBasicBlock(MBB); |
124 | 3 | } |
125 | 3 | return Changed; |
126 | 3 | } |
127 | | |
128 | | // Return the block that was fallen through to get to MBB, if any, |
129 | | // otherwise nullptr. |
130 | | static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB, |
131 | 3 | const TargetInstrInfo *TII) { |
132 | 3 | // Get the previous machine basic block in the function. |
133 | 3 | MachineFunction::iterator MBBI(MBB); |
134 | 3 | |
135 | 3 | // Can't go off top of function. |
136 | 3 | if (MBBI == MBB->getParent()->begin()) |
137 | 3 | return nullptr; |
138 | 0 | |
139 | 0 | MachineBasicBlock *TBB = nullptr, *FBB = nullptr; |
140 | 0 | SmallVector<MachineOperand, 2> Cond; |
141 | 0 |
|
142 | 0 | MachineBasicBlock *PrevBB = &*std::prev(MBBI); |
143 | 0 | for (MachineBasicBlock *S : MBB->predecessors()) |
144 | 0 | if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB && |
145 | 0 | !FBB) |
146 | 0 | return S; |
147 | 0 |
|
148 | 0 | return nullptr; |
149 | 0 | } |
150 | | |
151 | | // Iterate through fallen through blocks trying to find a previous non-pseudo if |
152 | | // there is one, otherwise return nullptr. Only look for instructions in |
153 | | // previous blocks, not the current block, since we only use this to look at |
154 | | // previous blocks. |
155 | | static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB, |
156 | 3 | const TargetInstrInfo *TII) { |
157 | 3 | MachineBasicBlock *FMBB = &MBB; |
158 | 3 | |
159 | 3 | // If there is no non-pseudo in the current block, loop back around and try |
160 | 3 | // the previous block (if there is one). |
161 | 3 | while ((FMBB = getBBFallenThrough(FMBB, TII))) { |
162 | 0 | for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend())) |
163 | 0 | if (!I.isPseudo()) |
164 | 0 | return &I; |
165 | 0 | } |
166 | 3 | |
167 | 3 | // There was no previous non-pseudo in the fallen through blocks |
168 | 3 | return nullptr; |
169 | 3 | } |
170 | | |
171 | | static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI, |
172 | 3 | const TargetInstrInfo *TII) { |
173 | 3 | // If we are the first instruction of the block, put the NOP at the end of |
174 | 3 | // the previous fallthrough block |
175 | 3 | if (MI == &MBB.front()) { |
176 | 0 | MachineInstr *I = getLastNonPseudo(MBB, TII); |
177 | 0 | assert(I && "Expected instruction"); |
178 | 0 | DebugLoc DL = I->getDebugLoc(); |
179 | 0 | BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0); |
180 | 0 | } |
181 | 3 | else { |
182 | 3 | DebugLoc DL = MI->getDebugLoc(); |
183 | 3 | BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0); |
184 | 3 | } |
185 | 3 | |
186 | 3 | ++NumNopsAdded; |
187 | 3 | } |
188 | | |
189 | | bool |
190 | 3 | AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) { |
191 | 3 | bool Changed = false; |
192 | 3 | LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB |
193 | 3 | << " - scanning instructions...\n"); |
194 | 3 | |
195 | 3 | // First, scan the basic block, looking for a sequence of 2 instructions |
196 | 3 | // that match the conditions under which the erratum may trigger. |
197 | 3 | |
198 | 3 | // List of terminating instructions in matching sequences |
199 | 3 | std::vector<MachineInstr*> Sequences; |
200 | 3 | unsigned Idx = 0; |
201 | 3 | MachineInstr *PrevInstr = nullptr; |
202 | 3 | |
203 | 3 | // Try and find the last non-pseudo instruction in any fallen through blocks, |
204 | 3 | // if there isn't one, then we use nullptr to represent that. |
205 | 3 | PrevInstr = getLastNonPseudo(MBB, TII); |
206 | 3 | |
207 | 9 | for (auto &MI : MBB) { |
208 | 9 | MachineInstr *CurrInstr = &MI; |
209 | 9 | LLVM_DEBUG(dbgs() << " Examining: " << MI); |
210 | 9 | if (PrevInstr) { |
211 | 6 | LLVM_DEBUG(dbgs() << " PrevInstr: " << *PrevInstr |
212 | 6 | << " CurrInstr: " << *CurrInstr |
213 | 6 | << " isFirstInstructionInSequence(PrevInstr): " |
214 | 6 | << isFirstInstructionInSequence(PrevInstr) << "\n" |
215 | 6 | << " isSecondInstructionInSequence(CurrInstr): " |
216 | 6 | << isSecondInstructionInSequence(CurrInstr) << "\n"); |
217 | 6 | if (isFirstInstructionInSequence(PrevInstr) && |
218 | 6 | isSecondInstructionInSequence(CurrInstr)3 ) { |
219 | 3 | LLVM_DEBUG(dbgs() << " ** pattern found at Idx " << Idx << "!\n"); |
220 | 3 | Sequences.push_back(CurrInstr); |
221 | 3 | } |
222 | 6 | } |
223 | 9 | if (!CurrInstr->isPseudo()) |
224 | 9 | PrevInstr = CurrInstr; |
225 | 9 | ++Idx; |
226 | 9 | } |
227 | 3 | |
228 | 3 | LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size() |
229 | 3 | << " occurrences of pattern found.\n"); |
230 | 3 | |
231 | 3 | // Then update the basic block, inserting nops between the detected sequences. |
232 | 3 | for (auto &MI : Sequences) { |
233 | 3 | Changed = true; |
234 | 3 | insertNopBeforeInstruction(MBB, MI, TII); |
235 | 3 | } |
236 | 3 | |
237 | 3 | return Changed; |
238 | 3 | } |
239 | | |
240 | | // Factory function used by AArch64TargetMachine to add the pass to |
241 | | // the passmanager. |
242 | 3 | FunctionPass *llvm::createAArch64A53Fix835769() { |
243 | 3 | return new AArch64A53Fix835769(); |
244 | 3 | } |