Coverage Report

Created: 2019-01-18 03:29

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/tools/lld/ELF/AArch64ErrataFix.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- AArch64ErrataFix.cpp -----------------------------------------------===//
2
//
3
//                             The LLVM Linker
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
// This file implements Section Patching for the purpose of working around
10
// errata in CPUs. The general principle is that an erratum sequence of one or
11
// more instructions is detected in the instruction stream, one of the
12
// instructions in the sequence is replaced with a branch to a patch sequence
13
// of replacement instructions. At the end of the replacement sequence the
14
// patch branches back to the instruction stream.
15
16
// This technique is only suitable for fixing an erratum when:
17
// - There is a set of necessary conditions required to trigger the erratum that
18
// can be detected at static link time.
19
// - There is a set of replacement instructions that can be used to remove at
20
// least one of the necessary conditions that trigger the erratum.
21
// - We can overwrite an instruction in the erratum sequence with a branch to
22
// the replacement sequence.
23
// - We can place the replacement sequence within range of the branch.
24
25
// FIXME:
26
// - The implementation here only supports one patch, the AArch64 Cortex-53
27
// errata 843419 that affects r0p0, r0p1, r0p2 and r0p4 versions of the core.
28
// To keep the initial version simple there is no support for multiple
29
// architectures or selection of different patches.
30
//===----------------------------------------------------------------------===//
31
32
#include "AArch64ErrataFix.h"
33
#include "Config.h"
34
#include "LinkerScript.h"
35
#include "OutputSections.h"
36
#include "Relocations.h"
37
#include "Symbols.h"
38
#include "SyntheticSections.h"
39
#include "Target.h"
40
#include "lld/Common/Memory.h"
41
#include "lld/Common/Strings.h"
42
#include "llvm/Support/Endian.h"
43
#include "llvm/Support/raw_ostream.h"
44
#include <algorithm>
45
46
using namespace llvm;
47
using namespace llvm::ELF;
48
using namespace llvm::object;
49
using namespace llvm::support;
50
using namespace llvm::support::endian;
51
52
using namespace lld;
53
using namespace lld::elf;
54
55
// Helper functions to identify instructions and conditions needed to trigger
56
// the Cortex-A53-843419 erratum.
57
58
// ADRP
59
// | 1 | immlo (2) | 1 | 0 0 0 0 | immhi (19) | Rd (5) |
60
209
static bool isADRP(uint32_t Instr) {
61
209
  return (Instr & 0x9f000000) == 0x90000000;
62
209
}
63
64
// Load and store bit patterns from ARMv8-A ARM ARM.
65
// Instructions appear in order of appearance starting from table in
66
// C4.1.3 Loads and Stores.
67
68
// All loads and stores have 1 (at bit postion 27), (0 at bit position 25).
69
// | op0 x op1 (2) | 1 op2 0 op3 (2) | x | op4 (5) | xxxx | op5 (2) | x (10) |
70
127
static bool isLoadStoreClass(uint32_t Instr) {
71
127
  return (Instr & 0x0a000000) == 0x08000000;
72
127
}
73
74
// LDN/STN multiple no offset
75
// | 0 Q 00 | 1100 | 0 L 00 | 0000 | opcode (4) | size (2) | Rn (5) | Rt (5) |
76
// LDN/STN multiple post-indexed
77
// | 0 Q 00 | 1100 | 1 L 0 | Rm (5)| opcode (4) | size (2) | Rn (5) | Rt (5) |
78
// L == 0 for stores.
79
80
// Utility routine to decode opcode field of LDN/STN multiple structure
81
// instructions to find the ST1 instructions.
82
// opcode == 0010 ST1 4 registers.
83
// opcode == 0110 ST1 3 registers.
84
// opcode == 0111 ST1 1 register.
85
// opcode == 1010 ST1 2 registers.
86
14
static bool isST1MultipleOpcode(uint32_t Instr) {
87
14
  return (Instr & 0x0000f000) == 0x00002000 ||
88
14
         (Instr & 0x0000f000) == 0x00006000 ||
89
14
         (Instr & 0x0000f000) == 0x00007000 ||
90
14
         
(Instr & 0x0000f000) == 0x0000a0006
;
91
14
}
92
93
22
static bool isST1Multiple(uint32_t Instr) {
94
22
  return (Instr & 0xbfff0000) == 0x0c000000 && 
isST1MultipleOpcode(Instr)4
;
95
22
}
96
97
// Writes to Rn (writeback).
98
113
static bool isST1MultiplePost(uint32_t Instr) {
99
113
  return (Instr & 0xbfe00000) == 0x0c800000 && 
isST1MultipleOpcode(Instr)10
;
100
113
}
101
102
// LDN/STN single no offset
103
// | 0 Q 00 | 1101 | 0 L R 0 | 0000 | opc (3) S | size (2) | Rn (5) | Rt (5)|
104
// LDN/STN single post-indexed
105
// | 0 Q 00 | 1101 | 1 L R | Rm (5) | opc (3) S | size (2) | Rn (5) | Rt (5)|
106
// L == 0 for stores
107
108
// Utility routine to decode opcode field of LDN/STN single structure
109
// instructions to find the ST1 instructions.
110
// R == 0 for ST1 and ST3, R == 1 for ST2 and ST4.
111
// opcode == 000 ST1 8-bit.
112
// opcode == 010 ST1 16-bit.
113
// opcode == 100 ST1 32 or 64-bit (Size determines which).
114
8
static bool isST1SingleOpcode(uint32_t Instr) {
115
8
  return (Instr & 0x0040e000) == 0x00000000 ||
116
8
         
(Instr & 0x0040e000) == 0x000040004
||
117
8
         
(Instr & 0x0040e000) == 0x000080004
;
118
8
}
119
120
16
static bool isST1Single(uint32_t Instr) {
121
16
  return (Instr & 0xbfff0000) == 0x0d000000 && 
isST1SingleOpcode(Instr)4
;
122
16
}
123
124
// Writes to Rn (writeback).
125
107
static bool isST1SinglePost(uint32_t Instr) {
126
107
  return (Instr & 0xbfe00000) == 0x0d800000 && 
isST1SingleOpcode(Instr)4
;
127
107
}
128
129
22
static bool isST1(uint32_t Instr) {
130
22
  return isST1Multiple(Instr) || 
isST1MultiplePost(Instr)20
||
131
22
         
isST1Single(Instr)16
||
isST1SinglePost(Instr)12
;
132
22
}
133
134
// Load/store exclusive
135
// | size (2) 00 | 1000 | o2 L o1 | Rs (5) | o0 | Rt2 (5) | Rn (5) | Rt (5) |
136
// L == 0 for Stores.
137
127
static bool isLoadStoreExclusive(uint32_t Instr) {
138
127
  return (Instr & 0x3f000000) == 0x08000000;
139
127
}
140
141
117
static bool isLoadExclusive(uint32_t Instr) {
142
117
  return (Instr & 0x3f400000) == 0x08400000;
143
117
}
144
145
// Load register literal
146
// | opc (2) 01 | 1 V 00 | imm19 | Rt (5) |
147
238
static bool isLoadLiteral(uint32_t Instr) {
148
238
  return (Instr & 0x3b000000) == 0x18000000;
149
238
}
150
151
// Load/store no-allocate pair
152
// (offset)
153
// | opc (2) 10 | 1 V 00 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |
154
// L == 0 for stores.
155
// Never writes to register
156
34
static bool isSTNP(uint32_t Instr) {
157
34
  return (Instr & 0x3bc00000) == 0x28000000;
158
34
}
159
160
// Load/store register pair
161
// (post-indexed)
162
// | opc (2) 10 | 1 V 00 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |
163
// L == 0 for stores, V == 0 for Scalar, V == 1 for Simd/FP
164
// Writes to Rn.
165
157
static bool isSTPPost(uint32_t Instr) {
166
157
  return (Instr & 0x3bc00000) == 0x28800000;
167
157
}
168
169
// (offset)
170
// | opc (2) 10 | 1 V 01 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |
171
50
static bool isSTPOffset(uint32_t Instr) {
172
50
  return (Instr & 0x3bc00000) == 0x29000000;
173
50
}
174
175
// (pre-index)
176
// | opc (2) 10 | 1 V 01 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |
177
// Writes to Rn.
178
147
static bool isSTPPre(uint32_t Instr) {
179
147
  return (Instr & 0x3bc00000) == 0x29800000;
180
147
}
181
182
56
static bool isSTP(uint32_t Instr) {
183
56
  return isSTPPost(Instr) || 
isSTPOffset(Instr)50
||
isSTPPre(Instr)40
;
184
56
}
185
186
// Load/store register (unscaled immediate)
187
// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 00 | Rn (5) | Rt (5) |
188
// V == 0 for Scalar, V == 1 for Simd/FP.
189
234
static bool isLoadStoreUnscaled(uint32_t Instr) {
190
234
  return (Instr & 0x3b000c00) == 0x38000000;
191
234
}
192
193
// Load/store register (immediate post-indexed)
194
// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 01 | Rn (5) | Rt (5) |
195
337
static bool isLoadStoreImmediatePost(uint32_t Instr) {
196
337
  return (Instr & 0x3b200c00) == 0x38000400;
197
337
}
198
199
// Load/store register (unprivileged)
200
// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 10 | Rn (5) | Rt (5) |
201
218
static bool isLoadStoreUnpriv(uint32_t Instr) {
202
218
  return (Instr & 0x3b200c00) == 0x38000800;
203
218
}
204
205
// Load/store register (immediate pre-indexed)
206
// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 11 | Rn (5) | Rt (5) |
207
329
static bool isLoadStoreImmediatePre(uint32_t Instr) {
208
329
  return (Instr & 0x3b200c00) == 0x38000c00;
209
329
}
210
211
// Load/store register (register offset)
212
// | size (2) 11 | 1 V 00 | opc (2) 1 | Rm (5) | option (3) S | 10 | Rn | Rt |
213
206
static bool isLoadStoreRegisterOff(uint32_t Instr) {
214
206
  return (Instr & 0x3b200c00) == 0x38200800;
215
206
}
216
217
// Load/store register (unsigned immediate)
218
// | size (2) 11 | 1 V 01 | opc (2) | imm12 | Rn (5) | Rt (5) |
219
307
static bool isLoadStoreRegisterUnsigned(uint32_t Instr) {
220
307
  return (Instr & 0x3b000000) == 0x39000000;
221
307
}
222
223
// Rt is always in bit position 0 - 4.
224
177
static uint32_t getRt(uint32_t Instr) { return (Instr & 0x1f); }
225
226
// Rn is always in bit position 5 - 9.
227
110
static uint32_t getRn(uint32_t Instr) { return (Instr >> 5) & 0x1f; }
228
229
// C4.1.2 Branches, Exception Generating and System instructions
230
// | op0 (3) 1 | 01 op1 (4) | x (22) |
231
// op0 == 010 101 op1 == 0xxx Conditional Branch.
232
// op0 == 110 101 op1 == 1xxx Unconditional Branch Register.
233
// op0 == x00 101 op1 == xxxx Unconditional Branch immediate.
234
// op0 == x01 101 op1 == 0xxx Compare and branch immediate.
235
// op0 == x01 101 op1 == 1xxx Test and branch immediate.
236
60
static bool isBranch(uint32_t Instr) {
237
60
  return ((Instr & 0xfe000000) == 0xd6000000) || // Cond branch.
238
60
         
((Instr & 0xfe000000) == 0x54000000)53
|| // Uncond branch reg.
239
60
         
((Instr & 0x7c000000) == 0x14000000)52
|| // Uncond branch imm.
240
60
         
((Instr & 0x7c000000) == 0x34000000)50
; // Compare and test branch.
241
60
}
242
243
234
static bool isV8SingleRegisterNonStructureLoadStore(uint32_t Instr) {
244
234
  return isLoadStoreUnscaled(Instr) || 
isLoadStoreImmediatePost(Instr)226
||
245
234
         
isLoadStoreUnpriv(Instr)218
||
isLoadStoreImmediatePre(Instr)214
||
246
234
         
isLoadStoreRegisterOff(Instr)206
||
isLoadStoreRegisterUnsigned(Instr)202
;
247
234
}
248
249
// Note that this function refers to v8.0 only and does not include the
250
// additional load and store instructions added for in later revisions of
251
// the architecture such as the Atomic memory operations introduced
252
// in v8.1.
253
117
static bool isV8NonStructureLoad(uint32_t Instr) {
254
117
  if (isLoadExclusive(Instr))
255
2
    return true;
256
115
  if (isLoadLiteral(Instr))
257
2
    return true;
258
113
  else if (isV8SingleRegisterNonStructureLoadStore(Instr)) {
259
65
    // For Load and Store single register, Loads are derived from a
260
65
    // combination of the Size, V and Opc fields.
261
65
    uint32_t Size = (Instr >> 30) & 0xff;
262
65
    uint32_t V = (Instr >> 26) & 0x1;
263
65
    uint32_t Opc = (Instr >> 22) & 0x3;
264
65
    // For the load and store instructions that we are decoding.
265
65
    // Opc == 0 are all stores.
266
65
    // Opc == 1 with a couple of exceptions are loads. The exceptions are:
267
65
    // Size == 00 (0), V == 1, Opc == 10 (2) which is a store and
268
65
    // Size == 11 (3), V == 0, Opc == 10 (2) which is a prefetch.
269
65
    return Opc != 0 && 
!(46
Size == 046
&&
V == 12
&&
Opc == 20
) &&
270
65
           
!(46
Size == 346
&&
V == 034
&&
Opc == 234
);
271
65
  }
272
48
  return false;
273
48
}
274
275
// The following decode instructions are only complete up to the instructions
276
// needed for errata 843419.
277
278
// Instruction with writeback updates the index register after the load/store.
279
115
static bool hasWriteback(uint32_t Instr) {
280
115
  return isLoadStoreImmediatePre(Instr) || 
isLoadStoreImmediatePost(Instr)111
||
281
115
         
isSTPPre(Instr)107
||
isSTPPost(Instr)101
||
isST1SinglePost(Instr)95
||
282
115
         
isST1MultiplePost(Instr)93
;
283
115
}
284
285
// For the load and store class of instructions, a load can write to the
286
// destination register, a load and a store can write to the base register when
287
// the instruction has writeback.
288
117
static bool doesLoadStoreWriteToReg(uint32_t Instr, uint32_t Reg) {
289
117
  return (isV8NonStructureLoad(Instr) && 
getRt(Instr) == Reg50
) ||
290
117
         
(115
hasWriteback(Instr)115
&&
getRn(Instr) == Reg26
);
291
117
}
292
293
// Scanner for Cortex-A53 errata 843419
294
// Full details are available in the Cortex A53 MPCore revision 0 Software
295
// Developers Errata Notice (ARM-EPM-048406).
296
//
297
// The instruction sequence that triggers the erratum is common in compiled
298
// AArch64 code, however it is sensitive to the offset of the sequence within
299
// a 4k page. This means that by scanning and fixing the patch after we have
300
// assigned addresses we only need to disassemble and fix instances of the
301
// sequence in the range of affected offsets.
302
//
303
// In summary the erratum conditions are a series of 4 instructions:
304
// 1.) An ADRP instruction that writes to register Rn with low 12 bits of
305
//     address of instruction either 0xff8 or 0xffc.
306
// 2.) A load or store instruction that can be:
307
// - A single register load or store, of either integer or vector registers.
308
// - An STP or STNP, of either integer or vector registers.
309
// - An Advanced SIMD ST1 store instruction.
310
// - Must not write to Rn, but may optionally read from it.
311
// 3.) An optional instruction that is not a branch and does not write to Rn.
312
// 4.) A load or store from the  Load/store register (unsigned immediate) class
313
//     that uses Rn as the base address register.
314
//
315
// Note that we do not attempt to scan for Sequence 2 as described in the
316
// Software Developers Errata Notice as this has been assessed to be extremely
317
// unlikely to occur in compiled code. This matches gold and ld.bfd behavior.
318
319
// Return true if the Instruction sequence Adrp, Instr2, and Instr4 match
320
// the erratum sequence. The Adrp, Instr2 and Instr4 correspond to 1.), 2.),
321
// and 4.) in the Scanner for Cortex-A53 errata comment above.
322
static bool is843419ErratumSequence(uint32_t Instr1, uint32_t Instr2,
323
209
                                    uint32_t Instr4) {
324
209
  if (!isADRP(Instr1))
325
82
    return false;
326
127
327
127
  uint32_t Rn = getRt(Instr1);
328
127
  return isLoadStoreClass(Instr2) &&
329
127
         (isLoadStoreExclusive(Instr2) || 
isLoadLiteral(Instr2)123
||
330
127
          
isV8SingleRegisterNonStructureLoadStore(Instr2)121
||
isSTP(Instr2)56
||
331
127
          
isSTNP(Instr2)34
||
isST1(Instr2)22
) &&
332
127
         
!doesLoadStoreWriteToReg(Instr2, Rn)117
&&
333
127
         
isLoadStoreRegisterUnsigned(Instr4)105
&&
getRn(Instr4) == Rn84
;
334
127
}
335
336
// Scan the instruction sequence starting at Offset Off from the base of
337
// InputSection IS. We update Off in this function rather than in the caller as
338
// we can skip ahead much further into the section when we know how many
339
// instructions we've scanned.
340
// Return the offset of the load or store instruction in IS that we want to
341
// patch or 0 if no patch required.
342
static uint64_t scanCortexA53Errata843419(InputSection *IS, uint64_t &Off,
343
185
                                          uint64_t Limit) {
344
185
  uint64_t ISAddr = IS->getVA(0);
345
185
346
185
  // Advance Off so that (ISAddr + Off) modulo 0x1000 is at least 0xff8.
347
185
  uint64_t InitialPageOff = (ISAddr + Off) & 0xfff;
348
185
  if (InitialPageOff < 0xff8)
349
24
    Off += 0xff8 - InitialPageOff;
350
185
351
185
  bool OptionalAllowed = Limit - Off > 12;
352
185
  if (Off >= Limit || 
Limit - Off < 12168
) {
353
25
    // Need at least 3 4-byte sized instructions to trigger erratum.
354
25
    Off = Limit;
355
25
    return 0;
356
25
  }
357
160
358
160
  uint64_t PatchOff = 0;
359
160
  const uint8_t *Buf = IS->data().begin();
360
160
  const ulittle32_t *InstBuf = reinterpret_cast<const ulittle32_t *>(Buf + Off);
361
160
  uint32_t Instr1 = *InstBuf++;
362
160
  uint32_t Instr2 = *InstBuf++;
363
160
  uint32_t Instr3 = *InstBuf++;
364
160
  if (is843419ErratumSequence(Instr1, Instr2, Instr3)) {
365
65
    PatchOff = Off + 8;
366
95
  } else if (OptionalAllowed && 
!isBranch(Instr3)60
) {
367
49
    uint32_t Instr4 = *InstBuf++;
368
49
    if (is843419ErratumSequence(Instr1, Instr2, Instr4))
369
17
      PatchOff = Off + 12;
370
49
  }
371
160
  if (((ISAddr + Off) & 0xfff) == 0xff8)
372
60
    Off += 4;
373
100
  else
374
100
    Off += 0xffc;
375
160
  return PatchOff;
376
160
}
377
378
class lld::elf::Patch843419Section : public SyntheticSection {
379
public:
380
  Patch843419Section(InputSection *P, uint64_t Off);
381
382
  void writeTo(uint8_t *Buf) override;
383
384
240
  size_t getSize() const override { return 8; }
385
386
  uint64_t getLDSTAddr() const;
387
388
  // The Section we are patching.
389
  const InputSection *Patchee;
390
  // The offset of the instruction in the Patchee section we are patching.
391
  uint64_t PatcheeOffset;
392
  // A label for the start of the Patch that we can use as a relocation target.
393
  Symbol *PatchSym;
394
};
395
396
lld::elf::Patch843419Section::Patch843419Section(InputSection *P, uint64_t Off)
397
    : SyntheticSection(SHF_ALLOC | SHF_EXECINSTR, SHT_PROGBITS, 4,
398
                       ".text.patch"),
399
36
      Patchee(P), PatcheeOffset(Off) {
400
36
  this->Parent = P->getParent();
401
36
  PatchSym = addSyntheticLocal(
402
36
      Saver.save("__CortexA53843419_" + utohexstr(getLDSTAddr())), STT_FUNC, 0,
403
36
      getSize(), *this);
404
36
  addSyntheticLocal(Saver.save("$x"), STT_NOTYPE, 0, 0, *this);
405
36
}
406
407
75
uint64_t lld::elf::Patch843419Section::getLDSTAddr() const {
408
75
  return Patchee->getVA(PatcheeOffset);
409
75
}
410
411
36
void lld::elf::Patch843419Section::writeTo(uint8_t *Buf) {
412
36
  // Copy the instruction that we will be replacing with a branch in the
413
36
  // Patchee Section.
414
36
  write32le(Buf, read32le(Patchee->data().begin() + PatcheeOffset));
415
36
416
36
  // Apply any relocation transferred from the original PatcheeSection.
417
36
  // For a SyntheticSection Buf already has OutSecOff added, but relocateAlloc
418
36
  // also adds OutSecOff so we need to subtract to avoid double counting.
419
36
  this->relocateAlloc(Buf - OutSecOff, Buf - OutSecOff + getSize());
420
36
421
36
  // Return address is the next instruction after the one we have just copied.
422
36
  uint64_t S = getLDSTAddr() + 4;
423
36
  uint64_t P = PatchSym->getVA() + 4;
424
36
  Target->relocateOne(Buf + 4, R_AARCH64_JUMP26, S - P);
425
36
}
426
427
8
void AArch64Err843419Patcher::init() {
428
8
  // The AArch64 ABI permits data in executable sections. We must avoid scanning
429
8
  // this data as if it were instructions to avoid false matches. We use the
430
8
  // mapping symbols in the InputObjects to identify this data, caching the
431
8
  // results in SectionMap so we don't have to recalculate it each pass.
432
8
433
8
  // The ABI Section 4.5.4 Mapping symbols; defines local symbols that describe
434
8
  // half open intervals [Symbol Value, Next Symbol Value) of code and data
435
8
  // within sections. If there is no next symbol then the half open interval is
436
8
  // [Symbol Value, End of section). The type, code or data, is determined by
437
8
  // the mapping symbol name, $x for code, $d for data.
438
218
  auto IsCodeMapSymbol = [](const Symbol *B) {
439
218
    return B->getName() == "$x" || B->getName().startswith("$x.");
440
218
  };
441
196
  auto IsDataMapSymbol = [](const Symbol *B) {
442
196
    return B->getName() == "$d" || B->getName().startswith("$d.");
443
196
  };
444
8
445
8
  // Collect mapping symbols for every executable InputSection.
446
8
  for (InputFile *File : ObjectFiles) {
447
8
    auto *F = cast<ObjFile<ELF64LE>>(File);
448
147
    for (Symbol *B : F->getLocalSymbols()) {
449
147
      auto *Def = dyn_cast<Defined>(B);
450
147
      if (!Def)
451
0
        continue;
452
147
      if (!IsCodeMapSymbol(Def) && 
!IsDataMapSymbol(Def)79
)
453
9
        continue;
454
138
      if (auto *Sec = dyn_cast_or_null<InputSection>(Def->Section))
455
137
        if (Sec->Flags & SHF_EXECINSTR)
456
131
          SectionMap[Sec].push_back(Def);
457
138
    }
458
8
  }
459
8
  // For each InputSection make sure the mapping symbols are in sorted in
460
8
  // ascending order and free from consecutive runs of mapping symbols with
461
8
  // the same type. For example we must remove the redundant $d.1 from $x.0
462
8
  // $d.0 $d.1 $x.1.
463
68
  for (auto &KV : SectionMap) {
464
68
    std::vector<const Defined *> &MapSyms = KV.second;
465
68
    if (MapSyms.size() <= 1)
466
17
      continue;
467
51
    std::stable_sort(
468
51
        MapSyms.begin(), MapSyms.end(),
469
73
        [](const Defined *A, const Defined *B) { return A->Value < B->Value; });
470
51
    MapSyms.erase(
471
51
        std::unique(MapSyms.begin(), MapSyms.end(),
472
63
                    [=](const Defined *A, const Defined *B) {
473
63
                      return (IsCodeMapSymbol(A) && 
IsCodeMapSymbol(B)8
) ||
474
63
                             
(62
IsDataMapSymbol(A)62
&&
IsDataMapSymbol(B)55
);
475
63
                    }),
476
51
        MapSyms.end());
477
51
  }
478
8
  Initialized = true;
479
8
}
480
481
// Insert the PatchSections we have created back into the
482
// InputSectionDescription. As inserting patches alters the addresses of
483
// InputSections that follow them, we try and place the patches after all the
484
// executable sections, although we may need to insert them earlier if the
485
// InputSectionDescription is larger than the maximum branch range.
486
void AArch64Err843419Patcher::insertPatches(
487
8
    InputSectionDescription &ISD, std::vector<Patch843419Section *> &Patches) {
488
8
  uint64_t ISLimit;
489
8
  uint64_t PrevISLimit = ISD.Sections.front()->OutSecOff;
490
8
  uint64_t PatchUpperBound = PrevISLimit + Target->getThunkSectionSpacing();
491
8
  uint64_t OutSecAddr = ISD.Sections.front()->getParent()->Addr;
492
8
493
8
  // Set the OutSecOff of patches to the place where we want to insert them.
494
8
  // We use a similar strategy to Thunk placement. Place patches roughly
495
8
  // every multiple of maximum branch range.
496
8
  auto PatchIt = Patches.begin();
497
8
  auto PatchEnd = Patches.end();
498
57
  for (const InputSection *IS : ISD.Sections) {
499
57
    ISLimit = IS->OutSecOff + IS->getSize();
500
57
    if (ISLimit > PatchUpperBound) {
501
6
      while (PatchIt != PatchEnd) {
502
4
        if ((*PatchIt)->getLDSTAddr() - OutSecAddr >= PrevISLimit)
503
1
          break;
504
3
        (*PatchIt)->OutSecOff = PrevISLimit;
505
3
        ++PatchIt;
506
3
      }
507
3
      PatchUpperBound = PrevISLimit + Target->getThunkSectionSpacing();
508
3
    }
509
57
    PrevISLimit = ISLimit;
510
57
  }
511
41
  for (; PatchIt != PatchEnd; 
++PatchIt33
) {
512
33
    (*PatchIt)->OutSecOff = ISLimit;
513
33
  }
514
8
515
8
  // merge all patch sections. We use the OutSecOff assigned above to
516
8
  // determine the insertion point. This is ok as we only merge into an
517
8
  // InputSectionDescription once per pass, and at the end of the pass
518
8
  // assignAddresses() will recalculate all the OutSecOff values.
519
8
  std::vector<InputSection *> Tmp;
520
8
  Tmp.reserve(ISD.Sections.size() + Patches.size());
521
56
  auto MergeCmp = [](const InputSection *A, const InputSection *B) {
522
56
    if (A->OutSecOff < B->OutSecOff)
523
0
      return true;
524
56
    if (A->OutSecOff == B->OutSecOff && 
isa<Patch843419Section>(A)3
&&
525
56
        
!isa<Patch843419Section>(B)3
)
526
3
      return true;
527
53
    return false;
528
53
  };
529
8
  std::merge(ISD.Sections.begin(), ISD.Sections.end(), Patches.begin(),
530
8
             Patches.end(), std::back_inserter(Tmp), MergeCmp);
531
8
  ISD.Sections = std::move(Tmp);
532
8
}
533
534
// Given an erratum sequence that starts at address AdrpAddr, with an
535
// instruction that we need to patch at PatcheeOffset from the start of
536
// InputSection IS, create a Patch843419 Section and add it to the
537
// Patches that we need to insert.
538
static void implementPatch(uint64_t AdrpAddr, uint64_t PatcheeOffset,
539
                           InputSection *IS,
540
82
                           std::vector<Patch843419Section *> &Patches) {
541
82
  // There may be a relocation at the same offset that we are patching. There
542
82
  // are four cases that we need to consider.
543
82
  // Case 1: R_AARCH64_JUMP26 branch relocation. We have already patched this
544
82
  // instance of the erratum on a previous patch and altered the relocation. We
545
82
  // have nothing more to do.
546
82
  // Case 2: A TLS Relaxation R_RELAX_TLS_IE_TO_LE. In this case the ADRP that
547
82
  // we read will be transformed into a MOVZ later so we actually don't match
548
82
  // the sequence and have nothing more to do.
549
82
  // Case 3: A load/store register (unsigned immediate) class relocation. There
550
82
  // are two of these R_AARCH_LD64_ABS_LO12_NC and R_AARCH_LD64_GOT_LO12_NC and
551
82
  // they are both absolute. We need to add the same relocation to the patch,
552
82
  // and replace the relocation with a R_AARCH_JUMP26 branch relocation.
553
82
  // Case 4: No relocation. We must create a new R_AARCH64_JUMP26 branch
554
82
  // relocation at the offset.
555
82
  auto RelIt = std::find_if(
556
82
      IS->Relocations.begin(), IS->Relocations.end(),
557
189
      [=](const Relocation &R) { return R.Offset == PatcheeOffset; });
558
82
  if (RelIt != IS->Relocations.end() &&
559
82
      
(75
RelIt->Type == R_AARCH64_JUMP2675
||
RelIt->Expr == R_RELAX_TLS_IE_TO_LE30
))
560
46
    return;
561
36
562
36
  log("detected cortex-a53-843419 erratum sequence starting at " +
563
36
      utohexstr(AdrpAddr) + " in unpatched output.");
564
36
565
36
  auto *PS = make<Patch843419Section>(IS, PatcheeOffset);
566
36
  Patches.push_back(PS);
567
36
568
36
  auto MakeRelToPatch = [](uint64_t Offset, Symbol *PatchSym) {
569
36
    return Relocation{R_PC, R_AARCH64_JUMP26, Offset, 0, PatchSym};
570
36
  };
571
36
572
36
  if (RelIt != IS->Relocations.end()) {
573
29
    PS->Relocations.push_back(
574
29
        {RelIt->Expr, RelIt->Type, 0, RelIt->Addend, RelIt->Sym});
575
29
    *RelIt = MakeRelToPatch(PatcheeOffset, PS->PatchSym);
576
29
  } else
577
7
    IS->Relocations.push_back(MakeRelToPatch(PatcheeOffset, PS->PatchSym));
578
36
}
579
580
// Scan all the instructions in InputSectionDescription, for each instance of
581
// the erratum sequence create a Patch843419Section. We return the list of
582
// Patch843419Sections that need to be applied to ISD.
583
std::vector<Patch843419Section *>
584
AArch64Err843419Patcher::patchInputSectionDescription(
585
35
    InputSectionDescription &ISD) {
586
35
  std::vector<Patch843419Section *> Patches;
587
190
  for (InputSection *IS : ISD.Sections) {
588
190
    //  LLD doesn't use the erratum sequence in SyntheticSections.
589
190
    if (isa<SyntheticSection>(IS))
590
49
      continue;
591
141
    // Use SectionMap to make sure we only scan code and not inline data.
592
141
    // We have already sorted MapSyms in ascending order and removed consecutive
593
141
    // mapping symbols of the same type. Our range of executable instructions to
594
141
    // scan is therefore [CodeSym->Value, DataSym->Value) or [CodeSym->Value,
595
141
    // section size).
596
141
    std::vector<const Defined *> &MapSyms = SectionMap[IS];
597
141
598
213
    auto CodeSym = llvm::find_if(MapSyms, [&](const Defined *MS) {
599
213
      return MS->getName().startswith("$x");
600
213
    });
601
141
602
159
    while (CodeSym != MapSyms.end()) {
603
122
      auto DataSym = std::next(CodeSym);
604
122
      uint64_t Off = (*CodeSym)->Value;
605
122
      uint64_t Limit =
606
122
          (DataSym == MapSyms.end()) ? 
IS->data().size()104
:
(*DataSym)->Value18
;
607
122
608
307
      while (Off < Limit) {
609
185
        uint64_t StartAddr = IS->getVA(Off);
610
185
        if (uint64_t PatcheeOffset = scanCortexA53Errata843419(IS, Off, Limit))
611
82
          implementPatch(StartAddr, PatcheeOffset, IS, Patches);
612
185
      }
613
122
      if (DataSym == MapSyms.end())
614
104
        break;
615
18
      CodeSym = std::next(DataSym);
616
18
    }
617
141
  }
618
35
  return Patches;
619
35
}
620
621
// For each InputSectionDescription make one pass over the executable sections
622
// looking for the erratum sequence; creating a synthetic Patch843419Section
623
// for each instance found. We insert these synthetic patch sections after the
624
// executable code in each InputSectionDescription.
625
//
626
// PreConditions:
627
// The Output and Input Sections have had their final addresses assigned.
628
//
629
// PostConditions:
630
// Returns true if at least one patch was added. The addresses of the
631
// Ouptut and Input Sections may have been changed.
632
// Returns false if no patches were required and no changes were made.
633
16
bool AArch64Err843419Patcher::createFixes() {
634
16
  if (Initialized == false)
635
8
    init();
636
16
637
16
  bool AddressesChanged = false;
638
114
  for (OutputSection *OS : OutputSections) {
639
114
    if (!(OS->Flags & SHF_ALLOC) || 
!(OS->Flags & SHF_EXECINSTR)50
)
640
91
      continue;
641
23
    for (BaseCommand *BC : OS->SectionCommands)
642
35
      if (auto *ISD = dyn_cast<InputSectionDescription>(BC)) {
643
35
        std::vector<Patch843419Section *> Patches =
644
35
            patchInputSectionDescription(*ISD);
645
35
        if (!Patches.empty()) {
646
8
          insertPatches(*ISD, Patches);
647
8
          AddressesChanged = true;
648
8
        }
649
35
      }
650
23
  }
651
16
  return AddressesChanged;
652
16
}