/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // This file contains a printer that converts from our internal representation |
10 | | // of machine-dependent LLVM code to the AArch64 assembly language. |
11 | | // |
12 | | //===----------------------------------------------------------------------===// |
13 | | |
14 | | #include "AArch64.h" |
15 | | #include "AArch64MCInstLower.h" |
16 | | #include "AArch64MachineFunctionInfo.h" |
17 | | #include "AArch64RegisterInfo.h" |
18 | | #include "AArch64Subtarget.h" |
19 | | #include "AArch64TargetObjectFile.h" |
20 | | #include "MCTargetDesc/AArch64AddressingModes.h" |
21 | | #include "MCTargetDesc/AArch64InstPrinter.h" |
22 | | #include "MCTargetDesc/AArch64MCExpr.h" |
23 | | #include "MCTargetDesc/AArch64MCTargetDesc.h" |
24 | | #include "MCTargetDesc/AArch64TargetStreamer.h" |
25 | | #include "TargetInfo/AArch64TargetInfo.h" |
26 | | #include "Utils/AArch64BaseInfo.h" |
27 | | #include "llvm/ADT/SmallString.h" |
28 | | #include "llvm/ADT/SmallVector.h" |
29 | | #include "llvm/ADT/StringRef.h" |
30 | | #include "llvm/ADT/Triple.h" |
31 | | #include "llvm/ADT/Twine.h" |
32 | | #include "llvm/BinaryFormat/COFF.h" |
33 | | #include "llvm/BinaryFormat/ELF.h" |
34 | | #include "llvm/CodeGen/AsmPrinter.h" |
35 | | #include "llvm/CodeGen/MachineBasicBlock.h" |
36 | | #include "llvm/CodeGen/MachineFunction.h" |
37 | | #include "llvm/CodeGen/MachineInstr.h" |
38 | | #include "llvm/CodeGen/MachineJumpTableInfo.h" |
39 | | #include "llvm/CodeGen/MachineModuleInfoImpls.h" |
40 | | #include "llvm/CodeGen/MachineOperand.h" |
41 | | #include "llvm/CodeGen/StackMaps.h" |
42 | | #include "llvm/CodeGen/TargetRegisterInfo.h" |
43 | | #include "llvm/IR/DataLayout.h" |
44 | | #include "llvm/IR/DebugInfoMetadata.h" |
45 | | #include "llvm/MC/MCAsmInfo.h" |
46 | | #include "llvm/MC/MCContext.h" |
47 | | #include "llvm/MC/MCInst.h" |
48 | | #include "llvm/MC/MCInstBuilder.h" |
49 | | #include "llvm/MC/MCSectionELF.h" |
50 | | #include "llvm/MC/MCStreamer.h" |
51 | | #include "llvm/MC/MCSymbol.h" |
52 | | #include "llvm/Support/Casting.h" |
53 | | #include "llvm/Support/ErrorHandling.h" |
54 | | #include "llvm/Support/TargetRegistry.h" |
55 | | #include "llvm/Support/raw_ostream.h" |
56 | | #include "llvm/Target/TargetMachine.h" |
57 | | #include <algorithm> |
58 | | #include <cassert> |
59 | | #include <cstdint> |
60 | | #include <map> |
61 | | #include <memory> |
62 | | |
63 | | using namespace llvm; |
64 | | |
65 | | #define DEBUG_TYPE "asm-printer" |
66 | | |
67 | | namespace { |
68 | | |
69 | | class AArch64AsmPrinter : public AsmPrinter { |
70 | | AArch64MCInstLower MCInstLowering; |
71 | | StackMaps SM; |
72 | | const AArch64Subtarget *STI; |
73 | | |
74 | | public: |
75 | | AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer) |
76 | | : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this), |
77 | 8.72k | SM(*this) {} |
78 | | |
79 | 257k | StringRef getPassName() const override { return "AArch64 Assembly Printer"; } |
80 | | |
81 | | /// Wrapper for MCInstLowering.lowerOperand() for the |
82 | | /// tblgen'erated pseudo lowering. |
83 | 0 | bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const { |
84 | 0 | return MCInstLowering.lowerOperand(MO, MCOp); |
85 | 0 | } |
86 | | |
87 | | void EmitJumpTableInfo() override; |
88 | | void emitJumpTableEntry(const MachineJumpTableInfo *MJTI, |
89 | | const MachineBasicBlock *MBB, unsigned JTI); |
90 | | |
91 | | void LowerJumpTableDestSmall(MCStreamer &OutStreamer, const MachineInstr &MI); |
92 | | |
93 | | void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, |
94 | | const MachineInstr &MI); |
95 | | void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, |
96 | | const MachineInstr &MI); |
97 | | |
98 | | void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI); |
99 | | void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); |
100 | | void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); |
101 | | |
102 | | std::map<std::pair<unsigned, uint32_t>, MCSymbol *> HwasanMemaccessSymbols; |
103 | | void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI); |
104 | | void EmitHwasanMemaccessSymbols(Module &M); |
105 | | |
106 | | void EmitSled(const MachineInstr &MI, SledKind Kind); |
107 | | |
108 | | /// tblgen'erated driver function for lowering simple MI->MC |
109 | | /// pseudo instructions. |
110 | | bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, |
111 | | const MachineInstr *MI); |
112 | | |
113 | | void EmitInstruction(const MachineInstr *MI) override; |
114 | | |
115 | 8.72k | void getAnalysisUsage(AnalysisUsage &AU) const override { |
116 | 8.72k | AsmPrinter::getAnalysisUsage(AU); |
117 | 8.72k | AU.setPreservesAll(); |
118 | 8.72k | } |
119 | | |
120 | 257k | bool runOnMachineFunction(MachineFunction &MF) override { |
121 | 257k | AArch64FI = MF.getInfo<AArch64FunctionInfo>(); |
122 | 257k | STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget()); |
123 | 257k | |
124 | 257k | SetupMachineFunction(MF); |
125 | 257k | |
126 | 257k | if (STI->isTargetCOFF()) { |
127 | 159 | bool Internal = MF.getFunction().hasInternalLinkage(); |
128 | 159 | COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC5 |
129 | 159 | : COFF::IMAGE_SYM_CLASS_EXTERNAL154 ; |
130 | 159 | int Type = |
131 | 159 | COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT; |
132 | 159 | |
133 | 159 | OutStreamer->BeginCOFFSymbolDef(CurrentFnSym); |
134 | 159 | OutStreamer->EmitCOFFSymbolStorageClass(Scl); |
135 | 159 | OutStreamer->EmitCOFFSymbolType(Type); |
136 | 159 | OutStreamer->EndCOFFSymbolDef(); |
137 | 159 | } |
138 | 257k | |
139 | 257k | // Emit the rest of the function body. |
140 | 257k | EmitFunctionBody(); |
141 | 257k | |
142 | 257k | // Emit the XRay table for this function. |
143 | 257k | emitXRayTable(); |
144 | 257k | |
145 | 257k | // We didn't modify anything. |
146 | 257k | return false; |
147 | 257k | } |
148 | | |
149 | | private: |
150 | | void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O); |
151 | | bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O); |
152 | | bool printAsmRegInClass(const MachineOperand &MO, |
153 | | const TargetRegisterClass *RC, bool isVector, |
154 | | raw_ostream &O); |
155 | | |
156 | | bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, |
157 | | const char *ExtraCode, raw_ostream &O) override; |
158 | | bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, |
159 | | const char *ExtraCode, raw_ostream &O) override; |
160 | | |
161 | | void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS); |
162 | | |
163 | | void EmitFunctionBodyEnd() override; |
164 | | |
165 | | MCSymbol *GetCPISymbol(unsigned CPID) const override; |
166 | | void EmitEndOfAsmFile(Module &M) override; |
167 | | |
168 | | AArch64FunctionInfo *AArch64FI = nullptr; |
169 | | |
170 | | /// Emit the LOHs contained in AArch64FI. |
171 | | void EmitLOHs(); |
172 | | |
173 | | /// Emit instruction to set float register to zero. |
174 | | void EmitFMov0(const MachineInstr &MI); |
175 | | |
176 | | using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>; |
177 | | |
178 | | MInstToMCSymbol LOHInstToLabel; |
179 | | }; |
180 | | |
181 | | } // end anonymous namespace |
182 | | |
183 | | void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI) |
184 | 3 | { |
185 | 3 | EmitSled(MI, SledKind::FUNCTION_ENTER); |
186 | 3 | } |
187 | | |
188 | | void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) |
189 | 3 | { |
190 | 3 | EmitSled(MI, SledKind::FUNCTION_EXIT); |
191 | 3 | } |
192 | | |
193 | | void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) |
194 | 0 | { |
195 | 0 | EmitSled(MI, SledKind::TAIL_CALL); |
196 | 0 | } |
197 | | |
198 | | void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind) |
199 | 6 | { |
200 | 6 | static const int8_t NoopsInSledCount = 7; |
201 | 6 | // We want to emit the following pattern: |
202 | 6 | // |
203 | 6 | // .Lxray_sled_N: |
204 | 6 | // ALIGN |
205 | 6 | // B #32 |
206 | 6 | // ; 7 NOP instructions (28 bytes) |
207 | 6 | // .tmpN |
208 | 6 | // |
209 | 6 | // We need the 28 bytes (7 instructions) because at runtime, we'd be patching |
210 | 6 | // over the full 32 bytes (8 instructions) with the following pattern: |
211 | 6 | // |
212 | 6 | // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack |
213 | 6 | // LDR W0, #12 ; W0 := function ID |
214 | 6 | // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit |
215 | 6 | // BLR X16 ; call the tracing trampoline |
216 | 6 | // ;DATA: 32 bits of function ID |
217 | 6 | // ;DATA: lower 32 bits of the address of the trampoline |
218 | 6 | // ;DATA: higher 32 bits of the address of the trampoline |
219 | 6 | // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack |
220 | 6 | // |
221 | 6 | OutStreamer->EmitCodeAlignment(4); |
222 | 6 | auto CurSled = OutContext.createTempSymbol("xray_sled_", true); |
223 | 6 | OutStreamer->EmitLabel(CurSled); |
224 | 6 | auto Target = OutContext.createTempSymbol(); |
225 | 6 | |
226 | 6 | // Emit "B #32" instruction, which jumps over the next 28 bytes. |
227 | 6 | // The operand has to be the number of 4-byte instructions to jump over, |
228 | 6 | // including the current instruction. |
229 | 6 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8)); |
230 | 6 | |
231 | 48 | for (int8_t I = 0; I < NoopsInSledCount; I++42 ) |
232 | 42 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); |
233 | 6 | |
234 | 6 | OutStreamer->EmitLabel(Target); |
235 | 6 | recordSled(CurSled, MI, Kind); |
236 | 6 | } |
237 | | |
238 | 2 | void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) { |
239 | 2 | unsigned Reg = MI.getOperand(0).getReg(); |
240 | 2 | uint32_t AccessInfo = MI.getOperand(1).getImm(); |
241 | 2 | MCSymbol *&Sym = HwasanMemaccessSymbols[{Reg, AccessInfo}]; |
242 | 2 | if (!Sym) { |
243 | 2 | // FIXME: Make this work on non-ELF. |
244 | 2 | if (!TM.getTargetTriple().isOSBinFormatELF()) |
245 | 0 | report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF"); |
246 | 2 | |
247 | 2 | std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" + |
248 | 2 | utostr(AccessInfo); |
249 | 2 | Sym = OutContext.getOrCreateSymbol(SymName); |
250 | 2 | } |
251 | 2 | |
252 | 2 | EmitToStreamer(*OutStreamer, |
253 | 2 | MCInstBuilder(AArch64::BL) |
254 | 2 | .addExpr(MCSymbolRefExpr::create(Sym, OutContext))); |
255 | 2 | } |
256 | | |
257 | 8.71k | void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) { |
258 | 8.71k | if (HwasanMemaccessSymbols.empty()) |
259 | 8.71k | return; |
260 | 1 | |
261 | 1 | const Triple &TT = TM.getTargetTriple(); |
262 | 1 | assert(TT.isOSBinFormatELF()); |
263 | 1 | std::unique_ptr<MCSubtargetInfo> STI( |
264 | 1 | TM.getTarget().createMCSubtargetInfo(TT.str(), "", "")); |
265 | 1 | |
266 | 1 | MCSymbol *HwasanTagMismatchSym = |
267 | 1 | OutContext.getOrCreateSymbol("__hwasan_tag_mismatch"); |
268 | 1 | |
269 | 1 | const MCSymbolRefExpr *HwasanTagMismatchRef = |
270 | 1 | MCSymbolRefExpr::create(HwasanTagMismatchSym, OutContext); |
271 | 1 | |
272 | 2 | for (auto &P : HwasanMemaccessSymbols) { |
273 | 2 | unsigned Reg = P.first.first; |
274 | 2 | uint32_t AccessInfo = P.first.second; |
275 | 2 | MCSymbol *Sym = P.second; |
276 | 2 | |
277 | 2 | OutStreamer->SwitchSection(OutContext.getELFSection( |
278 | 2 | ".text.hot", ELF::SHT_PROGBITS, |
279 | 2 | ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, |
280 | 2 | Sym->getName())); |
281 | 2 | |
282 | 2 | OutStreamer->EmitSymbolAttribute(Sym, MCSA_ELF_TypeFunction); |
283 | 2 | OutStreamer->EmitSymbolAttribute(Sym, MCSA_Weak); |
284 | 2 | OutStreamer->EmitSymbolAttribute(Sym, MCSA_Hidden); |
285 | 2 | OutStreamer->EmitLabel(Sym); |
286 | 2 | |
287 | 2 | OutStreamer->EmitInstruction(MCInstBuilder(AArch64::UBFMXri) |
288 | 2 | .addReg(AArch64::X16) |
289 | 2 | .addReg(Reg) |
290 | 2 | .addImm(4) |
291 | 2 | .addImm(55), |
292 | 2 | *STI); |
293 | 2 | OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBroX) |
294 | 2 | .addReg(AArch64::W16) |
295 | 2 | .addReg(AArch64::X9) |
296 | 2 | .addReg(AArch64::X16) |
297 | 2 | .addImm(0) |
298 | 2 | .addImm(0), |
299 | 2 | *STI); |
300 | 2 | OutStreamer->EmitInstruction( |
301 | 2 | MCInstBuilder(AArch64::SUBSXrs) |
302 | 2 | .addReg(AArch64::XZR) |
303 | 2 | .addReg(AArch64::X16) |
304 | 2 | .addReg(Reg) |
305 | 2 | .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), |
306 | 2 | *STI); |
307 | 2 | MCSymbol *HandlePartialSym = OutContext.createTempSymbol(); |
308 | 2 | OutStreamer->EmitInstruction( |
309 | 2 | MCInstBuilder(AArch64::Bcc) |
310 | 2 | .addImm(AArch64CC::NE) |
311 | 2 | .addExpr(MCSymbolRefExpr::create(HandlePartialSym, OutContext)), |
312 | 2 | *STI); |
313 | 2 | MCSymbol *ReturnSym = OutContext.createTempSymbol(); |
314 | 2 | OutStreamer->EmitLabel(ReturnSym); |
315 | 2 | OutStreamer->EmitInstruction( |
316 | 2 | MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI); |
317 | 2 | |
318 | 2 | OutStreamer->EmitLabel(HandlePartialSym); |
319 | 2 | OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWri) |
320 | 2 | .addReg(AArch64::WZR) |
321 | 2 | .addReg(AArch64::W16) |
322 | 2 | .addImm(15) |
323 | 2 | .addImm(0), |
324 | 2 | *STI); |
325 | 2 | MCSymbol *HandleMismatchSym = OutContext.createTempSymbol(); |
326 | 2 | OutStreamer->EmitInstruction( |
327 | 2 | MCInstBuilder(AArch64::Bcc) |
328 | 2 | .addImm(AArch64CC::HI) |
329 | 2 | .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), |
330 | 2 | *STI); |
331 | 2 | |
332 | 2 | OutStreamer->EmitInstruction( |
333 | 2 | MCInstBuilder(AArch64::ANDXri) |
334 | 2 | .addReg(AArch64::X17) |
335 | 2 | .addReg(Reg) |
336 | 2 | .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), |
337 | 2 | *STI); |
338 | 2 | unsigned Size = 1 << (AccessInfo & 0xf); |
339 | 2 | if (Size != 1) |
340 | 2 | OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ADDXri) |
341 | 2 | .addReg(AArch64::X17) |
342 | 2 | .addReg(AArch64::X17) |
343 | 2 | .addImm(Size - 1) |
344 | 2 | .addImm(0), |
345 | 2 | *STI); |
346 | 2 | OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWrs) |
347 | 2 | .addReg(AArch64::WZR) |
348 | 2 | .addReg(AArch64::W16) |
349 | 2 | .addReg(AArch64::W17) |
350 | 2 | .addImm(0), |
351 | 2 | *STI); |
352 | 2 | OutStreamer->EmitInstruction( |
353 | 2 | MCInstBuilder(AArch64::Bcc) |
354 | 2 | .addImm(AArch64CC::LS) |
355 | 2 | .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), |
356 | 2 | *STI); |
357 | 2 | |
358 | 2 | OutStreamer->EmitInstruction( |
359 | 2 | MCInstBuilder(AArch64::ORRXri) |
360 | 2 | .addReg(AArch64::X16) |
361 | 2 | .addReg(Reg) |
362 | 2 | .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), |
363 | 2 | *STI); |
364 | 2 | OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBui) |
365 | 2 | .addReg(AArch64::W16) |
366 | 2 | .addReg(AArch64::X16) |
367 | 2 | .addImm(0), |
368 | 2 | *STI); |
369 | 2 | OutStreamer->EmitInstruction( |
370 | 2 | MCInstBuilder(AArch64::SUBSXrs) |
371 | 2 | .addReg(AArch64::XZR) |
372 | 2 | .addReg(AArch64::X16) |
373 | 2 | .addReg(Reg) |
374 | 2 | .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), |
375 | 2 | *STI); |
376 | 2 | OutStreamer->EmitInstruction( |
377 | 2 | MCInstBuilder(AArch64::Bcc) |
378 | 2 | .addImm(AArch64CC::EQ) |
379 | 2 | .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), |
380 | 2 | *STI); |
381 | 2 | |
382 | 2 | OutStreamer->EmitLabel(HandleMismatchSym); |
383 | 2 | OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXpre) |
384 | 2 | .addReg(AArch64::SP) |
385 | 2 | .addReg(AArch64::X0) |
386 | 2 | .addReg(AArch64::X1) |
387 | 2 | .addReg(AArch64::SP) |
388 | 2 | .addImm(-32), |
389 | 2 | *STI); |
390 | 2 | OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXi) |
391 | 2 | .addReg(AArch64::FP) |
392 | 2 | .addReg(AArch64::LR) |
393 | 2 | .addReg(AArch64::SP) |
394 | 2 | .addImm(29), |
395 | 2 | *STI); |
396 | 2 | |
397 | 2 | if (Reg != AArch64::X0) |
398 | 1 | OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ORRXrs) |
399 | 1 | .addReg(AArch64::X0) |
400 | 1 | .addReg(AArch64::XZR) |
401 | 1 | .addReg(Reg) |
402 | 1 | .addImm(0), |
403 | 1 | *STI); |
404 | 2 | OutStreamer->EmitInstruction(MCInstBuilder(AArch64::MOVZXi) |
405 | 2 | .addReg(AArch64::X1) |
406 | 2 | .addImm(AccessInfo) |
407 | 2 | .addImm(0), |
408 | 2 | *STI); |
409 | 2 | |
410 | 2 | // Intentionally load the GOT entry and branch to it, rather than possibly |
411 | 2 | // late binding the function, which may clobber the registers before we have |
412 | 2 | // a chance to save them. |
413 | 2 | OutStreamer->EmitInstruction( |
414 | 2 | MCInstBuilder(AArch64::ADRP) |
415 | 2 | .addReg(AArch64::X16) |
416 | 2 | .addExpr(AArch64MCExpr::create( |
417 | 2 | HwasanTagMismatchRef, |
418 | 2 | AArch64MCExpr::VariantKind::VK_GOT_PAGE, OutContext)), |
419 | 2 | *STI); |
420 | 2 | OutStreamer->EmitInstruction( |
421 | 2 | MCInstBuilder(AArch64::LDRXui) |
422 | 2 | .addReg(AArch64::X16) |
423 | 2 | .addReg(AArch64::X16) |
424 | 2 | .addExpr(AArch64MCExpr::create( |
425 | 2 | HwasanTagMismatchRef, |
426 | 2 | AArch64MCExpr::VariantKind::VK_GOT_LO12, OutContext)), |
427 | 2 | *STI); |
428 | 2 | OutStreamer->EmitInstruction( |
429 | 2 | MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI); |
430 | 2 | } |
431 | 1 | } |
432 | | |
433 | 8.71k | void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) { |
434 | 8.71k | EmitHwasanMemaccessSymbols(M); |
435 | 8.71k | |
436 | 8.71k | const Triple &TT = TM.getTargetTriple(); |
437 | 8.71k | if (TT.isOSBinFormatMachO()) { |
438 | 7.59k | // Funny Darwin hack: This flag tells the linker that no global symbols |
439 | 7.59k | // contain code that falls through to other global symbols (e.g. the obvious |
440 | 7.59k | // implementation of multiple entry points). If this doesn't occur, the |
441 | 7.59k | // linker can safely perform dead code stripping. Since LLVM never |
442 | 7.59k | // generates code that does this, it is always safe to set. |
443 | 7.59k | OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols); |
444 | 7.59k | emitStackMaps(SM); |
445 | 7.59k | } |
446 | 8.71k | } |
447 | | |
448 | 140k | void AArch64AsmPrinter::EmitLOHs() { |
449 | 140k | SmallVector<MCSymbol *, 3> MCArgs; |
450 | 140k | |
451 | 636k | for (const auto &D : AArch64FI->getLOHContainer()) { |
452 | 1.32M | for (const MachineInstr *MI : D.getArgs()) { |
453 | 1.32M | MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI); |
454 | 1.32M | assert(LabelIt != LOHInstToLabel.end() && |
455 | 1.32M | "Label hasn't been inserted for LOH related instruction"); |
456 | 1.32M | MCArgs.push_back(LabelIt->second); |
457 | 1.32M | } |
458 | 636k | OutStreamer->EmitLOHDirective(D.getKind(), MCArgs); |
459 | 636k | MCArgs.clear(); |
460 | 636k | } |
461 | 140k | } |
462 | | |
463 | 257k | void AArch64AsmPrinter::EmitFunctionBodyEnd() { |
464 | 257k | if (!AArch64FI->getLOHRelated().empty()) |
465 | 140k | EmitLOHs(); |
466 | 257k | } |
467 | | |
468 | | /// GetCPISymbol - Return the symbol for the specified constant pool entry. |
469 | 67.7k | MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const { |
470 | 67.7k | // Darwin uses a linker-private symbol name for constant-pools (to |
471 | 67.7k | // avoid addends on the relocation?), ELF has no such concept and |
472 | 67.7k | // uses a normal private symbol. |
473 | 67.7k | if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty()) |
474 | 66.8k | return OutContext.getOrCreateSymbol( |
475 | 66.8k | Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" + |
476 | 66.8k | Twine(getFunctionNumber()) + "_" + Twine(CPID)); |
477 | 935 | |
478 | 935 | return AsmPrinter::GetCPISymbol(CPID); |
479 | 935 | } |
480 | | |
481 | | void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum, |
482 | 33 | raw_ostream &O) { |
483 | 33 | const MachineOperand &MO = MI->getOperand(OpNum); |
484 | 33 | switch (MO.getType()) { |
485 | 33 | default: |
486 | 0 | llvm_unreachable("<unknown operand type>"); |
487 | 33 | case MachineOperand::MO_Register: { |
488 | 0 | unsigned Reg = MO.getReg(); |
489 | 0 | assert(TargetRegisterInfo::isPhysicalRegister(Reg)); |
490 | 0 | assert(!MO.getSubReg() && "Subregs should be eliminated!"); |
491 | 0 | O << AArch64InstPrinter::getRegisterName(Reg); |
492 | 0 | break; |
493 | 33 | } |
494 | 33 | case MachineOperand::MO_Immediate: { |
495 | 23 | int64_t Imm = MO.getImm(); |
496 | 23 | O << '#' << Imm; |
497 | 23 | break; |
498 | 33 | } |
499 | 33 | case MachineOperand::MO_GlobalAddress: { |
500 | 8 | PrintSymbolOperand(MO, O); |
501 | 8 | break; |
502 | 33 | } |
503 | 33 | case MachineOperand::MO_BlockAddress: { |
504 | 2 | MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress()); |
505 | 2 | Sym->print(O, MAI); |
506 | 2 | break; |
507 | 33 | } |
508 | 33 | } |
509 | 33 | } |
510 | | |
511 | | bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode, |
512 | 62 | raw_ostream &O) { |
513 | 62 | unsigned Reg = MO.getReg(); |
514 | 62 | switch (Mode) { |
515 | 62 | default: |
516 | 0 | return true; // Unknown mode. |
517 | 62 | case 'w': |
518 | 24 | Reg = getWRegFromXReg(Reg); |
519 | 24 | break; |
520 | 62 | case 'x': |
521 | 38 | Reg = getXRegFromWReg(Reg); |
522 | 38 | break; |
523 | 62 | } |
524 | 62 | |
525 | 62 | O << AArch64InstPrinter::getRegisterName(Reg); |
526 | 62 | return false; |
527 | 62 | } |
528 | | |
529 | | // Prints the register in MO using class RC using the offset in the |
530 | | // new register class. This should not be used for cross class |
531 | | // printing. |
532 | | bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO, |
533 | | const TargetRegisterClass *RC, |
534 | 20 | bool isVector, raw_ostream &O) { |
535 | 20 | assert(MO.isReg() && "Should only get here with a register!"); |
536 | 20 | const TargetRegisterInfo *RI = STI->getRegisterInfo(); |
537 | 20 | unsigned Reg = MO.getReg(); |
538 | 20 | unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg)); |
539 | 20 | assert(RI->regsOverlap(RegToPrint, Reg)); |
540 | 20 | O << AArch64InstPrinter::getRegisterName( |
541 | 20 | RegToPrint, isVector ? AArch64::vreg8 : AArch64::NoRegAltName12 ); |
542 | 20 | return false; |
543 | 20 | } |
544 | | |
545 | | bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, |
546 | 125 | const char *ExtraCode, raw_ostream &O) { |
547 | 125 | const MachineOperand &MO = MI->getOperand(OpNum); |
548 | 125 | |
549 | 125 | // First try the generic code, which knows about modifiers like 'c' and 'n'. |
550 | 125 | if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O)) |
551 | 8 | return false; |
552 | 117 | |
553 | 117 | // Does this asm operand have a single letter operand modifier? |
554 | 117 | if (ExtraCode && ExtraCode[0]43 ) { |
555 | 43 | if (ExtraCode[1] != 0) |
556 | 0 | return true; // Unknown modifier. |
557 | 43 | |
558 | 43 | switch (ExtraCode[0]) { |
559 | 43 | default: |
560 | 0 | return true; // Unknown modifier. |
561 | 43 | case 'w': // Print W register |
562 | 31 | case 'x': // Print X register |
563 | 31 | if (MO.isReg()) |
564 | 29 | return printAsmMRegister(MO, ExtraCode[0], O); |
565 | 2 | if (MO.isImm() && MO.getImm() == 0) { |
566 | 2 | unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR1 : AArch64::XZR1 ; |
567 | 2 | O << AArch64InstPrinter::getRegisterName(Reg); |
568 | 2 | return false; |
569 | 2 | } |
570 | 0 | printOperand(MI, OpNum, O); |
571 | 0 | return false; |
572 | 12 | case 'b': // Print B register. |
573 | 12 | case 'h': // Print H register. |
574 | 12 | case 's': // Print S register. |
575 | 12 | case 'd': // Print D register. |
576 | 12 | case 'q': // Print Q register. |
577 | 12 | if (MO.isReg()) { |
578 | 12 | const TargetRegisterClass *RC; |
579 | 12 | switch (ExtraCode[0]) { |
580 | 12 | case 'b': |
581 | 1 | RC = &AArch64::FPR8RegClass; |
582 | 1 | break; |
583 | 12 | case 'h': |
584 | 4 | RC = &AArch64::FPR16RegClass; |
585 | 4 | break; |
586 | 12 | case 's': |
587 | 5 | RC = &AArch64::FPR32RegClass; |
588 | 5 | break; |
589 | 12 | case 'd': |
590 | 1 | RC = &AArch64::FPR64RegClass; |
591 | 1 | break; |
592 | 12 | case 'q': |
593 | 1 | RC = &AArch64::FPR128RegClass; |
594 | 1 | break; |
595 | 12 | default: |
596 | 0 | return true; |
597 | 12 | } |
598 | 12 | return printAsmRegInClass(MO, RC, false /* vector */, O); |
599 | 12 | } |
600 | 0 | printOperand(MI, OpNum, O); |
601 | 0 | return false; |
602 | 43 | } |
603 | 43 | } |
604 | 74 | |
605 | 74 | // According to ARM, we should emit x and v registers unless we have a |
606 | 74 | // modifier. |
607 | 74 | if (MO.isReg()) { |
608 | 41 | unsigned Reg = MO.getReg(); |
609 | 41 | |
610 | 41 | // If this is a w or x register, print an x register. |
611 | 41 | if (AArch64::GPR32allRegClass.contains(Reg) || |
612 | 41 | AArch64::GPR64allRegClass.contains(Reg)28 ) |
613 | 33 | return printAsmMRegister(MO, 'x', O); |
614 | 8 | |
615 | 8 | // If this is a b, h, s, d, or q register, print it as a v register. |
616 | 8 | return printAsmRegInClass(MO, &AArch64::FPR128RegClass, true /* vector */, |
617 | 8 | O); |
618 | 8 | } |
619 | 33 | |
620 | 33 | printOperand(MI, OpNum, O); |
621 | 33 | return false; |
622 | 33 | } |
623 | | |
624 | | bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, |
625 | | unsigned OpNum, |
626 | | const char *ExtraCode, |
627 | 3 | raw_ostream &O) { |
628 | 3 | if (ExtraCode && ExtraCode[0]0 && ExtraCode[0] != 'a'0 ) |
629 | 0 | return true; // Unknown modifier. |
630 | 3 | |
631 | 3 | const MachineOperand &MO = MI->getOperand(OpNum); |
632 | 3 | assert(MO.isReg() && "unexpected inline asm memory operand"); |
633 | 3 | O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]"; |
634 | 3 | return false; |
635 | 3 | } |
636 | | |
637 | | void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI, |
638 | 0 | raw_ostream &OS) { |
639 | 0 | unsigned NOps = MI->getNumOperands(); |
640 | 0 | assert(NOps == 4); |
641 | 0 | OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: "; |
642 | 0 | // cast away const; DIetc do not take const operands for some reason. |
643 | 0 | OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata()) |
644 | 0 | ->getName(); |
645 | 0 | OS << " <- "; |
646 | 0 | // Frame address. Currently handles register +- offset only. |
647 | 0 | assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm()); |
648 | 0 | OS << '['; |
649 | 0 | printOperand(MI, 0, OS); |
650 | 0 | OS << '+'; |
651 | 0 | printOperand(MI, 1, OS); |
652 | 0 | OS << ']'; |
653 | 0 | OS << "+"; |
654 | 0 | printOperand(MI, NOps - 2, OS); |
655 | 0 | } |
656 | | |
657 | 257k | void AArch64AsmPrinter::EmitJumpTableInfo() { |
658 | 257k | const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo(); |
659 | 257k | if (!MJTI) return255k ; |
660 | 2.23k | |
661 | 2.23k | const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); |
662 | 2.23k | if (JT.empty()) return0 ; |
663 | 2.23k | |
664 | 2.23k | const Function &F = MF->getFunction(); |
665 | 2.23k | const TargetLoweringObjectFile &TLOF = getObjFileLowering(); |
666 | 2.23k | bool JTInDiffSection = |
667 | 2.23k | !STI->isTargetCOFF() || |
668 | 2.23k | !TLOF.shouldPutJumpTableInFunctionSection( |
669 | 2 | MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32, |
670 | 2 | F); |
671 | 2.23k | if (JTInDiffSection) { |
672 | 2.23k | // Drop it in the readonly section. |
673 | 2.23k | MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM); |
674 | 2.23k | OutStreamer->SwitchSection(ReadOnlySec); |
675 | 2.23k | } |
676 | 2.23k | |
677 | 2.23k | auto AFI = MF->getInfo<AArch64FunctionInfo>(); |
678 | 4.64k | for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI2.41k ) { |
679 | 2.41k | const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs; |
680 | 2.41k | |
681 | 2.41k | // If this jump table was deleted, ignore it. |
682 | 2.41k | if (JTBBs.empty()) continue0 ; |
683 | 2.41k | |
684 | 2.41k | unsigned Size = AFI->getJumpTableEntrySize(JTI); |
685 | 2.41k | EmitAlignment(Log2_32(Size)); |
686 | 2.41k | OutStreamer->EmitLabel(GetJTISymbol(JTI)); |
687 | 2.41k | |
688 | 2.41k | for (auto *JTBB : JTBBs) |
689 | 36.2k | emitJumpTableEntry(MJTI, JTBB, JTI); |
690 | 2.41k | } |
691 | 2.23k | } |
692 | | |
693 | | void AArch64AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI, |
694 | | const MachineBasicBlock *MBB, |
695 | 36.2k | unsigned JTI) { |
696 | 36.2k | const MCExpr *Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext); |
697 | 36.2k | auto AFI = MF->getInfo<AArch64FunctionInfo>(); |
698 | 36.2k | unsigned Size = AFI->getJumpTableEntrySize(JTI); |
699 | 36.2k | |
700 | 36.2k | if (Size == 4) { |
701 | 142 | // .word LBB - LJTI |
702 | 142 | const TargetLowering *TLI = MF->getSubtarget().getTargetLowering(); |
703 | 142 | const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext); |
704 | 142 | Value = MCBinaryExpr::createSub(Value, Base, OutContext); |
705 | 36.0k | } else { |
706 | 36.0k | // .byte (LBB - LBB) >> 2 (or .hword) |
707 | 36.0k | const MCSymbol *BaseSym = AFI->getJumpTableEntryPCRelSymbol(JTI); |
708 | 36.0k | const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext); |
709 | 36.0k | Value = MCBinaryExpr::createSub(Value, Base, OutContext); |
710 | 36.0k | Value = MCBinaryExpr::createLShr( |
711 | 36.0k | Value, MCConstantExpr::create(2, OutContext), OutContext); |
712 | 36.0k | } |
713 | 36.2k | |
714 | 36.2k | OutStreamer->EmitValue(Value, Size); |
715 | 36.2k | } |
716 | | |
717 | | /// Small jump tables contain an unsigned byte or half, representing the offset |
718 | | /// from the lowest-addressed possible destination to the desired basic |
719 | | /// block. Since all instructions are 4-byte aligned, this is further compressed |
720 | | /// by counting in instructions rather than bytes (i.e. divided by 4). So, to |
721 | | /// materialize the correct destination we need: |
722 | | /// |
723 | | /// adr xDest, .LBB0_0 |
724 | | /// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh). |
725 | | /// add xDest, xDest, xScratch, lsl #2 |
726 | | void AArch64AsmPrinter::LowerJumpTableDestSmall(llvm::MCStreamer &OutStreamer, |
727 | 2.39k | const llvm::MachineInstr &MI) { |
728 | 2.39k | unsigned DestReg = MI.getOperand(0).getReg(); |
729 | 2.39k | unsigned ScratchReg = MI.getOperand(1).getReg(); |
730 | 2.39k | unsigned ScratchRegW = |
731 | 2.39k | STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32); |
732 | 2.39k | unsigned TableReg = MI.getOperand(2).getReg(); |
733 | 2.39k | unsigned EntryReg = MI.getOperand(3).getReg(); |
734 | 2.39k | int JTIdx = MI.getOperand(4).getIndex(); |
735 | 2.39k | bool IsByteEntry = MI.getOpcode() == AArch64::JumpTableDest8; |
736 | 2.39k | |
737 | 2.39k | // This has to be first because the compression pass based its reachability |
738 | 2.39k | // calculations on the start of the JumpTableDest instruction. |
739 | 2.39k | auto Label = |
740 | 2.39k | MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx); |
741 | 2.39k | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR) |
742 | 2.39k | .addReg(DestReg) |
743 | 2.39k | .addExpr(MCSymbolRefExpr::create( |
744 | 2.39k | Label, MF->getContext()))); |
745 | 2.39k | |
746 | 2.39k | // Load the number of instruction-steps to offset from the label. |
747 | 2.39k | unsigned LdrOpcode = IsByteEntry ? AArch64::LDRBBroX2.16k : AArch64::LDRHHroX234 ; |
748 | 2.39k | EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode) |
749 | 2.39k | .addReg(ScratchRegW) |
750 | 2.39k | .addReg(TableReg) |
751 | 2.39k | .addReg(EntryReg) |
752 | 2.39k | .addImm(0) |
753 | 2.39k | .addImm(IsByteEntry ? 02.16k : 1234 )); |
754 | 2.39k | |
755 | 2.39k | // Multiply the steps by 4 and add to the already materialized base label |
756 | 2.39k | // address. |
757 | 2.39k | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs) |
758 | 2.39k | .addReg(DestReg) |
759 | 2.39k | .addReg(DestReg) |
760 | 2.39k | .addReg(ScratchReg) |
761 | 2.39k | .addImm(2)); |
762 | 2.39k | } |
763 | | |
764 | | void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, |
765 | 15 | const MachineInstr &MI) { |
766 | 15 | unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes(); |
767 | 15 | |
768 | 15 | SM.recordStackMap(MI); |
769 | 15 | assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); |
770 | 15 | |
771 | 15 | // Scan ahead to trim the shadow. |
772 | 15 | const MachineBasicBlock &MBB = *MI.getParent(); |
773 | 15 | MachineBasicBlock::const_iterator MII(MI); |
774 | 15 | ++MII; |
775 | 45 | while (NumNOPBytes > 0) { |
776 | 33 | if (MII == MBB.end() || MII->isCall()32 || |
777 | 33 | MII->getOpcode() == AArch64::DBG_VALUE30 || |
778 | 33 | MII->getOpcode() == TargetOpcode::PATCHPOINT30 || |
779 | 33 | MII->getOpcode() == TargetOpcode::STACKMAP30 ) |
780 | 3 | break; |
781 | 30 | ++MII; |
782 | 30 | NumNOPBytes -= 4; |
783 | 30 | } |
784 | 15 | |
785 | 15 | // Emit nops. |
786 | 19 | for (unsigned i = 0; i < NumNOPBytes; i += 44 ) |
787 | 4 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); |
788 | 15 | } |
789 | | |
790 | | // Lower a patchpoint of the form: |
791 | | // [<def>], <id>, <numBytes>, <target>, <numArgs> |
792 | | void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, |
793 | 47 | const MachineInstr &MI) { |
794 | 47 | SM.recordPatchPoint(MI); |
795 | 47 | |
796 | 47 | PatchPointOpers Opers(&MI); |
797 | 47 | |
798 | 47 | int64_t CallTarget = Opers.getCallTarget().getImm(); |
799 | 47 | unsigned EncodedBytes = 0; |
800 | 47 | if (CallTarget) { |
801 | 33 | assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget && |
802 | 33 | "High 16 bits of call target should be zero."); |
803 | 33 | unsigned ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg(); |
804 | 33 | EncodedBytes = 16; |
805 | 33 | // Materialize the jump address: |
806 | 33 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi) |
807 | 33 | .addReg(ScratchReg) |
808 | 33 | .addImm((CallTarget >> 32) & 0xFFFF) |
809 | 33 | .addImm(32)); |
810 | 33 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi) |
811 | 33 | .addReg(ScratchReg) |
812 | 33 | .addReg(ScratchReg) |
813 | 33 | .addImm((CallTarget >> 16) & 0xFFFF) |
814 | 33 | .addImm(16)); |
815 | 33 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi) |
816 | 33 | .addReg(ScratchReg) |
817 | 33 | .addReg(ScratchReg) |
818 | 33 | .addImm(CallTarget & 0xFFFF) |
819 | 33 | .addImm(0)); |
820 | 33 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg)); |
821 | 33 | } |
822 | 47 | // Emit padding. |
823 | 47 | unsigned NumBytes = Opers.getNumPatchBytes(); |
824 | 47 | assert(NumBytes >= EncodedBytes && |
825 | 47 | "Patchpoint can't request size less than the length of a call."); |
826 | 47 | assert((NumBytes - EncodedBytes) % 4 == 0 && |
827 | 47 | "Invalid number of NOP bytes requested!"); |
828 | 161 | for (unsigned i = EncodedBytes; i < NumBytes; i += 4114 ) |
829 | 114 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); |
830 | 47 | } |
831 | | |
832 | 6.84k | void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) { |
833 | 6.84k | unsigned DestReg = MI.getOperand(0).getReg(); |
834 | 6.84k | if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()6.79k ) { |
835 | 51 | // Convert H/S/D register to corresponding Q register |
836 | 51 | if (AArch64::H0 <= DestReg && DestReg <= AArch64::H3130 ) |
837 | 2 | DestReg = AArch64::Q0 + (DestReg - AArch64::H0); |
838 | 49 | else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S3128 ) |
839 | 28 | DestReg = AArch64::Q0 + (DestReg - AArch64::S0); |
840 | 21 | else { |
841 | 21 | assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31); |
842 | 21 | DestReg = AArch64::Q0 + (DestReg - AArch64::D0); |
843 | 21 | } |
844 | 51 | MCInst MOVI; |
845 | 51 | MOVI.setOpcode(AArch64::MOVIv2d_ns); |
846 | 51 | MOVI.addOperand(MCOperand::createReg(DestReg)); |
847 | 51 | MOVI.addOperand(MCOperand::createImm(0)); |
848 | 51 | EmitToStreamer(*OutStreamer, MOVI); |
849 | 6.79k | } else { |
850 | 6.79k | MCInst FMov; |
851 | 6.79k | switch (MI.getOpcode()) { |
852 | 6.79k | default: 0 llvm_unreachable0 ("Unexpected opcode"); |
853 | 6.79k | case AArch64::FMOVH0: |
854 | 2 | FMov.setOpcode(AArch64::FMOVWHr); |
855 | 2 | FMov.addOperand(MCOperand::createReg(DestReg)); |
856 | 2 | FMov.addOperand(MCOperand::createReg(AArch64::WZR)); |
857 | 2 | break; |
858 | 6.79k | case AArch64::FMOVS0: |
859 | 2.47k | FMov.setOpcode(AArch64::FMOVWSr); |
860 | 2.47k | FMov.addOperand(MCOperand::createReg(DestReg)); |
861 | 2.47k | FMov.addOperand(MCOperand::createReg(AArch64::WZR)); |
862 | 2.47k | break; |
863 | 6.79k | case AArch64::FMOVD0: |
864 | 4.31k | FMov.setOpcode(AArch64::FMOVXDr); |
865 | 4.31k | FMov.addOperand(MCOperand::createReg(DestReg)); |
866 | 4.31k | FMov.addOperand(MCOperand::createReg(AArch64::XZR)); |
867 | 4.31k | break; |
868 | 6.79k | } |
869 | 6.79k | EmitToStreamer(*OutStreamer, FMov); |
870 | 6.79k | } |
871 | 6.84k | } |
872 | | |
873 | | // Simple pseudo-instructions have their lowering (with expansion to real |
874 | | // instructions) auto-generated. |
875 | | #include "AArch64GenMCPseudoLowering.inc" |
876 | | |
877 | 14.6M | void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) { |
878 | 14.6M | // Do any auto-generated pseudo lowerings. |
879 | 14.6M | if (emitPseudoExpansionLowering(*OutStreamer, MI)) |
880 | 0 | return; |
881 | 14.6M | |
882 | 14.6M | if (AArch64FI->getLOHRelated().count(MI)) { |
883 | 1.30M | // Generate a label for LOH related instruction |
884 | 1.30M | MCSymbol *LOHLabel = createTempSymbol("loh"); |
885 | 1.30M | // Associate the instruction with the label |
886 | 1.30M | LOHInstToLabel[MI] = LOHLabel; |
887 | 1.30M | OutStreamer->EmitLabel(LOHLabel); |
888 | 1.30M | } |
889 | 14.6M | |
890 | 14.6M | AArch64TargetStreamer *TS = |
891 | 14.6M | static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); |
892 | 14.6M | // Do any manual lowerings. |
893 | 14.6M | switch (MI->getOpcode()) { |
894 | 14.6M | default: |
895 | 14.5M | break; |
896 | 14.6M | case AArch64::MOVMCSym: { |
897 | 6 | unsigned DestReg = MI->getOperand(0).getReg(); |
898 | 6 | const MachineOperand &MO_Sym = MI->getOperand(1); |
899 | 6 | MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym); |
900 | 6 | MCOperand Hi_MCSym, Lo_MCSym; |
901 | 6 | |
902 | 6 | Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S); |
903 | 6 | Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC); |
904 | 6 | |
905 | 6 | MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym); |
906 | 6 | MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym); |
907 | 6 | |
908 | 6 | MCInst MovZ; |
909 | 6 | MovZ.setOpcode(AArch64::MOVZXi); |
910 | 6 | MovZ.addOperand(MCOperand::createReg(DestReg)); |
911 | 6 | MovZ.addOperand(Hi_MCSym); |
912 | 6 | MovZ.addOperand(MCOperand::createImm(16)); |
913 | 6 | EmitToStreamer(*OutStreamer, MovZ); |
914 | 6 | |
915 | 6 | MCInst MovK; |
916 | 6 | MovK.setOpcode(AArch64::MOVKXi); |
917 | 6 | MovK.addOperand(MCOperand::createReg(DestReg)); |
918 | 6 | MovK.addOperand(MCOperand::createReg(DestReg)); |
919 | 6 | MovK.addOperand(Lo_MCSym); |
920 | 6 | MovK.addOperand(MCOperand::createImm(0)); |
921 | 6 | EmitToStreamer(*OutStreamer, MovK); |
922 | 6 | return; |
923 | 14.6M | } |
924 | 14.6M | case AArch64::MOVIv2d_ns: |
925 | 2.55k | // If the target has <rdar://problem/16473581>, lower this |
926 | 2.55k | // instruction to movi.16b instead. |
927 | 2.55k | if (STI->hasZeroCycleZeroingFPWorkaround() && |
928 | 2.55k | MI->getOperand(1).getImm() == 02.29k ) { |
929 | 2.14k | MCInst TmpInst; |
930 | 2.14k | TmpInst.setOpcode(AArch64::MOVIv16b_ns); |
931 | 2.14k | TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); |
932 | 2.14k | TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm())); |
933 | 2.14k | EmitToStreamer(*OutStreamer, TmpInst); |
934 | 2.14k | return; |
935 | 2.14k | } |
936 | 413 | break; |
937 | 413 | |
938 | 413 | case AArch64::DBG_VALUE: { |
939 | 0 | if (isVerbose() && OutStreamer->hasRawTextSupport()) { |
940 | 0 | SmallString<128> TmpStr; |
941 | 0 | raw_svector_ostream OS(TmpStr); |
942 | 0 | PrintDebugValueComment(MI, OS); |
943 | 0 | OutStreamer->EmitRawText(StringRef(OS.str())); |
944 | 0 | } |
945 | 0 | return; |
946 | 413 | |
947 | 413 | case AArch64::EMITBKEY: { |
948 | 4 | ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType(); |
949 | 4 | if (ExceptionHandlingType != ExceptionHandling::DwarfCFI && |
950 | 4 | ExceptionHandlingType != ExceptionHandling::ARM0 ) |
951 | 0 | return; |
952 | 4 | |
953 | 4 | if (needsCFIMoves() == CFI_M_None) |
954 | 1 | return; |
955 | 3 | |
956 | 3 | OutStreamer->EmitCFIBKeyFrame(); |
957 | 3 | return; |
958 | 3 | } |
959 | 3 | } |
960 | 3 | |
961 | 3 | // Tail calls use pseudo instructions so they have the proper code-gen |
962 | 3 | // attributes (isCall, isReturn, etc.). We lower them to the real |
963 | 3 | // instruction here. |
964 | 108 | case AArch64::TCRETURNri: |
965 | 108 | case AArch64::TCRETURNriBTI: |
966 | 108 | case AArch64::TCRETURNriALL: { |
967 | 108 | MCInst TmpInst; |
968 | 108 | TmpInst.setOpcode(AArch64::BR); |
969 | 108 | TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); |
970 | 108 | EmitToStreamer(*OutStreamer, TmpInst); |
971 | 108 | return; |
972 | 108 | } |
973 | 48.4k | case AArch64::TCRETURNdi: { |
974 | 48.4k | MCOperand Dest; |
975 | 48.4k | MCInstLowering.lowerOperand(MI->getOperand(0), Dest); |
976 | 48.4k | MCInst TmpInst; |
977 | 48.4k | TmpInst.setOpcode(AArch64::B); |
978 | 48.4k | TmpInst.addOperand(Dest); |
979 | 48.4k | EmitToStreamer(*OutStreamer, TmpInst); |
980 | 48.4k | return; |
981 | 108 | } |
982 | 108 | case AArch64::TLSDESC_CALLSEQ: { |
983 | 29 | /// lower this to: |
984 | 29 | /// adrp x0, :tlsdesc:var |
985 | 29 | /// ldr x1, [x0, #:tlsdesc_lo12:var] |
986 | 29 | /// add x0, x0, #:tlsdesc_lo12:var |
987 | 29 | /// .tlsdesccall var |
988 | 29 | /// blr x1 |
989 | 29 | /// (TPIDR_EL0 offset now in x0) |
990 | 29 | const MachineOperand &MO_Sym = MI->getOperand(0); |
991 | 29 | MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym); |
992 | 29 | MCOperand Sym, SymTLSDescLo12, SymTLSDesc; |
993 | 29 | MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF); |
994 | 29 | MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE); |
995 | 29 | MCInstLowering.lowerOperand(MO_Sym, Sym); |
996 | 29 | MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12); |
997 | 29 | MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc); |
998 | 29 | |
999 | 29 | MCInst Adrp; |
1000 | 29 | Adrp.setOpcode(AArch64::ADRP); |
1001 | 29 | Adrp.addOperand(MCOperand::createReg(AArch64::X0)); |
1002 | 29 | Adrp.addOperand(SymTLSDesc); |
1003 | 29 | EmitToStreamer(*OutStreamer, Adrp); |
1004 | 29 | |
1005 | 29 | MCInst Ldr; |
1006 | 29 | Ldr.setOpcode(AArch64::LDRXui); |
1007 | 29 | Ldr.addOperand(MCOperand::createReg(AArch64::X1)); |
1008 | 29 | Ldr.addOperand(MCOperand::createReg(AArch64::X0)); |
1009 | 29 | Ldr.addOperand(SymTLSDescLo12); |
1010 | 29 | Ldr.addOperand(MCOperand::createImm(0)); |
1011 | 29 | EmitToStreamer(*OutStreamer, Ldr); |
1012 | 29 | |
1013 | 29 | MCInst Add; |
1014 | 29 | Add.setOpcode(AArch64::ADDXri); |
1015 | 29 | Add.addOperand(MCOperand::createReg(AArch64::X0)); |
1016 | 29 | Add.addOperand(MCOperand::createReg(AArch64::X0)); |
1017 | 29 | Add.addOperand(SymTLSDescLo12); |
1018 | 29 | Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0))); |
1019 | 29 | EmitToStreamer(*OutStreamer, Add); |
1020 | 29 | |
1021 | 29 | // Emit a relocation-annotation. This expands to no code, but requests |
1022 | 29 | // the following instruction gets an R_AARCH64_TLSDESC_CALL. |
1023 | 29 | MCInst TLSDescCall; |
1024 | 29 | TLSDescCall.setOpcode(AArch64::TLSDESCCALL); |
1025 | 29 | TLSDescCall.addOperand(Sym); |
1026 | 29 | EmitToStreamer(*OutStreamer, TLSDescCall); |
1027 | 29 | |
1028 | 29 | MCInst Blr; |
1029 | 29 | Blr.setOpcode(AArch64::BLR); |
1030 | 29 | Blr.addOperand(MCOperand::createReg(AArch64::X1)); |
1031 | 29 | EmitToStreamer(*OutStreamer, Blr); |
1032 | 29 | |
1033 | 29 | return; |
1034 | 108 | } |
1035 | 108 | |
1036 | 108 | case AArch64::JumpTableDest32: { |
1037 | 17 | // We want: |
1038 | 17 | // ldrsw xScratch, [xTable, xEntry, lsl #2] |
1039 | 17 | // add xDest, xTable, xScratch |
1040 | 17 | unsigned DestReg = MI->getOperand(0).getReg(), |
1041 | 17 | ScratchReg = MI->getOperand(1).getReg(), |
1042 | 17 | TableReg = MI->getOperand(2).getReg(), |
1043 | 17 | EntryReg = MI->getOperand(3).getReg(); |
1044 | 17 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX) |
1045 | 17 | .addReg(ScratchReg) |
1046 | 17 | .addReg(TableReg) |
1047 | 17 | .addReg(EntryReg) |
1048 | 17 | .addImm(0) |
1049 | 17 | .addImm(1)); |
1050 | 17 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs) |
1051 | 17 | .addReg(DestReg) |
1052 | 17 | .addReg(TableReg) |
1053 | 17 | .addReg(ScratchReg) |
1054 | 17 | .addImm(0)); |
1055 | 17 | return; |
1056 | 108 | } |
1057 | 2.39k | case AArch64::JumpTableDest16: |
1058 | 2.39k | case AArch64::JumpTableDest8: |
1059 | 2.39k | LowerJumpTableDestSmall(*OutStreamer, *MI); |
1060 | 2.39k | return; |
1061 | 2.39k | |
1062 | 6.84k | case AArch64::FMOVH0: |
1063 | 6.84k | case AArch64::FMOVS0: |
1064 | 6.84k | case AArch64::FMOVD0: |
1065 | 6.84k | EmitFMov0(*MI); |
1066 | 6.84k | return; |
1067 | 6.84k | |
1068 | 6.84k | case TargetOpcode::STACKMAP: |
1069 | 15 | return LowerSTACKMAP(*OutStreamer, SM, *MI); |
1070 | 6.84k | |
1071 | 6.84k | case TargetOpcode::PATCHPOINT: |
1072 | 47 | return LowerPATCHPOINT(*OutStreamer, SM, *MI); |
1073 | 6.84k | |
1074 | 6.84k | case TargetOpcode::PATCHABLE_FUNCTION_ENTER: |
1075 | 3 | LowerPATCHABLE_FUNCTION_ENTER(*MI); |
1076 | 3 | return; |
1077 | 6.84k | |
1078 | 6.84k | case TargetOpcode::PATCHABLE_FUNCTION_EXIT: |
1079 | 3 | LowerPATCHABLE_FUNCTION_EXIT(*MI); |
1080 | 3 | return; |
1081 | 6.84k | |
1082 | 6.84k | case TargetOpcode::PATCHABLE_TAIL_CALL: |
1083 | 0 | LowerPATCHABLE_TAIL_CALL(*MI); |
1084 | 0 | return; |
1085 | 6.84k | |
1086 | 6.84k | case AArch64::HWASAN_CHECK_MEMACCESS: |
1087 | 2 | LowerHWASAN_CHECK_MEMACCESS(*MI); |
1088 | 2 | return; |
1089 | 6.84k | |
1090 | 6.84k | case AArch64::SEH_StackAlloc: |
1091 | 93 | TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm()); |
1092 | 93 | return; |
1093 | 6.84k | |
1094 | 6.84k | case AArch64::SEH_SaveFPLR: |
1095 | 66 | TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm()); |
1096 | 66 | return; |
1097 | 6.84k | |
1098 | 6.84k | case AArch64::SEH_SaveFPLR_X: |
1099 | 34 | assert(MI->getOperand(0).getImm() < 0 && |
1100 | 34 | "Pre increment SEH opcode must have a negative offset"); |
1101 | 34 | TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm()); |
1102 | 34 | return; |
1103 | 6.84k | |
1104 | 6.84k | case AArch64::SEH_SaveReg: |
1105 | 58 | TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(), |
1106 | 58 | MI->getOperand(1).getImm()); |
1107 | 58 | return; |
1108 | 6.84k | |
1109 | 6.84k | case AArch64::SEH_SaveReg_X: |
1110 | 65 | assert(MI->getOperand(1).getImm() < 0 && |
1111 | 65 | "Pre increment SEH opcode must have a negative offset"); |
1112 | 65 | TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(), |
1113 | 65 | -MI->getOperand(1).getImm()); |
1114 | 65 | return; |
1115 | 6.84k | |
1116 | 6.84k | case AArch64::SEH_SaveRegP: |
1117 | 85 | assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && |
1118 | 85 | "Non-consecutive registers not allowed for save_regp"); |
1119 | 85 | TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(), |
1120 | 85 | MI->getOperand(2).getImm()); |
1121 | 85 | return; |
1122 | 6.84k | |
1123 | 6.84k | case AArch64::SEH_SaveRegP_X: |
1124 | 4 | assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && |
1125 | 4 | "Non-consecutive registers not allowed for save_regp_x"); |
1126 | 4 | assert(MI->getOperand(2).getImm() < 0 && |
1127 | 4 | "Pre increment SEH opcode must have a negative offset"); |
1128 | 4 | TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(), |
1129 | 4 | -MI->getOperand(2).getImm()); |
1130 | 4 | return; |
1131 | 6.84k | |
1132 | 6.84k | case AArch64::SEH_SaveFReg: |
1133 | 4 | TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(), |
1134 | 4 | MI->getOperand(1).getImm()); |
1135 | 4 | return; |
1136 | 6.84k | |
1137 | 6.84k | case AArch64::SEH_SaveFReg_X: |
1138 | 2 | assert(MI->getOperand(1).getImm() < 0 && |
1139 | 2 | "Pre increment SEH opcode must have a negative offset"); |
1140 | 2 | TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(), |
1141 | 2 | -MI->getOperand(1).getImm()); |
1142 | 2 | return; |
1143 | 6.84k | |
1144 | 6.84k | case AArch64::SEH_SaveFRegP: |
1145 | 10 | assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && |
1146 | 10 | "Non-consecutive registers not allowed for save_regp"); |
1147 | 10 | TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(), |
1148 | 10 | MI->getOperand(2).getImm()); |
1149 | 10 | return; |
1150 | 6.84k | |
1151 | 6.84k | case AArch64::SEH_SaveFRegP_X: |
1152 | 8 | assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && |
1153 | 8 | "Non-consecutive registers not allowed for save_regp_x"); |
1154 | 8 | assert(MI->getOperand(2).getImm() < 0 && |
1155 | 8 | "Pre increment SEH opcode must have a negative offset"); |
1156 | 8 | TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(), |
1157 | 8 | -MI->getOperand(2).getImm()); |
1158 | 8 | return; |
1159 | 6.84k | |
1160 | 6.84k | case AArch64::SEH_SetFP: |
1161 | 15 | TS->EmitARM64WinCFISetFP(); |
1162 | 15 | return; |
1163 | 6.84k | |
1164 | 6.84k | case AArch64::SEH_AddFP: |
1165 | 27 | TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm()); |
1166 | 27 | return; |
1167 | 6.84k | |
1168 | 6.84k | case AArch64::SEH_Nop: |
1169 | 18 | TS->EmitARM64WinCFINop(); |
1170 | 18 | return; |
1171 | 6.84k | |
1172 | 6.84k | case AArch64::SEH_PrologEnd: |
1173 | 91 | TS->EmitARM64WinCFIPrologEnd(); |
1174 | 91 | return; |
1175 | 6.84k | |
1176 | 6.84k | case AArch64::SEH_EpilogStart: |
1177 | 158 | TS->EmitARM64WinCFIEpilogStart(); |
1178 | 158 | return; |
1179 | 6.84k | |
1180 | 6.84k | case AArch64::SEH_EpilogEnd: |
1181 | 158 | TS->EmitARM64WinCFIEpilogEnd(); |
1182 | 158 | return; |
1183 | 14.5M | } |
1184 | 14.5M | |
1185 | 14.5M | // Finally, do the automated lowerings for everything else. |
1186 | 14.5M | MCInst TmpInst; |
1187 | 14.5M | MCInstLowering.Lower(MI, TmpInst); |
1188 | 14.5M | EmitToStreamer(*OutStreamer, TmpInst); |
1189 | 14.5M | } |
1190 | | |
1191 | | // Force static initialization. |
1192 | 78.9k | extern "C" void LLVMInitializeAArch64AsmPrinter() { |
1193 | 78.9k | RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget()); |
1194 | 78.9k | RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget()); |
1195 | 78.9k | RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target()); |
1196 | 78.9k | } |