Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file contains a printer that converts from our internal representation
10
// of machine-dependent LLVM code to the AArch64 assembly language.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "AArch64.h"
15
#include "AArch64MCInstLower.h"
16
#include "AArch64MachineFunctionInfo.h"
17
#include "AArch64RegisterInfo.h"
18
#include "AArch64Subtarget.h"
19
#include "AArch64TargetObjectFile.h"
20
#include "MCTargetDesc/AArch64AddressingModes.h"
21
#include "MCTargetDesc/AArch64InstPrinter.h"
22
#include "MCTargetDesc/AArch64MCExpr.h"
23
#include "MCTargetDesc/AArch64MCTargetDesc.h"
24
#include "MCTargetDesc/AArch64TargetStreamer.h"
25
#include "TargetInfo/AArch64TargetInfo.h"
26
#include "Utils/AArch64BaseInfo.h"
27
#include "llvm/ADT/SmallString.h"
28
#include "llvm/ADT/SmallVector.h"
29
#include "llvm/ADT/StringRef.h"
30
#include "llvm/ADT/Triple.h"
31
#include "llvm/ADT/Twine.h"
32
#include "llvm/BinaryFormat/COFF.h"
33
#include "llvm/BinaryFormat/ELF.h"
34
#include "llvm/CodeGen/AsmPrinter.h"
35
#include "llvm/CodeGen/MachineBasicBlock.h"
36
#include "llvm/CodeGen/MachineFunction.h"
37
#include "llvm/CodeGen/MachineInstr.h"
38
#include "llvm/CodeGen/MachineJumpTableInfo.h"
39
#include "llvm/CodeGen/MachineModuleInfoImpls.h"
40
#include "llvm/CodeGen/MachineOperand.h"
41
#include "llvm/CodeGen/StackMaps.h"
42
#include "llvm/CodeGen/TargetRegisterInfo.h"
43
#include "llvm/IR/DataLayout.h"
44
#include "llvm/IR/DebugInfoMetadata.h"
45
#include "llvm/MC/MCAsmInfo.h"
46
#include "llvm/MC/MCContext.h"
47
#include "llvm/MC/MCInst.h"
48
#include "llvm/MC/MCInstBuilder.h"
49
#include "llvm/MC/MCSectionELF.h"
50
#include "llvm/MC/MCStreamer.h"
51
#include "llvm/MC/MCSymbol.h"
52
#include "llvm/Support/Casting.h"
53
#include "llvm/Support/ErrorHandling.h"
54
#include "llvm/Support/TargetRegistry.h"
55
#include "llvm/Support/raw_ostream.h"
56
#include "llvm/Target/TargetMachine.h"
57
#include <algorithm>
58
#include <cassert>
59
#include <cstdint>
60
#include <map>
61
#include <memory>
62
63
using namespace llvm;
64
65
#define DEBUG_TYPE "asm-printer"
66
67
namespace {
68
69
class AArch64AsmPrinter : public AsmPrinter {
70
  AArch64MCInstLower MCInstLowering;
71
  StackMaps SM;
72
  const AArch64Subtarget *STI;
73
74
public:
75
  AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
76
      : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
77
8.72k
        SM(*this) {}
78
79
257k
  StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
80
81
  /// Wrapper for MCInstLowering.lowerOperand() for the
82
  /// tblgen'erated pseudo lowering.
83
0
  bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
84
0
    return MCInstLowering.lowerOperand(MO, MCOp);
85
0
  }
86
87
  void EmitJumpTableInfo() override;
88
  void emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
89
                          const MachineBasicBlock *MBB, unsigned JTI);
90
91
  void LowerJumpTableDestSmall(MCStreamer &OutStreamer, const MachineInstr &MI);
92
93
  void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
94
                     const MachineInstr &MI);
95
  void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
96
                       const MachineInstr &MI);
97
98
  void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
99
  void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
100
  void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
101
102
  std::map<std::pair<unsigned, uint32_t>, MCSymbol *> HwasanMemaccessSymbols;
103
  void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
104
  void EmitHwasanMemaccessSymbols(Module &M);
105
106
  void EmitSled(const MachineInstr &MI, SledKind Kind);
107
108
  /// tblgen'erated driver function for lowering simple MI->MC
109
  /// pseudo instructions.
110
  bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
111
                                   const MachineInstr *MI);
112
113
  void EmitInstruction(const MachineInstr *MI) override;
114
115
8.72k
  void getAnalysisUsage(AnalysisUsage &AU) const override {
116
8.72k
    AsmPrinter::getAnalysisUsage(AU);
117
8.72k
    AU.setPreservesAll();
118
8.72k
  }
119
120
257k
  bool runOnMachineFunction(MachineFunction &MF) override {
121
257k
    AArch64FI = MF.getInfo<AArch64FunctionInfo>();
122
257k
    STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
123
257k
124
257k
    SetupMachineFunction(MF);
125
257k
126
257k
    if (STI->isTargetCOFF()) {
127
159
      bool Internal = MF.getFunction().hasInternalLinkage();
128
159
      COFF::SymbolStorageClass Scl = Internal ? 
COFF::IMAGE_SYM_CLASS_STATIC5
129
159
                                              : 
COFF::IMAGE_SYM_CLASS_EXTERNAL154
;
130
159
      int Type =
131
159
        COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
132
159
133
159
      OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
134
159
      OutStreamer->EmitCOFFSymbolStorageClass(Scl);
135
159
      OutStreamer->EmitCOFFSymbolType(Type);
136
159
      OutStreamer->EndCOFFSymbolDef();
137
159
    }
138
257k
139
257k
    // Emit the rest of the function body.
140
257k
    EmitFunctionBody();
141
257k
142
257k
    // Emit the XRay table for this function.
143
257k
    emitXRayTable();
144
257k
145
257k
    // We didn't modify anything.
146
257k
    return false;
147
257k
  }
148
149
private:
150
  void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
151
  bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
152
  bool printAsmRegInClass(const MachineOperand &MO,
153
                          const TargetRegisterClass *RC, bool isVector,
154
                          raw_ostream &O);
155
156
  bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
157
                       const char *ExtraCode, raw_ostream &O) override;
158
  bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
159
                             const char *ExtraCode, raw_ostream &O) override;
160
161
  void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
162
163
  void EmitFunctionBodyEnd() override;
164
165
  MCSymbol *GetCPISymbol(unsigned CPID) const override;
166
  void EmitEndOfAsmFile(Module &M) override;
167
168
  AArch64FunctionInfo *AArch64FI = nullptr;
169
170
  /// Emit the LOHs contained in AArch64FI.
171
  void EmitLOHs();
172
173
  /// Emit instruction to set float register to zero.
174
  void EmitFMov0(const MachineInstr &MI);
175
176
  using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
177
178
  MInstToMCSymbol LOHInstToLabel;
179
};
180
181
} // end anonymous namespace
182
183
void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
184
3
{
185
3
  EmitSled(MI, SledKind::FUNCTION_ENTER);
186
3
}
187
188
void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
189
3
{
190
3
  EmitSled(MI, SledKind::FUNCTION_EXIT);
191
3
}
192
193
void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
194
0
{
195
0
  EmitSled(MI, SledKind::TAIL_CALL);
196
0
}
197
198
void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
199
6
{
200
6
  static const int8_t NoopsInSledCount = 7;
201
6
  // We want to emit the following pattern:
202
6
  //
203
6
  // .Lxray_sled_N:
204
6
  //   ALIGN
205
6
  //   B #32
206
6
  //   ; 7 NOP instructions (28 bytes)
207
6
  // .tmpN
208
6
  //
209
6
  // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
210
6
  // over the full 32 bytes (8 instructions) with the following pattern:
211
6
  //
212
6
  //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
213
6
  //   LDR W0, #12 ; W0 := function ID
214
6
  //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
215
6
  //   BLR X16 ; call the tracing trampoline
216
6
  //   ;DATA: 32 bits of function ID
217
6
  //   ;DATA: lower 32 bits of the address of the trampoline
218
6
  //   ;DATA: higher 32 bits of the address of the trampoline
219
6
  //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
220
6
  //
221
6
  OutStreamer->EmitCodeAlignment(4);
222
6
  auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
223
6
  OutStreamer->EmitLabel(CurSled);
224
6
  auto Target = OutContext.createTempSymbol();
225
6
226
6
  // Emit "B #32" instruction, which jumps over the next 28 bytes.
227
6
  // The operand has to be the number of 4-byte instructions to jump over,
228
6
  // including the current instruction.
229
6
  EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
230
6
231
48
  for (int8_t I = 0; I < NoopsInSledCount; 
I++42
)
232
42
    EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
233
6
234
6
  OutStreamer->EmitLabel(Target);
235
6
  recordSled(CurSled, MI, Kind);
236
6
}
237
238
2
void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
239
2
  unsigned Reg = MI.getOperand(0).getReg();
240
2
  uint32_t AccessInfo = MI.getOperand(1).getImm();
241
2
  MCSymbol *&Sym = HwasanMemaccessSymbols[{Reg, AccessInfo}];
242
2
  if (!Sym) {
243
2
    // FIXME: Make this work on non-ELF.
244
2
    if (!TM.getTargetTriple().isOSBinFormatELF())
245
0
      report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
246
2
247
2
    std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
248
2
                          utostr(AccessInfo);
249
2
    Sym = OutContext.getOrCreateSymbol(SymName);
250
2
  }
251
2
252
2
  EmitToStreamer(*OutStreamer,
253
2
                 MCInstBuilder(AArch64::BL)
254
2
                     .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
255
2
}
256
257
8.71k
void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
258
8.71k
  if (HwasanMemaccessSymbols.empty())
259
8.71k
    return;
260
1
261
1
  const Triple &TT = TM.getTargetTriple();
262
1
  assert(TT.isOSBinFormatELF());
263
1
  std::unique_ptr<MCSubtargetInfo> STI(
264
1
      TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
265
1
266
1
  MCSymbol *HwasanTagMismatchSym =
267
1
      OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
268
1
269
1
  const MCSymbolRefExpr *HwasanTagMismatchRef =
270
1
      MCSymbolRefExpr::create(HwasanTagMismatchSym, OutContext);
271
1
272
2
  for (auto &P : HwasanMemaccessSymbols) {
273
2
    unsigned Reg = P.first.first;
274
2
    uint32_t AccessInfo = P.first.second;
275
2
    MCSymbol *Sym = P.second;
276
2
277
2
    OutStreamer->SwitchSection(OutContext.getELFSection(
278
2
        ".text.hot", ELF::SHT_PROGBITS,
279
2
        ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
280
2
        Sym->getName()));
281
2
282
2
    OutStreamer->EmitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
283
2
    OutStreamer->EmitSymbolAttribute(Sym, MCSA_Weak);
284
2
    OutStreamer->EmitSymbolAttribute(Sym, MCSA_Hidden);
285
2
    OutStreamer->EmitLabel(Sym);
286
2
287
2
    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::UBFMXri)
288
2
                                     .addReg(AArch64::X16)
289
2
                                     .addReg(Reg)
290
2
                                     .addImm(4)
291
2
                                     .addImm(55),
292
2
                                 *STI);
293
2
    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBroX)
294
2
                                     .addReg(AArch64::W16)
295
2
                                     .addReg(AArch64::X9)
296
2
                                     .addReg(AArch64::X16)
297
2
                                     .addImm(0)
298
2
                                     .addImm(0),
299
2
                                 *STI);
300
2
    OutStreamer->EmitInstruction(
301
2
        MCInstBuilder(AArch64::SUBSXrs)
302
2
            .addReg(AArch64::XZR)
303
2
            .addReg(AArch64::X16)
304
2
            .addReg(Reg)
305
2
            .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
306
2
        *STI);
307
2
    MCSymbol *HandlePartialSym = OutContext.createTempSymbol();
308
2
    OutStreamer->EmitInstruction(
309
2
        MCInstBuilder(AArch64::Bcc)
310
2
            .addImm(AArch64CC::NE)
311
2
            .addExpr(MCSymbolRefExpr::create(HandlePartialSym, OutContext)),
312
2
        *STI);
313
2
    MCSymbol *ReturnSym = OutContext.createTempSymbol();
314
2
    OutStreamer->EmitLabel(ReturnSym);
315
2
    OutStreamer->EmitInstruction(
316
2
        MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
317
2
318
2
    OutStreamer->EmitLabel(HandlePartialSym);
319
2
    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWri)
320
2
                                     .addReg(AArch64::WZR)
321
2
                                     .addReg(AArch64::W16)
322
2
                                     .addImm(15)
323
2
                                     .addImm(0),
324
2
                                 *STI);
325
2
    MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
326
2
    OutStreamer->EmitInstruction(
327
2
        MCInstBuilder(AArch64::Bcc)
328
2
            .addImm(AArch64CC::HI)
329
2
            .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
330
2
        *STI);
331
2
332
2
    OutStreamer->EmitInstruction(
333
2
        MCInstBuilder(AArch64::ANDXri)
334
2
            .addReg(AArch64::X17)
335
2
            .addReg(Reg)
336
2
            .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
337
2
        *STI);
338
2
    unsigned Size = 1 << (AccessInfo & 0xf);
339
2
    if (Size != 1)
340
2
      OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ADDXri)
341
2
                                       .addReg(AArch64::X17)
342
2
                                       .addReg(AArch64::X17)
343
2
                                       .addImm(Size - 1)
344
2
                                       .addImm(0),
345
2
                                   *STI);
346
2
    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWrs)
347
2
                                     .addReg(AArch64::WZR)
348
2
                                     .addReg(AArch64::W16)
349
2
                                     .addReg(AArch64::W17)
350
2
                                     .addImm(0),
351
2
                                 *STI);
352
2
    OutStreamer->EmitInstruction(
353
2
        MCInstBuilder(AArch64::Bcc)
354
2
            .addImm(AArch64CC::LS)
355
2
            .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
356
2
        *STI);
357
2
358
2
    OutStreamer->EmitInstruction(
359
2
        MCInstBuilder(AArch64::ORRXri)
360
2
            .addReg(AArch64::X16)
361
2
            .addReg(Reg)
362
2
            .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
363
2
        *STI);
364
2
    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBui)
365
2
                                     .addReg(AArch64::W16)
366
2
                                     .addReg(AArch64::X16)
367
2
                                     .addImm(0),
368
2
                                 *STI);
369
2
    OutStreamer->EmitInstruction(
370
2
        MCInstBuilder(AArch64::SUBSXrs)
371
2
            .addReg(AArch64::XZR)
372
2
            .addReg(AArch64::X16)
373
2
            .addReg(Reg)
374
2
            .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
375
2
        *STI);
376
2
    OutStreamer->EmitInstruction(
377
2
        MCInstBuilder(AArch64::Bcc)
378
2
            .addImm(AArch64CC::EQ)
379
2
            .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
380
2
        *STI);
381
2
382
2
    OutStreamer->EmitLabel(HandleMismatchSym);
383
2
    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXpre)
384
2
                                     .addReg(AArch64::SP)
385
2
                                     .addReg(AArch64::X0)
386
2
                                     .addReg(AArch64::X1)
387
2
                                     .addReg(AArch64::SP)
388
2
                                     .addImm(-32),
389
2
                                 *STI);
390
2
    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXi)
391
2
                                     .addReg(AArch64::FP)
392
2
                                     .addReg(AArch64::LR)
393
2
                                     .addReg(AArch64::SP)
394
2
                                     .addImm(29),
395
2
                                 *STI);
396
2
397
2
    if (Reg != AArch64::X0)
398
1
      OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ORRXrs)
399
1
                                       .addReg(AArch64::X0)
400
1
                                       .addReg(AArch64::XZR)
401
1
                                       .addReg(Reg)
402
1
                                       .addImm(0),
403
1
                                   *STI);
404
2
    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::MOVZXi)
405
2
                                     .addReg(AArch64::X1)
406
2
                                     .addImm(AccessInfo)
407
2
                                     .addImm(0),
408
2
                                 *STI);
409
2
410
2
    // Intentionally load the GOT entry and branch to it, rather than possibly
411
2
    // late binding the function, which may clobber the registers before we have
412
2
    // a chance to save them.
413
2
    OutStreamer->EmitInstruction(
414
2
        MCInstBuilder(AArch64::ADRP)
415
2
            .addReg(AArch64::X16)
416
2
            .addExpr(AArch64MCExpr::create(
417
2
                HwasanTagMismatchRef,
418
2
                AArch64MCExpr::VariantKind::VK_GOT_PAGE, OutContext)),
419
2
        *STI);
420
2
    OutStreamer->EmitInstruction(
421
2
        MCInstBuilder(AArch64::LDRXui)
422
2
            .addReg(AArch64::X16)
423
2
            .addReg(AArch64::X16)
424
2
            .addExpr(AArch64MCExpr::create(
425
2
                HwasanTagMismatchRef,
426
2
                AArch64MCExpr::VariantKind::VK_GOT_LO12, OutContext)),
427
2
        *STI);
428
2
    OutStreamer->EmitInstruction(
429
2
        MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
430
2
  }
431
1
}
432
433
8.71k
void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) {
434
8.71k
  EmitHwasanMemaccessSymbols(M);
435
8.71k
436
8.71k
  const Triple &TT = TM.getTargetTriple();
437
8.71k
  if (TT.isOSBinFormatMachO()) {
438
7.59k
    // Funny Darwin hack: This flag tells the linker that no global symbols
439
7.59k
    // contain code that falls through to other global symbols (e.g. the obvious
440
7.59k
    // implementation of multiple entry points).  If this doesn't occur, the
441
7.59k
    // linker can safely perform dead code stripping.  Since LLVM never
442
7.59k
    // generates code that does this, it is always safe to set.
443
7.59k
    OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
444
7.59k
    emitStackMaps(SM);
445
7.59k
  }
446
8.71k
}
447
448
140k
void AArch64AsmPrinter::EmitLOHs() {
449
140k
  SmallVector<MCSymbol *, 3> MCArgs;
450
140k
451
636k
  for (const auto &D : AArch64FI->getLOHContainer()) {
452
1.32M
    for (const MachineInstr *MI : D.getArgs()) {
453
1.32M
      MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
454
1.32M
      assert(LabelIt != LOHInstToLabel.end() &&
455
1.32M
             "Label hasn't been inserted for LOH related instruction");
456
1.32M
      MCArgs.push_back(LabelIt->second);
457
1.32M
    }
458
636k
    OutStreamer->EmitLOHDirective(D.getKind(), MCArgs);
459
636k
    MCArgs.clear();
460
636k
  }
461
140k
}
462
463
257k
void AArch64AsmPrinter::EmitFunctionBodyEnd() {
464
257k
  if (!AArch64FI->getLOHRelated().empty())
465
140k
    EmitLOHs();
466
257k
}
467
468
/// GetCPISymbol - Return the symbol for the specified constant pool entry.
469
67.7k
MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
470
67.7k
  // Darwin uses a linker-private symbol name for constant-pools (to
471
67.7k
  // avoid addends on the relocation?), ELF has no such concept and
472
67.7k
  // uses a normal private symbol.
473
67.7k
  if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
474
66.8k
    return OutContext.getOrCreateSymbol(
475
66.8k
        Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
476
66.8k
        Twine(getFunctionNumber()) + "_" + Twine(CPID));
477
935
478
935
  return AsmPrinter::GetCPISymbol(CPID);
479
935
}
480
481
void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
482
33
                                     raw_ostream &O) {
483
33
  const MachineOperand &MO = MI->getOperand(OpNum);
484
33
  switch (MO.getType()) {
485
33
  default:
486
0
    llvm_unreachable("<unknown operand type>");
487
33
  case MachineOperand::MO_Register: {
488
0
    unsigned Reg = MO.getReg();
489
0
    assert(TargetRegisterInfo::isPhysicalRegister(Reg));
490
0
    assert(!MO.getSubReg() && "Subregs should be eliminated!");
491
0
    O << AArch64InstPrinter::getRegisterName(Reg);
492
0
    break;
493
33
  }
494
33
  case MachineOperand::MO_Immediate: {
495
23
    int64_t Imm = MO.getImm();
496
23
    O << '#' << Imm;
497
23
    break;
498
33
  }
499
33
  case MachineOperand::MO_GlobalAddress: {
500
8
    PrintSymbolOperand(MO, O);
501
8
    break;
502
33
  }
503
33
  case MachineOperand::MO_BlockAddress: {
504
2
    MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
505
2
    Sym->print(O, MAI);
506
2
    break;
507
33
  }
508
33
  }
509
33
}
510
511
bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
512
62
                                          raw_ostream &O) {
513
62
  unsigned Reg = MO.getReg();
514
62
  switch (Mode) {
515
62
  default:
516
0
    return true; // Unknown mode.
517
62
  case 'w':
518
24
    Reg = getWRegFromXReg(Reg);
519
24
    break;
520
62
  case 'x':
521
38
    Reg = getXRegFromWReg(Reg);
522
38
    break;
523
62
  }
524
62
525
62
  O << AArch64InstPrinter::getRegisterName(Reg);
526
62
  return false;
527
62
}
528
529
// Prints the register in MO using class RC using the offset in the
530
// new register class. This should not be used for cross class
531
// printing.
532
bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
533
                                           const TargetRegisterClass *RC,
534
20
                                           bool isVector, raw_ostream &O) {
535
20
  assert(MO.isReg() && "Should only get here with a register!");
536
20
  const TargetRegisterInfo *RI = STI->getRegisterInfo();
537
20
  unsigned Reg = MO.getReg();
538
20
  unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
539
20
  assert(RI->regsOverlap(RegToPrint, Reg));
540
20
  O << AArch64InstPrinter::getRegisterName(
541
20
           RegToPrint, isVector ? 
AArch64::vreg8
:
AArch64::NoRegAltName12
);
542
20
  return false;
543
20
}
544
545
bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
546
125
                                        const char *ExtraCode, raw_ostream &O) {
547
125
  const MachineOperand &MO = MI->getOperand(OpNum);
548
125
549
125
  // First try the generic code, which knows about modifiers like 'c' and 'n'.
550
125
  if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
551
8
    return false;
552
117
553
117
  // Does this asm operand have a single letter operand modifier?
554
117
  if (ExtraCode && 
ExtraCode[0]43
) {
555
43
    if (ExtraCode[1] != 0)
556
0
      return true; // Unknown modifier.
557
43
558
43
    switch (ExtraCode[0]) {
559
43
    default:
560
0
      return true; // Unknown modifier.
561
43
    case 'w':      // Print W register
562
31
    case 'x':      // Print X register
563
31
      if (MO.isReg())
564
29
        return printAsmMRegister(MO, ExtraCode[0], O);
565
2
      if (MO.isImm() && MO.getImm() == 0) {
566
2
        unsigned Reg = ExtraCode[0] == 'w' ? 
AArch64::WZR1
:
AArch64::XZR1
;
567
2
        O << AArch64InstPrinter::getRegisterName(Reg);
568
2
        return false;
569
2
      }
570
0
      printOperand(MI, OpNum, O);
571
0
      return false;
572
12
    case 'b': // Print B register.
573
12
    case 'h': // Print H register.
574
12
    case 's': // Print S register.
575
12
    case 'd': // Print D register.
576
12
    case 'q': // Print Q register.
577
12
      if (MO.isReg()) {
578
12
        const TargetRegisterClass *RC;
579
12
        switch (ExtraCode[0]) {
580
12
        case 'b':
581
1
          RC = &AArch64::FPR8RegClass;
582
1
          break;
583
12
        case 'h':
584
4
          RC = &AArch64::FPR16RegClass;
585
4
          break;
586
12
        case 's':
587
5
          RC = &AArch64::FPR32RegClass;
588
5
          break;
589
12
        case 'd':
590
1
          RC = &AArch64::FPR64RegClass;
591
1
          break;
592
12
        case 'q':
593
1
          RC = &AArch64::FPR128RegClass;
594
1
          break;
595
12
        default:
596
0
          return true;
597
12
        }
598
12
        return printAsmRegInClass(MO, RC, false /* vector */, O);
599
12
      }
600
0
      printOperand(MI, OpNum, O);
601
0
      return false;
602
43
    }
603
43
  }
604
74
605
74
  // According to ARM, we should emit x and v registers unless we have a
606
74
  // modifier.
607
74
  if (MO.isReg()) {
608
41
    unsigned Reg = MO.getReg();
609
41
610
41
    // If this is a w or x register, print an x register.
611
41
    if (AArch64::GPR32allRegClass.contains(Reg) ||
612
41
        
AArch64::GPR64allRegClass.contains(Reg)28
)
613
33
      return printAsmMRegister(MO, 'x', O);
614
8
615
8
    // If this is a b, h, s, d, or q register, print it as a v register.
616
8
    return printAsmRegInClass(MO, &AArch64::FPR128RegClass, true /* vector */,
617
8
                              O);
618
8
  }
619
33
620
33
  printOperand(MI, OpNum, O);
621
33
  return false;
622
33
}
623
624
bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
625
                                              unsigned OpNum,
626
                                              const char *ExtraCode,
627
3
                                              raw_ostream &O) {
628
3
  if (ExtraCode && 
ExtraCode[0]0
&&
ExtraCode[0] != 'a'0
)
629
0
    return true; // Unknown modifier.
630
3
631
3
  const MachineOperand &MO = MI->getOperand(OpNum);
632
3
  assert(MO.isReg() && "unexpected inline asm memory operand");
633
3
  O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
634
3
  return false;
635
3
}
636
637
void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
638
0
                                               raw_ostream &OS) {
639
0
  unsigned NOps = MI->getNumOperands();
640
0
  assert(NOps == 4);
641
0
  OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
642
0
  // cast away const; DIetc do not take const operands for some reason.
643
0
  OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata())
644
0
            ->getName();
645
0
  OS << " <- ";
646
0
  // Frame address.  Currently handles register +- offset only.
647
0
  assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm());
648
0
  OS << '[';
649
0
  printOperand(MI, 0, OS);
650
0
  OS << '+';
651
0
  printOperand(MI, 1, OS);
652
0
  OS << ']';
653
0
  OS << "+";
654
0
  printOperand(MI, NOps - 2, OS);
655
0
}
656
657
257k
void AArch64AsmPrinter::EmitJumpTableInfo() {
658
257k
  const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
659
257k
  if (!MJTI) 
return255k
;
660
2.23k
661
2.23k
  const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
662
2.23k
  if (JT.empty()) 
return0
;
663
2.23k
664
2.23k
  const Function &F = MF->getFunction();
665
2.23k
  const TargetLoweringObjectFile &TLOF = getObjFileLowering();
666
2.23k
  bool JTInDiffSection =
667
2.23k
      !STI->isTargetCOFF() ||
668
2.23k
      !TLOF.shouldPutJumpTableInFunctionSection(
669
2
          MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
670
2
          F);
671
2.23k
  if (JTInDiffSection) {
672
2.23k
      // Drop it in the readonly section.
673
2.23k
      MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
674
2.23k
      OutStreamer->SwitchSection(ReadOnlySec);
675
2.23k
  }
676
2.23k
677
2.23k
  auto AFI = MF->getInfo<AArch64FunctionInfo>();
678
4.64k
  for (unsigned JTI = 0, e = JT.size(); JTI != e; 
++JTI2.41k
) {
679
2.41k
    const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
680
2.41k
681
2.41k
    // If this jump table was deleted, ignore it.
682
2.41k
    if (JTBBs.empty()) 
continue0
;
683
2.41k
684
2.41k
    unsigned Size = AFI->getJumpTableEntrySize(JTI);
685
2.41k
    EmitAlignment(Log2_32(Size));
686
2.41k
    OutStreamer->EmitLabel(GetJTISymbol(JTI));
687
2.41k
688
2.41k
    for (auto *JTBB : JTBBs)
689
36.2k
      emitJumpTableEntry(MJTI, JTBB, JTI);
690
2.41k
  }
691
2.23k
}
692
693
void AArch64AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
694
                                           const MachineBasicBlock *MBB,
695
36.2k
                                           unsigned JTI) {
696
36.2k
  const MCExpr *Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
697
36.2k
  auto AFI = MF->getInfo<AArch64FunctionInfo>();
698
36.2k
  unsigned Size = AFI->getJumpTableEntrySize(JTI);
699
36.2k
700
36.2k
  if (Size == 4) {
701
142
    // .word LBB - LJTI
702
142
    const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
703
142
    const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
704
142
    Value = MCBinaryExpr::createSub(Value, Base, OutContext);
705
36.0k
  } else {
706
36.0k
    // .byte (LBB - LBB) >> 2 (or .hword)
707
36.0k
    const MCSymbol *BaseSym = AFI->getJumpTableEntryPCRelSymbol(JTI);
708
36.0k
    const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
709
36.0k
    Value = MCBinaryExpr::createSub(Value, Base, OutContext);
710
36.0k
    Value = MCBinaryExpr::createLShr(
711
36.0k
        Value, MCConstantExpr::create(2, OutContext), OutContext);
712
36.0k
  }
713
36.2k
714
36.2k
  OutStreamer->EmitValue(Value, Size);
715
36.2k
}
716
717
/// Small jump tables contain an unsigned byte or half, representing the offset
718
/// from the lowest-addressed possible destination to the desired basic
719
/// block. Since all instructions are 4-byte aligned, this is further compressed
720
/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
721
/// materialize the correct destination we need:
722
///
723
///             adr xDest, .LBB0_0
724
///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
725
///             add xDest, xDest, xScratch, lsl #2
726
void AArch64AsmPrinter::LowerJumpTableDestSmall(llvm::MCStreamer &OutStreamer,
727
2.39k
                                                const llvm::MachineInstr &MI) {
728
2.39k
  unsigned DestReg = MI.getOperand(0).getReg();
729
2.39k
  unsigned ScratchReg = MI.getOperand(1).getReg();
730
2.39k
  unsigned ScratchRegW =
731
2.39k
      STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
732
2.39k
  unsigned TableReg = MI.getOperand(2).getReg();
733
2.39k
  unsigned EntryReg = MI.getOperand(3).getReg();
734
2.39k
  int JTIdx = MI.getOperand(4).getIndex();
735
2.39k
  bool IsByteEntry = MI.getOpcode() == AArch64::JumpTableDest8;
736
2.39k
737
2.39k
  // This has to be first because the compression pass based its reachability
738
2.39k
  // calculations on the start of the JumpTableDest instruction.
739
2.39k
  auto Label =
740
2.39k
      MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
741
2.39k
  EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
742
2.39k
                                  .addReg(DestReg)
743
2.39k
                                  .addExpr(MCSymbolRefExpr::create(
744
2.39k
                                      Label, MF->getContext())));
745
2.39k
746
2.39k
  // Load the number of instruction-steps to offset from the label.
747
2.39k
  unsigned LdrOpcode = IsByteEntry ? 
AArch64::LDRBBroX2.16k
:
AArch64::LDRHHroX234
;
748
2.39k
  EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
749
2.39k
                                  .addReg(ScratchRegW)
750
2.39k
                                  .addReg(TableReg)
751
2.39k
                                  .addReg(EntryReg)
752
2.39k
                                  .addImm(0)
753
2.39k
                                  .addImm(IsByteEntry ? 
02.16k
:
1234
));
754
2.39k
755
2.39k
  // Multiply the steps by 4 and add to the already materialized base label
756
2.39k
  // address.
757
2.39k
  EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
758
2.39k
                                  .addReg(DestReg)
759
2.39k
                                  .addReg(DestReg)
760
2.39k
                                  .addReg(ScratchReg)
761
2.39k
                                  .addImm(2));
762
2.39k
}
763
764
void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
765
15
                                      const MachineInstr &MI) {
766
15
  unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
767
15
768
15
  SM.recordStackMap(MI);
769
15
  assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
770
15
771
15
  // Scan ahead to trim the shadow.
772
15
  const MachineBasicBlock &MBB = *MI.getParent();
773
15
  MachineBasicBlock::const_iterator MII(MI);
774
15
  ++MII;
775
45
  while (NumNOPBytes > 0) {
776
33
    if (MII == MBB.end() || 
MII->isCall()32
||
777
33
        
MII->getOpcode() == AArch64::DBG_VALUE30
||
778
33
        
MII->getOpcode() == TargetOpcode::PATCHPOINT30
||
779
33
        
MII->getOpcode() == TargetOpcode::STACKMAP30
)
780
3
      break;
781
30
    ++MII;
782
30
    NumNOPBytes -= 4;
783
30
  }
784
15
785
15
  // Emit nops.
786
19
  for (unsigned i = 0; i < NumNOPBytes; 
i += 44
)
787
4
    EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
788
15
}
789
790
// Lower a patchpoint of the form:
791
// [<def>], <id>, <numBytes>, <target>, <numArgs>
792
void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
793
47
                                        const MachineInstr &MI) {
794
47
  SM.recordPatchPoint(MI);
795
47
796
47
  PatchPointOpers Opers(&MI);
797
47
798
47
  int64_t CallTarget = Opers.getCallTarget().getImm();
799
47
  unsigned EncodedBytes = 0;
800
47
  if (CallTarget) {
801
33
    assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
802
33
           "High 16 bits of call target should be zero.");
803
33
    unsigned ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
804
33
    EncodedBytes = 16;
805
33
    // Materialize the jump address:
806
33
    EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
807
33
                                    .addReg(ScratchReg)
808
33
                                    .addImm((CallTarget >> 32) & 0xFFFF)
809
33
                                    .addImm(32));
810
33
    EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
811
33
                                    .addReg(ScratchReg)
812
33
                                    .addReg(ScratchReg)
813
33
                                    .addImm((CallTarget >> 16) & 0xFFFF)
814
33
                                    .addImm(16));
815
33
    EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
816
33
                                    .addReg(ScratchReg)
817
33
                                    .addReg(ScratchReg)
818
33
                                    .addImm(CallTarget & 0xFFFF)
819
33
                                    .addImm(0));
820
33
    EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
821
33
  }
822
47
  // Emit padding.
823
47
  unsigned NumBytes = Opers.getNumPatchBytes();
824
47
  assert(NumBytes >= EncodedBytes &&
825
47
         "Patchpoint can't request size less than the length of a call.");
826
47
  assert((NumBytes - EncodedBytes) % 4 == 0 &&
827
47
         "Invalid number of NOP bytes requested!");
828
161
  for (unsigned i = EncodedBytes; i < NumBytes; 
i += 4114
)
829
114
    EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
830
47
}
831
832
6.84k
void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
833
6.84k
  unsigned DestReg = MI.getOperand(0).getReg();
834
6.84k
  if (STI->hasZeroCycleZeroingFP() && 
!STI->hasZeroCycleZeroingFPWorkaround()6.79k
) {
835
51
    // Convert H/S/D register to corresponding Q register
836
51
    if (AArch64::H0 <= DestReg && 
DestReg <= AArch64::H3130
)
837
2
      DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
838
49
    else if (AArch64::S0 <= DestReg && 
DestReg <= AArch64::S3128
)
839
28
      DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
840
21
    else {
841
21
      assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
842
21
      DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
843
21
    }
844
51
    MCInst MOVI;
845
51
    MOVI.setOpcode(AArch64::MOVIv2d_ns);
846
51
    MOVI.addOperand(MCOperand::createReg(DestReg));
847
51
    MOVI.addOperand(MCOperand::createImm(0));
848
51
    EmitToStreamer(*OutStreamer, MOVI);
849
6.79k
  } else {
850
6.79k
    MCInst FMov;
851
6.79k
    switch (MI.getOpcode()) {
852
6.79k
    
default: 0
llvm_unreachable0
("Unexpected opcode");
853
6.79k
    case AArch64::FMOVH0:
854
2
      FMov.setOpcode(AArch64::FMOVWHr);
855
2
      FMov.addOperand(MCOperand::createReg(DestReg));
856
2
      FMov.addOperand(MCOperand::createReg(AArch64::WZR));
857
2
      break;
858
6.79k
    case AArch64::FMOVS0:
859
2.47k
      FMov.setOpcode(AArch64::FMOVWSr);
860
2.47k
      FMov.addOperand(MCOperand::createReg(DestReg));
861
2.47k
      FMov.addOperand(MCOperand::createReg(AArch64::WZR));
862
2.47k
      break;
863
6.79k
    case AArch64::FMOVD0:
864
4.31k
      FMov.setOpcode(AArch64::FMOVXDr);
865
4.31k
      FMov.addOperand(MCOperand::createReg(DestReg));
866
4.31k
      FMov.addOperand(MCOperand::createReg(AArch64::XZR));
867
4.31k
      break;
868
6.79k
    }
869
6.79k
    EmitToStreamer(*OutStreamer, FMov);
870
6.79k
  }
871
6.84k
}
872
873
// Simple pseudo-instructions have their lowering (with expansion to real
874
// instructions) auto-generated.
875
#include "AArch64GenMCPseudoLowering.inc"
876
877
14.6M
void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
878
14.6M
  // Do any auto-generated pseudo lowerings.
879
14.6M
  if (emitPseudoExpansionLowering(*OutStreamer, MI))
880
0
    return;
881
14.6M
882
14.6M
  if (AArch64FI->getLOHRelated().count(MI)) {
883
1.30M
    // Generate a label for LOH related instruction
884
1.30M
    MCSymbol *LOHLabel = createTempSymbol("loh");
885
1.30M
    // Associate the instruction with the label
886
1.30M
    LOHInstToLabel[MI] = LOHLabel;
887
1.30M
    OutStreamer->EmitLabel(LOHLabel);
888
1.30M
  }
889
14.6M
890
14.6M
  AArch64TargetStreamer *TS =
891
14.6M
    static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
892
14.6M
  // Do any manual lowerings.
893
14.6M
  switch (MI->getOpcode()) {
894
14.6M
  default:
895
14.5M
    break;
896
14.6M
    case AArch64::MOVMCSym: {
897
6
    unsigned DestReg = MI->getOperand(0).getReg();
898
6
    const MachineOperand &MO_Sym = MI->getOperand(1);
899
6
    MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
900
6
    MCOperand Hi_MCSym, Lo_MCSym;
901
6
902
6
    Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
903
6
    Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
904
6
905
6
    MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
906
6
    MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
907
6
908
6
    MCInst MovZ;
909
6
    MovZ.setOpcode(AArch64::MOVZXi);
910
6
    MovZ.addOperand(MCOperand::createReg(DestReg));
911
6
    MovZ.addOperand(Hi_MCSym);
912
6
    MovZ.addOperand(MCOperand::createImm(16));
913
6
    EmitToStreamer(*OutStreamer, MovZ);
914
6
915
6
    MCInst MovK;
916
6
    MovK.setOpcode(AArch64::MOVKXi);
917
6
    MovK.addOperand(MCOperand::createReg(DestReg));
918
6
    MovK.addOperand(MCOperand::createReg(DestReg));
919
6
    MovK.addOperand(Lo_MCSym);
920
6
    MovK.addOperand(MCOperand::createImm(0));
921
6
    EmitToStreamer(*OutStreamer, MovK);
922
6
    return;
923
14.6M
  }
924
14.6M
  case AArch64::MOVIv2d_ns:
925
2.55k
    // If the target has <rdar://problem/16473581>, lower this
926
2.55k
    // instruction to movi.16b instead.
927
2.55k
    if (STI->hasZeroCycleZeroingFPWorkaround() &&
928
2.55k
        
MI->getOperand(1).getImm() == 02.29k
) {
929
2.14k
      MCInst TmpInst;
930
2.14k
      TmpInst.setOpcode(AArch64::MOVIv16b_ns);
931
2.14k
      TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
932
2.14k
      TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
933
2.14k
      EmitToStreamer(*OutStreamer, TmpInst);
934
2.14k
      return;
935
2.14k
    }
936
413
    break;
937
413
938
413
  case AArch64::DBG_VALUE: {
939
0
    if (isVerbose() && OutStreamer->hasRawTextSupport()) {
940
0
      SmallString<128> TmpStr;
941
0
      raw_svector_ostream OS(TmpStr);
942
0
      PrintDebugValueComment(MI, OS);
943
0
      OutStreamer->EmitRawText(StringRef(OS.str()));
944
0
    }
945
0
    return;
946
413
947
413
  case AArch64::EMITBKEY: {
948
4
      ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
949
4
      if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
950
4
          
ExceptionHandlingType != ExceptionHandling::ARM0
)
951
0
        return;
952
4
953
4
      if (needsCFIMoves() == CFI_M_None)
954
1
        return;
955
3
956
3
      OutStreamer->EmitCFIBKeyFrame();
957
3
      return;
958
3
    }
959
3
  }
960
3
961
3
  // Tail calls use pseudo instructions so they have the proper code-gen
962
3
  // attributes (isCall, isReturn, etc.). We lower them to the real
963
3
  // instruction here.
964
108
  case AArch64::TCRETURNri:
965
108
  case AArch64::TCRETURNriBTI:
966
108
  case AArch64::TCRETURNriALL: {
967
108
    MCInst TmpInst;
968
108
    TmpInst.setOpcode(AArch64::BR);
969
108
    TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
970
108
    EmitToStreamer(*OutStreamer, TmpInst);
971
108
    return;
972
108
  }
973
48.4k
  case AArch64::TCRETURNdi: {
974
48.4k
    MCOperand Dest;
975
48.4k
    MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
976
48.4k
    MCInst TmpInst;
977
48.4k
    TmpInst.setOpcode(AArch64::B);
978
48.4k
    TmpInst.addOperand(Dest);
979
48.4k
    EmitToStreamer(*OutStreamer, TmpInst);
980
48.4k
    return;
981
108
  }
982
108
  case AArch64::TLSDESC_CALLSEQ: {
983
29
    /// lower this to:
984
29
    ///    adrp  x0, :tlsdesc:var
985
29
    ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
986
29
    ///    add   x0, x0, #:tlsdesc_lo12:var
987
29
    ///    .tlsdesccall var
988
29
    ///    blr   x1
989
29
    ///    (TPIDR_EL0 offset now in x0)
990
29
    const MachineOperand &MO_Sym = MI->getOperand(0);
991
29
    MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
992
29
    MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
993
29
    MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
994
29
    MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
995
29
    MCInstLowering.lowerOperand(MO_Sym, Sym);
996
29
    MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
997
29
    MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
998
29
999
29
    MCInst Adrp;
1000
29
    Adrp.setOpcode(AArch64::ADRP);
1001
29
    Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1002
29
    Adrp.addOperand(SymTLSDesc);
1003
29
    EmitToStreamer(*OutStreamer, Adrp);
1004
29
1005
29
    MCInst Ldr;
1006
29
    Ldr.setOpcode(AArch64::LDRXui);
1007
29
    Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1008
29
    Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1009
29
    Ldr.addOperand(SymTLSDescLo12);
1010
29
    Ldr.addOperand(MCOperand::createImm(0));
1011
29
    EmitToStreamer(*OutStreamer, Ldr);
1012
29
1013
29
    MCInst Add;
1014
29
    Add.setOpcode(AArch64::ADDXri);
1015
29
    Add.addOperand(MCOperand::createReg(AArch64::X0));
1016
29
    Add.addOperand(MCOperand::createReg(AArch64::X0));
1017
29
    Add.addOperand(SymTLSDescLo12);
1018
29
    Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1019
29
    EmitToStreamer(*OutStreamer, Add);
1020
29
1021
29
    // Emit a relocation-annotation. This expands to no code, but requests
1022
29
    // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1023
29
    MCInst TLSDescCall;
1024
29
    TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1025
29
    TLSDescCall.addOperand(Sym);
1026
29
    EmitToStreamer(*OutStreamer, TLSDescCall);
1027
29
1028
29
    MCInst Blr;
1029
29
    Blr.setOpcode(AArch64::BLR);
1030
29
    Blr.addOperand(MCOperand::createReg(AArch64::X1));
1031
29
    EmitToStreamer(*OutStreamer, Blr);
1032
29
1033
29
    return;
1034
108
  }
1035
108
1036
108
  case AArch64::JumpTableDest32: {
1037
17
    // We want:
1038
17
    //     ldrsw xScratch, [xTable, xEntry, lsl #2]
1039
17
    //     add xDest, xTable, xScratch
1040
17
    unsigned DestReg = MI->getOperand(0).getReg(),
1041
17
             ScratchReg = MI->getOperand(1).getReg(),
1042
17
             TableReg = MI->getOperand(2).getReg(),
1043
17
             EntryReg = MI->getOperand(3).getReg();
1044
17
    EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
1045
17
                                     .addReg(ScratchReg)
1046
17
                                     .addReg(TableReg)
1047
17
                                     .addReg(EntryReg)
1048
17
                                     .addImm(0)
1049
17
                                     .addImm(1));
1050
17
    EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1051
17
                                     .addReg(DestReg)
1052
17
                                     .addReg(TableReg)
1053
17
                                     .addReg(ScratchReg)
1054
17
                                     .addImm(0));
1055
17
    return;
1056
108
  }
1057
2.39k
  case AArch64::JumpTableDest16:
1058
2.39k
  case AArch64::JumpTableDest8:
1059
2.39k
    LowerJumpTableDestSmall(*OutStreamer, *MI);
1060
2.39k
    return;
1061
2.39k
1062
6.84k
  case AArch64::FMOVH0:
1063
6.84k
  case AArch64::FMOVS0:
1064
6.84k
  case AArch64::FMOVD0:
1065
6.84k
    EmitFMov0(*MI);
1066
6.84k
    return;
1067
6.84k
1068
6.84k
  case TargetOpcode::STACKMAP:
1069
15
    return LowerSTACKMAP(*OutStreamer, SM, *MI);
1070
6.84k
1071
6.84k
  case TargetOpcode::PATCHPOINT:
1072
47
    return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1073
6.84k
1074
6.84k
  case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1075
3
    LowerPATCHABLE_FUNCTION_ENTER(*MI);
1076
3
    return;
1077
6.84k
1078
6.84k
  case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1079
3
    LowerPATCHABLE_FUNCTION_EXIT(*MI);
1080
3
    return;
1081
6.84k
1082
6.84k
  case TargetOpcode::PATCHABLE_TAIL_CALL:
1083
0
    LowerPATCHABLE_TAIL_CALL(*MI);
1084
0
    return;
1085
6.84k
1086
6.84k
  case AArch64::HWASAN_CHECK_MEMACCESS:
1087
2
    LowerHWASAN_CHECK_MEMACCESS(*MI);
1088
2
    return;
1089
6.84k
1090
6.84k
  case AArch64::SEH_StackAlloc:
1091
93
    TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1092
93
    return;
1093
6.84k
1094
6.84k
  case AArch64::SEH_SaveFPLR:
1095
66
    TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1096
66
    return;
1097
6.84k
1098
6.84k
  case AArch64::SEH_SaveFPLR_X:
1099
34
    assert(MI->getOperand(0).getImm() < 0 &&
1100
34
           "Pre increment SEH opcode must have a negative offset");
1101
34
    TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1102
34
    return;
1103
6.84k
1104
6.84k
  case AArch64::SEH_SaveReg:
1105
58
    TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1106
58
                               MI->getOperand(1).getImm());
1107
58
    return;
1108
6.84k
1109
6.84k
  case AArch64::SEH_SaveReg_X:
1110
65
    assert(MI->getOperand(1).getImm() < 0 &&
1111
65
           "Pre increment SEH opcode must have a negative offset");
1112
65
    TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1113
65
                    -MI->getOperand(1).getImm());
1114
65
    return;
1115
6.84k
1116
6.84k
  case AArch64::SEH_SaveRegP:
1117
85
    assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1118
85
            "Non-consecutive registers not allowed for save_regp");
1119
85
    TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1120
85
                                MI->getOperand(2).getImm());
1121
85
    return;
1122
6.84k
1123
6.84k
  case AArch64::SEH_SaveRegP_X:
1124
4
    assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1125
4
            "Non-consecutive registers not allowed for save_regp_x");
1126
4
    assert(MI->getOperand(2).getImm() < 0 &&
1127
4
           "Pre increment SEH opcode must have a negative offset");
1128
4
    TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1129
4
                                 -MI->getOperand(2).getImm());
1130
4
    return;
1131
6.84k
1132
6.84k
  case AArch64::SEH_SaveFReg:
1133
4
    TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1134
4
                                MI->getOperand(1).getImm());
1135
4
    return;
1136
6.84k
1137
6.84k
  case AArch64::SEH_SaveFReg_X:
1138
2
    assert(MI->getOperand(1).getImm() < 0 &&
1139
2
           "Pre increment SEH opcode must have a negative offset");
1140
2
    TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1141
2
                                 -MI->getOperand(1).getImm());
1142
2
    return;
1143
6.84k
1144
6.84k
  case AArch64::SEH_SaveFRegP:
1145
10
    assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1146
10
            "Non-consecutive registers not allowed for save_regp");
1147
10
    TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1148
10
                                 MI->getOperand(2).getImm());
1149
10
    return;
1150
6.84k
1151
6.84k
  case AArch64::SEH_SaveFRegP_X:
1152
8
    assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1153
8
            "Non-consecutive registers not allowed for save_regp_x");
1154
8
    assert(MI->getOperand(2).getImm() < 0 &&
1155
8
           "Pre increment SEH opcode must have a negative offset");
1156
8
    TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1157
8
                                  -MI->getOperand(2).getImm());
1158
8
    return;
1159
6.84k
1160
6.84k
  case AArch64::SEH_SetFP:
1161
15
    TS->EmitARM64WinCFISetFP();
1162
15
    return;
1163
6.84k
1164
6.84k
  case AArch64::SEH_AddFP:
1165
27
    TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1166
27
    return;
1167
6.84k
1168
6.84k
  case AArch64::SEH_Nop:
1169
18
    TS->EmitARM64WinCFINop();
1170
18
    return;
1171
6.84k
1172
6.84k
  case AArch64::SEH_PrologEnd:
1173
91
    TS->EmitARM64WinCFIPrologEnd();
1174
91
    return;
1175
6.84k
1176
6.84k
  case AArch64::SEH_EpilogStart:
1177
158
    TS->EmitARM64WinCFIEpilogStart();
1178
158
    return;
1179
6.84k
1180
6.84k
  case AArch64::SEH_EpilogEnd:
1181
158
    TS->EmitARM64WinCFIEpilogEnd();
1182
158
    return;
1183
14.5M
  }
1184
14.5M
1185
14.5M
  // Finally, do the automated lowerings for everything else.
1186
14.5M
  MCInst TmpInst;
1187
14.5M
  MCInstLowering.Lower(MI, TmpInst);
1188
14.5M
  EmitToStreamer(*OutStreamer, TmpInst);
1189
14.5M
}
1190
1191
// Force static initialization.
1192
78.9k
extern "C" void LLVMInitializeAArch64AsmPrinter() {
1193
78.9k
  RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1194
78.9k
  RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1195
78.9k
  RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1196
78.9k
}