Coverage Report

Created: 2022-07-16 07:03

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/utils/TableGen/NeonEmitter.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- NeonEmitter.cpp - Generate arm_neon.h for use with clang -*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This tablegen backend is responsible for emitting arm_neon.h, which includes
10
// a declaration and definition of each function specified by the ARM NEON
11
// compiler interface.  See ARM document DUI0348B.
12
//
13
// Each NEON instruction is implemented in terms of 1 or more functions which
14
// are suffixed with the element type of the input vectors.  Functions may be
15
// implemented in terms of generic vector operations such as +, *, -, etc. or
16
// by calling a __builtin_-prefixed function which will be handled by clang's
17
// CodeGen library.
18
//
19
// Additional validation code can be generated by this file when runHeader() is
20
// called, rather than the normal run() entry point.
21
//
22
// See also the documentation in include/clang/Basic/arm_neon.td.
23
//
24
//===----------------------------------------------------------------------===//
25
26
#include "TableGenBackends.h"
27
#include "llvm/ADT/ArrayRef.h"
28
#include "llvm/ADT/DenseMap.h"
29
#include "llvm/ADT/None.h"
30
#include "llvm/ADT/Optional.h"
31
#include "llvm/ADT/STLExtras.h"
32
#include "llvm/ADT/SmallVector.h"
33
#include "llvm/ADT/StringExtras.h"
34
#include "llvm/ADT/StringRef.h"
35
#include "llvm/Support/Casting.h"
36
#include "llvm/Support/ErrorHandling.h"
37
#include "llvm/Support/raw_ostream.h"
38
#include "llvm/TableGen/Error.h"
39
#include "llvm/TableGen/Record.h"
40
#include "llvm/TableGen/SetTheory.h"
41
#include <algorithm>
42
#include <cassert>
43
#include <cctype>
44
#include <cstddef>
45
#include <cstdint>
46
#include <deque>
47
#include <map>
48
#include <set>
49
#include <sstream>
50
#include <string>
51
#include <utility>
52
#include <vector>
53
54
using namespace llvm;
55
56
namespace {
57
58
// While globals are generally bad, this one allows us to perform assertions
59
// liberally and somehow still trace them back to the def they indirectly
60
// came from.
61
static Record *CurrentRecord = nullptr;
62
0
static void assert_with_loc(bool Assertion, const std::string &Str) {
63
0
  if (!Assertion) {
64
0
    if (CurrentRecord)
65
0
      PrintFatalError(CurrentRecord->getLoc(), Str);
66
0
    else
67
0
      PrintFatalError(Str);
68
0
  }
69
0
}
70
71
enum ClassKind {
72
  ClassNone,
73
  ClassI,     // generic integer instruction, e.g., "i8" suffix
74
  ClassS,     // signed/unsigned/poly, e.g., "s8", "u8" or "p8" suffix
75
  ClassW,     // width-specific instruction, e.g., "8" suffix
76
  ClassB,     // bitcast arguments with enum argument to specify type
77
  ClassL,     // Logical instructions which are op instructions
78
              // but we need to not emit any suffix for in our
79
              // tests.
80
  ClassNoTest // Instructions which we do not test since they are
81
              // not TRUE instructions.
82
};
83
84
/// NeonTypeFlags - Flags to identify the types for overloaded Neon
85
/// builtins.  These must be kept in sync with the flags in
86
/// include/clang/Basic/TargetBuiltins.h.
87
namespace NeonTypeFlags {
88
89
enum { EltTypeMask = 0xf, UnsignedFlag = 0x10, QuadFlag = 0x20 };
90
91
enum EltType {
92
  Int8,
93
  Int16,
94
  Int32,
95
  Int64,
96
  Poly8,
97
  Poly16,
98
  Poly64,
99
  Poly128,
100
  Float16,
101
  Float32,
102
  Float64,
103
  BFloat16
104
};
105
106
} // end namespace NeonTypeFlags
107
108
class NeonEmitter;
109
110
//===----------------------------------------------------------------------===//
111
// TypeSpec
112
//===----------------------------------------------------------------------===//
113
114
/// A TypeSpec is just a simple wrapper around a string, but gets its own type
115
/// for strong typing purposes.
116
///
117
/// A TypeSpec can be used to create a type.
118
class TypeSpec : public std::string {
119
public:
120
0
  static std::vector<TypeSpec> fromTypeSpecs(StringRef Str) {
121
0
    std::vector<TypeSpec> Ret;
122
0
    TypeSpec Acc;
123
0
    for (char I : Str.str()) {
124
0
      if (islower(I)) {
125
0
        Acc.push_back(I);
126
0
        Ret.push_back(TypeSpec(Acc));
127
0
        Acc.clear();
128
0
      } else {
129
0
        Acc.push_back(I);
130
0
      }
131
0
    }
132
0
    return Ret;
133
0
  }
134
};
135
136
//===----------------------------------------------------------------------===//
137
// Type
138
//===----------------------------------------------------------------------===//
139
140
/// A Type. Not much more to say here.
141
class Type {
142
private:
143
  TypeSpec TS;
144
145
  enum TypeKind {
146
    Void,
147
    Float,
148
    SInt,
149
    UInt,
150
    Poly,
151
    BFloat16,
152
  };
153
  TypeKind Kind;
154
  bool Immediate, Constant, Pointer;
155
  // ScalarForMangling and NoManglingQ are really not suited to live here as
156
  // they are not related to the type. But they live in the TypeSpec (not the
157
  // prototype), so this is really the only place to store them.
158
  bool ScalarForMangling, NoManglingQ;
159
  unsigned Bitwidth, ElementBitwidth, NumVectors;
160
161
public:
162
  Type()
163
      : Kind(Void), Immediate(false), Constant(false),
164
        Pointer(false), ScalarForMangling(false), NoManglingQ(false),
165
0
        Bitwidth(0), ElementBitwidth(0), NumVectors(0) {}
166
167
  Type(TypeSpec TS, StringRef CharMods)
168
      : TS(std::move(TS)), Kind(Void), Immediate(false),
169
        Constant(false), Pointer(false), ScalarForMangling(false),
170
0
        NoManglingQ(false), Bitwidth(0), ElementBitwidth(0), NumVectors(0) {
171
0
    applyModifiers(CharMods);
172
0
  }
173
174
  /// Returns a type representing "void".
175
0
  static Type getVoid() { return Type(); }
176
177
0
  bool operator==(const Type &Other) const { return str() == Other.str(); }
178
0
  bool operator!=(const Type &Other) const { return !operator==(Other); }
179
180
  //
181
  // Query functions
182
  //
183
0
  bool isScalarForMangling() const { return ScalarForMangling; }
184
0
  bool noManglingQ() const { return NoManglingQ; }
185
186
0
  bool isPointer() const { return Pointer; }
187
0
  bool isValue() const { return !isVoid() && !isPointer(); }
188
0
  bool isScalar() const { return isValue() && NumVectors == 0; }
189
0
  bool isVector() const { return isValue() && NumVectors > 0; }
190
0
  bool isConstPointer() const { return Constant; }
191
0
  bool isFloating() const { return Kind == Float; }
192
0
  bool isInteger() const { return Kind == SInt || Kind == UInt; }
193
0
  bool isPoly() const { return Kind == Poly; }
194
0
  bool isSigned() const { return Kind == SInt; }
195
0
  bool isImmediate() const { return Immediate; }
196
0
  bool isFloat() const { return isFloating() && ElementBitwidth == 32; }
197
0
  bool isDouble() const { return isFloating() && ElementBitwidth == 64; }
198
0
  bool isHalf() const { return isFloating() && ElementBitwidth == 16; }
199
0
  bool isChar() const { return ElementBitwidth == 8; }
200
0
  bool isShort() const { return isInteger() && ElementBitwidth == 16; }
201
0
  bool isInt() const { return isInteger() && ElementBitwidth == 32; }
202
0
  bool isLong() const { return isInteger() && ElementBitwidth == 64; }
203
0
  bool isVoid() const { return Kind == Void; }
204
0
  bool isBFloat16() const { return Kind == BFloat16; }
205
0
  unsigned getNumElements() const { return Bitwidth / ElementBitwidth; }
206
0
  unsigned getSizeInBits() const { return Bitwidth; }
207
0
  unsigned getElementSizeInBits() const { return ElementBitwidth; }
208
0
  unsigned getNumVectors() const { return NumVectors; }
209
210
  //
211
  // Mutator functions
212
  //
213
0
  void makeUnsigned() {
214
0
    assert(!isVoid() && "not a potentially signed type");
215
0
    Kind = UInt;
216
0
  }
217
0
  void makeSigned() {
218
0
    assert(!isVoid() && "not a potentially signed type");
219
0
    Kind = SInt;
220
0
  }
221
222
0
  void makeInteger(unsigned ElemWidth, bool Sign) {
223
0
    assert(!isVoid() && "converting void to int probably not useful");
224
0
    Kind = Sign ? SInt : UInt;
225
0
    Immediate = false;
226
0
    ElementBitwidth = ElemWidth;
227
0
  }
228
229
0
  void makeImmediate(unsigned ElemWidth) {
230
0
    Kind = SInt;
231
0
    Immediate = true;
232
0
    ElementBitwidth = ElemWidth;
233
0
  }
234
235
0
  void makeScalar() {
236
0
    Bitwidth = ElementBitwidth;
237
0
    NumVectors = 0;
238
0
  }
239
240
0
  void makeOneVector() {
241
0
    assert(isVector());
242
0
    NumVectors = 1;
243
0
  }
244
245
0
  void make32BitElement() {
246
0
    assert_with_loc(Bitwidth > 32, "Not enough bits to make it 32!");
247
0
    ElementBitwidth = 32;
248
0
  }
249
250
0
  void doubleLanes() {
251
0
    assert_with_loc(Bitwidth != 128, "Can't get bigger than 128!");
252
0
    Bitwidth = 128;
253
0
  }
254
255
0
  void halveLanes() {
256
0
    assert_with_loc(Bitwidth != 64, "Can't get smaller than 64!");
257
0
    Bitwidth = 64;
258
0
  }
259
260
  /// Return the C string representation of a type, which is the typename
261
  /// defined in stdint.h or arm_neon.h.
262
  std::string str() const;
263
264
  /// Return the string representation of a type, which is an encoded
265
  /// string for passing to the BUILTIN() macro in Builtins.def.
266
  std::string builtin_str() const;
267
268
  /// Return the value in NeonTypeFlags for this type.
269
  unsigned getNeonEnum() const;
270
271
  /// Parse a type from a stdint.h or arm_neon.h typedef name,
272
  /// for example uint32x2_t or int64_t.
273
  static Type fromTypedefName(StringRef Name);
274
275
private:
276
  /// Creates the type based on the typespec string in TS.
277
  /// Sets "Quad" to true if the "Q" or "H" modifiers were
278
  /// seen. This is needed by applyModifier as some modifiers
279
  /// only take effect if the type size was changed by "Q" or "H".
280
  void applyTypespec(bool &Quad);
281
  /// Applies prototype modifiers to the type.
282
  void applyModifiers(StringRef Mods);
283
};
284
285
//===----------------------------------------------------------------------===//
286
// Variable
287
//===----------------------------------------------------------------------===//
288
289
/// A variable is a simple class that just has a type and a name.
290
class Variable {
291
  Type T;
292
  std::string N;
293
294
public:
295
0
  Variable() : T(Type::getVoid()) {}
296
0
  Variable(Type T, std::string N) : T(std::move(T)), N(std::move(N)) {}
297
298
0
  Type getType() const { return T; }
299
0
  std::string getName() const { return "__" + N; }
300
};
301
302
//===----------------------------------------------------------------------===//
303
// Intrinsic
304
//===----------------------------------------------------------------------===//
305
306
/// The main grunt class. This represents an instantiation of an intrinsic with
307
/// a particular typespec and prototype.
308
class Intrinsic {
309
  /// The Record this intrinsic was created from.
310
  Record *R;
311
  /// The unmangled name.
312
  std::string Name;
313
  /// The input and output typespecs. InTS == OutTS except when
314
  /// CartesianProductWith is non-empty - this is the case for vreinterpret.
315
  TypeSpec OutTS, InTS;
316
  /// The base class kind. Most intrinsics use ClassS, which has full type
317
  /// info for integers (s32/u32). Some use ClassI, which doesn't care about
318
  /// signedness (i32), while some (ClassB) have no type at all, only a width
319
  /// (32).
320
  ClassKind CK;
321
  /// The list of DAGs for the body. May be empty, in which case we should
322
  /// emit a builtin call.
323
  ListInit *Body;
324
  /// The architectural #ifdef guard.
325
  std::string Guard;
326
  /// Set if the Unavailable bit is 1. This means we don't generate a body,
327
  /// just an "unavailable" attribute on a declaration.
328
  bool IsUnavailable;
329
  /// Is this intrinsic safe for big-endian? or does it need its arguments
330
  /// reversing?
331
  bool BigEndianSafe;
332
333
  /// The types of return value [0] and parameters [1..].
334
  std::vector<Type> Types;
335
  /// The index of the key type passed to CGBuiltin.cpp for polymorphic calls.
336
  int PolymorphicKeyType;
337
  /// The local variables defined.
338
  std::map<std::string, Variable> Variables;
339
  /// NeededEarly - set if any other intrinsic depends on this intrinsic.
340
  bool NeededEarly;
341
  /// UseMacro - set if we should implement using a macro or unset for a
342
  ///            function.
343
  bool UseMacro;
344
  /// The set of intrinsics that this intrinsic uses/requires.
345
  std::set<Intrinsic *> Dependencies;
346
  /// The "base type", which is Type('d', OutTS). InBaseType is only
347
  /// different if CartesianProductWith is non-empty (for vreinterpret).
348
  Type BaseType, InBaseType;
349
  /// The return variable.
350
  Variable RetVar;
351
  /// A postfix to apply to every variable. Defaults to "".
352
  std::string VariablePostfix;
353
354
  NeonEmitter &Emitter;
355
  std::stringstream OS;
356
357
0
  bool isBigEndianSafe() const {
358
0
    if (BigEndianSafe)
359
0
      return true;
360
361
0
    for (const auto &T : Types){
362
0
      if (T.isVector() && T.getNumElements() > 1)
363
0
        return false;
364
0
    }
365
0
    return true;
366
0
  }
367
368
public:
369
  Intrinsic(Record *R, StringRef Name, StringRef Proto, TypeSpec OutTS,
370
            TypeSpec InTS, ClassKind CK, ListInit *Body, NeonEmitter &Emitter,
371
            StringRef Guard, bool IsUnavailable, bool BigEndianSafe)
372
      : R(R), Name(Name.str()), OutTS(OutTS), InTS(InTS), CK(CK), Body(Body),
373
        Guard(Guard.str()), IsUnavailable(IsUnavailable),
374
        BigEndianSafe(BigEndianSafe), PolymorphicKeyType(0), NeededEarly(false),
375
        UseMacro(false), BaseType(OutTS, "."), InBaseType(InTS, "."),
376
0
        Emitter(Emitter) {
377
    // Modify the TypeSpec per-argument to get a concrete Type, and create
378
    // known variables for each.
379
    // Types[0] is the return value.
380
0
    unsigned Pos = 0;
381
0
    Types.emplace_back(OutTS, getNextModifiers(Proto, Pos));
382
0
    StringRef Mods = getNextModifiers(Proto, Pos);
383
0
    while (!Mods.empty()) {
384
0
      Types.emplace_back(InTS, Mods);
385
0
      if (Mods.contains('!'))
386
0
        PolymorphicKeyType = Types.size() - 1;
387
388
0
      Mods = getNextModifiers(Proto, Pos);
389
0
    }
390
391
0
    for (auto Type : Types) {
392
      // If this builtin takes an immediate argument, we need to #define it rather
393
      // than use a standard declaration, so that SemaChecking can range check
394
      // the immediate passed by the user.
395
396
      // Pointer arguments need to use macros to avoid hiding aligned attributes
397
      // from the pointer type.
398
399
      // It is not permitted to pass or return an __fp16 by value, so intrinsics
400
      // taking a scalar float16_t must be implemented as macros.
401
0
      if (Type.isImmediate() || Type.isPointer() ||
402
0
          (Type.isScalar() && Type.isHalf()))
403
0
        UseMacro = true;
404
0
    }
405
0
  }
406
407
  /// Get the Record that this intrinsic is based off.
408
0
  Record *getRecord() const { return R; }
409
  /// Get the set of Intrinsics that this intrinsic calls.
410
  /// this is the set of immediate dependencies, NOT the
411
  /// transitive closure.
412
0
  const std::set<Intrinsic *> &getDependencies() const { return Dependencies; }
413
  /// Get the architectural guard string (#ifdef).
414
0
  std::string getGuard() const { return Guard; }
415
  /// Get the non-mangled name.
416
0
  std::string getName() const { return Name; }
417
418
  /// Return true if the intrinsic takes an immediate operand.
419
0
  bool hasImmediate() const {
420
0
    return llvm::any_of(Types, [](const Type &T) { return T.isImmediate(); });
421
0
  }
422
423
  /// Return the parameter index of the immediate operand.
424
0
  unsigned getImmediateIdx() const {
425
0
    for (unsigned Idx = 0; Idx < Types.size(); ++Idx)
426
0
      if (Types[Idx].isImmediate())
427
0
        return Idx - 1;
428
0
    llvm_unreachable("Intrinsic has no immediate");
429
0
  }
430
431
432
0
  unsigned getNumParams() const { return Types.size() - 1; }
433
0
  Type getReturnType() const { return Types[0]; }
434
0
  Type getParamType(unsigned I) const { return Types[I + 1]; }
435
0
  Type getBaseType() const { return BaseType; }
436
0
  Type getPolymorphicKeyType() const { return Types[PolymorphicKeyType]; }
437
438
  /// Return true if the prototype has a scalar argument.
439
  bool protoHasScalar() const;
440
441
  /// Return the index that parameter PIndex will sit at
442
  /// in a generated function call. This is often just PIndex,
443
  /// but may not be as things such as multiple-vector operands
444
  /// and sret parameters need to be taken into accont.
445
0
  unsigned getGeneratedParamIdx(unsigned PIndex) {
446
0
    unsigned Idx = 0;
447
0
    if (getReturnType().getNumVectors() > 1)
448
      // Multiple vectors are passed as sret.
449
0
      ++Idx;
450
451
0
    for (unsigned I = 0; I < PIndex; ++I)
452
0
      Idx += std::max(1U, getParamType(I).getNumVectors());
453
454
0
    return Idx;
455
0
  }
456
457
0
  bool hasBody() const { return Body && !Body->getValues().empty(); }
458
459
0
  void setNeededEarly() { NeededEarly = true; }
460
461
0
  bool operator<(const Intrinsic &Other) const {
462
    // Sort lexicographically on a two-tuple (Guard, Name)
463
0
    if (Guard != Other.Guard)
464
0
      return Guard < Other.Guard;
465
0
    return Name < Other.Name;
466
0
  }
467
468
0
  ClassKind getClassKind(bool UseClassBIfScalar = false) {
469
0
    if (UseClassBIfScalar && !protoHasScalar())
470
0
      return ClassB;
471
0
    return CK;
472
0
  }
473
474
  /// Return the name, mangled with type information.
475
  /// If ForceClassS is true, use ClassS (u32/s32) instead
476
  /// of the intrinsic's own type class.
477
  std::string getMangledName(bool ForceClassS = false) const;
478
  /// Return the type code for a builtin function call.
479
  std::string getInstTypeCode(Type T, ClassKind CK) const;
480
  /// Return the type string for a BUILTIN() macro in Builtins.def.
481
  std::string getBuiltinTypeStr();
482
483
  /// Generate the intrinsic, returning code.
484
  std::string generate();
485
  /// Perform type checking and populate the dependency graph, but
486
  /// don't generate code yet.
487
  void indexBody();
488
489
private:
490
  StringRef getNextModifiers(StringRef Proto, unsigned &Pos) const;
491
492
  std::string mangleName(std::string Name, ClassKind CK) const;
493
494
  void initVariables();
495
  std::string replaceParamsIn(std::string S);
496
497
  void emitBodyAsBuiltinCall();
498
499
  void generateImpl(bool ReverseArguments,
500
                    StringRef NamePrefix, StringRef CallPrefix);
501
  void emitReturn();
502
  void emitBody(StringRef CallPrefix);
503
  void emitShadowedArgs();
504
  void emitArgumentReversal();
505
  void emitReturnVarDecl();
506
  void emitReturnReversal();
507
  void emitReverseVariable(Variable &Dest, Variable &Src);
508
  void emitNewLine();
509
  void emitClosingBrace();
510
  void emitOpeningBrace();
511
  void emitPrototype(StringRef NamePrefix);
512
513
  class DagEmitter {
514
    Intrinsic &Intr;
515
    StringRef CallPrefix;
516
517
  public:
518
    DagEmitter(Intrinsic &Intr, StringRef CallPrefix) :
519
0
      Intr(Intr), CallPrefix(CallPrefix) {
520
0
    }
521
    std::pair<Type, std::string> emitDagArg(Init *Arg, std::string ArgName);
522
    std::pair<Type, std::string> emitDagSaveTemp(DagInit *DI);
523
    std::pair<Type, std::string> emitDagSplat(DagInit *DI);
524
    std::pair<Type, std::string> emitDagDup(DagInit *DI);
525
    std::pair<Type, std::string> emitDagDupTyped(DagInit *DI);
526
    std::pair<Type, std::string> emitDagShuffle(DagInit *DI);
527
    std::pair<Type, std::string> emitDagCast(DagInit *DI, bool IsBitCast);
528
    std::pair<Type, std::string> emitDagCall(DagInit *DI,
529
                                             bool MatchMangledName);
530
    std::pair<Type, std::string> emitDagNameReplace(DagInit *DI);
531
    std::pair<Type, std::string> emitDagLiteral(DagInit *DI);
532
    std::pair<Type, std::string> emitDagOp(DagInit *DI);
533
    std::pair<Type, std::string> emitDag(DagInit *DI);
534
  };
535
};
536
537
//===----------------------------------------------------------------------===//
538
// NeonEmitter
539
//===----------------------------------------------------------------------===//
540
541
class NeonEmitter {
542
  RecordKeeper &Records;
543
  DenseMap<Record *, ClassKind> ClassMap;
544
  std::map<std::string, std::deque<Intrinsic>> IntrinsicMap;
545
  unsigned UniqueNumber;
546
547
  void createIntrinsic(Record *R, SmallVectorImpl<Intrinsic *> &Out);
548
  void genBuiltinsDef(raw_ostream &OS, SmallVectorImpl<Intrinsic *> &Defs);
549
  void genOverloadTypeCheckCode(raw_ostream &OS,
550
                                SmallVectorImpl<Intrinsic *> &Defs);
551
  void genIntrinsicRangeCheckCode(raw_ostream &OS,
552
                                  SmallVectorImpl<Intrinsic *> &Defs);
553
554
public:
555
  /// Called by Intrinsic - this attempts to get an intrinsic that takes
556
  /// the given types as arguments.
557
  Intrinsic &getIntrinsic(StringRef Name, ArrayRef<Type> Types,
558
                          Optional<std::string> MangledName);
559
560
  /// Called by Intrinsic - returns a globally-unique number.
561
0
  unsigned getUniqueNumber() { return UniqueNumber++; }
562
563
0
  NeonEmitter(RecordKeeper &R) : Records(R), UniqueNumber(0) {
564
0
    Record *SI = R.getClass("SInst");
565
0
    Record *II = R.getClass("IInst");
566
0
    Record *WI = R.getClass("WInst");
567
0
    Record *SOpI = R.getClass("SOpInst");
568
0
    Record *IOpI = R.getClass("IOpInst");
569
0
    Record *WOpI = R.getClass("WOpInst");
570
0
    Record *LOpI = R.getClass("LOpInst");
571
0
    Record *NoTestOpI = R.getClass("NoTestOpInst");
572
573
0
    ClassMap[SI] = ClassS;
574
0
    ClassMap[II] = ClassI;
575
0
    ClassMap[WI] = ClassW;
576
0
    ClassMap[SOpI] = ClassS;
577
0
    ClassMap[IOpI] = ClassI;
578
0
    ClassMap[WOpI] = ClassW;
579
0
    ClassMap[LOpI] = ClassL;
580
0
    ClassMap[NoTestOpI] = ClassNoTest;
581
0
  }
582
583
  // Emit arm_neon.h.inc
584
  void run(raw_ostream &o);
585
586
  // Emit arm_fp16.h.inc
587
  void runFP16(raw_ostream &o);
588
589
  // Emit arm_bf16.h.inc
590
  void runBF16(raw_ostream &o);
591
592
  // Emit all the __builtin prototypes used in arm_neon.h, arm_fp16.h and
593
  // arm_bf16.h
594
  void runHeader(raw_ostream &o);
595
};
596
597
} // end anonymous namespace
598
599
//===----------------------------------------------------------------------===//
600
// Type implementation
601
//===----------------------------------------------------------------------===//
602
603
0
std::string Type::str() const {
604
0
  if (isVoid())
605
0
    return "void";
606
0
  std::string S;
607
608
0
  if (isInteger() && !isSigned())
609
0
    S += "u";
610
611
0
  if (isPoly())
612
0
    S += "poly";
613
0
  else if (isFloating())
614
0
    S += "float";
615
0
  else if (isBFloat16())
616
0
    S += "bfloat";
617
0
  else
618
0
    S += "int";
619
620
0
  S += utostr(ElementBitwidth);
621
0
  if (isVector())
622
0
    S += "x" + utostr(getNumElements());
623
0
  if (NumVectors > 1)
624
0
    S += "x" + utostr(NumVectors);
625
0
  S += "_t";
626
627
0
  if (Constant)
628
0
    S += " const";
629
0
  if (Pointer)
630
0
    S += " *";
631
632
0
  return S;
633
0
}
634
635
0
std::string Type::builtin_str() const {
636
0
  std::string S;
637
0
  if (isVoid())
638
0
    return "v";
639
640
0
  if (isPointer()) {
641
    // All pointers are void pointers.
642
0
    S = "v";
643
0
    if (isConstPointer())
644
0
      S += "C";
645
0
    S += "*";
646
0
    return S;
647
0
  } else if (isInteger())
648
0
    switch (ElementBitwidth) {
649
0
    case 8: S += "c"; break;
650
0
    case 16: S += "s"; break;
651
0
    case 32: S += "i"; break;
652
0
    case 64: S += "Wi"; break;
653
0
    case 128: S += "LLLi"; break;
654
0
    default: llvm_unreachable("Unhandled case!");
655
0
    }
656
0
  else if (isBFloat16()) {
657
0
    assert(ElementBitwidth == 16 && "BFloat16 can only be 16 bits");
658
0
    S += "y";
659
0
  } else
660
0
    switch (ElementBitwidth) {
661
0
    case 16: S += "h"; break;
662
0
    case 32: S += "f"; break;
663
0
    case 64: S += "d"; break;
664
0
    default: llvm_unreachable("Unhandled case!");
665
0
    }
666
667
  // FIXME: NECESSARY???????????????????????????????????????????????????????????????????????
668
0
  if (isChar() && !isPointer() && isSigned())
669
    // Make chars explicitly signed.
670
0
    S = "S" + S;
671
0
  else if (isInteger() && !isSigned())
672
0
    S = "U" + S;
673
674
  // Constant indices are "int", but have the "constant expression" modifier.
675
0
  if (isImmediate()) {
676
0
    assert(isInteger() && isSigned());
677
0
    S = "I" + S;
678
0
  }
679
680
0
  if (isScalar())
681
0
    return S;
682
683
0
  std::string Ret;
684
0
  for (unsigned I = 0; I < NumVectors; ++I)
685
0
    Ret += "V" + utostr(getNumElements()) + S;
686
687
0
  return Ret;
688
0
}
689
690
0
unsigned Type::getNeonEnum() const {
691
0
  unsigned Addend;
692
0
  switch (ElementBitwidth) {
693
0
  case 8: Addend = 0; break;
694
0
  case 16: Addend = 1; break;
695
0
  case 32: Addend = 2; break;
696
0
  case 64: Addend = 3; break;
697
0
  case 128: Addend = 4; break;
698
0
  default: llvm_unreachable("Unhandled element bitwidth!");
699
0
  }
700
701
0
  unsigned Base = (unsigned)NeonTypeFlags::Int8 + Addend;
702
0
  if (isPoly()) {
703
    // Adjustment needed because Poly32 doesn't exist.
704
0
    if (Addend >= 2)
705
0
      --Addend;
706
0
    Base = (unsigned)NeonTypeFlags::Poly8 + Addend;
707
0
  }
708
0
  if (isFloating()) {
709
0
    assert(Addend != 0 && "Float8 doesn't exist!");
710
0
    Base = (unsigned)NeonTypeFlags::Float16 + (Addend - 1);
711
0
  }
712
713
0
  if (isBFloat16()) {
714
0
    assert(Addend == 1 && "BFloat16 is only 16 bit");
715
0
    Base = (unsigned)NeonTypeFlags::BFloat16;
716
0
  }
717
718
0
  if (Bitwidth == 128)
719
0
    Base |= (unsigned)NeonTypeFlags::QuadFlag;
720
0
  if (isInteger() && !isSigned())
721
0
    Base |= (unsigned)NeonTypeFlags::UnsignedFlag;
722
723
0
  return Base;
724
0
}
725
726
0
Type Type::fromTypedefName(StringRef Name) {
727
0
  Type T;
728
0
  T.Kind = SInt;
729
730
0
  if (Name.front() == 'u') {
731
0
    T.Kind = UInt;
732
0
    Name = Name.drop_front();
733
0
  }
734
735
0
  if (Name.startswith("float")) {
736
0
    T.Kind = Float;
737
0
    Name = Name.drop_front(5);
738
0
  } else if (Name.startswith("poly")) {
739
0
    T.Kind = Poly;
740
0
    Name = Name.drop_front(4);
741
0
  } else if (Name.startswith("bfloat")) {
742
0
    T.Kind = BFloat16;
743
0
    Name = Name.drop_front(6);
744
0
  } else {
745
0
    assert(Name.startswith("int"));
746
0
    Name = Name.drop_front(3);
747
0
  }
748
749
0
  unsigned I = 0;
750
0
  for (I = 0; I < Name.size(); ++I) {
751
0
    if (!isdigit(Name[I]))
752
0
      break;
753
0
  }
754
0
  Name.substr(0, I).getAsInteger(10, T.ElementBitwidth);
755
0
  Name = Name.drop_front(I);
756
757
0
  T.Bitwidth = T.ElementBitwidth;
758
0
  T.NumVectors = 1;
759
760
0
  if (Name.front() == 'x') {
761
0
    Name = Name.drop_front();
762
0
    unsigned I = 0;
763
0
    for (I = 0; I < Name.size(); ++I) {
764
0
      if (!isdigit(Name[I]))
765
0
        break;
766
0
    }
767
0
    unsigned NumLanes;
768
0
    Name.substr(0, I).getAsInteger(10, NumLanes);
769
0
    Name = Name.drop_front(I);
770
0
    T.Bitwidth = T.ElementBitwidth * NumLanes;
771
0
  } else {
772
    // Was scalar.
773
0
    T.NumVectors = 0;
774
0
  }
775
0
  if (Name.front() == 'x') {
776
0
    Name = Name.drop_front();
777
0
    unsigned I = 0;
778
0
    for (I = 0; I < Name.size(); ++I) {
779
0
      if (!isdigit(Name[I]))
780
0
        break;
781
0
    }
782
0
    Name.substr(0, I).getAsInteger(10, T.NumVectors);
783
0
    Name = Name.drop_front(I);
784
0
  }
785
786
0
  assert(Name.startswith("_t") && "Malformed typedef!");
787
0
  return T;
788
0
}
789
790
0
void Type::applyTypespec(bool &Quad) {
791
0
  std::string S = TS;
792
0
  ScalarForMangling = false;
793
0
  Kind = SInt;
794
0
  ElementBitwidth = ~0U;
795
0
  NumVectors = 1;
796
797
0
  for (char I : S) {
798
0
    switch (I) {
799
0
    case 'S':
800
0
      ScalarForMangling = true;
801
0
      break;
802
0
    case 'H':
803
0
      NoManglingQ = true;
804
0
      Quad = true;
805
0
      break;
806
0
    case 'Q':
807
0
      Quad = true;
808
0
      break;
809
0
    case 'P':
810
0
      Kind = Poly;
811
0
      break;
812
0
    case 'U':
813
0
      Kind = UInt;
814
0
      break;
815
0
    case 'c':
816
0
      ElementBitwidth = 8;
817
0
      break;
818
0
    case 'h':
819
0
      Kind = Float;
820
0
      LLVM_FALLTHROUGH;
821
0
    case 's':
822
0
      ElementBitwidth = 16;
823
0
      break;
824
0
    case 'f':
825
0
      Kind = Float;
826
0
      LLVM_FALLTHROUGH;
827
0
    case 'i':
828
0
      ElementBitwidth = 32;
829
0
      break;
830
0
    case 'd':
831
0
      Kind = Float;
832
0
      LLVM_FALLTHROUGH;
833
0
    case 'l':
834
0
      ElementBitwidth = 64;
835
0
      break;
836
0
    case 'k':
837
0
      ElementBitwidth = 128;
838
      // Poly doesn't have a 128x1 type.
839
0
      if (isPoly())
840
0
        NumVectors = 0;
841
0
      break;
842
0
    case 'b':
843
0
      Kind = BFloat16;
844
0
      ElementBitwidth = 16;
845
0
      break;
846
0
    default:
847
0
      llvm_unreachable("Unhandled type code!");
848
0
    }
849
0
  }
850
0
  assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
851
852
0
  Bitwidth = Quad ? 128 : 64;
853
0
}
854
855
0
void Type::applyModifiers(StringRef Mods) {
856
0
  bool AppliedQuad = false;
857
0
  applyTypespec(AppliedQuad);
858
859
0
  for (char Mod : Mods) {
860
0
    switch (Mod) {
861
0
    case '.':
862
0
      break;
863
0
    case 'v':
864
0
      Kind = Void;
865
0
      break;
866
0
    case 'S':
867
0
      Kind = SInt;
868
0
      break;
869
0
    case 'U':
870
0
      Kind = UInt;
871
0
      break;
872
0
    case 'B':
873
0
      Kind = BFloat16;
874
0
      ElementBitwidth = 16;
875
0
      break;
876
0
    case 'F':
877
0
      Kind = Float;
878
0
      break;
879
0
    case 'P':
880
0
      Kind = Poly;
881
0
      break;
882
0
    case '>':
883
0
      assert(ElementBitwidth < 128);
884
0
      ElementBitwidth *= 2;
885
0
      break;
886
0
    case '<':
887
0
      assert(ElementBitwidth > 8);
888
0
      ElementBitwidth /= 2;
889
0
      break;
890
0
    case '1':
891
0
      NumVectors = 0;
892
0
      break;
893
0
    case '2':
894
0
      NumVectors = 2;
895
0
      break;
896
0
    case '3':
897
0
      NumVectors = 3;
898
0
      break;
899
0
    case '4':
900
0
      NumVectors = 4;
901
0
      break;
902
0
    case '*':
903
0
      Pointer = true;
904
0
      break;
905
0
    case 'c':
906
0
      Constant = true;
907
0
      break;
908
0
    case 'Q':
909
0
      Bitwidth = 128;
910
0
      break;
911
0
    case 'q':
912
0
      Bitwidth = 64;
913
0
      break;
914
0
    case 'I':
915
0
      Kind = SInt;
916
0
      ElementBitwidth = Bitwidth = 32;
917
0
      NumVectors = 0;
918
0
      Immediate = true;
919
0
      break;
920
0
    case 'p':
921
0
      if (isPoly())
922
0
        Kind = UInt;
923
0
      break;
924
0
    case '!':
925
      // Key type, handled elsewhere.
926
0
      break;
927
0
    default:
928
0
      llvm_unreachable("Unhandled character!");
929
0
    }
930
0
  }
931
0
}
932
933
//===----------------------------------------------------------------------===//
934
// Intrinsic implementation
935
//===----------------------------------------------------------------------===//
936
937
0
StringRef Intrinsic::getNextModifiers(StringRef Proto, unsigned &Pos) const {
938
0
  if (Proto.size() == Pos)
939
0
    return StringRef();
940
0
  else if (Proto[Pos] != '(')
941
0
    return Proto.substr(Pos++, 1);
942
943
0
  size_t Start = Pos + 1;
944
0
  size_t End = Proto.find(')', Start);
945
0
  assert_with_loc(End != StringRef::npos, "unmatched modifier group paren");
946
0
  Pos = End + 1;
947
0
  return Proto.slice(Start, End);
948
0
}
949
950
0
std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const {
951
0
  char typeCode = '\0';
952
0
  bool printNumber = true;
953
954
0
  if (CK == ClassB)
955
0
    return "";
956
957
0
  if (T.isBFloat16())
958
0
    return "bf16";
959
960
0
  if (T.isPoly())
961
0
    typeCode = 'p';
962
0
  else if (T.isInteger())
963
0
    typeCode = T.isSigned() ? 's' : 'u';
964
0
  else
965
0
    typeCode = 'f';
966
967
0
  if (CK == ClassI) {
968
0
    switch (typeCode) {
969
0
    default:
970
0
      break;
971
0
    case 's':
972
0
    case 'u':
973
0
    case 'p':
974
0
      typeCode = 'i';
975
0
      break;
976
0
    }
977
0
  }
978
0
  if (CK == ClassB) {
979
0
    typeCode = '\0';
980
0
  }
981
982
0
  std::string S;
983
0
  if (typeCode != '\0')
984
0
    S.push_back(typeCode);
985
0
  if (printNumber)
986
0
    S += utostr(T.getElementSizeInBits());
987
988
0
  return S;
989
0
}
990
991
0
std::string Intrinsic::getBuiltinTypeStr() {
992
0
  ClassKind LocalCK = getClassKind(true);
993
0
  std::string S;
994
995
0
  Type RetT = getReturnType();
996
0
  if ((LocalCK == ClassI || LocalCK == ClassW) && RetT.isScalar() &&
997
0
      !RetT.isFloating() && !RetT.isBFloat16())
998
0
    RetT.makeInteger(RetT.getElementSizeInBits(), false);
999
1000
  // Since the return value must be one type, return a vector type of the
1001
  // appropriate width which we will bitcast.  An exception is made for
1002
  // returning structs of 2, 3, or 4 vectors which are returned in a sret-like
1003
  // fashion, storing them to a pointer arg.
1004
0
  if (RetT.getNumVectors() > 1) {
1005
0
    S += "vv*"; // void result with void* first argument
1006
0
  } else {
1007
0
    if (RetT.isPoly())
1008
0
      RetT.makeInteger(RetT.getElementSizeInBits(), false);
1009
0
    if (!RetT.isScalar() && RetT.isInteger() && !RetT.isSigned())
1010
0
      RetT.makeSigned();
1011
1012
0
    if (LocalCK == ClassB && RetT.isValue() && !RetT.isScalar())
1013
      // Cast to vector of 8-bit elements.
1014
0
      RetT.makeInteger(8, true);
1015
1016
0
    S += RetT.builtin_str();
1017
0
  }
1018
1019
0
  for (unsigned I = 0; I < getNumParams(); ++I) {
1020
0
    Type T = getParamType(I);
1021
0
    if (T.isPoly())
1022
0
      T.makeInteger(T.getElementSizeInBits(), false);
1023
1024
0
    if (LocalCK == ClassB && !T.isScalar())
1025
0
      T.makeInteger(8, true);
1026
    // Halves always get converted to 8-bit elements.
1027
0
    if (T.isHalf() && T.isVector() && !T.isScalarForMangling())
1028
0
      T.makeInteger(8, true);
1029
1030
0
    if (LocalCK == ClassI && T.isInteger())
1031
0
      T.makeSigned();
1032
1033
0
    if (hasImmediate() && getImmediateIdx() == I)
1034
0
      T.makeImmediate(32);
1035
1036
0
    S += T.builtin_str();
1037
0
  }
1038
1039
  // Extra constant integer to hold type class enum for this function, e.g. s8
1040
0
  if (LocalCK == ClassB)
1041
0
    S += "i";
1042
1043
0
  return S;
1044
0
}
1045
1046
0
std::string Intrinsic::getMangledName(bool ForceClassS) const {
1047
  // Check if the prototype has a scalar operand with the type of the vector
1048
  // elements.  If not, bitcasting the args will take care of arg checking.
1049
  // The actual signedness etc. will be taken care of with special enums.
1050
0
  ClassKind LocalCK = CK;
1051
0
  if (!protoHasScalar())
1052
0
    LocalCK = ClassB;
1053
1054
0
  return mangleName(Name, ForceClassS ? ClassS : LocalCK);
1055
0
}
1056
1057
0
std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const {
1058
0
  std::string typeCode = getInstTypeCode(BaseType, LocalCK);
1059
0
  std::string S = Name;
1060
1061
0
  if (Name == "vcvt_f16_f32" || Name == "vcvt_f32_f16" ||
1062
0
      Name == "vcvt_f32_f64" || Name == "vcvt_f64_f32" ||
1063
0
      Name == "vcvt_f32_bf16")
1064
0
    return Name;
1065
1066
0
  if (!typeCode.empty()) {
1067
    // If the name ends with _xN (N = 2,3,4), insert the typeCode before _xN.
1068
0
    if (Name.size() >= 3 && isdigit(Name.back()) &&
1069
0
        Name[Name.length() - 2] == 'x' && Name[Name.length() - 3] == '_')
1070
0
      S.insert(S.length() - 3, "_" + typeCode);
1071
0
    else
1072
0
      S += "_" + typeCode;
1073
0
  }
1074
1075
0
  if (BaseType != InBaseType) {
1076
    // A reinterpret - out the input base type at the end.
1077
0
    S += "_" + getInstTypeCode(InBaseType, LocalCK);
1078
0
  }
1079
1080
0
  if (LocalCK == ClassB)
1081
0
    S += "_v";
1082
1083
  // Insert a 'q' before the first '_' character so that it ends up before
1084
  // _lane or _n on vector-scalar operations.
1085
0
  if (BaseType.getSizeInBits() == 128 && !BaseType.noManglingQ()) {
1086
0
    size_t Pos = S.find('_');
1087
0
    S.insert(Pos, "q");
1088
0
  }
1089
1090
0
  char Suffix = '\0';
1091
0
  if (BaseType.isScalarForMangling()) {
1092
0
    switch (BaseType.getElementSizeInBits()) {
1093
0
    case 8: Suffix = 'b'; break;
1094
0
    case 16: Suffix = 'h'; break;
1095
0
    case 32: Suffix = 's'; break;
1096
0
    case 64: Suffix = 'd'; break;
1097
0
    default: llvm_unreachable("Bad suffix!");
1098
0
    }
1099
0
  }
1100
0
  if (Suffix != '\0') {
1101
0
    size_t Pos = S.find('_');
1102
0
    S.insert(Pos, &Suffix, 1);
1103
0
  }
1104
1105
0
  return S;
1106
0
}
1107
1108
0
std::string Intrinsic::replaceParamsIn(std::string S) {
1109
0
  while (S.find('$') != std::string::npos) {
1110
0
    size_t Pos = S.find('$');
1111
0
    size_t End = Pos + 1;
1112
0
    while (isalpha(S[End]))
1113
0
      ++End;
1114
1115
0
    std::string VarName = S.substr(Pos + 1, End - Pos - 1);
1116
0
    assert_with_loc(Variables.find(VarName) != Variables.end(),
1117
0
                    "Variable not defined!");
1118
0
    S.replace(Pos, End - Pos, Variables.find(VarName)->second.getName());
1119
0
  }
1120
1121
0
  return S;
1122
0
}
1123
1124
0
void Intrinsic::initVariables() {
1125
0
  Variables.clear();
1126
1127
  // Modify the TypeSpec per-argument to get a concrete Type, and create
1128
  // known variables for each.
1129
0
  for (unsigned I = 1; I < Types.size(); ++I) {
1130
0
    char NameC = '0' + (I - 1);
1131
0
    std::string Name = "p";
1132
0
    Name.push_back(NameC);
1133
1134
0
    Variables[Name] = Variable(Types[I], Name + VariablePostfix);
1135
0
  }
1136
0
  RetVar = Variable(Types[0], "ret" + VariablePostfix);
1137
0
}
1138
1139
0
void Intrinsic::emitPrototype(StringRef NamePrefix) {
1140
0
  if (UseMacro)
1141
0
    OS << "#define ";
1142
0
  else
1143
0
    OS << "__ai " << Types[0].str() << " ";
1144
1145
0
  OS << NamePrefix.str() << mangleName(Name, ClassS) << "(";
1146
1147
0
  for (unsigned I = 0; I < getNumParams(); ++I) {
1148
0
    if (I != 0)
1149
0
      OS << ", ";
1150
1151
0
    char NameC = '0' + I;
1152
0
    std::string Name = "p";
1153
0
    Name.push_back(NameC);
1154
0
    assert(Variables.find(Name) != Variables.end());
1155
0
    Variable &V = Variables[Name];
1156
1157
0
    if (!UseMacro)
1158
0
      OS << V.getType().str() << " ";
1159
0
    OS << V.getName();
1160
0
  }
1161
1162
0
  OS << ")";
1163
0
}
1164
1165
0
void Intrinsic::emitOpeningBrace() {
1166
0
  if (UseMacro)
1167
0
    OS << " __extension__ ({";
1168
0
  else
1169
0
    OS << " {";
1170
0
  emitNewLine();
1171
0
}
1172
1173
0
void Intrinsic::emitClosingBrace() {
1174
0
  if (UseMacro)
1175
0
    OS << "})";
1176
0
  else
1177
0
    OS << "}";
1178
0
}
1179
1180
0
void Intrinsic::emitNewLine() {
1181
0
  if (UseMacro)
1182
0
    OS << " \\\n";
1183
0
  else
1184
0
    OS << "\n";
1185
0
}
1186
1187
0
void Intrinsic::emitReverseVariable(Variable &Dest, Variable &Src) {
1188
0
  if (Dest.getType().getNumVectors() > 1) {
1189
0
    emitNewLine();
1190
1191
0
    for (unsigned K = 0; K < Dest.getType().getNumVectors(); ++K) {
1192
0
      OS << "  " << Dest.getName() << ".val[" << K << "] = "
1193
0
         << "__builtin_shufflevector("
1194
0
         << Src.getName() << ".val[" << K << "], "
1195
0
         << Src.getName() << ".val[" << K << "]";
1196
0
      for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
1197
0
        OS << ", " << J;
1198
0
      OS << ");";
1199
0
      emitNewLine();
1200
0
    }
1201
0
  } else {
1202
0
    OS << "  " << Dest.getName()
1203
0
       << " = __builtin_shufflevector(" << Src.getName() << ", " << Src.getName();
1204
0
    for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
1205
0
      OS << ", " << J;
1206
0
    OS << ");";
1207
0
    emitNewLine();
1208
0
  }
1209
0
}
1210
1211
0
void Intrinsic::emitArgumentReversal() {
1212
0
  if (isBigEndianSafe())
1213
0
    return;
1214
1215
  // Reverse all vector arguments.
1216
0
  for (unsigned I = 0; I < getNumParams(); ++I) {
1217
0
    std::string Name = "p" + utostr(I);
1218
0
    std::string NewName = "rev" + utostr(I);
1219
1220
0
    Variable &V = Variables[Name];
1221
0
    Variable NewV(V.getType(), NewName + VariablePostfix);
1222
1223
0
    if (!NewV.getType().isVector() || NewV.getType().getNumElements() == 1)
1224
0
      continue;
1225
1226
0
    OS << "  " << NewV.getType().str() << " " << NewV.getName() << ";";
1227
0
    emitReverseVariable(NewV, V);
1228
0
    V = NewV;
1229
0
  }
1230
0
}
1231
1232
0
void Intrinsic::emitReturnVarDecl() {
1233
0
  assert(RetVar.getType() == Types[0]);
1234
  // Create a return variable, if we're not void.
1235
0
  if (!RetVar.getType().isVoid()) {
1236
0
    OS << "  " << RetVar.getType().str() << " " << RetVar.getName() << ";";
1237
0
    emitNewLine();
1238
0
  }
1239
0
}
1240
1241
0
void Intrinsic::emitReturnReversal() {
1242
0
  if (isBigEndianSafe())
1243
0
    return;
1244
0
  if (!getReturnType().isVector() || getReturnType().isVoid() ||
1245
0
      getReturnType().getNumElements() == 1)
1246
0
    return;
1247
0
  emitReverseVariable(RetVar, RetVar);
1248
0
}
1249
1250
0
void Intrinsic::emitShadowedArgs() {
1251
  // Macro arguments are not type-checked like inline function arguments,
1252
  // so assign them to local temporaries to get the right type checking.
1253
0
  if (!UseMacro)
1254
0
    return;
1255
1256
0
  for (unsigned I = 0; I < getNumParams(); ++I) {
1257
    // Do not create a temporary for an immediate argument.
1258
    // That would defeat the whole point of using a macro!
1259
0
    if (getParamType(I).isImmediate())
1260
0
      continue;
1261
    // Do not create a temporary for pointer arguments. The input
1262
    // pointer may have an alignment hint.
1263
0
    if (getParamType(I).isPointer())
1264
0
      continue;
1265
1266
0
    std::string Name = "p" + utostr(I);
1267
1268
0
    assert(Variables.find(Name) != Variables.end());
1269
0
    Variable &V = Variables[Name];
1270
1271
0
    std::string NewName = "s" + utostr(I);
1272
0
    Variable V2(V.getType(), NewName + VariablePostfix);
1273
1274
0
    OS << "  " << V2.getType().str() << " " << V2.getName() << " = "
1275
0
       << V.getName() << ";";
1276
0
    emitNewLine();
1277
1278
0
    V = V2;
1279
0
  }
1280
0
}
1281
1282
0
bool Intrinsic::protoHasScalar() const {
1283
0
  return llvm::any_of(
1284
0
      Types, [](const Type &T) { return T.isScalar() && !T.isImmediate(); });
1285
0
}
1286
1287
0
void Intrinsic::emitBodyAsBuiltinCall() {
1288
0
  std::string S;
1289
1290
  // If this builtin returns a struct 2, 3, or 4 vectors, pass it as an implicit
1291
  // sret-like argument.
1292
0
  bool SRet = getReturnType().getNumVectors() >= 2;
1293
1294
0
  StringRef N = Name;
1295
0
  ClassKind LocalCK = CK;
1296
0
  if (!protoHasScalar())
1297
0
    LocalCK = ClassB;
1298
1299
0
  if (!getReturnType().isVoid() && !SRet)
1300
0
    S += "(" + RetVar.getType().str() + ") ";
1301
1302
0
  S += "__builtin_neon_" + mangleName(std::string(N), LocalCK) + "(";
1303
1304
0
  if (SRet)
1305
0
    S += "&" + RetVar.getName() + ", ";
1306
1307
0
  for (unsigned I = 0; I < getNumParams(); ++I) {
1308
0
    Variable &V = Variables["p" + utostr(I)];
1309
0
    Type T = V.getType();
1310
1311
    // Handle multiple-vector values specially, emitting each subvector as an
1312
    // argument to the builtin.
1313
0
    if (T.getNumVectors() > 1) {
1314
      // Check if an explicit cast is needed.
1315
0
      std::string Cast;
1316
0
      if (LocalCK == ClassB) {
1317
0
        Type T2 = T;
1318
0
        T2.makeOneVector();
1319
0
        T2.makeInteger(8, /*Sign=*/true);
1320
0
        Cast = "(" + T2.str() + ")";
1321
0
      }
1322
1323
0
      for (unsigned J = 0; J < T.getNumVectors(); ++J)
1324
0
        S += Cast + V.getName() + ".val[" + utostr(J) + "], ";
1325
0
      continue;
1326
0
    }
1327
1328
0
    std::string Arg = V.getName();
1329
0
    Type CastToType = T;
1330
1331
    // Check if an explicit cast is needed.
1332
0
    if (CastToType.isVector() &&
1333
0
        (LocalCK == ClassB || (T.isHalf() && !T.isScalarForMangling()))) {
1334
0
      CastToType.makeInteger(8, true);
1335
0
      Arg = "(" + CastToType.str() + ")" + Arg;
1336
0
    } else if (CastToType.isVector() && LocalCK == ClassI) {
1337
0
      if (CastToType.isInteger())
1338
0
        CastToType.makeSigned();
1339
0
      Arg = "(" + CastToType.str() + ")" + Arg;
1340
0
    }
1341
1342
0
    S += Arg + ", ";
1343
0
  }
1344
1345
  // Extra constant integer to hold type class enum for this function, e.g. s8
1346
0
  if (getClassKind(true) == ClassB) {
1347
0
    S += utostr(getPolymorphicKeyType().getNeonEnum());
1348
0
  } else {
1349
    // Remove extraneous ", ".
1350
0
    S.pop_back();
1351
0
    S.pop_back();
1352
0
  }
1353
0
  S += ");";
1354
1355
0
  std::string RetExpr;
1356
0
  if (!SRet && !RetVar.getType().isVoid())
1357
0
    RetExpr = RetVar.getName() + " = ";
1358
1359
0
  OS << "  " << RetExpr << S;
1360
0
  emitNewLine();
1361
0
}
1362
1363
0
void Intrinsic::emitBody(StringRef CallPrefix) {
1364
0
  std::vector<std::string> Lines;
1365
1366
0
  if (!Body || Body->getValues().empty()) {
1367
    // Nothing specific to output - must output a builtin.
1368
0
    emitBodyAsBuiltinCall();
1369
0
    return;
1370
0
  }
1371
1372
  // We have a list of "things to output". The last should be returned.
1373
0
  for (auto *I : Body->getValues()) {
1374
0
    if (StringInit *SI = dyn_cast<StringInit>(I)) {
1375
0
      Lines.push_back(replaceParamsIn(SI->getAsString()));
1376
0
    } else if (DagInit *DI = dyn_cast<DagInit>(I)) {
1377
0
      DagEmitter DE(*this, CallPrefix);
1378
0
      Lines.push_back(DE.emitDag(DI).second + ";");
1379
0
    }
1380
0
  }
1381
1382
0
  assert(!Lines.empty() && "Empty def?");
1383
0
  if (!RetVar.getType().isVoid())
1384
0
    Lines.back().insert(0, RetVar.getName() + " = ");
1385
1386
0
  for (auto &L : Lines) {
1387
0
    OS << "  " << L;
1388
0
    emitNewLine();
1389
0
  }
1390
0
}
1391
1392
0
void Intrinsic::emitReturn() {
1393
0
  if (RetVar.getType().isVoid())
1394
0
    return;
1395
0
  if (UseMacro)
1396
0
    OS << "  " << RetVar.getName() << ";";
1397
0
  else
1398
0
    OS << "  return " << RetVar.getName() << ";";
1399
0
  emitNewLine();
1400
0
}
1401
1402
0
std::pair<Type, std::string> Intrinsic::DagEmitter::emitDag(DagInit *DI) {
1403
  // At this point we should only be seeing a def.
1404
0
  DefInit *DefI = cast<DefInit>(DI->getOperator());
1405
0
  std::string Op = DefI->getAsString();
1406
1407
0
  if (Op == "cast" || Op == "bitcast")
1408
0
    return emitDagCast(DI, Op == "bitcast");
1409
0
  if (Op == "shuffle")
1410
0
    return emitDagShuffle(DI);
1411
0
  if (Op == "dup")
1412
0
    return emitDagDup(DI);
1413
0
  if (Op == "dup_typed")
1414
0
    return emitDagDupTyped(DI);
1415
0
  if (Op == "splat")
1416
0
    return emitDagSplat(DI);
1417
0
  if (Op == "save_temp")
1418
0
    return emitDagSaveTemp(DI);
1419
0
  if (Op == "op")
1420
0
    return emitDagOp(DI);
1421
0
  if (Op == "call" || Op == "call_mangled")
1422
0
    return emitDagCall(DI, Op == "call_mangled");
1423
0
  if (Op == "name_replace")
1424
0
    return emitDagNameReplace(DI);
1425
0
  if (Op == "literal")
1426
0
    return emitDagLiteral(DI);
1427
0
  assert_with_loc(false, "Unknown operation!");
1428
0
  return std::make_pair(Type::getVoid(), "");
1429
0
}
1430
1431
0
std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagOp(DagInit *DI) {
1432
0
  std::string Op = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1433
0
  if (DI->getNumArgs() == 2) {
1434
    // Unary op.
1435
0
    std::pair<Type, std::string> R =
1436
0
        emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1437
0
    return std::make_pair(R.first, Op + R.second);
1438
0
  } else {
1439
0
    assert(DI->getNumArgs() == 3 && "Can only handle unary and binary ops!");
1440
0
    std::pair<Type, std::string> R1 =
1441
0
        emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1442
0
    std::pair<Type, std::string> R2 =
1443
0
        emitDagArg(DI->getArg(2), std::string(DI->getArgNameStr(2)));
1444
0
    assert_with_loc(R1.first == R2.first, "Argument type mismatch!");
1445
0
    return std::make_pair(R1.first, R1.second + " " + Op + " " + R2.second);
1446
0
  }
1447
0
}
1448
1449
std::pair<Type, std::string>
1450
0
Intrinsic::DagEmitter::emitDagCall(DagInit *DI, bool MatchMangledName) {
1451
0
  std::vector<Type> Types;
1452
0
  std::vector<std::string> Values;
1453
0
  for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
1454
0
    std::pair<Type, std::string> R =
1455
0
        emitDagArg(DI->getArg(I + 1), std::string(DI->getArgNameStr(I + 1)));
1456
0
    Types.push_back(R.first);
1457
0
    Values.push_back(R.second);
1458
0
  }
1459
1460
  // Look up the called intrinsic.
1461
0
  std::string N;
1462
0
  if (StringInit *SI = dyn_cast<StringInit>(DI->getArg(0)))
1463
0
    N = SI->getAsUnquotedString();
1464
0
  else
1465
0
    N = emitDagArg(DI->getArg(0), "").second;
1466
0
  Optional<std::string> MangledName;
1467
0
  if (MatchMangledName) {
1468
0
    if (Intr.getRecord()->getValueAsBit("isLaneQ"))
1469
0
      N += "q";
1470
0
    MangledName = Intr.mangleName(N, ClassS);
1471
0
  }
1472
0
  Intrinsic &Callee = Intr.Emitter.getIntrinsic(N, Types, MangledName);
1473
1474
  // Make sure the callee is known as an early def.
1475
0
  Callee.setNeededEarly();
1476
0
  Intr.Dependencies.insert(&Callee);
1477
1478
  // Now create the call itself.
1479
0
  std::string S;
1480
0
  if (!Callee.isBigEndianSafe())
1481
0
    S += CallPrefix.str();
1482
0
  S += Callee.getMangledName(true) + "(";
1483
0
  for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
1484
0
    if (I != 0)
1485
0
      S += ", ";
1486
0
    S += Values[I];
1487
0
  }
1488
0
  S += ")";
1489
1490
0
  return std::make_pair(Callee.getReturnType(), S);
1491
0
}
1492
1493
std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCast(DagInit *DI,
1494
0
                                                                bool IsBitCast){
1495
  // (cast MOD* VAL) -> cast VAL to type given by MOD.
1496
0
  std::pair<Type, std::string> R =
1497
0
      emitDagArg(DI->getArg(DI->getNumArgs() - 1),
1498
0
                 std::string(DI->getArgNameStr(DI->getNumArgs() - 1)));
1499
0
  Type castToType = R.first;
1500
0
  for (unsigned ArgIdx = 0; ArgIdx < DI->getNumArgs() - 1; ++ArgIdx) {
1501
1502
    // MOD can take several forms:
1503
    //   1. $X - take the type of parameter / variable X.
1504
    //   2. The value "R" - take the type of the return type.
1505
    //   3. a type string
1506
    //   4. The value "U" or "S" to switch the signedness.
1507
    //   5. The value "H" or "D" to half or double the bitwidth.
1508
    //   6. The value "8" to convert to 8-bit (signed) integer lanes.
1509
0
    if (!DI->getArgNameStr(ArgIdx).empty()) {
1510
0
      assert_with_loc(Intr.Variables.find(std::string(
1511
0
                          DI->getArgNameStr(ArgIdx))) != Intr.Variables.end(),
1512
0
                      "Variable not found");
1513
0
      castToType =
1514
0
          Intr.Variables[std::string(DI->getArgNameStr(ArgIdx))].getType();
1515
0
    } else {
1516
0
      StringInit *SI = dyn_cast<StringInit>(DI->getArg(ArgIdx));
1517
0
      assert_with_loc(SI, "Expected string type or $Name for cast type");
1518
1519
0
      if (SI->getAsUnquotedString() == "R") {
1520
0
        castToType = Intr.getReturnType();
1521
0
      } else if (SI->getAsUnquotedString() == "U") {
1522
0
        castToType.makeUnsigned();
1523
0
      } else if (SI->getAsUnquotedString() == "S") {
1524
0
        castToType.makeSigned();
1525
0
      } else if (SI->getAsUnquotedString() == "H") {
1526
0
        castToType.halveLanes();
1527
0
      } else if (SI->getAsUnquotedString() == "D") {
1528
0
        castToType.doubleLanes();
1529
0
      } else if (SI->getAsUnquotedString() == "8") {
1530
0
        castToType.makeInteger(8, true);
1531
0
      } else if (SI->getAsUnquotedString() == "32") {
1532
0
        castToType.make32BitElement();
1533
0
      } else {
1534
0
        castToType = Type::fromTypedefName(SI->getAsUnquotedString());
1535
0
        assert_with_loc(!castToType.isVoid(), "Unknown typedef");
1536
0
      }
1537
0
    }
1538
0
  }
1539
1540
0
  std::string S;
1541
0
  if (IsBitCast) {
1542
    // Emit a reinterpret cast. The second operand must be an lvalue, so create
1543
    // a temporary.
1544
0
    std::string N = "reint";
1545
0
    unsigned I = 0;
1546
0
    while (Intr.Variables.find(N) != Intr.Variables.end())
1547
0
      N = "reint" + utostr(++I);
1548
0
    Intr.Variables[N] = Variable(R.first, N + Intr.VariablePostfix);
1549
1550
0
    Intr.OS << R.first.str() << " " << Intr.Variables[N].getName() << " = "
1551
0
            << R.second << ";";
1552
0
    Intr.emitNewLine();
1553
1554
0
    S = "*(" + castToType.str() + " *) &" + Intr.Variables[N].getName() + "";
1555
0
  } else {
1556
    // Emit a normal (static) cast.
1557
0
    S = "(" + castToType.str() + ")(" + R.second + ")";
1558
0
  }
1559
1560
0
  return std::make_pair(castToType, S);
1561
0
}
1562
1563
0
std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagShuffle(DagInit *DI){
1564
  // See the documentation in arm_neon.td for a description of these operators.
1565
0
  class LowHalf : public SetTheory::Operator {
1566
0
  public:
1567
0
    void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1568
0
               ArrayRef<SMLoc> Loc) override {
1569
0
      SetTheory::RecSet Elts2;
1570
0
      ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc);
1571
0
      Elts.insert(Elts2.begin(), Elts2.begin() + (Elts2.size() / 2));
1572
0
    }
1573
0
  };
1574
1575
0
  class HighHalf : public SetTheory::Operator {
1576
0
  public:
1577
0
    void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1578
0
               ArrayRef<SMLoc> Loc) override {
1579
0
      SetTheory::RecSet Elts2;
1580
0
      ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc);
1581
0
      Elts.insert(Elts2.begin() + (Elts2.size() / 2), Elts2.end());
1582
0
    }
1583
0
  };
1584
1585
0
  class Rev : public SetTheory::Operator {
1586
0
    unsigned ElementSize;
1587
1588
0
  public:
1589
0
    Rev(unsigned ElementSize) : ElementSize(ElementSize) {}
1590
1591
0
    void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
1592
0
               ArrayRef<SMLoc> Loc) override {
1593
0
      SetTheory::RecSet Elts2;
1594
0
      ST.evaluate(Expr->arg_begin() + 1, Expr->arg_end(), Elts2, Loc);
1595
1596
0
      int64_t VectorSize = cast<IntInit>(Expr->getArg(0))->getValue();
1597
0
      VectorSize /= ElementSize;
1598
1599
0
      std::vector<Record *> Revved;
1600
0
      for (unsigned VI = 0; VI < Elts2.size(); VI += VectorSize) {
1601
0
        for (int LI = VectorSize - 1; LI >= 0; --LI) {
1602
0
          Revved.push_back(Elts2[VI + LI]);
1603
0
        }
1604
0
      }
1605
1606
0
      Elts.insert(Revved.begin(), Revved.end());
1607
0
    }
1608
0
  };
1609
1610
0
  class MaskExpander : public SetTheory::Expander {
1611
0
    unsigned N;
1612
1613
0
  public:
1614
0
    MaskExpander(unsigned N) : N(N) {}
1615
1616
0
    void expand(SetTheory &ST, Record *R, SetTheory::RecSet &Elts) override {
1617
0
      unsigned Addend = 0;
1618
0
      if (R->getName() == "mask0")
1619
0
        Addend = 0;
1620
0
      else if (R->getName() == "mask1")
1621
0
        Addend = N;
1622
0
      else
1623
0
        return;
1624
0
      for (unsigned I = 0; I < N; ++I)
1625
0
        Elts.insert(R->getRecords().getDef("sv" + utostr(I + Addend)));
1626
0
    }
1627
0
  };
1628
1629
  // (shuffle arg1, arg2, sequence)
1630
0
  std::pair<Type, std::string> Arg1 =
1631
0
      emitDagArg(DI->getArg(0), std::string(DI->getArgNameStr(0)));
1632
0
  std::pair<Type, std::string> Arg2 =
1633
0
      emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1634
0
  assert_with_loc(Arg1.first == Arg2.first,
1635
0
                  "Different types in arguments to shuffle!");
1636
1637
0
  SetTheory ST;
1638
0
  SetTheory::RecSet Elts;
1639
0
  ST.addOperator("lowhalf", std::make_unique<LowHalf>());
1640
0
  ST.addOperator("highhalf", std::make_unique<HighHalf>());
1641
0
  ST.addOperator("rev",
1642
0
                 std::make_unique<Rev>(Arg1.first.getElementSizeInBits()));
1643
0
  ST.addExpander("MaskExpand",
1644
0
                 std::make_unique<MaskExpander>(Arg1.first.getNumElements()));
1645
0
  ST.evaluate(DI->getArg(2), Elts, None);
1646
1647
0
  std::string S = "__builtin_shufflevector(" + Arg1.second + ", " + Arg2.second;
1648
0
  for (auto &E : Elts) {
1649
0
    StringRef Name = E->getName();
1650
0
    assert_with_loc(Name.startswith("sv"),
1651
0
                    "Incorrect element kind in shuffle mask!");
1652
0
    S += ", " + Name.drop_front(2).str();
1653
0
  }
1654
0
  S += ")";
1655
1656
  // Recalculate the return type - the shuffle may have halved or doubled it.
1657
0
  Type T(Arg1.first);
1658
0
  if (Elts.size() > T.getNumElements()) {
1659
0
    assert_with_loc(
1660
0
        Elts.size() == T.getNumElements() * 2,
1661
0
        "Can only double or half the number of elements in a shuffle!");
1662
0
    T.doubleLanes();
1663
0
  } else if (Elts.size() < T.getNumElements()) {
1664
0
    assert_with_loc(
1665
0
        Elts.size() == T.getNumElements() / 2,
1666
0
        "Can only double or half the number of elements in a shuffle!");
1667
0
    T.halveLanes();
1668
0
  }
1669
1670
0
  return std::make_pair(T, S);
1671
0
}
1672
1673
0
std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDup(DagInit *DI) {
1674
0
  assert_with_loc(DI->getNumArgs() == 1, "dup() expects one argument");
1675
0
  std::pair<Type, std::string> A =
1676
0
      emitDagArg(DI->getArg(0), std::string(DI->getArgNameStr(0)));
1677
0
  assert_with_loc(A.first.isScalar(), "dup() expects a scalar argument");
1678
1679
0
  Type T = Intr.getBaseType();
1680
0
  assert_with_loc(T.isVector(), "dup() used but default type is scalar!");
1681
0
  std::string S = "(" + T.str() + ") {";
1682
0
  for (unsigned I = 0; I < T.getNumElements(); ++I) {
1683
0
    if (I != 0)
1684
0
      S += ", ";
1685
0
    S += A.second;
1686
0
  }
1687
0
  S += "}";
1688
1689
0
  return std::make_pair(T, S);
1690
0
}
1691
1692
0
std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDupTyped(DagInit *DI) {
1693
0
  assert_with_loc(DI->getNumArgs() == 2, "dup_typed() expects two arguments");
1694
0
  std::pair<Type, std::string> B =
1695
0
      emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1696
0
  assert_with_loc(B.first.isScalar(),
1697
0
                  "dup_typed() requires a scalar as the second argument");
1698
0
  Type T;
1699
  // If the type argument is a constant string, construct the type directly.
1700
0
  if (StringInit *SI = dyn_cast<StringInit>(DI->getArg(0))) {
1701
0
    T = Type::fromTypedefName(SI->getAsUnquotedString());
1702
0
    assert_with_loc(!T.isVoid(), "Unknown typedef");
1703
0
  } else
1704
0
    T = emitDagArg(DI->getArg(0), std::string(DI->getArgNameStr(0))).first;
1705
1706
0
  assert_with_loc(T.isVector(), "dup_typed() used but target type is scalar!");
1707
0
  std::string S = "(" + T.str() + ") {";
1708
0
  for (unsigned I = 0; I < T.getNumElements(); ++I) {
1709
0
    if (I != 0)
1710
0
      S += ", ";
1711
0
    S += B.second;
1712
0
  }
1713
0
  S += "}";
1714
1715
0
  return std::make_pair(T, S);
1716
0
}
1717
1718
0
std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSplat(DagInit *DI) {
1719
0
  assert_with_loc(DI->getNumArgs() == 2, "splat() expects two arguments");
1720
0
  std::pair<Type, std::string> A =
1721
0
      emitDagArg(DI->getArg(0), std::string(DI->getArgNameStr(0)));
1722
0
  std::pair<Type, std::string> B =
1723
0
      emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1724
1725
0
  assert_with_loc(B.first.isScalar(),
1726
0
                  "splat() requires a scalar int as the second argument");
1727
1728
0
  std::string S = "__builtin_shufflevector(" + A.second + ", " + A.second;
1729
0
  for (unsigned I = 0; I < Intr.getBaseType().getNumElements(); ++I) {
1730
0
    S += ", " + B.second;
1731
0
  }
1732
0
  S += ")";
1733
1734
0
  return std::make_pair(Intr.getBaseType(), S);
1735
0
}
1736
1737
0
std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSaveTemp(DagInit *DI) {
1738
0
  assert_with_loc(DI->getNumArgs() == 2, "save_temp() expects two arguments");
1739
0
  std::pair<Type, std::string> A =
1740
0
      emitDagArg(DI->getArg(1), std::string(DI->getArgNameStr(1)));
1741
1742
0
  assert_with_loc(!A.first.isVoid(),
1743
0
                  "Argument to save_temp() must have non-void type!");
1744
1745
0
  std::string N = std::string(DI->getArgNameStr(0));
1746
0
  assert_with_loc(!N.empty(),
1747
0
                  "save_temp() expects a name as the first argument");
1748
1749
0
  assert_with_loc(Intr.Variables.find(N) == Intr.Variables.end(),
1750
0
                  "Variable already defined!");
1751
0
  Intr.Variables[N] = Variable(A.first, N + Intr.VariablePostfix);
1752
1753
0
  std::string S =
1754
0
      A.first.str() + " " + Intr.Variables[N].getName() + " = " + A.second;
1755
1756
0
  return std::make_pair(Type::getVoid(), S);
1757
0
}
1758
1759
std::pair<Type, std::string>
1760
0
Intrinsic::DagEmitter::emitDagNameReplace(DagInit *DI) {
1761
0
  std::string S = Intr.Name;
1762
1763
0
  assert_with_loc(DI->getNumArgs() == 2, "name_replace requires 2 arguments!");
1764
0
  std::string ToReplace = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1765
0
  std::string ReplaceWith = cast<StringInit>(DI->getArg(1))->getAsUnquotedString();
1766
1767
0
  size_t Idx = S.find(ToReplace);
1768
1769
0
  assert_with_loc(Idx != std::string::npos, "name should contain '" + ToReplace + "'!");
1770
0
  S.replace(Idx, ToReplace.size(), ReplaceWith);
1771
1772
0
  return std::make_pair(Type::getVoid(), S);
1773
0
}
1774
1775
0
std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagLiteral(DagInit *DI){
1776
0
  std::string Ty = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
1777
0
  std::string Value = cast<StringInit>(DI->getArg(1))->getAsUnquotedString();
1778
0
  return std::make_pair(Type::fromTypedefName(Ty), Value);
1779
0
}
1780
1781
std::pair<Type, std::string>
1782
0
Intrinsic::DagEmitter::emitDagArg(Init *Arg, std::string ArgName) {
1783
0
  if (!ArgName.empty()) {
1784
0
    assert_with_loc(!Arg->isComplete(),
1785
0
                    "Arguments must either be DAGs or names, not both!");
1786
0
    assert_with_loc(Intr.Variables.find(ArgName) != Intr.Variables.end(),
1787
0
                    "Variable not defined!");
1788
0
    Variable &V = Intr.Variables[ArgName];
1789
0
    return std::make_pair(V.getType(), V.getName());
1790
0
  }
1791
1792
0
  assert(Arg && "Neither ArgName nor Arg?!");
1793
0
  DagInit *DI = dyn_cast<DagInit>(Arg);
1794
0
  assert_with_loc(DI, "Arguments must either be DAGs or names!");
1795
1796
0
  return emitDag(DI);
1797
0
}
1798
1799
0
std::string Intrinsic::generate() {
1800
  // Avoid duplicated code for big and little endian
1801
0
  if (isBigEndianSafe()) {
1802
0
    generateImpl(false, "", "");
1803
0
    return OS.str();
1804
0
  }
1805
  // Little endian intrinsics are simple and don't require any argument
1806
  // swapping.
1807
0
  OS << "#ifdef __LITTLE_ENDIAN__\n";
1808
1809
0
  generateImpl(false, "", "");
1810
1811
0
  OS << "#else\n";
1812
1813
  // Big endian intrinsics are more complex. The user intended these
1814
  // intrinsics to operate on a vector "as-if" loaded by (V)LDR,
1815
  // but we load as-if (V)LD1. So we should swap all arguments and
1816
  // swap the return value too.
1817
  //
1818
  // If we call sub-intrinsics, we should call a version that does
1819
  // not re-swap the arguments!
1820
0
  generateImpl(true, "", "__noswap_");
1821
1822
  // If we're needed early, create a non-swapping variant for
1823
  // big-endian.
1824
0
  if (NeededEarly) {
1825
0
    generateImpl(false, "__noswap_", "__noswap_");
1826
0
  }
1827
0
  OS << "#endif\n\n";
1828
1829
0
  return OS.str();
1830
0
}
1831
1832
void Intrinsic::generateImpl(bool ReverseArguments,
1833
0
                             StringRef NamePrefix, StringRef CallPrefix) {
1834
0
  CurrentRecord = R;
1835
1836
  // If we call a macro, our local variables may be corrupted due to
1837
  // lack of proper lexical scoping. So, add a globally unique postfix
1838
  // to every variable.
1839
  //
1840
  // indexBody() should have set up the Dependencies set by now.
1841
0
  for (auto *I : Dependencies)
1842
0
    if (I->UseMacro) {
1843
0
      VariablePostfix = "_" + utostr(Emitter.getUniqueNumber());
1844
0
      break;
1845
0
    }
1846
1847
0
  initVariables();
1848
1849
0
  emitPrototype(NamePrefix);
1850
1851
0
  if (IsUnavailable) {
1852
0
    OS << " __attribute__((unavailable));";
1853
0
  } else {
1854
0
    emitOpeningBrace();
1855
    // Emit return variable declaration first as to not trigger
1856
    // -Wdeclaration-after-statement.
1857
0
    emitReturnVarDecl();
1858
0
    emitShadowedArgs();
1859
0
    if (ReverseArguments)
1860
0
      emitArgumentReversal();
1861
0
    emitBody(CallPrefix);
1862
0
    if (ReverseArguments)
1863
0
      emitReturnReversal();
1864
0
    emitReturn();
1865
0
    emitClosingBrace();
1866
0
  }
1867
0
  OS << "\n";
1868
1869
0
  CurrentRecord = nullptr;
1870
0
}
1871
1872
0
void Intrinsic::indexBody() {
1873
0
  CurrentRecord = R;
1874
1875
0
  initVariables();
1876
  // Emit return variable declaration first as to not trigger
1877
  // -Wdeclaration-after-statement.
1878
0
  emitReturnVarDecl();
1879
0
  emitBody("");
1880
0
  OS.str("");
1881
1882
0
  CurrentRecord = nullptr;
1883
0
}
1884
1885
//===----------------------------------------------------------------------===//
1886
// NeonEmitter implementation
1887
//===----------------------------------------------------------------------===//
1888
1889
Intrinsic &NeonEmitter::getIntrinsic(StringRef Name, ArrayRef<Type> Types,
1890
0
                                     Optional<std::string> MangledName) {
1891
  // First, look up the name in the intrinsic map.
1892
0
  assert_with_loc(IntrinsicMap.find(Name.str()) != IntrinsicMap.end(),
1893
0
                  ("Intrinsic '" + Name + "' not found!").str());
1894
0
  auto &V = IntrinsicMap.find(Name.str())->second;
1895
0
  std::vector<Intrinsic *> GoodVec;
1896
1897
  // Create a string to print if we end up failing.
1898
0
  std::string ErrMsg = "looking up intrinsic '" + Name.str() + "(";
1899
0
  for (unsigned I = 0; I < Types.size(); ++I) {
1900
0
    if (I != 0)
1901
0
      ErrMsg += ", ";
1902
0
    ErrMsg += Types[I].str();
1903
0
  }
1904
0
  ErrMsg += ")'\n";
1905
0
  ErrMsg += "Available overloads:\n";
1906
1907
  // Now, look through each intrinsic implementation and see if the types are
1908
  // compatible.
1909
0
  for (auto &I : V) {
1910
0
    ErrMsg += "  - " + I.getReturnType().str() + " " + I.getMangledName();
1911
0
    ErrMsg += "(";
1912
0
    for (unsigned A = 0; A < I.getNumParams(); ++A) {
1913
0
      if (A != 0)
1914
0
        ErrMsg += ", ";
1915
0
      ErrMsg += I.getParamType(A).str();
1916
0
    }
1917
0
    ErrMsg += ")\n";
1918
1919
0
    if (MangledName && MangledName != I.getMangledName(true))
1920
0
      continue;
1921
1922
0
    if (I.getNumParams() != Types.size())
1923
0
      continue;
1924
1925
0
    unsigned ArgNum = 0;
1926
0
    bool MatchingArgumentTypes = llvm::all_of(Types, [&](const auto &Type) {
1927
0
      return Type == I.getParamType(ArgNum++);
1928
0
    });
1929
1930
0
    if (MatchingArgumentTypes)
1931
0
      GoodVec.push_back(&I);
1932
0
  }
1933
1934
0
  assert_with_loc(!GoodVec.empty(),
1935
0
                  "No compatible intrinsic found - " + ErrMsg);
1936
0
  assert_with_loc(GoodVec.size() == 1, "Multiple overloads found - " + ErrMsg);
1937
1938
0
  return *GoodVec.front();
1939
0
}
1940
1941
void NeonEmitter::createIntrinsic(Record *R,
1942
0
                                  SmallVectorImpl<Intrinsic *> &Out) {
1943
0
  std::string Name = std::string(R->getValueAsString("Name"));
1944
0
  std::string Proto = std::string(R->getValueAsString("Prototype"));
1945
0
  std::string Types = std::string(R->getValueAsString("Types"));
1946
0
  Record *OperationRec = R->getValueAsDef("Operation");
1947
0
  bool BigEndianSafe  = R->getValueAsBit("BigEndianSafe");
1948
0
  std::string Guard = std::string(R->getValueAsString("ArchGuard"));
1949
0
  bool IsUnavailable = OperationRec->getValueAsBit("Unavailable");
1950
0
  std::string CartesianProductWith = std::string(R->getValueAsString("CartesianProductWith"));
1951
1952
  // Set the global current record. This allows assert_with_loc to produce
1953
  // decent location information even when highly nested.
1954
0
  CurrentRecord = R;
1955
1956
0
  ListInit *Body = OperationRec->getValueAsListInit("Ops");
1957
1958
0
  std::vector<TypeSpec> TypeSpecs = TypeSpec::fromTypeSpecs(Types);
1959
1960
0
  ClassKind CK = ClassNone;
1961
0
  if (R->getSuperClasses().size() >= 2)
1962
0
    CK = ClassMap[R->getSuperClasses()[1].first];
1963
1964
0
  std::vector<std::pair<TypeSpec, TypeSpec>> NewTypeSpecs;
1965
0
  if (!CartesianProductWith.empty()) {
1966
0
    std::vector<TypeSpec> ProductTypeSpecs = TypeSpec::fromTypeSpecs(CartesianProductWith);
1967
0
    for (auto TS : TypeSpecs) {
1968
0
      Type DefaultT(TS, ".");
1969
0
      for (auto SrcTS : ProductTypeSpecs) {
1970
0
        Type DefaultSrcT(SrcTS, ".");
1971
0
        if (TS == SrcTS ||
1972
0
            DefaultSrcT.getSizeInBits() != DefaultT.getSizeInBits())
1973
0
          continue;
1974
0
        NewTypeSpecs.push_back(std::make_pair(TS, SrcTS));
1975
0
      }
1976
0
    }
1977
0
  } else {
1978
0
    for (auto TS : TypeSpecs) {
1979
0
      NewTypeSpecs.push_back(std::make_pair(TS, TS));
1980
0
    }
1981
0
  }
1982
1983
0
  llvm::sort(NewTypeSpecs);
1984
0
  NewTypeSpecs.erase(std::unique(NewTypeSpecs.begin(), NewTypeSpecs.end()),
1985
0
         NewTypeSpecs.end());
1986
0
  auto &Entry = IntrinsicMap[Name];
1987
1988
0
  for (auto &I : NewTypeSpecs) {
1989
0
    Entry.emplace_back(R, Name, Proto, I.first, I.second, CK, Body, *this,
1990
0
                       Guard, IsUnavailable, BigEndianSafe);
1991
0
    Out.push_back(&Entry.back());
1992
0
  }
1993
1994
0
  CurrentRecord = nullptr;
1995
0
}
1996
1997
/// genBuiltinsDef: Generate the BuiltinsARM.def and  BuiltinsAArch64.def
1998
/// declaration of builtins, checking for unique builtin declarations.
1999
void NeonEmitter::genBuiltinsDef(raw_ostream &OS,
2000
0
                                 SmallVectorImpl<Intrinsic *> &Defs) {
2001
0
  OS << "#ifdef GET_NEON_BUILTINS\n";
2002
2003
  // We only want to emit a builtin once, and we want to emit them in
2004
  // alphabetical order, so use a std::set.
2005
0
  std::set<std::string> Builtins;
2006
2007
0
  for (auto *Def : Defs) {
2008
0
    if (Def->hasBody())
2009
0
      continue;
2010
2011
0
    std::string S = "BUILTIN(__builtin_neon_" + Def->getMangledName() + ", \"";
2012
2013
0
    S += Def->getBuiltinTypeStr();
2014
0
    S += "\", \"n\")";
2015
2016
0
    Builtins.insert(S);
2017
0
  }
2018
2019
0
  for (auto &S : Builtins)
2020
0
    OS << S << "\n";
2021
0
  OS << "#endif\n\n";
2022
0
}
2023
2024
/// Generate the ARM and AArch64 overloaded type checking code for
2025
/// SemaChecking.cpp, checking for unique builtin declarations.
2026
void NeonEmitter::genOverloadTypeCheckCode(raw_ostream &OS,
2027
0
                                           SmallVectorImpl<Intrinsic *> &Defs) {
2028
0
  OS << "#ifdef GET_NEON_OVERLOAD_CHECK\n";
2029
2030
  // We record each overload check line before emitting because subsequent Inst
2031
  // definitions may extend the number of permitted types (i.e. augment the
2032
  // Mask). Use std::map to avoid sorting the table by hash number.
2033
0
  struct OverloadInfo {
2034
0
    uint64_t Mask;
2035
0
    int PtrArgNum;
2036
0
    bool HasConstPtr;
2037
0
    OverloadInfo() : Mask(0ULL), PtrArgNum(0), HasConstPtr(false) {}
2038
0
  };
2039
0
  std::map<std::string, OverloadInfo> OverloadMap;
2040
2041
0
  for (auto *Def : Defs) {
2042
    // If the def has a body (that is, it has Operation DAGs), it won't call
2043
    // __builtin_neon_* so we don't need to generate a definition for it.
2044
0
    if (Def->hasBody())
2045
0
      continue;
2046
    // Functions which have a scalar argument cannot be overloaded, no need to
2047
    // check them if we are emitting the type checking code.
2048
0
    if (Def->protoHasScalar())
2049
0
      continue;
2050
2051
0
    uint64_t Mask = 0ULL;
2052
0
    Mask |= 1ULL << Def->getPolymorphicKeyType().getNeonEnum();
2053
2054
    // Check if the function has a pointer or const pointer argument.
2055
0
    int PtrArgNum = -1;
2056
0
    bool HasConstPtr = false;
2057
0
    for (unsigned I = 0; I < Def->getNumParams(); ++I) {
2058
0
      const auto &Type = Def->getParamType(I);
2059
0
      if (Type.isPointer()) {
2060
0
        PtrArgNum = I;
2061
0
        HasConstPtr = Type.isConstPointer();
2062
0
      }
2063
0
    }
2064
2065
    // For sret builtins, adjust the pointer argument index.
2066
0
    if (PtrArgNum >= 0 && Def->getReturnType().getNumVectors() > 1)
2067
0
      PtrArgNum += 1;
2068
2069
0
    std::string Name = Def->getName();
2070
    // Omit type checking for the pointer arguments of vld1_lane, vld1_dup,
2071
    // and vst1_lane intrinsics.  Using a pointer to the vector element
2072
    // type with one of those operations causes codegen to select an aligned
2073
    // load/store instruction.  If you want an unaligned operation,
2074
    // the pointer argument needs to have less alignment than element type,
2075
    // so just accept any pointer type.
2076
0
    if (Name == "vld1_lane" || Name == "vld1_dup" || Name == "vst1_lane") {
2077
0
      PtrArgNum = -1;
2078
0
      HasConstPtr = false;
2079
0
    }
2080
2081
0
    if (Mask) {
2082
0
      std::string Name = Def->getMangledName();
2083
0
      OverloadMap.insert(std::make_pair(Name, OverloadInfo()));
2084
0
      OverloadInfo &OI = OverloadMap[Name];
2085
0
      OI.Mask |= Mask;
2086
0
      OI.PtrArgNum |= PtrArgNum;
2087
0
      OI.HasConstPtr = HasConstPtr;
2088
0
    }
2089
0
  }
2090
2091
0
  for (auto &I : OverloadMap) {
2092
0
    OverloadInfo &OI = I.second;
2093
2094
0
    OS << "case NEON::BI__builtin_neon_" << I.first << ": ";
2095
0
    OS << "mask = 0x" << Twine::utohexstr(OI.Mask) << "ULL";
2096
0
    if (OI.PtrArgNum >= 0)
2097
0
      OS << "; PtrArgNum = " << OI.PtrArgNum;
2098
0
    if (OI.HasConstPtr)
2099
0
      OS << "; HasConstPtr = true";
2100
0
    OS << "; break;\n";
2101
0
  }
2102
0
  OS << "#endif\n\n";
2103
0
}
2104
2105
void NeonEmitter::genIntrinsicRangeCheckCode(raw_ostream &OS,
2106
0
                                        SmallVectorImpl<Intrinsic *> &Defs) {
2107
0
  OS << "#ifdef GET_NEON_IMMEDIATE_CHECK\n";
2108
2109
0
  std::set<std::string> Emitted;
2110
2111
0
  for (auto *Def : Defs) {
2112
0
    if (Def->hasBody())
2113
0
      continue;
2114
    // Functions which do not have an immediate do not need to have range
2115
    // checking code emitted.
2116
0
    if (!Def->hasImmediate())
2117
0
      continue;
2118
0
    if (Emitted.find(Def->getMangledName()) != Emitted.end())
2119
0
      continue;
2120
2121
0
    std::string LowerBound, UpperBound;
2122
2123
0
    Record *R = Def->getRecord();
2124
0
    if (R->getValueAsBit("isVXAR")) {
2125
      //VXAR takes an immediate in the range [0, 63]
2126
0
      LowerBound = "0";
2127
0
      UpperBound = "63";
2128
0
    } else if (R->getValueAsBit("isVCVT_N")) {
2129
      // VCVT between floating- and fixed-point values takes an immediate
2130
      // in the range [1, 32) for f32 or [1, 64) for f64 or [1, 16) for f16.
2131
0
      LowerBound = "1";
2132
0
    if (Def->getBaseType().getElementSizeInBits() == 16 ||
2133
0
      Def->getName().find('h') != std::string::npos)
2134
    // VCVTh operating on FP16 intrinsics in range [1, 16)
2135
0
    UpperBound = "15";
2136
0
    else if (Def->getBaseType().getElementSizeInBits() == 32)
2137
0
        UpperBound = "31";
2138
0
    else
2139
0
        UpperBound = "63";
2140
0
    } else if (R->getValueAsBit("isScalarShift")) {
2141
      // Right shifts have an 'r' in the name, left shifts do not. Convert
2142
      // instructions have the same bounds and right shifts.
2143
0
      if (Def->getName().find('r') != std::string::npos ||
2144
0
          Def->getName().find("cvt") != std::string::npos)
2145
0
        LowerBound = "1";
2146
2147
0
      UpperBound = utostr(Def->getReturnType().getElementSizeInBits() - 1);
2148
0
    } else if (R->getValueAsBit("isShift")) {
2149
      // Builtins which are overloaded by type will need to have their upper
2150
      // bound computed at Sema time based on the type constant.
2151
2152
      // Right shifts have an 'r' in the name, left shifts do not.
2153
0
      if (Def->getName().find('r') != std::string::npos)
2154
0
        LowerBound = "1";
2155
0
      UpperBound = "RFT(TV, true)";
2156
0
    } else if (Def->getClassKind(true) == ClassB) {
2157
      // ClassB intrinsics have a type (and hence lane number) that is only
2158
      // known at runtime.
2159
0
      if (R->getValueAsBit("isLaneQ"))
2160
0
        UpperBound = "RFT(TV, false, true)";
2161
0
      else
2162
0
        UpperBound = "RFT(TV, false, false)";
2163
0
    } else {
2164
      // The immediate generally refers to a lane in the preceding argument.
2165
0
      assert(Def->getImmediateIdx() > 0);
2166
0
      Type T = Def->getParamType(Def->getImmediateIdx() - 1);
2167
0
      UpperBound = utostr(T.getNumElements() - 1);
2168
0
    }
2169
2170
    // Calculate the index of the immediate that should be range checked.
2171
0
    unsigned Idx = Def->getNumParams();
2172
0
    if (Def->hasImmediate())
2173
0
      Idx = Def->getGeneratedParamIdx(Def->getImmediateIdx());
2174
2175
0
    OS << "case NEON::BI__builtin_neon_" << Def->getMangledName() << ": "
2176
0
       << "i = " << Idx << ";";
2177
0
    if (!LowerBound.empty())
2178
0
      OS << " l = " << LowerBound << ";";
2179
0
    if (!UpperBound.empty())
2180
0
      OS << " u = " << UpperBound << ";";
2181
0
    OS << " break;\n";
2182
2183
0
    Emitted.insert(Def->getMangledName());
2184
0
  }
2185
2186
0
  OS << "#endif\n\n";
2187
0
}
2188
2189
/// runHeader - Emit a file with sections defining:
2190
/// 1. the NEON section of BuiltinsARM.def and BuiltinsAArch64.def.
2191
/// 2. the SemaChecking code for the type overload checking.
2192
/// 3. the SemaChecking code for validation of intrinsic immediate arguments.
2193
0
void NeonEmitter::runHeader(raw_ostream &OS) {
2194
0
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2195
2196
0
  SmallVector<Intrinsic *, 128> Defs;
2197
0
  for (auto *R : RV)
2198
0
    createIntrinsic(R, Defs);
2199
2200
  // Generate shared BuiltinsXXX.def
2201
0
  genBuiltinsDef(OS, Defs);
2202
2203
  // Generate ARM overloaded type checking code for SemaChecking.cpp
2204
0
  genOverloadTypeCheckCode(OS, Defs);
2205
2206
  // Generate ARM range checking code for shift/lane immediates.
2207
0
  genIntrinsicRangeCheckCode(OS, Defs);
2208
0
}
2209
2210
0
static void emitNeonTypeDefs(const std::string& types, raw_ostream &OS) {
2211
0
  std::string TypedefTypes(types);
2212
0
  std::vector<TypeSpec> TDTypeVec = TypeSpec::fromTypeSpecs(TypedefTypes);
2213
2214
  // Emit vector typedefs.
2215
0
  bool InIfdef = false;
2216
0
  for (auto &TS : TDTypeVec) {
2217
0
    bool IsA64 = false;
2218
0
    Type T(TS, ".");
2219
0
    if (T.isDouble())
2220
0
      IsA64 = true;
2221
2222
0
    if (InIfdef && !IsA64) {
2223
0
      OS << "#endif\n";
2224
0
      InIfdef = false;
2225
0
    }
2226
0
    if (!InIfdef && IsA64) {
2227
0
      OS << "#ifdef __aarch64__\n";
2228
0
      InIfdef = true;
2229
0
    }
2230
2231
0
    if (T.isPoly())
2232
0
      OS << "typedef __attribute__((neon_polyvector_type(";
2233
0
    else
2234
0
      OS << "typedef __attribute__((neon_vector_type(";
2235
2236
0
    Type T2 = T;
2237
0
    T2.makeScalar();
2238
0
    OS << T.getNumElements() << "))) ";
2239
0
    OS << T2.str();
2240
0
    OS << " " << T.str() << ";\n";
2241
0
  }
2242
0
  if (InIfdef)
2243
0
    OS << "#endif\n";
2244
0
  OS << "\n";
2245
2246
  // Emit struct typedefs.
2247
0
  InIfdef = false;
2248
0
  for (unsigned NumMembers = 2; NumMembers <= 4; ++NumMembers) {
2249
0
    for (auto &TS : TDTypeVec) {
2250
0
      bool IsA64 = false;
2251
0
      Type T(TS, ".");
2252
0
      if (T.isDouble())
2253
0
        IsA64 = true;
2254
2255
0
      if (InIfdef && !IsA64) {
2256
0
        OS << "#endif\n";
2257
0
        InIfdef = false;
2258
0
      }
2259
0
      if (!InIfdef && IsA64) {
2260
0
        OS << "#ifdef __aarch64__\n";
2261
0
        InIfdef = true;
2262
0
      }
2263
2264
0
      const char Mods[] = { static_cast<char>('2' + (NumMembers - 2)), 0};
2265
0
      Type VT(TS, Mods);
2266
0
      OS << "typedef struct " << VT.str() << " {\n";
2267
0
      OS << "  " << T.str() << " val";
2268
0
      OS << "[" << NumMembers << "]";
2269
0
      OS << ";\n} ";
2270
0
      OS << VT.str() << ";\n";
2271
0
      OS << "\n";
2272
0
    }
2273
0
  }
2274
0
  if (InIfdef)
2275
0
    OS << "#endif\n";
2276
0
}
2277
2278
/// run - Read the records in arm_neon.td and output arm_neon.h.  arm_neon.h
2279
/// is comprised of type definitions and function declarations.
2280
0
void NeonEmitter::run(raw_ostream &OS) {
2281
0
  OS << "/*===---- arm_neon.h - ARM Neon intrinsics "
2282
0
        "------------------------------"
2283
0
        "---===\n"
2284
0
        " *\n"
2285
0
        " * Permission is hereby granted, free of charge, to any person "
2286
0
        "obtaining "
2287
0
        "a copy\n"
2288
0
        " * of this software and associated documentation files (the "
2289
0
        "\"Software\"),"
2290
0
        " to deal\n"
2291
0
        " * in the Software without restriction, including without limitation "
2292
0
        "the "
2293
0
        "rights\n"
2294
0
        " * to use, copy, modify, merge, publish, distribute, sublicense, "
2295
0
        "and/or sell\n"
2296
0
        " * copies of the Software, and to permit persons to whom the Software "
2297
0
        "is\n"
2298
0
        " * furnished to do so, subject to the following conditions:\n"
2299
0
        " *\n"
2300
0
        " * The above copyright notice and this permission notice shall be "
2301
0
        "included in\n"
2302
0
        " * all copies or substantial portions of the Software.\n"
2303
0
        " *\n"
2304
0
        " * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, "
2305
0
        "EXPRESS OR\n"
2306
0
        " * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF "
2307
0
        "MERCHANTABILITY,\n"
2308
0
        " * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT "
2309
0
        "SHALL THE\n"
2310
0
        " * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR "
2311
0
        "OTHER\n"
2312
0
        " * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, "
2313
0
        "ARISING FROM,\n"
2314
0
        " * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER "
2315
0
        "DEALINGS IN\n"
2316
0
        " * THE SOFTWARE.\n"
2317
0
        " *\n"
2318
0
        " *===-----------------------------------------------------------------"
2319
0
        "---"
2320
0
        "---===\n"
2321
0
        " */\n\n";
2322
2323
0
  OS << "#ifndef __ARM_NEON_H\n";
2324
0
  OS << "#define __ARM_NEON_H\n\n";
2325
2326
0
  OS << "#ifndef __ARM_FP\n";
2327
0
  OS << "#error \"NEON intrinsics not available with the soft-float ABI. "
2328
0
        "Please use -mfloat-abi=softfp or -mfloat-abi=hard\"\n";
2329
0
  OS << "#else\n\n";
2330
2331
0
  OS << "#if !defined(__ARM_NEON)\n";
2332
0
  OS << "#error \"NEON support not enabled\"\n";
2333
0
  OS << "#else\n\n";
2334
2335
0
  OS << "#include <stdint.h>\n\n";
2336
2337
0
  OS << "#ifdef __ARM_FEATURE_BF16\n";
2338
0
  OS << "#include <arm_bf16.h>\n";
2339
0
  OS << "typedef __bf16 bfloat16_t;\n";
2340
0
  OS << "#endif\n\n";
2341
2342
  // Emit NEON-specific scalar typedefs.
2343
0
  OS << "typedef float float32_t;\n";
2344
0
  OS << "typedef __fp16 float16_t;\n";
2345
2346
0
  OS << "#ifdef __aarch64__\n";
2347
0
  OS << "typedef double float64_t;\n";
2348
0
  OS << "#endif\n\n";
2349
2350
  // For now, signedness of polynomial types depends on target
2351
0
  OS << "#ifdef __aarch64__\n";
2352
0
  OS << "typedef uint8_t poly8_t;\n";
2353
0
  OS << "typedef uint16_t poly16_t;\n";
2354
0
  OS << "typedef uint64_t poly64_t;\n";
2355
0
  OS << "typedef __uint128_t poly128_t;\n";
2356
0
  OS << "#else\n";
2357
0
  OS << "typedef int8_t poly8_t;\n";
2358
0
  OS << "typedef int16_t poly16_t;\n";
2359
0
  OS << "typedef int64_t poly64_t;\n";
2360
0
  OS << "#endif\n";
2361
2362
0
  emitNeonTypeDefs("cQcsQsiQilQlUcQUcUsQUsUiQUiUlQUlhQhfQfdQdPcQPcPsQPsPlQPl", OS);
2363
2364
0
  OS << "#ifdef __ARM_FEATURE_BF16\n";
2365
0
  emitNeonTypeDefs("bQb", OS);
2366
0
  OS << "#endif\n\n";
2367
2368
0
  OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
2369
0
        "__nodebug__))\n\n";
2370
2371
0
  SmallVector<Intrinsic *, 128> Defs;
2372
0
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2373
0
  for (auto *R : RV)
2374
0
    createIntrinsic(R, Defs);
2375
2376
0
  for (auto *I : Defs)
2377
0
    I->indexBody();
2378
2379
0
  llvm::stable_sort(Defs, llvm::deref<std::less<>>());
2380
2381
  // Only emit a def when its requirements have been met.
2382
  // FIXME: This loop could be made faster, but it's fast enough for now.
2383
0
  bool MadeProgress = true;
2384
0
  std::string InGuard;
2385
0
  while (!Defs.empty() && MadeProgress) {
2386
0
    MadeProgress = false;
2387
2388
0
    for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2389
0
         I != Defs.end(); /*No step*/) {
2390
0
      bool DependenciesSatisfied = true;
2391
0
      for (auto *II : (*I)->getDependencies()) {
2392
0
        if (llvm::is_contained(Defs, II))
2393
0
          DependenciesSatisfied = false;
2394
0
      }
2395
0
      if (!DependenciesSatisfied) {
2396
        // Try the next one.
2397
0
        ++I;
2398
0
        continue;
2399
0
      }
2400
2401
      // Emit #endif/#if pair if needed.
2402
0
      if ((*I)->getGuard() != InGuard) {
2403
0
        if (!InGuard.empty())
2404
0
          OS << "#endif\n";
2405
0
        InGuard = (*I)->getGuard();
2406
0
        if (!InGuard.empty())
2407
0
          OS << "#if " << InGuard << "\n";
2408
0
      }
2409
2410
      // Actually generate the intrinsic code.
2411
0
      OS << (*I)->generate();
2412
2413
0
      MadeProgress = true;
2414
0
      I = Defs.erase(I);
2415
0
    }
2416
0
  }
2417
0
  assert(Defs.empty() && "Some requirements were not satisfied!");
2418
0
  if (!InGuard.empty())
2419
0
    OS << "#endif\n";
2420
2421
0
  OS << "\n";
2422
0
  OS << "#undef __ai\n\n";
2423
0
  OS << "#endif /* if !defined(__ARM_NEON) */\n";
2424
0
  OS << "#endif /* ifndef __ARM_FP */\n";
2425
0
  OS << "#endif /* __ARM_NEON_H */\n";
2426
0
}
2427
2428
/// run - Read the records in arm_fp16.td and output arm_fp16.h.  arm_fp16.h
2429
/// is comprised of type definitions and function declarations.
2430
0
void NeonEmitter::runFP16(raw_ostream &OS) {
2431
0
  OS << "/*===---- arm_fp16.h - ARM FP16 intrinsics "
2432
0
        "------------------------------"
2433
0
        "---===\n"
2434
0
        " *\n"
2435
0
        " * Permission is hereby granted, free of charge, to any person "
2436
0
        "obtaining a copy\n"
2437
0
        " * of this software and associated documentation files (the "
2438
0
        "\"Software\"), to deal\n"
2439
0
        " * in the Software without restriction, including without limitation "
2440
0
        "the rights\n"
2441
0
        " * to use, copy, modify, merge, publish, distribute, sublicense, "
2442
0
        "and/or sell\n"
2443
0
        " * copies of the Software, and to permit persons to whom the Software "
2444
0
        "is\n"
2445
0
        " * furnished to do so, subject to the following conditions:\n"
2446
0
        " *\n"
2447
0
        " * The above copyright notice and this permission notice shall be "
2448
0
        "included in\n"
2449
0
        " * all copies or substantial portions of the Software.\n"
2450
0
        " *\n"
2451
0
        " * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, "
2452
0
        "EXPRESS OR\n"
2453
0
        " * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF "
2454
0
        "MERCHANTABILITY,\n"
2455
0
        " * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT "
2456
0
        "SHALL THE\n"
2457
0
        " * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR "
2458
0
        "OTHER\n"
2459
0
        " * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, "
2460
0
        "ARISING FROM,\n"
2461
0
        " * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER "
2462
0
        "DEALINGS IN\n"
2463
0
        " * THE SOFTWARE.\n"
2464
0
        " *\n"
2465
0
        " *===-----------------------------------------------------------------"
2466
0
        "---"
2467
0
        "---===\n"
2468
0
        " */\n\n";
2469
2470
0
  OS << "#ifndef __ARM_FP16_H\n";
2471
0
  OS << "#define __ARM_FP16_H\n\n";
2472
2473
0
  OS << "#include <stdint.h>\n\n";
2474
2475
0
  OS << "typedef __fp16 float16_t;\n";
2476
2477
0
  OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
2478
0
        "__nodebug__))\n\n";
2479
2480
0
  SmallVector<Intrinsic *, 128> Defs;
2481
0
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2482
0
  for (auto *R : RV)
2483
0
    createIntrinsic(R, Defs);
2484
2485
0
  for (auto *I : Defs)
2486
0
    I->indexBody();
2487
2488
0
  llvm::stable_sort(Defs, llvm::deref<std::less<>>());
2489
2490
  // Only emit a def when its requirements have been met.
2491
  // FIXME: This loop could be made faster, but it's fast enough for now.
2492
0
  bool MadeProgress = true;
2493
0
  std::string InGuard;
2494
0
  while (!Defs.empty() && MadeProgress) {
2495
0
    MadeProgress = false;
2496
2497
0
    for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2498
0
         I != Defs.end(); /*No step*/) {
2499
0
      bool DependenciesSatisfied = true;
2500
0
      for (auto *II : (*I)->getDependencies()) {
2501
0
        if (llvm::is_contained(Defs, II))
2502
0
          DependenciesSatisfied = false;
2503
0
      }
2504
0
      if (!DependenciesSatisfied) {
2505
        // Try the next one.
2506
0
        ++I;
2507
0
        continue;
2508
0
      }
2509
2510
      // Emit #endif/#if pair if needed.
2511
0
      if ((*I)->getGuard() != InGuard) {
2512
0
        if (!InGuard.empty())
2513
0
          OS << "#endif\n";
2514
0
        InGuard = (*I)->getGuard();
2515
0
        if (!InGuard.empty())
2516
0
          OS << "#if " << InGuard << "\n";
2517
0
      }
2518
2519
      // Actually generate the intrinsic code.
2520
0
      OS << (*I)->generate();
2521
2522
0
      MadeProgress = true;
2523
0
      I = Defs.erase(I);
2524
0
    }
2525
0
  }
2526
0
  assert(Defs.empty() && "Some requirements were not satisfied!");
2527
0
  if (!InGuard.empty())
2528
0
    OS << "#endif\n";
2529
2530
0
  OS << "\n";
2531
0
  OS << "#undef __ai\n\n";
2532
0
  OS << "#endif /* __ARM_FP16_H */\n";
2533
0
}
2534
2535
0
void NeonEmitter::runBF16(raw_ostream &OS) {
2536
0
  OS << "/*===---- arm_bf16.h - ARM BF16 intrinsics "
2537
0
        "-----------------------------------===\n"
2538
0
        " *\n"
2539
0
        " *\n"
2540
0
        " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
2541
0
        "Exceptions.\n"
2542
0
        " * See https://llvm.org/LICENSE.txt for license information.\n"
2543
0
        " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
2544
0
        " *\n"
2545
0
        " *===-----------------------------------------------------------------"
2546
0
        "------===\n"
2547
0
        " */\n\n";
2548
2549
0
  OS << "#ifndef __ARM_BF16_H\n";
2550
0
  OS << "#define __ARM_BF16_H\n\n";
2551
2552
0
  OS << "typedef __bf16 bfloat16_t;\n";
2553
2554
0
  OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
2555
0
        "__nodebug__))\n\n";
2556
2557
0
  SmallVector<Intrinsic *, 128> Defs;
2558
0
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
2559
0
  for (auto *R : RV)
2560
0
    createIntrinsic(R, Defs);
2561
2562
0
  for (auto *I : Defs)
2563
0
    I->indexBody();
2564
2565
0
  llvm::stable_sort(Defs, llvm::deref<std::less<>>());
2566
2567
  // Only emit a def when its requirements have been met.
2568
  // FIXME: This loop could be made faster, but it's fast enough for now.
2569
0
  bool MadeProgress = true;
2570
0
  std::string InGuard;
2571
0
  while (!Defs.empty() && MadeProgress) {
2572
0
    MadeProgress = false;
2573
2574
0
    for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin();
2575
0
         I != Defs.end(); /*No step*/) {
2576
0
      bool DependenciesSatisfied = true;
2577
0
      for (auto *II : (*I)->getDependencies()) {
2578
0
        if (llvm::is_contained(Defs, II))
2579
0
          DependenciesSatisfied = false;
2580
0
      }
2581
0
      if (!DependenciesSatisfied) {
2582
        // Try the next one.
2583
0
        ++I;
2584
0
        continue;
2585
0
      }
2586
2587
      // Emit #endif/#if pair if needed.
2588
0
      if ((*I)->getGuard() != InGuard) {
2589
0
        if (!InGuard.empty())
2590
0
          OS << "#endif\n";
2591
0
        InGuard = (*I)->getGuard();
2592
0
        if (!InGuard.empty())
2593
0
          OS << "#if " << InGuard << "\n";
2594
0
      }
2595
2596
      // Actually generate the intrinsic code.
2597
0
      OS << (*I)->generate();
2598
2599
0
      MadeProgress = true;
2600
0
      I = Defs.erase(I);
2601
0
    }
2602
0
  }
2603
0
  assert(Defs.empty() && "Some requirements were not satisfied!");
2604
0
  if (!InGuard.empty())
2605
0
    OS << "#endif\n";
2606
2607
0
  OS << "\n";
2608
0
  OS << "#undef __ai\n\n";
2609
2610
0
  OS << "#endif\n";
2611
0
}
2612
2613
0
void clang::EmitNeon(RecordKeeper &Records, raw_ostream &OS) {
2614
0
  NeonEmitter(Records).run(OS);
2615
0
}
2616
2617
0
void clang::EmitFP16(RecordKeeper &Records, raw_ostream &OS) {
2618
0
  NeonEmitter(Records).runFP16(OS);
2619
0
}
2620
2621
0
void clang::EmitBF16(RecordKeeper &Records, raw_ostream &OS) {
2622
0
  NeonEmitter(Records).runBF16(OS);
2623
0
}
2624
2625
0
void clang::EmitNeonSema(RecordKeeper &Records, raw_ostream &OS) {
2626
0
  NeonEmitter(Records).runHeader(OS);
2627
0
}
2628
2629
0
void clang::EmitNeonTest(RecordKeeper &Records, raw_ostream &OS) {
2630
0
  llvm_unreachable("Neon test generation no longer implemented!");
2631
0
}