xref: /freebsd/contrib/llvm-project/clang/utils/TableGen/SveEmitter.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583) 
1 //===- SveEmitter.cpp - Generate arm_sve.h for use with clang -*- C++ -*-===//
2 //
3 //  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 //  See https://llvm.org/LICENSE.txt for license information.
5 //  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This tablegen backend is responsible for emitting arm_sve.h, which includes
10 // a declaration and definition of each function specified by the ARM C/C++
11 // Language Extensions (ACLE).
12 //
13 // For details, visit:
14 //  https://developer.arm.com/architectures/system-architectures/software-standards/acle
15 //
16 // Each SVE instruction is implemented in terms of 1 or more functions which
17 // are suffixed with the element type of the input vectors.  Functions may be
18 // implemented in terms of generic vector operations such as +, *, -, etc. or
19 // by calling a __builtin_-prefixed function which will be handled by clang's
20 // CodeGen library.
21 //
22 // See also the documentation in include/clang/Basic/arm_sve.td.
23 //
24 //===----------------------------------------------------------------------===//
25 
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/STLExtras.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/ADT/StringMap.h"
30 #include "llvm/TableGen/Error.h"
31 #include "llvm/TableGen/Record.h"
32 #include <array>
33 #include <cctype>
34 #include <set>
35 #include <sstream>
36 #include <string>
37 #include <tuple>
38 
39 using namespace llvm;
40 
41 enum ClassKind {
42   ClassNone,
43   ClassS,     // signed/unsigned, e.g., "_s8", "_u8" suffix
44   ClassG,     // Overloaded name without type suffix
45 };
46 
47 enum class ACLEKind { SVE, SME };
48 
49 using TypeSpec = std::string;
50 
51 namespace {
52 
53 class ImmCheck {
54   unsigned Arg;
55   unsigned Kind;
56   unsigned ElementSizeInBits;
57 
58 public:
ImmCheck(unsigned Arg,unsigned Kind,unsigned ElementSizeInBits=0)59   ImmCheck(unsigned Arg, unsigned Kind, unsigned ElementSizeInBits = 0)
60       : Arg(Arg), Kind(Kind), ElementSizeInBits(ElementSizeInBits) {}
61   ImmCheck(const ImmCheck &Other) = default;
62   ~ImmCheck() = default;
63 
getArg() const64   unsigned getArg() const { return Arg; }
getKind() const65   unsigned getKind() const { return Kind; }
getElementSizeInBits() const66   unsigned getElementSizeInBits() const { return ElementSizeInBits; }
67 };
68 
69 class SVEType {
70   bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat;
71   bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp,
72       Svcount;
73   unsigned Bitwidth, ElementBitwidth, NumVectors;
74 
75 public:
SVEType()76   SVEType() : SVEType("", 'v') {}
77 
SVEType(StringRef TS,char CharMod,unsigned NumVectors=1)78   SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
79       : Float(false), Signed(true), Immediate(false), Void(false),
80         Constant(false), Pointer(false), BFloat(false), DefaultType(false),
81         IsScalable(true), Predicate(false), PredicatePattern(false),
82         PrefetchOp(false), Svcount(false), Bitwidth(128), ElementBitwidth(~0U),
83         NumVectors(NumVectors) {
84     if (!TS.empty())
85       applyTypespec(TS);
86     applyModifier(CharMod);
87   }
88 
SVEType(const SVEType & Base,unsigned NumV)89   SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
90     NumVectors = NumV;
91   }
92 
isPointer() const93   bool isPointer() const { return Pointer; }
isVoidPointer() const94   bool isVoidPointer() const { return Pointer && Void; }
isSigned() const95   bool isSigned() const { return Signed; }
isImmediate() const96   bool isImmediate() const { return Immediate; }
isScalar() const97   bool isScalar() const { return NumVectors == 0; }
isVector() const98   bool isVector() const { return NumVectors > 0; }
isScalableVector() const99   bool isScalableVector() const { return isVector() && IsScalable; }
isFixedLengthVector() const100   bool isFixedLengthVector() const { return isVector() && !IsScalable; }
isChar() const101   bool isChar() const { return ElementBitwidth == 8; }
isVoid() const102   bool isVoid() const { return Void && !Pointer; }
isDefault() const103   bool isDefault() const { return DefaultType; }
isFloat() const104   bool isFloat() const { return Float && !BFloat; }
isBFloat() const105   bool isBFloat() const { return BFloat && !Float; }
isFloatingPoint() const106   bool isFloatingPoint() const { return Float || BFloat; }
isInteger() const107   bool isInteger() const {
108     return !isFloatingPoint() && !Predicate && !Svcount;
109   }
isScalarPredicate() const110   bool isScalarPredicate() const {
111     return !isFloatingPoint() && Predicate && NumVectors == 0;
112   }
isPredicateVector() const113   bool isPredicateVector() const { return Predicate; }
isPredicatePattern() const114   bool isPredicatePattern() const { return PredicatePattern; }
isPrefetchOp() const115   bool isPrefetchOp() const { return PrefetchOp; }
isSvcount() const116   bool isSvcount() const { return Svcount; }
isConstant() const117   bool isConstant() const { return Constant; }
getElementSizeInBits() const118   unsigned getElementSizeInBits() const { return ElementBitwidth; }
getNumVectors() const119   unsigned getNumVectors() const { return NumVectors; }
120 
getNumElements() const121   unsigned getNumElements() const {
122     assert(ElementBitwidth != ~0U);
123     return Bitwidth / ElementBitwidth;
124   }
getSizeInBits() const125   unsigned getSizeInBits() const {
126     return Bitwidth;
127   }
128 
129   /// Return the string representation of a type, which is an encoded
130   /// string for passing to the BUILTIN() macro in Builtins.def.
131   std::string builtin_str() const;
132 
133   /// Return the C/C++ string representation of a type for use in the
134   /// arm_sve.h header file.
135   std::string str() const;
136 
137 private:
138   /// Creates the type based on the typespec string in TS.
139   void applyTypespec(StringRef TS);
140 
141   /// Applies a prototype modifier to the type.
142   void applyModifier(char Mod);
143 };
144 
145 class SVEEmitter;
146 
147 /// The main grunt class. This represents an instantiation of an intrinsic with
148 /// a particular typespec and prototype.
149 class Intrinsic {
150   /// The unmangled name.
151   std::string Name;
152 
153   /// The name of the corresponding LLVM IR intrinsic.
154   std::string LLVMName;
155 
156   /// Intrinsic prototype.
157   std::string Proto;
158 
159   /// The base type spec for this intrinsic.
160   TypeSpec BaseTypeSpec;
161 
162   /// The base class kind. Most intrinsics use ClassS, which has full type
163   /// info for integers (_s32/_u32), or ClassG which is used for overloaded
164   /// intrinsics.
165   ClassKind Class;
166 
167   /// The architectural #ifdef guard.
168   std::string SVEGuard, SMEGuard;
169 
170   // The merge suffix such as _m, _x or _z.
171   std::string MergeSuffix;
172 
173   /// The types of return value [0] and parameters [1..].
174   std::vector<SVEType> Types;
175 
176   /// The "base type", which is VarType('d', BaseTypeSpec).
177   SVEType BaseType;
178 
179   uint64_t Flags;
180 
181   SmallVector<ImmCheck, 2> ImmChecks;
182 
183 public:
184   Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
185             StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
186             uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
187             ClassKind Class, SVEEmitter &Emitter, StringRef SVEGuard,
188             StringRef SMEGuard);
189 
190   ~Intrinsic()=default;
191 
getName() const192   std::string getName() const { return Name; }
getLLVMName() const193   std::string getLLVMName() const { return LLVMName; }
getProto() const194   std::string getProto() const { return Proto; }
getBaseTypeSpec() const195   TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
getBaseType() const196   SVEType getBaseType() const { return BaseType; }
197 
getSVEGuard() const198   StringRef getSVEGuard() const { return SVEGuard; }
getSMEGuard() const199   StringRef getSMEGuard() const { return SMEGuard; }
printGuard(raw_ostream & OS) const200   void printGuard(raw_ostream &OS) const {
201     if (!SVEGuard.empty() && SMEGuard.empty())
202       OS << SVEGuard;
203     else if (SVEGuard.empty() && !SMEGuard.empty())
204       OS << SMEGuard;
205     else {
206       if (SVEGuard.find(",") != std::string::npos ||
207           SVEGuard.find("|") != std::string::npos)
208         OS << "(" << SVEGuard << ")";
209       else
210         OS << SVEGuard;
211       OS << "|";
212       if (SMEGuard.find(",") != std::string::npos ||
213           SMEGuard.find("|") != std::string::npos)
214         OS << "(" << SMEGuard << ")";
215       else
216         OS << SMEGuard;
217     }
218   }
getClassKind() const219   ClassKind getClassKind() const { return Class; }
220 
getReturnType() const221   SVEType getReturnType() const { return Types[0]; }
getTypes() const222   ArrayRef<SVEType> getTypes() const { return Types; }
getParamType(unsigned I) const223   SVEType getParamType(unsigned I) const { return Types[I + 1]; }
getNumParams() const224   unsigned getNumParams() const {
225     return Proto.size() - (2 * llvm::count(Proto, '.')) - 1;
226   }
227 
getFlags() const228   uint64_t getFlags() const { return Flags; }
isFlagSet(uint64_t Flag) const229   bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
230 
getImmChecks() const231   ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
232 
233   /// Return the type string for a BUILTIN() macro in Builtins.def.
234   std::string getBuiltinTypeStr();
235 
236   /// Return the name, mangled with type information. The name is mangled for
237   /// ClassS, so will add type suffixes such as _u32/_s32.
getMangledName() const238   std::string getMangledName() const { return mangleName(ClassS); }
239 
240   /// As above, but mangles the LLVM name instead.
getMangledLLVMName() const241   std::string getMangledLLVMName() const { return mangleLLVMName(); }
242 
243   /// Returns true if the intrinsic is overloaded, in that it should also generate
244   /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
245   /// 'svld1_u32(..)'.
isOverloadedIntrinsic(StringRef Name)246   static bool isOverloadedIntrinsic(StringRef Name) {
247     auto BrOpen = Name.find('[');
248     auto BrClose = Name.find(']');
249     return BrOpen != std::string::npos && BrClose != std::string::npos;
250   }
251 
252   /// Return true if the intrinsic takes a splat operand.
hasSplat() const253   bool hasSplat() const {
254     // These prototype modifiers are described in arm_sve.td.
255     return Proto.find_first_of("ajfrKLR@") != std::string::npos;
256   }
257 
258   /// Return the parameter index of the splat operand.
getSplatIdx() const259   unsigned getSplatIdx() const {
260     unsigned I = 1, Param = 0;
261     for (; I < Proto.size(); ++I, ++Param) {
262       if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
263           Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
264           Proto[I] == 'R' || Proto[I] == '@')
265         break;
266 
267       // Multivector modifier can be skipped
268       if (Proto[I] == '.')
269         I += 2;
270     }
271     assert(I != Proto.size() && "Prototype has no splat operand");
272     return Param;
273   }
274 
275   /// Emits the intrinsic declaration to the ostream.
276   void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;
277 
278 private:
getMergeSuffix() const279   std::string getMergeSuffix() const { return MergeSuffix; }
280   std::string mangleName(ClassKind LocalCK) const;
281   std::string mangleLLVMName() const;
282   std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
283                                    std::string Proto) const;
284 };
285 
286 class SVEEmitter {
287 private:
288   // The reinterpret builtins are generated separately because they
289   // need the cross product of all types (121 functions in total),
290   // which is inconvenient to specify in the arm_sve.td file or
291   // generate in CGBuiltin.cpp.
292   struct ReinterpretTypeInfo {
293     SVEType BaseType;
294     const char *Suffix;
295   };
296 
297   static const std::array<ReinterpretTypeInfo, 12> Reinterprets;
298 
299   RecordKeeper &Records;
300   llvm::StringMap<uint64_t> EltTypes;
301   llvm::StringMap<uint64_t> MemEltTypes;
302   llvm::StringMap<uint64_t> FlagTypes;
303   llvm::StringMap<uint64_t> MergeTypes;
304   llvm::StringMap<uint64_t> ImmCheckTypes;
305 
306 public:
SVEEmitter(RecordKeeper & R)307   SVEEmitter(RecordKeeper &R) : Records(R) {
308     for (auto *RV : Records.getAllDerivedDefinitions("EltType"))
309       EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
310     for (auto *RV : Records.getAllDerivedDefinitions("MemEltType"))
311       MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
312     for (auto *RV : Records.getAllDerivedDefinitions("FlagType"))
313       FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
314     for (auto *RV : Records.getAllDerivedDefinitions("MergeType"))
315       MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
316     for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType"))
317       ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
318   }
319 
320   /// Returns the enum value for the immcheck type
getEnumValueForImmCheck(StringRef C) const321   unsigned getEnumValueForImmCheck(StringRef C) const {
322     auto It = ImmCheckTypes.find(C);
323     if (It != ImmCheckTypes.end())
324       return It->getValue();
325     llvm_unreachable("Unsupported imm check");
326   }
327 
328   /// Returns the enum value for the flag type
getEnumValueForFlag(StringRef C) const329   uint64_t getEnumValueForFlag(StringRef C) const {
330     auto Res = FlagTypes.find(C);
331     if (Res != FlagTypes.end())
332       return Res->getValue();
333     llvm_unreachable("Unsupported flag");
334   }
335 
336   // Returns the SVETypeFlags for a given value and mask.
encodeFlag(uint64_t V,StringRef MaskName) const337   uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
338     auto It = FlagTypes.find(MaskName);
339     if (It != FlagTypes.end()) {
340       uint64_t Mask = It->getValue();
341       unsigned Shift = llvm::countr_zero(Mask);
342       assert(Shift < 64 && "Mask value produced an invalid shift value");
343       return (V << Shift) & Mask;
344     }
345     llvm_unreachable("Unsupported flag");
346   }
347 
348   // Returns the SVETypeFlags for the given element type.
encodeEltType(StringRef EltName)349   uint64_t encodeEltType(StringRef EltName) {
350     auto It = EltTypes.find(EltName);
351     if (It != EltTypes.end())
352       return encodeFlag(It->getValue(), "EltTypeMask");
353     llvm_unreachable("Unsupported EltType");
354   }
355 
356   // Returns the SVETypeFlags for the given memory element type.
encodeMemoryElementType(uint64_t MT)357   uint64_t encodeMemoryElementType(uint64_t MT) {
358     return encodeFlag(MT, "MemEltTypeMask");
359   }
360 
361   // Returns the SVETypeFlags for the given merge type.
encodeMergeType(uint64_t MT)362   uint64_t encodeMergeType(uint64_t MT) {
363     return encodeFlag(MT, "MergeTypeMask");
364   }
365 
366   // Returns the SVETypeFlags for the given splat operand.
encodeSplatOperand(unsigned SplatIdx)367   unsigned encodeSplatOperand(unsigned SplatIdx) {
368     assert(SplatIdx < 7 && "SplatIdx out of encodable range");
369     return encodeFlag(SplatIdx + 1, "SplatOperandMask");
370   }
371 
372   // Returns the SVETypeFlags value for the given SVEType.
373   uint64_t encodeTypeFlags(const SVEType &T);
374 
375   /// Emit arm_sve.h.
376   void createHeader(raw_ostream &o);
377 
378   // Emits core intrinsics in both arm_sme.h and arm_sve.h
379   void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
380                                   ACLEKind Kind);
381 
382   /// Emit all the __builtin prototypes and code needed by Sema.
383   void createBuiltins(raw_ostream &o);
384 
385   /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
386   void createCodeGenMap(raw_ostream &o);
387 
388   /// Emit all the range checks for the immediates.
389   void createRangeChecks(raw_ostream &o);
390 
391   /// Create the SVETypeFlags used in CGBuiltins
392   void createTypeFlags(raw_ostream &o);
393 
394   /// Emit arm_sme.h.
395   void createSMEHeader(raw_ostream &o);
396 
397   /// Emit all the SME __builtin prototypes and code needed by Sema.
398   void createSMEBuiltins(raw_ostream &o);
399 
400   /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
401   void createSMECodeGenMap(raw_ostream &o);
402 
403   /// Create a table for a builtin's requirement for PSTATE.SM.
404   void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);
405 
406   /// Emit all the range checks for the immediates.
407   void createSMERangeChecks(raw_ostream &o);
408 
409   /// Create a table for a builtin's requirement for PSTATE.ZA.
410   void createBuiltinZAState(raw_ostream &OS);
411 
412   /// Create intrinsic and add it to \p Out
413   void createIntrinsic(Record *R,
414                        SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
415 };
416 
417 const std::array<SVEEmitter::ReinterpretTypeInfo, 12> SVEEmitter::Reinterprets =
418     {{{SVEType("c", 'd'), "s8"},
419       {SVEType("Uc", 'd'), "u8"},
420       {SVEType("s", 'd'), "s16"},
421       {SVEType("Us", 'd'), "u16"},
422       {SVEType("i", 'd'), "s32"},
423       {SVEType("Ui", 'd'), "u32"},
424       {SVEType("l", 'd'), "s64"},
425       {SVEType("Ul", 'd'), "u64"},
426       {SVEType("h", 'd'), "f16"},
427       {SVEType("b", 'd'), "bf16"},
428       {SVEType("f", 'd'), "f32"},
429       {SVEType("d", 'd'), "f64"}}};
430 
431 } // end anonymous namespace
432 
433 
434 //===----------------------------------------------------------------------===//
435 // Type implementation
436 //===----------------------------------------------------------------------===//
437 
builtin_str() const438 std::string SVEType::builtin_str() const {
439   std::string S;
440   if (isVoid())
441     return "v";
442 
443   if (isScalarPredicate())
444     return "b";
445 
446   if (isSvcount())
447     return "Qa";
448 
449   if (isVoidPointer())
450     S += "v";
451   else if (!isFloatingPoint())
452     switch (ElementBitwidth) {
453     case 1: S += "b"; break;
454     case 8: S += "c"; break;
455     case 16: S += "s"; break;
456     case 32: S += "i"; break;
457     case 64: S += "Wi"; break;
458     case 128: S += "LLLi"; break;
459     default: llvm_unreachable("Unhandled case!");
460     }
461   else if (isFloat())
462     switch (ElementBitwidth) {
463     case 16: S += "h"; break;
464     case 32: S += "f"; break;
465     case 64: S += "d"; break;
466     default: llvm_unreachable("Unhandled case!");
467     }
468   else if (isBFloat()) {
469     assert(ElementBitwidth == 16 && "Not a valid BFloat.");
470     S += "y";
471   }
472 
473   if (!isFloatingPoint()) {
474     if ((isChar() || isPointer()) && !isVoidPointer()) {
475       // Make chars and typed pointers explicitly signed.
476       if (Signed)
477         S = "S" + S;
478       else if (!Signed)
479         S = "U" + S;
480     } else if (!isVoidPointer() && !Signed) {
481       S = "U" + S;
482     }
483   }
484 
485   // Constant indices are "int", but have the "constant expression" modifier.
486   if (isImmediate()) {
487     assert(!isFloat() && "fp immediates are not supported");
488     S = "I" + S;
489   }
490 
491   if (isScalar()) {
492     if (Constant) S += "C";
493     if (Pointer) S += "*";
494     return S;
495   }
496 
497   if (isFixedLengthVector())
498     return "V" + utostr(getNumElements() * NumVectors) + S;
499   return "q" + utostr(getNumElements() * NumVectors) + S;
500 }
501 
str() const502 std::string SVEType::str() const {
503   if (isPredicatePattern())
504     return "enum svpattern";
505 
506   if (isPrefetchOp())
507     return "enum svprfop";
508 
509   std::string S;
510   if (Void)
511     S += "void";
512   else {
513     if (isScalableVector() || isSvcount())
514       S += "sv";
515     if (!Signed && !isFloatingPoint())
516       S += "u";
517 
518     if (Float)
519       S += "float";
520     else if (isSvcount())
521       S += "count";
522     else if (isScalarPredicate() || isPredicateVector())
523       S += "bool";
524     else if (isBFloat())
525       S += "bfloat";
526     else
527       S += "int";
528 
529     if (!isScalarPredicate() && !isPredicateVector() && !isSvcount())
530       S += utostr(ElementBitwidth);
531     if (isFixedLengthVector())
532       S += "x" + utostr(getNumElements());
533     if (NumVectors > 1)
534       S += "x" + utostr(NumVectors);
535     if (!isScalarPredicate())
536       S += "_t";
537   }
538 
539   if (Constant)
540     S += " const";
541   if (Pointer)
542     S += " *";
543 
544   return S;
545 }
546 
applyTypespec(StringRef TS)547 void SVEType::applyTypespec(StringRef TS) {
548   for (char I : TS) {
549     switch (I) {
550     case 'Q':
551       Svcount = true;
552       break;
553     case 'P':
554       Predicate = true;
555       break;
556     case 'U':
557       Signed = false;
558       break;
559     case 'c':
560       ElementBitwidth = 8;
561       break;
562     case 's':
563       ElementBitwidth = 16;
564       break;
565     case 'i':
566       ElementBitwidth = 32;
567       break;
568     case 'l':
569       ElementBitwidth = 64;
570       break;
571     case 'q':
572       ElementBitwidth = 128;
573       break;
574     case 'h':
575       Float = true;
576       ElementBitwidth = 16;
577       break;
578     case 'f':
579       Float = true;
580       ElementBitwidth = 32;
581       break;
582     case 'd':
583       Float = true;
584       ElementBitwidth = 64;
585       break;
586     case 'b':
587       BFloat = true;
588       Float = false;
589       ElementBitwidth = 16;
590       break;
591     default:
592       llvm_unreachable("Unhandled type code!");
593     }
594   }
595   assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
596 }
597 
applyModifier(char Mod)598 void SVEType::applyModifier(char Mod) {
599   switch (Mod) {
600   case 'v':
601     Void = true;
602     break;
603   case 'd':
604     DefaultType = true;
605     break;
606   case 'c':
607     Constant = true;
608     [[fallthrough]];
609   case 'p':
610     Pointer = true;
611     Bitwidth = ElementBitwidth;
612     NumVectors = 0;
613     break;
614   case 'e':
615     Signed = false;
616     ElementBitwidth /= 2;
617     break;
618   case 'h':
619     ElementBitwidth /= 2;
620     break;
621   case 'q':
622     ElementBitwidth /= 4;
623     break;
624   case 'b':
625     Signed = false;
626     Float = false;
627     BFloat = false;
628     ElementBitwidth /= 4;
629     break;
630   case 'o':
631     ElementBitwidth *= 4;
632     break;
633   case 'P':
634     Signed = true;
635     Float = false;
636     BFloat = false;
637     Predicate = true;
638     Svcount = false;
639     Bitwidth = 16;
640     ElementBitwidth = 1;
641     break;
642   case '{':
643     IsScalable = false;
644     Bitwidth = 128;
645     NumVectors = 1;
646     break;
647   case 's':
648   case 'a':
649     Bitwidth = ElementBitwidth;
650     NumVectors = 0;
651     break;
652   case 'R':
653     ElementBitwidth /= 2;
654     NumVectors = 0;
655     break;
656   case 'r':
657     ElementBitwidth /= 4;
658     NumVectors = 0;
659     break;
660   case '@':
661     Signed = false;
662     Float = false;
663     BFloat = false;
664     ElementBitwidth /= 4;
665     NumVectors = 0;
666     break;
667   case 'K':
668     Signed = true;
669     Float = false;
670     BFloat = false;
671     Bitwidth = ElementBitwidth;
672     NumVectors = 0;
673     break;
674   case 'L':
675     Signed = false;
676     Float = false;
677     BFloat = false;
678     Bitwidth = ElementBitwidth;
679     NumVectors = 0;
680     break;
681   case 'u':
682     Predicate = false;
683     Svcount = false;
684     Signed = false;
685     Float = false;
686     BFloat = false;
687     break;
688   case 'x':
689     Predicate = false;
690     Svcount = false;
691     Signed = true;
692     Float = false;
693     BFloat = false;
694     break;
695   case 'i':
696     Predicate = false;
697     Svcount = false;
698     Float = false;
699     BFloat = false;
700     ElementBitwidth = Bitwidth = 64;
701     NumVectors = 0;
702     Signed = false;
703     Immediate = true;
704     break;
705   case 'I':
706     Predicate = false;
707     Svcount = false;
708     Float = false;
709     BFloat = false;
710     ElementBitwidth = Bitwidth = 32;
711     NumVectors = 0;
712     Signed = true;
713     Immediate = true;
714     PredicatePattern = true;
715     break;
716   case 'J':
717     Predicate = false;
718     Svcount = false;
719     Float = false;
720     BFloat = false;
721     ElementBitwidth = Bitwidth = 32;
722     NumVectors = 0;
723     Signed = true;
724     Immediate = true;
725     PrefetchOp = true;
726     break;
727   case 'k':
728     Predicate = false;
729     Svcount = false;
730     Signed = true;
731     Float = false;
732     BFloat = false;
733     ElementBitwidth = Bitwidth = 32;
734     NumVectors = 0;
735     break;
736   case 'l':
737     Predicate = false;
738     Svcount = false;
739     Signed = true;
740     Float = false;
741     BFloat = false;
742     ElementBitwidth = Bitwidth = 64;
743     NumVectors = 0;
744     break;
745   case 'm':
746     Predicate = false;
747     Svcount = false;
748     Signed = false;
749     Float = false;
750     BFloat = false;
751     ElementBitwidth = Bitwidth = 32;
752     NumVectors = 0;
753     break;
754   case 'n':
755     Predicate = false;
756     Svcount = false;
757     Signed = false;
758     Float = false;
759     BFloat = false;
760     ElementBitwidth = Bitwidth = 64;
761     NumVectors = 0;
762     break;
763   case 'w':
764     ElementBitwidth = 64;
765     break;
766   case 'j':
767     ElementBitwidth = Bitwidth = 64;
768     NumVectors = 0;
769     break;
770   case 'f':
771     Signed = false;
772     ElementBitwidth = Bitwidth = 64;
773     NumVectors = 0;
774     break;
775   case 'g':
776     Signed = false;
777     Float = false;
778     BFloat = false;
779     ElementBitwidth = 64;
780     break;
781   case '[':
782     Signed = false;
783     Float = false;
784     BFloat = false;
785     ElementBitwidth = 8;
786     break;
787   case 't':
788     Signed = true;
789     Float = false;
790     BFloat = false;
791     ElementBitwidth = 32;
792     break;
793   case 'z':
794     Signed = false;
795     Float = false;
796     BFloat = false;
797     ElementBitwidth = 32;
798     break;
799   case 'O':
800     Predicate = false;
801     Svcount = false;
802     Float = true;
803     ElementBitwidth = 16;
804     break;
805   case 'M':
806     Predicate = false;
807     Svcount = false;
808     Float = true;
809     BFloat = false;
810     ElementBitwidth = 32;
811     break;
812   case 'N':
813     Predicate = false;
814     Svcount = false;
815     Float = true;
816     ElementBitwidth = 64;
817     break;
818   case 'Q':
819     Constant = true;
820     Pointer = true;
821     Void = true;
822     NumVectors = 0;
823     break;
824   case 'S':
825     Constant = true;
826     Pointer = true;
827     ElementBitwidth = Bitwidth = 8;
828     NumVectors = 0;
829     Signed = true;
830     break;
831   case 'W':
832     Constant = true;
833     Pointer = true;
834     ElementBitwidth = Bitwidth = 8;
835     NumVectors = 0;
836     Signed = false;
837     break;
838   case 'T':
839     Constant = true;
840     Pointer = true;
841     ElementBitwidth = Bitwidth = 16;
842     NumVectors = 0;
843     Signed = true;
844     break;
845   case 'X':
846     Constant = true;
847     Pointer = true;
848     ElementBitwidth = Bitwidth = 16;
849     NumVectors = 0;
850     Signed = false;
851     break;
852   case 'Y':
853     Constant = true;
854     Pointer = true;
855     ElementBitwidth = Bitwidth = 32;
856     NumVectors = 0;
857     Signed = false;
858     break;
859   case 'U':
860     Constant = true;
861     Pointer = true;
862     ElementBitwidth = Bitwidth = 32;
863     NumVectors = 0;
864     Signed = true;
865     break;
866   case '%':
867     Pointer = true;
868     Void = true;
869     NumVectors = 0;
870     break;
871   case 'A':
872     Pointer = true;
873     ElementBitwidth = Bitwidth = 8;
874     NumVectors = 0;
875     Signed = true;
876     break;
877   case 'B':
878     Pointer = true;
879     ElementBitwidth = Bitwidth = 16;
880     NumVectors = 0;
881     Signed = true;
882     break;
883   case 'C':
884     Pointer = true;
885     ElementBitwidth = Bitwidth = 32;
886     NumVectors = 0;
887     Signed = true;
888     break;
889   case 'D':
890     Pointer = true;
891     ElementBitwidth = Bitwidth = 64;
892     NumVectors = 0;
893     Signed = true;
894     break;
895   case 'E':
896     Pointer = true;
897     ElementBitwidth = Bitwidth = 8;
898     NumVectors = 0;
899     Signed = false;
900     break;
901   case 'F':
902     Pointer = true;
903     ElementBitwidth = Bitwidth = 16;
904     NumVectors = 0;
905     Signed = false;
906     break;
907   case 'G':
908     Pointer = true;
909     ElementBitwidth = Bitwidth = 32;
910     NumVectors = 0;
911     Signed = false;
912     break;
913   case '$':
914     Predicate = false;
915     Svcount = false;
916     Float = false;
917     BFloat = true;
918     ElementBitwidth = 16;
919     break;
920   case '}':
921     Predicate = false;
922     Signed = true;
923     Svcount = true;
924     NumVectors = 0;
925     Float = false;
926     BFloat = false;
927     break;
928   case '.':
929     llvm_unreachable(". is never a type in itself");
930     break;
931   default:
932     llvm_unreachable("Unhandled character!");
933   }
934 }
935 
936 /// Returns the modifier and number of vectors for the given operand \p Op.
getProtoModifier(StringRef Proto,unsigned Op)937 std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
938   for (unsigned P = 0; !Proto.empty(); ++P) {
939     unsigned NumVectors = 1;
940     unsigned CharsToSkip = 1;
941     char Mod = Proto[0];
942     if (Mod == '2' || Mod == '3' || Mod == '4') {
943       NumVectors = Mod - '0';
944       Mod = 'd';
945       if (Proto.size() > 1 && Proto[1] == '.') {
946         Mod = Proto[2];
947         CharsToSkip = 3;
948       }
949     }
950 
951     if (P == Op)
952       return {Mod, NumVectors};
953 
954     Proto = Proto.drop_front(CharsToSkip);
955   }
956   llvm_unreachable("Unexpected Op");
957 }
958 
959 //===----------------------------------------------------------------------===//
960 // Intrinsic implementation
961 //===----------------------------------------------------------------------===//
962 
Intrinsic(StringRef Name,StringRef Proto,uint64_t MergeTy,StringRef MergeSuffix,uint64_t MemoryElementTy,StringRef LLVMName,uint64_t Flags,ArrayRef<ImmCheck> Checks,TypeSpec BT,ClassKind Class,SVEEmitter & Emitter,StringRef SVEGuard,StringRef SMEGuard)963 Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
964                      StringRef MergeSuffix, uint64_t MemoryElementTy,
965                      StringRef LLVMName, uint64_t Flags,
966                      ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
967                      SVEEmitter &Emitter, StringRef SVEGuard,
968                      StringRef SMEGuard)
969     : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
970       BaseTypeSpec(BT), Class(Class), SVEGuard(SVEGuard.str()),
971       SMEGuard(SMEGuard.str()), MergeSuffix(MergeSuffix.str()),
972       BaseType(BT, 'd'), Flags(Flags), ImmChecks(Checks.begin(), Checks.end()) {
973   // Types[0] is the return value.
974   for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
975     char Mod;
976     unsigned NumVectors;
977     std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);
978     SVEType T(BaseTypeSpec, Mod, NumVectors);
979     Types.push_back(T);
980 
981     // Add range checks for immediates
982     if (I > 0) {
983       if (T.isPredicatePattern())
984         ImmChecks.emplace_back(
985             I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31"));
986       else if (T.isPrefetchOp())
987         ImmChecks.emplace_back(
988             I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13"));
989     }
990   }
991 
992   // Set flags based on properties
993   this->Flags |= Emitter.encodeTypeFlags(BaseType);
994   this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy);
995   this->Flags |= Emitter.encodeMergeType(MergeTy);
996   if (hasSplat())
997     this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());
998 }
999 
getBuiltinTypeStr()1000 std::string Intrinsic::getBuiltinTypeStr() {
1001   std::string S = getReturnType().builtin_str();
1002   for (unsigned I = 0; I < getNumParams(); ++I)
1003     S += getParamType(I).builtin_str();
1004 
1005   return S;
1006 }
1007 
replaceTemplatedArgs(std::string Name,TypeSpec TS,std::string Proto) const1008 std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
1009                                             std::string Proto) const {
1010   std::string Ret = Name;
1011   while (Ret.find('{') != std::string::npos) {
1012     size_t Pos = Ret.find('{');
1013     size_t End = Ret.find('}');
1014     unsigned NumChars = End - Pos + 1;
1015     assert(NumChars == 3 && "Unexpected template argument");
1016 
1017     SVEType T;
1018     char C = Ret[Pos+1];
1019     switch(C) {
1020     default:
1021       llvm_unreachable("Unknown predication specifier");
1022     case 'd':
1023       T = SVEType(TS, 'd');
1024       break;
1025     case '0':
1026     case '1':
1027     case '2':
1028     case '3':
1029       T = SVEType(TS, Proto[C - '0']);
1030       break;
1031     }
1032 
1033     // Replace templated arg with the right suffix (e.g. u32)
1034     std::string TypeCode;
1035     if (T.isInteger())
1036       TypeCode = T.isSigned() ? 's' : 'u';
1037     else if (T.isSvcount())
1038       TypeCode = 'c';
1039     else if (T.isPredicateVector())
1040       TypeCode = 'b';
1041     else if (T.isBFloat())
1042       TypeCode = "bf";
1043     else
1044       TypeCode = 'f';
1045     Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));
1046   }
1047 
1048   return Ret;
1049 }
1050 
mangleLLVMName() const1051 std::string Intrinsic::mangleLLVMName() const {
1052   std::string S = getLLVMName();
1053 
1054   // Replace all {d} like expressions with e.g. 'u32'
1055   return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto());
1056 }
1057 
mangleName(ClassKind LocalCK) const1058 std::string Intrinsic::mangleName(ClassKind LocalCK) const {
1059   std::string S = getName();
1060 
1061   if (LocalCK == ClassG) {
1062     // Remove the square brackets and everything in between.
1063     while (S.find('[') != std::string::npos) {
1064       auto Start = S.find('[');
1065       auto End = S.find(']');
1066       S.erase(Start, (End-Start)+1);
1067     }
1068   } else {
1069     // Remove the square brackets.
1070     while (S.find('[') != std::string::npos) {
1071       auto BrPos = S.find('[');
1072       if (BrPos != std::string::npos)
1073         S.erase(BrPos, 1);
1074       BrPos = S.find(']');
1075       if (BrPos != std::string::npos)
1076         S.erase(BrPos, 1);
1077     }
1078   }
1079 
1080   // Replace all {d} like expressions with e.g. 'u32'
1081   return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) +
1082          getMergeSuffix();
1083 }
1084 
emitIntrinsic(raw_ostream & OS,SVEEmitter & Emitter,ACLEKind Kind) const1085 void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
1086                               ACLEKind Kind) const {
1087   bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
1088 
1089   std::string FullName = mangleName(ClassS);
1090   std::string ProtoName = mangleName(getClassKind());
1091   OS << (IsOverloaded ? "__aio " : "__ai ")
1092      << "__attribute__((__clang_arm_builtin_alias(";
1093 
1094   switch (Kind) {
1095   case ACLEKind::SME:
1096     OS << "__builtin_sme_" << FullName << ")";
1097     break;
1098   case ACLEKind::SVE:
1099     OS << "__builtin_sve_" << FullName << ")";
1100     break;
1101   }
1102 
1103   OS << "))\n";
1104 
1105   OS << getTypes()[0].str() << " " << ProtoName << "(";
1106   for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
1107     if (I != 0)
1108       OS << ", ";
1109     OS << getTypes()[I + 1].str();
1110   }
1111   OS << ");\n";
1112 }
1113 
1114 //===----------------------------------------------------------------------===//
1115 // SVEEmitter implementation
1116 //===----------------------------------------------------------------------===//
encodeTypeFlags(const SVEType & T)1117 uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
1118   if (T.isFloat()) {
1119     switch (T.getElementSizeInBits()) {
1120     case 16:
1121       return encodeEltType("EltTyFloat16");
1122     case 32:
1123       return encodeEltType("EltTyFloat32");
1124     case 64:
1125       return encodeEltType("EltTyFloat64");
1126     default:
1127       llvm_unreachable("Unhandled float element bitwidth!");
1128     }
1129   }
1130 
1131   if (T.isBFloat()) {
1132     assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
1133     return encodeEltType("EltTyBFloat16");
1134   }
1135 
1136   if (T.isPredicateVector() || T.isSvcount()) {
1137     switch (T.getElementSizeInBits()) {
1138     case 8:
1139       return encodeEltType("EltTyBool8");
1140     case 16:
1141       return encodeEltType("EltTyBool16");
1142     case 32:
1143       return encodeEltType("EltTyBool32");
1144     case 64:
1145       return encodeEltType("EltTyBool64");
1146     default:
1147       llvm_unreachable("Unhandled predicate element bitwidth!");
1148     }
1149   }
1150 
1151   switch (T.getElementSizeInBits()) {
1152   case 8:
1153     return encodeEltType("EltTyInt8");
1154   case 16:
1155     return encodeEltType("EltTyInt16");
1156   case 32:
1157     return encodeEltType("EltTyInt32");
1158   case 64:
1159     return encodeEltType("EltTyInt64");
1160   case 128:
1161     return encodeEltType("EltTyInt128");
1162   default:
1163     llvm_unreachable("Unhandled integer element bitwidth!");
1164   }
1165 }
1166 
createIntrinsic(Record * R,SmallVectorImpl<std::unique_ptr<Intrinsic>> & Out)1167 void SVEEmitter::createIntrinsic(
1168     Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
1169   StringRef Name = R->getValueAsString("Name");
1170   StringRef Proto = R->getValueAsString("Prototype");
1171   StringRef Types = R->getValueAsString("Types");
1172   StringRef SVEGuard = R->getValueAsString("SVETargetGuard");
1173   StringRef SMEGuard = R->getValueAsString("SMETargetGuard");
1174   StringRef LLVMName = R->getValueAsString("LLVMIntrinsic");
1175   uint64_t Merge = R->getValueAsInt("Merge");
1176   StringRef MergeSuffix = R->getValueAsString("MergeSuffix");
1177   uint64_t MemEltType = R->getValueAsInt("MemEltType");
1178   std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags");
1179   std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks");
1180 
1181   int64_t Flags = 0;
1182   for (auto FlagRec : FlagsList)
1183     Flags |= FlagRec->getValueAsInt("Value");
1184 
1185   // Create a dummy TypeSpec for non-overloaded builtins.
1186   if (Types.empty()) {
1187     assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
1188            "Expect TypeSpec for overloaded builtin!");
1189     Types = "i";
1190   }
1191 
1192   // Extract type specs from string
1193   SmallVector<TypeSpec, 8> TypeSpecs;
1194   TypeSpec Acc;
1195   for (char I : Types) {
1196     Acc.push_back(I);
1197     if (islower(I)) {
1198       TypeSpecs.push_back(TypeSpec(Acc));
1199       Acc.clear();
1200     }
1201   }
1202 
1203   // Remove duplicate type specs.
1204   llvm::sort(TypeSpecs);
1205   TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()),
1206                   TypeSpecs.end());
1207 
1208   // Create an Intrinsic for each type spec.
1209   for (auto TS : TypeSpecs) {
1210     // Collate a list of range/option checks for the immediates.
1211     SmallVector<ImmCheck, 2> ImmChecks;
1212     for (auto *R : ImmCheckList) {
1213       int64_t Arg = R->getValueAsInt("Arg");
1214       int64_t EltSizeArg = R->getValueAsInt("EltSizeArg");
1215       int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value");
1216       assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative");
1217 
1218       unsigned ElementSizeInBits = 0;
1219       char Mod;
1220       unsigned NumVectors;
1221       std::tie(Mod, NumVectors) = getProtoModifier(Proto, EltSizeArg + 1);
1222       if (EltSizeArg >= 0)
1223         ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
1224       ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits));
1225     }
1226 
1227     Out.push_back(std::make_unique<Intrinsic>(
1228         Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks,
1229         TS, ClassS, *this, SVEGuard, SMEGuard));
1230 
1231     // Also generate the short-form (e.g. svadd_m) for the given type-spec.
1232     if (Intrinsic::isOverloadedIntrinsic(Name))
1233       Out.push_back(std::make_unique<Intrinsic>(
1234           Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags,
1235           ImmChecks, TS, ClassG, *this, SVEGuard, SMEGuard));
1236   }
1237 }
1238 
createCoreHeaderIntrinsics(raw_ostream & OS,SVEEmitter & Emitter,ACLEKind Kind)1239 void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
1240                                             SVEEmitter &Emitter,
1241                                             ACLEKind Kind) {
1242   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1243   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1244   for (auto *R : RV)
1245     createIntrinsic(R, Defs);
1246 
1247   // Sort intrinsics in header file by following order/priority:
1248   // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
1249   // - Class (is intrinsic overloaded or not)
1250   // - Intrinsic name
1251   std::stable_sort(Defs.begin(), Defs.end(),
1252                    [](const std::unique_ptr<Intrinsic> &A,
1253                       const std::unique_ptr<Intrinsic> &B) {
1254                      auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1255                        return std::make_tuple(
1256                            I->getSVEGuard().str() + I->getSMEGuard().str(),
1257                            (unsigned)I->getClassKind(), I->getName());
1258                      };
1259                      return ToTuple(A) < ToTuple(B);
1260                    });
1261 
1262   // Actually emit the intrinsic declarations.
1263   for (auto &I : Defs)
1264     I->emitIntrinsic(OS, Emitter, Kind);
1265 }
1266 
createHeader(raw_ostream & OS)1267 void SVEEmitter::createHeader(raw_ostream &OS) {
1268   OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
1269         "-----------------------------------===\n"
1270         " *\n"
1271         " *\n"
1272         " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1273         "Exceptions.\n"
1274         " * See https://llvm.org/LICENSE.txt for license information.\n"
1275         " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1276         " *\n"
1277         " *===-----------------------------------------------------------------"
1278         "------===\n"
1279         " */\n\n";
1280 
1281   OS << "#ifndef __ARM_SVE_H\n";
1282   OS << "#define __ARM_SVE_H\n\n";
1283 
1284   OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1285   OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
1286   OS << "#endif\n";
1287 
1288   OS << "#include <stdint.h>\n\n";
1289   OS << "#ifdef  __cplusplus\n";
1290   OS << "extern \"C\" {\n";
1291   OS << "#else\n";
1292   OS << "#include <stdbool.h>\n";
1293   OS << "#endif\n\n";
1294 
1295   OS << "typedef __fp16 float16_t;\n";
1296   OS << "typedef float float32_t;\n";
1297   OS << "typedef double float64_t;\n";
1298 
1299   OS << "typedef __SVInt8_t svint8_t;\n";
1300   OS << "typedef __SVInt16_t svint16_t;\n";
1301   OS << "typedef __SVInt32_t svint32_t;\n";
1302   OS << "typedef __SVInt64_t svint64_t;\n";
1303   OS << "typedef __SVUint8_t svuint8_t;\n";
1304   OS << "typedef __SVUint16_t svuint16_t;\n";
1305   OS << "typedef __SVUint32_t svuint32_t;\n";
1306   OS << "typedef __SVUint64_t svuint64_t;\n";
1307   OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
1308 
1309   OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
1310 
1311   OS << "#include <arm_bf16.h>\n";
1312   OS << "#include <arm_vector_types.h>\n";
1313 
1314   OS << "typedef __SVFloat32_t svfloat32_t;\n";
1315   OS << "typedef __SVFloat64_t svfloat64_t;\n";
1316   OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
1317   OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
1318   OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
1319   OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
1320   OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
1321   OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
1322   OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
1323   OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
1324   OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
1325   OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
1326   OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
1327   OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
1328   OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
1329   OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
1330   OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
1331   OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
1332   OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
1333   OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
1334   OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
1335   OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
1336   OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
1337   OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
1338   OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
1339   OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
1340   OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
1341   OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
1342   OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
1343   OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
1344   OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
1345   OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
1346   OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
1347   OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
1348   OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
1349   OS << "typedef __SVBool_t  svbool_t;\n";
1350   OS << "typedef __clang_svboolx2_t  svboolx2_t;\n";
1351   OS << "typedef __clang_svboolx4_t  svboolx4_t;\n\n";
1352 
1353   OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
1354   OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
1355   OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
1356 
1357   OS << "typedef __SVCount_t svcount_t;\n\n";
1358 
1359   OS << "enum svpattern\n";
1360   OS << "{\n";
1361   OS << "  SV_POW2 = 0,\n";
1362   OS << "  SV_VL1 = 1,\n";
1363   OS << "  SV_VL2 = 2,\n";
1364   OS << "  SV_VL3 = 3,\n";
1365   OS << "  SV_VL4 = 4,\n";
1366   OS << "  SV_VL5 = 5,\n";
1367   OS << "  SV_VL6 = 6,\n";
1368   OS << "  SV_VL7 = 7,\n";
1369   OS << "  SV_VL8 = 8,\n";
1370   OS << "  SV_VL16 = 9,\n";
1371   OS << "  SV_VL32 = 10,\n";
1372   OS << "  SV_VL64 = 11,\n";
1373   OS << "  SV_VL128 = 12,\n";
1374   OS << "  SV_VL256 = 13,\n";
1375   OS << "  SV_MUL4 = 29,\n";
1376   OS << "  SV_MUL3 = 30,\n";
1377   OS << "  SV_ALL = 31\n";
1378   OS << "};\n\n";
1379 
1380   OS << "enum svprfop\n";
1381   OS << "{\n";
1382   OS << "  SV_PLDL1KEEP = 0,\n";
1383   OS << "  SV_PLDL1STRM = 1,\n";
1384   OS << "  SV_PLDL2KEEP = 2,\n";
1385   OS << "  SV_PLDL2STRM = 3,\n";
1386   OS << "  SV_PLDL3KEEP = 4,\n";
1387   OS << "  SV_PLDL3STRM = 5,\n";
1388   OS << "  SV_PSTL1KEEP = 8,\n";
1389   OS << "  SV_PSTL1STRM = 9,\n";
1390   OS << "  SV_PSTL2KEEP = 10,\n";
1391   OS << "  SV_PSTL2STRM = 11,\n";
1392   OS << "  SV_PSTL3KEEP = 12,\n";
1393   OS << "  SV_PSTL3STRM = 13\n";
1394   OS << "};\n\n";
1395 
1396   OS << "/* Function attributes */\n";
1397   OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1398         "__nodebug__))\n\n";
1399   OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1400         "__nodebug__, __overloadable__))\n\n";
1401 
1402   // Add reinterpret functions.
1403   for (auto [N, Suffix] :
1404        std::initializer_list<std::pair<unsigned, const char *>>{
1405            {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1406     for (auto ShortForm : {false, true})
1407       for (const ReinterpretTypeInfo &To : Reinterprets) {
1408         SVEType ToV(To.BaseType, N);
1409         for (const ReinterpretTypeInfo &From : Reinterprets) {
1410           SVEType FromV(From.BaseType, N);
1411           OS << "__aio "
1412                 "__attribute__((__clang_arm_builtin_alias(__builtin_sve_"
1413                 "reinterpret_"
1414              << To.Suffix << "_" << From.Suffix << Suffix << ")))\n"
1415              << ToV.str() << " svreinterpret_" << To.Suffix;
1416           if (!ShortForm)
1417             OS << "_" << From.Suffix << Suffix;
1418           OS << "(" << FromV.str() << " op);\n";
1419         }
1420       }
1421   }
1422 
1423   createCoreHeaderIntrinsics(OS, *this, ACLEKind::SVE);
1424 
1425   OS << "#define svcvtnt_bf16_x      svcvtnt_bf16_m\n";
1426   OS << "#define svcvtnt_bf16_f32_x  svcvtnt_bf16_f32_m\n";
1427 
1428   OS << "#define svcvtnt_f16_x      svcvtnt_f16_m\n";
1429   OS << "#define svcvtnt_f16_f32_x  svcvtnt_f16_f32_m\n";
1430   OS << "#define svcvtnt_f32_x      svcvtnt_f32_m\n";
1431   OS << "#define svcvtnt_f32_f64_x  svcvtnt_f32_f64_m\n\n";
1432 
1433   OS << "#define svcvtxnt_f32_x     svcvtxnt_f32_m\n";
1434   OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
1435 
1436   OS << "#ifdef __cplusplus\n";
1437   OS << "} // extern \"C\"\n";
1438   OS << "#endif\n\n";
1439   OS << "#undef __ai\n\n";
1440   OS << "#undef __aio\n\n";
1441   OS << "#endif /* __ARM_SVE_H */\n";
1442 }
1443 
createBuiltins(raw_ostream & OS)1444 void SVEEmitter::createBuiltins(raw_ostream &OS) {
1445   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1446   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1447   for (auto *R : RV)
1448     createIntrinsic(R, Defs);
1449 
1450   // The mappings must be sorted based on BuiltinID.
1451   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1452                       const std::unique_ptr<Intrinsic> &B) {
1453     return A->getMangledName() < B->getMangledName();
1454   });
1455 
1456   OS << "#ifdef GET_SVE_BUILTINS\n";
1457   for (auto &Def : Defs) {
1458     // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1459     // declarations only live in the header file.
1460     if (Def->getClassKind() != ClassG) {
1461       OS << "TARGET_BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \""
1462          << Def->getBuiltinTypeStr() << "\", \"n\", \"";
1463       Def->printGuard(OS);
1464       OS << "\")\n";
1465     }
1466   }
1467 
1468   // Add reinterpret functions.
1469   for (auto [N, Suffix] :
1470        std::initializer_list<std::pair<unsigned, const char *>>{
1471            {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1472     for (const ReinterpretTypeInfo &To : Reinterprets) {
1473       SVEType ToV(To.BaseType, N);
1474       for (const ReinterpretTypeInfo &From : Reinterprets) {
1475         SVEType FromV(From.BaseType, N);
1476         OS << "TARGET_BUILTIN(__builtin_sve_reinterpret_" << To.Suffix << "_"
1477            << From.Suffix << Suffix << +", \"" << ToV.builtin_str()
1478            << FromV.builtin_str() << "\", \"n\", \"sme|sve\")\n";
1479       }
1480     }
1481   }
1482 
1483   OS << "#endif\n\n";
1484 }
1485 
createCodeGenMap(raw_ostream & OS)1486 void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
1487   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1488   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1489   for (auto *R : RV)
1490     createIntrinsic(R, Defs);
1491 
1492   // The mappings must be sorted based on BuiltinID.
1493   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1494                       const std::unique_ptr<Intrinsic> &B) {
1495     return A->getMangledName() < B->getMangledName();
1496   });
1497 
1498   OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
1499   for (auto &Def : Defs) {
1500     // Builtins only exist for non-overloaded intrinsics, overloaded
1501     // declarations only live in the header file.
1502     if (Def->getClassKind() == ClassG)
1503       continue;
1504 
1505     uint64_t Flags = Def->getFlags();
1506     auto FlagString = std::to_string(Flags);
1507 
1508     std::string LLVMName = Def->getMangledLLVMName();
1509     std::string Builtin = Def->getMangledName();
1510     if (!LLVMName.empty())
1511       OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1512          << "),\n";
1513     else
1514       OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
1515   }
1516   OS << "#endif\n\n";
1517 }
1518 
createRangeChecks(raw_ostream & OS)1519 void SVEEmitter::createRangeChecks(raw_ostream &OS) {
1520   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1521   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1522   for (auto *R : RV)
1523     createIntrinsic(R, Defs);
1524 
1525   // The mappings must be sorted based on BuiltinID.
1526   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1527                       const std::unique_ptr<Intrinsic> &B) {
1528     return A->getMangledName() < B->getMangledName();
1529   });
1530 
1531 
1532   OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
1533 
1534   // Ensure these are only emitted once.
1535   std::set<std::string> Emitted;
1536 
1537   for (auto &Def : Defs) {
1538     if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1539         Def->getImmChecks().empty())
1540       continue;
1541 
1542     OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
1543     for (auto &Check : Def->getImmChecks())
1544       OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
1545          << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n";
1546     OS << "  break;\n";
1547 
1548     Emitted.insert(Def->getMangledName());
1549   }
1550 
1551   OS << "#endif\n\n";
1552 }
1553 
1554 /// Create the SVETypeFlags used in CGBuiltins
createTypeFlags(raw_ostream & OS)1555 void SVEEmitter::createTypeFlags(raw_ostream &OS) {
1556   OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
1557   for (auto &KV : FlagTypes)
1558     OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
1559   OS << "#endif\n\n";
1560 
1561   OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
1562   for (auto &KV : EltTypes)
1563     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1564   OS << "#endif\n\n";
1565 
1566   OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
1567   for (auto &KV : MemEltTypes)
1568     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1569   OS << "#endif\n\n";
1570 
1571   OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
1572   for (auto &KV : MergeTypes)
1573     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1574   OS << "#endif\n\n";
1575 
1576   OS << "#ifdef LLVM_GET_SVE_IMMCHECKTYPES\n";
1577   for (auto &KV : ImmCheckTypes)
1578     OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
1579   OS << "#endif\n\n";
1580 }
1581 
createSMEHeader(raw_ostream & OS)1582 void SVEEmitter::createSMEHeader(raw_ostream &OS) {
1583   OS << "/*===---- arm_sme.h - ARM SME intrinsics "
1584         "------===\n"
1585         " *\n"
1586         " *\n"
1587         " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1588         "Exceptions.\n"
1589         " * See https://llvm.org/LICENSE.txt for license information.\n"
1590         " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1591         " *\n"
1592         " *===-----------------------------------------------------------------"
1593         "------===\n"
1594         " */\n\n";
1595 
1596   OS << "#ifndef __ARM_SME_H\n";
1597   OS << "#define __ARM_SME_H\n\n";
1598 
1599   OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1600   OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";
1601   OS << "#endif\n";
1602 
1603   OS << "#include <arm_sve.h>\n\n";
1604   OS << "#include <stddef.h>\n\n";
1605 
1606   OS << "/* Function attributes */\n";
1607   OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1608         "__nodebug__))\n\n";
1609   OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1610         "__nodebug__, __overloadable__))\n\n";
1611 
1612   OS << "#ifdef  __cplusplus\n";
1613   OS << "extern \"C\" {\n";
1614   OS << "#endif\n\n";
1615 
1616   OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";
1617 
1618   OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";
1619   OS << "  uint64_t x0, x1;\n";
1620   OS << "  __builtin_arm_get_sme_state(&x0, &x1);\n";
1621   OS << "  return x0 & (1ULL << 63);\n";
1622   OS << "}\n\n";
1623 
1624   OS << "__ai bool __arm_in_streaming_mode(void) __arm_streaming_compatible "
1625         "{\n";
1626   OS << "  uint64_t x0, x1;\n";
1627   OS << "  __builtin_arm_get_sme_state(&x0, &x1);\n";
1628   OS << "  return x0 & 1;\n";
1629   OS << "}\n\n";
1630 
1631   OS << "void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1632   OS << "void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1633   OS << "void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;\n";
1634   OS << "void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;\n\n";
1635 
1636   OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "
1637         "__arm_streaming_compatible __arm_out(\"za\") "
1638         "{ }\n\n";
1639 
1640   createCoreHeaderIntrinsics(OS, *this, ACLEKind::SME);
1641 
1642   OS << "#ifdef __cplusplus\n";
1643   OS << "} // extern \"C\"\n";
1644   OS << "#endif\n\n";
1645   OS << "#undef __ai\n\n";
1646   OS << "#endif /* __ARM_SME_H */\n";
1647 }
1648 
createSMEBuiltins(raw_ostream & OS)1649 void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
1650   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1651   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1652   for (auto *R : RV) {
1653     createIntrinsic(R, Defs);
1654   }
1655 
1656   // The mappings must be sorted based on BuiltinID.
1657   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1658                       const std::unique_ptr<Intrinsic> &B) {
1659     return A->getMangledName() < B->getMangledName();
1660   });
1661 
1662   OS << "#ifdef GET_SME_BUILTINS\n";
1663   for (auto &Def : Defs) {
1664     // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1665     // declarations only live in the header file.
1666     if (Def->getClassKind() != ClassG) {
1667       OS << "TARGET_BUILTIN(__builtin_sme_" << Def->getMangledName() << ", \""
1668          << Def->getBuiltinTypeStr() << "\", \"n\", \"";
1669       Def->printGuard(OS);
1670       OS << "\")\n";
1671     }
1672   }
1673 
1674   OS << "#endif\n\n";
1675 }
1676 
createSMECodeGenMap(raw_ostream & OS)1677 void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
1678   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1679   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1680   for (auto *R : RV) {
1681     createIntrinsic(R, Defs);
1682   }
1683 
1684   // The mappings must be sorted based on BuiltinID.
1685   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1686                       const std::unique_ptr<Intrinsic> &B) {
1687     return A->getMangledName() < B->getMangledName();
1688   });
1689 
1690   OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
1691   for (auto &Def : Defs) {
1692     // Builtins only exist for non-overloaded intrinsics, overloaded
1693     // declarations only live in the header file.
1694     if (Def->getClassKind() == ClassG)
1695       continue;
1696 
1697     uint64_t Flags = Def->getFlags();
1698     auto FlagString = std::to_string(Flags);
1699 
1700     std::string LLVMName = Def->getLLVMName();
1701     std::string Builtin = Def->getMangledName();
1702     if (!LLVMName.empty())
1703       OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1704          << "),\n";
1705     else
1706       OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
1707   }
1708   OS << "#endif\n\n";
1709 }
1710 
createSMERangeChecks(raw_ostream & OS)1711 void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
1712   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1713   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1714   for (auto *R : RV) {
1715     createIntrinsic(R, Defs);
1716   }
1717 
1718   // The mappings must be sorted based on BuiltinID.
1719   llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1720                       const std::unique_ptr<Intrinsic> &B) {
1721     return A->getMangledName() < B->getMangledName();
1722   });
1723 
1724 
1725   OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
1726 
1727   // Ensure these are only emitted once.
1728   std::set<std::string> Emitted;
1729 
1730   for (auto &Def : Defs) {
1731     if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1732         Def->getImmChecks().empty())
1733       continue;
1734 
1735     OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
1736     for (auto &Check : Def->getImmChecks())
1737       OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
1738          << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n";
1739     OS << "  break;\n";
1740 
1741     Emitted.insert(Def->getMangledName());
1742   }
1743 
1744   OS << "#endif\n\n";
1745 }
1746 
createBuiltinZAState(raw_ostream & OS)1747 void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
1748   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1749   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1750   for (auto *R : RV)
1751     createIntrinsic(R, Defs);
1752 
1753   std::map<std::string, std::set<std::string>> IntrinsicsPerState;
1754   for (auto &Def : Defs) {
1755     std::string Key;
1756     auto AddToKey = [&Key](const std::string &S) -> void {
1757       Key = Key.empty() ? S : (Key + " | " + S);
1758     };
1759 
1760     if (Def->isFlagSet(getEnumValueForFlag("IsInZA")))
1761       AddToKey("ArmInZA");
1762     else if (Def->isFlagSet(getEnumValueForFlag("IsOutZA")))
1763       AddToKey("ArmOutZA");
1764     else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZA")))
1765       AddToKey("ArmInOutZA");
1766 
1767     if (Def->isFlagSet(getEnumValueForFlag("IsInZT0")))
1768       AddToKey("ArmInZT0");
1769     else if (Def->isFlagSet(getEnumValueForFlag("IsOutZT0")))
1770       AddToKey("ArmOutZT0");
1771     else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZT0")))
1772       AddToKey("ArmInOutZT0");
1773 
1774     if (!Key.empty())
1775       IntrinsicsPerState[Key].insert(Def->getMangledName());
1776   }
1777 
1778   OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";
1779   for (auto &KV : IntrinsicsPerState) {
1780     for (StringRef Name : KV.second)
1781       OS << "case SME::BI__builtin_sme_" << Name << ":\n";
1782     OS << "  return " << KV.first << ";\n";
1783   }
1784   OS << "#endif\n\n";
1785 }
1786 
createStreamingAttrs(raw_ostream & OS,ACLEKind Kind)1787 void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
1788   std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
1789   SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1790   for (auto *R : RV)
1791     createIntrinsic(R, Defs);
1792 
1793   StringRef ExtensionKind;
1794   switch (Kind) {
1795   case ACLEKind::SME:
1796     ExtensionKind = "SME";
1797     break;
1798   case ACLEKind::SVE:
1799     ExtensionKind = "SVE";
1800     break;
1801   }
1802 
1803   OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";
1804 
1805   llvm::StringMap<std::set<std::string>> StreamingMap;
1806 
1807   uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming");
1808   uint64_t VerifyRuntimeMode = getEnumValueForFlag("VerifyRuntimeMode");
1809   uint64_t IsStreamingCompatibleFlag =
1810       getEnumValueForFlag("IsStreamingCompatible");
1811 
1812   for (auto &Def : Defs) {
1813     if (!Def->isFlagSet(VerifyRuntimeMode) && !Def->getSVEGuard().empty() &&
1814         !Def->getSMEGuard().empty())
1815       report_fatal_error("Missing VerifyRuntimeMode flag");
1816 
1817     if (Def->isFlagSet(IsStreamingFlag))
1818       StreamingMap["ArmStreaming"].insert(Def->getMangledName());
1819     else if (Def->isFlagSet(VerifyRuntimeMode))
1820       StreamingMap["VerifyRuntimeMode"].insert(Def->getMangledName());
1821     else if (Def->isFlagSet(IsStreamingCompatibleFlag))
1822       StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName());
1823     else
1824       StreamingMap["ArmNonStreaming"].insert(Def->getMangledName());
1825   }
1826 
1827   for (auto BuiltinType : StreamingMap.keys()) {
1828     for (auto Name : StreamingMap[BuiltinType]) {
1829       OS << "case " << ExtensionKind << "::BI__builtin_"
1830          << ExtensionKind.lower() << "_";
1831       OS << Name << ":\n";
1832     }
1833     OS << "  BuiltinType = " << BuiltinType << ";\n";
1834     OS << "  break;\n";
1835   }
1836 
1837   OS << "#endif\n\n";
1838 }
1839 
1840 namespace clang {
EmitSveHeader(RecordKeeper & Records,raw_ostream & OS)1841 void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) {
1842   SVEEmitter(Records).createHeader(OS);
1843 }
1844 
EmitSveBuiltins(RecordKeeper & Records,raw_ostream & OS)1845 void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) {
1846   SVEEmitter(Records).createBuiltins(OS);
1847 }
1848 
EmitSveBuiltinCG(RecordKeeper & Records,raw_ostream & OS)1849 void EmitSveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
1850   SVEEmitter(Records).createCodeGenMap(OS);
1851 }
1852 
EmitSveRangeChecks(RecordKeeper & Records,raw_ostream & OS)1853 void EmitSveRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
1854   SVEEmitter(Records).createRangeChecks(OS);
1855 }
1856 
EmitSveTypeFlags(RecordKeeper & Records,raw_ostream & OS)1857 void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) {
1858   SVEEmitter(Records).createTypeFlags(OS);
1859 }
1860 
EmitSveStreamingAttrs(RecordKeeper & Records,raw_ostream & OS)1861 void EmitSveStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
1862   SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SVE);
1863 }
1864 
EmitSmeHeader(RecordKeeper & Records,raw_ostream & OS)1865 void EmitSmeHeader(RecordKeeper &Records, raw_ostream &OS) {
1866   SVEEmitter(Records).createSMEHeader(OS);
1867 }
1868 
EmitSmeBuiltins(RecordKeeper & Records,raw_ostream & OS)1869 void EmitSmeBuiltins(RecordKeeper &Records, raw_ostream &OS) {
1870   SVEEmitter(Records).createSMEBuiltins(OS);
1871 }
1872 
EmitSmeBuiltinCG(RecordKeeper & Records,raw_ostream & OS)1873 void EmitSmeBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
1874   SVEEmitter(Records).createSMECodeGenMap(OS);
1875 }
1876 
EmitSmeRangeChecks(RecordKeeper & Records,raw_ostream & OS)1877 void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
1878   SVEEmitter(Records).createSMERangeChecks(OS);
1879 }
1880 
EmitSmeStreamingAttrs(RecordKeeper & Records,raw_ostream & OS)1881 void EmitSmeStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
1882   SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME);
1883 }
1884 
EmitSmeBuiltinZAState(RecordKeeper & Records,raw_ostream & OS)1885 void EmitSmeBuiltinZAState(RecordKeeper &Records, raw_ostream &OS) {
1886   SVEEmitter(Records).createBuiltinZAState(OS);
1887 }
1888 } // End namespace clang
1889