1 //===-- SveEmitter.cpp - Generate arm_sve.h for use with clang ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This tablegen backend is responsible for emitting arm_sve.h, which includes
10 // a declaration and definition of each function specified by the ARM C/C++
11 // Language Extensions (ACLE).
12 //
13 // For details, visit:
14 // https://developer.arm.com/architectures/system-architectures/software-standards/acle
15 //
16 // Each SVE instruction is implemented in terms of 1 or more functions which
17 // are suffixed with the element type of the input vectors. Functions may be
18 // implemented in terms of generic vector operations such as +, *, -, etc. or
19 // by calling a __builtin_-prefixed function which will be handled by clang's
20 // CodeGen library.
21 //
22 // See also the documentation in include/clang/Basic/arm_sve.td.
23 //
24 //===----------------------------------------------------------------------===//
25
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/STLExtras.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/ADT/StringMap.h"
30 #include "llvm/Support/raw_ostream.h"
31 #include "llvm/TableGen/AArch64ImmCheck.h"
32 #include "llvm/TableGen/Error.h"
33 #include "llvm/TableGen/Record.h"
34 #include "llvm/TableGen/StringToOffsetTable.h"
35 #include <array>
36 #include <cctype>
37 #include <set>
38 #include <string>
39 #include <tuple>
40
41 using namespace llvm;
42
43 enum ClassKind {
44 ClassNone,
45 ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix
46 ClassG, // Overloaded name without type suffix
47 };
48
49 enum class ACLEKind { SVE, SME };
50
51 using TypeSpec = std::string;
52
53 namespace {
54 class SVEType {
55
56 enum TypeKind {
57 Invalid,
58 Void,
59 Float,
60 SInt,
61 UInt,
62 BFloat16,
63 MFloat8,
64 Svcount,
65 PrefetchOp,
66 PredicatePattern,
67 Predicate,
68 Fpm
69 };
70 TypeKind Kind;
71 bool Immediate, Constant, Pointer, DefaultType, IsScalable;
72 unsigned Bitwidth, ElementBitwidth, NumVectors;
73
74 public:
SVEType()75 SVEType() : SVEType("", 'v') {}
76
SVEType(StringRef TS,char CharMod,unsigned NumVectors=1)77 SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
78 : Kind(Invalid), Immediate(false), Constant(false), Pointer(false),
79 DefaultType(false), IsScalable(true), Bitwidth(128),
80 ElementBitwidth(~0U), NumVectors(NumVectors) {
81 if (!TS.empty())
82 applyTypespec(TS);
83 applyModifier(CharMod);
84 }
85
SVEType(const SVEType & Base,unsigned NumV)86 SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
87 NumVectors = NumV;
88 }
89
isPointer() const90 bool isPointer() const { return Pointer; }
isConstant() const91 bool isConstant() const { return Constant; }
isImmediate() const92 bool isImmediate() const { return Immediate; }
isScalar() const93 bool isScalar() const { return NumVectors == 0; }
isVector() const94 bool isVector() const { return NumVectors > 0; }
isScalableVector() const95 bool isScalableVector() const { return isVector() && IsScalable; }
isFixedLengthVector() const96 bool isFixedLengthVector() const { return isVector() && !IsScalable; }
isChar() const97 bool isChar() const { return ElementBitwidth == 8 && isInteger(); }
isVoid() const98 bool isVoid() const { return Kind == Void; }
isDefault() const99 bool isDefault() const { return DefaultType; }
isFloat() const100 bool isFloat() const { return Kind == Float; }
isBFloat() const101 bool isBFloat() const { return Kind == BFloat16; }
isMFloat() const102 bool isMFloat() const { return Kind == MFloat8; }
isFloatingPoint() const103 bool isFloatingPoint() const {
104 return Kind == Float || Kind == BFloat16 || Kind == MFloat8;
105 }
isInteger() const106 bool isInteger() const { return Kind == SInt || Kind == UInt; }
isSignedInteger() const107 bool isSignedInteger() const { return Kind == SInt; }
isUnsignedInteger() const108 bool isUnsignedInteger() const { return Kind == UInt; }
isScalarPredicate() const109 bool isScalarPredicate() const {
110 return Kind == Predicate && NumVectors == 0;
111 }
isPredicate() const112 bool isPredicate() const { return Kind == Predicate; }
isPredicatePattern() const113 bool isPredicatePattern() const { return Kind == PredicatePattern; }
isPrefetchOp() const114 bool isPrefetchOp() const { return Kind == PrefetchOp; }
isSvcount() const115 bool isSvcount() const { return Kind == Svcount; }
isFpm() const116 bool isFpm() const { return Kind == Fpm; }
isInvalid() const117 bool isInvalid() const { return Kind == Invalid; }
getElementSizeInBits() const118 unsigned getElementSizeInBits() const { return ElementBitwidth; }
getNumVectors() const119 unsigned getNumVectors() const { return NumVectors; }
120
getNumElements() const121 unsigned getNumElements() const {
122 assert(ElementBitwidth != ~0U);
123 return isPredicate() ? 16 : (Bitwidth / ElementBitwidth);
124 }
getSizeInBits() const125 unsigned getSizeInBits() const {
126 return Bitwidth;
127 }
128
129 /// Return the string representation of a type, which is an encoded
130 /// string for passing to the BUILTIN() macro in Builtins.def.
131 std::string builtin_str() const;
132
133 /// Return the C/C++ string representation of a type for use in the
134 /// arm_sve.h header file.
135 std::string str() const;
136
137 private:
138 /// Creates the type based on the typespec string in TS.
139 void applyTypespec(StringRef TS);
140
141 /// Applies a prototype modifier to the type.
142 void applyModifier(char Mod);
143
144 /// Get the builtin base for this SVEType, e.g. 'Wi' for svint64_t.
145 std::string builtinBaseType() const;
146 };
147
148 class SVEEmitter;
149
150 /// The main grunt class. This represents an instantiation of an intrinsic with
151 /// a particular typespec and prototype.
152 class Intrinsic {
153 /// The unmangled name.
154 std::string Name;
155
156 /// The name of the corresponding LLVM IR intrinsic.
157 std::string LLVMName;
158
159 /// Intrinsic prototype.
160 std::string Proto;
161
162 /// The base type spec for this intrinsic.
163 TypeSpec BaseTypeSpec;
164
165 /// The base class kind. Most intrinsics use ClassS, which has full type
166 /// info for integers (_s32/_u32), or ClassG which is used for overloaded
167 /// intrinsics.
168 ClassKind Class;
169
170 /// The architectural #ifdef guard.
171 std::string SVEGuard, SMEGuard;
172
173 // The merge suffix such as _m, _x or _z.
174 std::string MergeSuffix;
175
176 /// The types of return value [0] and parameters [1..].
177 std::vector<SVEType> Types;
178
179 /// The "base type", which is VarType('d', BaseTypeSpec).
180 SVEType BaseType;
181
182 uint64_t Flags;
183
184 SmallVector<ImmCheck, 2> ImmChecks;
185
186 bool SetsFPMR;
187
188 public:
189 Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
190 StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
191 uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
192 ClassKind Class, SVEEmitter &Emitter, StringRef SVEGuard,
193 StringRef SMEGuard);
194
195 ~Intrinsic()=default;
196
getName() const197 std::string getName() const { return Name; }
getLLVMName() const198 std::string getLLVMName() const { return LLVMName; }
getProto() const199 std::string getProto() const { return Proto; }
getBaseTypeSpec() const200 TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
getBaseType() const201 SVEType getBaseType() const { return BaseType; }
202
getSVEGuard() const203 StringRef getSVEGuard() const { return SVEGuard; }
getSMEGuard() const204 StringRef getSMEGuard() const { return SMEGuard; }
getGuard() const205 std::string getGuard() const {
206 std::string Guard;
207 llvm::raw_string_ostream OS(Guard);
208 if (!SVEGuard.empty() && SMEGuard.empty())
209 OS << SVEGuard;
210 else if (SVEGuard.empty() && !SMEGuard.empty())
211 OS << SMEGuard;
212 else {
213 if (SVEGuard.find(",") != std::string::npos ||
214 SVEGuard.find("|") != std::string::npos)
215 OS << "(" << SVEGuard << ")";
216 else
217 OS << SVEGuard;
218 OS << "|";
219 if (SMEGuard.find(",") != std::string::npos ||
220 SMEGuard.find("|") != std::string::npos)
221 OS << "(" << SMEGuard << ")";
222 else
223 OS << SMEGuard;
224 }
225 return Guard;
226 }
getClassKind() const227 ClassKind getClassKind() const { return Class; }
228
getReturnType() const229 SVEType getReturnType() const { return Types[0]; }
getTypes() const230 ArrayRef<SVEType> getTypes() const { return Types; }
getParamType(unsigned I) const231 SVEType getParamType(unsigned I) const { return Types[I + 1]; }
getNumParams() const232 unsigned getNumParams() const {
233 return Proto.size() - (2 * count(Proto, '.')) - 1;
234 }
235
getFlags() const236 uint64_t getFlags() const { return Flags; }
isFlagSet(uint64_t Flag) const237 bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
238
getImmChecks() const239 ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
240
241 /// Return the type string for a BUILTIN() macro in Builtins.def.
242 std::string getBuiltinTypeStr();
243
244 /// Return the name, mangled with type information. The name is mangled for
245 /// ClassS, so will add type suffixes such as _u32/_s32.
getMangledName() const246 std::string getMangledName() const { return mangleName(ClassS); }
247
248 /// As above, but mangles the LLVM name instead.
getMangledLLVMName() const249 std::string getMangledLLVMName() const { return mangleLLVMName(); }
250
251 /// Returns true if the intrinsic is overloaded, in that it should also generate
252 /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
253 /// 'svld1_u32(..)'.
isOverloadedIntrinsic(StringRef Name)254 static bool isOverloadedIntrinsic(StringRef Name) {
255 return Name.contains('[') && Name.contains(']');
256 }
257
258 /// Return true if the intrinsic takes a splat operand.
hasSplat() const259 bool hasSplat() const {
260 // These prototype modifiers are described in arm_sve.td.
261 return Proto.find_first_of("ajfrKLR@!") != std::string::npos;
262 }
263
264 /// Return the parameter index of the splat operand.
getSplatIdx() const265 unsigned getSplatIdx() const {
266 unsigned I = 1, Param = 0;
267 for (; I < Proto.size(); ++I, ++Param) {
268 if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
269 Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
270 Proto[I] == 'R' || Proto[I] == '@' || Proto[I] == '!')
271 break;
272
273 // Multivector modifier can be skipped
274 if (Proto[I] == '.')
275 I += 2;
276 }
277 assert(I != Proto.size() && "Prototype has no splat operand");
278 return Param;
279 }
280
281 /// Emits the intrinsic declaration to the ostream.
282 void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;
283
284 private:
getMergeSuffix() const285 std::string getMergeSuffix() const { return MergeSuffix; }
getFPMSuffix() const286 StringRef getFPMSuffix() const { return SetsFPMR ? "_fpm" : ""; }
287 std::string mangleName(ClassKind LocalCK) const;
288 std::string mangleLLVMName() const;
289 std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
290 std::string Proto) const;
291 };
292
293 class SVEEmitter {
294 private:
295 // The reinterpret builtins are generated separately because they
296 // need the cross product of all types (121 functions in total),
297 // which is inconvenient to specify in the arm_sve.td file or
298 // generate in CGBuiltin.cpp.
299 struct ReinterpretTypeInfo {
300 SVEType BaseType;
301 const char *Suffix;
302 };
303
304 static const std::array<ReinterpretTypeInfo, 13> Reinterprets;
305
306 const RecordKeeper &Records;
307 StringMap<uint64_t> EltTypes;
308 StringMap<uint64_t> MemEltTypes;
309 StringMap<uint64_t> FlagTypes;
310 StringMap<uint64_t> MergeTypes;
311 StringMap<uint64_t> ImmCheckTypes;
312
313 public:
SVEEmitter(const RecordKeeper & R)314 SVEEmitter(const RecordKeeper &R) : Records(R) {
315 for (auto *RV : Records.getAllDerivedDefinitions("EltType"))
316 EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
317 for (auto *RV : Records.getAllDerivedDefinitions("MemEltType"))
318 MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
319 for (auto *RV : Records.getAllDerivedDefinitions("FlagType"))
320 FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
321 for (auto *RV : Records.getAllDerivedDefinitions("MergeType"))
322 MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
323 for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType"))
324 ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
325 }
326
327 /// Returns the enum value for the immcheck type
getEnumValueForImmCheck(StringRef C) const328 unsigned getEnumValueForImmCheck(StringRef C) const {
329 auto It = ImmCheckTypes.find(C);
330 if (It != ImmCheckTypes.end())
331 return It->getValue();
332 llvm_unreachable("Unsupported imm check");
333 }
334
335 /// Returns the enum value for the flag type
getEnumValueForFlag(StringRef C) const336 uint64_t getEnumValueForFlag(StringRef C) const {
337 auto Res = FlagTypes.find(C);
338 if (Res != FlagTypes.end())
339 return Res->getValue();
340 llvm_unreachable("Unsupported flag");
341 }
342
343 // Returns the SVETypeFlags for a given value and mask.
encodeFlag(uint64_t V,StringRef MaskName) const344 uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
345 auto It = FlagTypes.find(MaskName);
346 if (It != FlagTypes.end()) {
347 uint64_t Mask = It->getValue();
348 unsigned Shift = countr_zero(Mask);
349 assert(Shift < 64 && "Mask value produced an invalid shift value");
350 return (V << Shift) & Mask;
351 }
352 llvm_unreachable("Unsupported flag");
353 }
354
355 // Returns the SVETypeFlags for the given element type.
encodeEltType(StringRef EltName)356 uint64_t encodeEltType(StringRef EltName) {
357 auto It = EltTypes.find(EltName);
358 if (It != EltTypes.end())
359 return encodeFlag(It->getValue(), "EltTypeMask");
360 llvm_unreachable("Unsupported EltType");
361 }
362
363 // Returns the SVETypeFlags for the given memory element type.
encodeMemoryElementType(uint64_t MT)364 uint64_t encodeMemoryElementType(uint64_t MT) {
365 return encodeFlag(MT, "MemEltTypeMask");
366 }
367
368 // Returns the SVETypeFlags for the given merge type.
encodeMergeType(uint64_t MT)369 uint64_t encodeMergeType(uint64_t MT) {
370 return encodeFlag(MT, "MergeTypeMask");
371 }
372
373 // Returns the SVETypeFlags for the given splat operand.
encodeSplatOperand(unsigned SplatIdx)374 unsigned encodeSplatOperand(unsigned SplatIdx) {
375 assert(SplatIdx < 7 && "SplatIdx out of encodable range");
376 return encodeFlag(SplatIdx + 1, "SplatOperandMask");
377 }
378
379 // Returns the SVETypeFlags value for the given SVEType.
380 uint64_t encodeTypeFlags(const SVEType &T);
381
382 /// Emit arm_sve.h.
383 void createHeader(raw_ostream &o);
384
385 // Emits core intrinsics in both arm_sme.h and arm_sve.h
386 void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
387 ACLEKind Kind);
388
389 /// Emit all the __builtin prototypes and code needed by Sema.
390 void createBuiltins(raw_ostream &o);
391
392 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
393 void createCodeGenMap(raw_ostream &o);
394
395 /// Emit all the range checks for the immediates.
396 void createRangeChecks(raw_ostream &o);
397
398 // Emit all the ImmCheckTypes to arm_immcheck_types.inc
399 void createImmCheckTypes(raw_ostream &OS);
400
401 /// Create the SVETypeFlags used in CGBuiltins
402 void createTypeFlags(raw_ostream &o);
403
404 /// Emit arm_sme.h.
405 void createSMEHeader(raw_ostream &o);
406
407 /// Emit all the SME __builtin prototypes and code needed by Sema.
408 void createSMEBuiltins(raw_ostream &o);
409
410 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
411 void createSMECodeGenMap(raw_ostream &o);
412
413 /// Create a table for a builtin's requirement for PSTATE.SM.
414 void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);
415
416 /// Emit all the range checks for the immediates.
417 void createSMERangeChecks(raw_ostream &o);
418
419 /// Create a table for a builtin's requirement for PSTATE.ZA.
420 void createBuiltinZAState(raw_ostream &OS);
421
422 /// Create intrinsic and add it to \p Out
423 void createIntrinsic(const Record *R,
424 SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
425 };
426
427 const std::array<SVEEmitter::ReinterpretTypeInfo, 13> SVEEmitter::Reinterprets =
428 {{{SVEType("c", 'd'), "s8"},
429 {SVEType("Uc", 'd'), "u8"},
430 {SVEType("m", 'd'), "mf8"},
431 {SVEType("s", 'd'), "s16"},
432 {SVEType("Us", 'd'), "u16"},
433 {SVEType("i", 'd'), "s32"},
434 {SVEType("Ui", 'd'), "u32"},
435 {SVEType("l", 'd'), "s64"},
436 {SVEType("Ul", 'd'), "u64"},
437 {SVEType("h", 'd'), "f16"},
438 {SVEType("b", 'd'), "bf16"},
439 {SVEType("f", 'd'), "f32"},
440 {SVEType("d", 'd'), "f64"}}};
441
442 } // end anonymous namespace
443
444 //===----------------------------------------------------------------------===//
445 // Type implementation
446 //===----------------------------------------------------------------------===//
447
builtinBaseType() const448 std::string SVEType::builtinBaseType() const {
449 switch (Kind) {
450 case TypeKind::Void:
451 return "v";
452 case TypeKind::Svcount:
453 return "Qa";
454 case TypeKind::PrefetchOp:
455 case TypeKind::PredicatePattern:
456 return "i";
457 case TypeKind::Fpm:
458 return "UWi";
459 case TypeKind::Predicate:
460 return "b";
461 case TypeKind::BFloat16:
462 assert(ElementBitwidth == 16 && "Invalid BFloat16!");
463 return "y";
464 case TypeKind::MFloat8:
465 assert(ElementBitwidth == 8 && "Invalid MFloat8!");
466 return "m";
467 case TypeKind::Float:
468 switch (ElementBitwidth) {
469 case 16:
470 return "h";
471 case 32:
472 return "f";
473 case 64:
474 return "d";
475 default:
476 llvm_unreachable("Unhandled float width!");
477 }
478 case TypeKind::SInt:
479 case TypeKind::UInt:
480 switch (ElementBitwidth) {
481 case 1:
482 return "b";
483 case 8:
484 return "c";
485 case 16:
486 return "s";
487 case 32:
488 return "i";
489 case 64:
490 return "Wi";
491 case 128:
492 return "LLLi";
493 default:
494 llvm_unreachable("Unhandled bitwidth!");
495 }
496 case TypeKind::Invalid:
497 llvm_unreachable("Attempting to resolve builtin string from Invalid type!");
498 }
499 llvm_unreachable("Unhandled TypeKind!");
500 }
501
builtin_str() const502 std::string SVEType::builtin_str() const {
503 std::string Prefix;
504
505 if (isScalableVector())
506 Prefix = "q" + llvm::utostr(getNumElements() * NumVectors);
507 else if (isFixedLengthVector())
508 Prefix = "V" + llvm::utostr(getNumElements() * NumVectors);
509 else if (isImmediate()) {
510 assert(!isFloatingPoint() && "fp immediates are not supported");
511 Prefix = "I";
512 }
513
514 // Make chars and integer pointers explicitly signed.
515 if ((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
516 Prefix += "S";
517 else if (isUnsignedInteger())
518 Prefix += "U";
519
520 std::string BuiltinStr = Prefix + builtinBaseType();
521 if (isConstant())
522 BuiltinStr += "C";
523 if (isPointer())
524 BuiltinStr += "*";
525
526 return BuiltinStr;
527 }
528
str() const529 std::string SVEType::str() const {
530 std::string TypeStr;
531
532 switch (Kind) {
533 case TypeKind::PrefetchOp:
534 return "enum svprfop";
535 case TypeKind::PredicatePattern:
536 return "enum svpattern";
537 case TypeKind::Fpm:
538 TypeStr += "fpm";
539 break;
540 case TypeKind::Void:
541 TypeStr += "void";
542 break;
543 case TypeKind::Float:
544 TypeStr += "float" + llvm::utostr(ElementBitwidth);
545 break;
546 case TypeKind::Svcount:
547 TypeStr += "svcount";
548 break;
549 case TypeKind::Predicate:
550 TypeStr += "bool";
551 break;
552 case TypeKind::BFloat16:
553 TypeStr += "bfloat16";
554 break;
555 case TypeKind::MFloat8:
556 TypeStr += "mfloat8";
557 break;
558 case TypeKind::SInt:
559 TypeStr += "int" + llvm::utostr(ElementBitwidth);
560 break;
561 case TypeKind::UInt:
562 TypeStr += "uint" + llvm::utostr(ElementBitwidth);
563 break;
564 case TypeKind::Invalid:
565 llvm_unreachable("Attempting to resolve type name from Invalid type!");
566 }
567
568 if (isFixedLengthVector())
569 TypeStr += "x" + llvm::utostr(getNumElements());
570 else if (isScalableVector())
571 TypeStr = "sv" + TypeStr;
572
573 if (NumVectors > 1)
574 TypeStr += "x" + llvm::utostr(NumVectors);
575 if (!isScalarPredicate() && !isVoid())
576 TypeStr += "_t";
577 if (isConstant())
578 TypeStr += " const";
579 if (isPointer())
580 TypeStr += " *";
581
582 return TypeStr;
583 }
584
applyTypespec(StringRef TS)585 void SVEType::applyTypespec(StringRef TS) {
586 for (char I : TS) {
587 switch (I) {
588 case 'Q':
589 assert(isInvalid() && "Unexpected use of typespec modifier");
590 Kind = Svcount;
591 break;
592 case 'P':
593 assert(isInvalid() && "Unexpected use of typespec modifier");
594 Kind = Predicate;
595 break;
596 case 'U':
597 assert(isInvalid() && "Unexpected use of typespec modifier");
598 Kind = UInt;
599 break;
600 case 'c':
601 Kind = isInvalid() ? SInt : Kind;
602 ElementBitwidth = 8;
603 break;
604 case 's':
605 Kind = isInvalid() ? SInt : Kind;
606 ElementBitwidth = 16;
607 break;
608 case 'i':
609 Kind = isInvalid() ? SInt : Kind;
610 ElementBitwidth = 32;
611 break;
612 case 'l':
613 Kind = isInvalid() ? SInt : Kind;
614 ElementBitwidth = 64;
615 break;
616 case 'q':
617 Kind = isInvalid() ? SInt : Kind;
618 ElementBitwidth = 128;
619 break;
620 case 'h':
621 assert(isInvalid() && "Unexpected use of typespec modifier");
622 Kind = Float;
623 ElementBitwidth = 16;
624 break;
625 case 'f':
626 assert(isInvalid() && "Unexpected use of typespec modifier");
627 Kind = Float;
628 ElementBitwidth = 32;
629 break;
630 case 'd':
631 assert(isInvalid() && "Unexpected use of typespec modifier");
632 Kind = Float;
633 ElementBitwidth = 64;
634 break;
635 case 'b':
636 assert(isInvalid() && "Unexpected use of typespec modifier");
637 Kind = BFloat16;
638 ElementBitwidth = 16;
639 break;
640 case 'm':
641 assert(isInvalid() && "Unexpected use of typespec modifier");
642 Kind = MFloat8;
643 ElementBitwidth = 8;
644 break;
645 default:
646 llvm_unreachable("Unhandled type code!");
647 }
648 }
649 assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
650 }
651
applyModifier(char Mod)652 void SVEType::applyModifier(char Mod) {
653 switch (Mod) {
654 case 'v':
655 Kind = Void;
656 NumVectors = 0;
657 break;
658 case 'd':
659 DefaultType = true;
660 break;
661 case 'c':
662 Constant = true;
663 [[fallthrough]];
664 case 'p':
665 Pointer = true;
666 Bitwidth = ElementBitwidth;
667 NumVectors = 0;
668 break;
669 case 'e':
670 Kind = UInt;
671 ElementBitwidth /= 2;
672 break;
673 case 'h':
674 ElementBitwidth /= 2;
675 break;
676 case 'q':
677 ElementBitwidth /= 4;
678 break;
679 case 'b':
680 Kind = UInt;
681 ElementBitwidth /= 4;
682 break;
683 case 'o':
684 ElementBitwidth *= 4;
685 break;
686 case 'P':
687 Kind = Predicate;
688 Bitwidth = 16;
689 ElementBitwidth = 1;
690 break;
691 case '{':
692 IsScalable = false;
693 Bitwidth = 128;
694 NumVectors = 1;
695 break;
696 case 's':
697 case 'a':
698 Bitwidth = ElementBitwidth;
699 NumVectors = 0;
700 break;
701 case 'R':
702 ElementBitwidth /= 2;
703 NumVectors = 0;
704 break;
705 case 'r':
706 ElementBitwidth /= 4;
707 NumVectors = 0;
708 break;
709 case '@':
710 Kind = UInt;
711 ElementBitwidth /= 4;
712 NumVectors = 0;
713 break;
714 case 'K':
715 Kind = SInt;
716 Bitwidth = ElementBitwidth;
717 NumVectors = 0;
718 break;
719 case 'L':
720 Kind = UInt;
721 Bitwidth = ElementBitwidth;
722 NumVectors = 0;
723 break;
724 case 'u':
725 Kind = UInt;
726 break;
727 case 'x':
728 Kind = SInt;
729 break;
730 case 'i':
731 Kind = UInt;
732 ElementBitwidth = Bitwidth = 64;
733 NumVectors = 0;
734 Immediate = true;
735 break;
736 case 'I':
737 Kind = PredicatePattern;
738 ElementBitwidth = Bitwidth = 32;
739 NumVectors = 0;
740 Immediate = true;
741 break;
742 case 'J':
743 Kind = PrefetchOp;
744 ElementBitwidth = Bitwidth = 32;
745 NumVectors = 0;
746 Immediate = true;
747 break;
748 case 'k':
749 Kind = SInt;
750 ElementBitwidth = Bitwidth = 32;
751 NumVectors = 0;
752 break;
753 case 'l':
754 Kind = SInt;
755 ElementBitwidth = Bitwidth = 64;
756 NumVectors = 0;
757 break;
758 case 'm':
759 Kind = UInt;
760 ElementBitwidth = Bitwidth = 32;
761 NumVectors = 0;
762 break;
763 case '>':
764 Kind = Fpm;
765 ElementBitwidth = Bitwidth = 64;
766 NumVectors = 0;
767 break;
768 case 'n':
769 Kind = UInt;
770 ElementBitwidth = Bitwidth = 64;
771 NumVectors = 0;
772 break;
773 case 'w':
774 ElementBitwidth = 64;
775 break;
776 case 'j':
777 ElementBitwidth = Bitwidth = 64;
778 NumVectors = 0;
779 break;
780 case 'f':
781 Kind = UInt;
782 ElementBitwidth = Bitwidth = 64;
783 NumVectors = 0;
784 break;
785 case 'g':
786 Kind = UInt;
787 ElementBitwidth = 64;
788 break;
789 case '#':
790 Kind = SInt;
791 ElementBitwidth = 64;
792 break;
793 case '[':
794 Kind = UInt;
795 ElementBitwidth = 8;
796 break;
797 case 't':
798 Kind = SInt;
799 ElementBitwidth = 32;
800 break;
801 case 'z':
802 Kind = UInt;
803 ElementBitwidth = 32;
804 break;
805 case 'O':
806 Kind = Float;
807 ElementBitwidth = 16;
808 break;
809 case 'M':
810 Kind = Float;
811 ElementBitwidth = 32;
812 break;
813 case 'N':
814 Kind = Float;
815 ElementBitwidth = 64;
816 break;
817 case 'Q':
818 Kind = Void;
819 Constant = true;
820 Pointer = true;
821 NumVectors = 0;
822 break;
823 case 'S':
824 Kind = SInt;
825 Constant = true;
826 Pointer = true;
827 ElementBitwidth = Bitwidth = 8;
828 NumVectors = 0;
829 break;
830 case 'W':
831 Kind = UInt;
832 Constant = true;
833 Pointer = true;
834 ElementBitwidth = Bitwidth = 8;
835 NumVectors = 0;
836 break;
837 case 'T':
838 Kind = SInt;
839 Constant = true;
840 Pointer = true;
841 ElementBitwidth = Bitwidth = 16;
842 NumVectors = 0;
843 break;
844 case 'X':
845 Kind = UInt;
846 Constant = true;
847 Pointer = true;
848 ElementBitwidth = Bitwidth = 16;
849 NumVectors = 0;
850 break;
851 case 'Y':
852 Kind = UInt;
853 Constant = true;
854 Pointer = true;
855 ElementBitwidth = Bitwidth = 32;
856 NumVectors = 0;
857 break;
858 case 'U':
859 Kind = SInt;
860 Constant = true;
861 Pointer = true;
862 ElementBitwidth = Bitwidth = 32;
863 NumVectors = 0;
864 break;
865 case '%':
866 Kind = Void;
867 Pointer = true;
868 NumVectors = 0;
869 break;
870 case 'A':
871 Kind = SInt;
872 Pointer = true;
873 ElementBitwidth = Bitwidth = 8;
874 NumVectors = 0;
875 break;
876 case 'B':
877 Kind = SInt;
878 Pointer = true;
879 ElementBitwidth = Bitwidth = 16;
880 NumVectors = 0;
881 break;
882 case 'C':
883 Kind = SInt;
884 Pointer = true;
885 ElementBitwidth = Bitwidth = 32;
886 NumVectors = 0;
887 break;
888 case 'D':
889 Kind = SInt;
890 Pointer = true;
891 ElementBitwidth = Bitwidth = 64;
892 NumVectors = 0;
893 break;
894 case 'E':
895 Kind = UInt;
896 Pointer = true;
897 ElementBitwidth = Bitwidth = 8;
898 NumVectors = 0;
899 break;
900 case 'F':
901 Kind = UInt;
902 Pointer = true;
903 ElementBitwidth = Bitwidth = 16;
904 NumVectors = 0;
905 break;
906 case 'G':
907 Kind = UInt;
908 Pointer = true;
909 ElementBitwidth = Bitwidth = 32;
910 NumVectors = 0;
911 break;
912 case '$':
913 Kind = BFloat16;
914 ElementBitwidth = 16;
915 break;
916 case '}':
917 Kind = Svcount;
918 NumVectors = 0;
919 break;
920 case '~':
921 Kind = MFloat8;
922 ElementBitwidth = 8;
923 break;
924 case '!':
925 Kind = MFloat8;
926 Bitwidth = ElementBitwidth = 8;
927 NumVectors = 0;
928 break;
929 case '.':
930 llvm_unreachable(". is never a type in itself");
931 break;
932 default:
933 llvm_unreachable("Unhandled character!");
934 }
935 }
936
937 /// Returns the modifier and number of vectors for the given operand \p Op.
getProtoModifier(StringRef Proto,unsigned Op)938 std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
939 for (unsigned P = 0; !Proto.empty(); ++P) {
940 unsigned NumVectors = 1;
941 unsigned CharsToSkip = 1;
942 char Mod = Proto[0];
943 if (Mod == '2' || Mod == '3' || Mod == '4') {
944 NumVectors = Mod - '0';
945 Mod = 'd';
946 if (Proto.size() > 1 && Proto[1] == '.') {
947 Mod = Proto[2];
948 CharsToSkip = 3;
949 }
950 }
951
952 if (P == Op)
953 return {Mod, NumVectors};
954
955 Proto = Proto.drop_front(CharsToSkip);
956 }
957 llvm_unreachable("Unexpected Op");
958 }
959
960 //===----------------------------------------------------------------------===//
961 // Intrinsic implementation
962 //===----------------------------------------------------------------------===//
963
Intrinsic(StringRef Name,StringRef Proto,uint64_t MergeTy,StringRef MergeSuffix,uint64_t MemoryElementTy,StringRef LLVMName,uint64_t Flags,ArrayRef<ImmCheck> Checks,TypeSpec BT,ClassKind Class,SVEEmitter & Emitter,StringRef SVEGuard,StringRef SMEGuard)964 Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
965 StringRef MergeSuffix, uint64_t MemoryElementTy,
966 StringRef LLVMName, uint64_t Flags,
967 ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
968 SVEEmitter &Emitter, StringRef SVEGuard,
969 StringRef SMEGuard)
970 : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
971 BaseTypeSpec(BT), Class(Class), MergeSuffix(MergeSuffix.str()),
972 BaseType(BT, 'd'), Flags(Flags), ImmChecks(Checks) {
973
974 auto FormatGuard = [](StringRef Guard, StringRef Base) -> std::string {
975 if (Guard.contains('|'))
976 return Base.str() + ",(" + Guard.str() + ")";
977 if (Guard.empty() || Guard == Base || Guard.starts_with(Base.str() + ","))
978 return Guard.str();
979 return Base.str() + "," + Guard.str();
980 };
981
982 this->SVEGuard = FormatGuard(SVEGuard, "sve");
983 this->SMEGuard = FormatGuard(SMEGuard, "sme");
984
985 // Types[0] is the return value.
986 for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
987 char Mod;
988 unsigned NumVectors;
989 std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);
990 SVEType T(BaseTypeSpec, Mod, NumVectors);
991 Types.push_back(T);
992 SetsFPMR = T.isFpm();
993
994 // Add range checks for immediates
995 if (I > 0) {
996 if (T.isPredicatePattern())
997 ImmChecks.emplace_back(
998 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31"));
999 else if (T.isPrefetchOp())
1000 ImmChecks.emplace_back(
1001 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13"));
1002 }
1003 }
1004
1005 // Set flags based on properties
1006 this->Flags |= Emitter.encodeTypeFlags(BaseType);
1007 this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy);
1008 this->Flags |= Emitter.encodeMergeType(MergeTy);
1009 if (hasSplat())
1010 this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());
1011 if (SetsFPMR)
1012 this->Flags |= Emitter.getEnumValueForFlag("SetsFPMR");
1013 }
1014
getBuiltinTypeStr()1015 std::string Intrinsic::getBuiltinTypeStr() {
1016 std::string S = getReturnType().builtin_str();
1017 for (unsigned I = 0; I < getNumParams(); ++I)
1018 S += getParamType(I).builtin_str();
1019
1020 return S;
1021 }
1022
replaceTemplatedArgs(std::string Name,TypeSpec TS,std::string Proto) const1023 std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
1024 std::string Proto) const {
1025 std::string Ret = Name;
1026 while (Ret.find('{') != std::string::npos) {
1027 size_t Pos = Ret.find('{');
1028 size_t End = Ret.find('}');
1029 unsigned NumChars = End - Pos + 1;
1030 assert(NumChars == 3 && "Unexpected template argument");
1031
1032 SVEType T;
1033 char C = Ret[Pos+1];
1034 switch(C) {
1035 default:
1036 llvm_unreachable("Unknown predication specifier");
1037 case 'd':
1038 T = SVEType(TS, 'd');
1039 break;
1040 case '0':
1041 case '1':
1042 case '2':
1043 case '3':
1044 // Extract the modifier before passing to SVEType to handle numeric
1045 // modifiers
1046 auto [Mod, NumVectors] = getProtoModifier(Proto, (C - '0'));
1047 T = SVEType(TS, Mod);
1048 break;
1049 }
1050
1051 // Replace templated arg with the right suffix (e.g. u32)
1052 std::string TypeCode;
1053
1054 if (T.isSignedInteger())
1055 TypeCode = 's';
1056 else if (T.isUnsignedInteger())
1057 TypeCode = 'u';
1058 else if (T.isSvcount())
1059 TypeCode = 'c';
1060 else if (T.isPredicate())
1061 TypeCode = 'b';
1062 else if (T.isBFloat())
1063 TypeCode = "bf";
1064 else if (T.isMFloat())
1065 TypeCode = "mf";
1066 else
1067 TypeCode = 'f';
1068 Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));
1069 }
1070
1071 return Ret;
1072 }
1073
mangleLLVMName() const1074 std::string Intrinsic::mangleLLVMName() const {
1075 std::string S = getLLVMName();
1076
1077 // Replace all {d} like expressions with e.g. 'u32'
1078 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto());
1079 }
1080
mangleName(ClassKind LocalCK) const1081 std::string Intrinsic::mangleName(ClassKind LocalCK) const {
1082 std::string S = getName();
1083
1084 if (LocalCK == ClassG) {
1085 // Remove the square brackets and everything in between.
1086 while (S.find('[') != std::string::npos) {
1087 auto Start = S.find('[');
1088 auto End = S.find(']');
1089 S.erase(Start, (End-Start)+1);
1090 }
1091 } else {
1092 // Remove the square brackets.
1093 while (S.find('[') != std::string::npos) {
1094 auto BrPos = S.find('[');
1095 if (BrPos != std::string::npos)
1096 S.erase(BrPos, 1);
1097 BrPos = S.find(']');
1098 if (BrPos != std::string::npos)
1099 S.erase(BrPos, 1);
1100 }
1101 }
1102
1103 // Replace all {d} like expressions with e.g. 'u32'
1104 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto())
1105 .append(getMergeSuffix())
1106 .append(getFPMSuffix());
1107 }
1108
emitIntrinsic(raw_ostream & OS,SVEEmitter & Emitter,ACLEKind Kind) const1109 void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
1110 ACLEKind Kind) const {
1111 bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
1112
1113 std::string FullName = mangleName(ClassS);
1114 std::string ProtoName = mangleName(getClassKind());
1115 OS << (IsOverloaded ? "__aio " : "__ai ")
1116 << "__attribute__((__clang_arm_builtin_alias(";
1117
1118 switch (Kind) {
1119 case ACLEKind::SME:
1120 OS << "__builtin_sme_" << FullName << ")";
1121 break;
1122 case ACLEKind::SVE:
1123 OS << "__builtin_sve_" << FullName << ")";
1124 break;
1125 }
1126
1127 OS << "))\n";
1128
1129 OS << getTypes()[0].str() << " " << ProtoName << "(";
1130 for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
1131 if (I != 0)
1132 OS << ", ";
1133 OS << getTypes()[I + 1].str();
1134 }
1135 OS << ");\n";
1136 }
1137
1138 //===----------------------------------------------------------------------===//
1139 // SVEEmitter implementation
1140 //===----------------------------------------------------------------------===//
encodeTypeFlags(const SVEType & T)1141 uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
1142 if (T.isFloat()) {
1143 switch (T.getElementSizeInBits()) {
1144 case 16:
1145 return encodeEltType("EltTyFloat16");
1146 case 32:
1147 return encodeEltType("EltTyFloat32");
1148 case 64:
1149 return encodeEltType("EltTyFloat64");
1150 default:
1151 llvm_unreachable("Unhandled float element bitwidth!");
1152 }
1153 }
1154
1155 if (T.isBFloat()) {
1156 assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
1157 return encodeEltType("EltTyBFloat16");
1158 }
1159
1160 if (T.isMFloat()) {
1161 assert(T.getElementSizeInBits() == 8 && "Not a valid MFloat.");
1162 return encodeEltType("EltTyMFloat8");
1163 }
1164
1165 if (T.isPredicate() || T.isSvcount()) {
1166 switch (T.getElementSizeInBits()) {
1167 case 8:
1168 return encodeEltType("EltTyBool8");
1169 case 16:
1170 return encodeEltType("EltTyBool16");
1171 case 32:
1172 return encodeEltType("EltTyBool32");
1173 case 64:
1174 return encodeEltType("EltTyBool64");
1175 default:
1176 llvm_unreachable("Unhandled predicate element bitwidth!");
1177 }
1178 }
1179
1180 switch (T.getElementSizeInBits()) {
1181 case 8:
1182 return encodeEltType("EltTyInt8");
1183 case 16:
1184 return encodeEltType("EltTyInt16");
1185 case 32:
1186 return encodeEltType("EltTyInt32");
1187 case 64:
1188 return encodeEltType("EltTyInt64");
1189 case 128:
1190 return encodeEltType("EltTyInt128");
1191 default:
1192 llvm_unreachable("Unhandled integer element bitwidth!");
1193 }
1194 }
1195
createIntrinsic(const Record * R,SmallVectorImpl<std::unique_ptr<Intrinsic>> & Out)1196 void SVEEmitter::createIntrinsic(
1197 const Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
1198 StringRef Name = R->getValueAsString("Name");
1199 StringRef Proto = R->getValueAsString("Prototype");
1200 StringRef Types = R->getValueAsString("Types");
1201 StringRef SVEGuard = R->getValueAsString("SVETargetGuard");
1202 StringRef SMEGuard = R->getValueAsString("SMETargetGuard");
1203 StringRef LLVMName = R->getValueAsString("LLVMIntrinsic");
1204 uint64_t Merge = R->getValueAsInt("Merge");
1205 StringRef MergeSuffix = R->getValueAsString("MergeSuffix");
1206 uint64_t MemEltType = R->getValueAsInt("MemEltType");
1207
1208 int64_t Flags = 0;
1209 for (const Record *FlagRec : R->getValueAsListOfDefs("Flags"))
1210 Flags |= FlagRec->getValueAsInt("Value");
1211
1212 // Create a dummy TypeSpec for non-overloaded builtins.
1213 if (Types.empty()) {
1214 assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
1215 "Expect TypeSpec for overloaded builtin!");
1216 Types = "i";
1217 }
1218
1219 // Extract type specs from string
1220 SmallVector<TypeSpec, 8> TypeSpecs;
1221 TypeSpec Acc;
1222 for (char I : Types) {
1223 Acc.push_back(I);
1224 if (islower(I)) {
1225 TypeSpecs.push_back(TypeSpec(Acc));
1226 Acc.clear();
1227 }
1228 }
1229
1230 // Remove duplicate type specs.
1231 sort(TypeSpecs);
1232 TypeSpecs.erase(llvm::unique(TypeSpecs), TypeSpecs.end());
1233
1234 // Create an Intrinsic for each type spec.
1235 for (auto TS : TypeSpecs) {
1236 // Collate a list of range/option checks for the immediates.
1237 SmallVector<ImmCheck, 2> ImmChecks;
1238 for (const Record *ImmR : R->getValueAsListOfDefs("ImmChecks")) {
1239 int64_t ArgIdx = ImmR->getValueAsInt("ImmArgIdx");
1240 int64_t EltSizeArgIdx = ImmR->getValueAsInt("TypeContextArgIdx");
1241 int64_t Kind = ImmR->getValueAsDef("Kind")->getValueAsInt("Value");
1242 assert(ArgIdx >= 0 && Kind >= 0 &&
1243 "ImmArgIdx and Kind must be nonnegative");
1244
1245 unsigned ElementSizeInBits = 0;
1246 auto [Mod, NumVectors] = getProtoModifier(Proto, EltSizeArgIdx + 1);
1247 if (EltSizeArgIdx >= 0)
1248 ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
1249 ImmChecks.push_back(ImmCheck(ArgIdx, Kind, ElementSizeInBits));
1250 }
1251
1252 Out.push_back(std::make_unique<Intrinsic>(
1253 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks,
1254 TS, ClassS, *this, SVEGuard, SMEGuard));
1255
1256 // Also generate the short-form (e.g. svadd_m) for the given type-spec.
1257 if (Intrinsic::isOverloadedIntrinsic(Name))
1258 Out.push_back(std::make_unique<Intrinsic>(
1259 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags,
1260 ImmChecks, TS, ClassG, *this, SVEGuard, SMEGuard));
1261 }
1262 }
1263
createCoreHeaderIntrinsics(raw_ostream & OS,SVEEmitter & Emitter,ACLEKind Kind)1264 void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
1265 SVEEmitter &Emitter,
1266 ACLEKind Kind) {
1267 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1268 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1269 for (auto *R : RV)
1270 createIntrinsic(R, Defs);
1271
1272 // Sort intrinsics in header file by following order/priority:
1273 // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
1274 // - Class (is intrinsic overloaded or not)
1275 // - Intrinsic name
1276 llvm::stable_sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1277 const std::unique_ptr<Intrinsic> &B) {
1278 auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1279 return std::make_tuple(I->getSVEGuard().str() + I->getSMEGuard().str(),
1280 (unsigned)I->getClassKind(), I->getName());
1281 };
1282 return ToTuple(A) < ToTuple(B);
1283 });
1284
1285 // Actually emit the intrinsic declarations.
1286 for (auto &I : Defs)
1287 I->emitIntrinsic(OS, Emitter, Kind);
1288 }
1289
createHeader(raw_ostream & OS)1290 void SVEEmitter::createHeader(raw_ostream &OS) {
1291 OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
1292 "-----------------------------------===\n"
1293 " *\n"
1294 " *\n"
1295 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1296 "Exceptions.\n"
1297 " * See https://llvm.org/LICENSE.txt for license information.\n"
1298 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1299 " *\n"
1300 " *===-----------------------------------------------------------------"
1301 "------===\n"
1302 " */\n\n";
1303
1304 OS << "#ifndef __ARM_SVE_H\n";
1305 OS << "#define __ARM_SVE_H\n\n";
1306
1307 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1308 OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
1309 OS << "#endif\n";
1310
1311 OS << "#include <stdint.h>\n\n";
1312 OS << "#ifdef __cplusplus\n";
1313 OS << "extern \"C\" {\n";
1314 OS << "#else\n";
1315 OS << "#include <stdbool.h>\n";
1316 OS << "#endif\n\n";
1317
1318 OS << "typedef __fp16 float16_t;\n";
1319 OS << "typedef float float32_t;\n";
1320 OS << "typedef double float64_t;\n";
1321
1322 OS << "typedef __SVInt8_t svint8_t;\n";
1323 OS << "typedef __SVInt16_t svint16_t;\n";
1324 OS << "typedef __SVInt32_t svint32_t;\n";
1325 OS << "typedef __SVInt64_t svint64_t;\n";
1326 OS << "typedef __SVUint8_t svuint8_t;\n";
1327 OS << "typedef __SVUint16_t svuint16_t;\n";
1328 OS << "typedef __SVUint32_t svuint32_t;\n";
1329 OS << "typedef __SVUint64_t svuint64_t;\n";
1330 OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
1331
1332 OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
1333
1334 OS << "#include <arm_bf16.h>\n";
1335 OS << "#include <arm_vector_types.h>\n";
1336
1337 OS << "typedef __SVMfloat8_t svmfloat8_t;\n\n";
1338
1339 OS << "typedef __SVFloat32_t svfloat32_t;\n";
1340 OS << "typedef __SVFloat64_t svfloat64_t;\n";
1341 OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
1342 OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
1343 OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
1344 OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
1345 OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
1346 OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
1347 OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
1348 OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
1349 OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
1350 OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
1351 OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
1352 OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
1353 OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
1354 OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
1355 OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
1356 OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
1357 OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
1358 OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
1359 OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
1360 OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
1361 OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
1362 OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
1363 OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
1364 OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
1365 OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
1366 OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
1367 OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
1368 OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
1369 OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
1370 OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
1371 OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
1372 OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
1373 OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
1374 OS << "typedef __SVBool_t svbool_t;\n";
1375 OS << "typedef __clang_svboolx2_t svboolx2_t;\n";
1376 OS << "typedef __clang_svboolx4_t svboolx4_t;\n\n";
1377
1378 OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
1379 OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
1380 OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
1381
1382 OS << "typedef __clang_svmfloat8x2_t svmfloat8x2_t;\n";
1383 OS << "typedef __clang_svmfloat8x3_t svmfloat8x3_t;\n";
1384 OS << "typedef __clang_svmfloat8x4_t svmfloat8x4_t;\n";
1385
1386 OS << "typedef __SVCount_t svcount_t;\n\n";
1387
1388 OS << "enum svpattern\n";
1389 OS << "{\n";
1390 OS << " SV_POW2 = 0,\n";
1391 OS << " SV_VL1 = 1,\n";
1392 OS << " SV_VL2 = 2,\n";
1393 OS << " SV_VL3 = 3,\n";
1394 OS << " SV_VL4 = 4,\n";
1395 OS << " SV_VL5 = 5,\n";
1396 OS << " SV_VL6 = 6,\n";
1397 OS << " SV_VL7 = 7,\n";
1398 OS << " SV_VL8 = 8,\n";
1399 OS << " SV_VL16 = 9,\n";
1400 OS << " SV_VL32 = 10,\n";
1401 OS << " SV_VL64 = 11,\n";
1402 OS << " SV_VL128 = 12,\n";
1403 OS << " SV_VL256 = 13,\n";
1404 OS << " SV_MUL4 = 29,\n";
1405 OS << " SV_MUL3 = 30,\n";
1406 OS << " SV_ALL = 31\n";
1407 OS << "};\n\n";
1408
1409 OS << "enum svprfop\n";
1410 OS << "{\n";
1411 OS << " SV_PLDL1KEEP = 0,\n";
1412 OS << " SV_PLDL1STRM = 1,\n";
1413 OS << " SV_PLDL2KEEP = 2,\n";
1414 OS << " SV_PLDL2STRM = 3,\n";
1415 OS << " SV_PLDL3KEEP = 4,\n";
1416 OS << " SV_PLDL3STRM = 5,\n";
1417 OS << " SV_PSTL1KEEP = 8,\n";
1418 OS << " SV_PSTL1STRM = 9,\n";
1419 OS << " SV_PSTL2KEEP = 10,\n";
1420 OS << " SV_PSTL2STRM = 11,\n";
1421 OS << " SV_PSTL3KEEP = 12,\n";
1422 OS << " SV_PSTL3STRM = 13\n";
1423 OS << "};\n\n";
1424
1425 OS << "/* Function attributes */\n";
1426 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1427 "__nodebug__))\n\n";
1428 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1429 "__nodebug__, __overloadable__))\n\n";
1430
1431 // Add reinterpret functions.
1432 for (auto [N, Suffix] :
1433 std::initializer_list<std::pair<unsigned, const char *>>{
1434 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1435 for (auto ShortForm : {false, true})
1436 for (const ReinterpretTypeInfo &To : Reinterprets) {
1437 SVEType ToV(To.BaseType, N);
1438 for (const ReinterpretTypeInfo &From : Reinterprets) {
1439 SVEType FromV(From.BaseType, N);
1440 OS << "__aio "
1441 "__attribute__((__clang_arm_builtin_alias(__builtin_sve_"
1442 "reinterpret_"
1443 << To.Suffix << "_" << From.Suffix << Suffix << ")))\n"
1444 << ToV.str() << " svreinterpret_" << To.Suffix;
1445 if (!ShortForm)
1446 OS << "_" << From.Suffix << Suffix;
1447 OS << "(" << FromV.str() << " op);\n";
1448 }
1449 }
1450 }
1451
1452 createCoreHeaderIntrinsics(OS, *this, ACLEKind::SVE);
1453
1454 OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n";
1455 OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n";
1456
1457 OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n";
1458 OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n";
1459 OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n";
1460 OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n";
1461
1462 OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n";
1463 OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
1464
1465 OS << "#ifdef __cplusplus\n";
1466 OS << "} // extern \"C\"\n";
1467 OS << "#endif\n\n";
1468 OS << "#undef __ai\n\n";
1469 OS << "#undef __aio\n\n";
1470 OS << "#endif /* __ARM_SVE_H */\n";
1471 }
1472
createBuiltins(raw_ostream & OS)1473 void SVEEmitter::createBuiltins(raw_ostream &OS) {
1474 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1475 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1476 for (auto *R : RV)
1477 createIntrinsic(R, Defs);
1478
1479 // The mappings must be sorted based on BuiltinID.
1480 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1481 const std::unique_ptr<Intrinsic> &B) {
1482 return A->getMangledName() < B->getMangledName();
1483 });
1484
1485 llvm::StringToOffsetTable Table;
1486 Table.GetOrAddStringOffset("");
1487 Table.GetOrAddStringOffset("n");
1488
1489 for (const auto &Def : Defs)
1490 if (Def->getClassKind() != ClassG) {
1491 Table.GetOrAddStringOffset(Def->getMangledName());
1492 Table.GetOrAddStringOffset(Def->getBuiltinTypeStr());
1493 Table.GetOrAddStringOffset(Def->getGuard());
1494 }
1495
1496 Table.GetOrAddStringOffset("sme|sve");
1497 SmallVector<std::pair<std::string, std::string>> ReinterpretBuiltins;
1498 for (auto [N, Suffix] :
1499 std::initializer_list<std::pair<unsigned, const char *>>{
1500 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1501 for (const ReinterpretTypeInfo &To : Reinterprets) {
1502 SVEType ToV(To.BaseType, N);
1503 for (const ReinterpretTypeInfo &From : Reinterprets) {
1504 SVEType FromV(From.BaseType, N);
1505 std::string Name =
1506 (Twine("reinterpret_") + To.Suffix + "_" + From.Suffix + Suffix)
1507 .str();
1508 std::string Type = ToV.builtin_str() + FromV.builtin_str();
1509 Table.GetOrAddStringOffset(Name);
1510 Table.GetOrAddStringOffset(Type);
1511 ReinterpretBuiltins.push_back({Name, Type});
1512 }
1513 }
1514 }
1515
1516 OS << "#ifdef GET_SVE_BUILTIN_ENUMERATORS\n";
1517 for (const auto &Def : Defs)
1518 if (Def->getClassKind() != ClassG)
1519 OS << " BI__builtin_sve_" << Def->getMangledName() << ",\n";
1520 for (const auto &[Name, _] : ReinterpretBuiltins)
1521 OS << " BI__builtin_sve_" << Name << ",\n";
1522 OS << "#endif // GET_SVE_BUILTIN_ENUMERATORS\n\n";
1523
1524 OS << "#ifdef GET_SVE_BUILTIN_STR_TABLE\n";
1525 Table.EmitStringTableDef(OS, "BuiltinStrings");
1526 OS << "#endif // GET_SVE_BUILTIN_STR_TABLE\n\n";
1527
1528 OS << "#ifdef GET_SVE_BUILTIN_INFOS\n";
1529 for (const auto &Def : Defs) {
1530 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1531 // declarations only live in the header file.
1532 if (Def->getClassKind() != ClassG) {
1533 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1534 << Table.GetStringOffset(Def->getMangledName()) << " /* "
1535 << Def->getMangledName() << " */, ";
1536 OS << Table.GetStringOffset(Def->getBuiltinTypeStr()) << " /* "
1537 << Def->getBuiltinTypeStr() << " */, ";
1538 OS << Table.GetStringOffset("n") << " /* n */, ";
1539 OS << Table.GetStringOffset(Def->getGuard()) << " /* " << Def->getGuard()
1540 << " */}, ";
1541 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1542 }
1543 }
1544 for (const auto &[Name, Type] : ReinterpretBuiltins) {
1545 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1546 << Table.GetStringOffset(Name) << " /* " << Name << " */, ";
1547 OS << Table.GetStringOffset(Type) << " /* " << Type << " */, ";
1548 OS << Table.GetStringOffset("n") << " /* n */, ";
1549 OS << Table.GetStringOffset("sme|sve") << " /* sme|sve */}, ";
1550 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1551 }
1552 OS << "#endif // GET_SVE_BUILTIN_INFOS\n\n";
1553 }
1554
createCodeGenMap(raw_ostream & OS)1555 void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
1556 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1557 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1558 for (auto *R : RV)
1559 createIntrinsic(R, Defs);
1560
1561 // The mappings must be sorted based on BuiltinID.
1562 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1563 const std::unique_ptr<Intrinsic> &B) {
1564 return A->getMangledName() < B->getMangledName();
1565 });
1566
1567 OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
1568 for (auto &Def : Defs) {
1569 // Builtins only exist for non-overloaded intrinsics, overloaded
1570 // declarations only live in the header file.
1571 if (Def->getClassKind() == ClassG)
1572 continue;
1573
1574 uint64_t Flags = Def->getFlags();
1575 auto FlagString = std::to_string(Flags);
1576
1577 std::string LLVMName = Def->getMangledLLVMName();
1578 std::string Builtin = Def->getMangledName();
1579 if (!LLVMName.empty())
1580 OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1581 << "),\n";
1582 else
1583 OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
1584 }
1585 OS << "#endif\n\n";
1586 }
1587
createRangeChecks(raw_ostream & OS)1588 void SVEEmitter::createRangeChecks(raw_ostream &OS) {
1589 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1590 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1591 for (auto *R : RV)
1592 createIntrinsic(R, Defs);
1593
1594 // The mappings must be sorted based on BuiltinID.
1595 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1596 const std::unique_ptr<Intrinsic> &B) {
1597 return A->getMangledName() < B->getMangledName();
1598 });
1599
1600 OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
1601
1602 // Ensure these are only emitted once.
1603 std::set<std::string> Emitted;
1604
1605 for (auto &Def : Defs) {
1606 if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1607 Def->getImmChecks().empty())
1608 continue;
1609
1610 OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
1611 for (auto &Check : Def->getImmChecks())
1612 OS << "ImmChecks.emplace_back(" << Check.getImmArgIdx() << ", "
1613 << Check.getKind() << ", " << Check.getElementSizeInBits() << ");\n";
1614 OS << " break;\n";
1615
1616 Emitted.insert(Def->getMangledName());
1617 }
1618
1619 OS << "#endif\n\n";
1620 }
1621
1622 /// Create the SVETypeFlags used in CGBuiltins
createTypeFlags(raw_ostream & OS)1623 void SVEEmitter::createTypeFlags(raw_ostream &OS) {
1624 OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
1625 for (auto &KV : FlagTypes)
1626 OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
1627 OS << "#endif\n\n";
1628
1629 OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
1630 for (auto &KV : EltTypes)
1631 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1632 OS << "#endif\n\n";
1633
1634 OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
1635 for (auto &KV : MemEltTypes)
1636 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1637 OS << "#endif\n\n";
1638
1639 OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
1640 for (auto &KV : MergeTypes)
1641 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1642 OS << "#endif\n\n";
1643 }
1644
createImmCheckTypes(raw_ostream & OS)1645 void SVEEmitter::createImmCheckTypes(raw_ostream &OS) {
1646 OS << "#ifdef LLVM_GET_ARM_INTRIN_IMMCHECKTYPES\n";
1647 for (auto &KV : ImmCheckTypes)
1648 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1649 OS << "#endif\n\n";
1650 }
1651
createSMEHeader(raw_ostream & OS)1652 void SVEEmitter::createSMEHeader(raw_ostream &OS) {
1653 OS << "/*===---- arm_sme.h - ARM SME intrinsics "
1654 "------===\n"
1655 " *\n"
1656 " *\n"
1657 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1658 "Exceptions.\n"
1659 " * See https://llvm.org/LICENSE.txt for license information.\n"
1660 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1661 " *\n"
1662 " *===-----------------------------------------------------------------"
1663 "------===\n"
1664 " */\n\n";
1665
1666 OS << "#ifndef __ARM_SME_H\n";
1667 OS << "#define __ARM_SME_H\n\n";
1668
1669 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1670 OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";
1671 OS << "#endif\n";
1672
1673 OS << "#include <arm_sve.h>\n\n";
1674 OS << "#include <stddef.h>\n\n";
1675
1676 OS << "/* Function attributes */\n";
1677 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1678 "__nodebug__))\n\n";
1679 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1680 "__nodebug__, __overloadable__))\n\n";
1681
1682 OS << "#ifdef __cplusplus\n";
1683 OS << "extern \"C\" {\n";
1684 OS << "#endif\n\n";
1685
1686 OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";
1687
1688 OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";
1689 OS << " uint64_t x0, x1;\n";
1690 OS << " __builtin_arm_get_sme_state(&x0, &x1);\n";
1691 OS << " return x0 & (1ULL << 63);\n";
1692 OS << "}\n\n";
1693
1694 OS << "void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1695 OS << "void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1696 OS << "void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;\n";
1697 OS << "void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;\n\n";
1698
1699 OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "
1700 "__arm_streaming_compatible __arm_out(\"za\") "
1701 "{ }\n\n";
1702
1703 createCoreHeaderIntrinsics(OS, *this, ACLEKind::SME);
1704
1705 OS << "#ifdef __cplusplus\n";
1706 OS << "} // extern \"C\"\n";
1707 OS << "#endif\n\n";
1708 OS << "#undef __ai\n\n";
1709 OS << "#endif /* __ARM_SME_H */\n";
1710 }
1711
createSMEBuiltins(raw_ostream & OS)1712 void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
1713 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1714 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1715 for (auto *R : RV) {
1716 createIntrinsic(R, Defs);
1717 }
1718
1719 // The mappings must be sorted based on BuiltinID.
1720 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1721 const std::unique_ptr<Intrinsic> &B) {
1722 return A->getMangledName() < B->getMangledName();
1723 });
1724
1725 llvm::StringToOffsetTable Table;
1726 Table.GetOrAddStringOffset("");
1727 Table.GetOrAddStringOffset("n");
1728
1729 for (const auto &Def : Defs)
1730 if (Def->getClassKind() != ClassG) {
1731 Table.GetOrAddStringOffset(Def->getMangledName());
1732 Table.GetOrAddStringOffset(Def->getBuiltinTypeStr());
1733 Table.GetOrAddStringOffset(Def->getGuard());
1734 }
1735
1736 OS << "#ifdef GET_SME_BUILTIN_ENUMERATORS\n";
1737 for (const auto &Def : Defs)
1738 if (Def->getClassKind() != ClassG)
1739 OS << " BI__builtin_sme_" << Def->getMangledName() << ",\n";
1740 OS << "#endif // GET_SME_BUILTIN_ENUMERATORS\n\n";
1741
1742 OS << "#ifdef GET_SME_BUILTIN_STR_TABLE\n";
1743 Table.EmitStringTableDef(OS, "BuiltinStrings");
1744 OS << "#endif // GET_SME_BUILTIN_STR_TABLE\n\n";
1745
1746 OS << "#ifdef GET_SME_BUILTIN_INFOS\n";
1747 for (const auto &Def : Defs) {
1748 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1749 // declarations only live in the header file.
1750 if (Def->getClassKind() != ClassG) {
1751 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1752 << Table.GetStringOffset(Def->getMangledName()) << " /* "
1753 << Def->getMangledName() << " */, ";
1754 OS << Table.GetStringOffset(Def->getBuiltinTypeStr()) << " /* "
1755 << Def->getBuiltinTypeStr() << " */, ";
1756 OS << Table.GetStringOffset("n") << " /* n */, ";
1757 OS << Table.GetStringOffset(Def->getGuard()) << " /* " << Def->getGuard()
1758 << " */}, ";
1759 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1760 }
1761 }
1762 OS << "#endif // GET_SME_BUILTIN_INFOS\n\n";
1763 }
1764
createSMECodeGenMap(raw_ostream & OS)1765 void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
1766 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1767 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1768 for (auto *R : RV) {
1769 createIntrinsic(R, Defs);
1770 }
1771
1772 // The mappings must be sorted based on BuiltinID.
1773 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1774 const std::unique_ptr<Intrinsic> &B) {
1775 return A->getMangledName() < B->getMangledName();
1776 });
1777
1778 OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
1779 for (auto &Def : Defs) {
1780 // Builtins only exist for non-overloaded intrinsics, overloaded
1781 // declarations only live in the header file.
1782 if (Def->getClassKind() == ClassG)
1783 continue;
1784
1785 uint64_t Flags = Def->getFlags();
1786 auto FlagString = std::to_string(Flags);
1787
1788 std::string LLVMName = Def->getLLVMName();
1789 std::string Builtin = Def->getMangledName();
1790 if (!LLVMName.empty())
1791 OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1792 << "),\n";
1793 else
1794 OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
1795 }
1796 OS << "#endif\n\n";
1797 }
1798
createSMERangeChecks(raw_ostream & OS)1799 void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
1800 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1801 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1802 for (auto *R : RV) {
1803 createIntrinsic(R, Defs);
1804 }
1805
1806 // The mappings must be sorted based on BuiltinID.
1807 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
1808 const std::unique_ptr<Intrinsic> &B) {
1809 return A->getMangledName() < B->getMangledName();
1810 });
1811
1812 OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
1813
1814 // Ensure these are only emitted once.
1815 std::set<std::string> Emitted;
1816
1817 for (auto &Def : Defs) {
1818 if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
1819 Def->getImmChecks().empty())
1820 continue;
1821
1822 OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
1823 for (auto &Check : Def->getImmChecks())
1824 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getImmArgIdx()
1825 << ", " << Check.getKind() << ", " << Check.getElementSizeInBits()
1826 << "));\n";
1827 OS << " break;\n";
1828
1829 Emitted.insert(Def->getMangledName());
1830 }
1831
1832 OS << "#endif\n\n";
1833 }
1834
createBuiltinZAState(raw_ostream & OS)1835 void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
1836 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1837 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1838 for (auto *R : RV)
1839 createIntrinsic(R, Defs);
1840
1841 std::map<std::string, std::set<std::string>> IntrinsicsPerState;
1842 for (auto &Def : Defs) {
1843 std::string Key;
1844 auto AddToKey = [&Key](const std::string &S) -> void {
1845 Key = Key.empty() ? S : (Key + " | " + S);
1846 };
1847
1848 if (Def->isFlagSet(getEnumValueForFlag("IsInZA")))
1849 AddToKey("ArmInZA");
1850 else if (Def->isFlagSet(getEnumValueForFlag("IsOutZA")))
1851 AddToKey("ArmOutZA");
1852 else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZA")))
1853 AddToKey("ArmInOutZA");
1854
1855 if (Def->isFlagSet(getEnumValueForFlag("IsInZT0")))
1856 AddToKey("ArmInZT0");
1857 else if (Def->isFlagSet(getEnumValueForFlag("IsOutZT0")))
1858 AddToKey("ArmOutZT0");
1859 else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZT0")))
1860 AddToKey("ArmInOutZT0");
1861
1862 if (!Key.empty())
1863 IntrinsicsPerState[Key].insert(Def->getMangledName());
1864 }
1865
1866 OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";
1867 for (auto &KV : IntrinsicsPerState) {
1868 for (StringRef Name : KV.second)
1869 OS << "case SME::BI__builtin_sme_" << Name << ":\n";
1870 OS << " return " << KV.first << ";\n";
1871 }
1872 OS << "#endif\n\n";
1873 }
1874
createStreamingAttrs(raw_ostream & OS,ACLEKind Kind)1875 void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
1876 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");
1877 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1878 for (auto *R : RV)
1879 createIntrinsic(R, Defs);
1880
1881 StringRef ExtensionKind;
1882 switch (Kind) {
1883 case ACLEKind::SME:
1884 ExtensionKind = "SME";
1885 break;
1886 case ACLEKind::SVE:
1887 ExtensionKind = "SVE";
1888 break;
1889 }
1890
1891 OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";
1892
1893 StringMap<std::set<std::string>> StreamingMap;
1894
1895 uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming");
1896 uint64_t VerifyRuntimeMode = getEnumValueForFlag("VerifyRuntimeMode");
1897 uint64_t IsStreamingCompatibleFlag =
1898 getEnumValueForFlag("IsStreamingCompatible");
1899
1900 for (auto &Def : Defs) {
1901 if (!Def->isFlagSet(VerifyRuntimeMode) && !Def->getSVEGuard().empty() &&
1902 !Def->getSMEGuard().empty())
1903 report_fatal_error("Missing VerifyRuntimeMode flag");
1904 if (Def->isFlagSet(VerifyRuntimeMode) &&
1905 (Def->getSVEGuard().empty() || Def->getSMEGuard().empty()))
1906 report_fatal_error("VerifyRuntimeMode requires SVE and SME guards");
1907
1908 if (Def->isFlagSet(IsStreamingFlag))
1909 StreamingMap["ArmStreaming"].insert(Def->getMangledName());
1910 else if (Def->isFlagSet(VerifyRuntimeMode))
1911 StreamingMap["VerifyRuntimeMode"].insert(Def->getMangledName());
1912 else if (Def->isFlagSet(IsStreamingCompatibleFlag))
1913 StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName());
1914 else
1915 StreamingMap["ArmNonStreaming"].insert(Def->getMangledName());
1916 }
1917
1918 for (auto BuiltinType : StreamingMap.keys()) {
1919 for (auto Name : StreamingMap[BuiltinType]) {
1920 OS << "case " << ExtensionKind << "::BI__builtin_"
1921 << ExtensionKind.lower() << "_";
1922 OS << Name << ":\n";
1923 }
1924 OS << " BuiltinType = " << BuiltinType << ";\n";
1925 OS << " break;\n";
1926 }
1927
1928 OS << "#endif\n\n";
1929 }
1930
1931 namespace clang {
EmitSveHeader(const RecordKeeper & Records,raw_ostream & OS)1932 void EmitSveHeader(const RecordKeeper &Records, raw_ostream &OS) {
1933 SVEEmitter(Records).createHeader(OS);
1934 }
1935
EmitSveBuiltins(const RecordKeeper & Records,raw_ostream & OS)1936 void EmitSveBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
1937 SVEEmitter(Records).createBuiltins(OS);
1938 }
1939
EmitSveBuiltinCG(const RecordKeeper & Records,raw_ostream & OS)1940 void EmitSveBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
1941 SVEEmitter(Records).createCodeGenMap(OS);
1942 }
1943
EmitSveRangeChecks(const RecordKeeper & Records,raw_ostream & OS)1944 void EmitSveRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
1945 SVEEmitter(Records).createRangeChecks(OS);
1946 }
1947
EmitSveTypeFlags(const RecordKeeper & Records,raw_ostream & OS)1948 void EmitSveTypeFlags(const RecordKeeper &Records, raw_ostream &OS) {
1949 SVEEmitter(Records).createTypeFlags(OS);
1950 }
1951
EmitImmCheckTypes(const RecordKeeper & Records,raw_ostream & OS)1952 void EmitImmCheckTypes(const RecordKeeper &Records, raw_ostream &OS) {
1953 SVEEmitter(Records).createImmCheckTypes(OS);
1954 }
1955
EmitSveStreamingAttrs(const RecordKeeper & Records,raw_ostream & OS)1956 void EmitSveStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
1957 SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SVE);
1958 }
1959
EmitSmeHeader(const RecordKeeper & Records,raw_ostream & OS)1960 void EmitSmeHeader(const RecordKeeper &Records, raw_ostream &OS) {
1961 SVEEmitter(Records).createSMEHeader(OS);
1962 }
1963
EmitSmeBuiltins(const RecordKeeper & Records,raw_ostream & OS)1964 void EmitSmeBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
1965 SVEEmitter(Records).createSMEBuiltins(OS);
1966 }
1967
EmitSmeBuiltinCG(const RecordKeeper & Records,raw_ostream & OS)1968 void EmitSmeBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
1969 SVEEmitter(Records).createSMECodeGenMap(OS);
1970 }
1971
EmitSmeRangeChecks(const RecordKeeper & Records,raw_ostream & OS)1972 void EmitSmeRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
1973 SVEEmitter(Records).createSMERangeChecks(OS);
1974 }
1975
EmitSmeStreamingAttrs(const RecordKeeper & Records,raw_ostream & OS)1976 void EmitSmeStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
1977 SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME);
1978 }
1979
EmitSmeBuiltinZAState(const RecordKeeper & Records,raw_ostream & OS)1980 void EmitSmeBuiltinZAState(const RecordKeeper &Records, raw_ostream &OS) {
1981 SVEEmitter(Records).createBuiltinZAState(OS);
1982 }
1983 } // End namespace clang
1984