1 //===- NeonEmitter.cpp - Generate arm_neon.h for use with clang -*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This tablegen backend is responsible for emitting arm_neon.h, which includes 10 // a declaration and definition of each function specified by the ARM NEON 11 // compiler interface. See ARM document DUI0348B. 12 // 13 // Each NEON instruction is implemented in terms of 1 or more functions which 14 // are suffixed with the element type of the input vectors. Functions may be 15 // implemented in terms of generic vector operations such as +, *, -, etc. or 16 // by calling a __builtin_-prefixed function which will be handled by clang's 17 // CodeGen library. 18 // 19 // Additional validation code can be generated by this file when runHeader() is 20 // called, rather than the normal run() entry point. 21 // 22 // See also the documentation in include/clang/Basic/arm_neon.td. 23 // 24 //===----------------------------------------------------------------------===// 25 26 #include "TableGenBackends.h" 27 #include "llvm/ADT/ArrayRef.h" 28 #include "llvm/ADT/DenseMap.h" 29 #include "llvm/ADT/None.h" 30 #include "llvm/ADT/SmallVector.h" 31 #include "llvm/ADT/STLExtras.h" 32 #include "llvm/ADT/StringExtras.h" 33 #include "llvm/ADT/StringRef.h" 34 #include "llvm/Support/Casting.h" 35 #include "llvm/Support/ErrorHandling.h" 36 #include "llvm/Support/raw_ostream.h" 37 #include "llvm/TableGen/Error.h" 38 #include "llvm/TableGen/Record.h" 39 #include "llvm/TableGen/SetTheory.h" 40 #include <algorithm> 41 #include <cassert> 42 #include <cctype> 43 #include <cstddef> 44 #include <cstdint> 45 #include <deque> 46 #include <map> 47 #include <set> 48 #include <sstream> 49 #include <string> 50 #include <utility> 51 #include <vector> 52 53 using namespace llvm; 54 55 namespace { 56 57 // While globals are generally bad, this one allows us to perform assertions 58 // liberally and somehow still trace them back to the def they indirectly 59 // came from. 60 static Record *CurrentRecord = nullptr; 61 static void assert_with_loc(bool Assertion, const std::string &Str) { 62 if (!Assertion) { 63 if (CurrentRecord) 64 PrintFatalError(CurrentRecord->getLoc(), Str); 65 else 66 PrintFatalError(Str); 67 } 68 } 69 70 enum ClassKind { 71 ClassNone, 72 ClassI, // generic integer instruction, e.g., "i8" suffix 73 ClassS, // signed/unsigned/poly, e.g., "s8", "u8" or "p8" suffix 74 ClassW, // width-specific instruction, e.g., "8" suffix 75 ClassB, // bitcast arguments with enum argument to specify type 76 ClassL, // Logical instructions which are op instructions 77 // but we need to not emit any suffix for in our 78 // tests. 79 ClassNoTest // Instructions which we do not test since they are 80 // not TRUE instructions. 81 }; 82 83 /// NeonTypeFlags - Flags to identify the types for overloaded Neon 84 /// builtins. These must be kept in sync with the flags in 85 /// include/clang/Basic/TargetBuiltins.h. 86 namespace NeonTypeFlags { 87 88 enum { EltTypeMask = 0xf, UnsignedFlag = 0x10, QuadFlag = 0x20 }; 89 90 enum EltType { 91 Int8, 92 Int16, 93 Int32, 94 Int64, 95 Poly8, 96 Poly16, 97 Poly64, 98 Poly128, 99 Float16, 100 Float32, 101 Float64 102 }; 103 104 } // end namespace NeonTypeFlags 105 106 class NeonEmitter; 107 108 //===----------------------------------------------------------------------===// 109 // TypeSpec 110 //===----------------------------------------------------------------------===// 111 112 /// A TypeSpec is just a simple wrapper around a string, but gets its own type 113 /// for strong typing purposes. 114 /// 115 /// A TypeSpec can be used to create a type. 116 class TypeSpec : public std::string { 117 public: 118 static std::vector<TypeSpec> fromTypeSpecs(StringRef Str) { 119 std::vector<TypeSpec> Ret; 120 TypeSpec Acc; 121 for (char I : Str.str()) { 122 if (islower(I)) { 123 Acc.push_back(I); 124 Ret.push_back(TypeSpec(Acc)); 125 Acc.clear(); 126 } else { 127 Acc.push_back(I); 128 } 129 } 130 return Ret; 131 } 132 }; 133 134 //===----------------------------------------------------------------------===// 135 // Type 136 //===----------------------------------------------------------------------===// 137 138 /// A Type. Not much more to say here. 139 class Type { 140 private: 141 TypeSpec TS; 142 143 bool Float, Signed, Immediate, Void, Poly, Constant, Pointer; 144 // ScalarForMangling and NoManglingQ are really not suited to live here as 145 // they are not related to the type. But they live in the TypeSpec (not the 146 // prototype), so this is really the only place to store them. 147 bool ScalarForMangling, NoManglingQ; 148 unsigned Bitwidth, ElementBitwidth, NumVectors; 149 150 public: 151 Type() 152 : Float(false), Signed(false), Immediate(false), Void(true), Poly(false), 153 Constant(false), Pointer(false), ScalarForMangling(false), 154 NoManglingQ(false), Bitwidth(0), ElementBitwidth(0), NumVectors(0) {} 155 156 Type(TypeSpec TS, char CharMod) 157 : TS(std::move(TS)), Float(false), Signed(false), Immediate(false), 158 Void(false), Poly(false), Constant(false), Pointer(false), 159 ScalarForMangling(false), NoManglingQ(false), Bitwidth(0), 160 ElementBitwidth(0), NumVectors(0) { 161 applyModifier(CharMod); 162 } 163 164 /// Returns a type representing "void". 165 static Type getVoid() { return Type(); } 166 167 bool operator==(const Type &Other) const { return str() == Other.str(); } 168 bool operator!=(const Type &Other) const { return !operator==(Other); } 169 170 // 171 // Query functions 172 // 173 bool isScalarForMangling() const { return ScalarForMangling; } 174 bool noManglingQ() const { return NoManglingQ; } 175 176 bool isPointer() const { return Pointer; } 177 bool isFloating() const { return Float; } 178 bool isInteger() const { return !Float && !Poly; } 179 bool isSigned() const { return Signed; } 180 bool isImmediate() const { return Immediate; } 181 bool isScalar() const { return NumVectors == 0; } 182 bool isVector() const { return NumVectors > 0; } 183 bool isFloat() const { return Float && ElementBitwidth == 32; } 184 bool isDouble() const { return Float && ElementBitwidth == 64; } 185 bool isHalf() const { return Float && ElementBitwidth == 16; } 186 bool isPoly() const { return Poly; } 187 bool isChar() const { return ElementBitwidth == 8; } 188 bool isShort() const { return !Float && ElementBitwidth == 16; } 189 bool isInt() const { return !Float && ElementBitwidth == 32; } 190 bool isLong() const { return !Float && ElementBitwidth == 64; } 191 bool isVoid() const { return Void; } 192 unsigned getNumElements() const { return Bitwidth / ElementBitwidth; } 193 unsigned getSizeInBits() const { return Bitwidth; } 194 unsigned getElementSizeInBits() const { return ElementBitwidth; } 195 unsigned getNumVectors() const { return NumVectors; } 196 197 // 198 // Mutator functions 199 // 200 void makeUnsigned() { Signed = false; } 201 void makeSigned() { Signed = true; } 202 203 void makeInteger(unsigned ElemWidth, bool Sign) { 204 Float = false; 205 Poly = false; 206 Signed = Sign; 207 Immediate = false; 208 ElementBitwidth = ElemWidth; 209 } 210 211 void makeImmediate(unsigned ElemWidth) { 212 Float = false; 213 Poly = false; 214 Signed = true; 215 Immediate = true; 216 ElementBitwidth = ElemWidth; 217 } 218 219 void makeScalar() { 220 Bitwidth = ElementBitwidth; 221 NumVectors = 0; 222 } 223 224 void makeOneVector() { 225 assert(isVector()); 226 NumVectors = 1; 227 } 228 229 void doubleLanes() { 230 assert_with_loc(Bitwidth != 128, "Can't get bigger than 128!"); 231 Bitwidth = 128; 232 } 233 234 void halveLanes() { 235 assert_with_loc(Bitwidth != 64, "Can't get smaller than 64!"); 236 Bitwidth = 64; 237 } 238 239 /// Return the C string representation of a type, which is the typename 240 /// defined in stdint.h or arm_neon.h. 241 std::string str() const; 242 243 /// Return the string representation of a type, which is an encoded 244 /// string for passing to the BUILTIN() macro in Builtins.def. 245 std::string builtin_str() const; 246 247 /// Return the value in NeonTypeFlags for this type. 248 unsigned getNeonEnum() const; 249 250 /// Parse a type from a stdint.h or arm_neon.h typedef name, 251 /// for example uint32x2_t or int64_t. 252 static Type fromTypedefName(StringRef Name); 253 254 private: 255 /// Creates the type based on the typespec string in TS. 256 /// Sets "Quad" to true if the "Q" or "H" modifiers were 257 /// seen. This is needed by applyModifier as some modifiers 258 /// only take effect if the type size was changed by "Q" or "H". 259 void applyTypespec(bool &Quad); 260 /// Applies a prototype modifier to the type. 261 void applyModifier(char Mod); 262 }; 263 264 //===----------------------------------------------------------------------===// 265 // Variable 266 //===----------------------------------------------------------------------===// 267 268 /// A variable is a simple class that just has a type and a name. 269 class Variable { 270 Type T; 271 std::string N; 272 273 public: 274 Variable() : T(Type::getVoid()), N("") {} 275 Variable(Type T, std::string N) : T(std::move(T)), N(std::move(N)) {} 276 277 Type getType() const { return T; } 278 std::string getName() const { return "__" + N; } 279 }; 280 281 //===----------------------------------------------------------------------===// 282 // Intrinsic 283 //===----------------------------------------------------------------------===// 284 285 /// The main grunt class. This represents an instantiation of an intrinsic with 286 /// a particular typespec and prototype. 287 class Intrinsic { 288 friend class DagEmitter; 289 290 /// The Record this intrinsic was created from. 291 Record *R; 292 /// The unmangled name and prototype. 293 std::string Name, Proto; 294 /// The input and output typespecs. InTS == OutTS except when 295 /// CartesianProductOfTypes is 1 - this is the case for vreinterpret. 296 TypeSpec OutTS, InTS; 297 /// The base class kind. Most intrinsics use ClassS, which has full type 298 /// info for integers (s32/u32). Some use ClassI, which doesn't care about 299 /// signedness (i32), while some (ClassB) have no type at all, only a width 300 /// (32). 301 ClassKind CK; 302 /// The list of DAGs for the body. May be empty, in which case we should 303 /// emit a builtin call. 304 ListInit *Body; 305 /// The architectural #ifdef guard. 306 std::string Guard; 307 /// Set if the Unavailable bit is 1. This means we don't generate a body, 308 /// just an "unavailable" attribute on a declaration. 309 bool IsUnavailable; 310 /// Is this intrinsic safe for big-endian? or does it need its arguments 311 /// reversing? 312 bool BigEndianSafe; 313 314 /// The types of return value [0] and parameters [1..]. 315 std::vector<Type> Types; 316 /// The local variables defined. 317 std::map<std::string, Variable> Variables; 318 /// NeededEarly - set if any other intrinsic depends on this intrinsic. 319 bool NeededEarly; 320 /// UseMacro - set if we should implement using a macro or unset for a 321 /// function. 322 bool UseMacro; 323 /// The set of intrinsics that this intrinsic uses/requires. 324 std::set<Intrinsic *> Dependencies; 325 /// The "base type", which is Type('d', OutTS). InBaseType is only 326 /// different if CartesianProductOfTypes = 1 (for vreinterpret). 327 Type BaseType, InBaseType; 328 /// The return variable. 329 Variable RetVar; 330 /// A postfix to apply to every variable. Defaults to "". 331 std::string VariablePostfix; 332 333 NeonEmitter &Emitter; 334 std::stringstream OS; 335 336 bool isBigEndianSafe() const { 337 if (BigEndianSafe) 338 return true; 339 340 for (const auto &T : Types){ 341 if (T.isVector() && T.getNumElements() > 1) 342 return false; 343 } 344 return true; 345 } 346 347 public: 348 Intrinsic(Record *R, StringRef Name, StringRef Proto, TypeSpec OutTS, 349 TypeSpec InTS, ClassKind CK, ListInit *Body, NeonEmitter &Emitter, 350 StringRef Guard, bool IsUnavailable, bool BigEndianSafe) 351 : R(R), Name(Name.str()), Proto(Proto.str()), OutTS(OutTS), InTS(InTS), 352 CK(CK), Body(Body), Guard(Guard.str()), IsUnavailable(IsUnavailable), 353 BigEndianSafe(BigEndianSafe), NeededEarly(false), UseMacro(false), 354 BaseType(OutTS, 'd'), InBaseType(InTS, 'd'), Emitter(Emitter) { 355 // If this builtin takes an immediate argument, we need to #define it rather 356 // than use a standard declaration, so that SemaChecking can range check 357 // the immediate passed by the user. 358 if (Proto.find('i') != std::string::npos) 359 UseMacro = true; 360 361 // Pointer arguments need to use macros to avoid hiding aligned attributes 362 // from the pointer type. 363 if (Proto.find('p') != std::string::npos || 364 Proto.find('c') != std::string::npos) 365 UseMacro = true; 366 367 // It is not permitted to pass or return an __fp16 by value, so intrinsics 368 // taking a scalar float16_t must be implemented as macros. 369 if (OutTS.find('h') != std::string::npos && 370 Proto.find('s') != std::string::npos) 371 UseMacro = true; 372 373 // Modify the TypeSpec per-argument to get a concrete Type, and create 374 // known variables for each. 375 // Types[0] is the return value. 376 Types.emplace_back(OutTS, Proto[0]); 377 for (unsigned I = 1; I < Proto.size(); ++I) 378 Types.emplace_back(InTS, Proto[I]); 379 } 380 381 /// Get the Record that this intrinsic is based off. 382 Record *getRecord() const { return R; } 383 /// Get the set of Intrinsics that this intrinsic calls. 384 /// this is the set of immediate dependencies, NOT the 385 /// transitive closure. 386 const std::set<Intrinsic *> &getDependencies() const { return Dependencies; } 387 /// Get the architectural guard string (#ifdef). 388 std::string getGuard() const { return Guard; } 389 /// Get the non-mangled name. 390 std::string getName() const { return Name; } 391 392 /// Return true if the intrinsic takes an immediate operand. 393 bool hasImmediate() const { 394 return Proto.find('i') != std::string::npos; 395 } 396 397 /// Return the parameter index of the immediate operand. 398 unsigned getImmediateIdx() const { 399 assert(hasImmediate()); 400 unsigned Idx = Proto.find('i'); 401 assert(Idx > 0 && "Can't return an immediate!"); 402 return Idx - 1; 403 } 404 405 /// Return true if the intrinsic takes an splat operand. 406 bool hasSplat() const { return Proto.find('a') != std::string::npos; } 407 408 /// Return the parameter index of the splat operand. 409 unsigned getSplatIdx() const { 410 assert(hasSplat()); 411 unsigned Idx = Proto.find('a'); 412 assert(Idx > 0 && "Can't return a splat!"); 413 return Idx - 1; 414 } 415 416 unsigned getNumParams() const { return Proto.size() - 1; } 417 Type getReturnType() const { return Types[0]; } 418 Type getParamType(unsigned I) const { return Types[I + 1]; } 419 Type getBaseType() const { return BaseType; } 420 /// Return the raw prototype string. 421 std::string getProto() const { return Proto; } 422 423 /// Return true if the prototype has a scalar argument. 424 /// This does not return true for the "splat" code ('a'). 425 bool protoHasScalar() const; 426 427 /// Return the index that parameter PIndex will sit at 428 /// in a generated function call. This is often just PIndex, 429 /// but may not be as things such as multiple-vector operands 430 /// and sret parameters need to be taken into accont. 431 unsigned getGeneratedParamIdx(unsigned PIndex) { 432 unsigned Idx = 0; 433 if (getReturnType().getNumVectors() > 1) 434 // Multiple vectors are passed as sret. 435 ++Idx; 436 437 for (unsigned I = 0; I < PIndex; ++I) 438 Idx += std::max(1U, getParamType(I).getNumVectors()); 439 440 return Idx; 441 } 442 443 bool hasBody() const { return Body && !Body->getValues().empty(); } 444 445 void setNeededEarly() { NeededEarly = true; } 446 447 bool operator<(const Intrinsic &Other) const { 448 // Sort lexicographically on a two-tuple (Guard, Name) 449 if (Guard != Other.Guard) 450 return Guard < Other.Guard; 451 return Name < Other.Name; 452 } 453 454 ClassKind getClassKind(bool UseClassBIfScalar = false) { 455 if (UseClassBIfScalar && !protoHasScalar()) 456 return ClassB; 457 return CK; 458 } 459 460 /// Return the name, mangled with type information. 461 /// If ForceClassS is true, use ClassS (u32/s32) instead 462 /// of the intrinsic's own type class. 463 std::string getMangledName(bool ForceClassS = false) const; 464 /// Return the type code for a builtin function call. 465 std::string getInstTypeCode(Type T, ClassKind CK) const; 466 /// Return the type string for a BUILTIN() macro in Builtins.def. 467 std::string getBuiltinTypeStr(); 468 469 /// Generate the intrinsic, returning code. 470 std::string generate(); 471 /// Perform type checking and populate the dependency graph, but 472 /// don't generate code yet. 473 void indexBody(); 474 475 private: 476 std::string mangleName(std::string Name, ClassKind CK) const; 477 478 void initVariables(); 479 std::string replaceParamsIn(std::string S); 480 481 void emitBodyAsBuiltinCall(); 482 483 void generateImpl(bool ReverseArguments, 484 StringRef NamePrefix, StringRef CallPrefix); 485 void emitReturn(); 486 void emitBody(StringRef CallPrefix); 487 void emitShadowedArgs(); 488 void emitArgumentReversal(); 489 void emitReturnReversal(); 490 void emitReverseVariable(Variable &Dest, Variable &Src); 491 void emitNewLine(); 492 void emitClosingBrace(); 493 void emitOpeningBrace(); 494 void emitPrototype(StringRef NamePrefix); 495 496 class DagEmitter { 497 Intrinsic &Intr; 498 StringRef CallPrefix; 499 500 public: 501 DagEmitter(Intrinsic &Intr, StringRef CallPrefix) : 502 Intr(Intr), CallPrefix(CallPrefix) { 503 } 504 std::pair<Type, std::string> emitDagArg(Init *Arg, std::string ArgName); 505 std::pair<Type, std::string> emitDagSaveTemp(DagInit *DI); 506 std::pair<Type, std::string> emitDagSplat(DagInit *DI); 507 std::pair<Type, std::string> emitDagDup(DagInit *DI); 508 std::pair<Type, std::string> emitDagDupTyped(DagInit *DI); 509 std::pair<Type, std::string> emitDagShuffle(DagInit *DI); 510 std::pair<Type, std::string> emitDagCast(DagInit *DI, bool IsBitCast); 511 std::pair<Type, std::string> emitDagCall(DagInit *DI); 512 std::pair<Type, std::string> emitDagNameReplace(DagInit *DI); 513 std::pair<Type, std::string> emitDagLiteral(DagInit *DI); 514 std::pair<Type, std::string> emitDagOp(DagInit *DI); 515 std::pair<Type, std::string> emitDag(DagInit *DI); 516 }; 517 }; 518 519 //===----------------------------------------------------------------------===// 520 // NeonEmitter 521 //===----------------------------------------------------------------------===// 522 523 class NeonEmitter { 524 RecordKeeper &Records; 525 DenseMap<Record *, ClassKind> ClassMap; 526 std::map<std::string, std::deque<Intrinsic>> IntrinsicMap; 527 unsigned UniqueNumber; 528 529 void createIntrinsic(Record *R, SmallVectorImpl<Intrinsic *> &Out); 530 void genBuiltinsDef(raw_ostream &OS, SmallVectorImpl<Intrinsic *> &Defs); 531 void genOverloadTypeCheckCode(raw_ostream &OS, 532 SmallVectorImpl<Intrinsic *> &Defs); 533 void genIntrinsicRangeCheckCode(raw_ostream &OS, 534 SmallVectorImpl<Intrinsic *> &Defs); 535 536 public: 537 /// Called by Intrinsic - this attempts to get an intrinsic that takes 538 /// the given types as arguments. 539 Intrinsic &getIntrinsic(StringRef Name, ArrayRef<Type> Types); 540 541 /// Called by Intrinsic - returns a globally-unique number. 542 unsigned getUniqueNumber() { return UniqueNumber++; } 543 544 NeonEmitter(RecordKeeper &R) : Records(R), UniqueNumber(0) { 545 Record *SI = R.getClass("SInst"); 546 Record *II = R.getClass("IInst"); 547 Record *WI = R.getClass("WInst"); 548 Record *SOpI = R.getClass("SOpInst"); 549 Record *IOpI = R.getClass("IOpInst"); 550 Record *WOpI = R.getClass("WOpInst"); 551 Record *LOpI = R.getClass("LOpInst"); 552 Record *NoTestOpI = R.getClass("NoTestOpInst"); 553 554 ClassMap[SI] = ClassS; 555 ClassMap[II] = ClassI; 556 ClassMap[WI] = ClassW; 557 ClassMap[SOpI] = ClassS; 558 ClassMap[IOpI] = ClassI; 559 ClassMap[WOpI] = ClassW; 560 ClassMap[LOpI] = ClassL; 561 ClassMap[NoTestOpI] = ClassNoTest; 562 } 563 564 // run - Emit arm_neon.h.inc 565 void run(raw_ostream &o); 566 567 // runFP16 - Emit arm_fp16.h.inc 568 void runFP16(raw_ostream &o); 569 570 // runHeader - Emit all the __builtin prototypes used in arm_neon.h 571 // and arm_fp16.h 572 void runHeader(raw_ostream &o); 573 574 // runTests - Emit tests for all the Neon intrinsics. 575 void runTests(raw_ostream &o); 576 }; 577 578 } // end anonymous namespace 579 580 //===----------------------------------------------------------------------===// 581 // Type implementation 582 //===----------------------------------------------------------------------===// 583 584 std::string Type::str() const { 585 if (Void) 586 return "void"; 587 std::string S; 588 589 if (!Signed && isInteger()) 590 S += "u"; 591 592 if (Poly) 593 S += "poly"; 594 else if (Float) 595 S += "float"; 596 else 597 S += "int"; 598 599 S += utostr(ElementBitwidth); 600 if (isVector()) 601 S += "x" + utostr(getNumElements()); 602 if (NumVectors > 1) 603 S += "x" + utostr(NumVectors); 604 S += "_t"; 605 606 if (Constant) 607 S += " const"; 608 if (Pointer) 609 S += " *"; 610 611 return S; 612 } 613 614 std::string Type::builtin_str() const { 615 std::string S; 616 if (isVoid()) 617 return "v"; 618 619 if (Pointer) 620 // All pointers are void pointers. 621 S += "v"; 622 else if (isInteger()) 623 switch (ElementBitwidth) { 624 case 8: S += "c"; break; 625 case 16: S += "s"; break; 626 case 32: S += "i"; break; 627 case 64: S += "Wi"; break; 628 case 128: S += "LLLi"; break; 629 default: llvm_unreachable("Unhandled case!"); 630 } 631 else 632 switch (ElementBitwidth) { 633 case 16: S += "h"; break; 634 case 32: S += "f"; break; 635 case 64: S += "d"; break; 636 default: llvm_unreachable("Unhandled case!"); 637 } 638 639 if (isChar() && !Pointer && Signed) 640 // Make chars explicitly signed. 641 S = "S" + S; 642 else if (isInteger() && !Pointer && !Signed) 643 S = "U" + S; 644 645 // Constant indices are "int", but have the "constant expression" modifier. 646 if (isImmediate()) { 647 assert(isInteger() && isSigned()); 648 S = "I" + S; 649 } 650 651 if (isScalar()) { 652 if (Constant) S += "C"; 653 if (Pointer) S += "*"; 654 return S; 655 } 656 657 std::string Ret; 658 for (unsigned I = 0; I < NumVectors; ++I) 659 Ret += "V" + utostr(getNumElements()) + S; 660 661 return Ret; 662 } 663 664 unsigned Type::getNeonEnum() const { 665 unsigned Addend; 666 switch (ElementBitwidth) { 667 case 8: Addend = 0; break; 668 case 16: Addend = 1; break; 669 case 32: Addend = 2; break; 670 case 64: Addend = 3; break; 671 case 128: Addend = 4; break; 672 default: llvm_unreachable("Unhandled element bitwidth!"); 673 } 674 675 unsigned Base = (unsigned)NeonTypeFlags::Int8 + Addend; 676 if (Poly) { 677 // Adjustment needed because Poly32 doesn't exist. 678 if (Addend >= 2) 679 --Addend; 680 Base = (unsigned)NeonTypeFlags::Poly8 + Addend; 681 } 682 if (Float) { 683 assert(Addend != 0 && "Float8 doesn't exist!"); 684 Base = (unsigned)NeonTypeFlags::Float16 + (Addend - 1); 685 } 686 687 if (Bitwidth == 128) 688 Base |= (unsigned)NeonTypeFlags::QuadFlag; 689 if (isInteger() && !Signed) 690 Base |= (unsigned)NeonTypeFlags::UnsignedFlag; 691 692 return Base; 693 } 694 695 Type Type::fromTypedefName(StringRef Name) { 696 Type T; 697 T.Void = false; 698 T.Float = false; 699 T.Poly = false; 700 701 if (Name.front() == 'u') { 702 T.Signed = false; 703 Name = Name.drop_front(); 704 } else { 705 T.Signed = true; 706 } 707 708 if (Name.startswith("float")) { 709 T.Float = true; 710 Name = Name.drop_front(5); 711 } else if (Name.startswith("poly")) { 712 T.Poly = true; 713 Name = Name.drop_front(4); 714 } else { 715 assert(Name.startswith("int")); 716 Name = Name.drop_front(3); 717 } 718 719 unsigned I = 0; 720 for (I = 0; I < Name.size(); ++I) { 721 if (!isdigit(Name[I])) 722 break; 723 } 724 Name.substr(0, I).getAsInteger(10, T.ElementBitwidth); 725 Name = Name.drop_front(I); 726 727 T.Bitwidth = T.ElementBitwidth; 728 T.NumVectors = 1; 729 730 if (Name.front() == 'x') { 731 Name = Name.drop_front(); 732 unsigned I = 0; 733 for (I = 0; I < Name.size(); ++I) { 734 if (!isdigit(Name[I])) 735 break; 736 } 737 unsigned NumLanes; 738 Name.substr(0, I).getAsInteger(10, NumLanes); 739 Name = Name.drop_front(I); 740 T.Bitwidth = T.ElementBitwidth * NumLanes; 741 } else { 742 // Was scalar. 743 T.NumVectors = 0; 744 } 745 if (Name.front() == 'x') { 746 Name = Name.drop_front(); 747 unsigned I = 0; 748 for (I = 0; I < Name.size(); ++I) { 749 if (!isdigit(Name[I])) 750 break; 751 } 752 Name.substr(0, I).getAsInteger(10, T.NumVectors); 753 Name = Name.drop_front(I); 754 } 755 756 assert(Name.startswith("_t") && "Malformed typedef!"); 757 return T; 758 } 759 760 void Type::applyTypespec(bool &Quad) { 761 std::string S = TS; 762 ScalarForMangling = false; 763 Void = false; 764 Poly = Float = false; 765 ElementBitwidth = ~0U; 766 Signed = true; 767 NumVectors = 1; 768 769 for (char I : S) { 770 switch (I) { 771 case 'S': 772 ScalarForMangling = true; 773 break; 774 case 'H': 775 NoManglingQ = true; 776 Quad = true; 777 break; 778 case 'Q': 779 Quad = true; 780 break; 781 case 'P': 782 Poly = true; 783 break; 784 case 'U': 785 Signed = false; 786 break; 787 case 'c': 788 ElementBitwidth = 8; 789 break; 790 case 'h': 791 Float = true; 792 LLVM_FALLTHROUGH; 793 case 's': 794 ElementBitwidth = 16; 795 break; 796 case 'f': 797 Float = true; 798 LLVM_FALLTHROUGH; 799 case 'i': 800 ElementBitwidth = 32; 801 break; 802 case 'd': 803 Float = true; 804 LLVM_FALLTHROUGH; 805 case 'l': 806 ElementBitwidth = 64; 807 break; 808 case 'k': 809 ElementBitwidth = 128; 810 // Poly doesn't have a 128x1 type. 811 if (Poly) 812 NumVectors = 0; 813 break; 814 default: 815 llvm_unreachable("Unhandled type code!"); 816 } 817 } 818 assert(ElementBitwidth != ~0U && "Bad element bitwidth!"); 819 820 Bitwidth = Quad ? 128 : 64; 821 } 822 823 void Type::applyModifier(char Mod) { 824 bool AppliedQuad = false; 825 applyTypespec(AppliedQuad); 826 827 switch (Mod) { 828 case 'v': 829 Void = true; 830 break; 831 case 't': 832 if (Poly) { 833 Poly = false; 834 Signed = false; 835 } 836 break; 837 case 'b': 838 Signed = false; 839 Float = false; 840 Poly = false; 841 NumVectors = 0; 842 Bitwidth = ElementBitwidth; 843 break; 844 case '$': 845 Signed = true; 846 Float = false; 847 Poly = false; 848 NumVectors = 0; 849 Bitwidth = ElementBitwidth; 850 break; 851 case 'u': 852 Signed = false; 853 Poly = false; 854 Float = false; 855 break; 856 case 'x': 857 Signed = true; 858 assert(!Poly && "'u' can't be used with poly types!"); 859 Float = false; 860 break; 861 case 'o': 862 Bitwidth = ElementBitwidth = 64; 863 NumVectors = 0; 864 Float = true; 865 break; 866 case 'y': 867 Bitwidth = ElementBitwidth = 32; 868 NumVectors = 0; 869 Float = true; 870 break; 871 case 'Y': 872 Bitwidth = ElementBitwidth = 16; 873 NumVectors = 0; 874 Float = true; 875 break; 876 case 'I': 877 Bitwidth = ElementBitwidth = 32; 878 NumVectors = 0; 879 Float = false; 880 Signed = true; 881 break; 882 case 'L': 883 Bitwidth = ElementBitwidth = 64; 884 NumVectors = 0; 885 Float = false; 886 Signed = true; 887 break; 888 case 'U': 889 Bitwidth = ElementBitwidth = 32; 890 NumVectors = 0; 891 Float = false; 892 Signed = false; 893 break; 894 case 'O': 895 Bitwidth = ElementBitwidth = 64; 896 NumVectors = 0; 897 Float = false; 898 Signed = false; 899 break; 900 case 'f': 901 Float = true; 902 ElementBitwidth = 32; 903 break; 904 case 'F': 905 Float = true; 906 ElementBitwidth = 64; 907 break; 908 case 'H': 909 Float = true; 910 ElementBitwidth = 16; 911 break; 912 case '0': 913 Float = true; 914 if (AppliedQuad) 915 Bitwidth /= 2; 916 ElementBitwidth = 16; 917 break; 918 case '1': 919 Float = true; 920 if (!AppliedQuad) 921 Bitwidth *= 2; 922 ElementBitwidth = 16; 923 break; 924 case 'g': 925 if (AppliedQuad) 926 Bitwidth /= 2; 927 break; 928 case 'j': 929 if (!AppliedQuad) 930 Bitwidth *= 2; 931 break; 932 case 'w': 933 ElementBitwidth *= 2; 934 Bitwidth *= 2; 935 break; 936 case 'n': 937 ElementBitwidth *= 2; 938 break; 939 case 'i': 940 Float = false; 941 Poly = false; 942 ElementBitwidth = Bitwidth = 32; 943 NumVectors = 0; 944 Signed = true; 945 Immediate = true; 946 break; 947 case 'l': 948 Float = false; 949 Poly = false; 950 ElementBitwidth = Bitwidth = 64; 951 NumVectors = 0; 952 Signed = false; 953 Immediate = true; 954 break; 955 case 'z': 956 ElementBitwidth /= 2; 957 Bitwidth = ElementBitwidth; 958 NumVectors = 0; 959 break; 960 case 'r': 961 ElementBitwidth *= 2; 962 Bitwidth = ElementBitwidth; 963 NumVectors = 0; 964 break; 965 case 's': 966 case 'a': 967 Bitwidth = ElementBitwidth; 968 NumVectors = 0; 969 break; 970 case 'k': 971 Bitwidth *= 2; 972 break; 973 case 'c': 974 Constant = true; 975 LLVM_FALLTHROUGH; 976 case 'p': 977 Pointer = true; 978 Bitwidth = ElementBitwidth; 979 NumVectors = 0; 980 break; 981 case 'h': 982 ElementBitwidth /= 2; 983 break; 984 case 'q': 985 ElementBitwidth /= 2; 986 Bitwidth *= 2; 987 break; 988 case 'e': 989 ElementBitwidth /= 2; 990 Signed = false; 991 break; 992 case 'm': 993 ElementBitwidth /= 2; 994 Bitwidth /= 2; 995 break; 996 case 'd': 997 break; 998 case '2': 999 NumVectors = 2; 1000 break; 1001 case '3': 1002 NumVectors = 3; 1003 break; 1004 case '4': 1005 NumVectors = 4; 1006 break; 1007 case 'B': 1008 NumVectors = 2; 1009 if (!AppliedQuad) 1010 Bitwidth *= 2; 1011 break; 1012 case 'C': 1013 NumVectors = 3; 1014 if (!AppliedQuad) 1015 Bitwidth *= 2; 1016 break; 1017 case 'D': 1018 NumVectors = 4; 1019 if (!AppliedQuad) 1020 Bitwidth *= 2; 1021 break; 1022 case '7': 1023 if (AppliedQuad) 1024 Bitwidth /= 2; 1025 ElementBitwidth = 8; 1026 break; 1027 case '8': 1028 ElementBitwidth = 8; 1029 break; 1030 case '9': 1031 if (!AppliedQuad) 1032 Bitwidth *= 2; 1033 ElementBitwidth = 8; 1034 break; 1035 default: 1036 llvm_unreachable("Unhandled character!"); 1037 } 1038 } 1039 1040 //===----------------------------------------------------------------------===// 1041 // Intrinsic implementation 1042 //===----------------------------------------------------------------------===// 1043 1044 std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const { 1045 char typeCode = '\0'; 1046 bool printNumber = true; 1047 1048 if (CK == ClassB) 1049 return ""; 1050 1051 if (T.isPoly()) 1052 typeCode = 'p'; 1053 else if (T.isInteger()) 1054 typeCode = T.isSigned() ? 's' : 'u'; 1055 else 1056 typeCode = 'f'; 1057 1058 if (CK == ClassI) { 1059 switch (typeCode) { 1060 default: 1061 break; 1062 case 's': 1063 case 'u': 1064 case 'p': 1065 typeCode = 'i'; 1066 break; 1067 } 1068 } 1069 if (CK == ClassB) { 1070 typeCode = '\0'; 1071 } 1072 1073 std::string S; 1074 if (typeCode != '\0') 1075 S.push_back(typeCode); 1076 if (printNumber) 1077 S += utostr(T.getElementSizeInBits()); 1078 1079 return S; 1080 } 1081 1082 static bool isFloatingPointProtoModifier(char Mod) { 1083 return Mod == 'F' || Mod == 'f' || Mod == 'H' || Mod == 'Y' || Mod == 'I'; 1084 } 1085 1086 std::string Intrinsic::getBuiltinTypeStr() { 1087 ClassKind LocalCK = getClassKind(true); 1088 std::string S; 1089 1090 Type RetT = getReturnType(); 1091 if ((LocalCK == ClassI || LocalCK == ClassW) && RetT.isScalar() && 1092 !RetT.isFloating()) 1093 RetT.makeInteger(RetT.getElementSizeInBits(), false); 1094 1095 // Since the return value must be one type, return a vector type of the 1096 // appropriate width which we will bitcast. An exception is made for 1097 // returning structs of 2, 3, or 4 vectors which are returned in a sret-like 1098 // fashion, storing them to a pointer arg. 1099 if (RetT.getNumVectors() > 1) { 1100 S += "vv*"; // void result with void* first argument 1101 } else { 1102 if (RetT.isPoly()) 1103 RetT.makeInteger(RetT.getElementSizeInBits(), false); 1104 if (!RetT.isScalar() && !RetT.isSigned()) 1105 RetT.makeSigned(); 1106 1107 bool ForcedVectorFloatingType = isFloatingPointProtoModifier(Proto[0]); 1108 if (LocalCK == ClassB && !RetT.isScalar() && !ForcedVectorFloatingType) 1109 // Cast to vector of 8-bit elements. 1110 RetT.makeInteger(8, true); 1111 1112 S += RetT.builtin_str(); 1113 } 1114 1115 for (unsigned I = 0; I < getNumParams(); ++I) { 1116 Type T = getParamType(I); 1117 if (T.isPoly()) 1118 T.makeInteger(T.getElementSizeInBits(), false); 1119 1120 bool ForcedFloatingType = isFloatingPointProtoModifier(Proto[I + 1]); 1121 if (LocalCK == ClassB && !T.isScalar() && !ForcedFloatingType) 1122 T.makeInteger(8, true); 1123 // Halves always get converted to 8-bit elements. 1124 if (T.isHalf() && T.isVector() && !T.isScalarForMangling()) 1125 T.makeInteger(8, true); 1126 1127 if (LocalCK == ClassI) 1128 T.makeSigned(); 1129 1130 if (hasImmediate() && getImmediateIdx() == I) 1131 T.makeImmediate(32); 1132 1133 S += T.builtin_str(); 1134 } 1135 1136 // Extra constant integer to hold type class enum for this function, e.g. s8 1137 if (LocalCK == ClassB) 1138 S += "i"; 1139 1140 return S; 1141 } 1142 1143 std::string Intrinsic::getMangledName(bool ForceClassS) const { 1144 // Check if the prototype has a scalar operand with the type of the vector 1145 // elements. If not, bitcasting the args will take care of arg checking. 1146 // The actual signedness etc. will be taken care of with special enums. 1147 ClassKind LocalCK = CK; 1148 if (!protoHasScalar()) 1149 LocalCK = ClassB; 1150 1151 return mangleName(Name, ForceClassS ? ClassS : LocalCK); 1152 } 1153 1154 std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const { 1155 std::string typeCode = getInstTypeCode(BaseType, LocalCK); 1156 std::string S = Name; 1157 1158 if (Name == "vcvt_f16_f32" || Name == "vcvt_f32_f16" || 1159 Name == "vcvt_f32_f64" || Name == "vcvt_f64_f32") 1160 return Name; 1161 1162 if (!typeCode.empty()) { 1163 // If the name ends with _xN (N = 2,3,4), insert the typeCode before _xN. 1164 if (Name.size() >= 3 && isdigit(Name.back()) && 1165 Name[Name.length() - 2] == 'x' && Name[Name.length() - 3] == '_') 1166 S.insert(S.length() - 3, "_" + typeCode); 1167 else 1168 S += "_" + typeCode; 1169 } 1170 1171 if (BaseType != InBaseType) { 1172 // A reinterpret - out the input base type at the end. 1173 S += "_" + getInstTypeCode(InBaseType, LocalCK); 1174 } 1175 1176 if (LocalCK == ClassB) 1177 S += "_v"; 1178 1179 // Insert a 'q' before the first '_' character so that it ends up before 1180 // _lane or _n on vector-scalar operations. 1181 if (BaseType.getSizeInBits() == 128 && !BaseType.noManglingQ()) { 1182 size_t Pos = S.find('_'); 1183 S.insert(Pos, "q"); 1184 } 1185 1186 char Suffix = '\0'; 1187 if (BaseType.isScalarForMangling()) { 1188 switch (BaseType.getElementSizeInBits()) { 1189 case 8: Suffix = 'b'; break; 1190 case 16: Suffix = 'h'; break; 1191 case 32: Suffix = 's'; break; 1192 case 64: Suffix = 'd'; break; 1193 default: llvm_unreachable("Bad suffix!"); 1194 } 1195 } 1196 if (Suffix != '\0') { 1197 size_t Pos = S.find('_'); 1198 S.insert(Pos, &Suffix, 1); 1199 } 1200 1201 return S; 1202 } 1203 1204 std::string Intrinsic::replaceParamsIn(std::string S) { 1205 while (S.find('$') != std::string::npos) { 1206 size_t Pos = S.find('$'); 1207 size_t End = Pos + 1; 1208 while (isalpha(S[End])) 1209 ++End; 1210 1211 std::string VarName = S.substr(Pos + 1, End - Pos - 1); 1212 assert_with_loc(Variables.find(VarName) != Variables.end(), 1213 "Variable not defined!"); 1214 S.replace(Pos, End - Pos, Variables.find(VarName)->second.getName()); 1215 } 1216 1217 return S; 1218 } 1219 1220 void Intrinsic::initVariables() { 1221 Variables.clear(); 1222 1223 // Modify the TypeSpec per-argument to get a concrete Type, and create 1224 // known variables for each. 1225 for (unsigned I = 1; I < Proto.size(); ++I) { 1226 char NameC = '0' + (I - 1); 1227 std::string Name = "p"; 1228 Name.push_back(NameC); 1229 1230 Variables[Name] = Variable(Types[I], Name + VariablePostfix); 1231 } 1232 RetVar = Variable(Types[0], "ret" + VariablePostfix); 1233 } 1234 1235 void Intrinsic::emitPrototype(StringRef NamePrefix) { 1236 if (UseMacro) 1237 OS << "#define "; 1238 else 1239 OS << "__ai " << Types[0].str() << " "; 1240 1241 OS << NamePrefix.str() << mangleName(Name, ClassS) << "("; 1242 1243 for (unsigned I = 0; I < getNumParams(); ++I) { 1244 if (I != 0) 1245 OS << ", "; 1246 1247 char NameC = '0' + I; 1248 std::string Name = "p"; 1249 Name.push_back(NameC); 1250 assert(Variables.find(Name) != Variables.end()); 1251 Variable &V = Variables[Name]; 1252 1253 if (!UseMacro) 1254 OS << V.getType().str() << " "; 1255 OS << V.getName(); 1256 } 1257 1258 OS << ")"; 1259 } 1260 1261 void Intrinsic::emitOpeningBrace() { 1262 if (UseMacro) 1263 OS << " __extension__ ({"; 1264 else 1265 OS << " {"; 1266 emitNewLine(); 1267 } 1268 1269 void Intrinsic::emitClosingBrace() { 1270 if (UseMacro) 1271 OS << "})"; 1272 else 1273 OS << "}"; 1274 } 1275 1276 void Intrinsic::emitNewLine() { 1277 if (UseMacro) 1278 OS << " \\\n"; 1279 else 1280 OS << "\n"; 1281 } 1282 1283 void Intrinsic::emitReverseVariable(Variable &Dest, Variable &Src) { 1284 if (Dest.getType().getNumVectors() > 1) { 1285 emitNewLine(); 1286 1287 for (unsigned K = 0; K < Dest.getType().getNumVectors(); ++K) { 1288 OS << " " << Dest.getName() << ".val[" << K << "] = " 1289 << "__builtin_shufflevector(" 1290 << Src.getName() << ".val[" << K << "], " 1291 << Src.getName() << ".val[" << K << "]"; 1292 for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J) 1293 OS << ", " << J; 1294 OS << ");"; 1295 emitNewLine(); 1296 } 1297 } else { 1298 OS << " " << Dest.getName() 1299 << " = __builtin_shufflevector(" << Src.getName() << ", " << Src.getName(); 1300 for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J) 1301 OS << ", " << J; 1302 OS << ");"; 1303 emitNewLine(); 1304 } 1305 } 1306 1307 void Intrinsic::emitArgumentReversal() { 1308 if (isBigEndianSafe()) 1309 return; 1310 1311 // Reverse all vector arguments. 1312 for (unsigned I = 0; I < getNumParams(); ++I) { 1313 std::string Name = "p" + utostr(I); 1314 std::string NewName = "rev" + utostr(I); 1315 1316 Variable &V = Variables[Name]; 1317 Variable NewV(V.getType(), NewName + VariablePostfix); 1318 1319 if (!NewV.getType().isVector() || NewV.getType().getNumElements() == 1) 1320 continue; 1321 1322 OS << " " << NewV.getType().str() << " " << NewV.getName() << ";"; 1323 emitReverseVariable(NewV, V); 1324 V = NewV; 1325 } 1326 } 1327 1328 void Intrinsic::emitReturnReversal() { 1329 if (isBigEndianSafe()) 1330 return; 1331 if (!getReturnType().isVector() || getReturnType().isVoid() || 1332 getReturnType().getNumElements() == 1) 1333 return; 1334 emitReverseVariable(RetVar, RetVar); 1335 } 1336 1337 void Intrinsic::emitShadowedArgs() { 1338 // Macro arguments are not type-checked like inline function arguments, 1339 // so assign them to local temporaries to get the right type checking. 1340 if (!UseMacro) 1341 return; 1342 1343 for (unsigned I = 0; I < getNumParams(); ++I) { 1344 // Do not create a temporary for an immediate argument. 1345 // That would defeat the whole point of using a macro! 1346 if (hasImmediate() && Proto[I+1] == 'i') 1347 continue; 1348 // Do not create a temporary for pointer arguments. The input 1349 // pointer may have an alignment hint. 1350 if (getParamType(I).isPointer()) 1351 continue; 1352 1353 std::string Name = "p" + utostr(I); 1354 1355 assert(Variables.find(Name) != Variables.end()); 1356 Variable &V = Variables[Name]; 1357 1358 std::string NewName = "s" + utostr(I); 1359 Variable V2(V.getType(), NewName + VariablePostfix); 1360 1361 OS << " " << V2.getType().str() << " " << V2.getName() << " = " 1362 << V.getName() << ";"; 1363 emitNewLine(); 1364 1365 V = V2; 1366 } 1367 } 1368 1369 // We don't check 'a' in this function, because for builtin function the 1370 // argument matching to 'a' uses a vector type splatted from a scalar type. 1371 bool Intrinsic::protoHasScalar() const { 1372 return (Proto.find('s') != std::string::npos || 1373 Proto.find('z') != std::string::npos || 1374 Proto.find('r') != std::string::npos || 1375 Proto.find('b') != std::string::npos || 1376 Proto.find('$') != std::string::npos || 1377 Proto.find('y') != std::string::npos || 1378 Proto.find('o') != std::string::npos); 1379 } 1380 1381 void Intrinsic::emitBodyAsBuiltinCall() { 1382 std::string S; 1383 1384 // If this builtin returns a struct 2, 3, or 4 vectors, pass it as an implicit 1385 // sret-like argument. 1386 bool SRet = getReturnType().getNumVectors() >= 2; 1387 1388 StringRef N = Name; 1389 if (hasSplat()) { 1390 // Call the non-splat builtin: chop off the "_n" suffix from the name. 1391 assert(N.endswith("_n")); 1392 N = N.drop_back(2); 1393 } 1394 1395 ClassKind LocalCK = CK; 1396 if (!protoHasScalar()) 1397 LocalCK = ClassB; 1398 1399 if (!getReturnType().isVoid() && !SRet) 1400 S += "(" + RetVar.getType().str() + ") "; 1401 1402 S += "__builtin_neon_" + mangleName(N, LocalCK) + "("; 1403 1404 if (SRet) 1405 S += "&" + RetVar.getName() + ", "; 1406 1407 for (unsigned I = 0; I < getNumParams(); ++I) { 1408 Variable &V = Variables["p" + utostr(I)]; 1409 Type T = V.getType(); 1410 1411 // Handle multiple-vector values specially, emitting each subvector as an 1412 // argument to the builtin. 1413 if (T.getNumVectors() > 1) { 1414 // Check if an explicit cast is needed. 1415 std::string Cast; 1416 if (LocalCK == ClassB) { 1417 Type T2 = T; 1418 T2.makeOneVector(); 1419 T2.makeInteger(8, /*Signed=*/true); 1420 Cast = "(" + T2.str() + ")"; 1421 } 1422 1423 for (unsigned J = 0; J < T.getNumVectors(); ++J) 1424 S += Cast + V.getName() + ".val[" + utostr(J) + "], "; 1425 continue; 1426 } 1427 1428 std::string Arg; 1429 Type CastToType = T; 1430 if (hasSplat() && I == getSplatIdx()) { 1431 Arg = "(" + BaseType.str() + ") {"; 1432 for (unsigned J = 0; J < BaseType.getNumElements(); ++J) { 1433 if (J != 0) 1434 Arg += ", "; 1435 Arg += V.getName(); 1436 } 1437 Arg += "}"; 1438 1439 CastToType = BaseType; 1440 } else { 1441 Arg = V.getName(); 1442 } 1443 1444 // Check if an explicit cast is needed. 1445 if (CastToType.isVector() && 1446 (LocalCK == ClassB || (T.isHalf() && !T.isScalarForMangling()))) { 1447 CastToType.makeInteger(8, true); 1448 Arg = "(" + CastToType.str() + ")" + Arg; 1449 } else if (CastToType.isVector() && LocalCK == ClassI) { 1450 CastToType.makeSigned(); 1451 Arg = "(" + CastToType.str() + ")" + Arg; 1452 } 1453 1454 S += Arg + ", "; 1455 } 1456 1457 // Extra constant integer to hold type class enum for this function, e.g. s8 1458 if (getClassKind(true) == ClassB) { 1459 Type ThisTy = getReturnType(); 1460 if (Proto[0] == 'v' || isFloatingPointProtoModifier(Proto[0])) 1461 ThisTy = getParamType(0); 1462 if (ThisTy.isPointer()) 1463 ThisTy = getParamType(1); 1464 1465 S += utostr(ThisTy.getNeonEnum()); 1466 } else { 1467 // Remove extraneous ", ". 1468 S.pop_back(); 1469 S.pop_back(); 1470 } 1471 S += ");"; 1472 1473 std::string RetExpr; 1474 if (!SRet && !RetVar.getType().isVoid()) 1475 RetExpr = RetVar.getName() + " = "; 1476 1477 OS << " " << RetExpr << S; 1478 emitNewLine(); 1479 } 1480 1481 void Intrinsic::emitBody(StringRef CallPrefix) { 1482 std::vector<std::string> Lines; 1483 1484 assert(RetVar.getType() == Types[0]); 1485 // Create a return variable, if we're not void. 1486 if (!RetVar.getType().isVoid()) { 1487 OS << " " << RetVar.getType().str() << " " << RetVar.getName() << ";"; 1488 emitNewLine(); 1489 } 1490 1491 if (!Body || Body->getValues().empty()) { 1492 // Nothing specific to output - must output a builtin. 1493 emitBodyAsBuiltinCall(); 1494 return; 1495 } 1496 1497 // We have a list of "things to output". The last should be returned. 1498 for (auto *I : Body->getValues()) { 1499 if (StringInit *SI = dyn_cast<StringInit>(I)) { 1500 Lines.push_back(replaceParamsIn(SI->getAsString())); 1501 } else if (DagInit *DI = dyn_cast<DagInit>(I)) { 1502 DagEmitter DE(*this, CallPrefix); 1503 Lines.push_back(DE.emitDag(DI).second + ";"); 1504 } 1505 } 1506 1507 assert(!Lines.empty() && "Empty def?"); 1508 if (!RetVar.getType().isVoid()) 1509 Lines.back().insert(0, RetVar.getName() + " = "); 1510 1511 for (auto &L : Lines) { 1512 OS << " " << L; 1513 emitNewLine(); 1514 } 1515 } 1516 1517 void Intrinsic::emitReturn() { 1518 if (RetVar.getType().isVoid()) 1519 return; 1520 if (UseMacro) 1521 OS << " " << RetVar.getName() << ";"; 1522 else 1523 OS << " return " << RetVar.getName() << ";"; 1524 emitNewLine(); 1525 } 1526 1527 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDag(DagInit *DI) { 1528 // At this point we should only be seeing a def. 1529 DefInit *DefI = cast<DefInit>(DI->getOperator()); 1530 std::string Op = DefI->getAsString(); 1531 1532 if (Op == "cast" || Op == "bitcast") 1533 return emitDagCast(DI, Op == "bitcast"); 1534 if (Op == "shuffle") 1535 return emitDagShuffle(DI); 1536 if (Op == "dup") 1537 return emitDagDup(DI); 1538 if (Op == "dup_typed") 1539 return emitDagDupTyped(DI); 1540 if (Op == "splat") 1541 return emitDagSplat(DI); 1542 if (Op == "save_temp") 1543 return emitDagSaveTemp(DI); 1544 if (Op == "op") 1545 return emitDagOp(DI); 1546 if (Op == "call") 1547 return emitDagCall(DI); 1548 if (Op == "name_replace") 1549 return emitDagNameReplace(DI); 1550 if (Op == "literal") 1551 return emitDagLiteral(DI); 1552 assert_with_loc(false, "Unknown operation!"); 1553 return std::make_pair(Type::getVoid(), ""); 1554 } 1555 1556 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagOp(DagInit *DI) { 1557 std::string Op = cast<StringInit>(DI->getArg(0))->getAsUnquotedString(); 1558 if (DI->getNumArgs() == 2) { 1559 // Unary op. 1560 std::pair<Type, std::string> R = 1561 emitDagArg(DI->getArg(1), DI->getArgNameStr(1)); 1562 return std::make_pair(R.first, Op + R.second); 1563 } else { 1564 assert(DI->getNumArgs() == 3 && "Can only handle unary and binary ops!"); 1565 std::pair<Type, std::string> R1 = 1566 emitDagArg(DI->getArg(1), DI->getArgNameStr(1)); 1567 std::pair<Type, std::string> R2 = 1568 emitDagArg(DI->getArg(2), DI->getArgNameStr(2)); 1569 assert_with_loc(R1.first == R2.first, "Argument type mismatch!"); 1570 return std::make_pair(R1.first, R1.second + " " + Op + " " + R2.second); 1571 } 1572 } 1573 1574 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCall(DagInit *DI) { 1575 std::vector<Type> Types; 1576 std::vector<std::string> Values; 1577 for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) { 1578 std::pair<Type, std::string> R = 1579 emitDagArg(DI->getArg(I + 1), DI->getArgNameStr(I + 1)); 1580 Types.push_back(R.first); 1581 Values.push_back(R.second); 1582 } 1583 1584 // Look up the called intrinsic. 1585 std::string N; 1586 if (StringInit *SI = dyn_cast<StringInit>(DI->getArg(0))) 1587 N = SI->getAsUnquotedString(); 1588 else 1589 N = emitDagArg(DI->getArg(0), "").second; 1590 Intrinsic &Callee = Intr.Emitter.getIntrinsic(N, Types); 1591 1592 // Make sure the callee is known as an early def. 1593 Callee.setNeededEarly(); 1594 Intr.Dependencies.insert(&Callee); 1595 1596 // Now create the call itself. 1597 std::string S = ""; 1598 if (!Callee.isBigEndianSafe()) 1599 S += CallPrefix.str(); 1600 S += Callee.getMangledName(true) + "("; 1601 for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) { 1602 if (I != 0) 1603 S += ", "; 1604 S += Values[I]; 1605 } 1606 S += ")"; 1607 1608 return std::make_pair(Callee.getReturnType(), S); 1609 } 1610 1611 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCast(DagInit *DI, 1612 bool IsBitCast){ 1613 // (cast MOD* VAL) -> cast VAL to type given by MOD. 1614 std::pair<Type, std::string> R = emitDagArg( 1615 DI->getArg(DI->getNumArgs() - 1), 1616 DI->getArgNameStr(DI->getNumArgs() - 1)); 1617 Type castToType = R.first; 1618 for (unsigned ArgIdx = 0; ArgIdx < DI->getNumArgs() - 1; ++ArgIdx) { 1619 1620 // MOD can take several forms: 1621 // 1. $X - take the type of parameter / variable X. 1622 // 2. The value "R" - take the type of the return type. 1623 // 3. a type string 1624 // 4. The value "U" or "S" to switch the signedness. 1625 // 5. The value "H" or "D" to half or double the bitwidth. 1626 // 6. The value "8" to convert to 8-bit (signed) integer lanes. 1627 if (!DI->getArgNameStr(ArgIdx).empty()) { 1628 assert_with_loc(Intr.Variables.find(DI->getArgNameStr(ArgIdx)) != 1629 Intr.Variables.end(), 1630 "Variable not found"); 1631 castToType = Intr.Variables[DI->getArgNameStr(ArgIdx)].getType(); 1632 } else { 1633 StringInit *SI = dyn_cast<StringInit>(DI->getArg(ArgIdx)); 1634 assert_with_loc(SI, "Expected string type or $Name for cast type"); 1635 1636 if (SI->getAsUnquotedString() == "R") { 1637 castToType = Intr.getReturnType(); 1638 } else if (SI->getAsUnquotedString() == "U") { 1639 castToType.makeUnsigned(); 1640 } else if (SI->getAsUnquotedString() == "S") { 1641 castToType.makeSigned(); 1642 } else if (SI->getAsUnquotedString() == "H") { 1643 castToType.halveLanes(); 1644 } else if (SI->getAsUnquotedString() == "D") { 1645 castToType.doubleLanes(); 1646 } else if (SI->getAsUnquotedString() == "8") { 1647 castToType.makeInteger(8, true); 1648 } else { 1649 castToType = Type::fromTypedefName(SI->getAsUnquotedString()); 1650 assert_with_loc(!castToType.isVoid(), "Unknown typedef"); 1651 } 1652 } 1653 } 1654 1655 std::string S; 1656 if (IsBitCast) { 1657 // Emit a reinterpret cast. The second operand must be an lvalue, so create 1658 // a temporary. 1659 std::string N = "reint"; 1660 unsigned I = 0; 1661 while (Intr.Variables.find(N) != Intr.Variables.end()) 1662 N = "reint" + utostr(++I); 1663 Intr.Variables[N] = Variable(R.first, N + Intr.VariablePostfix); 1664 1665 Intr.OS << R.first.str() << " " << Intr.Variables[N].getName() << " = " 1666 << R.second << ";"; 1667 Intr.emitNewLine(); 1668 1669 S = "*(" + castToType.str() + " *) &" + Intr.Variables[N].getName() + ""; 1670 } else { 1671 // Emit a normal (static) cast. 1672 S = "(" + castToType.str() + ")(" + R.second + ")"; 1673 } 1674 1675 return std::make_pair(castToType, S); 1676 } 1677 1678 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagShuffle(DagInit *DI){ 1679 // See the documentation in arm_neon.td for a description of these operators. 1680 class LowHalf : public SetTheory::Operator { 1681 public: 1682 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts, 1683 ArrayRef<SMLoc> Loc) override { 1684 SetTheory::RecSet Elts2; 1685 ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc); 1686 Elts.insert(Elts2.begin(), Elts2.begin() + (Elts2.size() / 2)); 1687 } 1688 }; 1689 1690 class HighHalf : public SetTheory::Operator { 1691 public: 1692 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts, 1693 ArrayRef<SMLoc> Loc) override { 1694 SetTheory::RecSet Elts2; 1695 ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc); 1696 Elts.insert(Elts2.begin() + (Elts2.size() / 2), Elts2.end()); 1697 } 1698 }; 1699 1700 class Rev : public SetTheory::Operator { 1701 unsigned ElementSize; 1702 1703 public: 1704 Rev(unsigned ElementSize) : ElementSize(ElementSize) {} 1705 1706 void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts, 1707 ArrayRef<SMLoc> Loc) override { 1708 SetTheory::RecSet Elts2; 1709 ST.evaluate(Expr->arg_begin() + 1, Expr->arg_end(), Elts2, Loc); 1710 1711 int64_t VectorSize = cast<IntInit>(Expr->getArg(0))->getValue(); 1712 VectorSize /= ElementSize; 1713 1714 std::vector<Record *> Revved; 1715 for (unsigned VI = 0; VI < Elts2.size(); VI += VectorSize) { 1716 for (int LI = VectorSize - 1; LI >= 0; --LI) { 1717 Revved.push_back(Elts2[VI + LI]); 1718 } 1719 } 1720 1721 Elts.insert(Revved.begin(), Revved.end()); 1722 } 1723 }; 1724 1725 class MaskExpander : public SetTheory::Expander { 1726 unsigned N; 1727 1728 public: 1729 MaskExpander(unsigned N) : N(N) {} 1730 1731 void expand(SetTheory &ST, Record *R, SetTheory::RecSet &Elts) override { 1732 unsigned Addend = 0; 1733 if (R->getName() == "mask0") 1734 Addend = 0; 1735 else if (R->getName() == "mask1") 1736 Addend = N; 1737 else 1738 return; 1739 for (unsigned I = 0; I < N; ++I) 1740 Elts.insert(R->getRecords().getDef("sv" + utostr(I + Addend))); 1741 } 1742 }; 1743 1744 // (shuffle arg1, arg2, sequence) 1745 std::pair<Type, std::string> Arg1 = 1746 emitDagArg(DI->getArg(0), DI->getArgNameStr(0)); 1747 std::pair<Type, std::string> Arg2 = 1748 emitDagArg(DI->getArg(1), DI->getArgNameStr(1)); 1749 assert_with_loc(Arg1.first == Arg2.first, 1750 "Different types in arguments to shuffle!"); 1751 1752 SetTheory ST; 1753 SetTheory::RecSet Elts; 1754 ST.addOperator("lowhalf", std::make_unique<LowHalf>()); 1755 ST.addOperator("highhalf", std::make_unique<HighHalf>()); 1756 ST.addOperator("rev", 1757 std::make_unique<Rev>(Arg1.first.getElementSizeInBits())); 1758 ST.addExpander("MaskExpand", 1759 std::make_unique<MaskExpander>(Arg1.first.getNumElements())); 1760 ST.evaluate(DI->getArg(2), Elts, None); 1761 1762 std::string S = "__builtin_shufflevector(" + Arg1.second + ", " + Arg2.second; 1763 for (auto &E : Elts) { 1764 StringRef Name = E->getName(); 1765 assert_with_loc(Name.startswith("sv"), 1766 "Incorrect element kind in shuffle mask!"); 1767 S += ", " + Name.drop_front(2).str(); 1768 } 1769 S += ")"; 1770 1771 // Recalculate the return type - the shuffle may have halved or doubled it. 1772 Type T(Arg1.first); 1773 if (Elts.size() > T.getNumElements()) { 1774 assert_with_loc( 1775 Elts.size() == T.getNumElements() * 2, 1776 "Can only double or half the number of elements in a shuffle!"); 1777 T.doubleLanes(); 1778 } else if (Elts.size() < T.getNumElements()) { 1779 assert_with_loc( 1780 Elts.size() == T.getNumElements() / 2, 1781 "Can only double or half the number of elements in a shuffle!"); 1782 T.halveLanes(); 1783 } 1784 1785 return std::make_pair(T, S); 1786 } 1787 1788 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDup(DagInit *DI) { 1789 assert_with_loc(DI->getNumArgs() == 1, "dup() expects one argument"); 1790 std::pair<Type, std::string> A = emitDagArg(DI->getArg(0), 1791 DI->getArgNameStr(0)); 1792 assert_with_loc(A.first.isScalar(), "dup() expects a scalar argument"); 1793 1794 Type T = Intr.getBaseType(); 1795 assert_with_loc(T.isVector(), "dup() used but default type is scalar!"); 1796 std::string S = "(" + T.str() + ") {"; 1797 for (unsigned I = 0; I < T.getNumElements(); ++I) { 1798 if (I != 0) 1799 S += ", "; 1800 S += A.second; 1801 } 1802 S += "}"; 1803 1804 return std::make_pair(T, S); 1805 } 1806 1807 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDupTyped(DagInit *DI) { 1808 assert_with_loc(DI->getNumArgs() == 2, "dup_typed() expects two arguments"); 1809 std::pair<Type, std::string> A = emitDagArg(DI->getArg(0), 1810 DI->getArgNameStr(0)); 1811 std::pair<Type, std::string> B = emitDagArg(DI->getArg(1), 1812 DI->getArgNameStr(1)); 1813 assert_with_loc(B.first.isScalar(), 1814 "dup_typed() requires a scalar as the second argument"); 1815 1816 Type T = A.first; 1817 assert_with_loc(T.isVector(), "dup_typed() used but target type is scalar!"); 1818 std::string S = "(" + T.str() + ") {"; 1819 for (unsigned I = 0; I < T.getNumElements(); ++I) { 1820 if (I != 0) 1821 S += ", "; 1822 S += B.second; 1823 } 1824 S += "}"; 1825 1826 return std::make_pair(T, S); 1827 } 1828 1829 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSplat(DagInit *DI) { 1830 assert_with_loc(DI->getNumArgs() == 2, "splat() expects two arguments"); 1831 std::pair<Type, std::string> A = emitDagArg(DI->getArg(0), 1832 DI->getArgNameStr(0)); 1833 std::pair<Type, std::string> B = emitDagArg(DI->getArg(1), 1834 DI->getArgNameStr(1)); 1835 1836 assert_with_loc(B.first.isScalar(), 1837 "splat() requires a scalar int as the second argument"); 1838 1839 std::string S = "__builtin_shufflevector(" + A.second + ", " + A.second; 1840 for (unsigned I = 0; I < Intr.getBaseType().getNumElements(); ++I) { 1841 S += ", " + B.second; 1842 } 1843 S += ")"; 1844 1845 return std::make_pair(Intr.getBaseType(), S); 1846 } 1847 1848 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSaveTemp(DagInit *DI) { 1849 assert_with_loc(DI->getNumArgs() == 2, "save_temp() expects two arguments"); 1850 std::pair<Type, std::string> A = emitDagArg(DI->getArg(1), 1851 DI->getArgNameStr(1)); 1852 1853 assert_with_loc(!A.first.isVoid(), 1854 "Argument to save_temp() must have non-void type!"); 1855 1856 std::string N = DI->getArgNameStr(0); 1857 assert_with_loc(!N.empty(), 1858 "save_temp() expects a name as the first argument"); 1859 1860 assert_with_loc(Intr.Variables.find(N) == Intr.Variables.end(), 1861 "Variable already defined!"); 1862 Intr.Variables[N] = Variable(A.first, N + Intr.VariablePostfix); 1863 1864 std::string S = 1865 A.first.str() + " " + Intr.Variables[N].getName() + " = " + A.second; 1866 1867 return std::make_pair(Type::getVoid(), S); 1868 } 1869 1870 std::pair<Type, std::string> 1871 Intrinsic::DagEmitter::emitDagNameReplace(DagInit *DI) { 1872 std::string S = Intr.Name; 1873 1874 assert_with_loc(DI->getNumArgs() == 2, "name_replace requires 2 arguments!"); 1875 std::string ToReplace = cast<StringInit>(DI->getArg(0))->getAsUnquotedString(); 1876 std::string ReplaceWith = cast<StringInit>(DI->getArg(1))->getAsUnquotedString(); 1877 1878 size_t Idx = S.find(ToReplace); 1879 1880 assert_with_loc(Idx != std::string::npos, "name should contain '" + ToReplace + "'!"); 1881 S.replace(Idx, ToReplace.size(), ReplaceWith); 1882 1883 return std::make_pair(Type::getVoid(), S); 1884 } 1885 1886 std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagLiteral(DagInit *DI){ 1887 std::string Ty = cast<StringInit>(DI->getArg(0))->getAsUnquotedString(); 1888 std::string Value = cast<StringInit>(DI->getArg(1))->getAsUnquotedString(); 1889 return std::make_pair(Type::fromTypedefName(Ty), Value); 1890 } 1891 1892 std::pair<Type, std::string> 1893 Intrinsic::DagEmitter::emitDagArg(Init *Arg, std::string ArgName) { 1894 if (!ArgName.empty()) { 1895 assert_with_loc(!Arg->isComplete(), 1896 "Arguments must either be DAGs or names, not both!"); 1897 assert_with_loc(Intr.Variables.find(ArgName) != Intr.Variables.end(), 1898 "Variable not defined!"); 1899 Variable &V = Intr.Variables[ArgName]; 1900 return std::make_pair(V.getType(), V.getName()); 1901 } 1902 1903 assert(Arg && "Neither ArgName nor Arg?!"); 1904 DagInit *DI = dyn_cast<DagInit>(Arg); 1905 assert_with_loc(DI, "Arguments must either be DAGs or names!"); 1906 1907 return emitDag(DI); 1908 } 1909 1910 std::string Intrinsic::generate() { 1911 // Avoid duplicated code for big and little endian 1912 if (isBigEndianSafe()) { 1913 generateImpl(false, "", ""); 1914 return OS.str(); 1915 } 1916 // Little endian intrinsics are simple and don't require any argument 1917 // swapping. 1918 OS << "#ifdef __LITTLE_ENDIAN__\n"; 1919 1920 generateImpl(false, "", ""); 1921 1922 OS << "#else\n"; 1923 1924 // Big endian intrinsics are more complex. The user intended these 1925 // intrinsics to operate on a vector "as-if" loaded by (V)LDR, 1926 // but we load as-if (V)LD1. So we should swap all arguments and 1927 // swap the return value too. 1928 // 1929 // If we call sub-intrinsics, we should call a version that does 1930 // not re-swap the arguments! 1931 generateImpl(true, "", "__noswap_"); 1932 1933 // If we're needed early, create a non-swapping variant for 1934 // big-endian. 1935 if (NeededEarly) { 1936 generateImpl(false, "__noswap_", "__noswap_"); 1937 } 1938 OS << "#endif\n\n"; 1939 1940 return OS.str(); 1941 } 1942 1943 void Intrinsic::generateImpl(bool ReverseArguments, 1944 StringRef NamePrefix, StringRef CallPrefix) { 1945 CurrentRecord = R; 1946 1947 // If we call a macro, our local variables may be corrupted due to 1948 // lack of proper lexical scoping. So, add a globally unique postfix 1949 // to every variable. 1950 // 1951 // indexBody() should have set up the Dependencies set by now. 1952 for (auto *I : Dependencies) 1953 if (I->UseMacro) { 1954 VariablePostfix = "_" + utostr(Emitter.getUniqueNumber()); 1955 break; 1956 } 1957 1958 initVariables(); 1959 1960 emitPrototype(NamePrefix); 1961 1962 if (IsUnavailable) { 1963 OS << " __attribute__((unavailable));"; 1964 } else { 1965 emitOpeningBrace(); 1966 emitShadowedArgs(); 1967 if (ReverseArguments) 1968 emitArgumentReversal(); 1969 emitBody(CallPrefix); 1970 if (ReverseArguments) 1971 emitReturnReversal(); 1972 emitReturn(); 1973 emitClosingBrace(); 1974 } 1975 OS << "\n"; 1976 1977 CurrentRecord = nullptr; 1978 } 1979 1980 void Intrinsic::indexBody() { 1981 CurrentRecord = R; 1982 1983 initVariables(); 1984 emitBody(""); 1985 OS.str(""); 1986 1987 CurrentRecord = nullptr; 1988 } 1989 1990 //===----------------------------------------------------------------------===// 1991 // NeonEmitter implementation 1992 //===----------------------------------------------------------------------===// 1993 1994 Intrinsic &NeonEmitter::getIntrinsic(StringRef Name, ArrayRef<Type> Types) { 1995 // First, look up the name in the intrinsic map. 1996 assert_with_loc(IntrinsicMap.find(Name.str()) != IntrinsicMap.end(), 1997 ("Intrinsic '" + Name + "' not found!").str()); 1998 auto &V = IntrinsicMap.find(Name.str())->second; 1999 std::vector<Intrinsic *> GoodVec; 2000 2001 // Create a string to print if we end up failing. 2002 std::string ErrMsg = "looking up intrinsic '" + Name.str() + "("; 2003 for (unsigned I = 0; I < Types.size(); ++I) { 2004 if (I != 0) 2005 ErrMsg += ", "; 2006 ErrMsg += Types[I].str(); 2007 } 2008 ErrMsg += ")'\n"; 2009 ErrMsg += "Available overloads:\n"; 2010 2011 // Now, look through each intrinsic implementation and see if the types are 2012 // compatible. 2013 for (auto &I : V) { 2014 ErrMsg += " - " + I.getReturnType().str() + " " + I.getMangledName(); 2015 ErrMsg += "("; 2016 for (unsigned A = 0; A < I.getNumParams(); ++A) { 2017 if (A != 0) 2018 ErrMsg += ", "; 2019 ErrMsg += I.getParamType(A).str(); 2020 } 2021 ErrMsg += ")\n"; 2022 2023 if (I.getNumParams() != Types.size()) 2024 continue; 2025 2026 bool Good = true; 2027 for (unsigned Arg = 0; Arg < Types.size(); ++Arg) { 2028 if (I.getParamType(Arg) != Types[Arg]) { 2029 Good = false; 2030 break; 2031 } 2032 } 2033 if (Good) 2034 GoodVec.push_back(&I); 2035 } 2036 2037 assert_with_loc(!GoodVec.empty(), 2038 "No compatible intrinsic found - " + ErrMsg); 2039 assert_with_loc(GoodVec.size() == 1, "Multiple overloads found - " + ErrMsg); 2040 2041 return *GoodVec.front(); 2042 } 2043 2044 void NeonEmitter::createIntrinsic(Record *R, 2045 SmallVectorImpl<Intrinsic *> &Out) { 2046 std::string Name = R->getValueAsString("Name"); 2047 std::string Proto = R->getValueAsString("Prototype"); 2048 std::string Types = R->getValueAsString("Types"); 2049 Record *OperationRec = R->getValueAsDef("Operation"); 2050 bool CartesianProductOfTypes = R->getValueAsBit("CartesianProductOfTypes"); 2051 bool BigEndianSafe = R->getValueAsBit("BigEndianSafe"); 2052 std::string Guard = R->getValueAsString("ArchGuard"); 2053 bool IsUnavailable = OperationRec->getValueAsBit("Unavailable"); 2054 2055 // Set the global current record. This allows assert_with_loc to produce 2056 // decent location information even when highly nested. 2057 CurrentRecord = R; 2058 2059 ListInit *Body = OperationRec->getValueAsListInit("Ops"); 2060 2061 std::vector<TypeSpec> TypeSpecs = TypeSpec::fromTypeSpecs(Types); 2062 2063 ClassKind CK = ClassNone; 2064 if (R->getSuperClasses().size() >= 2) 2065 CK = ClassMap[R->getSuperClasses()[1].first]; 2066 2067 std::vector<std::pair<TypeSpec, TypeSpec>> NewTypeSpecs; 2068 for (auto TS : TypeSpecs) { 2069 if (CartesianProductOfTypes) { 2070 Type DefaultT(TS, 'd'); 2071 for (auto SrcTS : TypeSpecs) { 2072 Type DefaultSrcT(SrcTS, 'd'); 2073 if (TS == SrcTS || 2074 DefaultSrcT.getSizeInBits() != DefaultT.getSizeInBits()) 2075 continue; 2076 NewTypeSpecs.push_back(std::make_pair(TS, SrcTS)); 2077 } 2078 } else { 2079 NewTypeSpecs.push_back(std::make_pair(TS, TS)); 2080 } 2081 } 2082 2083 llvm::sort(NewTypeSpecs); 2084 NewTypeSpecs.erase(std::unique(NewTypeSpecs.begin(), NewTypeSpecs.end()), 2085 NewTypeSpecs.end()); 2086 auto &Entry = IntrinsicMap[Name]; 2087 2088 for (auto &I : NewTypeSpecs) { 2089 Entry.emplace_back(R, Name, Proto, I.first, I.second, CK, Body, *this, 2090 Guard, IsUnavailable, BigEndianSafe); 2091 Out.push_back(&Entry.back()); 2092 } 2093 2094 CurrentRecord = nullptr; 2095 } 2096 2097 /// genBuiltinsDef: Generate the BuiltinsARM.def and BuiltinsAArch64.def 2098 /// declaration of builtins, checking for unique builtin declarations. 2099 void NeonEmitter::genBuiltinsDef(raw_ostream &OS, 2100 SmallVectorImpl<Intrinsic *> &Defs) { 2101 OS << "#ifdef GET_NEON_BUILTINS\n"; 2102 2103 // We only want to emit a builtin once, and we want to emit them in 2104 // alphabetical order, so use a std::set. 2105 std::set<std::string> Builtins; 2106 2107 for (auto *Def : Defs) { 2108 if (Def->hasBody()) 2109 continue; 2110 // Functions with 'a' (the splat code) in the type prototype should not get 2111 // their own builtin as they use the non-splat variant. 2112 if (Def->hasSplat()) 2113 continue; 2114 2115 std::string S = "BUILTIN(__builtin_neon_" + Def->getMangledName() + ", \""; 2116 2117 S += Def->getBuiltinTypeStr(); 2118 S += "\", \"n\")"; 2119 2120 Builtins.insert(S); 2121 } 2122 2123 for (auto &S : Builtins) 2124 OS << S << "\n"; 2125 OS << "#endif\n\n"; 2126 } 2127 2128 /// Generate the ARM and AArch64 overloaded type checking code for 2129 /// SemaChecking.cpp, checking for unique builtin declarations. 2130 void NeonEmitter::genOverloadTypeCheckCode(raw_ostream &OS, 2131 SmallVectorImpl<Intrinsic *> &Defs) { 2132 OS << "#ifdef GET_NEON_OVERLOAD_CHECK\n"; 2133 2134 // We record each overload check line before emitting because subsequent Inst 2135 // definitions may extend the number of permitted types (i.e. augment the 2136 // Mask). Use std::map to avoid sorting the table by hash number. 2137 struct OverloadInfo { 2138 uint64_t Mask; 2139 int PtrArgNum; 2140 bool HasConstPtr; 2141 OverloadInfo() : Mask(0ULL), PtrArgNum(0), HasConstPtr(false) {} 2142 }; 2143 std::map<std::string, OverloadInfo> OverloadMap; 2144 2145 for (auto *Def : Defs) { 2146 // If the def has a body (that is, it has Operation DAGs), it won't call 2147 // __builtin_neon_* so we don't need to generate a definition for it. 2148 if (Def->hasBody()) 2149 continue; 2150 // Functions with 'a' (the splat code) in the type prototype should not get 2151 // their own builtin as they use the non-splat variant. 2152 if (Def->hasSplat()) 2153 continue; 2154 // Functions which have a scalar argument cannot be overloaded, no need to 2155 // check them if we are emitting the type checking code. 2156 if (Def->protoHasScalar()) 2157 continue; 2158 2159 uint64_t Mask = 0ULL; 2160 Type Ty = Def->getReturnType(); 2161 if (Def->getProto()[0] == 'v' || 2162 isFloatingPointProtoModifier(Def->getProto()[0])) 2163 Ty = Def->getParamType(0); 2164 if (Ty.isPointer()) 2165 Ty = Def->getParamType(1); 2166 2167 Mask |= 1ULL << Ty.getNeonEnum(); 2168 2169 // Check if the function has a pointer or const pointer argument. 2170 std::string Proto = Def->getProto(); 2171 int PtrArgNum = -1; 2172 bool HasConstPtr = false; 2173 for (unsigned I = 0; I < Def->getNumParams(); ++I) { 2174 char ArgType = Proto[I + 1]; 2175 if (ArgType == 'c') { 2176 HasConstPtr = true; 2177 PtrArgNum = I; 2178 break; 2179 } 2180 if (ArgType == 'p') { 2181 PtrArgNum = I; 2182 break; 2183 } 2184 } 2185 // For sret builtins, adjust the pointer argument index. 2186 if (PtrArgNum >= 0 && Def->getReturnType().getNumVectors() > 1) 2187 PtrArgNum += 1; 2188 2189 std::string Name = Def->getName(); 2190 // Omit type checking for the pointer arguments of vld1_lane, vld1_dup, 2191 // and vst1_lane intrinsics. Using a pointer to the vector element 2192 // type with one of those operations causes codegen to select an aligned 2193 // load/store instruction. If you want an unaligned operation, 2194 // the pointer argument needs to have less alignment than element type, 2195 // so just accept any pointer type. 2196 if (Name == "vld1_lane" || Name == "vld1_dup" || Name == "vst1_lane") { 2197 PtrArgNum = -1; 2198 HasConstPtr = false; 2199 } 2200 2201 if (Mask) { 2202 std::string Name = Def->getMangledName(); 2203 OverloadMap.insert(std::make_pair(Name, OverloadInfo())); 2204 OverloadInfo &OI = OverloadMap[Name]; 2205 OI.Mask |= Mask; 2206 OI.PtrArgNum |= PtrArgNum; 2207 OI.HasConstPtr = HasConstPtr; 2208 } 2209 } 2210 2211 for (auto &I : OverloadMap) { 2212 OverloadInfo &OI = I.second; 2213 2214 OS << "case NEON::BI__builtin_neon_" << I.first << ": "; 2215 OS << "mask = 0x" << Twine::utohexstr(OI.Mask) << "ULL"; 2216 if (OI.PtrArgNum >= 0) 2217 OS << "; PtrArgNum = " << OI.PtrArgNum; 2218 if (OI.HasConstPtr) 2219 OS << "; HasConstPtr = true"; 2220 OS << "; break;\n"; 2221 } 2222 OS << "#endif\n\n"; 2223 } 2224 2225 void NeonEmitter::genIntrinsicRangeCheckCode(raw_ostream &OS, 2226 SmallVectorImpl<Intrinsic *> &Defs) { 2227 OS << "#ifdef GET_NEON_IMMEDIATE_CHECK\n"; 2228 2229 std::set<std::string> Emitted; 2230 2231 for (auto *Def : Defs) { 2232 if (Def->hasBody()) 2233 continue; 2234 // Functions with 'a' (the splat code) in the type prototype should not get 2235 // their own builtin as they use the non-splat variant. 2236 if (Def->hasSplat()) 2237 continue; 2238 // Functions which do not have an immediate do not need to have range 2239 // checking code emitted. 2240 if (!Def->hasImmediate()) 2241 continue; 2242 if (Emitted.find(Def->getMangledName()) != Emitted.end()) 2243 continue; 2244 2245 std::string LowerBound, UpperBound; 2246 2247 Record *R = Def->getRecord(); 2248 if (R->getValueAsBit("isVCVT_N")) { 2249 // VCVT between floating- and fixed-point values takes an immediate 2250 // in the range [1, 32) for f32 or [1, 64) for f64 or [1, 16) for f16. 2251 LowerBound = "1"; 2252 if (Def->getBaseType().getElementSizeInBits() == 16 || 2253 Def->getName().find('h') != std::string::npos) 2254 // VCVTh operating on FP16 intrinsics in range [1, 16) 2255 UpperBound = "15"; 2256 else if (Def->getBaseType().getElementSizeInBits() == 32) 2257 UpperBound = "31"; 2258 else 2259 UpperBound = "63"; 2260 } else if (R->getValueAsBit("isScalarShift")) { 2261 // Right shifts have an 'r' in the name, left shifts do not. Convert 2262 // instructions have the same bounds and right shifts. 2263 if (Def->getName().find('r') != std::string::npos || 2264 Def->getName().find("cvt") != std::string::npos) 2265 LowerBound = "1"; 2266 2267 UpperBound = utostr(Def->getReturnType().getElementSizeInBits() - 1); 2268 } else if (R->getValueAsBit("isShift")) { 2269 // Builtins which are overloaded by type will need to have their upper 2270 // bound computed at Sema time based on the type constant. 2271 2272 // Right shifts have an 'r' in the name, left shifts do not. 2273 if (Def->getName().find('r') != std::string::npos) 2274 LowerBound = "1"; 2275 UpperBound = "RFT(TV, true)"; 2276 } else if (Def->getClassKind(true) == ClassB) { 2277 // ClassB intrinsics have a type (and hence lane number) that is only 2278 // known at runtime. 2279 if (R->getValueAsBit("isLaneQ")) 2280 UpperBound = "RFT(TV, false, true)"; 2281 else 2282 UpperBound = "RFT(TV, false, false)"; 2283 } else { 2284 // The immediate generally refers to a lane in the preceding argument. 2285 assert(Def->getImmediateIdx() > 0); 2286 Type T = Def->getParamType(Def->getImmediateIdx() - 1); 2287 UpperBound = utostr(T.getNumElements() - 1); 2288 } 2289 2290 // Calculate the index of the immediate that should be range checked. 2291 unsigned Idx = Def->getNumParams(); 2292 if (Def->hasImmediate()) 2293 Idx = Def->getGeneratedParamIdx(Def->getImmediateIdx()); 2294 2295 OS << "case NEON::BI__builtin_neon_" << Def->getMangledName() << ": " 2296 << "i = " << Idx << ";"; 2297 if (!LowerBound.empty()) 2298 OS << " l = " << LowerBound << ";"; 2299 if (!UpperBound.empty()) 2300 OS << " u = " << UpperBound << ";"; 2301 OS << " break;\n"; 2302 2303 Emitted.insert(Def->getMangledName()); 2304 } 2305 2306 OS << "#endif\n\n"; 2307 } 2308 2309 /// runHeader - Emit a file with sections defining: 2310 /// 1. the NEON section of BuiltinsARM.def and BuiltinsAArch64.def. 2311 /// 2. the SemaChecking code for the type overload checking. 2312 /// 3. the SemaChecking code for validation of intrinsic immediate arguments. 2313 void NeonEmitter::runHeader(raw_ostream &OS) { 2314 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 2315 2316 SmallVector<Intrinsic *, 128> Defs; 2317 for (auto *R : RV) 2318 createIntrinsic(R, Defs); 2319 2320 // Generate shared BuiltinsXXX.def 2321 genBuiltinsDef(OS, Defs); 2322 2323 // Generate ARM overloaded type checking code for SemaChecking.cpp 2324 genOverloadTypeCheckCode(OS, Defs); 2325 2326 // Generate ARM range checking code for shift/lane immediates. 2327 genIntrinsicRangeCheckCode(OS, Defs); 2328 } 2329 2330 /// run - Read the records in arm_neon.td and output arm_neon.h. arm_neon.h 2331 /// is comprised of type definitions and function declarations. 2332 void NeonEmitter::run(raw_ostream &OS) { 2333 OS << "/*===---- arm_neon.h - ARM Neon intrinsics " 2334 "------------------------------" 2335 "---===\n" 2336 " *\n" 2337 " * Permission is hereby granted, free of charge, to any person " 2338 "obtaining " 2339 "a copy\n" 2340 " * of this software and associated documentation files (the " 2341 "\"Software\")," 2342 " to deal\n" 2343 " * in the Software without restriction, including without limitation " 2344 "the " 2345 "rights\n" 2346 " * to use, copy, modify, merge, publish, distribute, sublicense, " 2347 "and/or sell\n" 2348 " * copies of the Software, and to permit persons to whom the Software " 2349 "is\n" 2350 " * furnished to do so, subject to the following conditions:\n" 2351 " *\n" 2352 " * The above copyright notice and this permission notice shall be " 2353 "included in\n" 2354 " * all copies or substantial portions of the Software.\n" 2355 " *\n" 2356 " * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, " 2357 "EXPRESS OR\n" 2358 " * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF " 2359 "MERCHANTABILITY,\n" 2360 " * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT " 2361 "SHALL THE\n" 2362 " * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR " 2363 "OTHER\n" 2364 " * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, " 2365 "ARISING FROM,\n" 2366 " * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER " 2367 "DEALINGS IN\n" 2368 " * THE SOFTWARE.\n" 2369 " *\n" 2370 " *===-----------------------------------------------------------------" 2371 "---" 2372 "---===\n" 2373 " */\n\n"; 2374 2375 OS << "#ifndef __ARM_NEON_H\n"; 2376 OS << "#define __ARM_NEON_H\n\n"; 2377 2378 OS << "#if !defined(__ARM_NEON)\n"; 2379 OS << "#error \"NEON support not enabled\"\n"; 2380 OS << "#endif\n\n"; 2381 2382 OS << "#include <stdint.h>\n\n"; 2383 2384 // Emit NEON-specific scalar typedefs. 2385 OS << "typedef float float32_t;\n"; 2386 OS << "typedef __fp16 float16_t;\n"; 2387 2388 OS << "#ifdef __aarch64__\n"; 2389 OS << "typedef double float64_t;\n"; 2390 OS << "#endif\n\n"; 2391 2392 // For now, signedness of polynomial types depends on target 2393 OS << "#ifdef __aarch64__\n"; 2394 OS << "typedef uint8_t poly8_t;\n"; 2395 OS << "typedef uint16_t poly16_t;\n"; 2396 OS << "typedef uint64_t poly64_t;\n"; 2397 OS << "typedef __uint128_t poly128_t;\n"; 2398 OS << "#else\n"; 2399 OS << "typedef int8_t poly8_t;\n"; 2400 OS << "typedef int16_t poly16_t;\n"; 2401 OS << "#endif\n"; 2402 2403 // Emit Neon vector typedefs. 2404 std::string TypedefTypes( 2405 "cQcsQsiQilQlUcQUcUsQUsUiQUiUlQUlhQhfQfdQdPcQPcPsQPsPlQPl"); 2406 std::vector<TypeSpec> TDTypeVec = TypeSpec::fromTypeSpecs(TypedefTypes); 2407 2408 // Emit vector typedefs. 2409 bool InIfdef = false; 2410 for (auto &TS : TDTypeVec) { 2411 bool IsA64 = false; 2412 Type T(TS, 'd'); 2413 if (T.isDouble() || (T.isPoly() && T.isLong())) 2414 IsA64 = true; 2415 2416 if (InIfdef && !IsA64) { 2417 OS << "#endif\n"; 2418 InIfdef = false; 2419 } 2420 if (!InIfdef && IsA64) { 2421 OS << "#ifdef __aarch64__\n"; 2422 InIfdef = true; 2423 } 2424 2425 if (T.isPoly()) 2426 OS << "typedef __attribute__((neon_polyvector_type("; 2427 else 2428 OS << "typedef __attribute__((neon_vector_type("; 2429 2430 Type T2 = T; 2431 T2.makeScalar(); 2432 OS << T.getNumElements() << "))) "; 2433 OS << T2.str(); 2434 OS << " " << T.str() << ";\n"; 2435 } 2436 if (InIfdef) 2437 OS << "#endif\n"; 2438 OS << "\n"; 2439 2440 // Emit struct typedefs. 2441 InIfdef = false; 2442 for (unsigned NumMembers = 2; NumMembers <= 4; ++NumMembers) { 2443 for (auto &TS : TDTypeVec) { 2444 bool IsA64 = false; 2445 Type T(TS, 'd'); 2446 if (T.isDouble() || (T.isPoly() && T.isLong())) 2447 IsA64 = true; 2448 2449 if (InIfdef && !IsA64) { 2450 OS << "#endif\n"; 2451 InIfdef = false; 2452 } 2453 if (!InIfdef && IsA64) { 2454 OS << "#ifdef __aarch64__\n"; 2455 InIfdef = true; 2456 } 2457 2458 char M = '2' + (NumMembers - 2); 2459 Type VT(TS, M); 2460 OS << "typedef struct " << VT.str() << " {\n"; 2461 OS << " " << T.str() << " val"; 2462 OS << "[" << NumMembers << "]"; 2463 OS << ";\n} "; 2464 OS << VT.str() << ";\n"; 2465 OS << "\n"; 2466 } 2467 } 2468 if (InIfdef) 2469 OS << "#endif\n"; 2470 OS << "\n"; 2471 2472 OS << "#define __ai static __inline__ __attribute__((__always_inline__, " 2473 "__nodebug__))\n\n"; 2474 2475 SmallVector<Intrinsic *, 128> Defs; 2476 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 2477 for (auto *R : RV) 2478 createIntrinsic(R, Defs); 2479 2480 for (auto *I : Defs) 2481 I->indexBody(); 2482 2483 llvm::stable_sort(Defs, llvm::deref<std::less<>>()); 2484 2485 // Only emit a def when its requirements have been met. 2486 // FIXME: This loop could be made faster, but it's fast enough for now. 2487 bool MadeProgress = true; 2488 std::string InGuard; 2489 while (!Defs.empty() && MadeProgress) { 2490 MadeProgress = false; 2491 2492 for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin(); 2493 I != Defs.end(); /*No step*/) { 2494 bool DependenciesSatisfied = true; 2495 for (auto *II : (*I)->getDependencies()) { 2496 if (llvm::is_contained(Defs, II)) 2497 DependenciesSatisfied = false; 2498 } 2499 if (!DependenciesSatisfied) { 2500 // Try the next one. 2501 ++I; 2502 continue; 2503 } 2504 2505 // Emit #endif/#if pair if needed. 2506 if ((*I)->getGuard() != InGuard) { 2507 if (!InGuard.empty()) 2508 OS << "#endif\n"; 2509 InGuard = (*I)->getGuard(); 2510 if (!InGuard.empty()) 2511 OS << "#if " << InGuard << "\n"; 2512 } 2513 2514 // Actually generate the intrinsic code. 2515 OS << (*I)->generate(); 2516 2517 MadeProgress = true; 2518 I = Defs.erase(I); 2519 } 2520 } 2521 assert(Defs.empty() && "Some requirements were not satisfied!"); 2522 if (!InGuard.empty()) 2523 OS << "#endif\n"; 2524 2525 OS << "\n"; 2526 OS << "#undef __ai\n\n"; 2527 OS << "#endif /* __ARM_NEON_H */\n"; 2528 } 2529 2530 /// run - Read the records in arm_fp16.td and output arm_fp16.h. arm_fp16.h 2531 /// is comprised of type definitions and function declarations. 2532 void NeonEmitter::runFP16(raw_ostream &OS) { 2533 OS << "/*===---- arm_fp16.h - ARM FP16 intrinsics " 2534 "------------------------------" 2535 "---===\n" 2536 " *\n" 2537 " * Permission is hereby granted, free of charge, to any person " 2538 "obtaining a copy\n" 2539 " * of this software and associated documentation files (the " 2540 "\"Software\"), to deal\n" 2541 " * in the Software without restriction, including without limitation " 2542 "the rights\n" 2543 " * to use, copy, modify, merge, publish, distribute, sublicense, " 2544 "and/or sell\n" 2545 " * copies of the Software, and to permit persons to whom the Software " 2546 "is\n" 2547 " * furnished to do so, subject to the following conditions:\n" 2548 " *\n" 2549 " * The above copyright notice and this permission notice shall be " 2550 "included in\n" 2551 " * all copies or substantial portions of the Software.\n" 2552 " *\n" 2553 " * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, " 2554 "EXPRESS OR\n" 2555 " * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF " 2556 "MERCHANTABILITY,\n" 2557 " * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT " 2558 "SHALL THE\n" 2559 " * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR " 2560 "OTHER\n" 2561 " * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, " 2562 "ARISING FROM,\n" 2563 " * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER " 2564 "DEALINGS IN\n" 2565 " * THE SOFTWARE.\n" 2566 " *\n" 2567 " *===-----------------------------------------------------------------" 2568 "---" 2569 "---===\n" 2570 " */\n\n"; 2571 2572 OS << "#ifndef __ARM_FP16_H\n"; 2573 OS << "#define __ARM_FP16_H\n\n"; 2574 2575 OS << "#include <stdint.h>\n\n"; 2576 2577 OS << "typedef __fp16 float16_t;\n"; 2578 2579 OS << "#define __ai static __inline__ __attribute__((__always_inline__, " 2580 "__nodebug__))\n\n"; 2581 2582 SmallVector<Intrinsic *, 128> Defs; 2583 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 2584 for (auto *R : RV) 2585 createIntrinsic(R, Defs); 2586 2587 for (auto *I : Defs) 2588 I->indexBody(); 2589 2590 llvm::stable_sort(Defs, llvm::deref<std::less<>>()); 2591 2592 // Only emit a def when its requirements have been met. 2593 // FIXME: This loop could be made faster, but it's fast enough for now. 2594 bool MadeProgress = true; 2595 std::string InGuard; 2596 while (!Defs.empty() && MadeProgress) { 2597 MadeProgress = false; 2598 2599 for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin(); 2600 I != Defs.end(); /*No step*/) { 2601 bool DependenciesSatisfied = true; 2602 for (auto *II : (*I)->getDependencies()) { 2603 if (llvm::is_contained(Defs, II)) 2604 DependenciesSatisfied = false; 2605 } 2606 if (!DependenciesSatisfied) { 2607 // Try the next one. 2608 ++I; 2609 continue; 2610 } 2611 2612 // Emit #endif/#if pair if needed. 2613 if ((*I)->getGuard() != InGuard) { 2614 if (!InGuard.empty()) 2615 OS << "#endif\n"; 2616 InGuard = (*I)->getGuard(); 2617 if (!InGuard.empty()) 2618 OS << "#if " << InGuard << "\n"; 2619 } 2620 2621 // Actually generate the intrinsic code. 2622 OS << (*I)->generate(); 2623 2624 MadeProgress = true; 2625 I = Defs.erase(I); 2626 } 2627 } 2628 assert(Defs.empty() && "Some requirements were not satisfied!"); 2629 if (!InGuard.empty()) 2630 OS << "#endif\n"; 2631 2632 OS << "\n"; 2633 OS << "#undef __ai\n\n"; 2634 OS << "#endif /* __ARM_FP16_H */\n"; 2635 } 2636 2637 void clang::EmitNeon(RecordKeeper &Records, raw_ostream &OS) { 2638 NeonEmitter(Records).run(OS); 2639 } 2640 2641 void clang::EmitFP16(RecordKeeper &Records, raw_ostream &OS) { 2642 NeonEmitter(Records).runFP16(OS); 2643 } 2644 2645 void clang::EmitNeonSema(RecordKeeper &Records, raw_ostream &OS) { 2646 NeonEmitter(Records).runHeader(OS); 2647 } 2648 2649 void clang::EmitNeonTest(RecordKeeper &Records, raw_ostream &OS) { 2650 llvm_unreachable("Neon test generation no longer implemented!"); 2651 } 2652