1 //===- MveEmitter.cpp - Generate arm_mve.h for use with clang -*- C++ -*-=====// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This set of linked tablegen backends is responsible for emitting the bits 10 // and pieces that implement <arm_mve.h>, which is defined by the ACLE standard 11 // and provides a set of types and functions for (more or less) direct access 12 // to the MVE instruction set, including the scalar shifts as well as the 13 // vector instructions. 14 // 15 // MVE's standard intrinsic functions are unusual in that they have a system of 16 // polymorphism. For example, the function vaddq() can behave like vaddq_u16(), 17 // vaddq_f32(), vaddq_s8(), etc., depending on the types of the vector 18 // arguments you give it. 19 // 20 // This constrains the implementation strategies. The usual approach to making 21 // the user-facing functions polymorphic would be to either use 22 // __attribute__((overloadable)) to make a set of vaddq() functions that are 23 // all inline wrappers on the underlying clang builtins, or to define a single 24 // vaddq() macro which expands to an instance of _Generic. 25 // 26 // The inline-wrappers approach would work fine for most intrinsics, except for 27 // the ones that take an argument required to be a compile-time constant, 28 // because if you wrap an inline function around a call to a builtin, the 29 // constant nature of the argument is not passed through. 30 // 31 // The _Generic approach can be made to work with enough effort, but it takes a 32 // lot of machinery, because of the design feature of _Generic that even the 33 // untaken branches are required to pass all front-end validity checks such as 34 // type-correctness. You can work around that by nesting further _Generics all 35 // over the place to coerce things to the right type in untaken branches, but 36 // what you get out is complicated, hard to guarantee its correctness, and 37 // worst of all, gives _completely unreadable_ error messages if the user gets 38 // the types wrong for an intrinsic call. 39 // 40 // Therefore, my strategy is to introduce a new __attribute__ that allows a 41 // function to be mapped to a clang builtin even though it doesn't have the 42 // same name, and then declare all the user-facing MVE function names with that 43 // attribute, mapping each one directly to the clang builtin. And the 44 // polymorphic ones have __attribute__((overloadable)) as well. So once the 45 // compiler has resolved the overload, it knows the internal builtin ID of the 46 // selected function, and can check the immediate arguments against that; and 47 // if the user gets the types wrong in a call to a polymorphic intrinsic, they 48 // get a completely clear error message showing all the declarations of that 49 // function in the header file and explaining why each one doesn't fit their 50 // call. 51 // 52 // The downside of this is that if every clang builtin has to correspond 53 // exactly to a user-facing ACLE intrinsic, then you can't save work in the 54 // frontend by doing it in the header file: CGBuiltin.cpp has to do the entire 55 // job of converting an ACLE intrinsic call into LLVM IR. So the Tablegen 56 // description for an MVE intrinsic has to contain a full description of the 57 // sequence of IRBuilder calls that clang will need to make. 58 // 59 //===----------------------------------------------------------------------===// 60 61 #include "llvm/ADT/APInt.h" 62 #include "llvm/ADT/StringRef.h" 63 #include "llvm/ADT/StringSwitch.h" 64 #include "llvm/Support/Casting.h" 65 #include "llvm/Support/raw_ostream.h" 66 #include "llvm/TableGen/Error.h" 67 #include "llvm/TableGen/Record.h" 68 #include "llvm/TableGen/StringToOffsetTable.h" 69 #include <cassert> 70 #include <cstddef> 71 #include <cstdint> 72 #include <list> 73 #include <map> 74 #include <memory> 75 #include <set> 76 #include <string> 77 #include <vector> 78 79 using namespace llvm; 80 81 namespace { 82 83 class EmitterBase; 84 class Result; 85 86 // ----------------------------------------------------------------------------- 87 // A system of classes to represent all the types we'll need to deal with in 88 // the prototypes of intrinsics. 89 // 90 // Query methods include finding out the C name of a type; the "LLVM name" in 91 // the sense of a C++ code snippet that can be used in the codegen function; 92 // the suffix that represents the type in the ACLE intrinsic naming scheme 93 // (e.g. 's32' represents int32_t in intrinsics such as vaddq_s32); whether the 94 // type is floating-point related (hence should be under #ifdef in the MVE 95 // header so that it isn't included in integer-only MVE mode); and the type's 96 // size in bits. Not all subtypes support all these queries. 97 98 class Type { 99 public: 100 enum class TypeKind { 101 // Void appears as a return type (for store intrinsics, which are pure 102 // side-effect). It's also used as the parameter type in the Tablegen 103 // when an intrinsic doesn't need to come in various suffixed forms like 104 // vfooq_s8,vfooq_u16,vfooq_f32. 105 Void, 106 107 // Scalar is used for ordinary int and float types of all sizes. 108 Scalar, 109 110 // Vector is used for anything that occupies exactly one MVE vector 111 // register, i.e. {uint,int,float}NxM_t. 112 Vector, 113 114 // MultiVector is used for the {uint,int,float}NxMxK_t types used by the 115 // interleaving load/store intrinsics v{ld,st}{2,4}q. 116 MultiVector, 117 118 // Predicate is used by all the predicated intrinsics. Its C 119 // representation is mve_pred16_t (which is just an alias for uint16_t). 120 // But we give more detail here, by indicating that a given predicate 121 // instruction is logically regarded as a vector of i1 containing the 122 // same number of lanes as the input vector type. So our Predicate type 123 // comes with a lane count, which we use to decide which kind of <n x i1> 124 // we'll invoke the pred_i2v IR intrinsic to translate it into. 125 Predicate, 126 127 // Pointer is used for pointer types (obviously), and comes with a flag 128 // indicating whether it's a pointer to a const or mutable instance of 129 // the pointee type. 130 Pointer, 131 }; 132 133 private: 134 const TypeKind TKind; 135 136 protected: 137 Type(TypeKind K) : TKind(K) {} 138 139 public: 140 TypeKind typeKind() const { return TKind; } 141 virtual ~Type() = default; 142 virtual bool requiresFloat() const = 0; 143 virtual bool requiresMVE() const = 0; 144 virtual unsigned sizeInBits() const = 0; 145 virtual std::string cName() const = 0; 146 virtual std::string llvmName() const { 147 PrintFatalError("no LLVM type name available for type " + cName()); 148 } 149 virtual std::string acleSuffix(std::string) const { 150 PrintFatalError("no ACLE suffix available for this type"); 151 } 152 }; 153 154 enum class ScalarTypeKind { SignedInt, UnsignedInt, Float }; 155 inline std::string toLetter(ScalarTypeKind kind) { 156 switch (kind) { 157 case ScalarTypeKind::SignedInt: 158 return "s"; 159 case ScalarTypeKind::UnsignedInt: 160 return "u"; 161 case ScalarTypeKind::Float: 162 return "f"; 163 } 164 llvm_unreachable("Unhandled ScalarTypeKind enum"); 165 } 166 inline std::string toCPrefix(ScalarTypeKind kind) { 167 switch (kind) { 168 case ScalarTypeKind::SignedInt: 169 return "int"; 170 case ScalarTypeKind::UnsignedInt: 171 return "uint"; 172 case ScalarTypeKind::Float: 173 return "float"; 174 } 175 llvm_unreachable("Unhandled ScalarTypeKind enum"); 176 } 177 178 class VoidType : public Type { 179 public: 180 VoidType() : Type(TypeKind::Void) {} 181 unsigned sizeInBits() const override { return 0; } 182 bool requiresFloat() const override { return false; } 183 bool requiresMVE() const override { return false; } 184 std::string cName() const override { return "void"; } 185 186 static bool classof(const Type *T) { return T->typeKind() == TypeKind::Void; } 187 std::string acleSuffix(std::string) const override { return ""; } 188 }; 189 190 class PointerType : public Type { 191 const Type *Pointee; 192 bool Const; 193 194 public: 195 PointerType(const Type *Pointee, bool Const) 196 : Type(TypeKind::Pointer), Pointee(Pointee), Const(Const) {} 197 unsigned sizeInBits() const override { return 32; } 198 bool requiresFloat() const override { return Pointee->requiresFloat(); } 199 bool requiresMVE() const override { return Pointee->requiresMVE(); } 200 std::string cName() const override { 201 std::string Name = Pointee->cName(); 202 203 // The syntax for a pointer in C is different when the pointee is 204 // itself a pointer. The MVE intrinsics don't contain any double 205 // pointers, so we don't need to worry about that wrinkle. 206 assert(!isa<PointerType>(Pointee) && "Pointer to pointer not supported"); 207 208 if (Const) 209 Name = "const " + Name; 210 return Name + " *"; 211 } 212 std::string llvmName() const override { 213 return "llvm::PointerType::getUnqual(" + Pointee->llvmName() + ")"; 214 } 215 216 static bool classof(const Type *T) { 217 return T->typeKind() == TypeKind::Pointer; 218 } 219 }; 220 221 // Base class for all the types that have a name of the form 222 // [prefix][numbers]_t, like int32_t, uint16x8_t, float32x4x2_t. 223 // 224 // For this sub-hierarchy we invent a cNameBase() method which returns the 225 // whole name except for the trailing "_t", so that Vector and MultiVector can 226 // append an extra "x2" or whatever to their element type's cNameBase(). Then 227 // the main cName() query method puts "_t" on the end for the final type name. 228 229 class CRegularNamedType : public Type { 230 using Type::Type; 231 virtual std::string cNameBase() const = 0; 232 233 public: 234 std::string cName() const override { return cNameBase() + "_t"; } 235 }; 236 237 class ScalarType : public CRegularNamedType { 238 ScalarTypeKind Kind; 239 unsigned Bits; 240 std::string NameOverride; 241 242 public: 243 ScalarType(const Record *Record) : CRegularNamedType(TypeKind::Scalar) { 244 Kind = StringSwitch<ScalarTypeKind>(Record->getValueAsString("kind")) 245 .Case("s", ScalarTypeKind::SignedInt) 246 .Case("u", ScalarTypeKind::UnsignedInt) 247 .Case("f", ScalarTypeKind::Float); 248 Bits = Record->getValueAsInt("size"); 249 NameOverride = std::string(Record->getValueAsString("nameOverride")); 250 } 251 unsigned sizeInBits() const override { return Bits; } 252 ScalarTypeKind kind() const { return Kind; } 253 std::string suffix() const { return toLetter(Kind) + utostr(Bits); } 254 std::string cNameBase() const override { 255 return toCPrefix(Kind) + utostr(Bits); 256 } 257 std::string cName() const override { 258 if (NameOverride.empty()) 259 return CRegularNamedType::cName(); 260 return NameOverride; 261 } 262 std::string llvmName() const override { 263 if (Kind == ScalarTypeKind::Float) { 264 if (Bits == 16) 265 return "HalfTy"; 266 if (Bits == 32) 267 return "FloatTy"; 268 if (Bits == 64) 269 return "DoubleTy"; 270 PrintFatalError("bad size for floating type"); 271 } 272 return "Int" + utostr(Bits) + "Ty"; 273 } 274 std::string acleSuffix(std::string overrideLetter) const override { 275 return "_" + (overrideLetter.size() ? overrideLetter : toLetter(Kind)) 276 + utostr(Bits); 277 } 278 bool isInteger() const { return Kind != ScalarTypeKind::Float; } 279 bool requiresFloat() const override { return !isInteger(); } 280 bool requiresMVE() const override { return false; } 281 bool hasNonstandardName() const { return !NameOverride.empty(); } 282 283 static bool classof(const Type *T) { 284 return T->typeKind() == TypeKind::Scalar; 285 } 286 }; 287 288 class VectorType : public CRegularNamedType { 289 const ScalarType *Element; 290 unsigned Lanes; 291 292 public: 293 VectorType(const ScalarType *Element, unsigned Lanes) 294 : CRegularNamedType(TypeKind::Vector), Element(Element), Lanes(Lanes) {} 295 unsigned sizeInBits() const override { return Lanes * Element->sizeInBits(); } 296 unsigned lanes() const { return Lanes; } 297 bool requiresFloat() const override { return Element->requiresFloat(); } 298 bool requiresMVE() const override { return true; } 299 std::string cNameBase() const override { 300 return Element->cNameBase() + "x" + utostr(Lanes); 301 } 302 std::string llvmName() const override { 303 return "llvm::FixedVectorType::get(" + Element->llvmName() + ", " + 304 utostr(Lanes) + ")"; 305 } 306 307 static bool classof(const Type *T) { 308 return T->typeKind() == TypeKind::Vector; 309 } 310 }; 311 312 class MultiVectorType : public CRegularNamedType { 313 const VectorType *Element; 314 unsigned Registers; 315 316 public: 317 MultiVectorType(unsigned Registers, const VectorType *Element) 318 : CRegularNamedType(TypeKind::MultiVector), Element(Element), 319 Registers(Registers) {} 320 unsigned sizeInBits() const override { 321 return Registers * Element->sizeInBits(); 322 } 323 unsigned registers() const { return Registers; } 324 bool requiresFloat() const override { return Element->requiresFloat(); } 325 bool requiresMVE() const override { return true; } 326 std::string cNameBase() const override { 327 return Element->cNameBase() + "x" + utostr(Registers); 328 } 329 330 // MultiVectorType doesn't override llvmName, because we don't expect to do 331 // automatic code generation for the MVE intrinsics that use it: the {vld2, 332 // vld4, vst2, vst4} family are the only ones that use these types, so it was 333 // easier to hand-write the codegen for dealing with these structs than to 334 // build in lots of extra automatic machinery that would only be used once. 335 336 static bool classof(const Type *T) { 337 return T->typeKind() == TypeKind::MultiVector; 338 } 339 }; 340 341 class PredicateType : public CRegularNamedType { 342 unsigned Lanes; 343 344 public: 345 PredicateType(unsigned Lanes) 346 : CRegularNamedType(TypeKind::Predicate), Lanes(Lanes) {} 347 unsigned sizeInBits() const override { return 16; } 348 std::string cNameBase() const override { return "mve_pred16"; } 349 bool requiresFloat() const override { return false; }; 350 bool requiresMVE() const override { return true; } 351 std::string llvmName() const override { 352 // Use <4 x i1> instead of <2 x i1> for two-lane vector types. See 353 // the comment in llvm/lib/Target/ARM/ARMInstrMVE.td for further 354 // explanation. 355 unsigned ModifiedLanes = (Lanes == 2 ? 4 : Lanes); 356 357 return "llvm::FixedVectorType::get(Builder.getInt1Ty(), " + 358 utostr(ModifiedLanes) + ")"; 359 } 360 361 static bool classof(const Type *T) { 362 return T->typeKind() == TypeKind::Predicate; 363 } 364 }; 365 366 // ----------------------------------------------------------------------------- 367 // Class to facilitate merging together the code generation for many intrinsics 368 // by means of varying a few constant or type parameters. 369 // 370 // Most obviously, the intrinsics in a single parametrised family will have 371 // code generation sequences that only differ in a type or two, e.g. vaddq_s8 372 // and vaddq_u16 will look the same apart from putting a different vector type 373 // in the call to CGM.getIntrinsic(). But also, completely different intrinsics 374 // will often code-generate in the same way, with only a different choice of 375 // _which_ IR intrinsic they lower to (e.g. vaddq_m_s8 and vmulq_m_s8), but 376 // marshalling the arguments and return values of the IR intrinsic in exactly 377 // the same way. And others might differ only in some other kind of constant, 378 // such as a lane index. 379 // 380 // So, when we generate the IR-building code for all these intrinsics, we keep 381 // track of every value that could possibly be pulled out of the code and 382 // stored ahead of time in a local variable. Then we group together intrinsics 383 // by textual equivalence of the code that would result if _all_ those 384 // parameters were stored in local variables. That gives us maximal sets that 385 // can be implemented by a single piece of IR-building code by changing 386 // parameter values ahead of time. 387 // 388 // After we've done that, we do a second pass in which we only allocate _some_ 389 // of the parameters into local variables, by tracking which ones have the same 390 // values as each other (so that a single variable can be reused) and which 391 // ones are the same across the whole set (so that no variable is needed at 392 // all). 393 // 394 // Hence the class below. Its allocParam method is invoked during code 395 // generation by every method of a Result subclass (see below) that wants to 396 // give it the opportunity to pull something out into a switchable parameter. 397 // It returns a variable name for the parameter, or (if it's being used in the 398 // second pass once we've decided that some parameters don't need to be stored 399 // in variables after all) it might just return the input expression unchanged. 400 401 struct CodeGenParamAllocator { 402 // Accumulated during code generation 403 std::vector<std::string> *ParamTypes = nullptr; 404 std::vector<std::string> *ParamValues = nullptr; 405 406 // Provided ahead of time in pass 2, to indicate which parameters are being 407 // assigned to what. This vector contains an entry for each call to 408 // allocParam expected during code gen (which we counted up in pass 1), and 409 // indicates the number of the parameter variable that should be returned, or 410 // -1 if this call shouldn't allocate a parameter variable at all. 411 // 412 // We rely on the recursive code generation working identically in passes 1 413 // and 2, so that the same list of calls to allocParam happen in the same 414 // order. That guarantees that the parameter numbers recorded in pass 1 will 415 // match the entries in this vector that store what EmitterBase::EmitBuiltinCG 416 // decided to do about each one in pass 2. 417 std::vector<int> *ParamNumberMap = nullptr; 418 419 // Internally track how many things we've allocated 420 unsigned nparams = 0; 421 422 std::string allocParam(StringRef Type, StringRef Value) { 423 unsigned ParamNumber; 424 425 if (!ParamNumberMap) { 426 // In pass 1, unconditionally assign a new parameter variable to every 427 // value we're asked to process. 428 ParamNumber = nparams++; 429 } else { 430 // In pass 2, consult the map provided by the caller to find out which 431 // variable we should be keeping things in. 432 int MapValue = (*ParamNumberMap)[nparams++]; 433 if (MapValue < 0) 434 return std::string(Value); 435 ParamNumber = MapValue; 436 } 437 438 // If we've allocated a new parameter variable for the first time, store 439 // its type and value to be retrieved after codegen. 440 if (ParamTypes && ParamTypes->size() == ParamNumber) 441 ParamTypes->push_back(std::string(Type)); 442 if (ParamValues && ParamValues->size() == ParamNumber) 443 ParamValues->push_back(std::string(Value)); 444 445 // Unimaginative naming scheme for parameter variables. 446 return "Param" + utostr(ParamNumber); 447 } 448 }; 449 450 // ----------------------------------------------------------------------------- 451 // System of classes that represent all the intermediate values used during 452 // code-generation for an intrinsic. 453 // 454 // The base class 'Result' can represent a value of the LLVM type 'Value', or 455 // sometimes 'Address' (for loads/stores, including an alignment requirement). 456 // 457 // In the case where the Tablegen provides a value in the codegen dag as a 458 // plain integer literal, the Result object we construct here will be one that 459 // returns true from hasIntegerConstantValue(). This allows the generated C++ 460 // code to use the constant directly in contexts which can take a literal 461 // integer, such as Builder.CreateExtractValue(thing, 1), without going to the 462 // effort of calling llvm::ConstantInt::get() and then pulling the constant 463 // back out of the resulting llvm:Value later. 464 465 class Result { 466 public: 467 // Convenient shorthand for the pointer type we'll be using everywhere. 468 using Ptr = std::shared_ptr<Result>; 469 470 private: 471 Ptr Predecessor; 472 std::string VarName; 473 bool VarNameUsed = false; 474 unsigned Visited = 0; 475 476 public: 477 virtual ~Result() = default; 478 using Scope = std::map<std::string, Ptr>; 479 virtual void genCode(raw_ostream &OS, CodeGenParamAllocator &) const = 0; 480 virtual bool hasIntegerConstantValue() const { return false; } 481 virtual uint32_t integerConstantValue() const { return 0; } 482 virtual bool hasIntegerValue() const { return false; } 483 virtual std::string getIntegerValue(const std::string &) { 484 llvm_unreachable("non-working Result::getIntegerValue called"); 485 } 486 virtual std::string typeName() const { return "Value *"; } 487 488 // Mostly, when a code-generation operation has a dependency on prior 489 // operations, it's because it uses the output values of those operations as 490 // inputs. But there's one exception, which is the use of 'seq' in Tablegen 491 // to indicate that operations have to be performed in sequence regardless of 492 // whether they use each others' output values. 493 // 494 // So, the actual generation of code is done by depth-first search, using the 495 // prerequisites() method to get a list of all the other Results that have to 496 // be computed before this one. That method divides into the 'predecessor', 497 // set by setPredecessor() while processing a 'seq' dag node, and the list 498 // returned by 'morePrerequisites', which each subclass implements to return 499 // a list of the Results it uses as input to whatever its own computation is 500 // doing. 501 502 virtual void morePrerequisites(std::vector<Ptr> &output) const {} 503 std::vector<Ptr> prerequisites() const { 504 std::vector<Ptr> ToRet; 505 if (Predecessor) 506 ToRet.push_back(Predecessor); 507 morePrerequisites(ToRet); 508 return ToRet; 509 } 510 511 void setPredecessor(Ptr p) { 512 // If the user has nested one 'seq' node inside another, and this 513 // method is called on the return value of the inner 'seq' (i.e. 514 // the final item inside it), then we can't link _this_ node to p, 515 // because it already has a predecessor. Instead, walk the chain 516 // until we find the first item in the inner seq, and link that to 517 // p, so that nesting seqs has the obvious effect of linking 518 // everything together into one long sequential chain. 519 Result *r = this; 520 while (r->Predecessor) 521 r = r->Predecessor.get(); 522 r->Predecessor = p; 523 } 524 525 // Each Result will be assigned a variable name in the output code, but not 526 // all those variable names will actually be used (e.g. the return value of 527 // Builder.CreateStore has void type, so nobody will want to refer to it). To 528 // prevent annoying compiler warnings, we track whether each Result's 529 // variable name was ever actually mentioned in subsequent statements, so 530 // that it can be left out of the final generated code. 531 std::string varname() { 532 VarNameUsed = true; 533 return VarName; 534 } 535 void setVarname(const StringRef s) { VarName = std::string(s); } 536 bool varnameUsed() const { return VarNameUsed; } 537 538 // Emit code to generate this result as a Value *. 539 virtual std::string asValue() { 540 return varname(); 541 } 542 543 // Code generation happens in multiple passes. This method tracks whether a 544 // Result has yet been visited in a given pass, without the need for a 545 // tedious loop in between passes that goes through and resets a 'visited' 546 // flag back to false: you just set Pass=1 the first time round, and Pass=2 547 // the second time. 548 bool needsVisiting(unsigned Pass) { 549 bool ToRet = Visited < Pass; 550 Visited = Pass; 551 return ToRet; 552 } 553 }; 554 555 // Result subclass that retrieves one of the arguments to the clang builtin 556 // function. In cases where the argument has pointer type, we call 557 // EmitPointerWithAlignment and store the result in a variable of type Address, 558 // so that load and store IR nodes can know the right alignment. Otherwise, we 559 // call EmitScalarExpr. 560 // 561 // There are aggregate parameters in the MVE intrinsics API, but we don't deal 562 // with them in this Tablegen back end: they only arise in the vld2q/vld4q and 563 // vst2q/vst4q family, which is few enough that we just write the code by hand 564 // for those in CGBuiltin.cpp. 565 class BuiltinArgResult : public Result { 566 public: 567 unsigned ArgNum; 568 bool AddressType; 569 bool Immediate; 570 BuiltinArgResult(unsigned ArgNum, bool AddressType, bool Immediate) 571 : ArgNum(ArgNum), AddressType(AddressType), Immediate(Immediate) {} 572 void genCode(raw_ostream &OS, CodeGenParamAllocator &) const override { 573 OS << (AddressType ? "EmitPointerWithAlignment" : "EmitScalarExpr") 574 << "(E->getArg(" << ArgNum << "))"; 575 } 576 std::string typeName() const override { 577 return AddressType ? "Address" : Result::typeName(); 578 } 579 // Emit code to generate this result as a Value *. 580 std::string asValue() override { 581 if (AddressType) 582 return "(" + varname() + ".getPointer())"; 583 return Result::asValue(); 584 } 585 bool hasIntegerValue() const override { return Immediate; } 586 std::string getIntegerValue(const std::string &IntType) override { 587 return "GetIntegerConstantValue<" + IntType + ">(E->getArg(" + 588 utostr(ArgNum) + "), getContext())"; 589 } 590 }; 591 592 // Result subclass for an integer literal appearing in Tablegen. This may need 593 // to be turned into an llvm::Result by means of llvm::ConstantInt::get(), or 594 // it may be used directly as an integer, depending on which IRBuilder method 595 // it's being passed to. 596 class IntLiteralResult : public Result { 597 public: 598 const ScalarType *IntegerType; 599 uint32_t IntegerValue; 600 IntLiteralResult(const ScalarType *IntegerType, uint32_t IntegerValue) 601 : IntegerType(IntegerType), IntegerValue(IntegerValue) {} 602 void genCode(raw_ostream &OS, 603 CodeGenParamAllocator &ParamAlloc) const override { 604 OS << "llvm::ConstantInt::get(" 605 << ParamAlloc.allocParam("llvm::Type *", IntegerType->llvmName()) 606 << ", "; 607 OS << ParamAlloc.allocParam(IntegerType->cName(), utostr(IntegerValue)) 608 << ")"; 609 } 610 bool hasIntegerConstantValue() const override { return true; } 611 uint32_t integerConstantValue() const override { return IntegerValue; } 612 }; 613 614 // Result subclass representing a cast between different integer types. We use 615 // our own ScalarType abstraction as the representation of the target type, 616 // which gives both size and signedness. 617 class IntCastResult : public Result { 618 public: 619 const ScalarType *IntegerType; 620 Ptr V; 621 IntCastResult(const ScalarType *IntegerType, Ptr V) 622 : IntegerType(IntegerType), V(V) {} 623 void genCode(raw_ostream &OS, 624 CodeGenParamAllocator &ParamAlloc) const override { 625 OS << "Builder.CreateIntCast(" << V->varname() << ", " 626 << ParamAlloc.allocParam("llvm::Type *", IntegerType->llvmName()) << ", " 627 << ParamAlloc.allocParam("bool", 628 IntegerType->kind() == ScalarTypeKind::SignedInt 629 ? "true" 630 : "false") 631 << ")"; 632 } 633 void morePrerequisites(std::vector<Ptr> &output) const override { 634 output.push_back(V); 635 } 636 }; 637 638 // Result subclass representing a cast between different pointer types. 639 class PointerCastResult : public Result { 640 public: 641 const PointerType *PtrType; 642 Ptr V; 643 PointerCastResult(const PointerType *PtrType, Ptr V) 644 : PtrType(PtrType), V(V) {} 645 void genCode(raw_ostream &OS, 646 CodeGenParamAllocator &ParamAlloc) const override { 647 OS << "Builder.CreatePointerCast(" << V->asValue() << ", " 648 << ParamAlloc.allocParam("llvm::Type *", PtrType->llvmName()) << ")"; 649 } 650 void morePrerequisites(std::vector<Ptr> &output) const override { 651 output.push_back(V); 652 } 653 }; 654 655 // Result subclass representing a call to an IRBuilder method. Each IRBuilder 656 // method we want to use will have a Tablegen record giving the method name and 657 // describing any important details of how to call it, such as whether a 658 // particular argument should be an integer constant instead of an llvm::Value. 659 class IRBuilderResult : public Result { 660 public: 661 StringRef CallPrefix; 662 std::vector<Ptr> Args; 663 std::set<unsigned> AddressArgs; 664 std::map<unsigned, std::string> IntegerArgs; 665 IRBuilderResult(StringRef CallPrefix, std::vector<Ptr> Args, 666 std::set<unsigned> AddressArgs, 667 std::map<unsigned, std::string> IntegerArgs) 668 : CallPrefix(CallPrefix), Args(Args), AddressArgs(AddressArgs), 669 IntegerArgs(IntegerArgs) {} 670 void genCode(raw_ostream &OS, 671 CodeGenParamAllocator &ParamAlloc) const override { 672 OS << CallPrefix; 673 const char *Sep = ""; 674 for (unsigned i = 0, e = Args.size(); i < e; ++i) { 675 Ptr Arg = Args[i]; 676 auto it = IntegerArgs.find(i); 677 678 OS << Sep; 679 Sep = ", "; 680 681 if (it != IntegerArgs.end()) { 682 if (Arg->hasIntegerConstantValue()) 683 OS << "static_cast<" << it->second << ">(" 684 << ParamAlloc.allocParam(it->second, 685 utostr(Arg->integerConstantValue())) 686 << ")"; 687 else if (Arg->hasIntegerValue()) 688 OS << ParamAlloc.allocParam(it->second, 689 Arg->getIntegerValue(it->second)); 690 } else { 691 OS << Arg->varname(); 692 } 693 } 694 OS << ")"; 695 } 696 void morePrerequisites(std::vector<Ptr> &output) const override { 697 for (unsigned i = 0, e = Args.size(); i < e; ++i) { 698 Ptr Arg = Args[i]; 699 if (IntegerArgs.find(i) != IntegerArgs.end()) 700 continue; 701 output.push_back(Arg); 702 } 703 } 704 }; 705 706 // Result subclass representing making an Address out of a Value. 707 class AddressResult : public Result { 708 public: 709 Ptr Arg; 710 unsigned Align; 711 AddressResult(Ptr Arg, unsigned Align) : Arg(Arg), Align(Align) {} 712 void genCode(raw_ostream &OS, 713 CodeGenParamAllocator &ParamAlloc) const override { 714 OS << "Address(" << Arg->varname() << ", CharUnits::fromQuantity(" 715 << Align << "))"; 716 } 717 std::string typeName() const override { 718 return "Address"; 719 } 720 void morePrerequisites(std::vector<Ptr> &output) const override { 721 output.push_back(Arg); 722 } 723 }; 724 725 // Result subclass representing a call to an IR intrinsic, which we first have 726 // to look up using an Intrinsic::ID constant and an array of types. 727 class IRIntrinsicResult : public Result { 728 public: 729 std::string IntrinsicID; 730 std::vector<const Type *> ParamTypes; 731 std::vector<Ptr> Args; 732 IRIntrinsicResult(StringRef IntrinsicID, std::vector<const Type *> ParamTypes, 733 std::vector<Ptr> Args) 734 : IntrinsicID(std::string(IntrinsicID)), ParamTypes(ParamTypes), 735 Args(Args) {} 736 void genCode(raw_ostream &OS, 737 CodeGenParamAllocator &ParamAlloc) const override { 738 std::string IntNo = ParamAlloc.allocParam( 739 "Intrinsic::ID", "Intrinsic::" + IntrinsicID); 740 OS << "Builder.CreateCall(CGM.getIntrinsic(" << IntNo; 741 if (!ParamTypes.empty()) { 742 OS << ", {"; 743 const char *Sep = ""; 744 for (auto T : ParamTypes) { 745 OS << Sep << ParamAlloc.allocParam("llvm::Type *", T->llvmName()); 746 Sep = ", "; 747 } 748 OS << "}"; 749 } 750 OS << "), {"; 751 const char *Sep = ""; 752 for (auto Arg : Args) { 753 OS << Sep << Arg->asValue(); 754 Sep = ", "; 755 } 756 OS << "})"; 757 } 758 void morePrerequisites(std::vector<Ptr> &output) const override { 759 output.insert(output.end(), Args.begin(), Args.end()); 760 } 761 }; 762 763 // Result subclass that specifies a type, for use in IRBuilder operations such 764 // as CreateBitCast that take a type argument. 765 class TypeResult : public Result { 766 public: 767 const Type *T; 768 TypeResult(const Type *T) : T(T) {} 769 void genCode(raw_ostream &OS, CodeGenParamAllocator &) const override { 770 OS << T->llvmName(); 771 } 772 std::string typeName() const override { 773 return "llvm::Type *"; 774 } 775 }; 776 777 // ----------------------------------------------------------------------------- 778 // Class that describes a single ACLE intrinsic. 779 // 780 // A Tablegen record will typically describe more than one ACLE intrinsic, by 781 // means of setting the 'list<Type> Params' field to a list of multiple 782 // parameter types, so as to define vaddq_{s8,u8,...,f16,f32} all in one go. 783 // We'll end up with one instance of ACLEIntrinsic for *each* parameter type, 784 // rather than a single one for all of them. Hence, the constructor takes both 785 // a Tablegen record and the current value of the parameter type. 786 787 class ACLEIntrinsic { 788 // Structure documenting that one of the intrinsic's arguments is required to 789 // be a compile-time constant integer, and what constraints there are on its 790 // value. Used when generating Sema checking code. 791 struct ImmediateArg { 792 enum class BoundsType { ExplicitRange, UInt }; 793 BoundsType boundsType; 794 int64_t i1, i2; 795 StringRef ExtraCheckType, ExtraCheckArgs; 796 const Type *ArgType; 797 }; 798 799 // For polymorphic intrinsics, FullName is the explicit name that uniquely 800 // identifies this variant of the intrinsic, and ShortName is the name it 801 // shares with at least one other intrinsic. 802 std::string ShortName, FullName; 803 804 // Name of the architecture extension, used in the Clang builtin name 805 StringRef BuiltinExtension; 806 807 // A very small number of intrinsics _only_ have a polymorphic 808 // variant (vuninitializedq taking an unevaluated argument). 809 bool PolymorphicOnly; 810 811 // Another rarely-used flag indicating that the builtin doesn't 812 // evaluate its argument(s) at all. 813 bool NonEvaluating; 814 815 // True if the intrinsic needs only the C header part (no codegen, semantic 816 // checks, etc). Used for redeclaring MVE intrinsics in the arm_cde.h header. 817 bool HeaderOnly; 818 819 const Type *ReturnType; 820 std::vector<const Type *> ArgTypes; 821 std::map<unsigned, ImmediateArg> ImmediateArgs; 822 Result::Ptr Code; 823 824 std::map<std::string, std::string> CustomCodeGenArgs; 825 826 // Recursive function that does the internals of code generation. 827 void genCodeDfs(Result::Ptr V, std::list<Result::Ptr> &Used, 828 unsigned Pass) const { 829 if (!V->needsVisiting(Pass)) 830 return; 831 832 for (Result::Ptr W : V->prerequisites()) 833 genCodeDfs(W, Used, Pass); 834 835 Used.push_back(V); 836 } 837 838 public: 839 const std::string &shortName() const { return ShortName; } 840 const std::string &fullName() const { return FullName; } 841 StringRef builtinExtension() const { return BuiltinExtension; } 842 const Type *returnType() const { return ReturnType; } 843 const std::vector<const Type *> &argTypes() const { return ArgTypes; } 844 bool requiresFloat() const { 845 if (ReturnType->requiresFloat()) 846 return true; 847 for (const Type *T : ArgTypes) 848 if (T->requiresFloat()) 849 return true; 850 return false; 851 } 852 bool requiresMVE() const { 853 return ReturnType->requiresMVE() || 854 any_of(ArgTypes, [](const Type *T) { return T->requiresMVE(); }); 855 } 856 bool polymorphic() const { return ShortName != FullName; } 857 bool polymorphicOnly() const { return PolymorphicOnly; } 858 bool nonEvaluating() const { return NonEvaluating; } 859 bool headerOnly() const { return HeaderOnly; } 860 861 // External entry point for code generation, called from EmitterBase. 862 void genCode(raw_ostream &OS, CodeGenParamAllocator &ParamAlloc, 863 unsigned Pass) const { 864 assert(!headerOnly() && "Called genCode for header-only intrinsic"); 865 if (!hasCode()) { 866 for (auto kv : CustomCodeGenArgs) 867 OS << " " << kv.first << " = " << kv.second << ";\n"; 868 OS << " break; // custom code gen\n"; 869 return; 870 } 871 std::list<Result::Ptr> Used; 872 genCodeDfs(Code, Used, Pass); 873 874 unsigned varindex = 0; 875 for (Result::Ptr V : Used) 876 if (V->varnameUsed()) 877 V->setVarname("Val" + utostr(varindex++)); 878 879 for (Result::Ptr V : Used) { 880 OS << " "; 881 if (V == Used.back()) { 882 assert(!V->varnameUsed()); 883 OS << "return "; // FIXME: what if the top-level thing is void? 884 } else if (V->varnameUsed()) { 885 std::string Type = V->typeName(); 886 OS << V->typeName(); 887 if (!StringRef(Type).endswith("*")) 888 OS << " "; 889 OS << V->varname() << " = "; 890 } 891 V->genCode(OS, ParamAlloc); 892 OS << ";\n"; 893 } 894 } 895 bool hasCode() const { return Code != nullptr; } 896 897 static std::string signedHexLiteral(const llvm::APInt &iOrig) { 898 llvm::APInt i = iOrig.trunc(64); 899 SmallString<40> s; 900 i.toString(s, 16, true, true); 901 return std::string(s.str()); 902 } 903 904 std::string genSema() const { 905 assert(!headerOnly() && "Called genSema for header-only intrinsic"); 906 std::vector<std::string> SemaChecks; 907 908 for (const auto &kv : ImmediateArgs) { 909 const ImmediateArg &IA = kv.second; 910 911 llvm::APInt lo(128, 0), hi(128, 0); 912 switch (IA.boundsType) { 913 case ImmediateArg::BoundsType::ExplicitRange: 914 lo = IA.i1; 915 hi = IA.i2; 916 break; 917 case ImmediateArg::BoundsType::UInt: 918 lo = 0; 919 hi = llvm::APInt::getMaxValue(IA.i1).zext(128); 920 break; 921 } 922 923 std::string Index = utostr(kv.first); 924 925 // Emit a range check if the legal range of values for the 926 // immediate is smaller than the _possible_ range of values for 927 // its type. 928 unsigned ArgTypeBits = IA.ArgType->sizeInBits(); 929 llvm::APInt ArgTypeRange = llvm::APInt::getMaxValue(ArgTypeBits).zext(128); 930 llvm::APInt ActualRange = (hi-lo).trunc(64).sext(128); 931 if (ActualRange.ult(ArgTypeRange)) 932 SemaChecks.push_back("SemaBuiltinConstantArgRange(TheCall, " + Index + 933 ", " + signedHexLiteral(lo) + ", " + 934 signedHexLiteral(hi) + ")"); 935 936 if (!IA.ExtraCheckType.empty()) { 937 std::string Suffix; 938 if (!IA.ExtraCheckArgs.empty()) { 939 std::string tmp; 940 StringRef Arg = IA.ExtraCheckArgs; 941 if (Arg == "!lanesize") { 942 tmp = utostr(IA.ArgType->sizeInBits()); 943 Arg = tmp; 944 } 945 Suffix = (Twine(", ") + Arg).str(); 946 } 947 SemaChecks.push_back((Twine("SemaBuiltinConstantArg") + 948 IA.ExtraCheckType + "(TheCall, " + Index + 949 Suffix + ")") 950 .str()); 951 } 952 953 assert(!SemaChecks.empty()); 954 } 955 if (SemaChecks.empty()) 956 return ""; 957 return join(std::begin(SemaChecks), std::end(SemaChecks), 958 " ||\n ") + 959 ";\n"; 960 } 961 962 ACLEIntrinsic(EmitterBase &ME, Record *R, const Type *Param); 963 }; 964 965 // ----------------------------------------------------------------------------- 966 // The top-level class that holds all the state from analyzing the entire 967 // Tablegen input. 968 969 class EmitterBase { 970 protected: 971 // EmitterBase holds a collection of all the types we've instantiated. 972 VoidType Void; 973 std::map<std::string, std::unique_ptr<ScalarType>> ScalarTypes; 974 std::map<std::tuple<ScalarTypeKind, unsigned, unsigned>, 975 std::unique_ptr<VectorType>> 976 VectorTypes; 977 std::map<std::pair<std::string, unsigned>, std::unique_ptr<MultiVectorType>> 978 MultiVectorTypes; 979 std::map<unsigned, std::unique_ptr<PredicateType>> PredicateTypes; 980 std::map<std::string, std::unique_ptr<PointerType>> PointerTypes; 981 982 // And all the ACLEIntrinsic instances we've created. 983 std::map<std::string, std::unique_ptr<ACLEIntrinsic>> ACLEIntrinsics; 984 985 public: 986 // Methods to create a Type object, or return the right existing one from the 987 // maps stored in this object. 988 const VoidType *getVoidType() { return &Void; } 989 const ScalarType *getScalarType(StringRef Name) { 990 return ScalarTypes[std::string(Name)].get(); 991 } 992 const ScalarType *getScalarType(Record *R) { 993 return getScalarType(R->getName()); 994 } 995 const VectorType *getVectorType(const ScalarType *ST, unsigned Lanes) { 996 std::tuple<ScalarTypeKind, unsigned, unsigned> key(ST->kind(), 997 ST->sizeInBits(), Lanes); 998 if (VectorTypes.find(key) == VectorTypes.end()) 999 VectorTypes[key] = std::make_unique<VectorType>(ST, Lanes); 1000 return VectorTypes[key].get(); 1001 } 1002 const VectorType *getVectorType(const ScalarType *ST) { 1003 return getVectorType(ST, 128 / ST->sizeInBits()); 1004 } 1005 const MultiVectorType *getMultiVectorType(unsigned Registers, 1006 const VectorType *VT) { 1007 std::pair<std::string, unsigned> key(VT->cNameBase(), Registers); 1008 if (MultiVectorTypes.find(key) == MultiVectorTypes.end()) 1009 MultiVectorTypes[key] = std::make_unique<MultiVectorType>(Registers, VT); 1010 return MultiVectorTypes[key].get(); 1011 } 1012 const PredicateType *getPredicateType(unsigned Lanes) { 1013 unsigned key = Lanes; 1014 if (PredicateTypes.find(key) == PredicateTypes.end()) 1015 PredicateTypes[key] = std::make_unique<PredicateType>(Lanes); 1016 return PredicateTypes[key].get(); 1017 } 1018 const PointerType *getPointerType(const Type *T, bool Const) { 1019 PointerType PT(T, Const); 1020 std::string key = PT.cName(); 1021 if (PointerTypes.find(key) == PointerTypes.end()) 1022 PointerTypes[key] = std::make_unique<PointerType>(PT); 1023 return PointerTypes[key].get(); 1024 } 1025 1026 // Methods to construct a type from various pieces of Tablegen. These are 1027 // always called in the context of setting up a particular ACLEIntrinsic, so 1028 // there's always an ambient parameter type (because we're iterating through 1029 // the Params list in the Tablegen record for the intrinsic), which is used 1030 // to expand Tablegen classes like 'Vector' which mean something different in 1031 // each member of a parametric family. 1032 const Type *getType(Record *R, const Type *Param); 1033 const Type *getType(DagInit *D, const Type *Param); 1034 const Type *getType(Init *I, const Type *Param); 1035 1036 // Functions that translate the Tablegen representation of an intrinsic's 1037 // code generation into a collection of Value objects (which will then be 1038 // reprocessed to read out the actual C++ code included by CGBuiltin.cpp). 1039 Result::Ptr getCodeForDag(DagInit *D, const Result::Scope &Scope, 1040 const Type *Param); 1041 Result::Ptr getCodeForDagArg(DagInit *D, unsigned ArgNum, 1042 const Result::Scope &Scope, const Type *Param); 1043 Result::Ptr getCodeForArg(unsigned ArgNum, const Type *ArgType, bool Promote, 1044 bool Immediate); 1045 1046 void GroupSemaChecks(std::map<std::string, std::set<std::string>> &Checks); 1047 1048 // Constructor and top-level functions. 1049 1050 EmitterBase(RecordKeeper &Records); 1051 virtual ~EmitterBase() = default; 1052 1053 virtual void EmitHeader(raw_ostream &OS) = 0; 1054 virtual void EmitBuiltinDef(raw_ostream &OS) = 0; 1055 virtual void EmitBuiltinSema(raw_ostream &OS) = 0; 1056 void EmitBuiltinCG(raw_ostream &OS); 1057 void EmitBuiltinAliases(raw_ostream &OS); 1058 }; 1059 1060 const Type *EmitterBase::getType(Init *I, const Type *Param) { 1061 if (auto Dag = dyn_cast<DagInit>(I)) 1062 return getType(Dag, Param); 1063 if (auto Def = dyn_cast<DefInit>(I)) 1064 return getType(Def->getDef(), Param); 1065 1066 PrintFatalError("Could not convert this value into a type"); 1067 } 1068 1069 const Type *EmitterBase::getType(Record *R, const Type *Param) { 1070 // Pass to a subfield of any wrapper records. We don't expect more than one 1071 // of these: immediate operands are used as plain numbers rather than as 1072 // llvm::Value, so it's meaningless to promote their type anyway. 1073 if (R->isSubClassOf("Immediate")) 1074 R = R->getValueAsDef("type"); 1075 else if (R->isSubClassOf("unpromoted")) 1076 R = R->getValueAsDef("underlying_type"); 1077 1078 if (R->getName() == "Void") 1079 return getVoidType(); 1080 if (R->isSubClassOf("PrimitiveType")) 1081 return getScalarType(R); 1082 if (R->isSubClassOf("ComplexType")) 1083 return getType(R->getValueAsDag("spec"), Param); 1084 1085 PrintFatalError(R->getLoc(), "Could not convert this record into a type"); 1086 } 1087 1088 const Type *EmitterBase::getType(DagInit *D, const Type *Param) { 1089 // The meat of the getType system: types in the Tablegen are represented by a 1090 // dag whose operators select sub-cases of this function. 1091 1092 Record *Op = cast<DefInit>(D->getOperator())->getDef(); 1093 if (!Op->isSubClassOf("ComplexTypeOp")) 1094 PrintFatalError( 1095 "Expected ComplexTypeOp as dag operator in type expression"); 1096 1097 if (Op->getName() == "CTO_Parameter") { 1098 if (isa<VoidType>(Param)) 1099 PrintFatalError("Parametric type in unparametrised context"); 1100 return Param; 1101 } 1102 1103 if (Op->getName() == "CTO_Vec") { 1104 const Type *Element = getType(D->getArg(0), Param); 1105 if (D->getNumArgs() == 1) { 1106 return getVectorType(cast<ScalarType>(Element)); 1107 } else { 1108 const Type *ExistingVector = getType(D->getArg(1), Param); 1109 return getVectorType(cast<ScalarType>(Element), 1110 cast<VectorType>(ExistingVector)->lanes()); 1111 } 1112 } 1113 1114 if (Op->getName() == "CTO_Pred") { 1115 const Type *Element = getType(D->getArg(0), Param); 1116 return getPredicateType(128 / Element->sizeInBits()); 1117 } 1118 1119 if (Op->isSubClassOf("CTO_Tuple")) { 1120 unsigned Registers = Op->getValueAsInt("n"); 1121 const Type *Element = getType(D->getArg(0), Param); 1122 return getMultiVectorType(Registers, cast<VectorType>(Element)); 1123 } 1124 1125 if (Op->isSubClassOf("CTO_Pointer")) { 1126 const Type *Pointee = getType(D->getArg(0), Param); 1127 return getPointerType(Pointee, Op->getValueAsBit("const")); 1128 } 1129 1130 if (Op->getName() == "CTO_CopyKind") { 1131 const ScalarType *STSize = cast<ScalarType>(getType(D->getArg(0), Param)); 1132 const ScalarType *STKind = cast<ScalarType>(getType(D->getArg(1), Param)); 1133 for (const auto &kv : ScalarTypes) { 1134 const ScalarType *RT = kv.second.get(); 1135 if (RT->kind() == STKind->kind() && RT->sizeInBits() == STSize->sizeInBits()) 1136 return RT; 1137 } 1138 PrintFatalError("Cannot find a type to satisfy CopyKind"); 1139 } 1140 1141 if (Op->isSubClassOf("CTO_ScaleSize")) { 1142 const ScalarType *STKind = cast<ScalarType>(getType(D->getArg(0), Param)); 1143 int Num = Op->getValueAsInt("num"), Denom = Op->getValueAsInt("denom"); 1144 unsigned DesiredSize = STKind->sizeInBits() * Num / Denom; 1145 for (const auto &kv : ScalarTypes) { 1146 const ScalarType *RT = kv.second.get(); 1147 if (RT->kind() == STKind->kind() && RT->sizeInBits() == DesiredSize) 1148 return RT; 1149 } 1150 PrintFatalError("Cannot find a type to satisfy ScaleSize"); 1151 } 1152 1153 PrintFatalError("Bad operator in type dag expression"); 1154 } 1155 1156 Result::Ptr EmitterBase::getCodeForDag(DagInit *D, const Result::Scope &Scope, 1157 const Type *Param) { 1158 Record *Op = cast<DefInit>(D->getOperator())->getDef(); 1159 1160 if (Op->getName() == "seq") { 1161 Result::Scope SubScope = Scope; 1162 Result::Ptr PrevV = nullptr; 1163 for (unsigned i = 0, e = D->getNumArgs(); i < e; ++i) { 1164 // We don't use getCodeForDagArg here, because the argument name 1165 // has different semantics in a seq 1166 Result::Ptr V = 1167 getCodeForDag(cast<DagInit>(D->getArg(i)), SubScope, Param); 1168 StringRef ArgName = D->getArgNameStr(i); 1169 if (!ArgName.empty()) 1170 SubScope[std::string(ArgName)] = V; 1171 if (PrevV) 1172 V->setPredecessor(PrevV); 1173 PrevV = V; 1174 } 1175 return PrevV; 1176 } else if (Op->isSubClassOf("Type")) { 1177 if (D->getNumArgs() != 1) 1178 PrintFatalError("Type casts should have exactly one argument"); 1179 const Type *CastType = getType(Op, Param); 1180 Result::Ptr Arg = getCodeForDagArg(D, 0, Scope, Param); 1181 if (const auto *ST = dyn_cast<ScalarType>(CastType)) { 1182 if (!ST->requiresFloat()) { 1183 if (Arg->hasIntegerConstantValue()) 1184 return std::make_shared<IntLiteralResult>( 1185 ST, Arg->integerConstantValue()); 1186 else 1187 return std::make_shared<IntCastResult>(ST, Arg); 1188 } 1189 } else if (const auto *PT = dyn_cast<PointerType>(CastType)) { 1190 return std::make_shared<PointerCastResult>(PT, Arg); 1191 } 1192 PrintFatalError("Unsupported type cast"); 1193 } else if (Op->getName() == "address") { 1194 if (D->getNumArgs() != 2) 1195 PrintFatalError("'address' should have two arguments"); 1196 Result::Ptr Arg = getCodeForDagArg(D, 0, Scope, Param); 1197 unsigned Alignment; 1198 if (auto *II = dyn_cast<IntInit>(D->getArg(1))) { 1199 Alignment = II->getValue(); 1200 } else { 1201 PrintFatalError("'address' alignment argument should be an integer"); 1202 } 1203 return std::make_shared<AddressResult>(Arg, Alignment); 1204 } else if (Op->getName() == "unsignedflag") { 1205 if (D->getNumArgs() != 1) 1206 PrintFatalError("unsignedflag should have exactly one argument"); 1207 Record *TypeRec = cast<DefInit>(D->getArg(0))->getDef(); 1208 if (!TypeRec->isSubClassOf("Type")) 1209 PrintFatalError("unsignedflag's argument should be a type"); 1210 if (const auto *ST = dyn_cast<ScalarType>(getType(TypeRec, Param))) { 1211 return std::make_shared<IntLiteralResult>( 1212 getScalarType("u32"), ST->kind() == ScalarTypeKind::UnsignedInt); 1213 } else { 1214 PrintFatalError("unsignedflag's argument should be a scalar type"); 1215 } 1216 } else if (Op->getName() == "bitsize") { 1217 if (D->getNumArgs() != 1) 1218 PrintFatalError("bitsize should have exactly one argument"); 1219 Record *TypeRec = cast<DefInit>(D->getArg(0))->getDef(); 1220 if (!TypeRec->isSubClassOf("Type")) 1221 PrintFatalError("bitsize's argument should be a type"); 1222 if (const auto *ST = dyn_cast<ScalarType>(getType(TypeRec, Param))) { 1223 return std::make_shared<IntLiteralResult>(getScalarType("u32"), 1224 ST->sizeInBits()); 1225 } else { 1226 PrintFatalError("bitsize's argument should be a scalar type"); 1227 } 1228 } else { 1229 std::vector<Result::Ptr> Args; 1230 for (unsigned i = 0, e = D->getNumArgs(); i < e; ++i) 1231 Args.push_back(getCodeForDagArg(D, i, Scope, Param)); 1232 if (Op->isSubClassOf("IRBuilderBase")) { 1233 std::set<unsigned> AddressArgs; 1234 std::map<unsigned, std::string> IntegerArgs; 1235 for (Record *sp : Op->getValueAsListOfDefs("special_params")) { 1236 unsigned Index = sp->getValueAsInt("index"); 1237 if (sp->isSubClassOf("IRBuilderAddrParam")) { 1238 AddressArgs.insert(Index); 1239 } else if (sp->isSubClassOf("IRBuilderIntParam")) { 1240 IntegerArgs[Index] = std::string(sp->getValueAsString("type")); 1241 } 1242 } 1243 return std::make_shared<IRBuilderResult>(Op->getValueAsString("prefix"), 1244 Args, AddressArgs, IntegerArgs); 1245 } else if (Op->isSubClassOf("IRIntBase")) { 1246 std::vector<const Type *> ParamTypes; 1247 for (Record *RParam : Op->getValueAsListOfDefs("params")) 1248 ParamTypes.push_back(getType(RParam, Param)); 1249 std::string IntName = std::string(Op->getValueAsString("intname")); 1250 if (Op->getValueAsBit("appendKind")) 1251 IntName += "_" + toLetter(cast<ScalarType>(Param)->kind()); 1252 return std::make_shared<IRIntrinsicResult>(IntName, ParamTypes, Args); 1253 } else { 1254 PrintFatalError("Unsupported dag node " + Op->getName()); 1255 } 1256 } 1257 } 1258 1259 Result::Ptr EmitterBase::getCodeForDagArg(DagInit *D, unsigned ArgNum, 1260 const Result::Scope &Scope, 1261 const Type *Param) { 1262 Init *Arg = D->getArg(ArgNum); 1263 StringRef Name = D->getArgNameStr(ArgNum); 1264 1265 if (!Name.empty()) { 1266 if (!isa<UnsetInit>(Arg)) 1267 PrintFatalError( 1268 "dag operator argument should not have both a value and a name"); 1269 auto it = Scope.find(std::string(Name)); 1270 if (it == Scope.end()) 1271 PrintFatalError("unrecognized variable name '" + Name + "'"); 1272 return it->second; 1273 } 1274 1275 if (auto *II = dyn_cast<IntInit>(Arg)) 1276 return std::make_shared<IntLiteralResult>(getScalarType("u32"), 1277 II->getValue()); 1278 1279 if (auto *DI = dyn_cast<DagInit>(Arg)) 1280 return getCodeForDag(DI, Scope, Param); 1281 1282 if (auto *DI = dyn_cast<DefInit>(Arg)) { 1283 Record *Rec = DI->getDef(); 1284 if (Rec->isSubClassOf("Type")) { 1285 const Type *T = getType(Rec, Param); 1286 return std::make_shared<TypeResult>(T); 1287 } 1288 } 1289 1290 PrintFatalError("bad dag argument type for code generation"); 1291 } 1292 1293 Result::Ptr EmitterBase::getCodeForArg(unsigned ArgNum, const Type *ArgType, 1294 bool Promote, bool Immediate) { 1295 Result::Ptr V = std::make_shared<BuiltinArgResult>( 1296 ArgNum, isa<PointerType>(ArgType), Immediate); 1297 1298 if (Promote) { 1299 if (const auto *ST = dyn_cast<ScalarType>(ArgType)) { 1300 if (ST->isInteger() && ST->sizeInBits() < 32) 1301 V = std::make_shared<IntCastResult>(getScalarType("u32"), V); 1302 } else if (const auto *PT = dyn_cast<PredicateType>(ArgType)) { 1303 V = std::make_shared<IntCastResult>(getScalarType("u32"), V); 1304 V = std::make_shared<IRIntrinsicResult>("arm_mve_pred_i2v", 1305 std::vector<const Type *>{PT}, 1306 std::vector<Result::Ptr>{V}); 1307 } 1308 } 1309 1310 return V; 1311 } 1312 1313 ACLEIntrinsic::ACLEIntrinsic(EmitterBase &ME, Record *R, const Type *Param) 1314 : ReturnType(ME.getType(R->getValueAsDef("ret"), Param)) { 1315 // Derive the intrinsic's full name, by taking the name of the 1316 // Tablegen record (or override) and appending the suffix from its 1317 // parameter type. (If the intrinsic is unparametrised, its 1318 // parameter type will be given as Void, which returns the empty 1319 // string for acleSuffix.) 1320 StringRef BaseName = 1321 (R->isSubClassOf("NameOverride") ? R->getValueAsString("basename") 1322 : R->getName()); 1323 StringRef overrideLetter = R->getValueAsString("overrideKindLetter"); 1324 FullName = 1325 (Twine(BaseName) + Param->acleSuffix(std::string(overrideLetter))).str(); 1326 1327 // Derive the intrinsic's polymorphic name, by removing components from the 1328 // full name as specified by its 'pnt' member ('polymorphic name type'), 1329 // which indicates how many type suffixes to remove, and any other piece of 1330 // the name that should be removed. 1331 Record *PolymorphicNameType = R->getValueAsDef("pnt"); 1332 SmallVector<StringRef, 8> NameParts; 1333 StringRef(FullName).split(NameParts, '_'); 1334 for (unsigned i = 0, e = PolymorphicNameType->getValueAsInt( 1335 "NumTypeSuffixesToDiscard"); 1336 i < e; ++i) 1337 NameParts.pop_back(); 1338 if (!PolymorphicNameType->isValueUnset("ExtraSuffixToDiscard")) { 1339 StringRef ExtraSuffix = 1340 PolymorphicNameType->getValueAsString("ExtraSuffixToDiscard"); 1341 auto it = NameParts.end(); 1342 while (it != NameParts.begin()) { 1343 --it; 1344 if (*it == ExtraSuffix) { 1345 NameParts.erase(it); 1346 break; 1347 } 1348 } 1349 } 1350 ShortName = join(std::begin(NameParts), std::end(NameParts), "_"); 1351 1352 BuiltinExtension = R->getValueAsString("builtinExtension"); 1353 1354 PolymorphicOnly = R->getValueAsBit("polymorphicOnly"); 1355 NonEvaluating = R->getValueAsBit("nonEvaluating"); 1356 HeaderOnly = R->getValueAsBit("headerOnly"); 1357 1358 // Process the intrinsic's argument list. 1359 DagInit *ArgsDag = R->getValueAsDag("args"); 1360 Result::Scope Scope; 1361 for (unsigned i = 0, e = ArgsDag->getNumArgs(); i < e; ++i) { 1362 Init *TypeInit = ArgsDag->getArg(i); 1363 1364 bool Promote = true; 1365 if (auto TypeDI = dyn_cast<DefInit>(TypeInit)) 1366 if (TypeDI->getDef()->isSubClassOf("unpromoted")) 1367 Promote = false; 1368 1369 // Work out the type of the argument, for use in the function prototype in 1370 // the header file. 1371 const Type *ArgType = ME.getType(TypeInit, Param); 1372 ArgTypes.push_back(ArgType); 1373 1374 // If the argument is a subclass of Immediate, record the details about 1375 // what values it can take, for Sema checking. 1376 bool Immediate = false; 1377 if (auto TypeDI = dyn_cast<DefInit>(TypeInit)) { 1378 Record *TypeRec = TypeDI->getDef(); 1379 if (TypeRec->isSubClassOf("Immediate")) { 1380 Immediate = true; 1381 1382 Record *Bounds = TypeRec->getValueAsDef("bounds"); 1383 ImmediateArg &IA = ImmediateArgs[i]; 1384 if (Bounds->isSubClassOf("IB_ConstRange")) { 1385 IA.boundsType = ImmediateArg::BoundsType::ExplicitRange; 1386 IA.i1 = Bounds->getValueAsInt("lo"); 1387 IA.i2 = Bounds->getValueAsInt("hi"); 1388 } else if (Bounds->getName() == "IB_UEltValue") { 1389 IA.boundsType = ImmediateArg::BoundsType::UInt; 1390 IA.i1 = Param->sizeInBits(); 1391 } else if (Bounds->getName() == "IB_LaneIndex") { 1392 IA.boundsType = ImmediateArg::BoundsType::ExplicitRange; 1393 IA.i1 = 0; 1394 IA.i2 = 128 / Param->sizeInBits() - 1; 1395 } else if (Bounds->isSubClassOf("IB_EltBit")) { 1396 IA.boundsType = ImmediateArg::BoundsType::ExplicitRange; 1397 IA.i1 = Bounds->getValueAsInt("base"); 1398 const Type *T = ME.getType(Bounds->getValueAsDef("type"), Param); 1399 IA.i2 = IA.i1 + T->sizeInBits() - 1; 1400 } else { 1401 PrintFatalError("unrecognised ImmediateBounds subclass"); 1402 } 1403 1404 IA.ArgType = ArgType; 1405 1406 if (!TypeRec->isValueUnset("extra")) { 1407 IA.ExtraCheckType = TypeRec->getValueAsString("extra"); 1408 if (!TypeRec->isValueUnset("extraarg")) 1409 IA.ExtraCheckArgs = TypeRec->getValueAsString("extraarg"); 1410 } 1411 } 1412 } 1413 1414 // The argument will usually have a name in the arguments dag, which goes 1415 // into the variable-name scope that the code gen will refer to. 1416 StringRef ArgName = ArgsDag->getArgNameStr(i); 1417 if (!ArgName.empty()) 1418 Scope[std::string(ArgName)] = 1419 ME.getCodeForArg(i, ArgType, Promote, Immediate); 1420 } 1421 1422 // Finally, go through the codegen dag and translate it into a Result object 1423 // (with an arbitrary DAG of depended-on Results hanging off it). 1424 DagInit *CodeDag = R->getValueAsDag("codegen"); 1425 Record *MainOp = cast<DefInit>(CodeDag->getOperator())->getDef(); 1426 if (MainOp->isSubClassOf("CustomCodegen")) { 1427 // Or, if it's the special case of CustomCodegen, just accumulate 1428 // a list of parameters we're going to assign to variables before 1429 // breaking from the loop. 1430 CustomCodeGenArgs["CustomCodeGenType"] = 1431 (Twine("CustomCodeGen::") + MainOp->getValueAsString("type")).str(); 1432 for (unsigned i = 0, e = CodeDag->getNumArgs(); i < e; ++i) { 1433 StringRef Name = CodeDag->getArgNameStr(i); 1434 if (Name.empty()) { 1435 PrintFatalError("Operands to CustomCodegen should have names"); 1436 } else if (auto *II = dyn_cast<IntInit>(CodeDag->getArg(i))) { 1437 CustomCodeGenArgs[std::string(Name)] = itostr(II->getValue()); 1438 } else if (auto *SI = dyn_cast<StringInit>(CodeDag->getArg(i))) { 1439 CustomCodeGenArgs[std::string(Name)] = std::string(SI->getValue()); 1440 } else { 1441 PrintFatalError("Operands to CustomCodegen should be integers"); 1442 } 1443 } 1444 } else { 1445 Code = ME.getCodeForDag(CodeDag, Scope, Param); 1446 } 1447 } 1448 1449 EmitterBase::EmitterBase(RecordKeeper &Records) { 1450 // Construct the whole EmitterBase. 1451 1452 // First, look up all the instances of PrimitiveType. This gives us the list 1453 // of vector typedefs we have to put in arm_mve.h, and also allows us to 1454 // collect all the useful ScalarType instances into a big list so that we can 1455 // use it for operations such as 'find the unsigned version of this signed 1456 // integer type'. 1457 for (Record *R : Records.getAllDerivedDefinitions("PrimitiveType")) 1458 ScalarTypes[std::string(R->getName())] = std::make_unique<ScalarType>(R); 1459 1460 // Now go through the instances of Intrinsic, and for each one, iterate 1461 // through its list of type parameters making an ACLEIntrinsic for each one. 1462 for (Record *R : Records.getAllDerivedDefinitions("Intrinsic")) { 1463 for (Record *RParam : R->getValueAsListOfDefs("params")) { 1464 const Type *Param = getType(RParam, getVoidType()); 1465 auto Intrinsic = std::make_unique<ACLEIntrinsic>(*this, R, Param); 1466 ACLEIntrinsics[Intrinsic->fullName()] = std::move(Intrinsic); 1467 } 1468 } 1469 } 1470 1471 /// A wrapper on raw_string_ostream that contains its own buffer rather than 1472 /// having to point it at one elsewhere. (In other words, it works just like 1473 /// std::ostringstream; also, this makes it convenient to declare a whole array 1474 /// of them at once.) 1475 /// 1476 /// We have to set this up using multiple inheritance, to ensure that the 1477 /// string member has been constructed before raw_string_ostream's constructor 1478 /// is given a pointer to it. 1479 class string_holder { 1480 protected: 1481 std::string S; 1482 }; 1483 class raw_self_contained_string_ostream : private string_holder, 1484 public raw_string_ostream { 1485 public: 1486 raw_self_contained_string_ostream() 1487 : string_holder(), raw_string_ostream(S) {} 1488 }; 1489 1490 const char LLVMLicenseHeader[] = 1491 " *\n" 1492 " *\n" 1493 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM" 1494 " Exceptions.\n" 1495 " * See https://llvm.org/LICENSE.txt for license information.\n" 1496 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n" 1497 " *\n" 1498 " *===-----------------------------------------------------------------" 1499 "------===\n" 1500 " */\n" 1501 "\n"; 1502 1503 // Machinery for the grouping of intrinsics by similar codegen. 1504 // 1505 // The general setup is that 'MergeableGroup' stores the things that a set of 1506 // similarly shaped intrinsics have in common: the text of their code 1507 // generation, and the number and type of their parameter variables. 1508 // MergeableGroup is the key in a std::map whose value is a set of 1509 // OutputIntrinsic, which stores the ways in which a particular intrinsic 1510 // specializes the MergeableGroup's generic description: the function name and 1511 // the _values_ of the parameter variables. 1512 1513 struct ComparableStringVector : std::vector<std::string> { 1514 // Infrastructure: a derived class of vector<string> which comes with an 1515 // ordering, so that it can be used as a key in maps and an element in sets. 1516 // There's no requirement on the ordering beyond being deterministic. 1517 bool operator<(const ComparableStringVector &rhs) const { 1518 if (size() != rhs.size()) 1519 return size() < rhs.size(); 1520 for (size_t i = 0, e = size(); i < e; ++i) 1521 if ((*this)[i] != rhs[i]) 1522 return (*this)[i] < rhs[i]; 1523 return false; 1524 } 1525 }; 1526 1527 struct OutputIntrinsic { 1528 const ACLEIntrinsic *Int; 1529 std::string Name; 1530 ComparableStringVector ParamValues; 1531 bool operator<(const OutputIntrinsic &rhs) const { 1532 if (Name != rhs.Name) 1533 return Name < rhs.Name; 1534 return ParamValues < rhs.ParamValues; 1535 } 1536 }; 1537 struct MergeableGroup { 1538 std::string Code; 1539 ComparableStringVector ParamTypes; 1540 bool operator<(const MergeableGroup &rhs) const { 1541 if (Code != rhs.Code) 1542 return Code < rhs.Code; 1543 return ParamTypes < rhs.ParamTypes; 1544 } 1545 }; 1546 1547 void EmitterBase::EmitBuiltinCG(raw_ostream &OS) { 1548 // Pass 1: generate code for all the intrinsics as if every type or constant 1549 // that can possibly be abstracted out into a parameter variable will be. 1550 // This identifies the sets of intrinsics we'll group together into a single 1551 // piece of code generation. 1552 1553 std::map<MergeableGroup, std::set<OutputIntrinsic>> MergeableGroupsPrelim; 1554 1555 for (const auto &kv : ACLEIntrinsics) { 1556 const ACLEIntrinsic &Int = *kv.second; 1557 if (Int.headerOnly()) 1558 continue; 1559 1560 MergeableGroup MG; 1561 OutputIntrinsic OI; 1562 1563 OI.Int = ∬ 1564 OI.Name = Int.fullName(); 1565 CodeGenParamAllocator ParamAllocPrelim{&MG.ParamTypes, &OI.ParamValues}; 1566 raw_string_ostream OS(MG.Code); 1567 Int.genCode(OS, ParamAllocPrelim, 1); 1568 OS.flush(); 1569 1570 MergeableGroupsPrelim[MG].insert(OI); 1571 } 1572 1573 // Pass 2: for each of those groups, optimize the parameter variable set by 1574 // eliminating 'parameters' that are the same for all intrinsics in the 1575 // group, and merging together pairs of parameter variables that take the 1576 // same values as each other for all intrinsics in the group. 1577 1578 std::map<MergeableGroup, std::set<OutputIntrinsic>> MergeableGroups; 1579 1580 for (const auto &kv : MergeableGroupsPrelim) { 1581 const MergeableGroup &MG = kv.first; 1582 std::vector<int> ParamNumbers; 1583 std::map<ComparableStringVector, int> ParamNumberMap; 1584 1585 // Loop over the parameters for this group. 1586 for (size_t i = 0, e = MG.ParamTypes.size(); i < e; ++i) { 1587 // Is this parameter the same for all intrinsics in the group? 1588 const OutputIntrinsic &OI_first = *kv.second.begin(); 1589 bool Constant = all_of(kv.second, [&](const OutputIntrinsic &OI) { 1590 return OI.ParamValues[i] == OI_first.ParamValues[i]; 1591 }); 1592 1593 // If so, record it as -1, meaning 'no parameter variable needed'. Then 1594 // the corresponding call to allocParam in pass 2 will not generate a 1595 // variable at all, and just use the value inline. 1596 if (Constant) { 1597 ParamNumbers.push_back(-1); 1598 continue; 1599 } 1600 1601 // Otherwise, make a list of the values this parameter takes for each 1602 // intrinsic, and see if that value vector matches anything we already 1603 // have. We also record the parameter type, so that we don't accidentally 1604 // match up two parameter variables with different types. (Not that 1605 // there's much chance of them having textually equivalent values, but in 1606 // _principle_ it could happen.) 1607 ComparableStringVector key; 1608 key.push_back(MG.ParamTypes[i]); 1609 for (const auto &OI : kv.second) 1610 key.push_back(OI.ParamValues[i]); 1611 1612 auto Found = ParamNumberMap.find(key); 1613 if (Found != ParamNumberMap.end()) { 1614 // Yes, an existing parameter variable can be reused for this. 1615 ParamNumbers.push_back(Found->second); 1616 continue; 1617 } 1618 1619 // No, we need a new parameter variable. 1620 int ExistingIndex = ParamNumberMap.size(); 1621 ParamNumberMap[key] = ExistingIndex; 1622 ParamNumbers.push_back(ExistingIndex); 1623 } 1624 1625 // Now we're ready to do the pass 2 code generation, which will emit the 1626 // reduced set of parameter variables we've just worked out. 1627 1628 for (const auto &OI_prelim : kv.second) { 1629 const ACLEIntrinsic *Int = OI_prelim.Int; 1630 1631 MergeableGroup MG; 1632 OutputIntrinsic OI; 1633 1634 OI.Int = OI_prelim.Int; 1635 OI.Name = OI_prelim.Name; 1636 CodeGenParamAllocator ParamAlloc{&MG.ParamTypes, &OI.ParamValues, 1637 &ParamNumbers}; 1638 raw_string_ostream OS(MG.Code); 1639 Int->genCode(OS, ParamAlloc, 2); 1640 OS.flush(); 1641 1642 MergeableGroups[MG].insert(OI); 1643 } 1644 } 1645 1646 // Output the actual C++ code. 1647 1648 for (const auto &kv : MergeableGroups) { 1649 const MergeableGroup &MG = kv.first; 1650 1651 // List of case statements in the main switch on BuiltinID, and an open 1652 // brace. 1653 const char *prefix = ""; 1654 for (const auto &OI : kv.second) { 1655 OS << prefix << "case ARM::BI__builtin_arm_" << OI.Int->builtinExtension() 1656 << "_" << OI.Name << ":"; 1657 1658 prefix = "\n"; 1659 } 1660 OS << " {\n"; 1661 1662 if (!MG.ParamTypes.empty()) { 1663 // If we've got some parameter variables, then emit their declarations... 1664 for (size_t i = 0, e = MG.ParamTypes.size(); i < e; ++i) { 1665 StringRef Type = MG.ParamTypes[i]; 1666 OS << " " << Type; 1667 if (!Type.endswith("*")) 1668 OS << " "; 1669 OS << " Param" << utostr(i) << ";\n"; 1670 } 1671 1672 // ... and an inner switch on BuiltinID that will fill them in with each 1673 // individual intrinsic's values. 1674 OS << " switch (BuiltinID) {\n"; 1675 for (const auto &OI : kv.second) { 1676 OS << " case ARM::BI__builtin_arm_" << OI.Int->builtinExtension() 1677 << "_" << OI.Name << ":\n"; 1678 for (size_t i = 0, e = MG.ParamTypes.size(); i < e; ++i) 1679 OS << " Param" << utostr(i) << " = " << OI.ParamValues[i] << ";\n"; 1680 OS << " break;\n"; 1681 } 1682 OS << " }\n"; 1683 } 1684 1685 // And finally, output the code, and close the outer pair of braces. (The 1686 // code will always end with a 'return' statement, so we need not insert a 1687 // 'break' here.) 1688 OS << MG.Code << "}\n"; 1689 } 1690 } 1691 1692 void EmitterBase::EmitBuiltinAliases(raw_ostream &OS) { 1693 // Build a sorted table of: 1694 // - intrinsic id number 1695 // - full name 1696 // - polymorphic name or -1 1697 StringToOffsetTable StringTable; 1698 OS << "static const IntrinToName MapData[] = {\n"; 1699 for (const auto &kv : ACLEIntrinsics) { 1700 const ACLEIntrinsic &Int = *kv.second; 1701 if (Int.headerOnly()) 1702 continue; 1703 int32_t ShortNameOffset = 1704 Int.polymorphic() ? StringTable.GetOrAddStringOffset(Int.shortName()) 1705 : -1; 1706 OS << " { ARM::BI__builtin_arm_" << Int.builtinExtension() << "_" 1707 << Int.fullName() << ", " 1708 << StringTable.GetOrAddStringOffset(Int.fullName()) << ", " 1709 << ShortNameOffset << "},\n"; 1710 } 1711 OS << "};\n\n"; 1712 1713 OS << "ArrayRef<IntrinToName> Map(MapData);\n\n"; 1714 1715 OS << "static const char IntrinNames[] = {\n"; 1716 StringTable.EmitString(OS); 1717 OS << "};\n\n"; 1718 } 1719 1720 void EmitterBase::GroupSemaChecks( 1721 std::map<std::string, std::set<std::string>> &Checks) { 1722 for (const auto &kv : ACLEIntrinsics) { 1723 const ACLEIntrinsic &Int = *kv.second; 1724 if (Int.headerOnly()) 1725 continue; 1726 std::string Check = Int.genSema(); 1727 if (!Check.empty()) 1728 Checks[Check].insert(Int.fullName()); 1729 } 1730 } 1731 1732 // ----------------------------------------------------------------------------- 1733 // The class used for generating arm_mve.h and related Clang bits 1734 // 1735 1736 class MveEmitter : public EmitterBase { 1737 public: 1738 MveEmitter(RecordKeeper &Records) : EmitterBase(Records){}; 1739 void EmitHeader(raw_ostream &OS) override; 1740 void EmitBuiltinDef(raw_ostream &OS) override; 1741 void EmitBuiltinSema(raw_ostream &OS) override; 1742 }; 1743 1744 void MveEmitter::EmitHeader(raw_ostream &OS) { 1745 // Accumulate pieces of the header file that will be enabled under various 1746 // different combinations of #ifdef. The index into parts[] is made up of 1747 // the following bit flags. 1748 constexpr unsigned Float = 1; 1749 constexpr unsigned UseUserNamespace = 2; 1750 1751 constexpr unsigned NumParts = 4; 1752 raw_self_contained_string_ostream parts[NumParts]; 1753 1754 // Write typedefs for all the required vector types, and a few scalar 1755 // types that don't already have the name we want them to have. 1756 1757 parts[0] << "typedef uint16_t mve_pred16_t;\n"; 1758 parts[Float] << "typedef __fp16 float16_t;\n" 1759 "typedef float float32_t;\n"; 1760 for (const auto &kv : ScalarTypes) { 1761 const ScalarType *ST = kv.second.get(); 1762 if (ST->hasNonstandardName()) 1763 continue; 1764 raw_ostream &OS = parts[ST->requiresFloat() ? Float : 0]; 1765 const VectorType *VT = getVectorType(ST); 1766 1767 OS << "typedef __attribute__((__neon_vector_type__(" << VT->lanes() 1768 << "), __clang_arm_mve_strict_polymorphism)) " << ST->cName() << " " 1769 << VT->cName() << ";\n"; 1770 1771 // Every vector type also comes with a pair of multi-vector types for 1772 // the VLD2 and VLD4 instructions. 1773 for (unsigned n = 2; n <= 4; n += 2) { 1774 const MultiVectorType *MT = getMultiVectorType(n, VT); 1775 OS << "typedef struct { " << VT->cName() << " val[" << n << "]; } " 1776 << MT->cName() << ";\n"; 1777 } 1778 } 1779 parts[0] << "\n"; 1780 parts[Float] << "\n"; 1781 1782 // Write declarations for all the intrinsics. 1783 1784 for (const auto &kv : ACLEIntrinsics) { 1785 const ACLEIntrinsic &Int = *kv.second; 1786 1787 // We generate each intrinsic twice, under its full unambiguous 1788 // name and its shorter polymorphic name (if the latter exists). 1789 for (bool Polymorphic : {false, true}) { 1790 if (Polymorphic && !Int.polymorphic()) 1791 continue; 1792 if (!Polymorphic && Int.polymorphicOnly()) 1793 continue; 1794 1795 // We also generate each intrinsic under a name like __arm_vfooq 1796 // (which is in C language implementation namespace, so it's 1797 // safe to define in any conforming user program) and a shorter 1798 // one like vfooq (which is in user namespace, so a user might 1799 // reasonably have used it for something already). If so, they 1800 // can #define __ARM_MVE_PRESERVE_USER_NAMESPACE before 1801 // including the header, which will suppress the shorter names 1802 // and leave only the implementation-namespace ones. Then they 1803 // have to write __arm_vfooq everywhere, of course. 1804 1805 for (bool UserNamespace : {false, true}) { 1806 raw_ostream &OS = parts[(Int.requiresFloat() ? Float : 0) | 1807 (UserNamespace ? UseUserNamespace : 0)]; 1808 1809 // Make the name of the function in this declaration. 1810 1811 std::string FunctionName = 1812 Polymorphic ? Int.shortName() : Int.fullName(); 1813 if (!UserNamespace) 1814 FunctionName = "__arm_" + FunctionName; 1815 1816 // Make strings for the types involved in the function's 1817 // prototype. 1818 1819 std::string RetTypeName = Int.returnType()->cName(); 1820 if (!StringRef(RetTypeName).endswith("*")) 1821 RetTypeName += " "; 1822 1823 std::vector<std::string> ArgTypeNames; 1824 for (const Type *ArgTypePtr : Int.argTypes()) 1825 ArgTypeNames.push_back(ArgTypePtr->cName()); 1826 std::string ArgTypesString = 1827 join(std::begin(ArgTypeNames), std::end(ArgTypeNames), ", "); 1828 1829 // Emit the actual declaration. All these functions are 1830 // declared 'static inline' without a body, which is fine 1831 // provided clang recognizes them as builtins, and has the 1832 // effect that this type signature is used in place of the one 1833 // that Builtins.def didn't provide. That's how we can get 1834 // structure types that weren't defined until this header was 1835 // included to be part of the type signature of a builtin that 1836 // was known to clang already. 1837 // 1838 // The declarations use __attribute__(__clang_arm_builtin_alias), 1839 // so that each function declared will be recognized as the 1840 // appropriate MVE builtin in spite of its user-facing name. 1841 // 1842 // (That's better than making them all wrapper functions, 1843 // partly because it avoids any compiler error message citing 1844 // the wrapper function definition instead of the user's code, 1845 // and mostly because some MVE intrinsics have arguments 1846 // required to be compile-time constants, and that property 1847 // can't be propagated through a wrapper function. It can be 1848 // propagated through a macro, but macros can't be overloaded 1849 // on argument types very easily - you have to use _Generic, 1850 // which makes error messages very confusing when the user 1851 // gets it wrong.) 1852 // 1853 // Finally, the polymorphic versions of the intrinsics are 1854 // also defined with __attribute__(overloadable), so that when 1855 // the same name is defined with several type signatures, the 1856 // right thing happens. Each one of the overloaded 1857 // declarations is given a different builtin id, which 1858 // has exactly the effect we want: first clang resolves the 1859 // overload to the right function, then it knows which builtin 1860 // it's referring to, and then the Sema checking for that 1861 // builtin can check further things like the constant 1862 // arguments. 1863 // 1864 // One more subtlety is the newline just before the return 1865 // type name. That's a cosmetic tweak to make the error 1866 // messages legible if the user gets the types wrong in a call 1867 // to a polymorphic function: this way, clang will print just 1868 // the _final_ line of each declaration in the header, to show 1869 // the type signatures that would have been legal. So all the 1870 // confusing machinery with __attribute__ is left out of the 1871 // error message, and the user sees something that's more or 1872 // less self-documenting: "here's a list of actually readable 1873 // type signatures for vfooq(), and here's why each one didn't 1874 // match your call". 1875 1876 OS << "static __inline__ __attribute__((" 1877 << (Polymorphic ? "__overloadable__, " : "") 1878 << "__clang_arm_builtin_alias(__builtin_arm_mve_" << Int.fullName() 1879 << ")))\n" 1880 << RetTypeName << FunctionName << "(" << ArgTypesString << ");\n"; 1881 } 1882 } 1883 } 1884 for (auto &part : parts) 1885 part << "\n"; 1886 1887 // Now we've finished accumulating bits and pieces into the parts[] array. 1888 // Put it all together to write the final output file. 1889 1890 OS << "/*===---- arm_mve.h - ARM MVE intrinsics " 1891 "-----------------------------------===\n" 1892 << LLVMLicenseHeader 1893 << "#ifndef __ARM_MVE_H\n" 1894 "#define __ARM_MVE_H\n" 1895 "\n" 1896 "#if !__ARM_FEATURE_MVE\n" 1897 "#error \"MVE support not enabled\"\n" 1898 "#endif\n" 1899 "\n" 1900 "#include <stdint.h>\n" 1901 "\n" 1902 "#ifdef __cplusplus\n" 1903 "extern \"C\" {\n" 1904 "#endif\n" 1905 "\n"; 1906 1907 for (size_t i = 0; i < NumParts; ++i) { 1908 std::vector<std::string> conditions; 1909 if (i & Float) 1910 conditions.push_back("(__ARM_FEATURE_MVE & 2)"); 1911 if (i & UseUserNamespace) 1912 conditions.push_back("(!defined __ARM_MVE_PRESERVE_USER_NAMESPACE)"); 1913 1914 std::string condition = 1915 join(std::begin(conditions), std::end(conditions), " && "); 1916 if (!condition.empty()) 1917 OS << "#if " << condition << "\n\n"; 1918 OS << parts[i].str(); 1919 if (!condition.empty()) 1920 OS << "#endif /* " << condition << " */\n\n"; 1921 } 1922 1923 OS << "#ifdef __cplusplus\n" 1924 "} /* extern \"C\" */\n" 1925 "#endif\n" 1926 "\n" 1927 "#endif /* __ARM_MVE_H */\n"; 1928 } 1929 1930 void MveEmitter::EmitBuiltinDef(raw_ostream &OS) { 1931 for (const auto &kv : ACLEIntrinsics) { 1932 const ACLEIntrinsic &Int = *kv.second; 1933 OS << "TARGET_HEADER_BUILTIN(__builtin_arm_mve_" << Int.fullName() 1934 << ", \"\", \"n\", \"arm_mve.h\", ALL_LANGUAGES, \"\")\n"; 1935 } 1936 1937 std::set<std::string> ShortNamesSeen; 1938 1939 for (const auto &kv : ACLEIntrinsics) { 1940 const ACLEIntrinsic &Int = *kv.second; 1941 if (Int.polymorphic()) { 1942 StringRef Name = Int.shortName(); 1943 if (ShortNamesSeen.find(std::string(Name)) == ShortNamesSeen.end()) { 1944 OS << "BUILTIN(__builtin_arm_mve_" << Name << ", \"vi.\", \"nt"; 1945 if (Int.nonEvaluating()) 1946 OS << "u"; // indicate that this builtin doesn't evaluate its args 1947 OS << "\")\n"; 1948 ShortNamesSeen.insert(std::string(Name)); 1949 } 1950 } 1951 } 1952 } 1953 1954 void MveEmitter::EmitBuiltinSema(raw_ostream &OS) { 1955 std::map<std::string, std::set<std::string>> Checks; 1956 GroupSemaChecks(Checks); 1957 1958 for (const auto &kv : Checks) { 1959 for (StringRef Name : kv.second) 1960 OS << "case ARM::BI__builtin_arm_mve_" << Name << ":\n"; 1961 OS << " return " << kv.first; 1962 } 1963 } 1964 1965 // ----------------------------------------------------------------------------- 1966 // Class that describes an ACLE intrinsic implemented as a macro. 1967 // 1968 // This class is used when the intrinsic is polymorphic in 2 or 3 types, but we 1969 // want to avoid a combinatorial explosion by reinterpreting the arguments to 1970 // fixed types. 1971 1972 class FunctionMacro { 1973 std::vector<StringRef> Params; 1974 StringRef Definition; 1975 1976 public: 1977 FunctionMacro(const Record &R); 1978 1979 const std::vector<StringRef> &getParams() const { return Params; } 1980 StringRef getDefinition() const { return Definition; } 1981 }; 1982 1983 FunctionMacro::FunctionMacro(const Record &R) { 1984 Params = R.getValueAsListOfStrings("params"); 1985 Definition = R.getValueAsString("definition"); 1986 } 1987 1988 // ----------------------------------------------------------------------------- 1989 // The class used for generating arm_cde.h and related Clang bits 1990 // 1991 1992 class CdeEmitter : public EmitterBase { 1993 std::map<StringRef, FunctionMacro> FunctionMacros; 1994 1995 public: 1996 CdeEmitter(RecordKeeper &Records); 1997 void EmitHeader(raw_ostream &OS) override; 1998 void EmitBuiltinDef(raw_ostream &OS) override; 1999 void EmitBuiltinSema(raw_ostream &OS) override; 2000 }; 2001 2002 CdeEmitter::CdeEmitter(RecordKeeper &Records) : EmitterBase(Records) { 2003 for (Record *R : Records.getAllDerivedDefinitions("FunctionMacro")) 2004 FunctionMacros.emplace(R->getName(), FunctionMacro(*R)); 2005 } 2006 2007 void CdeEmitter::EmitHeader(raw_ostream &OS) { 2008 // Accumulate pieces of the header file that will be enabled under various 2009 // different combinations of #ifdef. The index into parts[] is one of the 2010 // following: 2011 constexpr unsigned None = 0; 2012 constexpr unsigned MVE = 1; 2013 constexpr unsigned MVEFloat = 2; 2014 2015 constexpr unsigned NumParts = 3; 2016 raw_self_contained_string_ostream parts[NumParts]; 2017 2018 // Write typedefs for all the required vector types, and a few scalar 2019 // types that don't already have the name we want them to have. 2020 2021 parts[MVE] << "typedef uint16_t mve_pred16_t;\n"; 2022 parts[MVEFloat] << "typedef __fp16 float16_t;\n" 2023 "typedef float float32_t;\n"; 2024 for (const auto &kv : ScalarTypes) { 2025 const ScalarType *ST = kv.second.get(); 2026 if (ST->hasNonstandardName()) 2027 continue; 2028 // We don't have float64x2_t 2029 if (ST->kind() == ScalarTypeKind::Float && ST->sizeInBits() == 64) 2030 continue; 2031 raw_ostream &OS = parts[ST->requiresFloat() ? MVEFloat : MVE]; 2032 const VectorType *VT = getVectorType(ST); 2033 2034 OS << "typedef __attribute__((__neon_vector_type__(" << VT->lanes() 2035 << "), __clang_arm_mve_strict_polymorphism)) " << ST->cName() << " " 2036 << VT->cName() << ";\n"; 2037 } 2038 parts[MVE] << "\n"; 2039 parts[MVEFloat] << "\n"; 2040 2041 // Write declarations for all the intrinsics. 2042 2043 for (const auto &kv : ACLEIntrinsics) { 2044 const ACLEIntrinsic &Int = *kv.second; 2045 2046 // We generate each intrinsic twice, under its full unambiguous 2047 // name and its shorter polymorphic name (if the latter exists). 2048 for (bool Polymorphic : {false, true}) { 2049 if (Polymorphic && !Int.polymorphic()) 2050 continue; 2051 if (!Polymorphic && Int.polymorphicOnly()) 2052 continue; 2053 2054 raw_ostream &OS = 2055 parts[Int.requiresFloat() ? MVEFloat 2056 : Int.requiresMVE() ? MVE : None]; 2057 2058 // Make the name of the function in this declaration. 2059 std::string FunctionName = 2060 "__arm_" + (Polymorphic ? Int.shortName() : Int.fullName()); 2061 2062 // Make strings for the types involved in the function's 2063 // prototype. 2064 std::string RetTypeName = Int.returnType()->cName(); 2065 if (!StringRef(RetTypeName).endswith("*")) 2066 RetTypeName += " "; 2067 2068 std::vector<std::string> ArgTypeNames; 2069 for (const Type *ArgTypePtr : Int.argTypes()) 2070 ArgTypeNames.push_back(ArgTypePtr->cName()); 2071 std::string ArgTypesString = 2072 join(std::begin(ArgTypeNames), std::end(ArgTypeNames), ", "); 2073 2074 // Emit the actual declaration. See MveEmitter::EmitHeader for detailed 2075 // comments 2076 OS << "static __inline__ __attribute__((" 2077 << (Polymorphic ? "__overloadable__, " : "") 2078 << "__clang_arm_builtin_alias(__builtin_arm_" << Int.builtinExtension() 2079 << "_" << Int.fullName() << ")))\n" 2080 << RetTypeName << FunctionName << "(" << ArgTypesString << ");\n"; 2081 } 2082 } 2083 2084 for (const auto &kv : FunctionMacros) { 2085 StringRef Name = kv.first; 2086 const FunctionMacro &FM = kv.second; 2087 2088 raw_ostream &OS = parts[MVE]; 2089 OS << "#define " 2090 << "__arm_" << Name << "(" << join(FM.getParams(), ", ") << ") " 2091 << FM.getDefinition() << "\n"; 2092 } 2093 2094 for (auto &part : parts) 2095 part << "\n"; 2096 2097 // Now we've finished accumulating bits and pieces into the parts[] array. 2098 // Put it all together to write the final output file. 2099 2100 OS << "/*===---- arm_cde.h - ARM CDE intrinsics " 2101 "-----------------------------------===\n" 2102 << LLVMLicenseHeader 2103 << "#ifndef __ARM_CDE_H\n" 2104 "#define __ARM_CDE_H\n" 2105 "\n" 2106 "#if !__ARM_FEATURE_CDE\n" 2107 "#error \"CDE support not enabled\"\n" 2108 "#endif\n" 2109 "\n" 2110 "#include <stdint.h>\n" 2111 "\n" 2112 "#ifdef __cplusplus\n" 2113 "extern \"C\" {\n" 2114 "#endif\n" 2115 "\n"; 2116 2117 for (size_t i = 0; i < NumParts; ++i) { 2118 std::string condition; 2119 if (i == MVEFloat) 2120 condition = "__ARM_FEATURE_MVE & 2"; 2121 else if (i == MVE) 2122 condition = "__ARM_FEATURE_MVE"; 2123 2124 if (!condition.empty()) 2125 OS << "#if " << condition << "\n\n"; 2126 OS << parts[i].str(); 2127 if (!condition.empty()) 2128 OS << "#endif /* " << condition << " */\n\n"; 2129 } 2130 2131 OS << "#ifdef __cplusplus\n" 2132 "} /* extern \"C\" */\n" 2133 "#endif\n" 2134 "\n" 2135 "#endif /* __ARM_CDE_H */\n"; 2136 } 2137 2138 void CdeEmitter::EmitBuiltinDef(raw_ostream &OS) { 2139 for (const auto &kv : ACLEIntrinsics) { 2140 if (kv.second->headerOnly()) 2141 continue; 2142 const ACLEIntrinsic &Int = *kv.second; 2143 OS << "TARGET_HEADER_BUILTIN(__builtin_arm_cde_" << Int.fullName() 2144 << ", \"\", \"ncU\", \"arm_cde.h\", ALL_LANGUAGES, \"\")\n"; 2145 } 2146 } 2147 2148 void CdeEmitter::EmitBuiltinSema(raw_ostream &OS) { 2149 std::map<std::string, std::set<std::string>> Checks; 2150 GroupSemaChecks(Checks); 2151 2152 for (const auto &kv : Checks) { 2153 for (StringRef Name : kv.second) 2154 OS << "case ARM::BI__builtin_arm_cde_" << Name << ":\n"; 2155 OS << " Err = " << kv.first << " break;\n"; 2156 } 2157 } 2158 2159 } // namespace 2160 2161 namespace clang { 2162 2163 // MVE 2164 2165 void EmitMveHeader(RecordKeeper &Records, raw_ostream &OS) { 2166 MveEmitter(Records).EmitHeader(OS); 2167 } 2168 2169 void EmitMveBuiltinDef(RecordKeeper &Records, raw_ostream &OS) { 2170 MveEmitter(Records).EmitBuiltinDef(OS); 2171 } 2172 2173 void EmitMveBuiltinSema(RecordKeeper &Records, raw_ostream &OS) { 2174 MveEmitter(Records).EmitBuiltinSema(OS); 2175 } 2176 2177 void EmitMveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) { 2178 MveEmitter(Records).EmitBuiltinCG(OS); 2179 } 2180 2181 void EmitMveBuiltinAliases(RecordKeeper &Records, raw_ostream &OS) { 2182 MveEmitter(Records).EmitBuiltinAliases(OS); 2183 } 2184 2185 // CDE 2186 2187 void EmitCdeHeader(RecordKeeper &Records, raw_ostream &OS) { 2188 CdeEmitter(Records).EmitHeader(OS); 2189 } 2190 2191 void EmitCdeBuiltinDef(RecordKeeper &Records, raw_ostream &OS) { 2192 CdeEmitter(Records).EmitBuiltinDef(OS); 2193 } 2194 2195 void EmitCdeBuiltinSema(RecordKeeper &Records, raw_ostream &OS) { 2196 CdeEmitter(Records).EmitBuiltinSema(OS); 2197 } 2198 2199 void EmitCdeBuiltinCG(RecordKeeper &Records, raw_ostream &OS) { 2200 CdeEmitter(Records).EmitBuiltinCG(OS); 2201 } 2202 2203 void EmitCdeBuiltinAliases(RecordKeeper &Records, raw_ostream &OS) { 2204 CdeEmitter(Records).EmitBuiltinAliases(OS); 2205 } 2206 2207 } // end namespace clang 2208