1 //===--- RISCVVIntrinsicUtils.h - RISC-V Vector Intrinsic Utils -*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef CLANG_SUPPORT_RISCVVINTRINSICUTILS_H 10 #define CLANG_SUPPORT_RISCVVINTRINSICUTILS_H 11 12 #include "llvm/ADT/ArrayRef.h" 13 #include "llvm/ADT/BitmaskEnum.h" 14 #include "llvm/ADT/SmallVector.h" 15 #include "llvm/ADT/StringRef.h" 16 #include <cstdint> 17 #include <optional> 18 #include <set> 19 #include <string> 20 #include <unordered_map> 21 #include <vector> 22 23 namespace llvm { 24 class raw_ostream; 25 } // end namespace llvm 26 27 namespace clang { 28 namespace RISCV { 29 30 using VScaleVal = std::optional<unsigned>; 31 32 // Modifier for vector type. 33 enum class VectorTypeModifier : uint8_t { 34 NoModifier, 35 Widening2XVector, 36 Widening4XVector, 37 Widening8XVector, 38 MaskVector, 39 Log2EEW3, 40 Log2EEW4, 41 Log2EEW5, 42 Log2EEW6, 43 FixedSEW8, 44 FixedSEW16, 45 FixedSEW32, 46 FixedSEW64, 47 LFixedLog2LMULN3, 48 LFixedLog2LMULN2, 49 LFixedLog2LMULN1, 50 LFixedLog2LMUL0, 51 LFixedLog2LMUL1, 52 LFixedLog2LMUL2, 53 LFixedLog2LMUL3, 54 SFixedLog2LMULN3, 55 SFixedLog2LMULN2, 56 SFixedLog2LMULN1, 57 SFixedLog2LMUL0, 58 SFixedLog2LMUL1, 59 SFixedLog2LMUL2, 60 SFixedLog2LMUL3, 61 SEFixedLog2LMULN3, 62 SEFixedLog2LMULN2, 63 SEFixedLog2LMULN1, 64 SEFixedLog2LMUL0, 65 SEFixedLog2LMUL1, 66 SEFixedLog2LMUL2, 67 SEFixedLog2LMUL3, 68 Tuple2, 69 Tuple3, 70 Tuple4, 71 Tuple5, 72 Tuple6, 73 Tuple7, 74 Tuple8, 75 }; 76 77 // Similar to basic type but used to describe what's kind of type related to 78 // basic vector type, used to compute type info of arguments. 79 enum class BaseTypeModifier : uint8_t { 80 Invalid, 81 Scalar, 82 Vector, 83 Void, 84 SizeT, 85 Ptrdiff, 86 UnsignedLong, 87 SignedLong, 88 Float32 89 }; 90 91 // Modifier for type, used for both scalar and vector types. 92 enum class TypeModifier : uint8_t { 93 NoModifier = 0, 94 Pointer = 1 << 0, 95 Const = 1 << 1, 96 Immediate = 1 << 2, 97 UnsignedInteger = 1 << 3, 98 SignedInteger = 1 << 4, 99 Float = 1 << 5, 100 BFloat = 1 << 6, 101 // LMUL1 should be kind of VectorTypeModifier, but that might come with 102 // Widening2XVector for widening reduction. 103 // However that might require VectorTypeModifier become bitmask rather than 104 // simple enum, so we decide keek LMUL1 in TypeModifier for code size 105 // optimization of clang binary size. 106 LMUL1 = 1 << 7, 107 MaxOffset = 7, 108 LLVM_MARK_AS_BITMASK_ENUM(LMUL1), 109 }; 110 111 class Policy { 112 public: 113 enum PolicyType { 114 Undisturbed, 115 Agnostic, 116 }; 117 118 private: 119 // The default assumption for an RVV instruction is TAMA, as an undisturbed 120 // policy generally will affect the performance of an out-of-order core. 121 const PolicyType TailPolicy = Agnostic; 122 const PolicyType MaskPolicy = Agnostic; 123 124 public: 125 Policy() = default; Policy(PolicyType TailPolicy)126 Policy(PolicyType TailPolicy) : TailPolicy(TailPolicy) {} Policy(PolicyType TailPolicy,PolicyType MaskPolicy)127 Policy(PolicyType TailPolicy, PolicyType MaskPolicy) 128 : TailPolicy(TailPolicy), MaskPolicy(MaskPolicy) {} 129 isTAMAPolicy()130 bool isTAMAPolicy() const { 131 return TailPolicy == Agnostic && MaskPolicy == Agnostic; 132 } 133 isTAMUPolicy()134 bool isTAMUPolicy() const { 135 return TailPolicy == Agnostic && MaskPolicy == Undisturbed; 136 } 137 isTUMAPolicy()138 bool isTUMAPolicy() const { 139 return TailPolicy == Undisturbed && MaskPolicy == Agnostic; 140 } 141 isTUMUPolicy()142 bool isTUMUPolicy() const { 143 return TailPolicy == Undisturbed && MaskPolicy == Undisturbed; 144 } 145 isTAPolicy()146 bool isTAPolicy() const { return TailPolicy == Agnostic; } 147 isTUPolicy()148 bool isTUPolicy() const { return TailPolicy == Undisturbed; } 149 isMAPolicy()150 bool isMAPolicy() const { return MaskPolicy == Agnostic; } 151 isMUPolicy()152 bool isMUPolicy() const { return MaskPolicy == Undisturbed; } 153 154 bool operator==(const Policy &Other) const { 155 return TailPolicy == Other.TailPolicy && MaskPolicy == Other.MaskPolicy; 156 } 157 158 bool operator!=(const Policy &Other) const { return !(*this == Other); } 159 160 bool operator<(const Policy &Other) const { 161 // Just for maintain the old order for quick test. 162 if (MaskPolicy != Other.MaskPolicy) 163 return Other.MaskPolicy < MaskPolicy; 164 return TailPolicy < Other.TailPolicy; 165 } 166 }; 167 168 // PrototypeDescriptor is used to compute type info of arguments or return 169 // value. 170 struct PrototypeDescriptor { 171 constexpr PrototypeDescriptor() = default; 172 constexpr PrototypeDescriptor( 173 BaseTypeModifier PT, 174 VectorTypeModifier VTM = VectorTypeModifier::NoModifier, 175 TypeModifier TM = TypeModifier::NoModifier) PTPrototypeDescriptor176 : PT(static_cast<uint8_t>(PT)), VTM(static_cast<uint8_t>(VTM)), 177 TM(static_cast<uint8_t>(TM)) {} PrototypeDescriptorPrototypeDescriptor178 constexpr PrototypeDescriptor(uint8_t PT, uint8_t VTM, uint8_t TM) 179 : PT(PT), VTM(VTM), TM(TM) {} 180 181 uint8_t PT = static_cast<uint8_t>(BaseTypeModifier::Invalid); 182 uint8_t VTM = static_cast<uint8_t>(VectorTypeModifier::NoModifier); 183 uint8_t TM = static_cast<uint8_t>(TypeModifier::NoModifier); 184 185 bool operator!=(const PrototypeDescriptor &PD) const { 186 return !(*this == PD); 187 } 188 bool operator==(const PrototypeDescriptor &PD) const { 189 return PD.PT == PT && PD.VTM == VTM && PD.TM == TM; 190 } 191 bool operator<(const PrototypeDescriptor &PD) const { 192 return std::tie(PT, VTM, TM) < std::tie(PD.PT, PD.VTM, PD.TM); 193 } 194 static const PrototypeDescriptor Mask; 195 static const PrototypeDescriptor Vector; 196 static const PrototypeDescriptor VL; 197 static std::optional<PrototypeDescriptor> 198 parsePrototypeDescriptor(llvm::StringRef PrototypeStr); 199 }; 200 201 llvm::SmallVector<PrototypeDescriptor> 202 parsePrototypes(llvm::StringRef Prototypes); 203 204 // Basic type of vector type. 205 enum class BasicType : uint8_t { 206 Unknown = 0, 207 Int8 = 1 << 0, 208 Int16 = 1 << 1, 209 Int32 = 1 << 2, 210 Int64 = 1 << 3, 211 BFloat16 = 1 << 4, 212 Float16 = 1 << 5, 213 Float32 = 1 << 6, 214 Float64 = 1 << 7, 215 MaxOffset = 7, 216 LLVM_MARK_AS_BITMASK_ENUM(Float64), 217 }; 218 219 // Type of vector type. 220 enum ScalarTypeKind : uint8_t { 221 Void, 222 Size_t, 223 Ptrdiff_t, 224 UnsignedLong, 225 SignedLong, 226 Boolean, 227 SignedInteger, 228 UnsignedInteger, 229 Float, 230 BFloat, 231 Invalid, 232 Undefined, 233 }; 234 235 // Exponential LMUL 236 struct LMULType { 237 int Log2LMUL; 238 LMULType(int Log2LMUL); 239 // Return the C/C++ string representation of LMUL 240 std::string str() const; 241 std::optional<unsigned> getScale(unsigned ElementBitwidth) const; 242 void MulLog2LMUL(int Log2LMUL); 243 }; 244 245 class RVVType; 246 using RVVTypePtr = RVVType *; 247 using RVVTypes = std::vector<RVVTypePtr>; 248 class RVVTypeCache; 249 250 // This class is compact representation of a valid and invalid RVVType. 251 class RVVType { 252 friend class RVVTypeCache; 253 254 BasicType BT; 255 ScalarTypeKind ScalarType = Undefined; 256 LMULType LMUL; 257 bool IsPointer = false; 258 // IsConstant indices are "int", but have the constant expression. 259 bool IsImmediate = false; 260 // Const qualifier for pointer to const object or object of const type. 261 bool IsConstant = false; 262 unsigned ElementBitwidth = 0; 263 VScaleVal Scale = 0; 264 bool Valid; 265 bool IsTuple = false; 266 unsigned NF = 0; 267 268 std::string BuiltinStr; 269 std::string ClangBuiltinStr; 270 std::string Str; 271 std::string ShortStr; 272 273 enum class FixedLMULType { LargerThan, SmallerThan, SmallerOrEqual }; 274 275 RVVType(BasicType BT, int Log2LMUL, const PrototypeDescriptor &Profile); 276 277 public: 278 // Return the string representation of a type, which is an encoded string for 279 // passing to the BUILTIN() macro in Builtins.def. getBuiltinStr()280 const std::string &getBuiltinStr() const { return BuiltinStr; } 281 282 // Return the clang builtin type for RVV vector type which are used in the 283 // riscv_vector.h header file. getClangBuiltinStr()284 const std::string &getClangBuiltinStr() const { return ClangBuiltinStr; } 285 286 // Return the C/C++ string representation of a type for use in the 287 // riscv_vector.h header file. getTypeStr()288 const std::string &getTypeStr() const { return Str; } 289 290 // Return the short name of a type for C/C++ name suffix. getShortStr()291 const std::string &getShortStr() { 292 // Not all types are used in short name, so compute the short name by 293 // demanded. 294 if (ShortStr.empty()) 295 initShortStr(); 296 return ShortStr; 297 } 298 isValid()299 bool isValid() const { return Valid; } isScalar()300 bool isScalar() const { return Scale && *Scale == 0; } isVector()301 bool isVector() const { return Scale && *Scale != 0; } isVector(unsigned Width)302 bool isVector(unsigned Width) const { 303 return isVector() && ElementBitwidth == Width; 304 } isFloat()305 bool isFloat() const { return ScalarType == ScalarTypeKind::Float; } isBFloat()306 bool isBFloat() const { return ScalarType == ScalarTypeKind::BFloat; } isSignedInteger()307 bool isSignedInteger() const { 308 return ScalarType == ScalarTypeKind::SignedInteger; 309 } isFloatVector(unsigned Width)310 bool isFloatVector(unsigned Width) const { 311 return isVector() && isFloat() && ElementBitwidth == Width; 312 } isFloat(unsigned Width)313 bool isFloat(unsigned Width) const { 314 return isFloat() && ElementBitwidth == Width; 315 } isConstant()316 bool isConstant() const { return IsConstant; } isPointer()317 bool isPointer() const { return IsPointer; } isTuple()318 bool isTuple() const { return IsTuple; } getElementBitwidth()319 unsigned getElementBitwidth() const { return ElementBitwidth; } 320 getScalarType()321 ScalarTypeKind getScalarType() const { return ScalarType; } getScale()322 VScaleVal getScale() const { return Scale; } getNF()323 unsigned getNF() const { 324 assert(NF > 1 && NF <= 8 && "Only legal NF should be fetched"); 325 return NF; 326 } 327 328 private: 329 // Verify RVV vector type and set Valid. 330 bool verifyType() const; 331 332 // Creates a type based on basic types of TypeRange 333 void applyBasicType(); 334 335 // Applies a prototype modifier to the current type. The result maybe an 336 // invalid type. 337 void applyModifier(const PrototypeDescriptor &prototype); 338 339 void applyLog2EEW(unsigned Log2EEW); 340 void applyFixedSEW(unsigned NewSEW); 341 void applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type); 342 343 // Compute and record a string for legal type. 344 void initBuiltinStr(); 345 // Compute and record a builtin RVV vector type string. 346 void initClangBuiltinStr(); 347 // Compute and record a type string for used in the header. 348 void initTypeStr(); 349 // Compute and record a short name of a type for C/C++ name suffix. 350 void initShortStr(); 351 }; 352 353 // This class is used to manage RVVType, RVVType should only created by this 354 // class, also provided thread-safe cache capability. 355 class RVVTypeCache { 356 private: 357 std::unordered_map<uint64_t, RVVType> LegalTypes; 358 std::set<uint64_t> IllegalTypes; 359 360 public: 361 /// Compute output and input types by applying different config (basic type 362 /// and LMUL with type transformers). It also record result of type in legal 363 /// or illegal set to avoid compute the same config again. The result maybe 364 /// have illegal RVVType. 365 std::optional<RVVTypes> 366 computeTypes(BasicType BT, int Log2LMUL, unsigned NF, 367 llvm::ArrayRef<PrototypeDescriptor> Prototype); 368 std::optional<RVVTypePtr> computeType(BasicType BT, int Log2LMUL, 369 PrototypeDescriptor Proto); 370 }; 371 372 enum PolicyScheme : uint8_t { 373 SchemeNone, 374 // Passthru operand is at first parameter in C builtin. 375 HasPassthruOperand, 376 HasPolicyOperand, 377 }; 378 379 // TODO refactor RVVIntrinsic class design after support all intrinsic 380 // combination. This represents an instantiation of an intrinsic with a 381 // particular type and prototype 382 class RVVIntrinsic { 383 384 private: 385 std::string BuiltinName; // Builtin name 386 std::string Name; // C intrinsic name. 387 std::string OverloadedName; 388 std::string IRName; 389 bool IsMasked; 390 bool HasMaskedOffOperand; 391 bool HasVL; 392 PolicyScheme Scheme; 393 bool SupportOverloading; 394 bool HasBuiltinAlias; 395 std::string ManualCodegen; 396 RVVTypePtr OutputType; // Builtin output type 397 RVVTypes InputTypes; // Builtin input types 398 // The types we use to obtain the specific LLVM intrinsic. They are index of 399 // InputTypes. -1 means the return type. 400 std::vector<int64_t> IntrinsicTypes; 401 unsigned NF = 1; 402 Policy PolicyAttrs; 403 404 public: 405 RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix, 406 llvm::StringRef OverloadedName, llvm::StringRef OverloadedSuffix, 407 llvm::StringRef IRName, bool IsMasked, bool HasMaskedOffOperand, 408 bool HasVL, PolicyScheme Scheme, bool SupportOverloading, 409 bool HasBuiltinAlias, llvm::StringRef ManualCodegen, 410 const RVVTypes &Types, 411 const std::vector<int64_t> &IntrinsicTypes, 412 unsigned NF, Policy PolicyAttrs, bool HasFRMRoundModeOp); 413 ~RVVIntrinsic() = default; 414 getOutputType()415 RVVTypePtr getOutputType() const { return OutputType; } getInputTypes()416 const RVVTypes &getInputTypes() const { return InputTypes; } getBuiltinName()417 llvm::StringRef getBuiltinName() const { return BuiltinName; } hasMaskedOffOperand()418 bool hasMaskedOffOperand() const { return HasMaskedOffOperand; } hasVL()419 bool hasVL() const { return HasVL; } hasPolicy()420 bool hasPolicy() const { return Scheme != PolicyScheme::SchemeNone; } hasPassthruOperand()421 bool hasPassthruOperand() const { 422 return Scheme == PolicyScheme::HasPassthruOperand; 423 } hasPolicyOperand()424 bool hasPolicyOperand() const { 425 return Scheme == PolicyScheme::HasPolicyOperand; 426 } supportOverloading()427 bool supportOverloading() const { return SupportOverloading; } hasBuiltinAlias()428 bool hasBuiltinAlias() const { return HasBuiltinAlias; } hasManualCodegen()429 bool hasManualCodegen() const { return !ManualCodegen.empty(); } isMasked()430 bool isMasked() const { return IsMasked; } getIRName()431 llvm::StringRef getIRName() const { return IRName; } getManualCodegen()432 llvm::StringRef getManualCodegen() const { return ManualCodegen; } getPolicyScheme()433 PolicyScheme getPolicyScheme() const { return Scheme; } getNF()434 unsigned getNF() const { return NF; } getIntrinsicTypes()435 const std::vector<int64_t> &getIntrinsicTypes() const { 436 return IntrinsicTypes; 437 } getPolicyAttrs()438 Policy getPolicyAttrs() const { 439 return PolicyAttrs; 440 } getPolicyAttrsBits()441 unsigned getPolicyAttrsBits() const { 442 // CGBuiltin.cpp 443 // The 0th bit simulates the `vta` of RVV 444 // The 1st bit simulates the `vma` of RVV 445 // int PolicyAttrs = 0; 446 447 if (PolicyAttrs.isTUMAPolicy()) 448 return 2; 449 if (PolicyAttrs.isTAMAPolicy()) 450 return 3; 451 if (PolicyAttrs.isTUMUPolicy()) 452 return 0; 453 if (PolicyAttrs.isTAMUPolicy()) 454 return 1; 455 456 llvm_unreachable("unsupport policy"); 457 return 0; 458 } 459 460 // Return the type string for a BUILTIN() macro in Builtins.def. 461 std::string getBuiltinTypeStr() const; 462 463 static std::string 464 getSuffixStr(RVVTypeCache &TypeCache, BasicType Type, int Log2LMUL, 465 llvm::ArrayRef<PrototypeDescriptor> PrototypeDescriptors); 466 467 static llvm::SmallVector<PrototypeDescriptor> 468 computeBuiltinTypes(llvm::ArrayRef<PrototypeDescriptor> Prototype, 469 bool IsMasked, bool HasMaskedOffOperand, bool HasVL, 470 unsigned NF, PolicyScheme DefaultScheme, 471 Policy PolicyAttrs, bool IsTuple); 472 473 static llvm::SmallVector<Policy> getSupportedUnMaskedPolicies(); 474 static llvm::SmallVector<Policy> 475 getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy); 476 477 static void updateNamesAndPolicy(bool IsMasked, bool HasPolicy, 478 std::string &Name, std::string &BuiltinName, 479 std::string &OverloadedName, 480 Policy &PolicyAttrs, bool HasFRMRoundModeOp); 481 }; 482 483 // RVVRequire should be sync'ed with target features, but only 484 // required features used in riscv_vector.td. 485 enum RVVRequire : uint32_t { 486 RVV_REQ_None = 0, 487 RVV_REQ_RV64 = 1 << 0, 488 RVV_REQ_Zvfhmin = 1 << 1, 489 RVV_REQ_Xsfvcp = 1 << 2, 490 RVV_REQ_Xsfvfnrclipxfqf = 1 << 3, 491 RVV_REQ_Xsfvfwmaccqqq = 1 << 4, 492 RVV_REQ_Xsfvqmaccdod = 1 << 5, 493 RVV_REQ_Xsfvqmaccqoq = 1 << 6, 494 RVV_REQ_Zvbb = 1 << 7, 495 RVV_REQ_Zvbc = 1 << 8, 496 RVV_REQ_Zvkb = 1 << 9, 497 RVV_REQ_Zvkg = 1 << 10, 498 RVV_REQ_Zvkned = 1 << 11, 499 RVV_REQ_Zvknha = 1 << 12, 500 RVV_REQ_Zvknhb = 1 << 13, 501 RVV_REQ_Zvksed = 1 << 14, 502 RVV_REQ_Zvksh = 1 << 15, 503 RVV_REQ_Zvfbfwma = 1 << 16, 504 RVV_REQ_Zvfbfmin = 1 << 17, 505 RVV_REQ_Experimental = 1 << 18, 506 507 LLVM_MARK_AS_BITMASK_ENUM(RVV_REQ_Experimental) 508 }; 509 510 // Raw RVV intrinsic info, used to expand later. 511 // This struct is highly compact for minimized code size. 512 struct RVVIntrinsicRecord { 513 // Intrinsic name, e.g. vadd_vv 514 const char *Name; 515 516 // Overloaded intrinsic name, could be empty if it can be computed from Name. 517 // e.g. vadd 518 const char *OverloadedName; 519 520 // Prototype for this intrinsic, index of RVVSignatureTable. 521 uint16_t PrototypeIndex; 522 523 // Suffix of intrinsic name, index of RVVSignatureTable. 524 uint16_t SuffixIndex; 525 526 // Suffix of overloaded intrinsic name, index of RVVSignatureTable. 527 uint16_t OverloadedSuffixIndex; 528 529 // Length of the prototype. 530 uint8_t PrototypeLength; 531 532 // Length of intrinsic name suffix. 533 uint8_t SuffixLength; 534 535 // Length of overloaded intrinsic suffix. 536 uint8_t OverloadedSuffixSize; 537 538 // Required target features for this intrinsic. 539 uint32_t RequiredExtensions; 540 541 // Supported type, mask of BasicType. 542 uint8_t TypeRangeMask; 543 544 // Supported LMUL. 545 uint8_t Log2LMULMask; 546 547 // Number of fields, greater than 1 if it's segment load/store. 548 uint8_t NF; 549 550 bool HasMasked : 1; 551 bool HasVL : 1; 552 bool HasMaskedOffOperand : 1; 553 bool HasTailPolicy : 1; 554 bool HasMaskPolicy : 1; 555 bool HasFRMRoundModeOp : 1; 556 bool IsTuple : 1; 557 LLVM_PREFERRED_TYPE(PolicyScheme) 558 uint8_t UnMaskedPolicyScheme : 2; 559 LLVM_PREFERRED_TYPE(PolicyScheme) 560 uint8_t MaskedPolicyScheme : 2; 561 }; 562 563 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, 564 const RVVIntrinsicRecord &RVVInstrRecord); 565 566 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); 567 } // end namespace RISCV 568 569 } // end namespace clang 570 571 #endif // CLANG_SUPPORT_RISCVVINTRINSICUTILS_H 572