1 //===-- AMDGPULibFunc.h ----------------------------------------*- C++ -*--===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef _AMDGPU_LIBFUNC_H_ 10 #define _AMDGPU_LIBFUNC_H_ 11 12 #include "llvm/ADT/StringRef.h" 13 #include <memory> 14 15 namespace llvm { 16 17 class FunctionCallee; 18 class FunctionType; 19 class Function; 20 class Module; 21 class Type; 22 23 class AMDGPULibFuncBase { 24 public: 25 enum EFuncId { 26 EI_NONE, 27 28 // IMPORTANT: enums below should go in ascending by 1 value order 29 // because they are used as indexes in the mangling rules table. 30 // don't use explicit value assignment. 31 // 32 // There are two types of library functions: those with mangled 33 // name and those with unmangled name. The enums for the library 34 // functions with mangled name are defined before enums for the 35 // library functions with unmangled name. The enum for the last 36 // library function with mangled name is EI_LAST_MANGLED. 37 // 38 // Library functions with mangled name. 39 EI_ABS, 40 EI_ABS_DIFF, 41 EI_ACOS, 42 EI_ACOSH, 43 EI_ACOSPI, 44 EI_ADD_SAT, 45 EI_ALL, 46 EI_ANY, 47 EI_ASIN, 48 EI_ASINH, 49 EI_ASINPI, 50 EI_ASYNC_WORK_GROUP_COPY, 51 EI_ASYNC_WORK_GROUP_STRIDED_COPY, 52 EI_ATAN, 53 EI_ATAN2, 54 EI_ATAN2PI, 55 EI_ATANH, 56 EI_ATANPI, 57 EI_ATOMIC_ADD, 58 EI_ATOMIC_AND, 59 EI_ATOMIC_CMPXCHG, 60 EI_ATOMIC_DEC, 61 EI_ATOMIC_INC, 62 EI_ATOMIC_MAX, 63 EI_ATOMIC_MIN, 64 EI_ATOMIC_OR, 65 EI_ATOMIC_SUB, 66 EI_ATOMIC_XCHG, 67 EI_ATOMIC_XOR, 68 EI_BITSELECT, 69 EI_CBRT, 70 EI_CEIL, 71 EI_CLAMP, 72 EI_CLZ, 73 EI_COMMIT_READ_PIPE, 74 EI_COMMIT_WRITE_PIPE, 75 EI_COPYSIGN, 76 EI_COS, 77 EI_COSH, 78 EI_COSPI, 79 EI_CROSS, 80 EI_CTZ, 81 EI_DEGREES, 82 EI_DISTANCE, 83 EI_DIVIDE, 84 EI_DOT, 85 EI_ERF, 86 EI_ERFC, 87 EI_EXP, 88 EI_EXP10, 89 EI_EXP2, 90 EI_EXPM1, 91 EI_FABS, 92 EI_FAST_DISTANCE, 93 EI_FAST_LENGTH, 94 EI_FAST_NORMALIZE, 95 EI_FDIM, 96 EI_FLOOR, 97 EI_FMA, 98 EI_FMAX, 99 EI_FMIN, 100 EI_FMOD, 101 EI_FRACT, 102 EI_FREXP, 103 EI_GET_IMAGE_ARRAY_SIZE, 104 EI_GET_IMAGE_CHANNEL_DATA_TYPE, 105 EI_GET_IMAGE_CHANNEL_ORDER, 106 EI_GET_IMAGE_DIM, 107 EI_GET_IMAGE_HEIGHT, 108 EI_GET_IMAGE_WIDTH, 109 EI_GET_PIPE_MAX_PACKETS, 110 EI_GET_PIPE_NUM_PACKETS, 111 EI_HADD, 112 EI_HYPOT, 113 EI_ILOGB, 114 EI_ISEQUAL, 115 EI_ISFINITE, 116 EI_ISGREATER, 117 EI_ISGREATEREQUAL, 118 EI_ISINF, 119 EI_ISLESS, 120 EI_ISLESSEQUAL, 121 EI_ISLESSGREATER, 122 EI_ISNAN, 123 EI_ISNORMAL, 124 EI_ISNOTEQUAL, 125 EI_ISORDERED, 126 EI_ISUNORDERED, 127 EI_LDEXP, 128 EI_LENGTH, 129 EI_LGAMMA, 130 EI_LGAMMA_R, 131 EI_LOG, 132 EI_LOG10, 133 EI_LOG1P, 134 EI_LOG2, 135 EI_LOGB, 136 EI_MAD, 137 EI_MAD24, 138 EI_MAD_HI, 139 EI_MAD_SAT, 140 EI_MAX, 141 EI_MAXMAG, 142 EI_MIN, 143 EI_MINMAG, 144 EI_MIX, 145 EI_MODF, 146 EI_MUL24, 147 EI_MUL_HI, 148 EI_NAN, 149 EI_NEXTAFTER, 150 EI_NORMALIZE, 151 EI_POPCOUNT, 152 EI_POW, 153 EI_POWN, 154 EI_POWR, 155 EI_PREFETCH, 156 EI_RADIANS, 157 EI_RECIP, 158 EI_REMAINDER, 159 EI_REMQUO, 160 EI_RESERVE_READ_PIPE, 161 EI_RESERVE_WRITE_PIPE, 162 EI_RHADD, 163 EI_RINT, 164 EI_ROOTN, 165 EI_ROTATE, 166 EI_ROUND, 167 EI_RSQRT, 168 EI_SELECT, 169 EI_SHUFFLE, 170 EI_SHUFFLE2, 171 EI_SIGN, 172 EI_SIGNBIT, 173 EI_SIN, 174 EI_SINCOS, 175 EI_SINH, 176 EI_SINPI, 177 EI_SMOOTHSTEP, 178 EI_SQRT, 179 EI_STEP, 180 EI_SUB_GROUP_BROADCAST, 181 EI_SUB_GROUP_COMMIT_READ_PIPE, 182 EI_SUB_GROUP_COMMIT_WRITE_PIPE, 183 EI_SUB_GROUP_REDUCE_ADD, 184 EI_SUB_GROUP_REDUCE_MAX, 185 EI_SUB_GROUP_REDUCE_MIN, 186 EI_SUB_GROUP_RESERVE_READ_PIPE, 187 EI_SUB_GROUP_RESERVE_WRITE_PIPE, 188 EI_SUB_GROUP_SCAN_EXCLUSIVE_ADD, 189 EI_SUB_GROUP_SCAN_EXCLUSIVE_MAX, 190 EI_SUB_GROUP_SCAN_EXCLUSIVE_MIN, 191 EI_SUB_GROUP_SCAN_INCLUSIVE_ADD, 192 EI_SUB_GROUP_SCAN_INCLUSIVE_MAX, 193 EI_SUB_GROUP_SCAN_INCLUSIVE_MIN, 194 EI_SUB_SAT, 195 EI_TAN, 196 EI_TANH, 197 EI_TANPI, 198 EI_TGAMMA, 199 EI_TRUNC, 200 EI_UPSAMPLE, 201 EI_VEC_STEP, 202 EI_VSTORE, 203 EI_VSTORE16, 204 EI_VSTORE2, 205 EI_VSTORE3, 206 EI_VSTORE4, 207 EI_VSTORE8, 208 EI_WORK_GROUP_COMMIT_READ_PIPE, 209 EI_WORK_GROUP_COMMIT_WRITE_PIPE, 210 EI_WORK_GROUP_REDUCE_ADD, 211 EI_WORK_GROUP_REDUCE_MAX, 212 EI_WORK_GROUP_REDUCE_MIN, 213 EI_WORK_GROUP_RESERVE_READ_PIPE, 214 EI_WORK_GROUP_RESERVE_WRITE_PIPE, 215 EI_WORK_GROUP_SCAN_EXCLUSIVE_ADD, 216 EI_WORK_GROUP_SCAN_EXCLUSIVE_MAX, 217 EI_WORK_GROUP_SCAN_EXCLUSIVE_MIN, 218 EI_WORK_GROUP_SCAN_INCLUSIVE_ADD, 219 EI_WORK_GROUP_SCAN_INCLUSIVE_MAX, 220 EI_WORK_GROUP_SCAN_INCLUSIVE_MIN, 221 EI_WRITE_IMAGEF, 222 EI_WRITE_IMAGEI, 223 EI_WRITE_IMAGEUI, 224 EI_NCOS, 225 EI_NEXP2, 226 EI_NFMA, 227 EI_NLOG2, 228 EI_NRCP, 229 EI_NRSQRT, 230 EI_NSIN, 231 EI_NSQRT, 232 EI_FTZ, 233 EI_FLDEXP, 234 EI_CLASS, 235 EI_RCBRT, 236 EI_LAST_MANGLED = 237 EI_RCBRT, /* The last library function with mangled name */ 238 239 // Library functions with unmangled name. 240 EI_READ_PIPE_2, 241 EI_READ_PIPE_4, 242 EI_WRITE_PIPE_2, 243 EI_WRITE_PIPE_4, 244 245 EX_INTRINSICS_COUNT 246 }; 247 248 enum ENamePrefix { 249 NOPFX, 250 NATIVE, 251 HALF 252 }; 253 254 enum EType { 255 B8 = 1, 256 B16 = 2, 257 B32 = 3, 258 B64 = 4, 259 SIZE_MASK = 7, 260 FLOAT = 0x10, 261 INT = 0x20, 262 UINT = 0x30, 263 BASE_TYPE_MASK = 0x30, 264 U8 = UINT | B8, 265 U16 = UINT | B16, 266 U32 = UINT | B32, 267 U64 = UINT | B64, 268 I8 = INT | B8, 269 I16 = INT | B16, 270 I32 = INT | B32, 271 I64 = INT | B64, 272 F16 = FLOAT | B16, 273 F32 = FLOAT | B32, 274 F64 = FLOAT | B64, 275 IMG1DA = 0x80, 276 IMG1DB, 277 IMG2DA, 278 IMG1D, 279 IMG2D, 280 IMG3D, 281 SAMPLER, 282 EVENT, 283 DUMMY 284 }; 285 286 enum EPtrKind { 287 BYVALUE = 0, 288 ADDR_SPACE = 0xF, // Address space takes value 0x1 ~ 0xF. 289 CONST = 0x10, 290 VOLATILE = 0x20 291 }; 292 293 struct Param { 294 unsigned char ArgType = 0; 295 unsigned char VectorSize = 1; 296 unsigned char PtrKind = 0; 297 298 unsigned char Reserved = 0; 299 300 void reset() { 301 ArgType = 0; 302 VectorSize = 1; 303 PtrKind = 0; 304 } 305 306 static Param getIntN(unsigned char NumElts) { 307 return Param{I32, NumElts, 0, 0}; 308 } 309 310 static Param getFromTy(Type *Ty, bool Signed); 311 312 template <typename Stream> 313 void mangleItanium(Stream& os); 314 }; 315 static bool isMangled(EFuncId Id) { 316 return static_cast<unsigned>(Id) <= static_cast<unsigned>(EI_LAST_MANGLED); 317 } 318 319 static unsigned getEPtrKindFromAddrSpace(unsigned AS) { 320 assert(((AS + 1) & ~ADDR_SPACE) == 0); 321 return AS + 1; 322 } 323 324 static unsigned getAddrSpaceFromEPtrKind(unsigned Kind) { 325 Kind = Kind & ADDR_SPACE; 326 assert(Kind >= 1); 327 return Kind - 1; 328 } 329 }; 330 331 class AMDGPULibFuncImpl : public AMDGPULibFuncBase { 332 public: 333 AMDGPULibFuncImpl() = default; 334 virtual ~AMDGPULibFuncImpl() = default; 335 336 /// Get unmangled name for mangled library function and name for unmangled 337 /// library function. 338 virtual std::string getName() const = 0; 339 virtual unsigned getNumArgs() const = 0; 340 EFuncId getId() const { return FuncId; } 341 ENamePrefix getPrefix() const { return FKind; } 342 343 bool isMangled() const { return AMDGPULibFuncBase::isMangled(FuncId); } 344 345 void setId(EFuncId id) { FuncId = id; } 346 virtual bool parseFuncName(StringRef &mangledName) = 0; 347 348 /// \return The mangled function name for mangled library functions 349 /// and unmangled function name for unmangled library functions. 350 virtual std::string mangle() const = 0; 351 352 void setName(StringRef N) { Name = std::string(N); } 353 void setPrefix(ENamePrefix pfx) { FKind = pfx; } 354 355 virtual FunctionType *getFunctionType(Module &M) const = 0; 356 357 protected: 358 EFuncId FuncId; 359 std::string Name; 360 ENamePrefix FKind = NOPFX; 361 }; 362 363 /// Wrapper class for AMDGPULIbFuncImpl 364 class AMDGPULibFunc : public AMDGPULibFuncBase { 365 public: 366 explicit AMDGPULibFunc() : Impl(std::unique_ptr<AMDGPULibFuncImpl>()) {} 367 AMDGPULibFunc(const AMDGPULibFunc &F); 368 /// Clone a mangled library func with the Id \p Id and argument info from \p 369 /// CopyFrom. 370 explicit AMDGPULibFunc(EFuncId Id, const AMDGPULibFunc &CopyFrom); 371 explicit AMDGPULibFunc(EFuncId Id, FunctionType *FT, bool SignedInts); 372 373 /// Construct an unmangled library function on the fly. 374 explicit AMDGPULibFunc(StringRef FName, FunctionType *FT); 375 376 AMDGPULibFunc &operator=(const AMDGPULibFunc &F); 377 378 /// Get unmangled name for mangled library function and name for unmangled 379 /// library function. 380 std::string getName() const { return Impl->getName(); } 381 unsigned getNumArgs() const { return Impl->getNumArgs(); } 382 EFuncId getId() const { return Impl->getId(); } 383 ENamePrefix getPrefix() const { return Impl->getPrefix(); } 384 /// Get leading parameters for mangled lib functions. 385 Param *getLeads(); 386 const Param *getLeads() const; 387 388 bool isMangled() const { return Impl->isMangled(); } 389 void setId(EFuncId Id) { Impl->setId(Id); } 390 bool parseFuncName(StringRef &MangledName) { 391 return Impl->parseFuncName(MangledName); 392 } 393 394 // Validate the call type matches the expected libfunc type. 395 bool isCompatibleSignature(const FunctionType *FuncTy) const; 396 397 /// \return The mangled function name for mangled library functions 398 /// and unmangled function name for unmangled library functions. 399 std::string mangle() const { return Impl->mangle(); } 400 401 void setName(StringRef N) { Impl->setName(N); } 402 void setPrefix(ENamePrefix PFX) { Impl->setPrefix(PFX); } 403 404 FunctionType *getFunctionType(Module &M) const { 405 return Impl->getFunctionType(M); 406 } 407 static Function *getFunction(llvm::Module *M, const AMDGPULibFunc &fInfo); 408 409 static FunctionCallee getOrInsertFunction(llvm::Module *M, 410 const AMDGPULibFunc &fInfo); 411 static bool parse(StringRef MangledName, AMDGPULibFunc &Ptr); 412 413 private: 414 /// Initialize as a mangled library function. 415 void initMangled(); 416 std::unique_ptr<AMDGPULibFuncImpl> Impl; 417 }; 418 419 class AMDGPUMangledLibFunc : public AMDGPULibFuncImpl { 420 public: 421 Param Leads[2]; 422 423 explicit AMDGPUMangledLibFunc(); 424 explicit AMDGPUMangledLibFunc(EFuncId id, 425 const AMDGPUMangledLibFunc ©From); 426 explicit AMDGPUMangledLibFunc(EFuncId id, FunctionType *FT, 427 bool SignedInts = true); 428 429 std::string getName() const override; 430 unsigned getNumArgs() const override; 431 FunctionType *getFunctionType(Module &M) const override; 432 static StringRef getUnmangledName(StringRef MangledName); 433 434 bool parseFuncName(StringRef &mangledName) override; 435 436 // Methods for support type inquiry through isa, cast, and dyn_cast: 437 static bool classof(const AMDGPULibFuncImpl *F) { return F->isMangled(); } 438 439 std::string mangle() const override; 440 441 private: 442 std::string mangleNameItanium() const; 443 444 std::string mangleName(StringRef Name) const; 445 bool parseUnmangledName(StringRef MangledName); 446 447 template <typename Stream> void writeName(Stream &OS) const; 448 }; 449 450 class AMDGPUUnmangledLibFunc : public AMDGPULibFuncImpl { 451 FunctionType *FuncTy; 452 453 public: 454 explicit AMDGPUUnmangledLibFunc(); 455 explicit AMDGPUUnmangledLibFunc(StringRef FName, FunctionType *FT) { 456 Name = std::string(FName); 457 FuncTy = FT; 458 } 459 std::string getName() const override { return Name; } 460 unsigned getNumArgs() const override; 461 FunctionType *getFunctionType(Module &M) const override { return FuncTy; } 462 463 bool parseFuncName(StringRef &Name) override; 464 465 // Methods for support type inquiry through isa, cast, and dyn_cast: 466 static bool classof(const AMDGPULibFuncImpl *F) { return !F->isMangled(); } 467 468 std::string mangle() const override { return Name; } 469 470 void setFunctionType(FunctionType *FT) { FuncTy = FT; } 471 }; 472 } 473 #endif // _AMDGPU_LIBFUNC_H_ 474