1 //===- NVPTXUtilities.cpp - Utility Functions -----------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains miscellaneous utility functions 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "NVPTXUtilities.h" 14 #include "NVPTX.h" 15 #include "NVPTXTargetMachine.h" 16 #include "llvm/IR/Constants.h" 17 #include "llvm/IR/Function.h" 18 #include "llvm/IR/GlobalVariable.h" 19 #include "llvm/IR/InstIterator.h" 20 #include "llvm/IR/Module.h" 21 #include "llvm/IR/Operator.h" 22 #include "llvm/Support/Alignment.h" 23 #include "llvm/Support/Mutex.h" 24 #include <algorithm> 25 #include <cstring> 26 #include <map> 27 #include <mutex> 28 #include <optional> 29 #include <string> 30 #include <vector> 31 32 namespace llvm { 33 34 namespace { 35 typedef std::map<std::string, std::vector<unsigned> > key_val_pair_t; 36 typedef std::map<const GlobalValue *, key_val_pair_t> global_val_annot_t; 37 38 struct AnnotationCache { 39 sys::Mutex Lock; 40 std::map<const Module *, global_val_annot_t> Cache; 41 }; 42 43 AnnotationCache &getAnnotationCache() { 44 static AnnotationCache AC; 45 return AC; 46 } 47 } // anonymous namespace 48 49 void clearAnnotationCache(const Module *Mod) { 50 auto &AC = getAnnotationCache(); 51 std::lock_guard<sys::Mutex> Guard(AC.Lock); 52 AC.Cache.erase(Mod); 53 } 54 55 static void readIntVecFromMDNode(const MDNode *MetadataNode, 56 std::vector<unsigned> &Vec) { 57 for (unsigned i = 0, e = MetadataNode->getNumOperands(); i != e; ++i) { 58 ConstantInt *Val = 59 mdconst::extract<ConstantInt>(MetadataNode->getOperand(i)); 60 Vec.push_back(Val->getZExtValue()); 61 } 62 } 63 64 static void cacheAnnotationFromMD(const MDNode *MetadataNode, 65 key_val_pair_t &retval) { 66 auto &AC = getAnnotationCache(); 67 std::lock_guard<sys::Mutex> Guard(AC.Lock); 68 assert(MetadataNode && "Invalid mdnode for annotation"); 69 assert((MetadataNode->getNumOperands() % 2) == 1 && 70 "Invalid number of operands"); 71 // start index = 1, to skip the global variable key 72 // increment = 2, to skip the value for each property-value pairs 73 for (unsigned i = 1, e = MetadataNode->getNumOperands(); i != e; i += 2) { 74 // property 75 const MDString *prop = dyn_cast<MDString>(MetadataNode->getOperand(i)); 76 assert(prop && "Annotation property not a string"); 77 std::string Key = prop->getString().str(); 78 79 // value 80 if (ConstantInt *Val = mdconst::dyn_extract<ConstantInt>( 81 MetadataNode->getOperand(i + 1))) { 82 retval[Key].push_back(Val->getZExtValue()); 83 } else if (MDNode *VecMd = 84 dyn_cast<MDNode>(MetadataNode->getOperand(i + 1))) { 85 // note: only "grid_constant" annotations support vector MDNodes. 86 // assert: there can only exist one unique key value pair of 87 // the form (string key, MDNode node). Operands of such a node 88 // shall always be unsigned ints. 89 if (retval.find(Key) == retval.end()) { 90 readIntVecFromMDNode(VecMd, retval[Key]); 91 continue; 92 } 93 } else { 94 llvm_unreachable("Value operand not a constant int or an mdnode"); 95 } 96 } 97 } 98 99 static void cacheAnnotationFromMD(const Module *m, const GlobalValue *gv) { 100 auto &AC = getAnnotationCache(); 101 std::lock_guard<sys::Mutex> Guard(AC.Lock); 102 NamedMDNode *NMD = m->getNamedMetadata("nvvm.annotations"); 103 if (!NMD) 104 return; 105 key_val_pair_t tmp; 106 for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) { 107 const MDNode *elem = NMD->getOperand(i); 108 109 GlobalValue *entity = 110 mdconst::dyn_extract_or_null<GlobalValue>(elem->getOperand(0)); 111 // entity may be null due to DCE 112 if (!entity) 113 continue; 114 if (entity != gv) 115 continue; 116 117 // accumulate annotations for entity in tmp 118 cacheAnnotationFromMD(elem, tmp); 119 } 120 121 if (tmp.empty()) // no annotations for this gv 122 return; 123 124 if (AC.Cache.find(m) != AC.Cache.end()) 125 AC.Cache[m][gv] = std::move(tmp); 126 else { 127 global_val_annot_t tmp1; 128 tmp1[gv] = std::move(tmp); 129 AC.Cache[m] = std::move(tmp1); 130 } 131 } 132 133 bool findOneNVVMAnnotation(const GlobalValue *gv, const std::string &prop, 134 unsigned &retval) { 135 auto &AC = getAnnotationCache(); 136 std::lock_guard<sys::Mutex> Guard(AC.Lock); 137 const Module *m = gv->getParent(); 138 if (AC.Cache.find(m) == AC.Cache.end()) 139 cacheAnnotationFromMD(m, gv); 140 else if (AC.Cache[m].find(gv) == AC.Cache[m].end()) 141 cacheAnnotationFromMD(m, gv); 142 if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end()) 143 return false; 144 retval = AC.Cache[m][gv][prop][0]; 145 return true; 146 } 147 148 static std::optional<unsigned> 149 findOneNVVMAnnotation(const GlobalValue &GV, const std::string &PropName) { 150 unsigned RetVal; 151 if (findOneNVVMAnnotation(&GV, PropName, RetVal)) 152 return RetVal; 153 return std::nullopt; 154 } 155 156 bool findAllNVVMAnnotation(const GlobalValue *gv, const std::string &prop, 157 std::vector<unsigned> &retval) { 158 auto &AC = getAnnotationCache(); 159 std::lock_guard<sys::Mutex> Guard(AC.Lock); 160 const Module *m = gv->getParent(); 161 if (AC.Cache.find(m) == AC.Cache.end()) 162 cacheAnnotationFromMD(m, gv); 163 else if (AC.Cache[m].find(gv) == AC.Cache[m].end()) 164 cacheAnnotationFromMD(m, gv); 165 if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end()) 166 return false; 167 retval = AC.Cache[m][gv][prop]; 168 return true; 169 } 170 171 bool isTexture(const Value &val) { 172 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) { 173 unsigned Annot; 174 if (findOneNVVMAnnotation(gv, "texture", Annot)) { 175 assert((Annot == 1) && "Unexpected annotation on a texture symbol"); 176 return true; 177 } 178 } 179 return false; 180 } 181 182 bool isSurface(const Value &val) { 183 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) { 184 unsigned Annot; 185 if (findOneNVVMAnnotation(gv, "surface", Annot)) { 186 assert((Annot == 1) && "Unexpected annotation on a surface symbol"); 187 return true; 188 } 189 } 190 return false; 191 } 192 193 static bool argHasNVVMAnnotation(const Value &Val, 194 const std::string &Annotation, 195 const bool StartArgIndexAtOne = false) { 196 if (const Argument *Arg = dyn_cast<Argument>(&Val)) { 197 const Function *Func = Arg->getParent(); 198 std::vector<unsigned> Annot; 199 if (findAllNVVMAnnotation(Func, Annotation, Annot)) { 200 const unsigned BaseOffset = StartArgIndexAtOne ? 1 : 0; 201 if (is_contained(Annot, BaseOffset + Arg->getArgNo())) { 202 return true; 203 } 204 } 205 } 206 return false; 207 } 208 209 bool isParamGridConstant(const Value &V) { 210 if (const Argument *Arg = dyn_cast<Argument>(&V)) { 211 // "grid_constant" counts argument indices starting from 1 212 if (Arg->hasByValAttr() && 213 argHasNVVMAnnotation(*Arg, "grid_constant", 214 /*StartArgIndexAtOne*/ true)) { 215 assert(isKernelFunction(*Arg->getParent()) && 216 "only kernel arguments can be grid_constant"); 217 return true; 218 } 219 } 220 return false; 221 } 222 223 bool isSampler(const Value &val) { 224 const char *AnnotationName = "sampler"; 225 226 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) { 227 unsigned Annot; 228 if (findOneNVVMAnnotation(gv, AnnotationName, Annot)) { 229 assert((Annot == 1) && "Unexpected annotation on a sampler symbol"); 230 return true; 231 } 232 } 233 return argHasNVVMAnnotation(val, AnnotationName); 234 } 235 236 bool isImageReadOnly(const Value &val) { 237 return argHasNVVMAnnotation(val, "rdoimage"); 238 } 239 240 bool isImageWriteOnly(const Value &val) { 241 return argHasNVVMAnnotation(val, "wroimage"); 242 } 243 244 bool isImageReadWrite(const Value &val) { 245 return argHasNVVMAnnotation(val, "rdwrimage"); 246 } 247 248 bool isImage(const Value &val) { 249 return isImageReadOnly(val) || isImageWriteOnly(val) || isImageReadWrite(val); 250 } 251 252 bool isManaged(const Value &val) { 253 if(const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) { 254 unsigned Annot; 255 if (findOneNVVMAnnotation(gv, "managed", Annot)) { 256 assert((Annot == 1) && "Unexpected annotation on a managed symbol"); 257 return true; 258 } 259 } 260 return false; 261 } 262 263 std::string getTextureName(const Value &val) { 264 assert(val.hasName() && "Found texture variable with no name"); 265 return std::string(val.getName()); 266 } 267 268 std::string getSurfaceName(const Value &val) { 269 assert(val.hasName() && "Found surface variable with no name"); 270 return std::string(val.getName()); 271 } 272 273 std::string getSamplerName(const Value &val) { 274 assert(val.hasName() && "Found sampler variable with no name"); 275 return std::string(val.getName()); 276 } 277 278 std::optional<unsigned> getMaxNTIDx(const Function &F) { 279 return findOneNVVMAnnotation(F, "maxntidx"); 280 } 281 282 std::optional<unsigned> getMaxNTIDy(const Function &F) { 283 return findOneNVVMAnnotation(F, "maxntidy"); 284 } 285 286 std::optional<unsigned> getMaxNTIDz(const Function &F) { 287 return findOneNVVMAnnotation(F, "maxntidz"); 288 } 289 290 std::optional<unsigned> getMaxNTID(const Function &F) { 291 // Note: The semantics here are a bit strange. The PTX ISA states the 292 // following (11.4.2. Performance-Tuning Directives: .maxntid): 293 // 294 // Note that this directive guarantees that the total number of threads does 295 // not exceed the maximum, but does not guarantee that the limit in any 296 // particular dimension is not exceeded. 297 std::optional<unsigned> MaxNTIDx = getMaxNTIDx(F); 298 std::optional<unsigned> MaxNTIDy = getMaxNTIDy(F); 299 std::optional<unsigned> MaxNTIDz = getMaxNTIDz(F); 300 if (MaxNTIDx || MaxNTIDy || MaxNTIDz) 301 return MaxNTIDx.value_or(1) * MaxNTIDy.value_or(1) * MaxNTIDz.value_or(1); 302 return std::nullopt; 303 } 304 305 bool getMaxClusterRank(const Function &F, unsigned &x) { 306 return findOneNVVMAnnotation(&F, "maxclusterrank", x); 307 } 308 309 std::optional<unsigned> getReqNTIDx(const Function &F) { 310 return findOneNVVMAnnotation(F, "reqntidx"); 311 } 312 313 std::optional<unsigned> getReqNTIDy(const Function &F) { 314 return findOneNVVMAnnotation(F, "reqntidy"); 315 } 316 317 std::optional<unsigned> getReqNTIDz(const Function &F) { 318 return findOneNVVMAnnotation(F, "reqntidz"); 319 } 320 321 std::optional<unsigned> getReqNTID(const Function &F) { 322 // Note: The semantics here are a bit strange. See getMaxNTID. 323 std::optional<unsigned> ReqNTIDx = getReqNTIDx(F); 324 std::optional<unsigned> ReqNTIDy = getReqNTIDy(F); 325 std::optional<unsigned> ReqNTIDz = getReqNTIDz(F); 326 if (ReqNTIDx || ReqNTIDy || ReqNTIDz) 327 return ReqNTIDx.value_or(1) * ReqNTIDy.value_or(1) * ReqNTIDz.value_or(1); 328 return std::nullopt; 329 } 330 331 bool getMinCTASm(const Function &F, unsigned &x) { 332 return findOneNVVMAnnotation(&F, "minctasm", x); 333 } 334 335 bool getMaxNReg(const Function &F, unsigned &x) { 336 return findOneNVVMAnnotation(&F, "maxnreg", x); 337 } 338 339 bool isKernelFunction(const Function &F) { 340 unsigned x = 0; 341 if (!findOneNVVMAnnotation(&F, "kernel", x)) { 342 // There is no NVVM metadata, check the calling convention 343 return F.getCallingConv() == CallingConv::PTX_Kernel; 344 } 345 return (x == 1); 346 } 347 348 MaybeAlign getAlign(const Function &F, unsigned Index) { 349 // First check the alignstack metadata 350 if (MaybeAlign StackAlign = 351 F.getAttributes().getAttributes(Index).getStackAlignment()) 352 return StackAlign; 353 354 // If that is missing, check the legacy nvvm metadata 355 std::vector<unsigned> Vs; 356 bool retval = findAllNVVMAnnotation(&F, "align", Vs); 357 if (!retval) 358 return std::nullopt; 359 for (unsigned V : Vs) 360 if ((V >> 16) == Index) 361 return Align(V & 0xFFFF); 362 363 return std::nullopt; 364 } 365 366 MaybeAlign getAlign(const CallInst &I, unsigned Index) { 367 // First check the alignstack metadata 368 if (MaybeAlign StackAlign = 369 I.getAttributes().getAttributes(Index).getStackAlignment()) 370 return StackAlign; 371 372 // If that is missing, check the legacy nvvm metadata 373 if (MDNode *alignNode = I.getMetadata("callalign")) { 374 for (int i = 0, n = alignNode->getNumOperands(); i < n; i++) { 375 if (const ConstantInt *CI = 376 mdconst::dyn_extract<ConstantInt>(alignNode->getOperand(i))) { 377 unsigned V = CI->getZExtValue(); 378 if ((V >> 16) == Index) 379 return Align(V & 0xFFFF); 380 if ((V >> 16) > Index) 381 return std::nullopt; 382 } 383 } 384 } 385 return std::nullopt; 386 } 387 388 Function *getMaybeBitcastedCallee(const CallBase *CB) { 389 return dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts()); 390 } 391 392 bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM) { 393 const auto &ST = 394 *static_cast<const NVPTXTargetMachine &>(TM).getSubtargetImpl(); 395 if (!ST.hasNoReturn()) 396 return false; 397 398 assert((isa<Function>(V) || isa<CallInst>(V)) && 399 "Expect either a call instruction or a function"); 400 401 if (const CallInst *CallI = dyn_cast<CallInst>(V)) 402 return CallI->doesNotReturn() && 403 CallI->getFunctionType()->getReturnType()->isVoidTy(); 404 405 const Function *F = cast<Function>(V); 406 return F->doesNotReturn() && 407 F->getFunctionType()->getReturnType()->isVoidTy() && 408 !isKernelFunction(*F); 409 } 410 411 bool Isv2x16VT(EVT VT) { 412 return (VT == MVT::v2f16 || VT == MVT::v2bf16 || VT == MVT::v2i16); 413 } 414 415 } // namespace llvm 416