1 //=== AMDGPUPrintfRuntimeBinding.cpp - OpenCL printf implementation -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // \file 9 // 10 // The pass bind printfs to a kernel arg pointer that will be bound to a buffer 11 // later by the runtime. 12 // 13 // This pass traverses the functions in the module and converts 14 // each call to printf to a sequence of operations that 15 // store the following into the printf buffer: 16 // - format string (passed as a module's metadata unique ID) 17 // - bitwise copies of printf arguments 18 // The backend passes will need to store metadata in the kernel 19 //===----------------------------------------------------------------------===// 20 21 #include "AMDGPU.h" 22 #include "llvm/ADT/StringExtras.h" 23 #include "llvm/Analysis/ValueTracking.h" 24 #include "llvm/IR/DiagnosticInfo.h" 25 #include "llvm/IR/Dominators.h" 26 #include "llvm/IR/IRBuilder.h" 27 #include "llvm/IR/Instructions.h" 28 #include "llvm/IR/Module.h" 29 #include "llvm/InitializePasses.h" 30 #include "llvm/Support/DataExtractor.h" 31 #include "llvm/TargetParser/Triple.h" 32 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 33 34 using namespace llvm; 35 36 #define DEBUG_TYPE "printfToRuntime" 37 enum { DWORD_ALIGN = 4 }; 38 39 namespace { 40 class AMDGPUPrintfRuntimeBinding final : public ModulePass { 41 42 public: 43 static char ID; 44 45 explicit AMDGPUPrintfRuntimeBinding(); 46 47 private: 48 bool runOnModule(Module &M) override; 49 }; 50 51 class AMDGPUPrintfRuntimeBindingImpl { 52 public: 53 AMDGPUPrintfRuntimeBindingImpl() = default; 54 bool run(Module &M); 55 56 private: 57 void getConversionSpecifiers(SmallVectorImpl<char> &OpConvSpecifiers, 58 StringRef fmt, size_t num_ops) const; 59 60 bool lowerPrintfForGpu(Module &M); 61 62 const DataLayout *TD; 63 SmallVector<CallInst *, 32> Printfs; 64 }; 65 } // namespace 66 67 char AMDGPUPrintfRuntimeBinding::ID = 0; 68 69 INITIALIZE_PASS_BEGIN(AMDGPUPrintfRuntimeBinding, 70 "amdgpu-printf-runtime-binding", "AMDGPU Printf lowering", 71 false, false) 72 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 73 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 74 INITIALIZE_PASS_END(AMDGPUPrintfRuntimeBinding, "amdgpu-printf-runtime-binding", 75 "AMDGPU Printf lowering", false, false) 76 77 char &llvm::AMDGPUPrintfRuntimeBindingID = AMDGPUPrintfRuntimeBinding::ID; 78 79 namespace llvm { 80 ModulePass *createAMDGPUPrintfRuntimeBinding() { 81 return new AMDGPUPrintfRuntimeBinding(); 82 } 83 } // namespace llvm 84 85 AMDGPUPrintfRuntimeBinding::AMDGPUPrintfRuntimeBinding() : ModulePass(ID) { 86 initializeAMDGPUPrintfRuntimeBindingPass(*PassRegistry::getPassRegistry()); 87 } 88 89 void AMDGPUPrintfRuntimeBindingImpl::getConversionSpecifiers( 90 SmallVectorImpl<char> &OpConvSpecifiers, StringRef Fmt, 91 size_t NumOps) const { 92 // not all format characters are collected. 93 // At this time the format characters of interest 94 // are %p and %s, which use to know if we 95 // are either storing a literal string or a 96 // pointer to the printf buffer. 97 static const char ConvSpecifiers[] = "cdieEfgGaosuxXp"; 98 size_t CurFmtSpecifierIdx = 0; 99 size_t PrevFmtSpecifierIdx = 0; 100 101 while ((CurFmtSpecifierIdx = Fmt.find_first_of( 102 ConvSpecifiers, CurFmtSpecifierIdx)) != StringRef::npos) { 103 bool ArgDump = false; 104 StringRef CurFmt = Fmt.substr(PrevFmtSpecifierIdx, 105 CurFmtSpecifierIdx - PrevFmtSpecifierIdx); 106 size_t pTag = CurFmt.find_last_of('%'); 107 if (pTag != StringRef::npos) { 108 ArgDump = true; 109 while (pTag && CurFmt[--pTag] == '%') { 110 ArgDump = !ArgDump; 111 } 112 } 113 114 if (ArgDump) 115 OpConvSpecifiers.push_back(Fmt[CurFmtSpecifierIdx]); 116 117 PrevFmtSpecifierIdx = ++CurFmtSpecifierIdx; 118 } 119 } 120 121 static bool shouldPrintAsStr(char Specifier, Type *OpType) { 122 return Specifier == 's' && isa<PointerType>(OpType); 123 } 124 125 constexpr StringLiteral NonLiteralStr("???"); 126 static_assert(NonLiteralStr.size() == 3); 127 128 static StringRef getAsConstantStr(Value *V) { 129 StringRef S; 130 if (!getConstantStringInfo(V, S)) 131 S = NonLiteralStr; 132 133 return S; 134 } 135 136 static void diagnoseInvalidFormatString(const CallBase *CI) { 137 DiagnosticInfoUnsupported UnsupportedFormatStr( 138 *CI->getParent()->getParent(), 139 "printf format string must be a trivially resolved constant string " 140 "global variable", 141 CI->getDebugLoc()); 142 CI->getContext().diagnose(UnsupportedFormatStr); 143 } 144 145 bool AMDGPUPrintfRuntimeBindingImpl::lowerPrintfForGpu(Module &M) { 146 LLVMContext &Ctx = M.getContext(); 147 IRBuilder<> Builder(Ctx); 148 Type *I32Ty = Type::getInt32Ty(Ctx); 149 150 // Instead of creating global variables, the printf format strings are 151 // extracted and passed as metadata. This avoids polluting llvm's symbol 152 // tables in this module. Metadata is going to be extracted by the backend 153 // passes and inserted into the OpenCL binary as appropriate. 154 NamedMDNode *metaD = M.getOrInsertNamedMetadata("llvm.printf.fmts"); 155 unsigned UniqID = metaD->getNumOperands(); 156 157 for (auto *CI : Printfs) { 158 unsigned NumOps = CI->arg_size(); 159 160 SmallString<16> OpConvSpecifiers; 161 Value *Op = CI->getArgOperand(0); 162 163 StringRef FormatStr; 164 if (!getConstantStringInfo(Op, FormatStr)) { 165 Value *Stripped = Op->stripPointerCasts(); 166 if (!isa<UndefValue>(Stripped) && !isa<ConstantPointerNull>(Stripped)) 167 diagnoseInvalidFormatString(CI); 168 continue; 169 } 170 171 // We need this call to ascertain that we are printing a string or a 172 // pointer. It takes out the specifiers and fills up the first arg. 173 getConversionSpecifiers(OpConvSpecifiers, FormatStr, NumOps - 1); 174 175 // Add metadata for the string 176 std::string AStreamHolder; 177 raw_string_ostream Sizes(AStreamHolder); 178 int Sum = DWORD_ALIGN; 179 Sizes << CI->arg_size() - 1; 180 Sizes << ':'; 181 for (unsigned ArgCount = 1; 182 ArgCount < CI->arg_size() && ArgCount <= OpConvSpecifiers.size(); 183 ArgCount++) { 184 Value *Arg = CI->getArgOperand(ArgCount); 185 Type *ArgType = Arg->getType(); 186 unsigned ArgSize = TD->getTypeAllocSize(ArgType); 187 // 188 // ArgSize by design should be a multiple of DWORD_ALIGN, 189 // expand the arguments that do not follow this rule. 190 // 191 if (ArgSize % DWORD_ALIGN != 0) { 192 Type *ResType = Type::getInt32Ty(Ctx); 193 if (auto *VecType = dyn_cast<VectorType>(ArgType)) 194 ResType = VectorType::get(ResType, VecType->getElementCount()); 195 Builder.SetInsertPoint(CI); 196 Builder.SetCurrentDebugLocation(CI->getDebugLoc()); 197 198 if (ArgType->isFloatingPointTy()) { 199 Arg = Builder.CreateBitCast( 200 Arg, 201 IntegerType::getIntNTy(Ctx, ArgType->getPrimitiveSizeInBits())); 202 } 203 204 if (OpConvSpecifiers[ArgCount - 1] == 'x' || 205 OpConvSpecifiers[ArgCount - 1] == 'X' || 206 OpConvSpecifiers[ArgCount - 1] == 'u' || 207 OpConvSpecifiers[ArgCount - 1] == 'o') 208 Arg = Builder.CreateZExt(Arg, ResType); 209 else 210 Arg = Builder.CreateSExt(Arg, ResType); 211 ArgType = Arg->getType(); 212 ArgSize = TD->getTypeAllocSize(ArgType); 213 CI->setOperand(ArgCount, Arg); 214 } 215 if (OpConvSpecifiers[ArgCount - 1] == 'f') { 216 ConstantFP *FpCons = dyn_cast<ConstantFP>(Arg); 217 if (FpCons) 218 ArgSize = 4; 219 else { 220 FPExtInst *FpExt = dyn_cast<FPExtInst>(Arg); 221 if (FpExt && FpExt->getType()->isDoubleTy() && 222 FpExt->getOperand(0)->getType()->isFloatTy()) 223 ArgSize = 4; 224 } 225 } 226 if (shouldPrintAsStr(OpConvSpecifiers[ArgCount - 1], ArgType)) 227 ArgSize = alignTo(getAsConstantStr(Arg).size() + 1, 4); 228 229 LLVM_DEBUG(dbgs() << "Printf ArgSize (in buffer) = " << ArgSize 230 << " for type: " << *ArgType << '\n'); 231 Sizes << ArgSize << ':'; 232 Sum += ArgSize; 233 } 234 LLVM_DEBUG(dbgs() << "Printf format string in source = " << FormatStr 235 << '\n'); 236 for (char C : FormatStr) { 237 // Rest of the C escape sequences (e.g. \') are handled correctly 238 // by the MDParser 239 switch (C) { 240 case '\a': 241 Sizes << "\\a"; 242 break; 243 case '\b': 244 Sizes << "\\b"; 245 break; 246 case '\f': 247 Sizes << "\\f"; 248 break; 249 case '\n': 250 Sizes << "\\n"; 251 break; 252 case '\r': 253 Sizes << "\\r"; 254 break; 255 case '\v': 256 Sizes << "\\v"; 257 break; 258 case ':': 259 // ':' cannot be scanned by Flex, as it is defined as a delimiter 260 // Replace it with it's octal representation \72 261 Sizes << "\\72"; 262 break; 263 default: 264 Sizes << C; 265 break; 266 } 267 } 268 269 // Insert the printf_alloc call 270 Builder.SetInsertPoint(CI); 271 Builder.SetCurrentDebugLocation(CI->getDebugLoc()); 272 273 AttributeList Attr = AttributeList::get(Ctx, AttributeList::FunctionIndex, 274 Attribute::NoUnwind); 275 276 Type *SizetTy = Type::getInt32Ty(Ctx); 277 278 Type *Tys_alloc[1] = {SizetTy}; 279 Type *I8Ty = Type::getInt8Ty(Ctx); 280 Type *I8Ptr = PointerType::get(I8Ty, 1); 281 FunctionType *FTy_alloc = FunctionType::get(I8Ptr, Tys_alloc, false); 282 FunctionCallee PrintfAllocFn = 283 M.getOrInsertFunction(StringRef("__printf_alloc"), FTy_alloc, Attr); 284 285 LLVM_DEBUG(dbgs() << "Printf metadata = " << Sizes.str() << '\n'); 286 std::string fmtstr = itostr(++UniqID) + ":" + Sizes.str(); 287 MDString *fmtStrArray = MDString::get(Ctx, fmtstr); 288 289 MDNode *myMD = MDNode::get(Ctx, fmtStrArray); 290 metaD->addOperand(myMD); 291 Value *sumC = ConstantInt::get(SizetTy, Sum, false); 292 SmallVector<Value *, 1> alloc_args; 293 alloc_args.push_back(sumC); 294 CallInst *pcall = CallInst::Create(PrintfAllocFn, alloc_args, 295 "printf_alloc_fn", CI->getIterator()); 296 297 // 298 // Insert code to split basicblock with a 299 // piece of hammock code. 300 // basicblock splits after buffer overflow check 301 // 302 ConstantPointerNull *zeroIntPtr = 303 ConstantPointerNull::get(PointerType::get(I8Ty, 1)); 304 auto *cmp = cast<ICmpInst>(Builder.CreateICmpNE(pcall, zeroIntPtr, "")); 305 if (!CI->use_empty()) { 306 Value *result = 307 Builder.CreateSExt(Builder.CreateNot(cmp), I32Ty, "printf_res"); 308 CI->replaceAllUsesWith(result); 309 } 310 SplitBlock(CI->getParent(), cmp); 311 Instruction *Brnch = 312 SplitBlockAndInsertIfThen(cmp, cmp->getNextNode(), false); 313 BasicBlock::iterator BrnchPoint = Brnch->getIterator(); 314 315 Builder.SetInsertPoint(Brnch); 316 317 // store unique printf id in the buffer 318 // 319 GetElementPtrInst *BufferIdx = GetElementPtrInst::Create( 320 I8Ty, pcall, ConstantInt::get(Ctx, APInt(32, 0)), "PrintBuffID", 321 BrnchPoint); 322 323 Type *idPointer = PointerType::get(I32Ty, AMDGPUAS::GLOBAL_ADDRESS); 324 Value *id_gep_cast = 325 new BitCastInst(BufferIdx, idPointer, "PrintBuffIdCast", BrnchPoint); 326 327 new StoreInst(ConstantInt::get(I32Ty, UniqID), id_gep_cast, BrnchPoint); 328 329 // 1st 4 bytes hold the printf_id 330 // the following GEP is the buffer pointer 331 BufferIdx = GetElementPtrInst::Create(I8Ty, pcall, 332 ConstantInt::get(Ctx, APInt(32, 4)), 333 "PrintBuffGep", BrnchPoint); 334 335 Type *Int32Ty = Type::getInt32Ty(Ctx); 336 for (unsigned ArgCount = 1; 337 ArgCount < CI->arg_size() && ArgCount <= OpConvSpecifiers.size(); 338 ArgCount++) { 339 Value *Arg = CI->getArgOperand(ArgCount); 340 Type *ArgType = Arg->getType(); 341 SmallVector<Value *, 32> WhatToStore; 342 if (ArgType->isFPOrFPVectorTy() && !isa<VectorType>(ArgType)) { 343 if (OpConvSpecifiers[ArgCount - 1] == 'f') { 344 if (auto *FpCons = dyn_cast<ConstantFP>(Arg)) { 345 APFloat Val(FpCons->getValueAPF()); 346 bool Lost = false; 347 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, 348 &Lost); 349 Arg = ConstantFP::get(Ctx, Val); 350 } else if (auto *FpExt = dyn_cast<FPExtInst>(Arg)) { 351 if (FpExt->getType()->isDoubleTy() && 352 FpExt->getOperand(0)->getType()->isFloatTy()) { 353 Arg = FpExt->getOperand(0); 354 } 355 } 356 } 357 WhatToStore.push_back(Arg); 358 } else if (isa<PointerType>(ArgType)) { 359 if (shouldPrintAsStr(OpConvSpecifiers[ArgCount - 1], ArgType)) { 360 StringRef S = getAsConstantStr(Arg); 361 if (!S.empty()) { 362 const uint64_t ReadSize = 4; 363 364 DataExtractor Extractor(S, /*IsLittleEndian=*/true, 8); 365 DataExtractor::Cursor Offset(0); 366 while (Offset && Offset.tell() < S.size()) { 367 uint64_t ReadNow = std::min(ReadSize, S.size() - Offset.tell()); 368 uint64_t ReadBytes = 0; 369 switch (ReadNow) { 370 default: llvm_unreachable("min(4, X) > 4?"); 371 case 1: 372 ReadBytes = Extractor.getU8(Offset); 373 break; 374 case 2: 375 ReadBytes = Extractor.getU16(Offset); 376 break; 377 case 3: 378 ReadBytes = Extractor.getU24(Offset); 379 break; 380 case 4: 381 ReadBytes = Extractor.getU32(Offset); 382 break; 383 } 384 385 cantFail(Offset.takeError(), 386 "failed to read bytes from constant array"); 387 388 APInt IntVal(8 * ReadSize, ReadBytes); 389 390 // TODO: Should not bothering aligning up. 391 if (ReadNow < ReadSize) 392 IntVal = IntVal.zext(8 * ReadSize); 393 394 Type *IntTy = Type::getIntNTy(Ctx, IntVal.getBitWidth()); 395 WhatToStore.push_back(ConstantInt::get(IntTy, IntVal)); 396 } 397 } else { 398 // Empty string, give a hint to RT it is no NULL 399 Value *ANumV = ConstantInt::get(Int32Ty, 0xFFFFFF00, false); 400 WhatToStore.push_back(ANumV); 401 } 402 } else { 403 WhatToStore.push_back(Arg); 404 } 405 } else { 406 WhatToStore.push_back(Arg); 407 } 408 for (unsigned I = 0, E = WhatToStore.size(); I != E; ++I) { 409 Value *TheBtCast = WhatToStore[I]; 410 unsigned ArgSize = TD->getTypeAllocSize(TheBtCast->getType()); 411 StoreInst *StBuff = new StoreInst(TheBtCast, BufferIdx, BrnchPoint); 412 LLVM_DEBUG(dbgs() << "inserting store to printf buffer:\n" 413 << *StBuff << '\n'); 414 (void)StBuff; 415 if (I + 1 == E && ArgCount + 1 == CI->arg_size()) 416 break; 417 BufferIdx = GetElementPtrInst::Create( 418 I8Ty, BufferIdx, {ConstantInt::get(I32Ty, ArgSize)}, 419 "PrintBuffNextPtr", BrnchPoint); 420 LLVM_DEBUG(dbgs() << "inserting gep to the printf buffer:\n" 421 << *BufferIdx << '\n'); 422 } 423 } 424 } 425 426 // erase the printf calls 427 for (auto *CI : Printfs) 428 CI->eraseFromParent(); 429 430 Printfs.clear(); 431 return true; 432 } 433 434 bool AMDGPUPrintfRuntimeBindingImpl::run(Module &M) { 435 Triple TT(M.getTargetTriple()); 436 if (TT.getArch() == Triple::r600) 437 return false; 438 439 auto PrintfFunction = M.getFunction("printf"); 440 if (!PrintfFunction || !PrintfFunction->isDeclaration()) 441 return false; 442 443 for (auto &U : PrintfFunction->uses()) { 444 if (auto *CI = dyn_cast<CallInst>(U.getUser())) { 445 if (CI->isCallee(&U) && !CI->isNoBuiltin()) 446 Printfs.push_back(CI); 447 } 448 } 449 450 if (Printfs.empty()) 451 return false; 452 453 TD = &M.getDataLayout(); 454 455 return lowerPrintfForGpu(M); 456 } 457 458 bool AMDGPUPrintfRuntimeBinding::runOnModule(Module &M) { 459 return AMDGPUPrintfRuntimeBindingImpl().run(M); 460 } 461 462 PreservedAnalyses 463 AMDGPUPrintfRuntimeBindingPass::run(Module &M, ModuleAnalysisManager &AM) { 464 bool Changed = AMDGPUPrintfRuntimeBindingImpl().run(M); 465 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 466 } 467