1 //=== AMDGPUPrintfRuntimeBinding.cpp - OpenCL printf implementation -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // \file
9 //
10 // The pass bind printfs to a kernel arg pointer that will be bound to a buffer
11 // later by the runtime.
12 //
13 // This pass traverses the functions in the module and converts
14 // each call to printf to a sequence of operations that
15 // store the following into the printf buffer:
16 // - format string (passed as a module's metadata unique ID)
17 // - bitwise copies of printf arguments
18 // The backend passes will need to store metadata in the kernel
19 //===----------------------------------------------------------------------===//
20
21 #include "AMDGPU.h"
22 #include "llvm/ADT/StringExtras.h"
23 #include "llvm/Analysis/ValueTracking.h"
24 #include "llvm/IR/DiagnosticInfo.h"
25 #include "llvm/IR/Dominators.h"
26 #include "llvm/IR/IRBuilder.h"
27 #include "llvm/IR/Instructions.h"
28 #include "llvm/IR/Module.h"
29 #include "llvm/InitializePasses.h"
30 #include "llvm/Support/DataExtractor.h"
31 #include "llvm/TargetParser/Triple.h"
32 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
33
34 using namespace llvm;
35
36 #define DEBUG_TYPE "printfToRuntime"
37 enum { DWORD_ALIGN = 4 };
38
39 namespace {
40 class AMDGPUPrintfRuntimeBinding final : public ModulePass {
41
42 public:
43 static char ID;
44
45 explicit AMDGPUPrintfRuntimeBinding();
46
47 private:
48 bool runOnModule(Module &M) override;
49 };
50
51 class AMDGPUPrintfRuntimeBindingImpl {
52 public:
53 AMDGPUPrintfRuntimeBindingImpl() = default;
54 bool run(Module &M);
55
56 private:
57 void getConversionSpecifiers(SmallVectorImpl<char> &OpConvSpecifiers,
58 StringRef fmt, size_t num_ops) const;
59
60 bool lowerPrintfForGpu(Module &M);
61
62 const DataLayout *TD;
63 SmallVector<CallInst *, 32> Printfs;
64 };
65 } // namespace
66
67 char AMDGPUPrintfRuntimeBinding::ID = 0;
68
69 INITIALIZE_PASS_BEGIN(AMDGPUPrintfRuntimeBinding,
70 "amdgpu-printf-runtime-binding", "AMDGPU Printf lowering",
71 false, false)
72 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
73 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
74 INITIALIZE_PASS_END(AMDGPUPrintfRuntimeBinding, "amdgpu-printf-runtime-binding",
75 "AMDGPU Printf lowering", false, false)
76
77 char &llvm::AMDGPUPrintfRuntimeBindingID = AMDGPUPrintfRuntimeBinding::ID;
78
79 namespace llvm {
createAMDGPUPrintfRuntimeBinding()80 ModulePass *createAMDGPUPrintfRuntimeBinding() {
81 return new AMDGPUPrintfRuntimeBinding();
82 }
83 } // namespace llvm
84
AMDGPUPrintfRuntimeBinding()85 AMDGPUPrintfRuntimeBinding::AMDGPUPrintfRuntimeBinding() : ModulePass(ID) {
86 initializeAMDGPUPrintfRuntimeBindingPass(*PassRegistry::getPassRegistry());
87 }
88
getConversionSpecifiers(SmallVectorImpl<char> & OpConvSpecifiers,StringRef Fmt,size_t NumOps) const89 void AMDGPUPrintfRuntimeBindingImpl::getConversionSpecifiers(
90 SmallVectorImpl<char> &OpConvSpecifiers, StringRef Fmt,
91 size_t NumOps) const {
92 // not all format characters are collected.
93 // At this time the format characters of interest
94 // are %p and %s, which use to know if we
95 // are either storing a literal string or a
96 // pointer to the printf buffer.
97 static const char ConvSpecifiers[] = "cdieEfgGaosuxXp";
98 size_t CurFmtSpecifierIdx = 0;
99 size_t PrevFmtSpecifierIdx = 0;
100
101 while ((CurFmtSpecifierIdx = Fmt.find_first_of(
102 ConvSpecifiers, CurFmtSpecifierIdx)) != StringRef::npos) {
103 bool ArgDump = false;
104 StringRef CurFmt = Fmt.substr(PrevFmtSpecifierIdx,
105 CurFmtSpecifierIdx - PrevFmtSpecifierIdx);
106 size_t pTag = CurFmt.find_last_of('%');
107 if (pTag != StringRef::npos) {
108 ArgDump = true;
109 while (pTag && CurFmt[--pTag] == '%') {
110 ArgDump = !ArgDump;
111 }
112 }
113
114 if (ArgDump)
115 OpConvSpecifiers.push_back(Fmt[CurFmtSpecifierIdx]);
116
117 PrevFmtSpecifierIdx = ++CurFmtSpecifierIdx;
118 }
119 }
120
shouldPrintAsStr(char Specifier,Type * OpType)121 static bool shouldPrintAsStr(char Specifier, Type *OpType) {
122 return Specifier == 's' && isa<PointerType>(OpType);
123 }
124
125 constexpr StringLiteral NonLiteralStr("???");
126 static_assert(NonLiteralStr.size() == 3);
127
getAsConstantStr(Value * V)128 static StringRef getAsConstantStr(Value *V) {
129 StringRef S;
130 if (!getConstantStringInfo(V, S))
131 S = NonLiteralStr;
132
133 return S;
134 }
135
diagnoseInvalidFormatString(const CallBase * CI)136 static void diagnoseInvalidFormatString(const CallBase *CI) {
137 DiagnosticInfoUnsupported UnsupportedFormatStr(
138 *CI->getParent()->getParent(),
139 "printf format string must be a trivially resolved constant string "
140 "global variable",
141 CI->getDebugLoc());
142 CI->getContext().diagnose(UnsupportedFormatStr);
143 }
144
lowerPrintfForGpu(Module & M)145 bool AMDGPUPrintfRuntimeBindingImpl::lowerPrintfForGpu(Module &M) {
146 LLVMContext &Ctx = M.getContext();
147 IRBuilder<> Builder(Ctx);
148 Type *I32Ty = Type::getInt32Ty(Ctx);
149
150 // Instead of creating global variables, the printf format strings are
151 // extracted and passed as metadata. This avoids polluting llvm's symbol
152 // tables in this module. Metadata is going to be extracted by the backend
153 // passes and inserted into the OpenCL binary as appropriate.
154 NamedMDNode *metaD = M.getOrInsertNamedMetadata("llvm.printf.fmts");
155 unsigned UniqID = metaD->getNumOperands();
156
157 for (auto *CI : Printfs) {
158 unsigned NumOps = CI->arg_size();
159
160 SmallString<16> OpConvSpecifiers;
161 Value *Op = CI->getArgOperand(0);
162
163 StringRef FormatStr;
164 if (!getConstantStringInfo(Op, FormatStr)) {
165 Value *Stripped = Op->stripPointerCasts();
166 if (!isa<UndefValue>(Stripped) && !isa<ConstantPointerNull>(Stripped))
167 diagnoseInvalidFormatString(CI);
168 continue;
169 }
170
171 // We need this call to ascertain that we are printing a string or a
172 // pointer. It takes out the specifiers and fills up the first arg.
173 getConversionSpecifiers(OpConvSpecifiers, FormatStr, NumOps - 1);
174
175 // Add metadata for the string
176 std::string AStreamHolder;
177 raw_string_ostream Sizes(AStreamHolder);
178 int Sum = DWORD_ALIGN;
179 Sizes << CI->arg_size() - 1;
180 Sizes << ':';
181 for (unsigned ArgCount = 1;
182 ArgCount < CI->arg_size() && ArgCount <= OpConvSpecifiers.size();
183 ArgCount++) {
184 Value *Arg = CI->getArgOperand(ArgCount);
185 Type *ArgType = Arg->getType();
186 unsigned ArgSize = TD->getTypeAllocSize(ArgType);
187 //
188 // ArgSize by design should be a multiple of DWORD_ALIGN,
189 // expand the arguments that do not follow this rule.
190 //
191 if (ArgSize % DWORD_ALIGN != 0) {
192 Type *ResType = Type::getInt32Ty(Ctx);
193 if (auto *VecType = dyn_cast<VectorType>(ArgType))
194 ResType = VectorType::get(ResType, VecType->getElementCount());
195 Builder.SetInsertPoint(CI);
196 Builder.SetCurrentDebugLocation(CI->getDebugLoc());
197
198 if (ArgType->isFloatingPointTy()) {
199 Arg = Builder.CreateBitCast(
200 Arg,
201 IntegerType::getIntNTy(Ctx, ArgType->getPrimitiveSizeInBits()));
202 }
203
204 if (OpConvSpecifiers[ArgCount - 1] == 'x' ||
205 OpConvSpecifiers[ArgCount - 1] == 'X' ||
206 OpConvSpecifiers[ArgCount - 1] == 'u' ||
207 OpConvSpecifiers[ArgCount - 1] == 'o')
208 Arg = Builder.CreateZExt(Arg, ResType);
209 else
210 Arg = Builder.CreateSExt(Arg, ResType);
211 ArgType = Arg->getType();
212 ArgSize = TD->getTypeAllocSize(ArgType);
213 CI->setOperand(ArgCount, Arg);
214 }
215 if (OpConvSpecifiers[ArgCount - 1] == 'f') {
216 ConstantFP *FpCons = dyn_cast<ConstantFP>(Arg);
217 if (FpCons)
218 ArgSize = 4;
219 else {
220 FPExtInst *FpExt = dyn_cast<FPExtInst>(Arg);
221 if (FpExt && FpExt->getType()->isDoubleTy() &&
222 FpExt->getOperand(0)->getType()->isFloatTy())
223 ArgSize = 4;
224 }
225 }
226 if (shouldPrintAsStr(OpConvSpecifiers[ArgCount - 1], ArgType))
227 ArgSize = alignTo(getAsConstantStr(Arg).size() + 1, 4);
228
229 LLVM_DEBUG(dbgs() << "Printf ArgSize (in buffer) = " << ArgSize
230 << " for type: " << *ArgType << '\n');
231 Sizes << ArgSize << ':';
232 Sum += ArgSize;
233 }
234 LLVM_DEBUG(dbgs() << "Printf format string in source = " << FormatStr
235 << '\n');
236 for (char C : FormatStr) {
237 // Rest of the C escape sequences (e.g. \') are handled correctly
238 // by the MDParser
239 switch (C) {
240 case '\a':
241 Sizes << "\\a";
242 break;
243 case '\b':
244 Sizes << "\\b";
245 break;
246 case '\f':
247 Sizes << "\\f";
248 break;
249 case '\n':
250 Sizes << "\\n";
251 break;
252 case '\r':
253 Sizes << "\\r";
254 break;
255 case '\v':
256 Sizes << "\\v";
257 break;
258 case ':':
259 // ':' cannot be scanned by Flex, as it is defined as a delimiter
260 // Replace it with it's octal representation \72
261 Sizes << "\\72";
262 break;
263 default:
264 Sizes << C;
265 break;
266 }
267 }
268
269 // Insert the printf_alloc call
270 Builder.SetInsertPoint(CI);
271 Builder.SetCurrentDebugLocation(CI->getDebugLoc());
272
273 AttributeList Attr = AttributeList::get(Ctx, AttributeList::FunctionIndex,
274 Attribute::NoUnwind);
275
276 Type *SizetTy = Type::getInt32Ty(Ctx);
277
278 Type *Tys_alloc[1] = {SizetTy};
279 Type *I8Ty = Type::getInt8Ty(Ctx);
280 Type *I8Ptr = PointerType::get(I8Ty, 1);
281 FunctionType *FTy_alloc = FunctionType::get(I8Ptr, Tys_alloc, false);
282 FunctionCallee PrintfAllocFn =
283 M.getOrInsertFunction(StringRef("__printf_alloc"), FTy_alloc, Attr);
284
285 LLVM_DEBUG(dbgs() << "Printf metadata = " << Sizes.str() << '\n');
286 std::string fmtstr = itostr(++UniqID) + ":" + Sizes.str();
287 MDString *fmtStrArray = MDString::get(Ctx, fmtstr);
288
289 MDNode *myMD = MDNode::get(Ctx, fmtStrArray);
290 metaD->addOperand(myMD);
291 Value *sumC = ConstantInt::get(SizetTy, Sum, false);
292 SmallVector<Value *, 1> alloc_args;
293 alloc_args.push_back(sumC);
294 CallInst *pcall = CallInst::Create(PrintfAllocFn, alloc_args,
295 "printf_alloc_fn", CI->getIterator());
296
297 //
298 // Insert code to split basicblock with a
299 // piece of hammock code.
300 // basicblock splits after buffer overflow check
301 //
302 ConstantPointerNull *zeroIntPtr =
303 ConstantPointerNull::get(PointerType::get(I8Ty, 1));
304 auto *cmp = cast<ICmpInst>(Builder.CreateICmpNE(pcall, zeroIntPtr, ""));
305 if (!CI->use_empty()) {
306 Value *result =
307 Builder.CreateSExt(Builder.CreateNot(cmp), I32Ty, "printf_res");
308 CI->replaceAllUsesWith(result);
309 }
310 SplitBlock(CI->getParent(), cmp);
311 Instruction *Brnch =
312 SplitBlockAndInsertIfThen(cmp, cmp->getNextNode(), false);
313 BasicBlock::iterator BrnchPoint = Brnch->getIterator();
314
315 Builder.SetInsertPoint(Brnch);
316
317 // store unique printf id in the buffer
318 //
319 GetElementPtrInst *BufferIdx = GetElementPtrInst::Create(
320 I8Ty, pcall, ConstantInt::get(Ctx, APInt(32, 0)), "PrintBuffID",
321 BrnchPoint);
322
323 Type *idPointer = PointerType::get(I32Ty, AMDGPUAS::GLOBAL_ADDRESS);
324 Value *id_gep_cast =
325 new BitCastInst(BufferIdx, idPointer, "PrintBuffIdCast", BrnchPoint);
326
327 new StoreInst(ConstantInt::get(I32Ty, UniqID), id_gep_cast, BrnchPoint);
328
329 // 1st 4 bytes hold the printf_id
330 // the following GEP is the buffer pointer
331 BufferIdx = GetElementPtrInst::Create(I8Ty, pcall,
332 ConstantInt::get(Ctx, APInt(32, 4)),
333 "PrintBuffGep", BrnchPoint);
334
335 Type *Int32Ty = Type::getInt32Ty(Ctx);
336 for (unsigned ArgCount = 1;
337 ArgCount < CI->arg_size() && ArgCount <= OpConvSpecifiers.size();
338 ArgCount++) {
339 Value *Arg = CI->getArgOperand(ArgCount);
340 Type *ArgType = Arg->getType();
341 SmallVector<Value *, 32> WhatToStore;
342 if (ArgType->isFPOrFPVectorTy() && !isa<VectorType>(ArgType)) {
343 if (OpConvSpecifiers[ArgCount - 1] == 'f') {
344 if (auto *FpCons = dyn_cast<ConstantFP>(Arg)) {
345 APFloat Val(FpCons->getValueAPF());
346 bool Lost = false;
347 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
348 &Lost);
349 Arg = ConstantFP::get(Ctx, Val);
350 } else if (auto *FpExt = dyn_cast<FPExtInst>(Arg)) {
351 if (FpExt->getType()->isDoubleTy() &&
352 FpExt->getOperand(0)->getType()->isFloatTy()) {
353 Arg = FpExt->getOperand(0);
354 }
355 }
356 }
357 WhatToStore.push_back(Arg);
358 } else if (isa<PointerType>(ArgType)) {
359 if (shouldPrintAsStr(OpConvSpecifiers[ArgCount - 1], ArgType)) {
360 StringRef S = getAsConstantStr(Arg);
361 if (!S.empty()) {
362 const uint64_t ReadSize = 4;
363
364 DataExtractor Extractor(S, /*IsLittleEndian=*/true, 8);
365 DataExtractor::Cursor Offset(0);
366 while (Offset && Offset.tell() < S.size()) {
367 uint64_t ReadNow = std::min(ReadSize, S.size() - Offset.tell());
368 uint64_t ReadBytes = 0;
369 switch (ReadNow) {
370 default: llvm_unreachable("min(4, X) > 4?");
371 case 1:
372 ReadBytes = Extractor.getU8(Offset);
373 break;
374 case 2:
375 ReadBytes = Extractor.getU16(Offset);
376 break;
377 case 3:
378 ReadBytes = Extractor.getU24(Offset);
379 break;
380 case 4:
381 ReadBytes = Extractor.getU32(Offset);
382 break;
383 }
384
385 cantFail(Offset.takeError(),
386 "failed to read bytes from constant array");
387
388 APInt IntVal(8 * ReadSize, ReadBytes);
389
390 // TODO: Should not bothering aligning up.
391 if (ReadNow < ReadSize)
392 IntVal = IntVal.zext(8 * ReadSize);
393
394 Type *IntTy = Type::getIntNTy(Ctx, IntVal.getBitWidth());
395 WhatToStore.push_back(ConstantInt::get(IntTy, IntVal));
396 }
397 } else {
398 // Empty string, give a hint to RT it is no NULL
399 Value *ANumV = ConstantInt::get(Int32Ty, 0xFFFFFF00, false);
400 WhatToStore.push_back(ANumV);
401 }
402 } else {
403 WhatToStore.push_back(Arg);
404 }
405 } else {
406 WhatToStore.push_back(Arg);
407 }
408 for (unsigned I = 0, E = WhatToStore.size(); I != E; ++I) {
409 Value *TheBtCast = WhatToStore[I];
410 unsigned ArgSize = TD->getTypeAllocSize(TheBtCast->getType());
411 StoreInst *StBuff = new StoreInst(TheBtCast, BufferIdx, BrnchPoint);
412 LLVM_DEBUG(dbgs() << "inserting store to printf buffer:\n"
413 << *StBuff << '\n');
414 (void)StBuff;
415 if (I + 1 == E && ArgCount + 1 == CI->arg_size())
416 break;
417 BufferIdx = GetElementPtrInst::Create(
418 I8Ty, BufferIdx, {ConstantInt::get(I32Ty, ArgSize)},
419 "PrintBuffNextPtr", BrnchPoint);
420 LLVM_DEBUG(dbgs() << "inserting gep to the printf buffer:\n"
421 << *BufferIdx << '\n');
422 }
423 }
424 }
425
426 // erase the printf calls
427 for (auto *CI : Printfs)
428 CI->eraseFromParent();
429
430 Printfs.clear();
431 return true;
432 }
433
run(Module & M)434 bool AMDGPUPrintfRuntimeBindingImpl::run(Module &M) {
435 Triple TT(M.getTargetTriple());
436 if (TT.getArch() == Triple::r600)
437 return false;
438
439 auto PrintfFunction = M.getFunction("printf");
440 if (!PrintfFunction || !PrintfFunction->isDeclaration())
441 return false;
442
443 for (auto &U : PrintfFunction->uses()) {
444 if (auto *CI = dyn_cast<CallInst>(U.getUser())) {
445 if (CI->isCallee(&U) && !CI->isNoBuiltin())
446 Printfs.push_back(CI);
447 }
448 }
449
450 if (Printfs.empty())
451 return false;
452
453 TD = &M.getDataLayout();
454
455 return lowerPrintfForGpu(M);
456 }
457
runOnModule(Module & M)458 bool AMDGPUPrintfRuntimeBinding::runOnModule(Module &M) {
459 return AMDGPUPrintfRuntimeBindingImpl().run(M);
460 }
461
462 PreservedAnalyses
run(Module & M,ModuleAnalysisManager & AM)463 AMDGPUPrintfRuntimeBindingPass::run(Module &M, ModuleAnalysisManager &AM) {
464 bool Changed = AMDGPUPrintfRuntimeBindingImpl().run(M);
465 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
466 }
467