1 //===- AMDGPUOpenCLEnqueuedBlockLowering.cpp - Lower enqueued block -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // \file 10 // This post-linking pass replaces the function pointer of enqueued 11 // block kernel with a global variable (runtime handle) and adds 12 // "runtime-handle" attribute to the enqueued block kernel. 13 // 14 // In LLVM CodeGen the runtime-handle metadata will be translated to 15 // RuntimeHandle metadata in code object. Runtime allocates a global buffer 16 // for each kernel with RuntimeHandle metadata and saves the kernel address 17 // required for the AQL packet into the buffer. __enqueue_kernel function 18 // in device library knows that the invoke function pointer in the block 19 // literal is actually runtime handle and loads the kernel address from it 20 // and put it into AQL packet for dispatching. 21 // 22 // This cannot be done in FE since FE cannot create a unique global variable 23 // with external linkage across LLVM modules. The global variable with internal 24 // linkage does not work since optimization passes will try to replace loads 25 // of the global variable with its initialization value. 26 // 27 // It also identifies the kernels directly or indirectly enqueues kernels 28 // and adds "calls-enqueue-kernel" function attribute to them, which will 29 // be used to determine whether to emit runtime metadata for the kernel 30 // enqueue related hidden kernel arguments. 31 // 32 //===----------------------------------------------------------------------===// 33 34 #include "AMDGPU.h" 35 #include "llvm/ADT/DenseSet.h" 36 #include "llvm/ADT/SmallString.h" 37 #include "llvm/IR/Constants.h" 38 #include "llvm/IR/Instructions.h" 39 #include "llvm/IR/Mangler.h" 40 #include "llvm/IR/Module.h" 41 #include "llvm/Pass.h" 42 #include "llvm/Support/Debug.h" 43 44 #define DEBUG_TYPE "amdgpu-lower-enqueued-block" 45 46 using namespace llvm; 47 48 namespace { 49 50 /// Lower enqueued blocks. 51 class AMDGPUOpenCLEnqueuedBlockLowering : public ModulePass { 52 public: 53 static char ID; 54 55 explicit AMDGPUOpenCLEnqueuedBlockLowering() : ModulePass(ID) {} 56 57 private: 58 bool runOnModule(Module &M) override; 59 }; 60 61 } // end anonymous namespace 62 63 char AMDGPUOpenCLEnqueuedBlockLowering::ID = 0; 64 65 char &llvm::AMDGPUOpenCLEnqueuedBlockLoweringID = 66 AMDGPUOpenCLEnqueuedBlockLowering::ID; 67 68 INITIALIZE_PASS(AMDGPUOpenCLEnqueuedBlockLowering, DEBUG_TYPE, 69 "Lower OpenCL enqueued blocks", false, false) 70 71 ModulePass* llvm::createAMDGPUOpenCLEnqueuedBlockLoweringPass() { 72 return new AMDGPUOpenCLEnqueuedBlockLowering(); 73 } 74 75 /// Collect direct or indirect callers of \p F and save them 76 /// to \p Callers. 77 static void collectCallers(Function *F, DenseSet<Function *> &Callers) { 78 for (auto U : F->users()) { 79 if (auto *CI = dyn_cast<CallInst>(&*U)) { 80 auto *Caller = CI->getParent()->getParent(); 81 if (Callers.insert(Caller).second) 82 collectCallers(Caller, Callers); 83 } 84 } 85 } 86 87 /// If \p U is instruction or constant, collect functions which directly or 88 /// indirectly use it. 89 static void collectFunctionUsers(User *U, DenseSet<Function *> &Funcs) { 90 if (auto *I = dyn_cast<Instruction>(U)) { 91 auto *F = I->getParent()->getParent(); 92 if (Funcs.insert(F).second) 93 collectCallers(F, Funcs); 94 return; 95 } 96 if (!isa<Constant>(U)) 97 return; 98 for (auto UU : U->users()) 99 collectFunctionUsers(&*UU, Funcs); 100 } 101 102 bool AMDGPUOpenCLEnqueuedBlockLowering::runOnModule(Module &M) { 103 DenseSet<Function *> Callers; 104 auto &C = M.getContext(); 105 bool Changed = false; 106 for (auto &F : M.functions()) { 107 if (F.hasFnAttribute("enqueued-block")) { 108 if (!F.hasName()) { 109 SmallString<64> Name; 110 Mangler::getNameWithPrefix(Name, "__amdgpu_enqueued_kernel", 111 M.getDataLayout()); 112 F.setName(Name); 113 } 114 LLVM_DEBUG(dbgs() << "found enqueued kernel: " << F.getName() << '\n'); 115 auto RuntimeHandle = (F.getName() + ".runtime_handle").str(); 116 auto T = ArrayType::get(Type::getInt64Ty(C), 2); 117 auto *GV = new GlobalVariable( 118 M, T, 119 /*isConstant=*/false, GlobalValue::ExternalLinkage, 120 /*Initializer=*/Constant::getNullValue(T), RuntimeHandle, 121 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal, 122 AMDGPUAS::GLOBAL_ADDRESS, 123 /*isExternallyInitialized=*/false); 124 LLVM_DEBUG(dbgs() << "runtime handle created: " << *GV << '\n'); 125 126 for (auto U : F.users()) { 127 auto *UU = &*U; 128 if (!isa<ConstantExpr>(UU)) 129 continue; 130 collectFunctionUsers(UU, Callers); 131 auto *BitCast = cast<ConstantExpr>(UU); 132 auto *NewPtr = ConstantExpr::getPointerCast(GV, BitCast->getType()); 133 BitCast->replaceAllUsesWith(NewPtr); 134 F.addFnAttr("runtime-handle", RuntimeHandle); 135 F.setLinkage(GlobalValue::ExternalLinkage); 136 Changed = true; 137 } 138 } 139 } 140 141 for (auto F : Callers) { 142 if (F->getCallingConv() != CallingConv::AMDGPU_KERNEL) 143 continue; 144 F->addFnAttr("calls-enqueue-kernel"); 145 LLVM_DEBUG(dbgs() << "mark enqueue_kernel caller:" << F->getName() << '\n'); 146 } 147 return Changed; 148 } 149