1 //===- AMDGPUOpenCLEnqueuedBlockLowering.cpp - Lower enqueued block -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // \file 10 // This post-linking pass replaces the function pointer of enqueued 11 // block kernel with a global variable (runtime handle) and adds 12 // "runtime-handle" attribute to the enqueued block kernel. 13 // 14 // In LLVM CodeGen the runtime-handle metadata will be translated to 15 // RuntimeHandle metadata in code object. Runtime allocates a global buffer 16 // for each kernel with RuntimeHandle metadata and saves the kernel address 17 // required for the AQL packet into the buffer. __enqueue_kernel function 18 // in device library knows that the invoke function pointer in the block 19 // literal is actually runtime handle and loads the kernel address from it 20 // and put it into AQL packet for dispatching. 21 // 22 // This cannot be done in FE since FE cannot create a unique global variable 23 // with external linkage across LLVM modules. The global variable with internal 24 // linkage does not work since optimization passes will try to replace loads 25 // of the global variable with its initialization value. 26 // 27 // It also identifies the kernels directly or indirectly enqueues kernels 28 // and adds "calls-enqueue-kernel" function attribute to them, which will 29 // be used to determine whether to emit runtime metadata for the kernel 30 // enqueue related hidden kernel arguments. 31 // 32 //===----------------------------------------------------------------------===// 33 34 #include "AMDGPU.h" 35 #include "llvm/ADT/DenseSet.h" 36 #include "llvm/ADT/SmallString.h" 37 #include "llvm/IR/Constants.h" 38 #include "llvm/IR/Instructions.h" 39 #include "llvm/IR/Mangler.h" 40 #include "llvm/IR/Module.h" 41 #include "llvm/Pass.h" 42 #include "llvm/Support/Debug.h" 43 44 #define DEBUG_TYPE "amdgpu-lower-enqueued-block" 45 46 using namespace llvm; 47 48 namespace { 49 50 /// Lower enqueued blocks. 51 class AMDGPUOpenCLEnqueuedBlockLowering : public ModulePass { 52 public: 53 static char ID; 54 55 explicit AMDGPUOpenCLEnqueuedBlockLowering() : ModulePass(ID) {} 56 57 private: 58 bool runOnModule(Module &M) override; 59 }; 60 61 } // end anonymous namespace 62 63 char AMDGPUOpenCLEnqueuedBlockLowering::ID = 0; 64 65 char &llvm::AMDGPUOpenCLEnqueuedBlockLoweringID = 66 AMDGPUOpenCLEnqueuedBlockLowering::ID; 67 68 INITIALIZE_PASS(AMDGPUOpenCLEnqueuedBlockLowering, DEBUG_TYPE, 69 "Lower OpenCL enqueued blocks", false, false) 70 71 ModulePass* llvm::createAMDGPUOpenCLEnqueuedBlockLoweringPass() { 72 return new AMDGPUOpenCLEnqueuedBlockLowering(); 73 } 74 75 bool AMDGPUOpenCLEnqueuedBlockLowering::runOnModule(Module &M) { 76 DenseSet<Function *> Callers; 77 auto &C = M.getContext(); 78 bool Changed = false; 79 80 // ptr kernel_object, i32 private_segment_size, i32 group_segment_size 81 StructType *HandleTy = nullptr; 82 83 for (auto &F : M.functions()) { 84 if (F.hasFnAttribute("enqueued-block")) { 85 if (!F.hasName()) { 86 SmallString<64> Name; 87 Mangler::getNameWithPrefix(Name, "__amdgpu_enqueued_kernel", 88 M.getDataLayout()); 89 F.setName(Name); 90 } 91 LLVM_DEBUG(dbgs() << "found enqueued kernel: " << F.getName() << '\n'); 92 auto RuntimeHandle = (F.getName() + ".runtime_handle").str(); 93 if (!HandleTy) { 94 Type *Int32 = Type::getInt32Ty(C); 95 HandleTy = 96 StructType::create(C, {PointerType::getUnqual(C), Int32, Int32}, 97 "block.runtime.handle.t"); 98 } 99 100 auto *GV = new GlobalVariable( 101 M, HandleTy, 102 /*isConstant=*/true, GlobalValue::ExternalLinkage, 103 /*Initializer=*/Constant::getNullValue(HandleTy), RuntimeHandle, 104 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal, 105 AMDGPUAS::GLOBAL_ADDRESS, 106 /*isExternallyInitialized=*/true); 107 LLVM_DEBUG(dbgs() << "runtime handle created: " << *GV << '\n'); 108 109 F.replaceAllUsesWith(ConstantExpr::getAddrSpaceCast(GV, F.getType())); 110 F.addFnAttr("runtime-handle", RuntimeHandle); 111 F.setLinkage(GlobalValue::ExternalLinkage); 112 Changed = true; 113 } 114 } 115 116 return Changed; 117 } 118