1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Populates the VFABI attribute with the scalar-to-vector mappings 10 // from the TargetLibraryInfo. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/Utils/InjectTLIMappings.h" 15 #include "llvm/ADT/Statistic.h" 16 #include "llvm/Analysis/VectorUtils.h" 17 #include "llvm/IR/InstIterator.h" 18 #include "llvm/Transforms/Utils.h" 19 #include "llvm/Transforms/Utils/ModuleUtils.h" 20 21 using namespace llvm; 22 23 #define DEBUG_TYPE "inject-tli-mappings" 24 25 STATISTIC(NumCallInjected, 26 "Number of calls in which the mappings have been injected."); 27 28 STATISTIC(NumVFDeclAdded, 29 "Number of function declarations that have been added."); 30 STATISTIC(NumCompUsedAdded, 31 "Number of `@llvm.compiler.used` operands that have been added."); 32 33 /// Helper function to map the TLI name to a strings that holds 34 /// scalar-to-vector mapping. 35 /// 36 /// _ZGV<isa><mask><vlen><vparams>_<scalarname>(<vectorname>) 37 /// 38 /// where: 39 /// 40 /// <isa> = "_LLVM_" 41 /// <mask> = "N". Note: TLI does not support masked interfaces. 42 /// <vlen> = Number of concurrent lanes, stored in the `VectorizationFactor` 43 /// field of the `VecDesc` struct. 44 /// <vparams> = "v", as many as are the number of parameters of CI. 45 /// <scalarname> = the name of the scalar function called by CI. 46 /// <vectorname> = the name of the vector function mapped by the TLI. 47 static std::string mangleTLIName(StringRef VectorName, const CallInst &CI, 48 unsigned VF) { 49 SmallString<256> Buffer; 50 llvm::raw_svector_ostream Out(Buffer); 51 Out << "_ZGV" << VFABI::_LLVM_ << "N" << VF; 52 for (unsigned I = 0; I < CI.getNumArgOperands(); ++I) 53 Out << "v"; 54 Out << "_" << CI.getCalledFunction()->getName() << "(" << VectorName << ")"; 55 return Out.str(); 56 } 57 58 /// A helper function for converting Scalar types to vector types. 59 /// If the incoming type is void, we return void. If the VF is 1, we return 60 /// the scalar type. 61 static Type *ToVectorTy(Type *Scalar, unsigned VF, bool isScalable = false) { 62 if (Scalar->isVoidTy() || VF == 1) 63 return Scalar; 64 return VectorType::get(Scalar, {VF, isScalable}); 65 } 66 67 /// A helper function that adds the vector function declaration that 68 /// vectorizes the CallInst CI with a vectorization factor of VF 69 /// lanes. The TLI assumes that all parameters and the return type of 70 /// CI (other than void) need to be widened to a VectorType of VF 71 /// lanes. 72 static void addVariantDeclaration(CallInst &CI, const unsigned VF, 73 const StringRef VFName) { 74 Module *M = CI.getModule(); 75 76 // Add function declaration. 77 Type *RetTy = ToVectorTy(CI.getType(), VF); 78 SmallVector<Type *, 4> Tys; 79 for (Value *ArgOperand : CI.arg_operands()) 80 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); 81 assert(!CI.getFunctionType()->isVarArg() && 82 "VarArg functions are not supported."); 83 FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false); 84 Function *VectorF = 85 Function::Create(FTy, Function::ExternalLinkage, VFName, M); 86 VectorF->copyAttributesFrom(CI.getCalledFunction()); 87 ++NumVFDeclAdded; 88 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName 89 << "` of type " << *(VectorF->getType()) << "\n"); 90 91 // Make function declaration (without a body) "sticky" in the IR by 92 // listing it in the @llvm.compiler.used intrinsic. 93 assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` " 94 "only on declarations."); 95 appendToCompilerUsed(*M, {VectorF}); 96 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName 97 << "` to `@llvm.compiler.used`.\n"); 98 ++NumCompUsedAdded; 99 } 100 101 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) { 102 // This is needed to make sure we don't query the TLI for calls to 103 // bitcast of function pointers, like `%call = call i32 (i32*, ...) 104 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`, 105 // as such calls make the `isFunctionVectorizable` raise an 106 // exception. 107 if (CI.isNoBuiltin() || !CI.getCalledFunction()) 108 return; 109 110 const std::string ScalarName = CI.getCalledFunction()->getName(); 111 // Nothing to be done if the TLI thinks the function is not 112 // vectorizable. 113 if (!TLI.isFunctionVectorizable(ScalarName)) 114 return; 115 SmallVector<std::string, 8> Mappings; 116 VFABI::getVectorVariantNames(CI, Mappings); 117 Module *M = CI.getModule(); 118 const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(), 119 Mappings.end()); 120 // All VFs in the TLI are powers of 2. 121 for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); VF <= WidestVF; 122 VF *= 2) { 123 const std::string TLIName = TLI.getVectorizedFunction(ScalarName, VF); 124 if (!TLIName.empty()) { 125 std::string MangledName = mangleTLIName(TLIName, CI, VF); 126 if (!OriginalSetOfMappings.count(MangledName)) { 127 Mappings.push_back(MangledName); 128 ++NumCallInjected; 129 } 130 Function *VariantF = M->getFunction(TLIName); 131 if (!VariantF) 132 addVariantDeclaration(CI, VF, TLIName); 133 } 134 } 135 136 VFABI::setVectorVariantNames(&CI, Mappings); 137 } 138 139 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { 140 for (auto &I : instructions(F)) 141 if (auto CI = dyn_cast<CallInst>(&I)) 142 addMappingsFromTLI(TLI, *CI); 143 // Even if the pass adds IR attributes, the analyses are preserved. 144 return false; 145 } 146 147 //////////////////////////////////////////////////////////////////////////////// 148 // New pass manager implementation. 149 //////////////////////////////////////////////////////////////////////////////// 150 PreservedAnalyses InjectTLIMappings::run(Function &F, 151 FunctionAnalysisManager &AM) { 152 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F); 153 runImpl(TLI, F); 154 // Even if the pass adds IR attributes, the analyses are preserved. 155 return PreservedAnalyses::all(); 156 } 157 158 //////////////////////////////////////////////////////////////////////////////// 159 // Legacy PM Implementation. 160 //////////////////////////////////////////////////////////////////////////////// 161 bool InjectTLIMappingsLegacy::runOnFunction(Function &F) { 162 const TargetLibraryInfo &TLI = 163 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); 164 return runImpl(TLI, F); 165 } 166 167 void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const { 168 AU.setPreservesCFG(); 169 AU.addRequired<TargetLibraryInfoWrapperPass>(); 170 AU.addPreserved<TargetLibraryInfoWrapperPass>(); 171 } 172 173 //////////////////////////////////////////////////////////////////////////////// 174 // Legacy Pass manager initialization 175 //////////////////////////////////////////////////////////////////////////////// 176 char InjectTLIMappingsLegacy::ID = 0; 177 178 INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE, 179 "Inject TLI Mappings", false, false) 180 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 181 INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings", 182 false, false) 183 184 FunctionPass *llvm::createInjectTLIMappingsLegacyPass() { 185 return new InjectTLIMappingsLegacy(); 186 } 187