1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Populates the VFABI attribute with the scalar-to-vector mappings 10 // from the TargetLibraryInfo. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/Utils/InjectTLIMappings.h" 15 #include "llvm/ADT/Statistic.h" 16 #include "llvm/Analysis/DemandedBits.h" 17 #include "llvm/Analysis/GlobalsModRef.h" 18 #include "llvm/Analysis/TargetLibraryInfo.h" 19 #include "llvm/Analysis/VectorUtils.h" 20 #include "llvm/IR/InstIterator.h" 21 #include "llvm/IR/VFABIDemangler.h" 22 #include "llvm/Transforms/Utils/ModuleUtils.h" 23 24 using namespace llvm; 25 26 #define DEBUG_TYPE "inject-tli-mappings" 27 28 STATISTIC(NumCallInjected, 29 "Number of calls in which the mappings have been injected."); 30 31 STATISTIC(NumVFDeclAdded, 32 "Number of function declarations that have been added."); 33 STATISTIC(NumCompUsedAdded, 34 "Number of `@llvm.compiler.used` operands that have been added."); 35 36 /// A helper function that adds the vector variant declaration for vectorizing 37 /// the CallInst \p CI with a vectorization factor of \p VF lanes. For each 38 /// mapping, TLI provides a VABI prefix, which contains all information required 39 /// to create vector function declaration. 40 static void addVariantDeclaration(CallInst &CI, const ElementCount &VF, 41 const VecDesc *VD) { 42 Module *M = CI.getModule(); 43 FunctionType *ScalarFTy = CI.getFunctionType(); 44 45 assert(!ScalarFTy->isVarArg() && "VarArg functions are not supported."); 46 47 const std::optional<VFInfo> Info = VFABI::tryDemangleForVFABI( 48 VD->getVectorFunctionABIVariantString(), ScalarFTy); 49 50 assert(Info && "Failed to demangle vector variant"); 51 assert(Info->Shape.VF == VF && "Mangled name does not match VF"); 52 53 const StringRef VFName = VD->getVectorFnName(); 54 FunctionType *VectorFTy = VFABI::createFunctionType(*Info, ScalarFTy); 55 Function *VecFunc = 56 Function::Create(VectorFTy, Function::ExternalLinkage, VFName, M); 57 VecFunc->copyAttributesFrom(CI.getCalledFunction()); 58 if (auto CC = VD->getCallingConv()) 59 VecFunc->setCallingConv(*CC); 60 ++NumVFDeclAdded; 61 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName 62 << "` of type " << *VectorFTy << "\n"); 63 64 // Make function declaration (without a body) "sticky" in the IR by 65 // listing it in the @llvm.compiler.used intrinsic. 66 assert(!VecFunc->size() && "VFABI attribute requires `@llvm.compiler.used` " 67 "only on declarations."); 68 appendToCompilerUsed(*M, {VecFunc}); 69 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName 70 << "` to `@llvm.compiler.used`.\n"); 71 ++NumCompUsedAdded; 72 } 73 74 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) { 75 // This is needed to make sure we don't query the TLI for calls to 76 // bitcast of function pointers, like `%call = call i32 (i32*, ...) 77 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`, 78 // as such calls make the `isFunctionVectorizable` raise an 79 // exception. 80 if (CI.isNoBuiltin() || !CI.getCalledFunction()) 81 return; 82 83 StringRef ScalarName = CI.getCalledFunction()->getName(); 84 85 // Nothing to be done if the TLI thinks the function is not 86 // vectorizable. 87 if (!TLI.isFunctionVectorizable(ScalarName)) 88 return; 89 SmallVector<std::string, 8> Mappings; 90 VFABI::getVectorVariantNames(CI, Mappings); 91 Module *M = CI.getModule(); 92 const SetVector<StringRef> OriginalSetOfMappings(llvm::from_range, Mappings); 93 94 auto AddVariantDecl = [&](const ElementCount &VF, bool Predicate) { 95 const VecDesc *VD = TLI.getVectorMappingInfo(ScalarName, VF, Predicate); 96 if (VD && !VD->getVectorFnName().empty()) { 97 std::string MangledName = VD->getVectorFunctionABIVariantString(); 98 if (!OriginalSetOfMappings.count(MangledName)) { 99 Mappings.push_back(MangledName); 100 ++NumCallInjected; 101 } 102 Function *VariantF = M->getFunction(VD->getVectorFnName()); 103 if (!VariantF) 104 addVariantDeclaration(CI, VF, VD); 105 } 106 }; 107 108 // All VFs in the TLI are powers of 2. 109 ElementCount WidestFixedVF, WidestScalableVF; 110 TLI.getWidestVF(ScalarName, WidestFixedVF, WidestScalableVF); 111 112 for (bool Predicated : {false, true}) { 113 for (ElementCount VF = ElementCount::getFixed(2); 114 ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2) 115 AddVariantDecl(VF, Predicated); 116 117 for (ElementCount VF = ElementCount::getScalable(2); 118 ElementCount::isKnownLE(VF, WidestScalableVF); VF *= 2) 119 AddVariantDecl(VF, Predicated); 120 } 121 122 VFABI::setVectorVariantNames(&CI, Mappings); 123 } 124 125 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { 126 for (auto &I : instructions(F)) 127 if (auto CI = dyn_cast<CallInst>(&I)) 128 addMappingsFromTLI(TLI, *CI); 129 // Even if the pass adds IR attributes, the analyses are preserved. 130 return false; 131 } 132 133 //////////////////////////////////////////////////////////////////////////////// 134 // New pass manager implementation. 135 //////////////////////////////////////////////////////////////////////////////// 136 PreservedAnalyses InjectTLIMappings::run(Function &F, 137 FunctionAnalysisManager &AM) { 138 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F); 139 runImpl(TLI, F); 140 // Even if the pass adds IR attributes, the analyses are preserved. 141 return PreservedAnalyses::all(); 142 } 143