1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Populates the VFABI attribute with the scalar-to-vector mappings 10 // from the TargetLibraryInfo. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/Utils/InjectTLIMappings.h" 15 #include "llvm/ADT/Statistic.h" 16 #include "llvm/Analysis/DemandedBits.h" 17 #include "llvm/Analysis/GlobalsModRef.h" 18 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 19 #include "llvm/Analysis/TargetLibraryInfo.h" 20 #include "llvm/Analysis/VectorUtils.h" 21 #include "llvm/IR/InstIterator.h" 22 #include "llvm/Transforms/Utils/ModuleUtils.h" 23 24 using namespace llvm; 25 26 #define DEBUG_TYPE "inject-tli-mappings" 27 28 STATISTIC(NumCallInjected, 29 "Number of calls in which the mappings have been injected."); 30 31 STATISTIC(NumVFDeclAdded, 32 "Number of function declarations that have been added."); 33 STATISTIC(NumCompUsedAdded, 34 "Number of `@llvm.compiler.used` operands that have been added."); 35 36 /// A helper function that adds the vector function declaration that 37 /// vectorizes the CallInst CI with a vectorization factor of VF 38 /// lanes. The TLI assumes that all parameters and the return type of 39 /// CI (other than void) need to be widened to a VectorType of VF 40 /// lanes. 41 static void addVariantDeclaration(CallInst &CI, const ElementCount &VF, 42 bool Predicate, const StringRef VFName) { 43 Module *M = CI.getModule(); 44 45 // Add function declaration. 46 Type *RetTy = ToVectorTy(CI.getType(), VF); 47 SmallVector<Type *, 4> Tys; 48 for (Value *ArgOperand : CI.args()) 49 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); 50 assert(!CI.getFunctionType()->isVarArg() && 51 "VarArg functions are not supported."); 52 if (Predicate) 53 Tys.push_back(ToVectorTy(Type::getInt1Ty(RetTy->getContext()), VF)); 54 FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false); 55 Function *VectorF = 56 Function::Create(FTy, Function::ExternalLinkage, VFName, M); 57 VectorF->copyAttributesFrom(CI.getCalledFunction()); 58 ++NumVFDeclAdded; 59 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName 60 << "` of type " << *(VectorF->getType()) << "\n"); 61 62 // Make function declaration (without a body) "sticky" in the IR by 63 // listing it in the @llvm.compiler.used intrinsic. 64 assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` " 65 "only on declarations."); 66 appendToCompilerUsed(*M, {VectorF}); 67 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName 68 << "` to `@llvm.compiler.used`.\n"); 69 ++NumCompUsedAdded; 70 } 71 72 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) { 73 // This is needed to make sure we don't query the TLI for calls to 74 // bitcast of function pointers, like `%call = call i32 (i32*, ...) 75 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`, 76 // as such calls make the `isFunctionVectorizable` raise an 77 // exception. 78 if (CI.isNoBuiltin() || !CI.getCalledFunction()) 79 return; 80 81 StringRef ScalarName = CI.getCalledFunction()->getName(); 82 83 // Nothing to be done if the TLI thinks the function is not 84 // vectorizable. 85 if (!TLI.isFunctionVectorizable(ScalarName)) 86 return; 87 SmallVector<std::string, 8> Mappings; 88 VFABI::getVectorVariantNames(CI, Mappings); 89 Module *M = CI.getModule(); 90 const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(), 91 Mappings.end()); 92 93 auto AddVariantDecl = [&](const ElementCount &VF, bool Predicate) { 94 const std::string TLIName = 95 std::string(TLI.getVectorizedFunction(ScalarName, VF, Predicate)); 96 if (!TLIName.empty()) { 97 std::string MangledName = VFABI::mangleTLIVectorName( 98 TLIName, ScalarName, CI.arg_size(), VF, Predicate); 99 if (!OriginalSetOfMappings.count(MangledName)) { 100 Mappings.push_back(MangledName); 101 ++NumCallInjected; 102 } 103 Function *VariantF = M->getFunction(TLIName); 104 if (!VariantF) 105 addVariantDeclaration(CI, VF, Predicate, TLIName); 106 } 107 }; 108 109 // All VFs in the TLI are powers of 2. 110 ElementCount WidestFixedVF, WidestScalableVF; 111 TLI.getWidestVF(ScalarName, WidestFixedVF, WidestScalableVF); 112 113 for (bool Predicated : {false, true}) { 114 for (ElementCount VF = ElementCount::getFixed(2); 115 ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2) 116 AddVariantDecl(VF, Predicated); 117 118 for (ElementCount VF = ElementCount::getScalable(2); 119 ElementCount::isKnownLE(VF, WidestScalableVF); VF *= 2) 120 AddVariantDecl(VF, Predicated); 121 } 122 123 VFABI::setVectorVariantNames(&CI, Mappings); 124 } 125 126 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { 127 for (auto &I : instructions(F)) 128 if (auto CI = dyn_cast<CallInst>(&I)) 129 addMappingsFromTLI(TLI, *CI); 130 // Even if the pass adds IR attributes, the analyses are preserved. 131 return false; 132 } 133 134 //////////////////////////////////////////////////////////////////////////////// 135 // New pass manager implementation. 136 //////////////////////////////////////////////////////////////////////////////// 137 PreservedAnalyses InjectTLIMappings::run(Function &F, 138 FunctionAnalysisManager &AM) { 139 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F); 140 runImpl(TLI, F); 141 // Even if the pass adds IR attributes, the analyses are preserved. 142 return PreservedAnalyses::all(); 143 } 144