1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Populates the VFABI attribute with the scalar-to-vector mappings 10 // from the TargetLibraryInfo. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/Utils/InjectTLIMappings.h" 15 #include "llvm/ADT/Statistic.h" 16 #include "llvm/Analysis/DemandedBits.h" 17 #include "llvm/Analysis/GlobalsModRef.h" 18 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 19 #include "llvm/Analysis/TargetLibraryInfo.h" 20 #include "llvm/Analysis/VectorUtils.h" 21 #include "llvm/IR/InstIterator.h" 22 #include "llvm/Transforms/Utils.h" 23 #include "llvm/Transforms/Utils/ModuleUtils.h" 24 25 using namespace llvm; 26 27 #define DEBUG_TYPE "inject-tli-mappings" 28 29 STATISTIC(NumCallInjected, 30 "Number of calls in which the mappings have been injected."); 31 32 STATISTIC(NumVFDeclAdded, 33 "Number of function declarations that have been added."); 34 STATISTIC(NumCompUsedAdded, 35 "Number of `@llvm.compiler.used` operands that have been added."); 36 37 /// A helper function that adds the vector function declaration that 38 /// vectorizes the CallInst CI with a vectorization factor of VF 39 /// lanes. The TLI assumes that all parameters and the return type of 40 /// CI (other than void) need to be widened to a VectorType of VF 41 /// lanes. 42 static void addVariantDeclaration(CallInst &CI, const ElementCount &VF, 43 const StringRef VFName) { 44 Module *M = CI.getModule(); 45 46 // Add function declaration. 47 Type *RetTy = ToVectorTy(CI.getType(), VF); 48 SmallVector<Type *, 4> Tys; 49 for (Value *ArgOperand : CI.args()) 50 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); 51 assert(!CI.getFunctionType()->isVarArg() && 52 "VarArg functions are not supported."); 53 FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false); 54 Function *VectorF = 55 Function::Create(FTy, Function::ExternalLinkage, VFName, M); 56 VectorF->copyAttributesFrom(CI.getCalledFunction()); 57 ++NumVFDeclAdded; 58 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName 59 << "` of type " << *(VectorF->getType()) << "\n"); 60 61 // Make function declaration (without a body) "sticky" in the IR by 62 // listing it in the @llvm.compiler.used intrinsic. 63 assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` " 64 "only on declarations."); 65 appendToCompilerUsed(*M, {VectorF}); 66 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName 67 << "` to `@llvm.compiler.used`.\n"); 68 ++NumCompUsedAdded; 69 } 70 71 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) { 72 // This is needed to make sure we don't query the TLI for calls to 73 // bitcast of function pointers, like `%call = call i32 (i32*, ...) 74 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`, 75 // as such calls make the `isFunctionVectorizable` raise an 76 // exception. 77 if (CI.isNoBuiltin() || !CI.getCalledFunction()) 78 return; 79 80 StringRef ScalarName = CI.getCalledFunction()->getName(); 81 82 // Nothing to be done if the TLI thinks the function is not 83 // vectorizable. 84 if (!TLI.isFunctionVectorizable(ScalarName)) 85 return; 86 SmallVector<std::string, 8> Mappings; 87 VFABI::getVectorVariantNames(CI, Mappings); 88 Module *M = CI.getModule(); 89 const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(), 90 Mappings.end()); 91 92 auto AddVariantDecl = [&](const ElementCount &VF) { 93 const std::string TLIName = 94 std::string(TLI.getVectorizedFunction(ScalarName, VF)); 95 if (!TLIName.empty()) { 96 std::string MangledName = 97 VFABI::mangleTLIVectorName(TLIName, ScalarName, CI.arg_size(), VF); 98 if (!OriginalSetOfMappings.count(MangledName)) { 99 Mappings.push_back(MangledName); 100 ++NumCallInjected; 101 } 102 Function *VariantF = M->getFunction(TLIName); 103 if (!VariantF) 104 addVariantDeclaration(CI, VF, TLIName); 105 } 106 }; 107 108 // All VFs in the TLI are powers of 2. 109 ElementCount WidestFixedVF, WidestScalableVF; 110 TLI.getWidestVF(ScalarName, WidestFixedVF, WidestScalableVF); 111 112 for (ElementCount VF = ElementCount::getFixed(2); 113 ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2) 114 AddVariantDecl(VF); 115 116 // TODO: Add scalable variants once we're able to test them. 117 assert(WidestScalableVF.isZero() && 118 "Scalable vector mappings not yet supported"); 119 120 VFABI::setVectorVariantNames(&CI, Mappings); 121 } 122 123 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { 124 for (auto &I : instructions(F)) 125 if (auto CI = dyn_cast<CallInst>(&I)) 126 addMappingsFromTLI(TLI, *CI); 127 // Even if the pass adds IR attributes, the analyses are preserved. 128 return false; 129 } 130 131 //////////////////////////////////////////////////////////////////////////////// 132 // New pass manager implementation. 133 //////////////////////////////////////////////////////////////////////////////// 134 PreservedAnalyses InjectTLIMappings::run(Function &F, 135 FunctionAnalysisManager &AM) { 136 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F); 137 runImpl(TLI, F); 138 // Even if the pass adds IR attributes, the analyses are preserved. 139 return PreservedAnalyses::all(); 140 } 141 142 //////////////////////////////////////////////////////////////////////////////// 143 // Legacy PM Implementation. 144 //////////////////////////////////////////////////////////////////////////////// 145 bool InjectTLIMappingsLegacy::runOnFunction(Function &F) { 146 const TargetLibraryInfo &TLI = 147 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); 148 return runImpl(TLI, F); 149 } 150 151 void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const { 152 AU.setPreservesCFG(); 153 AU.addRequired<TargetLibraryInfoWrapperPass>(); 154 AU.addPreserved<TargetLibraryInfoWrapperPass>(); 155 AU.addPreserved<ScalarEvolutionWrapperPass>(); 156 AU.addPreserved<AAResultsWrapperPass>(); 157 AU.addPreserved<LoopAccessLegacyAnalysis>(); 158 AU.addPreserved<DemandedBitsWrapperPass>(); 159 AU.addPreserved<OptimizationRemarkEmitterWrapperPass>(); 160 AU.addPreserved<GlobalsAAWrapperPass>(); 161 } 162 163 //////////////////////////////////////////////////////////////////////////////// 164 // Legacy Pass manager initialization 165 //////////////////////////////////////////////////////////////////////////////// 166 char InjectTLIMappingsLegacy::ID = 0; 167 168 INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE, 169 "Inject TLI Mappings", false, false) 170 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 171 INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings", 172 false, false) 173 174 FunctionPass *llvm::createInjectTLIMappingsLegacyPass() { 175 return new InjectTLIMappingsLegacy(); 176 } 177