1 //=== ReplaceWithVeclib.cpp - Replace vector instrinsics with veclib calls ===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Replaces calls to LLVM vector intrinsics (i.e., calls to LLVM intrinsics 10 // with vector operands) with matching calls to functions from a vector 11 // library (e.g., libmvec, SVML) according to TargetLibraryInfo. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "llvm/CodeGen/ReplaceWithVeclib.h" 16 #include "llvm/ADT/STLExtras.h" 17 #include "llvm/ADT/Statistic.h" 18 #include "llvm/Analysis/DemandedBits.h" 19 #include "llvm/Analysis/GlobalsModRef.h" 20 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 21 #include "llvm/Analysis/TargetLibraryInfo.h" 22 #include "llvm/Analysis/VectorUtils.h" 23 #include "llvm/CodeGen/Passes.h" 24 #include "llvm/IR/IRBuilder.h" 25 #include "llvm/IR/InstIterator.h" 26 #include "llvm/IR/IntrinsicInst.h" 27 #include "llvm/Transforms/Utils/ModuleUtils.h" 28 29 using namespace llvm; 30 31 #define DEBUG_TYPE "replace-with-veclib" 32 33 STATISTIC(NumCallsReplaced, 34 "Number of calls to intrinsics that have been replaced."); 35 36 STATISTIC(NumTLIFuncDeclAdded, 37 "Number of vector library function declarations added."); 38 39 STATISTIC(NumFuncUsedAdded, 40 "Number of functions added to `llvm.compiler.used`"); 41 42 static bool replaceWithTLIFunction(CallInst &CI, const StringRef TLIName) { 43 Module *M = CI.getModule(); 44 45 Function *OldFunc = CI.getCalledFunction(); 46 47 // Check if the vector library function is already declared in this module, 48 // otherwise insert it. 49 Function *TLIFunc = M->getFunction(TLIName); 50 if (!TLIFunc) { 51 TLIFunc = Function::Create(OldFunc->getFunctionType(), 52 Function::ExternalLinkage, TLIName, *M); 53 TLIFunc->copyAttributesFrom(OldFunc); 54 55 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added vector library function `" 56 << TLIName << "` of type `" << *(TLIFunc->getType()) 57 << "` to module.\n"); 58 59 ++NumTLIFuncDeclAdded; 60 61 // Add the freshly created function to llvm.compiler.used, 62 // similar to as it is done in InjectTLIMappings 63 appendToCompilerUsed(*M, {TLIFunc}); 64 65 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << TLIName 66 << "` to `@llvm.compiler.used`.\n"); 67 ++NumFuncUsedAdded; 68 } 69 70 // Replace the call to the vector intrinsic with a call 71 // to the corresponding function from the vector library. 72 IRBuilder<> IRBuilder(&CI); 73 SmallVector<Value *> Args(CI.args()); 74 // Preserve the operand bundles. 75 SmallVector<OperandBundleDef, 1> OpBundles; 76 CI.getOperandBundlesAsDefs(OpBundles); 77 CallInst *Replacement = IRBuilder.CreateCall(TLIFunc, Args, OpBundles); 78 assert(OldFunc->getFunctionType() == TLIFunc->getFunctionType() && 79 "Expecting function types to be identical"); 80 CI.replaceAllUsesWith(Replacement); 81 if (isa<FPMathOperator>(Replacement)) { 82 // Preserve fast math flags for FP math. 83 Replacement->copyFastMathFlags(&CI); 84 } 85 86 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Replaced call to `" 87 << OldFunc->getName() << "` with call to `" << TLIName 88 << "`.\n"); 89 ++NumCallsReplaced; 90 return true; 91 } 92 93 static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, 94 CallInst &CI) { 95 if (!CI.getCalledFunction()) { 96 return false; 97 } 98 99 auto IntrinsicID = CI.getCalledFunction()->getIntrinsicID(); 100 if (IntrinsicID == Intrinsic::not_intrinsic) { 101 // Replacement is only performed for intrinsic functions 102 return false; 103 } 104 105 // Convert vector arguments to scalar type and check that 106 // all vector operands have identical vector width. 107 ElementCount VF = ElementCount::getFixed(0); 108 SmallVector<Type *> ScalarTypes; 109 for (auto Arg : enumerate(CI.args())) { 110 auto *ArgType = Arg.value()->getType(); 111 // Vector calls to intrinsics can still have 112 // scalar operands for specific arguments. 113 if (hasVectorInstrinsicScalarOpd(IntrinsicID, Arg.index())) { 114 ScalarTypes.push_back(ArgType); 115 } else { 116 // The argument in this place should be a vector if 117 // this is a call to a vector intrinsic. 118 auto *VectorArgTy = dyn_cast<VectorType>(ArgType); 119 if (!VectorArgTy) { 120 // The argument is not a vector, do not perform 121 // the replacement. 122 return false; 123 } 124 ElementCount NumElements = VectorArgTy->getElementCount(); 125 if (NumElements.isScalable()) { 126 // The current implementation does not support 127 // scalable vectors. 128 return false; 129 } 130 if (VF.isNonZero() && VF != NumElements) { 131 // The different arguments differ in vector size. 132 return false; 133 } else { 134 VF = NumElements; 135 } 136 ScalarTypes.push_back(VectorArgTy->getElementType()); 137 } 138 } 139 140 // Try to reconstruct the name for the scalar version of this 141 // intrinsic using the intrinsic ID and the argument types 142 // converted to scalar above. 143 std::string ScalarName; 144 if (Intrinsic::isOverloaded(IntrinsicID)) { 145 ScalarName = Intrinsic::getName(IntrinsicID, ScalarTypes, CI.getModule()); 146 } else { 147 ScalarName = Intrinsic::getName(IntrinsicID).str(); 148 } 149 150 if (!TLI.isFunctionVectorizable(ScalarName)) { 151 // The TargetLibraryInfo does not contain a vectorized version of 152 // the scalar function. 153 return false; 154 } 155 156 // Try to find the mapping for the scalar version of this intrinsic 157 // and the exact vector width of the call operands in the 158 // TargetLibraryInfo. 159 const std::string TLIName = 160 std::string(TLI.getVectorizedFunction(ScalarName, VF)); 161 162 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Looking up TLI mapping for `" 163 << ScalarName << "` and vector width " << VF << ".\n"); 164 165 if (!TLIName.empty()) { 166 // Found the correct mapping in the TargetLibraryInfo, 167 // replace the call to the intrinsic with a call to 168 // the vector library function. 169 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Found TLI function `" << TLIName 170 << "`.\n"); 171 return replaceWithTLIFunction(CI, TLIName); 172 } 173 174 return false; 175 } 176 177 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { 178 bool Changed = false; 179 SmallVector<CallInst *> ReplacedCalls; 180 for (auto &I : instructions(F)) { 181 if (auto *CI = dyn_cast<CallInst>(&I)) { 182 if (replaceWithCallToVeclib(TLI, *CI)) { 183 ReplacedCalls.push_back(CI); 184 Changed = true; 185 } 186 } 187 } 188 // Erase the calls to the intrinsics that have been replaced 189 // with calls to the vector library. 190 for (auto *CI : ReplacedCalls) { 191 CI->eraseFromParent(); 192 } 193 return Changed; 194 } 195 196 //////////////////////////////////////////////////////////////////////////////// 197 // New pass manager implementation. 198 //////////////////////////////////////////////////////////////////////////////// 199 PreservedAnalyses ReplaceWithVeclib::run(Function &F, 200 FunctionAnalysisManager &AM) { 201 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F); 202 auto Changed = runImpl(TLI, F); 203 if (Changed) { 204 PreservedAnalyses PA; 205 PA.preserveSet<CFGAnalyses>(); 206 PA.preserve<TargetLibraryAnalysis>(); 207 PA.preserve<ScalarEvolutionAnalysis>(); 208 PA.preserve<LoopAccessAnalysis>(); 209 PA.preserve<DemandedBitsAnalysis>(); 210 PA.preserve<OptimizationRemarkEmitterAnalysis>(); 211 return PA; 212 } else { 213 // The pass did not replace any calls, hence it preserves all analyses. 214 return PreservedAnalyses::all(); 215 } 216 } 217 218 //////////////////////////////////////////////////////////////////////////////// 219 // Legacy PM Implementation. 220 //////////////////////////////////////////////////////////////////////////////// 221 bool ReplaceWithVeclibLegacy::runOnFunction(Function &F) { 222 const TargetLibraryInfo &TLI = 223 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); 224 return runImpl(TLI, F); 225 } 226 227 void ReplaceWithVeclibLegacy::getAnalysisUsage(AnalysisUsage &AU) const { 228 AU.setPreservesCFG(); 229 AU.addRequired<TargetLibraryInfoWrapperPass>(); 230 AU.addPreserved<TargetLibraryInfoWrapperPass>(); 231 AU.addPreserved<ScalarEvolutionWrapperPass>(); 232 AU.addPreserved<AAResultsWrapperPass>(); 233 AU.addPreserved<LoopAccessLegacyAnalysis>(); 234 AU.addPreserved<DemandedBitsWrapperPass>(); 235 AU.addPreserved<OptimizationRemarkEmitterWrapperPass>(); 236 AU.addPreserved<GlobalsAAWrapperPass>(); 237 } 238 239 //////////////////////////////////////////////////////////////////////////////// 240 // Legacy Pass manager initialization 241 //////////////////////////////////////////////////////////////////////////////// 242 char ReplaceWithVeclibLegacy::ID = 0; 243 244 INITIALIZE_PASS_BEGIN(ReplaceWithVeclibLegacy, DEBUG_TYPE, 245 "Replace intrinsics with calls to vector library", false, 246 false) 247 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 248 INITIALIZE_PASS_END(ReplaceWithVeclibLegacy, DEBUG_TYPE, 249 "Replace intrinsics with calls to vector library", false, 250 false) 251 252 FunctionPass *llvm::createReplaceWithVeclibLegacyPass() { 253 return new ReplaceWithVeclibLegacy(); 254 } 255