1 //=== ReplaceWithVeclib.cpp - Replace vector intrinsics with veclib calls -===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Replaces calls to LLVM vector intrinsics (i.e., calls to LLVM intrinsics 10 // with vector operands) with matching calls to functions from a vector 11 // library (e.g., libmvec, SVML) according to TargetLibraryInfo. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "llvm/CodeGen/ReplaceWithVeclib.h" 16 #include "llvm/ADT/STLExtras.h" 17 #include "llvm/ADT/Statistic.h" 18 #include "llvm/Analysis/DemandedBits.h" 19 #include "llvm/Analysis/GlobalsModRef.h" 20 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 21 #include "llvm/Analysis/TargetLibraryInfo.h" 22 #include "llvm/Analysis/VectorUtils.h" 23 #include "llvm/CodeGen/Passes.h" 24 #include "llvm/IR/IRBuilder.h" 25 #include "llvm/IR/InstIterator.h" 26 #include "llvm/Transforms/Utils/ModuleUtils.h" 27 28 using namespace llvm; 29 30 #define DEBUG_TYPE "replace-with-veclib" 31 32 STATISTIC(NumCallsReplaced, 33 "Number of calls to intrinsics that have been replaced."); 34 35 STATISTIC(NumTLIFuncDeclAdded, 36 "Number of vector library function declarations added."); 37 38 STATISTIC(NumFuncUsedAdded, 39 "Number of functions added to `llvm.compiler.used`"); 40 41 static bool replaceWithTLIFunction(CallInst &CI, const StringRef TLIName) { 42 Module *M = CI.getModule(); 43 44 Function *OldFunc = CI.getCalledFunction(); 45 46 // Check if the vector library function is already declared in this module, 47 // otherwise insert it. 48 Function *TLIFunc = M->getFunction(TLIName); 49 if (!TLIFunc) { 50 TLIFunc = Function::Create(OldFunc->getFunctionType(), 51 Function::ExternalLinkage, TLIName, *M); 52 TLIFunc->copyAttributesFrom(OldFunc); 53 54 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added vector library function `" 55 << TLIName << "` of type `" << *(TLIFunc->getType()) 56 << "` to module.\n"); 57 58 ++NumTLIFuncDeclAdded; 59 60 // Add the freshly created function to llvm.compiler.used, 61 // similar to as it is done in InjectTLIMappings 62 appendToCompilerUsed(*M, {TLIFunc}); 63 64 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << TLIName 65 << "` to `@llvm.compiler.used`.\n"); 66 ++NumFuncUsedAdded; 67 } 68 69 // Replace the call to the vector intrinsic with a call 70 // to the corresponding function from the vector library. 71 IRBuilder<> IRBuilder(&CI); 72 SmallVector<Value *> Args(CI.args()); 73 // Preserve the operand bundles. 74 SmallVector<OperandBundleDef, 1> OpBundles; 75 CI.getOperandBundlesAsDefs(OpBundles); 76 CallInst *Replacement = IRBuilder.CreateCall(TLIFunc, Args, OpBundles); 77 assert(OldFunc->getFunctionType() == TLIFunc->getFunctionType() && 78 "Expecting function types to be identical"); 79 CI.replaceAllUsesWith(Replacement); 80 if (isa<FPMathOperator>(Replacement)) { 81 // Preserve fast math flags for FP math. 82 Replacement->copyFastMathFlags(&CI); 83 } 84 85 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Replaced call to `" 86 << OldFunc->getName() << "` with call to `" << TLIName 87 << "`.\n"); 88 ++NumCallsReplaced; 89 return true; 90 } 91 92 static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, 93 CallInst &CI) { 94 if (!CI.getCalledFunction()) { 95 return false; 96 } 97 98 auto IntrinsicID = CI.getCalledFunction()->getIntrinsicID(); 99 if (IntrinsicID == Intrinsic::not_intrinsic) { 100 // Replacement is only performed for intrinsic functions 101 return false; 102 } 103 104 // Convert vector arguments to scalar type and check that 105 // all vector operands have identical vector width. 106 ElementCount VF = ElementCount::getFixed(0); 107 SmallVector<Type *> ScalarTypes; 108 for (auto Arg : enumerate(CI.args())) { 109 auto *ArgType = Arg.value()->getType(); 110 // Vector calls to intrinsics can still have 111 // scalar operands for specific arguments. 112 if (isVectorIntrinsicWithScalarOpAtArg(IntrinsicID, Arg.index())) { 113 ScalarTypes.push_back(ArgType); 114 } else { 115 // The argument in this place should be a vector if 116 // this is a call to a vector intrinsic. 117 auto *VectorArgTy = dyn_cast<VectorType>(ArgType); 118 if (!VectorArgTy) { 119 // The argument is not a vector, do not perform 120 // the replacement. 121 return false; 122 } 123 ElementCount NumElements = VectorArgTy->getElementCount(); 124 if (NumElements.isScalable()) { 125 // The current implementation does not support 126 // scalable vectors. 127 return false; 128 } 129 if (VF.isNonZero() && VF != NumElements) { 130 // The different arguments differ in vector size. 131 return false; 132 } else { 133 VF = NumElements; 134 } 135 ScalarTypes.push_back(VectorArgTy->getElementType()); 136 } 137 } 138 139 // Try to reconstruct the name for the scalar version of this 140 // intrinsic using the intrinsic ID and the argument types 141 // converted to scalar above. 142 std::string ScalarName; 143 if (Intrinsic::isOverloaded(IntrinsicID)) { 144 ScalarName = Intrinsic::getName(IntrinsicID, ScalarTypes, CI.getModule()); 145 } else { 146 ScalarName = Intrinsic::getName(IntrinsicID).str(); 147 } 148 149 if (!TLI.isFunctionVectorizable(ScalarName)) { 150 // The TargetLibraryInfo does not contain a vectorized version of 151 // the scalar function. 152 return false; 153 } 154 155 // Try to find the mapping for the scalar version of this intrinsic 156 // and the exact vector width of the call operands in the 157 // TargetLibraryInfo. 158 const std::string TLIName = 159 std::string(TLI.getVectorizedFunction(ScalarName, VF)); 160 161 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Looking up TLI mapping for `" 162 << ScalarName << "` and vector width " << VF << ".\n"); 163 164 if (!TLIName.empty()) { 165 // Found the correct mapping in the TargetLibraryInfo, 166 // replace the call to the intrinsic with a call to 167 // the vector library function. 168 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Found TLI function `" << TLIName 169 << "`.\n"); 170 return replaceWithTLIFunction(CI, TLIName); 171 } 172 173 return false; 174 } 175 176 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { 177 bool Changed = false; 178 SmallVector<CallInst *> ReplacedCalls; 179 for (auto &I : instructions(F)) { 180 if (auto *CI = dyn_cast<CallInst>(&I)) { 181 if (replaceWithCallToVeclib(TLI, *CI)) { 182 ReplacedCalls.push_back(CI); 183 Changed = true; 184 } 185 } 186 } 187 // Erase the calls to the intrinsics that have been replaced 188 // with calls to the vector library. 189 for (auto *CI : ReplacedCalls) { 190 CI->eraseFromParent(); 191 } 192 return Changed; 193 } 194 195 //////////////////////////////////////////////////////////////////////////////// 196 // New pass manager implementation. 197 //////////////////////////////////////////////////////////////////////////////// 198 PreservedAnalyses ReplaceWithVeclib::run(Function &F, 199 FunctionAnalysisManager &AM) { 200 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F); 201 auto Changed = runImpl(TLI, F); 202 if (Changed) { 203 PreservedAnalyses PA; 204 PA.preserveSet<CFGAnalyses>(); 205 PA.preserve<TargetLibraryAnalysis>(); 206 PA.preserve<ScalarEvolutionAnalysis>(); 207 PA.preserve<LoopAccessAnalysis>(); 208 PA.preserve<DemandedBitsAnalysis>(); 209 PA.preserve<OptimizationRemarkEmitterAnalysis>(); 210 return PA; 211 } else { 212 // The pass did not replace any calls, hence it preserves all analyses. 213 return PreservedAnalyses::all(); 214 } 215 } 216 217 //////////////////////////////////////////////////////////////////////////////// 218 // Legacy PM Implementation. 219 //////////////////////////////////////////////////////////////////////////////// 220 bool ReplaceWithVeclibLegacy::runOnFunction(Function &F) { 221 const TargetLibraryInfo &TLI = 222 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); 223 return runImpl(TLI, F); 224 } 225 226 void ReplaceWithVeclibLegacy::getAnalysisUsage(AnalysisUsage &AU) const { 227 AU.setPreservesCFG(); 228 AU.addRequired<TargetLibraryInfoWrapperPass>(); 229 AU.addPreserved<TargetLibraryInfoWrapperPass>(); 230 AU.addPreserved<ScalarEvolutionWrapperPass>(); 231 AU.addPreserved<AAResultsWrapperPass>(); 232 AU.addPreserved<LoopAccessLegacyAnalysis>(); 233 AU.addPreserved<DemandedBitsWrapperPass>(); 234 AU.addPreserved<OptimizationRemarkEmitterWrapperPass>(); 235 AU.addPreserved<GlobalsAAWrapperPass>(); 236 } 237 238 //////////////////////////////////////////////////////////////////////////////// 239 // Legacy Pass manager initialization 240 //////////////////////////////////////////////////////////////////////////////// 241 char ReplaceWithVeclibLegacy::ID = 0; 242 243 INITIALIZE_PASS_BEGIN(ReplaceWithVeclibLegacy, DEBUG_TYPE, 244 "Replace intrinsics with calls to vector library", false, 245 false) 246 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 247 INITIALIZE_PASS_END(ReplaceWithVeclibLegacy, DEBUG_TYPE, 248 "Replace intrinsics with calls to vector library", false, 249 false) 250 251 FunctionPass *llvm::createReplaceWithVeclibLegacyPass() { 252 return new ReplaceWithVeclibLegacy(); 253 } 254