1 //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass implements IR lowering for the llvm.memcpy, llvm.memmove, 10 // llvm.memset, llvm.load.relative and llvm.objc.* intrinsics. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/PreISelIntrinsicLowering.h" 15 #include "llvm/Analysis/ObjCARCInstKind.h" 16 #include "llvm/Analysis/ObjCARCUtil.h" 17 #include "llvm/Analysis/TargetTransformInfo.h" 18 #include "llvm/CodeGen/Passes.h" 19 #include "llvm/CodeGen/TargetLowering.h" 20 #include "llvm/CodeGen/TargetPassConfig.h" 21 #include "llvm/IR/Function.h" 22 #include "llvm/IR/IRBuilder.h" 23 #include "llvm/IR/Instructions.h" 24 #include "llvm/IR/IntrinsicInst.h" 25 #include "llvm/IR/Module.h" 26 #include "llvm/IR/Type.h" 27 #include "llvm/InitializePasses.h" 28 #include "llvm/Pass.h" 29 #include "llvm/Support/Casting.h" 30 #include "llvm/Target/TargetMachine.h" 31 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" 32 33 using namespace llvm; 34 35 /// Threshold to leave statically sized memory intrinsic calls. Calls of known 36 /// size larger than this will be expanded by the pass. Calls of unknown or 37 /// lower size will be left for expansion in codegen. 38 static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt( 39 "mem-intrinsic-expand-size", 40 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1), 41 cl::Hidden); 42 43 namespace { 44 45 struct PreISelIntrinsicLowering { 46 const TargetMachine &TM; 47 const function_ref<TargetTransformInfo &(Function &)> LookupTTI; 48 49 /// If this is true, assume it's preferably to leave memory intrinsic calls 50 /// for replacement with a library call later. Otherwise this depends on 51 /// TargetLoweringInfo availability of the corresponding function. 52 const bool UseMemIntrinsicLibFunc; 53 54 explicit PreISelIntrinsicLowering( 55 const TargetMachine &TM_, 56 function_ref<TargetTransformInfo &(Function &)> LookupTTI_, 57 bool UseMemIntrinsicLibFunc_ = true) 58 : TM(TM_), LookupTTI(LookupTTI_), 59 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {} 60 61 static bool shouldExpandMemIntrinsicWithSize(Value *Size, 62 const TargetTransformInfo &TTI); 63 bool expandMemIntrinsicUses(Function &F) const; 64 bool lowerIntrinsics(Module &M) const; 65 }; 66 67 } // namespace 68 69 static bool lowerLoadRelative(Function &F) { 70 if (F.use_empty()) 71 return false; 72 73 bool Changed = false; 74 Type *Int32Ty = Type::getInt32Ty(F.getContext()); 75 76 for (Use &U : llvm::make_early_inc_range(F.uses())) { 77 auto CI = dyn_cast<CallInst>(U.getUser()); 78 if (!CI || CI->getCalledOperand() != &F) 79 continue; 80 81 IRBuilder<> B(CI); 82 Value *OffsetPtr = 83 B.CreatePtrAdd(CI->getArgOperand(0), CI->getArgOperand(1)); 84 Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtr, Align(4)); 85 86 Value *ResultPtr = B.CreatePtrAdd(CI->getArgOperand(0), OffsetI32); 87 88 CI->replaceAllUsesWith(ResultPtr); 89 CI->eraseFromParent(); 90 Changed = true; 91 } 92 93 return Changed; 94 } 95 96 // ObjCARC has knowledge about whether an obj-c runtime function needs to be 97 // always tail-called or never tail-called. 98 static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) { 99 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(&F); 100 if (objcarc::IsAlwaysTail(Kind)) 101 return CallInst::TCK_Tail; 102 else if (objcarc::IsNeverTail(Kind)) 103 return CallInst::TCK_NoTail; 104 return CallInst::TCK_None; 105 } 106 107 static bool lowerObjCCall(Function &F, const char *NewFn, 108 bool setNonLazyBind = false) { 109 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) && 110 "Pre-ISel intrinsics do lower into regular function calls"); 111 if (F.use_empty()) 112 return false; 113 114 // If we haven't already looked up this function, check to see if the 115 // program already contains a function with this name. 116 Module *M = F.getParent(); 117 FunctionCallee FCache = M->getOrInsertFunction(NewFn, F.getFunctionType()); 118 119 if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) { 120 Fn->setLinkage(F.getLinkage()); 121 if (setNonLazyBind && !Fn->isWeakForLinker()) { 122 // If we have Native ARC, set nonlazybind attribute for these APIs for 123 // performance. 124 Fn->addFnAttr(Attribute::NonLazyBind); 125 } 126 } 127 128 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F); 129 130 for (Use &U : llvm::make_early_inc_range(F.uses())) { 131 auto *CB = cast<CallBase>(U.getUser()); 132 133 if (CB->getCalledFunction() != &F) { 134 objcarc::ARCInstKind Kind = objcarc::getAttachedARCFunctionKind(CB); 135 (void)Kind; 136 assert((Kind == objcarc::ARCInstKind::RetainRV || 137 Kind == objcarc::ARCInstKind::UnsafeClaimRV) && 138 "use expected to be the argument of operand bundle " 139 "\"clang.arc.attachedcall\""); 140 U.set(FCache.getCallee()); 141 continue; 142 } 143 144 auto *CI = cast<CallInst>(CB); 145 assert(CI->getCalledFunction() && "Cannot lower an indirect call!"); 146 147 IRBuilder<> Builder(CI->getParent(), CI->getIterator()); 148 SmallVector<Value *, 8> Args(CI->args()); 149 SmallVector<llvm::OperandBundleDef, 1> BundleList; 150 CI->getOperandBundlesAsDefs(BundleList); 151 CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList); 152 NewCI->setName(CI->getName()); 153 154 // Try to set the most appropriate TailCallKind based on both the current 155 // attributes and the ones that we could get from ObjCARC's special 156 // knowledge of the runtime functions. 157 // 158 // std::max respects both requirements of notail and tail here: 159 // * notail on either the call or from ObjCARC becomes notail 160 // * tail on either side is stronger than none, but not notail 161 CallInst::TailCallKind TCK = CI->getTailCallKind(); 162 NewCI->setTailCallKind(std::max(TCK, OverridingTCK)); 163 164 // Transfer the 'returned' attribute from the intrinsic to the call site. 165 // By applying this only to intrinsic call sites, we avoid applying it to 166 // non-ARC explicit calls to things like objc_retain which have not been 167 // auto-upgraded to use the intrinsics. 168 unsigned Index; 169 if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) && 170 Index) 171 NewCI->addParamAttr(Index - AttributeList::FirstArgIndex, 172 Attribute::Returned); 173 174 if (!CI->use_empty()) 175 CI->replaceAllUsesWith(NewCI); 176 CI->eraseFromParent(); 177 } 178 179 return true; 180 } 181 182 // TODO: Should refine based on estimated number of accesses (e.g. does it 183 // require splitting based on alignment) 184 bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize( 185 Value *Size, const TargetTransformInfo &TTI) { 186 ConstantInt *CI = dyn_cast<ConstantInt>(Size); 187 if (!CI) 188 return true; 189 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences() 190 ? MemIntrinsicExpandSizeThresholdOpt 191 : TTI.getMaxMemIntrinsicInlineSizeThreshold(); 192 uint64_t SizeVal = CI->getZExtValue(); 193 194 // Treat a threshold of 0 as a special case to force expansion of all 195 // intrinsics, including size 0. 196 return SizeVal > Threshold || Threshold == 0; 197 } 198 199 static bool canEmitLibcall(const TargetMachine &TM, Function *F, 200 RTLIB::Libcall LC) { 201 // TODO: Should this consider the address space of the memcpy? 202 const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering(); 203 return TLI->getLibcallName(LC) != nullptr; 204 } 205 206 // TODO: Handle atomic memcpy and memcpy.inline 207 // TODO: Pass ScalarEvolution 208 bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const { 209 Intrinsic::ID ID = F.getIntrinsicID(); 210 bool Changed = false; 211 212 for (User *U : llvm::make_early_inc_range(F.users())) { 213 Instruction *Inst = cast<Instruction>(U); 214 215 switch (ID) { 216 case Intrinsic::memcpy: { 217 auto *Memcpy = cast<MemCpyInst>(Inst); 218 Function *ParentFunc = Memcpy->getFunction(); 219 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 220 if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) { 221 if (UseMemIntrinsicLibFunc && 222 canEmitLibcall(TM, ParentFunc, RTLIB::MEMCPY)) 223 break; 224 225 // TODO: For optsize, emit the loop into a separate function 226 expandMemCpyAsLoop(Memcpy, TTI); 227 Changed = true; 228 Memcpy->eraseFromParent(); 229 } 230 231 break; 232 } 233 case Intrinsic::memmove: { 234 auto *Memmove = cast<MemMoveInst>(Inst); 235 Function *ParentFunc = Memmove->getFunction(); 236 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 237 if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) { 238 if (UseMemIntrinsicLibFunc && 239 canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE)) 240 break; 241 242 if (expandMemMoveAsLoop(Memmove, TTI)) { 243 Changed = true; 244 Memmove->eraseFromParent(); 245 } 246 } 247 248 break; 249 } 250 case Intrinsic::memset: { 251 auto *Memset = cast<MemSetInst>(Inst); 252 Function *ParentFunc = Memset->getFunction(); 253 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 254 if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) { 255 if (UseMemIntrinsicLibFunc && 256 canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET)) 257 break; 258 259 expandMemSetAsLoop(Memset); 260 Changed = true; 261 Memset->eraseFromParent(); 262 } 263 264 break; 265 } 266 default: 267 llvm_unreachable("unhandled intrinsic"); 268 } 269 } 270 271 return Changed; 272 } 273 274 bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { 275 bool Changed = false; 276 for (Function &F : M) { 277 switch (F.getIntrinsicID()) { 278 default: 279 break; 280 case Intrinsic::memcpy: 281 case Intrinsic::memmove: 282 case Intrinsic::memset: 283 Changed |= expandMemIntrinsicUses(F); 284 break; 285 case Intrinsic::load_relative: 286 Changed |= lowerLoadRelative(F); 287 break; 288 case Intrinsic::objc_autorelease: 289 Changed |= lowerObjCCall(F, "objc_autorelease"); 290 break; 291 case Intrinsic::objc_autoreleasePoolPop: 292 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPop"); 293 break; 294 case Intrinsic::objc_autoreleasePoolPush: 295 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPush"); 296 break; 297 case Intrinsic::objc_autoreleaseReturnValue: 298 Changed |= lowerObjCCall(F, "objc_autoreleaseReturnValue"); 299 break; 300 case Intrinsic::objc_copyWeak: 301 Changed |= lowerObjCCall(F, "objc_copyWeak"); 302 break; 303 case Intrinsic::objc_destroyWeak: 304 Changed |= lowerObjCCall(F, "objc_destroyWeak"); 305 break; 306 case Intrinsic::objc_initWeak: 307 Changed |= lowerObjCCall(F, "objc_initWeak"); 308 break; 309 case Intrinsic::objc_loadWeak: 310 Changed |= lowerObjCCall(F, "objc_loadWeak"); 311 break; 312 case Intrinsic::objc_loadWeakRetained: 313 Changed |= lowerObjCCall(F, "objc_loadWeakRetained"); 314 break; 315 case Intrinsic::objc_moveWeak: 316 Changed |= lowerObjCCall(F, "objc_moveWeak"); 317 break; 318 case Intrinsic::objc_release: 319 Changed |= lowerObjCCall(F, "objc_release", true); 320 break; 321 case Intrinsic::objc_retain: 322 Changed |= lowerObjCCall(F, "objc_retain", true); 323 break; 324 case Intrinsic::objc_retainAutorelease: 325 Changed |= lowerObjCCall(F, "objc_retainAutorelease"); 326 break; 327 case Intrinsic::objc_retainAutoreleaseReturnValue: 328 Changed |= lowerObjCCall(F, "objc_retainAutoreleaseReturnValue"); 329 break; 330 case Intrinsic::objc_retainAutoreleasedReturnValue: 331 Changed |= lowerObjCCall(F, "objc_retainAutoreleasedReturnValue"); 332 break; 333 case Intrinsic::objc_retainBlock: 334 Changed |= lowerObjCCall(F, "objc_retainBlock"); 335 break; 336 case Intrinsic::objc_storeStrong: 337 Changed |= lowerObjCCall(F, "objc_storeStrong"); 338 break; 339 case Intrinsic::objc_storeWeak: 340 Changed |= lowerObjCCall(F, "objc_storeWeak"); 341 break; 342 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue: 343 Changed |= lowerObjCCall(F, "objc_unsafeClaimAutoreleasedReturnValue"); 344 break; 345 case Intrinsic::objc_retainedObject: 346 Changed |= lowerObjCCall(F, "objc_retainedObject"); 347 break; 348 case Intrinsic::objc_unretainedObject: 349 Changed |= lowerObjCCall(F, "objc_unretainedObject"); 350 break; 351 case Intrinsic::objc_unretainedPointer: 352 Changed |= lowerObjCCall(F, "objc_unretainedPointer"); 353 break; 354 case Intrinsic::objc_retain_autorelease: 355 Changed |= lowerObjCCall(F, "objc_retain_autorelease"); 356 break; 357 case Intrinsic::objc_sync_enter: 358 Changed |= lowerObjCCall(F, "objc_sync_enter"); 359 break; 360 case Intrinsic::objc_sync_exit: 361 Changed |= lowerObjCCall(F, "objc_sync_exit"); 362 break; 363 } 364 } 365 return Changed; 366 } 367 368 namespace { 369 370 class PreISelIntrinsicLoweringLegacyPass : public ModulePass { 371 public: 372 static char ID; 373 374 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {} 375 376 void getAnalysisUsage(AnalysisUsage &AU) const override { 377 AU.addRequired<TargetTransformInfoWrapperPass>(); 378 AU.addRequired<TargetPassConfig>(); 379 } 380 381 bool runOnModule(Module &M) override { 382 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & { 383 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 384 }; 385 386 const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>(); 387 PreISelIntrinsicLowering Lowering(TM, LookupTTI); 388 return Lowering.lowerIntrinsics(M); 389 } 390 }; 391 392 } // end anonymous namespace 393 394 char PreISelIntrinsicLoweringLegacyPass::ID; 395 396 INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass, 397 "pre-isel-intrinsic-lowering", 398 "Pre-ISel Intrinsic Lowering", false, false) 399 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) 400 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) 401 INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass, 402 "pre-isel-intrinsic-lowering", 403 "Pre-ISel Intrinsic Lowering", false, false) 404 405 ModulePass *llvm::createPreISelIntrinsicLoweringPass() { 406 return new PreISelIntrinsicLoweringLegacyPass(); 407 } 408 409 PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M, 410 ModuleAnalysisManager &AM) { 411 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 412 413 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & { 414 return FAM.getResult<TargetIRAnalysis>(F); 415 }; 416 417 PreISelIntrinsicLowering Lowering(TM, LookupTTI); 418 if (!Lowering.lowerIntrinsics(M)) 419 return PreservedAnalyses::all(); 420 else 421 return PreservedAnalyses::none(); 422 } 423