1 //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass implements IR lowering for the llvm.memcpy, llvm.memmove, 10 // llvm.memset, llvm.load.relative and llvm.objc.* intrinsics. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/PreISelIntrinsicLowering.h" 15 #include "llvm/Analysis/ObjCARCInstKind.h" 16 #include "llvm/Analysis/ObjCARCUtil.h" 17 #include "llvm/Analysis/TargetTransformInfo.h" 18 #include "llvm/CodeGen/Passes.h" 19 #include "llvm/CodeGen/TargetLowering.h" 20 #include "llvm/CodeGen/TargetPassConfig.h" 21 #include "llvm/IR/Function.h" 22 #include "llvm/IR/IRBuilder.h" 23 #include "llvm/IR/Instructions.h" 24 #include "llvm/IR/IntrinsicInst.h" 25 #include "llvm/IR/Module.h" 26 #include "llvm/IR/Type.h" 27 #include "llvm/InitializePasses.h" 28 #include "llvm/Pass.h" 29 #include "llvm/Support/Casting.h" 30 #include "llvm/Target/TargetMachine.h" 31 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" 32 33 using namespace llvm; 34 35 /// Threshold to leave statically sized memory intrinsic calls. Calls of known 36 /// size larger than this will be expanded by the pass. Calls of unknown or 37 /// lower size will be left for expansion in codegen. 38 static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt( 39 "mem-intrinsic-expand-size", 40 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1), 41 cl::Hidden); 42 43 namespace { 44 45 struct PreISelIntrinsicLowering { 46 const TargetMachine &TM; 47 const function_ref<TargetTransformInfo &(Function &)> LookupTTI; 48 49 /// If this is true, assume it's preferably to leave memory intrinsic calls 50 /// for replacement with a library call later. Otherwise this depends on 51 /// TargetLoweringInfo availability of the corresponding function. 52 const bool UseMemIntrinsicLibFunc; 53 54 explicit PreISelIntrinsicLowering( 55 const TargetMachine &TM_, 56 function_ref<TargetTransformInfo &(Function &)> LookupTTI_, 57 bool UseMemIntrinsicLibFunc_ = true) 58 : TM(TM_), LookupTTI(LookupTTI_), 59 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {} 60 61 static bool shouldExpandMemIntrinsicWithSize(Value *Size, 62 const TargetTransformInfo &TTI); 63 bool expandMemIntrinsicUses(Function &F) const; 64 bool lowerIntrinsics(Module &M) const; 65 }; 66 67 } // namespace 68 69 static bool lowerLoadRelative(Function &F) { 70 if (F.use_empty()) 71 return false; 72 73 bool Changed = false; 74 Type *Int32Ty = Type::getInt32Ty(F.getContext()); 75 76 for (Use &U : llvm::make_early_inc_range(F.uses())) { 77 auto CI = dyn_cast<CallInst>(U.getUser()); 78 if (!CI || CI->getCalledOperand() != &F) 79 continue; 80 81 IRBuilder<> B(CI); 82 Value *OffsetPtr = 83 B.CreatePtrAdd(CI->getArgOperand(0), CI->getArgOperand(1)); 84 Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtr, Align(4)); 85 86 Value *ResultPtr = B.CreatePtrAdd(CI->getArgOperand(0), OffsetI32); 87 88 CI->replaceAllUsesWith(ResultPtr); 89 CI->eraseFromParent(); 90 Changed = true; 91 } 92 93 return Changed; 94 } 95 96 // ObjCARC has knowledge about whether an obj-c runtime function needs to be 97 // always tail-called or never tail-called. 98 static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) { 99 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(&F); 100 if (objcarc::IsAlwaysTail(Kind)) 101 return CallInst::TCK_Tail; 102 else if (objcarc::IsNeverTail(Kind)) 103 return CallInst::TCK_NoTail; 104 return CallInst::TCK_None; 105 } 106 107 static bool lowerObjCCall(Function &F, const char *NewFn, 108 bool setNonLazyBind = false) { 109 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) && 110 "Pre-ISel intrinsics do lower into regular function calls"); 111 if (F.use_empty()) 112 return false; 113 114 // If we haven't already looked up this function, check to see if the 115 // program already contains a function with this name. 116 Module *M = F.getParent(); 117 FunctionCallee FCache = M->getOrInsertFunction(NewFn, F.getFunctionType()); 118 119 if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) { 120 Fn->setLinkage(F.getLinkage()); 121 if (setNonLazyBind && !Fn->isWeakForLinker()) { 122 // If we have Native ARC, set nonlazybind attribute for these APIs for 123 // performance. 124 Fn->addFnAttr(Attribute::NonLazyBind); 125 } 126 } 127 128 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F); 129 130 for (Use &U : llvm::make_early_inc_range(F.uses())) { 131 auto *CB = cast<CallBase>(U.getUser()); 132 133 if (CB->getCalledFunction() != &F) { 134 objcarc::ARCInstKind Kind = objcarc::getAttachedARCFunctionKind(CB); 135 (void)Kind; 136 assert((Kind == objcarc::ARCInstKind::RetainRV || 137 Kind == objcarc::ARCInstKind::UnsafeClaimRV) && 138 "use expected to be the argument of operand bundle " 139 "\"clang.arc.attachedcall\""); 140 U.set(FCache.getCallee()); 141 continue; 142 } 143 144 auto *CI = cast<CallInst>(CB); 145 assert(CI->getCalledFunction() && "Cannot lower an indirect call!"); 146 147 IRBuilder<> Builder(CI->getParent(), CI->getIterator()); 148 SmallVector<Value *, 8> Args(CI->args()); 149 SmallVector<llvm::OperandBundleDef, 1> BundleList; 150 CI->getOperandBundlesAsDefs(BundleList); 151 CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList); 152 NewCI->setName(CI->getName()); 153 154 // Try to set the most appropriate TailCallKind based on both the current 155 // attributes and the ones that we could get from ObjCARC's special 156 // knowledge of the runtime functions. 157 // 158 // std::max respects both requirements of notail and tail here: 159 // * notail on either the call or from ObjCARC becomes notail 160 // * tail on either side is stronger than none, but not notail 161 CallInst::TailCallKind TCK = CI->getTailCallKind(); 162 NewCI->setTailCallKind(std::max(TCK, OverridingTCK)); 163 164 // Transfer the 'returned' attribute from the intrinsic to the call site. 165 // By applying this only to intrinsic call sites, we avoid applying it to 166 // non-ARC explicit calls to things like objc_retain which have not been 167 // auto-upgraded to use the intrinsics. 168 unsigned Index; 169 if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) && 170 Index) 171 NewCI->addParamAttr(Index - AttributeList::FirstArgIndex, 172 Attribute::Returned); 173 174 if (!CI->use_empty()) 175 CI->replaceAllUsesWith(NewCI); 176 CI->eraseFromParent(); 177 } 178 179 return true; 180 } 181 182 // TODO: Should refine based on estimated number of accesses (e.g. does it 183 // require splitting based on alignment) 184 bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize( 185 Value *Size, const TargetTransformInfo &TTI) { 186 ConstantInt *CI = dyn_cast<ConstantInt>(Size); 187 if (!CI) 188 return true; 189 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences() 190 ? MemIntrinsicExpandSizeThresholdOpt 191 : TTI.getMaxMemIntrinsicInlineSizeThreshold(); 192 uint64_t SizeVal = CI->getZExtValue(); 193 194 // Treat a threshold of 0 as a special case to force expansion of all 195 // intrinsics, including size 0. 196 return SizeVal > Threshold || Threshold == 0; 197 } 198 199 static bool canEmitLibcall(const TargetMachine &TM, Function *F, 200 RTLIB::Libcall LC) { 201 // TODO: Should this consider the address space of the memcpy? 202 const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering(); 203 return TLI->getLibcallName(LC) != nullptr; 204 } 205 206 // TODO: Handle atomic memcpy and memcpy.inline 207 // TODO: Pass ScalarEvolution 208 bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const { 209 Intrinsic::ID ID = F.getIntrinsicID(); 210 bool Changed = false; 211 212 for (User *U : llvm::make_early_inc_range(F.users())) { 213 Instruction *Inst = cast<Instruction>(U); 214 215 switch (ID) { 216 case Intrinsic::memcpy: { 217 auto *Memcpy = cast<MemCpyInst>(Inst); 218 Function *ParentFunc = Memcpy->getFunction(); 219 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 220 if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) { 221 if (UseMemIntrinsicLibFunc && 222 canEmitLibcall(TM, ParentFunc, RTLIB::MEMCPY)) 223 break; 224 225 // TODO: For optsize, emit the loop into a separate function 226 expandMemCpyAsLoop(Memcpy, TTI); 227 Changed = true; 228 Memcpy->eraseFromParent(); 229 } 230 231 break; 232 } 233 case Intrinsic::memcpy_inline: { 234 // Only expand llvm.memcpy.inline with non-constant length in this 235 // codepath, leaving the current SelectionDAG expansion for constant 236 // length memcpy intrinsics undisturbed. 237 auto *Memcpy = cast<MemCpyInlineInst>(Inst); 238 if (isa<ConstantInt>(Memcpy->getLength())) 239 break; 240 241 Function *ParentFunc = Memcpy->getFunction(); 242 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 243 expandMemCpyAsLoop(Memcpy, TTI); 244 Changed = true; 245 Memcpy->eraseFromParent(); 246 break; 247 } 248 case Intrinsic::memmove: { 249 auto *Memmove = cast<MemMoveInst>(Inst); 250 Function *ParentFunc = Memmove->getFunction(); 251 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 252 if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) { 253 if (UseMemIntrinsicLibFunc && 254 canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE)) 255 break; 256 257 if (expandMemMoveAsLoop(Memmove, TTI)) { 258 Changed = true; 259 Memmove->eraseFromParent(); 260 } 261 } 262 263 break; 264 } 265 case Intrinsic::memset: { 266 auto *Memset = cast<MemSetInst>(Inst); 267 Function *ParentFunc = Memset->getFunction(); 268 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 269 if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) { 270 if (UseMemIntrinsicLibFunc && 271 canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET)) 272 break; 273 274 expandMemSetAsLoop(Memset); 275 Changed = true; 276 Memset->eraseFromParent(); 277 } 278 279 break; 280 } 281 case Intrinsic::memset_inline: { 282 // Only expand llvm.memset.inline with non-constant length in this 283 // codepath, leaving the current SelectionDAG expansion for constant 284 // length memset intrinsics undisturbed. 285 auto *Memset = cast<MemSetInlineInst>(Inst); 286 if (isa<ConstantInt>(Memset->getLength())) 287 break; 288 289 expandMemSetAsLoop(Memset); 290 Changed = true; 291 Memset->eraseFromParent(); 292 break; 293 } 294 default: 295 llvm_unreachable("unhandled intrinsic"); 296 } 297 } 298 299 return Changed; 300 } 301 302 bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { 303 bool Changed = false; 304 for (Function &F : M) { 305 switch (F.getIntrinsicID()) { 306 default: 307 break; 308 case Intrinsic::memcpy: 309 case Intrinsic::memcpy_inline: 310 case Intrinsic::memmove: 311 case Intrinsic::memset: 312 case Intrinsic::memset_inline: 313 Changed |= expandMemIntrinsicUses(F); 314 break; 315 case Intrinsic::load_relative: 316 Changed |= lowerLoadRelative(F); 317 break; 318 case Intrinsic::objc_autorelease: 319 Changed |= lowerObjCCall(F, "objc_autorelease"); 320 break; 321 case Intrinsic::objc_autoreleasePoolPop: 322 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPop"); 323 break; 324 case Intrinsic::objc_autoreleasePoolPush: 325 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPush"); 326 break; 327 case Intrinsic::objc_autoreleaseReturnValue: 328 Changed |= lowerObjCCall(F, "objc_autoreleaseReturnValue"); 329 break; 330 case Intrinsic::objc_copyWeak: 331 Changed |= lowerObjCCall(F, "objc_copyWeak"); 332 break; 333 case Intrinsic::objc_destroyWeak: 334 Changed |= lowerObjCCall(F, "objc_destroyWeak"); 335 break; 336 case Intrinsic::objc_initWeak: 337 Changed |= lowerObjCCall(F, "objc_initWeak"); 338 break; 339 case Intrinsic::objc_loadWeak: 340 Changed |= lowerObjCCall(F, "objc_loadWeak"); 341 break; 342 case Intrinsic::objc_loadWeakRetained: 343 Changed |= lowerObjCCall(F, "objc_loadWeakRetained"); 344 break; 345 case Intrinsic::objc_moveWeak: 346 Changed |= lowerObjCCall(F, "objc_moveWeak"); 347 break; 348 case Intrinsic::objc_release: 349 Changed |= lowerObjCCall(F, "objc_release", true); 350 break; 351 case Intrinsic::objc_retain: 352 Changed |= lowerObjCCall(F, "objc_retain", true); 353 break; 354 case Intrinsic::objc_retainAutorelease: 355 Changed |= lowerObjCCall(F, "objc_retainAutorelease"); 356 break; 357 case Intrinsic::objc_retainAutoreleaseReturnValue: 358 Changed |= lowerObjCCall(F, "objc_retainAutoreleaseReturnValue"); 359 break; 360 case Intrinsic::objc_retainAutoreleasedReturnValue: 361 Changed |= lowerObjCCall(F, "objc_retainAutoreleasedReturnValue"); 362 break; 363 case Intrinsic::objc_retainBlock: 364 Changed |= lowerObjCCall(F, "objc_retainBlock"); 365 break; 366 case Intrinsic::objc_storeStrong: 367 Changed |= lowerObjCCall(F, "objc_storeStrong"); 368 break; 369 case Intrinsic::objc_storeWeak: 370 Changed |= lowerObjCCall(F, "objc_storeWeak"); 371 break; 372 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue: 373 Changed |= lowerObjCCall(F, "objc_unsafeClaimAutoreleasedReturnValue"); 374 break; 375 case Intrinsic::objc_retainedObject: 376 Changed |= lowerObjCCall(F, "objc_retainedObject"); 377 break; 378 case Intrinsic::objc_unretainedObject: 379 Changed |= lowerObjCCall(F, "objc_unretainedObject"); 380 break; 381 case Intrinsic::objc_unretainedPointer: 382 Changed |= lowerObjCCall(F, "objc_unretainedPointer"); 383 break; 384 case Intrinsic::objc_retain_autorelease: 385 Changed |= lowerObjCCall(F, "objc_retain_autorelease"); 386 break; 387 case Intrinsic::objc_sync_enter: 388 Changed |= lowerObjCCall(F, "objc_sync_enter"); 389 break; 390 case Intrinsic::objc_sync_exit: 391 Changed |= lowerObjCCall(F, "objc_sync_exit"); 392 break; 393 } 394 } 395 return Changed; 396 } 397 398 namespace { 399 400 class PreISelIntrinsicLoweringLegacyPass : public ModulePass { 401 public: 402 static char ID; 403 404 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {} 405 406 void getAnalysisUsage(AnalysisUsage &AU) const override { 407 AU.addRequired<TargetTransformInfoWrapperPass>(); 408 AU.addRequired<TargetPassConfig>(); 409 } 410 411 bool runOnModule(Module &M) override { 412 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & { 413 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 414 }; 415 416 const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>(); 417 PreISelIntrinsicLowering Lowering(TM, LookupTTI); 418 return Lowering.lowerIntrinsics(M); 419 } 420 }; 421 422 } // end anonymous namespace 423 424 char PreISelIntrinsicLoweringLegacyPass::ID; 425 426 INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass, 427 "pre-isel-intrinsic-lowering", 428 "Pre-ISel Intrinsic Lowering", false, false) 429 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) 430 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) 431 INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass, 432 "pre-isel-intrinsic-lowering", 433 "Pre-ISel Intrinsic Lowering", false, false) 434 435 ModulePass *llvm::createPreISelIntrinsicLoweringPass() { 436 return new PreISelIntrinsicLoweringLegacyPass(); 437 } 438 439 PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M, 440 ModuleAnalysisManager &AM) { 441 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 442 443 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & { 444 return FAM.getResult<TargetIRAnalysis>(F); 445 }; 446 447 PreISelIntrinsicLowering Lowering(TM, LookupTTI); 448 if (!Lowering.lowerIntrinsics(M)) 449 return PreservedAnalyses::all(); 450 else 451 return PreservedAnalyses::none(); 452 } 453