1 //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass implements IR lowering for the llvm.memcpy, llvm.memmove, 10 // llvm.memset, llvm.load.relative and llvm.objc.* intrinsics. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/PreISelIntrinsicLowering.h" 15 #include "llvm/Analysis/ObjCARCInstKind.h" 16 #include "llvm/Analysis/ObjCARCUtil.h" 17 #include "llvm/Analysis/TargetLibraryInfo.h" 18 #include "llvm/Analysis/TargetTransformInfo.h" 19 #include "llvm/CodeGen/ExpandVectorPredication.h" 20 #include "llvm/CodeGen/Passes.h" 21 #include "llvm/CodeGen/TargetLowering.h" 22 #include "llvm/CodeGen/TargetPassConfig.h" 23 #include "llvm/IR/Function.h" 24 #include "llvm/IR/IRBuilder.h" 25 #include "llvm/IR/Instructions.h" 26 #include "llvm/IR/IntrinsicInst.h" 27 #include "llvm/IR/Module.h" 28 #include "llvm/IR/RuntimeLibcalls.h" 29 #include "llvm/IR/Type.h" 30 #include "llvm/IR/Use.h" 31 #include "llvm/InitializePasses.h" 32 #include "llvm/Pass.h" 33 #include "llvm/Support/Casting.h" 34 #include "llvm/Target/TargetMachine.h" 35 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h" 36 #include "llvm/Transforms/Utils/BuildLibCalls.h" 37 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" 38 #include "llvm/Transforms/Utils/LowerVectorIntrinsics.h" 39 40 using namespace llvm; 41 42 /// Threshold to leave statically sized memory intrinsic calls. Calls of known 43 /// size larger than this will be expanded by the pass. Calls of unknown or 44 /// lower size will be left for expansion in codegen. 45 static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt( 46 "mem-intrinsic-expand-size", 47 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1), 48 cl::Hidden); 49 50 namespace { 51 52 struct PreISelIntrinsicLowering { 53 const TargetMachine *TM; 54 const function_ref<TargetTransformInfo &(Function &)> LookupTTI; 55 const function_ref<TargetLibraryInfo &(Function &)> LookupTLI; 56 57 /// If this is true, assume it's preferably to leave memory intrinsic calls 58 /// for replacement with a library call later. Otherwise this depends on 59 /// TargetLoweringInfo availability of the corresponding function. 60 const bool UseMemIntrinsicLibFunc; 61 62 explicit PreISelIntrinsicLowering( 63 const TargetMachine *TM_, 64 function_ref<TargetTransformInfo &(Function &)> LookupTTI_, 65 function_ref<TargetLibraryInfo &(Function &)> LookupTLI_, 66 bool UseMemIntrinsicLibFunc_ = true) 67 : TM(TM_), LookupTTI(LookupTTI_), LookupTLI(LookupTLI_), 68 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {} 69 70 static bool shouldExpandMemIntrinsicWithSize(Value *Size, 71 const TargetTransformInfo &TTI); 72 bool 73 expandMemIntrinsicUses(Function &F, 74 DenseMap<Constant *, GlobalVariable *> &CMap) const; 75 bool lowerIntrinsics(Module &M) const; 76 }; 77 78 } // namespace 79 80 template <class T> static bool forEachCall(Function &Intrin, T Callback) { 81 // Lowering all intrinsics in a function will delete multiple uses, so we 82 // can't use an early-inc-range. In case some remain, we don't want to look 83 // at them again. Unfortunately, Value::UseList is private, so we can't use a 84 // simple Use**. If LastUse is null, the next use to consider is 85 // Intrin.use_begin(), otherwise it's LastUse->getNext(). 86 Use *LastUse = nullptr; 87 bool Changed = false; 88 while (!Intrin.use_empty() && (!LastUse || LastUse->getNext())) { 89 Use *U = LastUse ? LastUse->getNext() : &*Intrin.use_begin(); 90 bool Removed = false; 91 // An intrinsic cannot have its address taken, so it cannot be an argument 92 // operand. It might be used as operand in debug metadata, though. 93 if (auto CI = dyn_cast<CallInst>(U->getUser())) 94 Changed |= Removed = Callback(CI); 95 if (!Removed) 96 LastUse = U; 97 } 98 return Changed; 99 } 100 101 static bool lowerLoadRelative(Function &F) { 102 if (F.use_empty()) 103 return false; 104 105 bool Changed = false; 106 Type *Int32Ty = Type::getInt32Ty(F.getContext()); 107 108 for (Use &U : llvm::make_early_inc_range(F.uses())) { 109 auto CI = dyn_cast<CallInst>(U.getUser()); 110 if (!CI || CI->getCalledOperand() != &F) 111 continue; 112 113 IRBuilder<> B(CI); 114 Value *OffsetPtr = 115 B.CreatePtrAdd(CI->getArgOperand(0), CI->getArgOperand(1)); 116 Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtr, Align(4)); 117 118 Value *ResultPtr = B.CreatePtrAdd(CI->getArgOperand(0), OffsetI32); 119 120 CI->replaceAllUsesWith(ResultPtr); 121 CI->eraseFromParent(); 122 Changed = true; 123 } 124 125 return Changed; 126 } 127 128 // ObjCARC has knowledge about whether an obj-c runtime function needs to be 129 // always tail-called or never tail-called. 130 static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) { 131 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(&F); 132 if (objcarc::IsAlwaysTail(Kind)) 133 return CallInst::TCK_Tail; 134 else if (objcarc::IsNeverTail(Kind)) 135 return CallInst::TCK_NoTail; 136 return CallInst::TCK_None; 137 } 138 139 static bool lowerObjCCall(Function &F, RTLIB::LibcallImpl NewFn, 140 bool setNonLazyBind = false) { 141 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) && 142 "Pre-ISel intrinsics do lower into regular function calls"); 143 if (F.use_empty()) 144 return false; 145 146 // FIXME: When RuntimeLibcalls is an analysis, check if the function is really 147 // supported, and go through RTLIB::Libcall. 148 const char *NewFnName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(NewFn); 149 150 // If we haven't already looked up this function, check to see if the 151 // program already contains a function with this name. 152 Module *M = F.getParent(); 153 FunctionCallee FCache = 154 M->getOrInsertFunction(NewFnName, F.getFunctionType()); 155 156 if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) { 157 Fn->setLinkage(F.getLinkage()); 158 if (setNonLazyBind && !Fn->isWeakForLinker()) { 159 // If we have Native ARC, set nonlazybind attribute for these APIs for 160 // performance. 161 Fn->addFnAttr(Attribute::NonLazyBind); 162 } 163 } 164 165 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F); 166 167 for (Use &U : llvm::make_early_inc_range(F.uses())) { 168 auto *CB = cast<CallBase>(U.getUser()); 169 170 if (CB->getCalledFunction() != &F) { 171 assert(objcarc::getAttachedARCFunction(CB) == &F && 172 "use expected to be the argument of operand bundle " 173 "\"clang.arc.attachedcall\""); 174 U.set(FCache.getCallee()); 175 continue; 176 } 177 178 auto *CI = cast<CallInst>(CB); 179 assert(CI->getCalledFunction() && "Cannot lower an indirect call!"); 180 181 IRBuilder<> Builder(CI->getParent(), CI->getIterator()); 182 SmallVector<Value *, 8> Args(CI->args()); 183 SmallVector<llvm::OperandBundleDef, 1> BundleList; 184 CI->getOperandBundlesAsDefs(BundleList); 185 CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList); 186 NewCI->setName(CI->getName()); 187 188 // Try to set the most appropriate TailCallKind based on both the current 189 // attributes and the ones that we could get from ObjCARC's special 190 // knowledge of the runtime functions. 191 // 192 // std::max respects both requirements of notail and tail here: 193 // * notail on either the call or from ObjCARC becomes notail 194 // * tail on either side is stronger than none, but not notail 195 CallInst::TailCallKind TCK = CI->getTailCallKind(); 196 NewCI->setTailCallKind(std::max(TCK, OverridingTCK)); 197 198 // Transfer the 'returned' attribute from the intrinsic to the call site. 199 // By applying this only to intrinsic call sites, we avoid applying it to 200 // non-ARC explicit calls to things like objc_retain which have not been 201 // auto-upgraded to use the intrinsics. 202 unsigned Index; 203 if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) && 204 Index) 205 NewCI->addParamAttr(Index - AttributeList::FirstArgIndex, 206 Attribute::Returned); 207 208 if (!CI->use_empty()) 209 CI->replaceAllUsesWith(NewCI); 210 CI->eraseFromParent(); 211 } 212 213 return true; 214 } 215 216 // TODO: Should refine based on estimated number of accesses (e.g. does it 217 // require splitting based on alignment) 218 bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize( 219 Value *Size, const TargetTransformInfo &TTI) { 220 ConstantInt *CI = dyn_cast<ConstantInt>(Size); 221 if (!CI) 222 return true; 223 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences() 224 ? MemIntrinsicExpandSizeThresholdOpt 225 : TTI.getMaxMemIntrinsicInlineSizeThreshold(); 226 uint64_t SizeVal = CI->getZExtValue(); 227 228 // Treat a threshold of 0 as a special case to force expansion of all 229 // intrinsics, including size 0. 230 return SizeVal > Threshold || Threshold == 0; 231 } 232 233 static bool canEmitLibcall(const TargetMachine *TM, Function *F, 234 RTLIB::Libcall LC) { 235 // TODO: Should this consider the address space of the memcpy? 236 if (!TM) 237 return true; 238 const TargetLowering *TLI = TM->getSubtargetImpl(*F)->getTargetLowering(); 239 return TLI->getLibcallName(LC) != nullptr; 240 } 241 242 static bool canEmitMemcpy(const TargetMachine *TM, Function *F) { 243 // TODO: Should this consider the address space of the memcpy? 244 if (!TM) 245 return true; 246 const TargetLowering *TLI = TM->getSubtargetImpl(*F)->getTargetLowering(); 247 return TLI->getMemcpyName() != nullptr; 248 } 249 250 // Return a value appropriate for use with the memset_pattern16 libcall, if 251 // possible and if we know how. (Adapted from equivalent helper in 252 // LoopIdiomRecognize). 253 static Constant *getMemSetPattern16Value(MemSetPatternInst *Inst, 254 const TargetLibraryInfo &TLI) { 255 // TODO: This could check for UndefValue because it can be merged into any 256 // other valid pattern. 257 258 // Don't emit libcalls if a non-default address space is being used. 259 if (Inst->getRawDest()->getType()->getPointerAddressSpace() != 0) 260 return nullptr; 261 262 Value *V = Inst->getValue(); 263 Type *VTy = V->getType(); 264 const DataLayout &DL = Inst->getDataLayout(); 265 Module *M = Inst->getModule(); 266 267 if (!isLibFuncEmittable(M, &TLI, LibFunc_memset_pattern16)) 268 return nullptr; 269 270 // If the value isn't a constant, we can't promote it to being in a constant 271 // array. We could theoretically do a store to an alloca or something, but 272 // that doesn't seem worthwhile. 273 Constant *C = dyn_cast<Constant>(V); 274 if (!C || isa<ConstantExpr>(C)) 275 return nullptr; 276 277 // Only handle simple values that are a power of two bytes in size. 278 uint64_t Size = DL.getTypeSizeInBits(VTy); 279 if (!DL.typeSizeEqualsStoreSize(VTy) || !isPowerOf2_64(Size)) 280 return nullptr; 281 282 // Don't care enough about darwin/ppc to implement this. 283 if (DL.isBigEndian()) 284 return nullptr; 285 286 // Convert to size in bytes. 287 Size /= 8; 288 289 // TODO: If CI is larger than 16-bytes, we can try slicing it in half to see 290 // if the top and bottom are the same (e.g. for vectors and large integers). 291 if (Size > 16) 292 return nullptr; 293 294 // If the constant is exactly 16 bytes, just use it. 295 if (Size == 16) 296 return C; 297 298 // Otherwise, we'll use an array of the constants. 299 uint64_t ArraySize = 16 / Size; 300 ArrayType *AT = ArrayType::get(V->getType(), ArraySize); 301 return ConstantArray::get(AT, std::vector<Constant *>(ArraySize, C)); 302 } 303 304 // TODO: Handle atomic memcpy and memcpy.inline 305 // TODO: Pass ScalarEvolution 306 bool PreISelIntrinsicLowering::expandMemIntrinsicUses( 307 Function &F, DenseMap<Constant *, GlobalVariable *> &CMap) const { 308 Intrinsic::ID ID = F.getIntrinsicID(); 309 bool Changed = false; 310 311 for (User *U : llvm::make_early_inc_range(F.users())) { 312 Instruction *Inst = cast<Instruction>(U); 313 314 switch (ID) { 315 case Intrinsic::memcpy: { 316 auto *Memcpy = cast<MemCpyInst>(Inst); 317 Function *ParentFunc = Memcpy->getFunction(); 318 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 319 if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) { 320 if (UseMemIntrinsicLibFunc && canEmitMemcpy(TM, ParentFunc)) 321 break; 322 323 // TODO: For optsize, emit the loop into a separate function 324 expandMemCpyAsLoop(Memcpy, TTI); 325 Changed = true; 326 Memcpy->eraseFromParent(); 327 } 328 329 break; 330 } 331 case Intrinsic::memcpy_inline: { 332 // Only expand llvm.memcpy.inline with non-constant length in this 333 // codepath, leaving the current SelectionDAG expansion for constant 334 // length memcpy intrinsics undisturbed. 335 auto *Memcpy = cast<MemCpyInst>(Inst); 336 if (isa<ConstantInt>(Memcpy->getLength())) 337 break; 338 339 Function *ParentFunc = Memcpy->getFunction(); 340 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 341 expandMemCpyAsLoop(Memcpy, TTI); 342 Changed = true; 343 Memcpy->eraseFromParent(); 344 break; 345 } 346 case Intrinsic::memmove: { 347 auto *Memmove = cast<MemMoveInst>(Inst); 348 Function *ParentFunc = Memmove->getFunction(); 349 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 350 if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) { 351 if (UseMemIntrinsicLibFunc && 352 canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE)) 353 break; 354 355 if (expandMemMoveAsLoop(Memmove, TTI)) { 356 Changed = true; 357 Memmove->eraseFromParent(); 358 } 359 } 360 361 break; 362 } 363 case Intrinsic::memset: { 364 auto *Memset = cast<MemSetInst>(Inst); 365 Function *ParentFunc = Memset->getFunction(); 366 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 367 if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) { 368 if (UseMemIntrinsicLibFunc && 369 canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET)) 370 break; 371 372 expandMemSetAsLoop(Memset); 373 Changed = true; 374 Memset->eraseFromParent(); 375 } 376 377 break; 378 } 379 case Intrinsic::memset_inline: { 380 // Only expand llvm.memset.inline with non-constant length in this 381 // codepath, leaving the current SelectionDAG expansion for constant 382 // length memset intrinsics undisturbed. 383 auto *Memset = cast<MemSetInst>(Inst); 384 if (isa<ConstantInt>(Memset->getLength())) 385 break; 386 387 expandMemSetAsLoop(Memset); 388 Changed = true; 389 Memset->eraseFromParent(); 390 break; 391 } 392 case Intrinsic::experimental_memset_pattern: { 393 auto *Memset = cast<MemSetPatternInst>(Inst); 394 const TargetLibraryInfo &TLI = LookupTLI(*Memset->getFunction()); 395 Constant *PatternValue = getMemSetPattern16Value(Memset, TLI); 396 if (!PatternValue) { 397 // If it isn't possible to emit a memset_pattern16 libcall, expand to 398 // a loop instead. 399 expandMemSetPatternAsLoop(Memset); 400 Changed = true; 401 Memset->eraseFromParent(); 402 break; 403 } 404 // FIXME: There is currently no profitability calculation for emitting 405 // the libcall vs expanding the memset.pattern directly. 406 IRBuilder<> Builder(Inst); 407 Module *M = Memset->getModule(); 408 const DataLayout &DL = Memset->getDataLayout(); 409 410 Type *DestPtrTy = Memset->getRawDest()->getType(); 411 Type *SizeTTy = TLI.getSizeTType(*M); 412 StringRef FuncName = "memset_pattern16"; 413 FunctionCallee MSP = getOrInsertLibFunc(M, TLI, LibFunc_memset_pattern16, 414 Builder.getVoidTy(), DestPtrTy, 415 Builder.getPtrTy(), SizeTTy); 416 inferNonMandatoryLibFuncAttrs(M, FuncName, TLI); 417 418 // Otherwise we should form a memset_pattern16. PatternValue is known 419 // to be an constant array of 16-bytes. Put the value into a mergable 420 // global. 421 assert(Memset->getRawDest()->getType()->getPointerAddressSpace() == 0 && 422 "Should have skipped if non-zero AS"); 423 GlobalVariable *GV; 424 auto It = CMap.find(PatternValue); 425 if (It != CMap.end()) { 426 GV = It->second; 427 } else { 428 GV = new GlobalVariable( 429 *M, PatternValue->getType(), /*isConstant=*/true, 430 GlobalValue::PrivateLinkage, PatternValue, ".memset_pattern"); 431 GV->setUnnamedAddr( 432 GlobalValue::UnnamedAddr::Global); // Ok to merge these. 433 // TODO: Consider relaxing alignment requirement. 434 GV->setAlignment(Align(16)); 435 CMap[PatternValue] = GV; 436 } 437 Value *PatternPtr = GV; 438 Value *NumBytes = Builder.CreateMul( 439 TLI.getAsSizeT(DL.getTypeAllocSize(Memset->getValue()->getType()), 440 *M), 441 Builder.CreateZExtOrTrunc(Memset->getLength(), SizeTTy)); 442 CallInst *MemsetPattern16Call = 443 Builder.CreateCall(MSP, {Memset->getRawDest(), PatternPtr, NumBytes}); 444 MemsetPattern16Call->setAAMetadata(Memset->getAAMetadata()); 445 // Preserve any call site attributes on the destination pointer 446 // argument (e.g. alignment). 447 AttrBuilder ArgAttrs(Memset->getContext(), 448 Memset->getAttributes().getParamAttrs(0)); 449 MemsetPattern16Call->setAttributes( 450 MemsetPattern16Call->getAttributes().addParamAttributes( 451 Memset->getContext(), 0, ArgAttrs)); 452 Changed = true; 453 Memset->eraseFromParent(); 454 break; 455 } 456 default: 457 llvm_unreachable("unhandled intrinsic"); 458 } 459 } 460 461 return Changed; 462 } 463 464 bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { 465 // Map unique constants to globals. 466 DenseMap<Constant *, GlobalVariable *> CMap; 467 bool Changed = false; 468 for (Function &F : M) { 469 switch (F.getIntrinsicID()) { 470 default: 471 break; 472 case Intrinsic::memcpy: 473 case Intrinsic::memcpy_inline: 474 case Intrinsic::memmove: 475 case Intrinsic::memset: 476 case Intrinsic::memset_inline: 477 case Intrinsic::experimental_memset_pattern: 478 Changed |= expandMemIntrinsicUses(F, CMap); 479 break; 480 case Intrinsic::load_relative: 481 Changed |= lowerLoadRelative(F); 482 break; 483 case Intrinsic::is_constant: 484 case Intrinsic::objectsize: 485 Changed |= forEachCall(F, [&](CallInst *CI) { 486 Function *Parent = CI->getParent()->getParent(); 487 TargetLibraryInfo &TLI = LookupTLI(*Parent); 488 // Intrinsics in unreachable code are not lowered. 489 bool Changed = lowerConstantIntrinsics(*Parent, TLI, /*DT=*/nullptr); 490 return Changed; 491 }); 492 break; 493 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \ 494 case Intrinsic::VPID: 495 #include "llvm/IR/VPIntrinsics.def" 496 forEachCall(F, [&](CallInst *CI) { 497 Function *Parent = CI->getParent()->getParent(); 498 const TargetTransformInfo &TTI = LookupTTI(*Parent); 499 auto *VPI = cast<VPIntrinsic>(CI); 500 VPExpansionDetails ED = expandVectorPredicationIntrinsic(*VPI, TTI); 501 // Expansion of VP intrinsics may change the IR but not actually 502 // replace the intrinsic, so update Changed for the pass 503 // and compute Removed for forEachCall. 504 Changed |= ED != VPExpansionDetails::IntrinsicUnchanged; 505 bool Removed = ED == VPExpansionDetails::IntrinsicReplaced; 506 return Removed; 507 }); 508 break; 509 case Intrinsic::objc_autorelease: 510 Changed |= lowerObjCCall(F, RTLIB::objc_autorelease); 511 break; 512 case Intrinsic::objc_autoreleasePoolPop: 513 Changed |= lowerObjCCall(F, RTLIB::objc_autoreleasePoolPop); 514 break; 515 case Intrinsic::objc_autoreleasePoolPush: 516 Changed |= lowerObjCCall(F, RTLIB::objc_autoreleasePoolPush); 517 break; 518 case Intrinsic::objc_autoreleaseReturnValue: 519 Changed |= lowerObjCCall(F, RTLIB::objc_autoreleaseReturnValue); 520 break; 521 case Intrinsic::objc_copyWeak: 522 Changed |= lowerObjCCall(F, RTLIB::objc_copyWeak); 523 break; 524 case Intrinsic::objc_destroyWeak: 525 Changed |= lowerObjCCall(F, RTLIB::objc_destroyWeak); 526 break; 527 case Intrinsic::objc_initWeak: 528 Changed |= lowerObjCCall(F, RTLIB::objc_initWeak); 529 break; 530 case Intrinsic::objc_loadWeak: 531 Changed |= lowerObjCCall(F, RTLIB::objc_loadWeak); 532 break; 533 case Intrinsic::objc_loadWeakRetained: 534 Changed |= lowerObjCCall(F, RTLIB::objc_loadWeakRetained); 535 break; 536 case Intrinsic::objc_moveWeak: 537 Changed |= lowerObjCCall(F, RTLIB::objc_moveWeak); 538 break; 539 case Intrinsic::objc_release: 540 Changed |= lowerObjCCall(F, RTLIB::objc_release, true); 541 break; 542 case Intrinsic::objc_retain: 543 Changed |= lowerObjCCall(F, RTLIB::objc_retain, true); 544 break; 545 case Intrinsic::objc_retainAutorelease: 546 Changed |= lowerObjCCall(F, RTLIB::objc_retainAutorelease); 547 break; 548 case Intrinsic::objc_retainAutoreleaseReturnValue: 549 Changed |= lowerObjCCall(F, RTLIB::objc_retainAutoreleaseReturnValue); 550 break; 551 case Intrinsic::objc_retainAutoreleasedReturnValue: 552 Changed |= lowerObjCCall(F, RTLIB::objc_retainAutoreleasedReturnValue); 553 break; 554 case Intrinsic::objc_claimAutoreleasedReturnValue: 555 Changed |= lowerObjCCall(F, RTLIB::objc_claimAutoreleasedReturnValue); 556 break; 557 case Intrinsic::objc_retainBlock: 558 Changed |= lowerObjCCall(F, RTLIB::objc_retainBlock); 559 break; 560 case Intrinsic::objc_storeStrong: 561 Changed |= lowerObjCCall(F, RTLIB::objc_storeStrong); 562 break; 563 case Intrinsic::objc_storeWeak: 564 Changed |= lowerObjCCall(F, RTLIB::objc_storeWeak); 565 break; 566 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue: 567 Changed |= 568 lowerObjCCall(F, RTLIB::objc_unsafeClaimAutoreleasedReturnValue); 569 break; 570 case Intrinsic::objc_retainedObject: 571 Changed |= lowerObjCCall(F, RTLIB::objc_retainedObject); 572 break; 573 case Intrinsic::objc_unretainedObject: 574 Changed |= lowerObjCCall(F, RTLIB::objc_unretainedObject); 575 break; 576 case Intrinsic::objc_unretainedPointer: 577 Changed |= lowerObjCCall(F, RTLIB::objc_unretainedPointer); 578 break; 579 case Intrinsic::objc_retain_autorelease: 580 Changed |= lowerObjCCall(F, RTLIB::objc_retain_autorelease); 581 break; 582 case Intrinsic::objc_sync_enter: 583 Changed |= lowerObjCCall(F, RTLIB::objc_sync_enter); 584 break; 585 case Intrinsic::objc_sync_exit: 586 Changed |= lowerObjCCall(F, RTLIB::objc_sync_exit); 587 break; 588 case Intrinsic::exp: 589 case Intrinsic::exp2: 590 Changed |= forEachCall(F, [&](CallInst *CI) { 591 Type *Ty = CI->getArgOperand(0)->getType(); 592 if (!isa<ScalableVectorType>(Ty)) 593 return false; 594 const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering(); 595 unsigned Op = TL->IntrinsicIDToISD(F.getIntrinsicID()); 596 if (!TL->isOperationExpand(Op, EVT::getEVT(Ty))) 597 return false; 598 return lowerUnaryVectorIntrinsicAsLoop(M, CI); 599 }); 600 break; 601 } 602 } 603 return Changed; 604 } 605 606 namespace { 607 608 class PreISelIntrinsicLoweringLegacyPass : public ModulePass { 609 public: 610 static char ID; 611 612 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {} 613 614 void getAnalysisUsage(AnalysisUsage &AU) const override { 615 AU.addRequired<TargetTransformInfoWrapperPass>(); 616 AU.addRequired<TargetLibraryInfoWrapperPass>(); 617 AU.addRequired<TargetPassConfig>(); 618 } 619 620 bool runOnModule(Module &M) override { 621 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & { 622 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 623 }; 624 auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & { 625 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); 626 }; 627 628 const auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>(); 629 PreISelIntrinsicLowering Lowering(TM, LookupTTI, LookupTLI); 630 return Lowering.lowerIntrinsics(M); 631 } 632 }; 633 634 } // end anonymous namespace 635 636 char PreISelIntrinsicLoweringLegacyPass::ID; 637 638 INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass, 639 "pre-isel-intrinsic-lowering", 640 "Pre-ISel Intrinsic Lowering", false, false) 641 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 642 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) 643 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) 644 INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass, 645 "pre-isel-intrinsic-lowering", 646 "Pre-ISel Intrinsic Lowering", false, false) 647 648 ModulePass *llvm::createPreISelIntrinsicLoweringPass() { 649 return new PreISelIntrinsicLoweringLegacyPass(); 650 } 651 652 PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M, 653 ModuleAnalysisManager &AM) { 654 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 655 656 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & { 657 return FAM.getResult<TargetIRAnalysis>(F); 658 }; 659 auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & { 660 return FAM.getResult<TargetLibraryAnalysis>(F); 661 }; 662 663 PreISelIntrinsicLowering Lowering(TM, LookupTTI, LookupTLI); 664 if (!Lowering.lowerIntrinsics(M)) 665 return PreservedAnalyses::all(); 666 else 667 return PreservedAnalyses::none(); 668 } 669