1 //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10 // llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
15 #include "llvm/Analysis/ObjCARCInstKind.h"
16 #include "llvm/Analysis/ObjCARCUtil.h"
17 #include "llvm/Analysis/TargetTransformInfo.h"
18 #include "llvm/CodeGen/Passes.h"
19 #include "llvm/CodeGen/TargetLowering.h"
20 #include "llvm/CodeGen/TargetPassConfig.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/IRBuilder.h"
23 #include "llvm/IR/Instructions.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/Module.h"
26 #include "llvm/IR/Type.h"
27 #include "llvm/InitializePasses.h"
28 #include "llvm/Pass.h"
29 #include "llvm/Support/Casting.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
32
33 using namespace llvm;
34
35 /// Threshold to leave statically sized memory intrinsic calls. Calls of known
36 /// size larger than this will be expanded by the pass. Calls of unknown or
37 /// lower size will be left for expansion in codegen.
38 static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
39 "mem-intrinsic-expand-size",
40 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1),
41 cl::Hidden);
42
43 namespace {
44
45 struct PreISelIntrinsicLowering {
46 const TargetMachine &TM;
47 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
48
49 /// If this is true, assume it's preferably to leave memory intrinsic calls
50 /// for replacement with a library call later. Otherwise this depends on
51 /// TargetLoweringInfo availability of the corresponding function.
52 const bool UseMemIntrinsicLibFunc;
53
PreISelIntrinsicLowering__anonad5192420111::PreISelIntrinsicLowering54 explicit PreISelIntrinsicLowering(
55 const TargetMachine &TM_,
56 function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
57 bool UseMemIntrinsicLibFunc_ = true)
58 : TM(TM_), LookupTTI(LookupTTI_),
59 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
60
61 static bool shouldExpandMemIntrinsicWithSize(Value *Size,
62 const TargetTransformInfo &TTI);
63 bool expandMemIntrinsicUses(Function &F) const;
64 bool lowerIntrinsics(Module &M) const;
65 };
66
67 } // namespace
68
lowerLoadRelative(Function & F)69 static bool lowerLoadRelative(Function &F) {
70 if (F.use_empty())
71 return false;
72
73 bool Changed = false;
74 Type *Int32Ty = Type::getInt32Ty(F.getContext());
75
76 for (Use &U : llvm::make_early_inc_range(F.uses())) {
77 auto CI = dyn_cast<CallInst>(U.getUser());
78 if (!CI || CI->getCalledOperand() != &F)
79 continue;
80
81 IRBuilder<> B(CI);
82 Value *OffsetPtr =
83 B.CreatePtrAdd(CI->getArgOperand(0), CI->getArgOperand(1));
84 Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtr, Align(4));
85
86 Value *ResultPtr = B.CreatePtrAdd(CI->getArgOperand(0), OffsetI32);
87
88 CI->replaceAllUsesWith(ResultPtr);
89 CI->eraseFromParent();
90 Changed = true;
91 }
92
93 return Changed;
94 }
95
96 // ObjCARC has knowledge about whether an obj-c runtime function needs to be
97 // always tail-called or never tail-called.
getOverridingTailCallKind(const Function & F)98 static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
99 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(&F);
100 if (objcarc::IsAlwaysTail(Kind))
101 return CallInst::TCK_Tail;
102 else if (objcarc::IsNeverTail(Kind))
103 return CallInst::TCK_NoTail;
104 return CallInst::TCK_None;
105 }
106
lowerObjCCall(Function & F,const char * NewFn,bool setNonLazyBind=false)107 static bool lowerObjCCall(Function &F, const char *NewFn,
108 bool setNonLazyBind = false) {
109 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
110 "Pre-ISel intrinsics do lower into regular function calls");
111 if (F.use_empty())
112 return false;
113
114 // If we haven't already looked up this function, check to see if the
115 // program already contains a function with this name.
116 Module *M = F.getParent();
117 FunctionCallee FCache = M->getOrInsertFunction(NewFn, F.getFunctionType());
118
119 if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) {
120 Fn->setLinkage(F.getLinkage());
121 if (setNonLazyBind && !Fn->isWeakForLinker()) {
122 // If we have Native ARC, set nonlazybind attribute for these APIs for
123 // performance.
124 Fn->addFnAttr(Attribute::NonLazyBind);
125 }
126 }
127
128 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
129
130 for (Use &U : llvm::make_early_inc_range(F.uses())) {
131 auto *CB = cast<CallBase>(U.getUser());
132
133 if (CB->getCalledFunction() != &F) {
134 objcarc::ARCInstKind Kind = objcarc::getAttachedARCFunctionKind(CB);
135 (void)Kind;
136 assert((Kind == objcarc::ARCInstKind::RetainRV ||
137 Kind == objcarc::ARCInstKind::UnsafeClaimRV) &&
138 "use expected to be the argument of operand bundle "
139 "\"clang.arc.attachedcall\"");
140 U.set(FCache.getCallee());
141 continue;
142 }
143
144 auto *CI = cast<CallInst>(CB);
145 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
146
147 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
148 SmallVector<Value *, 8> Args(CI->args());
149 SmallVector<llvm::OperandBundleDef, 1> BundleList;
150 CI->getOperandBundlesAsDefs(BundleList);
151 CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList);
152 NewCI->setName(CI->getName());
153
154 // Try to set the most appropriate TailCallKind based on both the current
155 // attributes and the ones that we could get from ObjCARC's special
156 // knowledge of the runtime functions.
157 //
158 // std::max respects both requirements of notail and tail here:
159 // * notail on either the call or from ObjCARC becomes notail
160 // * tail on either side is stronger than none, but not notail
161 CallInst::TailCallKind TCK = CI->getTailCallKind();
162 NewCI->setTailCallKind(std::max(TCK, OverridingTCK));
163
164 // Transfer the 'returned' attribute from the intrinsic to the call site.
165 // By applying this only to intrinsic call sites, we avoid applying it to
166 // non-ARC explicit calls to things like objc_retain which have not been
167 // auto-upgraded to use the intrinsics.
168 unsigned Index;
169 if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) &&
170 Index)
171 NewCI->addParamAttr(Index - AttributeList::FirstArgIndex,
172 Attribute::Returned);
173
174 if (!CI->use_empty())
175 CI->replaceAllUsesWith(NewCI);
176 CI->eraseFromParent();
177 }
178
179 return true;
180 }
181
182 // TODO: Should refine based on estimated number of accesses (e.g. does it
183 // require splitting based on alignment)
shouldExpandMemIntrinsicWithSize(Value * Size,const TargetTransformInfo & TTI)184 bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
185 Value *Size, const TargetTransformInfo &TTI) {
186 ConstantInt *CI = dyn_cast<ConstantInt>(Size);
187 if (!CI)
188 return true;
189 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
190 ? MemIntrinsicExpandSizeThresholdOpt
191 : TTI.getMaxMemIntrinsicInlineSizeThreshold();
192 uint64_t SizeVal = CI->getZExtValue();
193
194 // Treat a threshold of 0 as a special case to force expansion of all
195 // intrinsics, including size 0.
196 return SizeVal > Threshold || Threshold == 0;
197 }
198
canEmitLibcall(const TargetMachine & TM,Function * F,RTLIB::Libcall LC)199 static bool canEmitLibcall(const TargetMachine &TM, Function *F,
200 RTLIB::Libcall LC) {
201 // TODO: Should this consider the address space of the memcpy?
202 const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering();
203 return TLI->getLibcallName(LC) != nullptr;
204 }
205
206 // TODO: Handle atomic memcpy and memcpy.inline
207 // TODO: Pass ScalarEvolution
expandMemIntrinsicUses(Function & F) const208 bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
209 Intrinsic::ID ID = F.getIntrinsicID();
210 bool Changed = false;
211
212 for (User *U : llvm::make_early_inc_range(F.users())) {
213 Instruction *Inst = cast<Instruction>(U);
214
215 switch (ID) {
216 case Intrinsic::memcpy: {
217 auto *Memcpy = cast<MemCpyInst>(Inst);
218 Function *ParentFunc = Memcpy->getFunction();
219 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
220 if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {
221 if (UseMemIntrinsicLibFunc &&
222 canEmitLibcall(TM, ParentFunc, RTLIB::MEMCPY))
223 break;
224
225 // TODO: For optsize, emit the loop into a separate function
226 expandMemCpyAsLoop(Memcpy, TTI);
227 Changed = true;
228 Memcpy->eraseFromParent();
229 }
230
231 break;
232 }
233 case Intrinsic::memcpy_inline: {
234 // Only expand llvm.memcpy.inline with non-constant length in this
235 // codepath, leaving the current SelectionDAG expansion for constant
236 // length memcpy intrinsics undisturbed.
237 auto *Memcpy = cast<MemCpyInlineInst>(Inst);
238 if (isa<ConstantInt>(Memcpy->getLength()))
239 break;
240
241 Function *ParentFunc = Memcpy->getFunction();
242 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
243 expandMemCpyAsLoop(Memcpy, TTI);
244 Changed = true;
245 Memcpy->eraseFromParent();
246 break;
247 }
248 case Intrinsic::memmove: {
249 auto *Memmove = cast<MemMoveInst>(Inst);
250 Function *ParentFunc = Memmove->getFunction();
251 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
252 if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {
253 if (UseMemIntrinsicLibFunc &&
254 canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE))
255 break;
256
257 if (expandMemMoveAsLoop(Memmove, TTI)) {
258 Changed = true;
259 Memmove->eraseFromParent();
260 }
261 }
262
263 break;
264 }
265 case Intrinsic::memset: {
266 auto *Memset = cast<MemSetInst>(Inst);
267 Function *ParentFunc = Memset->getFunction();
268 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
269 if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) {
270 if (UseMemIntrinsicLibFunc &&
271 canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET))
272 break;
273
274 expandMemSetAsLoop(Memset);
275 Changed = true;
276 Memset->eraseFromParent();
277 }
278
279 break;
280 }
281 case Intrinsic::memset_inline: {
282 // Only expand llvm.memset.inline with non-constant length in this
283 // codepath, leaving the current SelectionDAG expansion for constant
284 // length memset intrinsics undisturbed.
285 auto *Memset = cast<MemSetInlineInst>(Inst);
286 if (isa<ConstantInt>(Memset->getLength()))
287 break;
288
289 expandMemSetAsLoop(Memset);
290 Changed = true;
291 Memset->eraseFromParent();
292 break;
293 }
294 default:
295 llvm_unreachable("unhandled intrinsic");
296 }
297 }
298
299 return Changed;
300 }
301
lowerIntrinsics(Module & M) const302 bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
303 bool Changed = false;
304 for (Function &F : M) {
305 switch (F.getIntrinsicID()) {
306 default:
307 break;
308 case Intrinsic::memcpy:
309 case Intrinsic::memcpy_inline:
310 case Intrinsic::memmove:
311 case Intrinsic::memset:
312 case Intrinsic::memset_inline:
313 Changed |= expandMemIntrinsicUses(F);
314 break;
315 case Intrinsic::load_relative:
316 Changed |= lowerLoadRelative(F);
317 break;
318 case Intrinsic::objc_autorelease:
319 Changed |= lowerObjCCall(F, "objc_autorelease");
320 break;
321 case Intrinsic::objc_autoreleasePoolPop:
322 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPop");
323 break;
324 case Intrinsic::objc_autoreleasePoolPush:
325 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPush");
326 break;
327 case Intrinsic::objc_autoreleaseReturnValue:
328 Changed |= lowerObjCCall(F, "objc_autoreleaseReturnValue");
329 break;
330 case Intrinsic::objc_copyWeak:
331 Changed |= lowerObjCCall(F, "objc_copyWeak");
332 break;
333 case Intrinsic::objc_destroyWeak:
334 Changed |= lowerObjCCall(F, "objc_destroyWeak");
335 break;
336 case Intrinsic::objc_initWeak:
337 Changed |= lowerObjCCall(F, "objc_initWeak");
338 break;
339 case Intrinsic::objc_loadWeak:
340 Changed |= lowerObjCCall(F, "objc_loadWeak");
341 break;
342 case Intrinsic::objc_loadWeakRetained:
343 Changed |= lowerObjCCall(F, "objc_loadWeakRetained");
344 break;
345 case Intrinsic::objc_moveWeak:
346 Changed |= lowerObjCCall(F, "objc_moveWeak");
347 break;
348 case Intrinsic::objc_release:
349 Changed |= lowerObjCCall(F, "objc_release", true);
350 break;
351 case Intrinsic::objc_retain:
352 Changed |= lowerObjCCall(F, "objc_retain", true);
353 break;
354 case Intrinsic::objc_retainAutorelease:
355 Changed |= lowerObjCCall(F, "objc_retainAutorelease");
356 break;
357 case Intrinsic::objc_retainAutoreleaseReturnValue:
358 Changed |= lowerObjCCall(F, "objc_retainAutoreleaseReturnValue");
359 break;
360 case Intrinsic::objc_retainAutoreleasedReturnValue:
361 Changed |= lowerObjCCall(F, "objc_retainAutoreleasedReturnValue");
362 break;
363 case Intrinsic::objc_retainBlock:
364 Changed |= lowerObjCCall(F, "objc_retainBlock");
365 break;
366 case Intrinsic::objc_storeStrong:
367 Changed |= lowerObjCCall(F, "objc_storeStrong");
368 break;
369 case Intrinsic::objc_storeWeak:
370 Changed |= lowerObjCCall(F, "objc_storeWeak");
371 break;
372 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
373 Changed |= lowerObjCCall(F, "objc_unsafeClaimAutoreleasedReturnValue");
374 break;
375 case Intrinsic::objc_retainedObject:
376 Changed |= lowerObjCCall(F, "objc_retainedObject");
377 break;
378 case Intrinsic::objc_unretainedObject:
379 Changed |= lowerObjCCall(F, "objc_unretainedObject");
380 break;
381 case Intrinsic::objc_unretainedPointer:
382 Changed |= lowerObjCCall(F, "objc_unretainedPointer");
383 break;
384 case Intrinsic::objc_retain_autorelease:
385 Changed |= lowerObjCCall(F, "objc_retain_autorelease");
386 break;
387 case Intrinsic::objc_sync_enter:
388 Changed |= lowerObjCCall(F, "objc_sync_enter");
389 break;
390 case Intrinsic::objc_sync_exit:
391 Changed |= lowerObjCCall(F, "objc_sync_exit");
392 break;
393 }
394 }
395 return Changed;
396 }
397
398 namespace {
399
400 class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
401 public:
402 static char ID;
403
PreISelIntrinsicLoweringLegacyPass()404 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
405
getAnalysisUsage(AnalysisUsage & AU) const406 void getAnalysisUsage(AnalysisUsage &AU) const override {
407 AU.addRequired<TargetTransformInfoWrapperPass>();
408 AU.addRequired<TargetPassConfig>();
409 }
410
runOnModule(Module & M)411 bool runOnModule(Module &M) override {
412 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
413 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
414 };
415
416 const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
417 PreISelIntrinsicLowering Lowering(TM, LookupTTI);
418 return Lowering.lowerIntrinsics(M);
419 }
420 };
421
422 } // end anonymous namespace
423
424 char PreISelIntrinsicLoweringLegacyPass::ID;
425
426 INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
427 "pre-isel-intrinsic-lowering",
428 "Pre-ISel Intrinsic Lowering", false, false)
INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)429 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
430 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
431 INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
432 "pre-isel-intrinsic-lowering",
433 "Pre-ISel Intrinsic Lowering", false, false)
434
435 ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
436 return new PreISelIntrinsicLoweringLegacyPass();
437 }
438
run(Module & M,ModuleAnalysisManager & AM)439 PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M,
440 ModuleAnalysisManager &AM) {
441 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
442
443 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
444 return FAM.getResult<TargetIRAnalysis>(F);
445 };
446
447 PreISelIntrinsicLowering Lowering(TM, LookupTTI);
448 if (!Lowering.lowerIntrinsics(M))
449 return PreservedAnalyses::all();
450 else
451 return PreservedAnalyses::none();
452 }
453