1 //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10 // llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
15 #include "llvm/Analysis/ObjCARCInstKind.h"
16 #include "llvm/Analysis/ObjCARCUtil.h"
17 #include "llvm/Analysis/TargetLibraryInfo.h"
18 #include "llvm/Analysis/TargetTransformInfo.h"
19 #include "llvm/CodeGen/ExpandVectorPredication.h"
20 #include "llvm/CodeGen/Passes.h"
21 #include "llvm/CodeGen/TargetLowering.h"
22 #include "llvm/CodeGen/TargetPassConfig.h"
23 #include "llvm/IR/Function.h"
24 #include "llvm/IR/IRBuilder.h"
25 #include "llvm/IR/Instructions.h"
26 #include "llvm/IR/IntrinsicInst.h"
27 #include "llvm/IR/Module.h"
28 #include "llvm/IR/RuntimeLibcalls.h"
29 #include "llvm/IR/Type.h"
30 #include "llvm/IR/Use.h"
31 #include "llvm/InitializePasses.h"
32 #include "llvm/Pass.h"
33 #include "llvm/Support/Casting.h"
34 #include "llvm/Target/TargetMachine.h"
35 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
36 #include "llvm/Transforms/Utils/BuildLibCalls.h"
37 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
38 #include "llvm/Transforms/Utils/LowerVectorIntrinsics.h"
39
40 using namespace llvm;
41
42 /// Threshold to leave statically sized memory intrinsic calls. Calls of known
43 /// size larger than this will be expanded by the pass. Calls of unknown or
44 /// lower size will be left for expansion in codegen.
45 static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
46 "mem-intrinsic-expand-size",
47 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1),
48 cl::Hidden);
49
50 namespace {
51
52 struct PreISelIntrinsicLowering {
53 const TargetMachine *TM;
54 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
55 const function_ref<TargetLibraryInfo &(Function &)> LookupTLI;
56
57 /// If this is true, assume it's preferably to leave memory intrinsic calls
58 /// for replacement with a library call later. Otherwise this depends on
59 /// TargetLoweringInfo availability of the corresponding function.
60 const bool UseMemIntrinsicLibFunc;
61
PreISelIntrinsicLowering__anonad5192420111::PreISelIntrinsicLowering62 explicit PreISelIntrinsicLowering(
63 const TargetMachine *TM_,
64 function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
65 function_ref<TargetLibraryInfo &(Function &)> LookupTLI_,
66 bool UseMemIntrinsicLibFunc_ = true)
67 : TM(TM_), LookupTTI(LookupTTI_), LookupTLI(LookupTLI_),
68 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
69
70 static bool shouldExpandMemIntrinsicWithSize(Value *Size,
71 const TargetTransformInfo &TTI);
72 bool
73 expandMemIntrinsicUses(Function &F,
74 DenseMap<Constant *, GlobalVariable *> &CMap) const;
75 bool lowerIntrinsics(Module &M) const;
76 };
77
78 } // namespace
79
forEachCall(Function & Intrin,T Callback)80 template <class T> static bool forEachCall(Function &Intrin, T Callback) {
81 // Lowering all intrinsics in a function will delete multiple uses, so we
82 // can't use an early-inc-range. In case some remain, we don't want to look
83 // at them again. Unfortunately, Value::UseList is private, so we can't use a
84 // simple Use**. If LastUse is null, the next use to consider is
85 // Intrin.use_begin(), otherwise it's LastUse->getNext().
86 Use *LastUse = nullptr;
87 bool Changed = false;
88 while (!Intrin.use_empty() && (!LastUse || LastUse->getNext())) {
89 Use *U = LastUse ? LastUse->getNext() : &*Intrin.use_begin();
90 bool Removed = false;
91 // An intrinsic cannot have its address taken, so it cannot be an argument
92 // operand. It might be used as operand in debug metadata, though.
93 if (auto CI = dyn_cast<CallInst>(U->getUser()))
94 Changed |= Removed = Callback(CI);
95 if (!Removed)
96 LastUse = U;
97 }
98 return Changed;
99 }
100
lowerLoadRelative(Function & F)101 static bool lowerLoadRelative(Function &F) {
102 if (F.use_empty())
103 return false;
104
105 bool Changed = false;
106 Type *Int32Ty = Type::getInt32Ty(F.getContext());
107
108 for (Use &U : llvm::make_early_inc_range(F.uses())) {
109 auto CI = dyn_cast<CallInst>(U.getUser());
110 if (!CI || CI->getCalledOperand() != &F)
111 continue;
112
113 IRBuilder<> B(CI);
114 Value *OffsetPtr =
115 B.CreatePtrAdd(CI->getArgOperand(0), CI->getArgOperand(1));
116 Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtr, Align(4));
117
118 Value *ResultPtr = B.CreatePtrAdd(CI->getArgOperand(0), OffsetI32);
119
120 CI->replaceAllUsesWith(ResultPtr);
121 CI->eraseFromParent();
122 Changed = true;
123 }
124
125 return Changed;
126 }
127
128 // ObjCARC has knowledge about whether an obj-c runtime function needs to be
129 // always tail-called or never tail-called.
getOverridingTailCallKind(const Function & F)130 static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
131 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(&F);
132 if (objcarc::IsAlwaysTail(Kind))
133 return CallInst::TCK_Tail;
134 else if (objcarc::IsNeverTail(Kind))
135 return CallInst::TCK_NoTail;
136 return CallInst::TCK_None;
137 }
138
lowerObjCCall(Function & F,RTLIB::LibcallImpl NewFn,bool setNonLazyBind=false)139 static bool lowerObjCCall(Function &F, RTLIB::LibcallImpl NewFn,
140 bool setNonLazyBind = false) {
141 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
142 "Pre-ISel intrinsics do lower into regular function calls");
143 if (F.use_empty())
144 return false;
145
146 // FIXME: When RuntimeLibcalls is an analysis, check if the function is really
147 // supported, and go through RTLIB::Libcall.
148 const char *NewFnName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(NewFn);
149
150 // If we haven't already looked up this function, check to see if the
151 // program already contains a function with this name.
152 Module *M = F.getParent();
153 FunctionCallee FCache =
154 M->getOrInsertFunction(NewFnName, F.getFunctionType());
155
156 if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) {
157 Fn->setLinkage(F.getLinkage());
158 if (setNonLazyBind && !Fn->isWeakForLinker()) {
159 // If we have Native ARC, set nonlazybind attribute for these APIs for
160 // performance.
161 Fn->addFnAttr(Attribute::NonLazyBind);
162 }
163 }
164
165 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
166
167 for (Use &U : llvm::make_early_inc_range(F.uses())) {
168 auto *CB = cast<CallBase>(U.getUser());
169
170 if (CB->getCalledFunction() != &F) {
171 assert(objcarc::getAttachedARCFunction(CB) == &F &&
172 "use expected to be the argument of operand bundle "
173 "\"clang.arc.attachedcall\"");
174 U.set(FCache.getCallee());
175 continue;
176 }
177
178 auto *CI = cast<CallInst>(CB);
179 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
180
181 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
182 SmallVector<Value *, 8> Args(CI->args());
183 SmallVector<llvm::OperandBundleDef, 1> BundleList;
184 CI->getOperandBundlesAsDefs(BundleList);
185 CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList);
186 NewCI->setName(CI->getName());
187
188 // Try to set the most appropriate TailCallKind based on both the current
189 // attributes and the ones that we could get from ObjCARC's special
190 // knowledge of the runtime functions.
191 //
192 // std::max respects both requirements of notail and tail here:
193 // * notail on either the call or from ObjCARC becomes notail
194 // * tail on either side is stronger than none, but not notail
195 CallInst::TailCallKind TCK = CI->getTailCallKind();
196 NewCI->setTailCallKind(std::max(TCK, OverridingTCK));
197
198 // Transfer the 'returned' attribute from the intrinsic to the call site.
199 // By applying this only to intrinsic call sites, we avoid applying it to
200 // non-ARC explicit calls to things like objc_retain which have not been
201 // auto-upgraded to use the intrinsics.
202 unsigned Index;
203 if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) &&
204 Index)
205 NewCI->addParamAttr(Index - AttributeList::FirstArgIndex,
206 Attribute::Returned);
207
208 if (!CI->use_empty())
209 CI->replaceAllUsesWith(NewCI);
210 CI->eraseFromParent();
211 }
212
213 return true;
214 }
215
216 // TODO: Should refine based on estimated number of accesses (e.g. does it
217 // require splitting based on alignment)
shouldExpandMemIntrinsicWithSize(Value * Size,const TargetTransformInfo & TTI)218 bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
219 Value *Size, const TargetTransformInfo &TTI) {
220 ConstantInt *CI = dyn_cast<ConstantInt>(Size);
221 if (!CI)
222 return true;
223 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
224 ? MemIntrinsicExpandSizeThresholdOpt
225 : TTI.getMaxMemIntrinsicInlineSizeThreshold();
226 uint64_t SizeVal = CI->getZExtValue();
227
228 // Treat a threshold of 0 as a special case to force expansion of all
229 // intrinsics, including size 0.
230 return SizeVal > Threshold || Threshold == 0;
231 }
232
canEmitLibcall(const TargetMachine * TM,Function * F,RTLIB::Libcall LC)233 static bool canEmitLibcall(const TargetMachine *TM, Function *F,
234 RTLIB::Libcall LC) {
235 // TODO: Should this consider the address space of the memcpy?
236 if (!TM)
237 return true;
238 const TargetLowering *TLI = TM->getSubtargetImpl(*F)->getTargetLowering();
239 return TLI->getLibcallName(LC) != nullptr;
240 }
241
canEmitMemcpy(const TargetMachine * TM,Function * F)242 static bool canEmitMemcpy(const TargetMachine *TM, Function *F) {
243 // TODO: Should this consider the address space of the memcpy?
244 if (!TM)
245 return true;
246 const TargetLowering *TLI = TM->getSubtargetImpl(*F)->getTargetLowering();
247 return TLI->getMemcpyName() != nullptr;
248 }
249
250 // Return a value appropriate for use with the memset_pattern16 libcall, if
251 // possible and if we know how. (Adapted from equivalent helper in
252 // LoopIdiomRecognize).
getMemSetPattern16Value(MemSetPatternInst * Inst,const TargetLibraryInfo & TLI)253 static Constant *getMemSetPattern16Value(MemSetPatternInst *Inst,
254 const TargetLibraryInfo &TLI) {
255 // TODO: This could check for UndefValue because it can be merged into any
256 // other valid pattern.
257
258 // Don't emit libcalls if a non-default address space is being used.
259 if (Inst->getRawDest()->getType()->getPointerAddressSpace() != 0)
260 return nullptr;
261
262 Value *V = Inst->getValue();
263 Type *VTy = V->getType();
264 const DataLayout &DL = Inst->getDataLayout();
265 Module *M = Inst->getModule();
266
267 if (!isLibFuncEmittable(M, &TLI, LibFunc_memset_pattern16))
268 return nullptr;
269
270 // If the value isn't a constant, we can't promote it to being in a constant
271 // array. We could theoretically do a store to an alloca or something, but
272 // that doesn't seem worthwhile.
273 Constant *C = dyn_cast<Constant>(V);
274 if (!C || isa<ConstantExpr>(C))
275 return nullptr;
276
277 // Only handle simple values that are a power of two bytes in size.
278 uint64_t Size = DL.getTypeSizeInBits(VTy);
279 if (!DL.typeSizeEqualsStoreSize(VTy) || !isPowerOf2_64(Size))
280 return nullptr;
281
282 // Don't care enough about darwin/ppc to implement this.
283 if (DL.isBigEndian())
284 return nullptr;
285
286 // Convert to size in bytes.
287 Size /= 8;
288
289 // TODO: If CI is larger than 16-bytes, we can try slicing it in half to see
290 // if the top and bottom are the same (e.g. for vectors and large integers).
291 if (Size > 16)
292 return nullptr;
293
294 // If the constant is exactly 16 bytes, just use it.
295 if (Size == 16)
296 return C;
297
298 // Otherwise, we'll use an array of the constants.
299 uint64_t ArraySize = 16 / Size;
300 ArrayType *AT = ArrayType::get(V->getType(), ArraySize);
301 return ConstantArray::get(AT, std::vector<Constant *>(ArraySize, C));
302 }
303
304 // TODO: Handle atomic memcpy and memcpy.inline
305 // TODO: Pass ScalarEvolution
expandMemIntrinsicUses(Function & F,DenseMap<Constant *,GlobalVariable * > & CMap) const306 bool PreISelIntrinsicLowering::expandMemIntrinsicUses(
307 Function &F, DenseMap<Constant *, GlobalVariable *> &CMap) const {
308 Intrinsic::ID ID = F.getIntrinsicID();
309 bool Changed = false;
310
311 for (User *U : llvm::make_early_inc_range(F.users())) {
312 Instruction *Inst = cast<Instruction>(U);
313
314 switch (ID) {
315 case Intrinsic::memcpy: {
316 auto *Memcpy = cast<MemCpyInst>(Inst);
317 Function *ParentFunc = Memcpy->getFunction();
318 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
319 if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {
320 if (UseMemIntrinsicLibFunc && canEmitMemcpy(TM, ParentFunc))
321 break;
322
323 // TODO: For optsize, emit the loop into a separate function
324 expandMemCpyAsLoop(Memcpy, TTI);
325 Changed = true;
326 Memcpy->eraseFromParent();
327 }
328
329 break;
330 }
331 case Intrinsic::memcpy_inline: {
332 // Only expand llvm.memcpy.inline with non-constant length in this
333 // codepath, leaving the current SelectionDAG expansion for constant
334 // length memcpy intrinsics undisturbed.
335 auto *Memcpy = cast<MemCpyInst>(Inst);
336 if (isa<ConstantInt>(Memcpy->getLength()))
337 break;
338
339 Function *ParentFunc = Memcpy->getFunction();
340 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
341 expandMemCpyAsLoop(Memcpy, TTI);
342 Changed = true;
343 Memcpy->eraseFromParent();
344 break;
345 }
346 case Intrinsic::memmove: {
347 auto *Memmove = cast<MemMoveInst>(Inst);
348 Function *ParentFunc = Memmove->getFunction();
349 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
350 if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {
351 if (UseMemIntrinsicLibFunc &&
352 canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE))
353 break;
354
355 if (expandMemMoveAsLoop(Memmove, TTI)) {
356 Changed = true;
357 Memmove->eraseFromParent();
358 }
359 }
360
361 break;
362 }
363 case Intrinsic::memset: {
364 auto *Memset = cast<MemSetInst>(Inst);
365 Function *ParentFunc = Memset->getFunction();
366 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
367 if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) {
368 if (UseMemIntrinsicLibFunc &&
369 canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET))
370 break;
371
372 expandMemSetAsLoop(Memset);
373 Changed = true;
374 Memset->eraseFromParent();
375 }
376
377 break;
378 }
379 case Intrinsic::memset_inline: {
380 // Only expand llvm.memset.inline with non-constant length in this
381 // codepath, leaving the current SelectionDAG expansion for constant
382 // length memset intrinsics undisturbed.
383 auto *Memset = cast<MemSetInst>(Inst);
384 if (isa<ConstantInt>(Memset->getLength()))
385 break;
386
387 expandMemSetAsLoop(Memset);
388 Changed = true;
389 Memset->eraseFromParent();
390 break;
391 }
392 case Intrinsic::experimental_memset_pattern: {
393 auto *Memset = cast<MemSetPatternInst>(Inst);
394 const TargetLibraryInfo &TLI = LookupTLI(*Memset->getFunction());
395 Constant *PatternValue = getMemSetPattern16Value(Memset, TLI);
396 if (!PatternValue) {
397 // If it isn't possible to emit a memset_pattern16 libcall, expand to
398 // a loop instead.
399 expandMemSetPatternAsLoop(Memset);
400 Changed = true;
401 Memset->eraseFromParent();
402 break;
403 }
404 // FIXME: There is currently no profitability calculation for emitting
405 // the libcall vs expanding the memset.pattern directly.
406 IRBuilder<> Builder(Inst);
407 Module *M = Memset->getModule();
408 const DataLayout &DL = Memset->getDataLayout();
409
410 Type *DestPtrTy = Memset->getRawDest()->getType();
411 Type *SizeTTy = TLI.getSizeTType(*M);
412 StringRef FuncName = "memset_pattern16";
413 FunctionCallee MSP = getOrInsertLibFunc(M, TLI, LibFunc_memset_pattern16,
414 Builder.getVoidTy(), DestPtrTy,
415 Builder.getPtrTy(), SizeTTy);
416 inferNonMandatoryLibFuncAttrs(M, FuncName, TLI);
417
418 // Otherwise we should form a memset_pattern16. PatternValue is known
419 // to be an constant array of 16-bytes. Put the value into a mergable
420 // global.
421 assert(Memset->getRawDest()->getType()->getPointerAddressSpace() == 0 &&
422 "Should have skipped if non-zero AS");
423 GlobalVariable *GV;
424 auto It = CMap.find(PatternValue);
425 if (It != CMap.end()) {
426 GV = It->second;
427 } else {
428 GV = new GlobalVariable(
429 *M, PatternValue->getType(), /*isConstant=*/true,
430 GlobalValue::PrivateLinkage, PatternValue, ".memset_pattern");
431 GV->setUnnamedAddr(
432 GlobalValue::UnnamedAddr::Global); // Ok to merge these.
433 // TODO: Consider relaxing alignment requirement.
434 GV->setAlignment(Align(16));
435 CMap[PatternValue] = GV;
436 }
437 Value *PatternPtr = GV;
438 Value *NumBytes = Builder.CreateMul(
439 TLI.getAsSizeT(DL.getTypeAllocSize(Memset->getValue()->getType()),
440 *M),
441 Builder.CreateZExtOrTrunc(Memset->getLength(), SizeTTy));
442 CallInst *MemsetPattern16Call =
443 Builder.CreateCall(MSP, {Memset->getRawDest(), PatternPtr, NumBytes});
444 MemsetPattern16Call->setAAMetadata(Memset->getAAMetadata());
445 // Preserve any call site attributes on the destination pointer
446 // argument (e.g. alignment).
447 AttrBuilder ArgAttrs(Memset->getContext(),
448 Memset->getAttributes().getParamAttrs(0));
449 MemsetPattern16Call->setAttributes(
450 MemsetPattern16Call->getAttributes().addParamAttributes(
451 Memset->getContext(), 0, ArgAttrs));
452 Changed = true;
453 Memset->eraseFromParent();
454 break;
455 }
456 default:
457 llvm_unreachable("unhandled intrinsic");
458 }
459 }
460
461 return Changed;
462 }
463
lowerIntrinsics(Module & M) const464 bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
465 // Map unique constants to globals.
466 DenseMap<Constant *, GlobalVariable *> CMap;
467 bool Changed = false;
468 for (Function &F : M) {
469 switch (F.getIntrinsicID()) {
470 default:
471 break;
472 case Intrinsic::memcpy:
473 case Intrinsic::memcpy_inline:
474 case Intrinsic::memmove:
475 case Intrinsic::memset:
476 case Intrinsic::memset_inline:
477 case Intrinsic::experimental_memset_pattern:
478 Changed |= expandMemIntrinsicUses(F, CMap);
479 break;
480 case Intrinsic::load_relative:
481 Changed |= lowerLoadRelative(F);
482 break;
483 case Intrinsic::is_constant:
484 case Intrinsic::objectsize:
485 Changed |= forEachCall(F, [&](CallInst *CI) {
486 Function *Parent = CI->getParent()->getParent();
487 TargetLibraryInfo &TLI = LookupTLI(*Parent);
488 // Intrinsics in unreachable code are not lowered.
489 bool Changed = lowerConstantIntrinsics(*Parent, TLI, /*DT=*/nullptr);
490 return Changed;
491 });
492 break;
493 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
494 case Intrinsic::VPID:
495 #include "llvm/IR/VPIntrinsics.def"
496 forEachCall(F, [&](CallInst *CI) {
497 Function *Parent = CI->getParent()->getParent();
498 const TargetTransformInfo &TTI = LookupTTI(*Parent);
499 auto *VPI = cast<VPIntrinsic>(CI);
500 VPExpansionDetails ED = expandVectorPredicationIntrinsic(*VPI, TTI);
501 // Expansion of VP intrinsics may change the IR but not actually
502 // replace the intrinsic, so update Changed for the pass
503 // and compute Removed for forEachCall.
504 Changed |= ED != VPExpansionDetails::IntrinsicUnchanged;
505 bool Removed = ED == VPExpansionDetails::IntrinsicReplaced;
506 return Removed;
507 });
508 break;
509 case Intrinsic::objc_autorelease:
510 Changed |= lowerObjCCall(F, RTLIB::objc_autorelease);
511 break;
512 case Intrinsic::objc_autoreleasePoolPop:
513 Changed |= lowerObjCCall(F, RTLIB::objc_autoreleasePoolPop);
514 break;
515 case Intrinsic::objc_autoreleasePoolPush:
516 Changed |= lowerObjCCall(F, RTLIB::objc_autoreleasePoolPush);
517 break;
518 case Intrinsic::objc_autoreleaseReturnValue:
519 Changed |= lowerObjCCall(F, RTLIB::objc_autoreleaseReturnValue);
520 break;
521 case Intrinsic::objc_copyWeak:
522 Changed |= lowerObjCCall(F, RTLIB::objc_copyWeak);
523 break;
524 case Intrinsic::objc_destroyWeak:
525 Changed |= lowerObjCCall(F, RTLIB::objc_destroyWeak);
526 break;
527 case Intrinsic::objc_initWeak:
528 Changed |= lowerObjCCall(F, RTLIB::objc_initWeak);
529 break;
530 case Intrinsic::objc_loadWeak:
531 Changed |= lowerObjCCall(F, RTLIB::objc_loadWeak);
532 break;
533 case Intrinsic::objc_loadWeakRetained:
534 Changed |= lowerObjCCall(F, RTLIB::objc_loadWeakRetained);
535 break;
536 case Intrinsic::objc_moveWeak:
537 Changed |= lowerObjCCall(F, RTLIB::objc_moveWeak);
538 break;
539 case Intrinsic::objc_release:
540 Changed |= lowerObjCCall(F, RTLIB::objc_release, true);
541 break;
542 case Intrinsic::objc_retain:
543 Changed |= lowerObjCCall(F, RTLIB::objc_retain, true);
544 break;
545 case Intrinsic::objc_retainAutorelease:
546 Changed |= lowerObjCCall(F, RTLIB::objc_retainAutorelease);
547 break;
548 case Intrinsic::objc_retainAutoreleaseReturnValue:
549 Changed |= lowerObjCCall(F, RTLIB::objc_retainAutoreleaseReturnValue);
550 break;
551 case Intrinsic::objc_retainAutoreleasedReturnValue:
552 Changed |= lowerObjCCall(F, RTLIB::objc_retainAutoreleasedReturnValue);
553 break;
554 case Intrinsic::objc_claimAutoreleasedReturnValue:
555 Changed |= lowerObjCCall(F, RTLIB::objc_claimAutoreleasedReturnValue);
556 break;
557 case Intrinsic::objc_retainBlock:
558 Changed |= lowerObjCCall(F, RTLIB::objc_retainBlock);
559 break;
560 case Intrinsic::objc_storeStrong:
561 Changed |= lowerObjCCall(F, RTLIB::objc_storeStrong);
562 break;
563 case Intrinsic::objc_storeWeak:
564 Changed |= lowerObjCCall(F, RTLIB::objc_storeWeak);
565 break;
566 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
567 Changed |=
568 lowerObjCCall(F, RTLIB::objc_unsafeClaimAutoreleasedReturnValue);
569 break;
570 case Intrinsic::objc_retainedObject:
571 Changed |= lowerObjCCall(F, RTLIB::objc_retainedObject);
572 break;
573 case Intrinsic::objc_unretainedObject:
574 Changed |= lowerObjCCall(F, RTLIB::objc_unretainedObject);
575 break;
576 case Intrinsic::objc_unretainedPointer:
577 Changed |= lowerObjCCall(F, RTLIB::objc_unretainedPointer);
578 break;
579 case Intrinsic::objc_retain_autorelease:
580 Changed |= lowerObjCCall(F, RTLIB::objc_retain_autorelease);
581 break;
582 case Intrinsic::objc_sync_enter:
583 Changed |= lowerObjCCall(F, RTLIB::objc_sync_enter);
584 break;
585 case Intrinsic::objc_sync_exit:
586 Changed |= lowerObjCCall(F, RTLIB::objc_sync_exit);
587 break;
588 case Intrinsic::exp:
589 case Intrinsic::exp2:
590 case Intrinsic::log:
591 Changed |= forEachCall(F, [&](CallInst *CI) {
592 Type *Ty = CI->getArgOperand(0)->getType();
593 if (!isa<ScalableVectorType>(Ty))
594 return false;
595 const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering();
596 unsigned Op = TL->IntrinsicIDToISD(F.getIntrinsicID());
597 assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");
598 if (!TL->isOperationExpand(Op, EVT::getEVT(Ty)))
599 return false;
600 return lowerUnaryVectorIntrinsicAsLoop(M, CI);
601 });
602 break;
603 }
604 }
605 return Changed;
606 }
607
608 namespace {
609
610 class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
611 public:
612 static char ID;
613
PreISelIntrinsicLoweringLegacyPass()614 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
615
getAnalysisUsage(AnalysisUsage & AU) const616 void getAnalysisUsage(AnalysisUsage &AU) const override {
617 AU.addRequired<TargetTransformInfoWrapperPass>();
618 AU.addRequired<TargetLibraryInfoWrapperPass>();
619 AU.addRequired<TargetPassConfig>();
620 }
621
runOnModule(Module & M)622 bool runOnModule(Module &M) override {
623 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
624 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
625 };
626 auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & {
627 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
628 };
629
630 const auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
631 PreISelIntrinsicLowering Lowering(TM, LookupTTI, LookupTLI);
632 return Lowering.lowerIntrinsics(M);
633 }
634 };
635
636 } // end anonymous namespace
637
638 char PreISelIntrinsicLoweringLegacyPass::ID;
639
640 INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
641 "pre-isel-intrinsic-lowering",
642 "Pre-ISel Intrinsic Lowering", false, false)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)643 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
644 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
645 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
646 INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
647 "pre-isel-intrinsic-lowering",
648 "Pre-ISel Intrinsic Lowering", false, false)
649
650 ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
651 return new PreISelIntrinsicLoweringLegacyPass();
652 }
653
run(Module & M,ModuleAnalysisManager & AM)654 PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M,
655 ModuleAnalysisManager &AM) {
656 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
657
658 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
659 return FAM.getResult<TargetIRAnalysis>(F);
660 };
661 auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
662 return FAM.getResult<TargetLibraryAnalysis>(F);
663 };
664
665 PreISelIntrinsicLowering Lowering(TM, LookupTTI, LookupTLI);
666 if (!Lowering.lowerIntrinsics(M))
667 return PreservedAnalyses::all();
668 else
669 return PreservedAnalyses::none();
670 }
671