1 //===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IntrinsicLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/CodeGen/IntrinsicLowering.h"
14 #include "llvm/ADT/SmallVector.h"
15 #include "llvm/IR/Constants.h"
16 #include "llvm/IR/DataLayout.h"
17 #include "llvm/IR/DerivedTypes.h"
18 #include "llvm/IR/IRBuilder.h"
19 #include "llvm/IR/Module.h"
20 #include "llvm/IR/Type.h"
21 #include "llvm/Support/ErrorHandling.h"
22 #include "llvm/Support/raw_ostream.h"
23 using namespace llvm;
24
25 /// This function is used when we want to lower an intrinsic call to a call of
26 /// an external function. This handles hard cases such as when there was already
27 /// a prototype for the external function, but that prototype doesn't match the
28 /// arguments we expect to pass in.
29 template <class ArgIt>
ReplaceCallWith(const char * NewFn,CallInst * CI,ArgIt ArgBegin,ArgIt ArgEnd,Type * RetTy)30 static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI,
31 ArgIt ArgBegin, ArgIt ArgEnd,
32 Type *RetTy) {
33 // If we haven't already looked up this function, check to see if the
34 // program already contains a function with this name.
35 Module *M = CI->getModule();
36 // Get or insert the definition now.
37 std::vector<Type *> ParamTys;
38 for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
39 ParamTys.push_back((*I)->getType());
40 FunctionCallee FCache =
41 M->getOrInsertFunction(NewFn, FunctionType::get(RetTy, ParamTys, false));
42
43 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
44 SmallVector<Value *, 8> Args(ArgBegin, ArgEnd);
45 CallInst *NewCI = Builder.CreateCall(FCache, Args);
46 NewCI->setName(CI->getName());
47 if (!CI->use_empty())
48 CI->replaceAllUsesWith(NewCI);
49 return NewCI;
50 }
51
52 /// Emit the code to lower bswap of V before the specified instruction IP.
LowerBSWAP(LLVMContext & Context,Value * V,Instruction * IP)53 static Value *LowerBSWAP(LLVMContext &Context, Value *V, Instruction *IP) {
54 assert(V->getType()->isIntOrIntVectorTy() && "Can't bswap a non-integer type!");
55
56 unsigned BitSize = V->getType()->getScalarSizeInBits();
57
58 IRBuilder<> Builder(IP);
59
60 switch(BitSize) {
61 default: llvm_unreachable("Unhandled type size of value to byteswap!");
62 case 16: {
63 Value *Tmp1 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8),
64 "bswap.2");
65 Value *Tmp2 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8),
66 "bswap.1");
67 V = Builder.CreateOr(Tmp1, Tmp2, "bswap.i16");
68 break;
69 }
70 case 32: {
71 Value *Tmp4 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 24),
72 "bswap.4");
73 Value *Tmp3 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8),
74 "bswap.3");
75 Value *Tmp2 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8),
76 "bswap.2");
77 Value *Tmp1 = Builder.CreateLShr(V,ConstantInt::get(V->getType(), 24),
78 "bswap.1");
79 Tmp3 = Builder.CreateAnd(Tmp3,
80 ConstantInt::get(V->getType(), 0xFF0000),
81 "bswap.and3");
82 Tmp2 = Builder.CreateAnd(Tmp2,
83 ConstantInt::get(V->getType(), 0xFF00),
84 "bswap.and2");
85 Tmp4 = Builder.CreateOr(Tmp4, Tmp3, "bswap.or1");
86 Tmp2 = Builder.CreateOr(Tmp2, Tmp1, "bswap.or2");
87 V = Builder.CreateOr(Tmp4, Tmp2, "bswap.i32");
88 break;
89 }
90 case 64: {
91 Value *Tmp8 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 56),
92 "bswap.8");
93 Value *Tmp7 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 40),
94 "bswap.7");
95 Value *Tmp6 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 24),
96 "bswap.6");
97 Value *Tmp5 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8),
98 "bswap.5");
99 Value* Tmp4 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8),
100 "bswap.4");
101 Value* Tmp3 = Builder.CreateLShr(V,
102 ConstantInt::get(V->getType(), 24),
103 "bswap.3");
104 Value* Tmp2 = Builder.CreateLShr(V,
105 ConstantInt::get(V->getType(), 40),
106 "bswap.2");
107 Value* Tmp1 = Builder.CreateLShr(V,
108 ConstantInt::get(V->getType(), 56),
109 "bswap.1");
110 Tmp7 = Builder.CreateAnd(Tmp7,
111 ConstantInt::get(V->getType(),
112 0xFF000000000000ULL),
113 "bswap.and7");
114 Tmp6 = Builder.CreateAnd(Tmp6,
115 ConstantInt::get(V->getType(),
116 0xFF0000000000ULL),
117 "bswap.and6");
118 Tmp5 = Builder.CreateAnd(Tmp5,
119 ConstantInt::get(V->getType(),
120 0xFF00000000ULL),
121 "bswap.and5");
122 Tmp4 = Builder.CreateAnd(Tmp4,
123 ConstantInt::get(V->getType(),
124 0xFF000000ULL),
125 "bswap.and4");
126 Tmp3 = Builder.CreateAnd(Tmp3,
127 ConstantInt::get(V->getType(),
128 0xFF0000ULL),
129 "bswap.and3");
130 Tmp2 = Builder.CreateAnd(Tmp2,
131 ConstantInt::get(V->getType(),
132 0xFF00ULL),
133 "bswap.and2");
134 Tmp8 = Builder.CreateOr(Tmp8, Tmp7, "bswap.or1");
135 Tmp6 = Builder.CreateOr(Tmp6, Tmp5, "bswap.or2");
136 Tmp4 = Builder.CreateOr(Tmp4, Tmp3, "bswap.or3");
137 Tmp2 = Builder.CreateOr(Tmp2, Tmp1, "bswap.or4");
138 Tmp8 = Builder.CreateOr(Tmp8, Tmp6, "bswap.or5");
139 Tmp4 = Builder.CreateOr(Tmp4, Tmp2, "bswap.or6");
140 V = Builder.CreateOr(Tmp8, Tmp4, "bswap.i64");
141 break;
142 }
143 }
144 return V;
145 }
146
147 /// Emit the code to lower ctpop of V before the specified instruction IP.
LowerCTPOP(LLVMContext & Context,Value * V,Instruction * IP)148 static Value *LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP) {
149 assert(V->getType()->isIntegerTy() && "Can't ctpop a non-integer type!");
150
151 static const uint64_t MaskValues[6] = {
152 0x5555555555555555ULL, 0x3333333333333333ULL,
153 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
154 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL
155 };
156
157 IRBuilder<> Builder(IP);
158
159 unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
160 unsigned WordSize = (BitSize + 63) / 64;
161 Value *Count = ConstantInt::get(V->getType(), 0);
162
163 for (unsigned n = 0; n < WordSize; ++n) {
164 Value *PartValue = V;
165 for (unsigned i = 1, ct = 0; i < (BitSize>64 ? 64 : BitSize);
166 i <<= 1, ++ct) {
167 Value *MaskCst = ConstantInt::get(V->getType(), MaskValues[ct]);
168 Value *LHS = Builder.CreateAnd(PartValue, MaskCst, "cppop.and1");
169 Value *VShift = Builder.CreateLShr(PartValue,
170 ConstantInt::get(V->getType(), i),
171 "ctpop.sh");
172 Value *RHS = Builder.CreateAnd(VShift, MaskCst, "cppop.and2");
173 PartValue = Builder.CreateAdd(LHS, RHS, "ctpop.step");
174 }
175 Count = Builder.CreateAdd(PartValue, Count, "ctpop.part");
176 if (BitSize > 64) {
177 V = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 64),
178 "ctpop.part.sh");
179 BitSize -= 64;
180 }
181 }
182
183 return Count;
184 }
185
186 /// Emit the code to lower ctlz of V before the specified instruction IP.
LowerCTLZ(LLVMContext & Context,Value * V,Instruction * IP)187 static Value *LowerCTLZ(LLVMContext &Context, Value *V, Instruction *IP) {
188
189 IRBuilder<> Builder(IP);
190
191 unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
192 for (unsigned i = 1; i < BitSize; i <<= 1) {
193 Value *ShVal = ConstantInt::get(V->getType(), i);
194 ShVal = Builder.CreateLShr(V, ShVal, "ctlz.sh");
195 V = Builder.CreateOr(V, ShVal, "ctlz.step");
196 }
197
198 V = Builder.CreateNot(V);
199 return LowerCTPOP(Context, V, IP);
200 }
201
ReplaceFPIntrinsicWithCall(CallInst * CI,const char * Fname,const char * Dname,const char * LDname)202 static void ReplaceFPIntrinsicWithCall(CallInst *CI, const char *Fname,
203 const char *Dname,
204 const char *LDname) {
205 switch (CI->getArgOperand(0)->getType()->getTypeID()) {
206 default: llvm_unreachable("Invalid type in intrinsic");
207 case Type::FloatTyID:
208 ReplaceCallWith(Fname, CI, CI->arg_begin(), CI->arg_end(),
209 Type::getFloatTy(CI->getContext()));
210 break;
211 case Type::DoubleTyID:
212 ReplaceCallWith(Dname, CI, CI->arg_begin(), CI->arg_end(),
213 Type::getDoubleTy(CI->getContext()));
214 break;
215 case Type::X86_FP80TyID:
216 case Type::FP128TyID:
217 case Type::PPC_FP128TyID:
218 ReplaceCallWith(LDname, CI, CI->arg_begin(), CI->arg_end(),
219 CI->getArgOperand(0)->getType());
220 break;
221 }
222 }
223
LowerIntrinsicCall(CallInst * CI)224 void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
225 IRBuilder<> Builder(CI);
226 LLVMContext &Context = CI->getContext();
227
228 const Function *Callee = CI->getCalledFunction();
229 assert(Callee && "Cannot lower an indirect call!");
230
231 switch (Callee->getIntrinsicID()) {
232 case Intrinsic::not_intrinsic:
233 report_fatal_error("Cannot lower a call to a non-intrinsic function '"+
234 Callee->getName() + "'!");
235 default:
236 report_fatal_error("Code generator does not support intrinsic function '"+
237 Callee->getName()+"'!");
238
239 case Intrinsic::expect: {
240 // Just replace __builtin_expect(exp, c) with EXP.
241 Value *V = CI->getArgOperand(0);
242 CI->replaceAllUsesWith(V);
243 break;
244 }
245
246 case Intrinsic::allow_runtime_check:
247 case Intrinsic::allow_ubsan_check:
248 CI->replaceAllUsesWith(ConstantInt::getTrue(CI->getType()));
249 return;
250
251 case Intrinsic::ctpop:
252 CI->replaceAllUsesWith(LowerCTPOP(Context, CI->getArgOperand(0), CI));
253 break;
254
255 case Intrinsic::bswap:
256 CI->replaceAllUsesWith(LowerBSWAP(Context, CI->getArgOperand(0), CI));
257 break;
258
259 case Intrinsic::ctlz:
260 CI->replaceAllUsesWith(LowerCTLZ(Context, CI->getArgOperand(0), CI));
261 break;
262
263 case Intrinsic::cttz: {
264 // cttz(x) -> ctpop(~X & (X-1))
265 Value *Src = CI->getArgOperand(0);
266 Value *NotSrc = Builder.CreateNot(Src);
267 NotSrc->setName(Src->getName() + ".not");
268 Value *SrcM1 = ConstantInt::get(Src->getType(), 1);
269 SrcM1 = Builder.CreateSub(Src, SrcM1);
270 Src = LowerCTPOP(Context, Builder.CreateAnd(NotSrc, SrcM1), CI);
271 CI->replaceAllUsesWith(Src);
272 break;
273 }
274
275 case Intrinsic::stacksave:
276 case Intrinsic::stackrestore: {
277 if (!Warned)
278 errs() << "WARNING: this target does not support the llvm.stack"
279 << (Callee->getIntrinsicID() == Intrinsic::stacksave ?
280 "save" : "restore") << " intrinsic.\n";
281 Warned = true;
282 if (Callee->getIntrinsicID() == Intrinsic::stacksave)
283 CI->replaceAllUsesWith(Constant::getNullValue(CI->getType()));
284 break;
285 }
286
287 case Intrinsic::get_dynamic_area_offset:
288 errs() << "WARNING: this target does not support the custom llvm.get."
289 "dynamic.area.offset. It is being lowered to a constant 0\n";
290 // Just lower it to a constant 0 because for most targets
291 // @llvm.get.dynamic.area.offset is lowered to zero.
292 CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 0));
293 break;
294 case Intrinsic::returnaddress:
295 case Intrinsic::frameaddress:
296 errs() << "WARNING: this target does not support the llvm."
297 << (Callee->getIntrinsicID() == Intrinsic::returnaddress ?
298 "return" : "frame") << "address intrinsic.\n";
299 CI->replaceAllUsesWith(
300 ConstantPointerNull::get(cast<PointerType>(CI->getType())));
301 break;
302 case Intrinsic::addressofreturnaddress:
303 errs() << "WARNING: this target does not support the "
304 "llvm.addressofreturnaddress intrinsic.\n";
305 CI->replaceAllUsesWith(
306 ConstantPointerNull::get(cast<PointerType>(CI->getType())));
307 break;
308
309 case Intrinsic::prefetch:
310 break; // Simply strip out prefetches on unsupported architectures
311
312 case Intrinsic::pcmarker:
313 break; // Simply strip out pcmarker on unsupported architectures
314 case Intrinsic::readcyclecounter: {
315 errs() << "WARNING: this target does not support the llvm.readcyclecoun"
316 << "ter intrinsic. It is being lowered to a constant 0\n";
317 CI->replaceAllUsesWith(ConstantInt::get(Type::getInt64Ty(Context), 0));
318 break;
319 }
320 case Intrinsic::readsteadycounter: {
321 errs() << "WARNING: this target does not support the llvm.readsteadycounter"
322 << " intrinsic. It is being lowered to a constant 0\n";
323 CI->replaceAllUsesWith(ConstantInt::get(Type::getInt64Ty(Context), 0));
324 break;
325 }
326
327 case Intrinsic::dbg_declare:
328 case Intrinsic::dbg_label:
329 break; // Simply strip out debugging intrinsics
330
331 case Intrinsic::eh_typeid_for:
332 // Return something different to eh_selector.
333 CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 1));
334 break;
335
336 case Intrinsic::annotation:
337 case Intrinsic::ptr_annotation:
338 // Just drop the annotation, but forward the value
339 CI->replaceAllUsesWith(CI->getOperand(0));
340 break;
341
342 case Intrinsic::assume:
343 case Intrinsic::experimental_noalias_scope_decl:
344 case Intrinsic::var_annotation:
345 break; // Strip out these intrinsics
346
347 case Intrinsic::memcpy: {
348 Type *IntPtr = DL.getIntPtrType(Context);
349 Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
350 /* isSigned */ false);
351 Value *Ops[3];
352 Ops[0] = CI->getArgOperand(0);
353 Ops[1] = CI->getArgOperand(1);
354 Ops[2] = Size;
355 ReplaceCallWith("memcpy", CI, Ops, Ops+3, CI->getArgOperand(0)->getType());
356 break;
357 }
358 case Intrinsic::memmove: {
359 Type *IntPtr = DL.getIntPtrType(Context);
360 Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
361 /* isSigned */ false);
362 Value *Ops[3];
363 Ops[0] = CI->getArgOperand(0);
364 Ops[1] = CI->getArgOperand(1);
365 Ops[2] = Size;
366 ReplaceCallWith("memmove", CI, Ops, Ops+3, CI->getArgOperand(0)->getType());
367 break;
368 }
369 case Intrinsic::memset: {
370 Value *Op0 = CI->getArgOperand(0);
371 Type *IntPtr = DL.getIntPtrType(Op0->getType());
372 Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
373 /* isSigned */ false);
374 Value *Ops[3];
375 Ops[0] = Op0;
376 // Extend the amount to i32.
377 Ops[1] = Builder.CreateIntCast(CI->getArgOperand(1),
378 Type::getInt32Ty(Context),
379 /* isSigned */ false);
380 Ops[2] = Size;
381 ReplaceCallWith("memset", CI, Ops, Ops+3, CI->getArgOperand(0)->getType());
382 break;
383 }
384 case Intrinsic::sqrt: {
385 ReplaceFPIntrinsicWithCall(CI, "sqrtf", "sqrt", "sqrtl");
386 break;
387 }
388 case Intrinsic::log: {
389 ReplaceFPIntrinsicWithCall(CI, "logf", "log", "logl");
390 break;
391 }
392 case Intrinsic::log2: {
393 ReplaceFPIntrinsicWithCall(CI, "log2f", "log2", "log2l");
394 break;
395 }
396 case Intrinsic::log10: {
397 ReplaceFPIntrinsicWithCall(CI, "log10f", "log10", "log10l");
398 break;
399 }
400 case Intrinsic::exp: {
401 ReplaceFPIntrinsicWithCall(CI, "expf", "exp", "expl");
402 break;
403 }
404 case Intrinsic::exp2: {
405 ReplaceFPIntrinsicWithCall(CI, "exp2f", "exp2", "exp2l");
406 break;
407 }
408 case Intrinsic::pow: {
409 ReplaceFPIntrinsicWithCall(CI, "powf", "pow", "powl");
410 break;
411 }
412 case Intrinsic::sin: {
413 ReplaceFPIntrinsicWithCall(CI, "sinf", "sin", "sinl");
414 break;
415 }
416 case Intrinsic::cos: {
417 ReplaceFPIntrinsicWithCall(CI, "cosf", "cos", "cosl");
418 break;
419 }
420 case Intrinsic::floor: {
421 ReplaceFPIntrinsicWithCall(CI, "floorf", "floor", "floorl");
422 break;
423 }
424 case Intrinsic::ceil: {
425 ReplaceFPIntrinsicWithCall(CI, "ceilf", "ceil", "ceill");
426 break;
427 }
428 case Intrinsic::trunc: {
429 ReplaceFPIntrinsicWithCall(CI, "truncf", "trunc", "truncl");
430 break;
431 }
432 case Intrinsic::round: {
433 ReplaceFPIntrinsicWithCall(CI, "roundf", "round", "roundl");
434 break;
435 }
436 case Intrinsic::roundeven: {
437 ReplaceFPIntrinsicWithCall(CI, "roundevenf", "roundeven", "roundevenl");
438 break;
439 }
440 case Intrinsic::copysign: {
441 ReplaceFPIntrinsicWithCall(CI, "copysignf", "copysign", "copysignl");
442 break;
443 }
444 case Intrinsic::get_rounding:
445 // Lower to "round to the nearest"
446 if (!CI->getType()->isVoidTy())
447 CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 1));
448 break;
449 case Intrinsic::invariant_start:
450 case Intrinsic::lifetime_start:
451 // Discard region information.
452 CI->replaceAllUsesWith(UndefValue::get(CI->getType()));
453 break;
454 case Intrinsic::invariant_end:
455 case Intrinsic::lifetime_end:
456 // Discard region information.
457 break;
458 }
459
460 assert(CI->use_empty() &&
461 "Lowering should have eliminated any uses of the intrinsic call!");
462 CI->eraseFromParent();
463 }
464
LowerToByteSwap(CallInst * CI)465 bool IntrinsicLowering::LowerToByteSwap(CallInst *CI) {
466 // Verify this is a simple bswap.
467 if (CI->arg_size() != 1 || CI->getType() != CI->getArgOperand(0)->getType() ||
468 !CI->getType()->isIntegerTy())
469 return false;
470
471 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
472 if (!Ty)
473 return false;
474
475 // Okay, we can do this xform, do so now.
476 Module *M = CI->getModule();
477 Function *Int = Intrinsic::getDeclaration(M, Intrinsic::bswap, Ty);
478
479 Value *Op = CI->getArgOperand(0);
480 Op = CallInst::Create(Int, Op, CI->getName(), CI->getIterator());
481
482 CI->replaceAllUsesWith(Op);
483 CI->eraseFromParent();
484 return true;
485 }
486