1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file implements some simple delegations needed for call lowering. 11 /// 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/Analysis.h" 15 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 16 #include "llvm/CodeGen/GlobalISel/Utils.h" 17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 18 #include "llvm/CodeGen/MachineOperand.h" 19 #include "llvm/CodeGen/MachineRegisterInfo.h" 20 #include "llvm/CodeGen/TargetLowering.h" 21 #include "llvm/IR/DataLayout.h" 22 #include "llvm/IR/Instructions.h" 23 #include "llvm/IR/LLVMContext.h" 24 #include "llvm/IR/Module.h" 25 26 #define DEBUG_TYPE "call-lowering" 27 28 using namespace llvm; 29 30 void CallLowering::anchor() {} 31 32 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS, 33 ArrayRef<Register> ResRegs, 34 ArrayRef<ArrayRef<Register>> ArgRegs, 35 Register SwiftErrorVReg, 36 std::function<unsigned()> GetCalleeReg) const { 37 CallLoweringInfo Info; 38 auto &DL = CS.getParent()->getParent()->getParent()->getDataLayout(); 39 40 // First step is to marshall all the function's parameters into the correct 41 // physregs and memory locations. Gather the sequence of argument types that 42 // we'll pass to the assigner function. 43 unsigned i = 0; 44 unsigned NumFixedArgs = CS.getFunctionType()->getNumParams(); 45 for (auto &Arg : CS.args()) { 46 ArgInfo OrigArg{ArgRegs[i], Arg->getType(), ISD::ArgFlagsTy{}, 47 i < NumFixedArgs}; 48 setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CS); 49 Info.OrigArgs.push_back(OrigArg); 50 ++i; 51 } 52 53 if (const Function *F = CS.getCalledFunction()) 54 Info.Callee = MachineOperand::CreateGA(F, 0); 55 else 56 Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false); 57 58 Info.OrigRet = ArgInfo{ResRegs, CS.getType(), ISD::ArgFlagsTy{}}; 59 if (!Info.OrigRet.Ty->isVoidTy()) 60 setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CS); 61 62 Info.KnownCallees = 63 CS.getInstruction()->getMetadata(LLVMContext::MD_callees); 64 Info.CallConv = CS.getCallingConv(); 65 Info.SwiftErrorVReg = SwiftErrorVReg; 66 Info.IsMustTailCall = CS.isMustTailCall(); 67 Info.IsTailCall = CS.isTailCall() && 68 isInTailCallPosition(CS, MIRBuilder.getMF().getTarget()) && 69 (MIRBuilder.getMF() 70 .getFunction() 71 .getFnAttribute("disable-tail-calls") 72 .getValueAsString() != "true"); 73 Info.IsVarArg = CS.getFunctionType()->isVarArg(); 74 return lowerCall(MIRBuilder, Info); 75 } 76 77 template <typename FuncInfoTy> 78 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, 79 const DataLayout &DL, 80 const FuncInfoTy &FuncInfo) const { 81 auto &Flags = Arg.Flags[0]; 82 const AttributeList &Attrs = FuncInfo.getAttributes(); 83 if (Attrs.hasAttribute(OpIdx, Attribute::ZExt)) 84 Flags.setZExt(); 85 if (Attrs.hasAttribute(OpIdx, Attribute::SExt)) 86 Flags.setSExt(); 87 if (Attrs.hasAttribute(OpIdx, Attribute::InReg)) 88 Flags.setInReg(); 89 if (Attrs.hasAttribute(OpIdx, Attribute::StructRet)) 90 Flags.setSRet(); 91 if (Attrs.hasAttribute(OpIdx, Attribute::SwiftSelf)) 92 Flags.setSwiftSelf(); 93 if (Attrs.hasAttribute(OpIdx, Attribute::SwiftError)) 94 Flags.setSwiftError(); 95 if (Attrs.hasAttribute(OpIdx, Attribute::ByVal)) 96 Flags.setByVal(); 97 if (Attrs.hasAttribute(OpIdx, Attribute::InAlloca)) 98 Flags.setInAlloca(); 99 100 if (Flags.isByVal() || Flags.isInAlloca()) { 101 Type *ElementTy = cast<PointerType>(Arg.Ty)->getElementType(); 102 103 auto Ty = Attrs.getAttribute(OpIdx, Attribute::ByVal).getValueAsType(); 104 Flags.setByValSize(DL.getTypeAllocSize(Ty ? Ty : ElementTy)); 105 106 // For ByVal, alignment should be passed from FE. BE will guess if 107 // this info is not there but there are cases it cannot get right. 108 unsigned FrameAlign; 109 if (FuncInfo.getParamAlignment(OpIdx - 2)) 110 FrameAlign = FuncInfo.getParamAlignment(OpIdx - 2); 111 else 112 FrameAlign = getTLI()->getByValTypeAlignment(ElementTy, DL); 113 Flags.setByValAlign(Align(FrameAlign)); 114 } 115 if (Attrs.hasAttribute(OpIdx, Attribute::Nest)) 116 Flags.setNest(); 117 Flags.setOrigAlign(Align(DL.getABITypeAlignment(Arg.Ty))); 118 } 119 120 template void 121 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx, 122 const DataLayout &DL, 123 const Function &FuncInfo) const; 124 125 template void 126 CallLowering::setArgFlags<CallInst>(CallLowering::ArgInfo &Arg, unsigned OpIdx, 127 const DataLayout &DL, 128 const CallInst &FuncInfo) const; 129 130 Register CallLowering::packRegs(ArrayRef<Register> SrcRegs, Type *PackedTy, 131 MachineIRBuilder &MIRBuilder) const { 132 assert(SrcRegs.size() > 1 && "Nothing to pack"); 133 134 const DataLayout &DL = MIRBuilder.getMF().getDataLayout(); 135 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 136 137 LLT PackedLLT = getLLTForType(*PackedTy, DL); 138 139 SmallVector<LLT, 8> LLTs; 140 SmallVector<uint64_t, 8> Offsets; 141 computeValueLLTs(DL, *PackedTy, LLTs, &Offsets); 142 assert(LLTs.size() == SrcRegs.size() && "Regs / types mismatch"); 143 144 Register Dst = MRI->createGenericVirtualRegister(PackedLLT); 145 MIRBuilder.buildUndef(Dst); 146 for (unsigned i = 0; i < SrcRegs.size(); ++i) { 147 Register NewDst = MRI->createGenericVirtualRegister(PackedLLT); 148 MIRBuilder.buildInsert(NewDst, Dst, SrcRegs[i], Offsets[i]); 149 Dst = NewDst; 150 } 151 152 return Dst; 153 } 154 155 void CallLowering::unpackRegs(ArrayRef<Register> DstRegs, Register SrcReg, 156 Type *PackedTy, 157 MachineIRBuilder &MIRBuilder) const { 158 assert(DstRegs.size() > 1 && "Nothing to unpack"); 159 160 const DataLayout &DL = MIRBuilder.getMF().getDataLayout(); 161 162 SmallVector<LLT, 8> LLTs; 163 SmallVector<uint64_t, 8> Offsets; 164 computeValueLLTs(DL, *PackedTy, LLTs, &Offsets); 165 assert(LLTs.size() == DstRegs.size() && "Regs / types mismatch"); 166 167 for (unsigned i = 0; i < DstRegs.size(); ++i) 168 MIRBuilder.buildExtract(DstRegs[i], SrcReg, Offsets[i]); 169 } 170 171 bool CallLowering::handleAssignments(MachineIRBuilder &MIRBuilder, 172 SmallVectorImpl<ArgInfo> &Args, 173 ValueHandler &Handler) const { 174 MachineFunction &MF = MIRBuilder.getMF(); 175 const Function &F = MF.getFunction(); 176 SmallVector<CCValAssign, 16> ArgLocs; 177 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext()); 178 return handleAssignments(CCInfo, ArgLocs, MIRBuilder, Args, Handler); 179 } 180 181 bool CallLowering::handleAssignments(CCState &CCInfo, 182 SmallVectorImpl<CCValAssign> &ArgLocs, 183 MachineIRBuilder &MIRBuilder, 184 SmallVectorImpl<ArgInfo> &Args, 185 ValueHandler &Handler) const { 186 MachineFunction &MF = MIRBuilder.getMF(); 187 const Function &F = MF.getFunction(); 188 const DataLayout &DL = F.getParent()->getDataLayout(); 189 190 unsigned NumArgs = Args.size(); 191 for (unsigned i = 0; i != NumArgs; ++i) { 192 MVT CurVT = MVT::getVT(Args[i].Ty); 193 if (Handler.assignArg(i, CurVT, CurVT, CCValAssign::Full, Args[i], 194 Args[i].Flags[0], CCInfo)) { 195 if (!CurVT.isValid()) 196 return false; 197 MVT NewVT = TLI->getRegisterTypeForCallingConv( 198 F.getContext(), F.getCallingConv(), EVT(CurVT)); 199 200 // If we need to split the type over multiple regs, check it's a scenario 201 // we currently support. 202 unsigned NumParts = TLI->getNumRegistersForCallingConv( 203 F.getContext(), F.getCallingConv(), CurVT); 204 if (NumParts > 1) { 205 // For now only handle exact splits. 206 if (NewVT.getSizeInBits() * NumParts != CurVT.getSizeInBits()) 207 return false; 208 } 209 210 // For incoming arguments (physregs to vregs), we could have values in 211 // physregs (or memlocs) which we want to extract and copy to vregs. 212 // During this, we might have to deal with the LLT being split across 213 // multiple regs, so we have to record this information for later. 214 // 215 // If we have outgoing args, then we have the opposite case. We have a 216 // vreg with an LLT which we want to assign to a physical location, and 217 // we might have to record that the value has to be split later. 218 if (Handler.isIncomingArgumentHandler()) { 219 if (NumParts == 1) { 220 // Try to use the register type if we couldn't assign the VT. 221 if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full, Args[i], 222 Args[i].Flags[0], CCInfo)) 223 return false; 224 } else { 225 // We're handling an incoming arg which is split over multiple regs. 226 // E.g. passing an s128 on AArch64. 227 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0]; 228 Args[i].OrigRegs.push_back(Args[i].Regs[0]); 229 Args[i].Regs.clear(); 230 Args[i].Flags.clear(); 231 LLT NewLLT = getLLTForMVT(NewVT); 232 // For each split register, create and assign a vreg that will store 233 // the incoming component of the larger value. These will later be 234 // merged to form the final vreg. 235 for (unsigned Part = 0; Part < NumParts; ++Part) { 236 Register Reg = 237 MIRBuilder.getMRI()->createGenericVirtualRegister(NewLLT); 238 ISD::ArgFlagsTy Flags = OrigFlags; 239 if (Part == 0) { 240 Flags.setSplit(); 241 } else { 242 Flags.setOrigAlign(Align::None()); 243 if (Part == NumParts - 1) 244 Flags.setSplitEnd(); 245 } 246 Args[i].Regs.push_back(Reg); 247 Args[i].Flags.push_back(Flags); 248 if (Handler.assignArg(i + Part, NewVT, NewVT, CCValAssign::Full, 249 Args[i], Args[i].Flags[Part], CCInfo)) { 250 // Still couldn't assign this smaller part type for some reason. 251 return false; 252 } 253 } 254 } 255 } else { 256 // Handling an outgoing arg that might need to be split. 257 if (NumParts < 2) 258 return false; // Don't know how to deal with this type combination. 259 260 // This type is passed via multiple registers in the calling convention. 261 // We need to extract the individual parts. 262 Register LargeReg = Args[i].Regs[0]; 263 LLT SmallTy = LLT::scalar(NewVT.getSizeInBits()); 264 auto Unmerge = MIRBuilder.buildUnmerge(SmallTy, LargeReg); 265 assert(Unmerge->getNumOperands() == NumParts + 1); 266 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0]; 267 // We're going to replace the regs and flags with the split ones. 268 Args[i].Regs.clear(); 269 Args[i].Flags.clear(); 270 for (unsigned PartIdx = 0; PartIdx < NumParts; ++PartIdx) { 271 ISD::ArgFlagsTy Flags = OrigFlags; 272 if (PartIdx == 0) { 273 Flags.setSplit(); 274 } else { 275 Flags.setOrigAlign(Align::None()); 276 if (PartIdx == NumParts - 1) 277 Flags.setSplitEnd(); 278 } 279 Args[i].Regs.push_back(Unmerge.getReg(PartIdx)); 280 Args[i].Flags.push_back(Flags); 281 if (Handler.assignArg(i + PartIdx, NewVT, NewVT, CCValAssign::Full, 282 Args[i], Args[i].Flags[PartIdx], CCInfo)) 283 return false; 284 } 285 } 286 } 287 } 288 289 for (unsigned i = 0, e = Args.size(), j = 0; i != e; ++i, ++j) { 290 assert(j < ArgLocs.size() && "Skipped too many arg locs"); 291 292 CCValAssign &VA = ArgLocs[j]; 293 assert(VA.getValNo() == i && "Location doesn't correspond to current arg"); 294 295 if (VA.needsCustom()) { 296 j += Handler.assignCustomValue(Args[i], makeArrayRef(ArgLocs).slice(j)); 297 continue; 298 } 299 300 // FIXME: Pack registers if we have more than one. 301 Register ArgReg = Args[i].Regs[0]; 302 303 MVT OrigVT = MVT::getVT(Args[i].Ty); 304 MVT VAVT = VA.getValVT(); 305 if (VA.isRegLoc()) { 306 if (Handler.isIncomingArgumentHandler() && VAVT != OrigVT) { 307 if (VAVT.getSizeInBits() < OrigVT.getSizeInBits()) { 308 // Expected to be multiple regs for a single incoming arg. 309 unsigned NumArgRegs = Args[i].Regs.size(); 310 if (NumArgRegs < 2) 311 return false; 312 313 assert((j + (NumArgRegs - 1)) < ArgLocs.size() && 314 "Too many regs for number of args"); 315 for (unsigned Part = 0; Part < NumArgRegs; ++Part) { 316 // There should be Regs.size() ArgLocs per argument. 317 VA = ArgLocs[j + Part]; 318 Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA); 319 } 320 j += NumArgRegs - 1; 321 // Merge the split registers into the expected larger result vreg 322 // of the original call. 323 MIRBuilder.buildMerge(Args[i].OrigRegs[0], Args[i].Regs); 324 continue; 325 } 326 const LLT VATy(VAVT); 327 Register NewReg = 328 MIRBuilder.getMRI()->createGenericVirtualRegister(VATy); 329 Handler.assignValueToReg(NewReg, VA.getLocReg(), VA); 330 // If it's a vector type, we either need to truncate the elements 331 // or do an unmerge to get the lower block of elements. 332 if (VATy.isVector() && 333 VATy.getNumElements() > OrigVT.getVectorNumElements()) { 334 const LLT OrigTy(OrigVT); 335 // Just handle the case where the VA type is 2 * original type. 336 if (VATy.getNumElements() != OrigVT.getVectorNumElements() * 2) { 337 LLVM_DEBUG(dbgs() 338 << "Incoming promoted vector arg has too many elts"); 339 return false; 340 } 341 auto Unmerge = MIRBuilder.buildUnmerge({OrigTy, OrigTy}, {NewReg}); 342 MIRBuilder.buildCopy(ArgReg, Unmerge.getReg(0)); 343 } else { 344 MIRBuilder.buildTrunc(ArgReg, {NewReg}).getReg(0); 345 } 346 } else if (!Handler.isIncomingArgumentHandler()) { 347 assert((j + (Args[i].Regs.size() - 1)) < ArgLocs.size() && 348 "Too many regs for number of args"); 349 // This is an outgoing argument that might have been split. 350 for (unsigned Part = 0; Part < Args[i].Regs.size(); ++Part) { 351 // There should be Regs.size() ArgLocs per argument. 352 VA = ArgLocs[j + Part]; 353 Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA); 354 } 355 j += Args[i].Regs.size() - 1; 356 } else { 357 Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA); 358 } 359 } else if (VA.isMemLoc()) { 360 // Don't currently support loading/storing a type that needs to be split 361 // to the stack. Should be easy, just not implemented yet. 362 if (Args[i].Regs.size() > 1) { 363 LLVM_DEBUG( 364 dbgs() 365 << "Load/store a split arg to/from the stack not implemented yet"); 366 return false; 367 } 368 MVT VT = MVT::getVT(Args[i].Ty); 369 unsigned Size = VT == MVT::iPTR ? DL.getPointerSize() 370 : alignTo(VT.getSizeInBits(), 8) / 8; 371 unsigned Offset = VA.getLocMemOffset(); 372 MachinePointerInfo MPO; 373 Register StackAddr = Handler.getStackAddress(Size, Offset, MPO); 374 Handler.assignValueToAddress(ArgReg, StackAddr, Size, MPO, VA); 375 } else { 376 // FIXME: Support byvals and other weirdness 377 return false; 378 } 379 } 380 return true; 381 } 382 383 bool CallLowering::analyzeArgInfo(CCState &CCState, 384 SmallVectorImpl<ArgInfo> &Args, 385 CCAssignFn &AssignFnFixed, 386 CCAssignFn &AssignFnVarArg) const { 387 for (unsigned i = 0, e = Args.size(); i < e; ++i) { 388 MVT VT = MVT::getVT(Args[i].Ty); 389 CCAssignFn &Fn = Args[i].IsFixed ? AssignFnFixed : AssignFnVarArg; 390 if (Fn(i, VT, VT, CCValAssign::Full, Args[i].Flags[0], CCState)) { 391 // Bail out on anything we can't handle. 392 LLVM_DEBUG(dbgs() << "Cannot analyze " << EVT(VT).getEVTString() 393 << " (arg number = " << i << "\n"); 394 return false; 395 } 396 } 397 return true; 398 } 399 400 bool CallLowering::resultsCompatible(CallLoweringInfo &Info, 401 MachineFunction &MF, 402 SmallVectorImpl<ArgInfo> &InArgs, 403 CCAssignFn &CalleeAssignFnFixed, 404 CCAssignFn &CalleeAssignFnVarArg, 405 CCAssignFn &CallerAssignFnFixed, 406 CCAssignFn &CallerAssignFnVarArg) const { 407 const Function &F = MF.getFunction(); 408 CallingConv::ID CalleeCC = Info.CallConv; 409 CallingConv::ID CallerCC = F.getCallingConv(); 410 411 if (CallerCC == CalleeCC) 412 return true; 413 414 SmallVector<CCValAssign, 16> ArgLocs1; 415 CCState CCInfo1(CalleeCC, false, MF, ArgLocs1, F.getContext()); 416 if (!analyzeArgInfo(CCInfo1, InArgs, CalleeAssignFnFixed, 417 CalleeAssignFnVarArg)) 418 return false; 419 420 SmallVector<CCValAssign, 16> ArgLocs2; 421 CCState CCInfo2(CallerCC, false, MF, ArgLocs2, F.getContext()); 422 if (!analyzeArgInfo(CCInfo2, InArgs, CallerAssignFnFixed, 423 CalleeAssignFnVarArg)) 424 return false; 425 426 // We need the argument locations to match up exactly. If there's more in 427 // one than the other, then we are done. 428 if (ArgLocs1.size() != ArgLocs2.size()) 429 return false; 430 431 // Make sure that each location is passed in exactly the same way. 432 for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) { 433 const CCValAssign &Loc1 = ArgLocs1[i]; 434 const CCValAssign &Loc2 = ArgLocs2[i]; 435 436 // We need both of them to be the same. So if one is a register and one 437 // isn't, we're done. 438 if (Loc1.isRegLoc() != Loc2.isRegLoc()) 439 return false; 440 441 if (Loc1.isRegLoc()) { 442 // If they don't have the same register location, we're done. 443 if (Loc1.getLocReg() != Loc2.getLocReg()) 444 return false; 445 446 // They matched, so we can move to the next ArgLoc. 447 continue; 448 } 449 450 // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match. 451 if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset()) 452 return false; 453 } 454 455 return true; 456 } 457 458 Register CallLowering::ValueHandler::extendRegister(Register ValReg, 459 CCValAssign &VA) { 460 LLT LocTy{VA.getLocVT()}; 461 if (LocTy.getSizeInBits() == MRI.getType(ValReg).getSizeInBits()) 462 return ValReg; 463 switch (VA.getLocInfo()) { 464 default: break; 465 case CCValAssign::Full: 466 case CCValAssign::BCvt: 467 // FIXME: bitconverting between vector types may or may not be a 468 // nop in big-endian situations. 469 return ValReg; 470 case CCValAssign::AExt: { 471 auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg); 472 return MIB->getOperand(0).getReg(); 473 } 474 case CCValAssign::SExt: { 475 Register NewReg = MRI.createGenericVirtualRegister(LocTy); 476 MIRBuilder.buildSExt(NewReg, ValReg); 477 return NewReg; 478 } 479 case CCValAssign::ZExt: { 480 Register NewReg = MRI.createGenericVirtualRegister(LocTy); 481 MIRBuilder.buildZExt(NewReg, ValReg); 482 return NewReg; 483 } 484 } 485 llvm_unreachable("unable to extend register"); 486 } 487 488 void CallLowering::ValueHandler::anchor() {} 489