1 //===- llvm/lib/Target/X86/X86CallLowering.cpp - Call lowering ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file implements the lowering of LLVM calls to machine code calls for 11 /// GlobalISel. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86CallLowering.h" 16 #include "X86CallingConv.h" 17 #include "X86ISelLowering.h" 18 #include "X86InstrInfo.h" 19 #include "X86MachineFunctionInfo.h" 20 #include "X86RegisterInfo.h" 21 #include "X86Subtarget.h" 22 #include "llvm/ADT/ArrayRef.h" 23 #include "llvm/ADT/SmallVector.h" 24 #include "llvm/CodeGen/Analysis.h" 25 #include "llvm/CodeGen/CallingConvLower.h" 26 #include "llvm/CodeGen/FunctionLoweringInfo.h" 27 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 28 #include "llvm/CodeGen/GlobalISel/Utils.h" 29 #include "llvm/CodeGen/LowLevelTypeUtils.h" 30 #include "llvm/CodeGen/MachineBasicBlock.h" 31 #include "llvm/CodeGen/MachineFrameInfo.h" 32 #include "llvm/CodeGen/MachineFunction.h" 33 #include "llvm/CodeGen/MachineInstrBuilder.h" 34 #include "llvm/CodeGen/MachineMemOperand.h" 35 #include "llvm/CodeGen/MachineOperand.h" 36 #include "llvm/CodeGen/MachineRegisterInfo.h" 37 #include "llvm/CodeGen/TargetInstrInfo.h" 38 #include "llvm/CodeGen/TargetSubtargetInfo.h" 39 #include "llvm/CodeGen/ValueTypes.h" 40 #include "llvm/CodeGenTypes/LowLevelType.h" 41 #include "llvm/CodeGenTypes/MachineValueType.h" 42 #include "llvm/IR/Attributes.h" 43 #include "llvm/IR/DataLayout.h" 44 #include "llvm/IR/Function.h" 45 #include "llvm/IR/Value.h" 46 #include "llvm/MC/MCRegisterInfo.h" 47 #include <cassert> 48 #include <cstdint> 49 50 using namespace llvm; 51 52 X86CallLowering::X86CallLowering(const X86TargetLowering &TLI) 53 : CallLowering(&TLI) {} 54 55 namespace { 56 57 struct X86OutgoingValueAssigner : public CallLowering::OutgoingValueAssigner { 58 private: 59 uint64_t StackSize = 0; 60 unsigned NumXMMRegs = 0; 61 62 public: 63 uint64_t getStackSize() { return StackSize; } 64 unsigned getNumXmmRegs() { return NumXMMRegs; } 65 66 X86OutgoingValueAssigner(CCAssignFn *AssignFn_) 67 : CallLowering::OutgoingValueAssigner(AssignFn_) {} 68 69 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT, 70 CCValAssign::LocInfo LocInfo, 71 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags, 72 CCState &State) override { 73 bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); 74 StackSize = State.getStackSize(); 75 76 static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, 77 X86::XMM3, X86::XMM4, X86::XMM5, 78 X86::XMM6, X86::XMM7}; 79 if (!Info.IsFixed) 80 NumXMMRegs = State.getFirstUnallocated(XMMArgRegs); 81 82 return Res; 83 } 84 }; 85 86 struct X86OutgoingValueHandler : public CallLowering::OutgoingValueHandler { 87 X86OutgoingValueHandler(MachineIRBuilder &MIRBuilder, 88 MachineRegisterInfo &MRI, MachineInstrBuilder &MIB) 89 : OutgoingValueHandler(MIRBuilder, MRI), MIB(MIB), 90 DL(MIRBuilder.getMF().getDataLayout()), 91 STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {} 92 93 Register getStackAddress(uint64_t Size, int64_t Offset, 94 MachinePointerInfo &MPO, 95 ISD::ArgFlagsTy Flags) override { 96 LLT p0 = LLT::pointer(0, DL.getPointerSizeInBits(0)); 97 LLT SType = LLT::scalar(DL.getPointerSizeInBits(0)); 98 auto SPReg = 99 MIRBuilder.buildCopy(p0, STI.getRegisterInfo()->getStackRegister()); 100 101 auto OffsetReg = MIRBuilder.buildConstant(SType, Offset); 102 103 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg); 104 105 MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); 106 return AddrReg.getReg(0); 107 } 108 109 void assignValueToReg(Register ValVReg, Register PhysReg, 110 const CCValAssign &VA) override { 111 MIB.addUse(PhysReg, RegState::Implicit); 112 Register ExtReg = extendRegister(ValVReg, VA); 113 MIRBuilder.buildCopy(PhysReg, ExtReg); 114 } 115 116 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy, 117 const MachinePointerInfo &MPO, 118 const CCValAssign &VA) override { 119 MachineFunction &MF = MIRBuilder.getMF(); 120 Register ExtReg = extendRegister(ValVReg, VA); 121 122 auto *MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy, 123 inferAlignFromPtrInfo(MF, MPO)); 124 MIRBuilder.buildStore(ExtReg, Addr, *MMO); 125 } 126 127 protected: 128 MachineInstrBuilder &MIB; 129 const DataLayout &DL; 130 const X86Subtarget &STI; 131 }; 132 133 } // end anonymous namespace 134 135 bool X86CallLowering::canLowerReturn( 136 MachineFunction &MF, CallingConv::ID CallConv, 137 SmallVectorImpl<CallLowering::BaseArgInfo> &Outs, bool IsVarArg) const { 138 LLVMContext &Context = MF.getFunction().getContext(); 139 SmallVector<CCValAssign, 16> RVLocs; 140 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); 141 return checkReturn(CCInfo, Outs, RetCC_X86); 142 } 143 144 bool X86CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, 145 const Value *Val, ArrayRef<Register> VRegs, 146 FunctionLoweringInfo &FLI) const { 147 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && 148 "Return value without a vreg"); 149 MachineFunction &MF = MIRBuilder.getMF(); 150 auto MIB = MIRBuilder.buildInstrNoInsert(X86::RET).addImm(0); 151 auto FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 152 const auto &STI = MF.getSubtarget<X86Subtarget>(); 153 Register RetReg = STI.is64Bit() ? X86::RAX : X86::EAX; 154 155 if (!FLI.CanLowerReturn) { 156 insertSRetStores(MIRBuilder, Val->getType(), VRegs, FLI.DemoteRegister); 157 MIRBuilder.buildCopy(RetReg, FLI.DemoteRegister); 158 MIB.addReg(RetReg); 159 } else if (Register Reg = FuncInfo->getSRetReturnReg()) { 160 MIRBuilder.buildCopy(RetReg, Reg); 161 MIB.addReg(RetReg); 162 } else if (!VRegs.empty()) { 163 const Function &F = MF.getFunction(); 164 MachineRegisterInfo &MRI = MF.getRegInfo(); 165 const DataLayout &DL = MF.getDataLayout(); 166 167 ArgInfo OrigRetInfo(VRegs, Val->getType(), 0); 168 setArgFlags(OrigRetInfo, AttributeList::ReturnIndex, DL, F); 169 170 SmallVector<ArgInfo, 4> SplitRetInfos; 171 splitToValueTypes(OrigRetInfo, SplitRetInfos, DL, F.getCallingConv()); 172 173 X86OutgoingValueAssigner Assigner(RetCC_X86); 174 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB); 175 if (!determineAndHandleAssignments(Handler, Assigner, SplitRetInfos, 176 MIRBuilder, F.getCallingConv(), 177 F.isVarArg())) 178 return false; 179 } 180 181 MIRBuilder.insertInstr(MIB); 182 return true; 183 } 184 185 namespace { 186 187 struct X86IncomingValueHandler : public CallLowering::IncomingValueHandler { 188 X86IncomingValueHandler(MachineIRBuilder &MIRBuilder, 189 MachineRegisterInfo &MRI) 190 : IncomingValueHandler(MIRBuilder, MRI), 191 DL(MIRBuilder.getMF().getDataLayout()) {} 192 193 Register getStackAddress(uint64_t Size, int64_t Offset, 194 MachinePointerInfo &MPO, 195 ISD::ArgFlagsTy Flags) override { 196 auto &MFI = MIRBuilder.getMF().getFrameInfo(); 197 198 // Byval is assumed to be writable memory, but other stack passed arguments 199 // are not. 200 const bool IsImmutable = !Flags.isByVal(); 201 202 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable); 203 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); 204 205 return MIRBuilder 206 .buildFrameIndex(LLT::pointer(0, DL.getPointerSizeInBits(0)), FI) 207 .getReg(0); 208 } 209 210 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy, 211 const MachinePointerInfo &MPO, 212 const CCValAssign &VA) override { 213 MachineFunction &MF = MIRBuilder.getMF(); 214 auto *MMO = MF.getMachineMemOperand( 215 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemTy, 216 inferAlignFromPtrInfo(MF, MPO)); 217 MIRBuilder.buildLoad(ValVReg, Addr, *MMO); 218 } 219 220 void assignValueToReg(Register ValVReg, Register PhysReg, 221 const CCValAssign &VA) override { 222 markPhysRegUsed(PhysReg); 223 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA); 224 } 225 226 /// How the physical register gets marked varies between formal 227 /// parameters (it's a basic-block live-in), and a call instruction 228 /// (it's an implicit-def of the BL). 229 virtual void markPhysRegUsed(unsigned PhysReg) = 0; 230 231 protected: 232 const DataLayout &DL; 233 }; 234 235 struct FormalArgHandler : public X86IncomingValueHandler { 236 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI) 237 : X86IncomingValueHandler(MIRBuilder, MRI) {} 238 239 void markPhysRegUsed(unsigned PhysReg) override { 240 MIRBuilder.getMRI()->addLiveIn(PhysReg); 241 MIRBuilder.getMBB().addLiveIn(PhysReg); 242 } 243 }; 244 245 struct CallReturnHandler : public X86IncomingValueHandler { 246 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 247 MachineInstrBuilder &MIB) 248 : X86IncomingValueHandler(MIRBuilder, MRI), MIB(MIB) {} 249 250 void markPhysRegUsed(unsigned PhysReg) override { 251 MIB.addDef(PhysReg, RegState::Implicit); 252 } 253 254 protected: 255 MachineInstrBuilder &MIB; 256 }; 257 258 } // end anonymous namespace 259 260 bool X86CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, 261 const Function &F, 262 ArrayRef<ArrayRef<Register>> VRegs, 263 FunctionLoweringInfo &FLI) const { 264 MachineFunction &MF = MIRBuilder.getMF(); 265 MachineRegisterInfo &MRI = MF.getRegInfo(); 266 auto DL = MF.getDataLayout(); 267 auto FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 268 269 SmallVector<ArgInfo, 8> SplitArgs; 270 271 if (!FLI.CanLowerReturn) 272 insertSRetIncomingArgument(F, SplitArgs, FLI.DemoteRegister, MRI, DL); 273 274 // TODO: handle variadic function 275 if (F.isVarArg()) 276 return false; 277 278 unsigned Idx = 0; 279 for (const auto &Arg : F.args()) { 280 // TODO: handle not simple cases. 281 if (Arg.hasAttribute(Attribute::ByVal) || 282 Arg.hasAttribute(Attribute::InReg) || 283 Arg.hasAttribute(Attribute::SwiftSelf) || 284 Arg.hasAttribute(Attribute::SwiftError) || 285 Arg.hasAttribute(Attribute::Nest) || VRegs[Idx].size() > 1) 286 return false; 287 288 if (Arg.hasAttribute(Attribute::StructRet)) { 289 assert(VRegs[Idx].size() == 1 && 290 "Unexpected amount of registers for sret argument."); 291 FuncInfo->setSRetReturnReg(VRegs[Idx][0]); 292 } 293 294 ArgInfo OrigArg(VRegs[Idx], Arg.getType(), Idx); 295 setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F); 296 splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv()); 297 Idx++; 298 } 299 300 if (SplitArgs.empty()) 301 return true; 302 303 MachineBasicBlock &MBB = MIRBuilder.getMBB(); 304 if (!MBB.empty()) 305 MIRBuilder.setInstr(*MBB.begin()); 306 307 X86OutgoingValueAssigner Assigner(CC_X86); 308 FormalArgHandler Handler(MIRBuilder, MRI); 309 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 310 F.getCallingConv(), F.isVarArg())) 311 return false; 312 313 // Move back to the end of the basic block. 314 MIRBuilder.setMBB(MBB); 315 316 return true; 317 } 318 319 bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, 320 CallLoweringInfo &Info) const { 321 MachineFunction &MF = MIRBuilder.getMF(); 322 const Function &F = MF.getFunction(); 323 MachineRegisterInfo &MRI = MF.getRegInfo(); 324 const DataLayout &DL = F.getDataLayout(); 325 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 326 const TargetInstrInfo &TII = *STI.getInstrInfo(); 327 const X86RegisterInfo *TRI = STI.getRegisterInfo(); 328 329 // Handle only Linux C, X86_64_SysV calling conventions for now. 330 if (!STI.isTargetLinux() || !(Info.CallConv == CallingConv::C || 331 Info.CallConv == CallingConv::X86_64_SysV)) 332 return false; 333 334 unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); 335 auto CallSeqStart = MIRBuilder.buildInstr(AdjStackDown); 336 337 // Create a temporarily-floating call instruction so we can add the implicit 338 // uses of arg registers. 339 bool Is64Bit = STI.is64Bit(); 340 unsigned CallOpc = Info.Callee.isReg() 341 ? (Is64Bit ? X86::CALL64r : X86::CALL32r) 342 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32); 343 344 auto MIB = MIRBuilder.buildInstrNoInsert(CallOpc) 345 .add(Info.Callee) 346 .addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv)); 347 348 SmallVector<ArgInfo, 8> SplitArgs; 349 for (const auto &OrigArg : Info.OrigArgs) { 350 351 // TODO: handle not simple cases. 352 if (OrigArg.Flags[0].isByVal()) 353 return false; 354 355 if (OrigArg.Regs.size() > 1) 356 return false; 357 358 splitToValueTypes(OrigArg, SplitArgs, DL, Info.CallConv); 359 } 360 // Do the actual argument marshalling. 361 X86OutgoingValueAssigner Assigner(CC_X86); 362 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB); 363 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 364 Info.CallConv, Info.IsVarArg)) 365 return false; 366 367 bool IsFixed = Info.OrigArgs.empty() ? true : Info.OrigArgs.back().IsFixed; 368 if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) { 369 // From AMD64 ABI document: 370 // For calls that may call functions that use varargs or stdargs 371 // (prototype-less calls or calls to functions containing ellipsis (...) in 372 // the declaration) %al is used as hidden argument to specify the number 373 // of SSE registers used. The contents of %al do not need to match exactly 374 // the number of registers, but must be an ubound on the number of SSE 375 // registers used and is in the range 0 - 8 inclusive. 376 377 MIRBuilder.buildInstr(X86::MOV8ri) 378 .addDef(X86::AL) 379 .addImm(Assigner.getNumXmmRegs()); 380 MIB.addUse(X86::AL, RegState::Implicit); 381 } 382 383 // Now we can add the actual call instruction to the correct basic block. 384 MIRBuilder.insertInstr(MIB); 385 386 // If Callee is a reg, since it is used by a target specific 387 // instruction, it must have a register class matching the 388 // constraint of that instruction. 389 if (Info.Callee.isReg()) 390 MIB->getOperand(0).setReg(constrainOperandRegClass( 391 MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(), 392 *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee, 393 0)); 394 395 // Finally we can copy the returned value back into its virtual-register. In 396 // symmetry with the arguments, the physical register must be an 397 // implicit-define of the call instruction. 398 399 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) { 400 if (Info.OrigRet.Regs.size() > 1) 401 return false; 402 403 SplitArgs.clear(); 404 SmallVector<Register, 8> NewRegs; 405 406 splitToValueTypes(Info.OrigRet, SplitArgs, DL, Info.CallConv); 407 408 X86OutgoingValueAssigner Assigner(RetCC_X86); 409 CallReturnHandler Handler(MIRBuilder, MRI, MIB); 410 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 411 Info.CallConv, Info.IsVarArg)) 412 return false; 413 414 if (!NewRegs.empty()) 415 MIRBuilder.buildMergeLikeInstr(Info.OrigRet.Regs[0], NewRegs); 416 } 417 418 CallSeqStart.addImm(Assigner.getStackSize()) 419 .addImm(0 /* see getFrameTotalSize */) 420 .addImm(0 /* see getFrameAdjustment */); 421 422 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); 423 MIRBuilder.buildInstr(AdjStackUp) 424 .addImm(Assigner.getStackSize()) 425 .addImm(0 /* NumBytesForCalleeToPop */); 426 427 if (!Info.CanLowerReturn) 428 insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs, 429 Info.DemoteRegister, Info.DemoteStackIndex); 430 431 return true; 432 } 433