1 //===- llvm/lib/Target/X86/X86CallLowering.cpp - Call lowering ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file implements the lowering of LLVM calls to machine code calls for 11 /// GlobalISel. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86CallLowering.h" 16 #include "X86CallingConv.h" 17 #include "X86ISelLowering.h" 18 #include "X86InstrInfo.h" 19 #include "X86RegisterInfo.h" 20 #include "X86Subtarget.h" 21 #include "llvm/ADT/ArrayRef.h" 22 #include "llvm/ADT/SmallVector.h" 23 #include "llvm/CodeGen/Analysis.h" 24 #include "llvm/CodeGen/CallingConvLower.h" 25 #include "llvm/CodeGen/FunctionLoweringInfo.h" 26 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 27 #include "llvm/CodeGen/GlobalISel/Utils.h" 28 #include "llvm/CodeGen/LowLevelType.h" 29 #include "llvm/CodeGen/LowLevelTypeUtils.h" 30 #include "llvm/CodeGen/MachineBasicBlock.h" 31 #include "llvm/CodeGen/MachineFrameInfo.h" 32 #include "llvm/CodeGen/MachineFunction.h" 33 #include "llvm/CodeGen/MachineInstrBuilder.h" 34 #include "llvm/CodeGen/MachineMemOperand.h" 35 #include "llvm/CodeGen/MachineOperand.h" 36 #include "llvm/CodeGen/MachineRegisterInfo.h" 37 #include "llvm/CodeGen/MachineValueType.h" 38 #include "llvm/CodeGen/TargetInstrInfo.h" 39 #include "llvm/CodeGen/TargetSubtargetInfo.h" 40 #include "llvm/CodeGen/ValueTypes.h" 41 #include "llvm/IR/Attributes.h" 42 #include "llvm/IR/DataLayout.h" 43 #include "llvm/IR/Function.h" 44 #include "llvm/IR/Value.h" 45 #include "llvm/MC/MCRegisterInfo.h" 46 #include <cassert> 47 #include <cstdint> 48 49 using namespace llvm; 50 51 X86CallLowering::X86CallLowering(const X86TargetLowering &TLI) 52 : CallLowering(&TLI) {} 53 54 namespace { 55 56 struct X86OutgoingValueAssigner : public CallLowering::OutgoingValueAssigner { 57 private: 58 uint64_t StackSize = 0; 59 unsigned NumXMMRegs = 0; 60 61 public: 62 uint64_t getStackSize() { return StackSize; } 63 unsigned getNumXmmRegs() { return NumXMMRegs; } 64 65 X86OutgoingValueAssigner(CCAssignFn *AssignFn_) 66 : CallLowering::OutgoingValueAssigner(AssignFn_) {} 67 68 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT, 69 CCValAssign::LocInfo LocInfo, 70 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags, 71 CCState &State) override { 72 bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); 73 StackSize = State.getStackSize(); 74 75 static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, 76 X86::XMM3, X86::XMM4, X86::XMM5, 77 X86::XMM6, X86::XMM7}; 78 if (!Info.IsFixed) 79 NumXMMRegs = State.getFirstUnallocated(XMMArgRegs); 80 81 return Res; 82 } 83 }; 84 85 struct X86OutgoingValueHandler : public CallLowering::OutgoingValueHandler { 86 X86OutgoingValueHandler(MachineIRBuilder &MIRBuilder, 87 MachineRegisterInfo &MRI, MachineInstrBuilder &MIB) 88 : OutgoingValueHandler(MIRBuilder, MRI), MIB(MIB), 89 DL(MIRBuilder.getMF().getDataLayout()), 90 STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {} 91 92 Register getStackAddress(uint64_t Size, int64_t Offset, 93 MachinePointerInfo &MPO, 94 ISD::ArgFlagsTy Flags) override { 95 LLT p0 = LLT::pointer(0, DL.getPointerSizeInBits(0)); 96 LLT SType = LLT::scalar(DL.getPointerSizeInBits(0)); 97 auto SPReg = 98 MIRBuilder.buildCopy(p0, STI.getRegisterInfo()->getStackRegister()); 99 100 auto OffsetReg = MIRBuilder.buildConstant(SType, Offset); 101 102 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg); 103 104 MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); 105 return AddrReg.getReg(0); 106 } 107 108 void assignValueToReg(Register ValVReg, Register PhysReg, 109 const CCValAssign &VA) override { 110 MIB.addUse(PhysReg, RegState::Implicit); 111 Register ExtReg = extendRegister(ValVReg, VA); 112 MIRBuilder.buildCopy(PhysReg, ExtReg); 113 } 114 115 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy, 116 const MachinePointerInfo &MPO, 117 const CCValAssign &VA) override { 118 MachineFunction &MF = MIRBuilder.getMF(); 119 Register ExtReg = extendRegister(ValVReg, VA); 120 121 auto *MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy, 122 inferAlignFromPtrInfo(MF, MPO)); 123 MIRBuilder.buildStore(ExtReg, Addr, *MMO); 124 } 125 126 protected: 127 MachineInstrBuilder &MIB; 128 const DataLayout &DL; 129 const X86Subtarget &STI; 130 }; 131 132 } // end anonymous namespace 133 134 bool X86CallLowering::canLowerReturn( 135 MachineFunction &MF, CallingConv::ID CallConv, 136 SmallVectorImpl<CallLowering::BaseArgInfo> &Outs, bool IsVarArg) const { 137 LLVMContext &Context = MF.getFunction().getContext(); 138 SmallVector<CCValAssign, 16> RVLocs; 139 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); 140 return checkReturn(CCInfo, Outs, RetCC_X86); 141 } 142 143 bool X86CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, 144 const Value *Val, ArrayRef<Register> VRegs, 145 FunctionLoweringInfo &FLI) const { 146 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && 147 "Return value without a vreg"); 148 MachineFunction &MF = MIRBuilder.getMF(); 149 auto MIB = MIRBuilder.buildInstrNoInsert(X86::RET).addImm(0); 150 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 151 bool Is64Bit = STI.is64Bit(); 152 153 if (!FLI.CanLowerReturn) { 154 insertSRetStores(MIRBuilder, Val->getType(), VRegs, FLI.DemoteRegister); 155 MIRBuilder.buildCopy(Is64Bit ? X86::RAX : X86::EAX, FLI.DemoteRegister); 156 } else if (!VRegs.empty()) { 157 const Function &F = MF.getFunction(); 158 MachineRegisterInfo &MRI = MF.getRegInfo(); 159 const DataLayout &DL = MF.getDataLayout(); 160 161 ArgInfo OrigRetInfo(VRegs, Val->getType(), 0); 162 setArgFlags(OrigRetInfo, AttributeList::ReturnIndex, DL, F); 163 164 SmallVector<ArgInfo, 4> SplitRetInfos; 165 splitToValueTypes(OrigRetInfo, SplitRetInfos, DL, F.getCallingConv()); 166 167 X86OutgoingValueAssigner Assigner(RetCC_X86); 168 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB); 169 if (!determineAndHandleAssignments(Handler, Assigner, SplitRetInfos, 170 MIRBuilder, F.getCallingConv(), 171 F.isVarArg())) 172 return false; 173 } 174 175 MIRBuilder.insertInstr(MIB); 176 return true; 177 } 178 179 namespace { 180 181 struct X86IncomingValueHandler : public CallLowering::IncomingValueHandler { 182 X86IncomingValueHandler(MachineIRBuilder &MIRBuilder, 183 MachineRegisterInfo &MRI) 184 : IncomingValueHandler(MIRBuilder, MRI), 185 DL(MIRBuilder.getMF().getDataLayout()) {} 186 187 Register getStackAddress(uint64_t Size, int64_t Offset, 188 MachinePointerInfo &MPO, 189 ISD::ArgFlagsTy Flags) override { 190 auto &MFI = MIRBuilder.getMF().getFrameInfo(); 191 192 // Byval is assumed to be writable memory, but other stack passed arguments 193 // are not. 194 const bool IsImmutable = !Flags.isByVal(); 195 196 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable); 197 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); 198 199 return MIRBuilder 200 .buildFrameIndex(LLT::pointer(0, DL.getPointerSizeInBits(0)), FI) 201 .getReg(0); 202 } 203 204 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy, 205 const MachinePointerInfo &MPO, 206 const CCValAssign &VA) override { 207 MachineFunction &MF = MIRBuilder.getMF(); 208 auto *MMO = MF.getMachineMemOperand( 209 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemTy, 210 inferAlignFromPtrInfo(MF, MPO)); 211 MIRBuilder.buildLoad(ValVReg, Addr, *MMO); 212 } 213 214 void assignValueToReg(Register ValVReg, Register PhysReg, 215 const CCValAssign &VA) override { 216 markPhysRegUsed(PhysReg); 217 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA); 218 } 219 220 /// How the physical register gets marked varies between formal 221 /// parameters (it's a basic-block live-in), and a call instruction 222 /// (it's an implicit-def of the BL). 223 virtual void markPhysRegUsed(unsigned PhysReg) = 0; 224 225 protected: 226 const DataLayout &DL; 227 }; 228 229 struct FormalArgHandler : public X86IncomingValueHandler { 230 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI) 231 : X86IncomingValueHandler(MIRBuilder, MRI) {} 232 233 void markPhysRegUsed(unsigned PhysReg) override { 234 MIRBuilder.getMRI()->addLiveIn(PhysReg); 235 MIRBuilder.getMBB().addLiveIn(PhysReg); 236 } 237 }; 238 239 struct CallReturnHandler : public X86IncomingValueHandler { 240 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 241 MachineInstrBuilder &MIB) 242 : X86IncomingValueHandler(MIRBuilder, MRI), MIB(MIB) {} 243 244 void markPhysRegUsed(unsigned PhysReg) override { 245 MIB.addDef(PhysReg, RegState::Implicit); 246 } 247 248 protected: 249 MachineInstrBuilder &MIB; 250 }; 251 252 } // end anonymous namespace 253 254 bool X86CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, 255 const Function &F, 256 ArrayRef<ArrayRef<Register>> VRegs, 257 FunctionLoweringInfo &FLI) const { 258 MachineFunction &MF = MIRBuilder.getMF(); 259 MachineRegisterInfo &MRI = MF.getRegInfo(); 260 auto DL = MF.getDataLayout(); 261 262 SmallVector<ArgInfo, 8> SplitArgs; 263 264 if (!FLI.CanLowerReturn) 265 insertSRetIncomingArgument(F, SplitArgs, FLI.DemoteRegister, MRI, DL); 266 267 // TODO: handle variadic function 268 if (F.isVarArg()) 269 return false; 270 271 unsigned Idx = 0; 272 for (const auto &Arg : F.args()) { 273 // TODO: handle not simple cases. 274 if (Arg.hasAttribute(Attribute::ByVal) || 275 Arg.hasAttribute(Attribute::InReg) || 276 Arg.hasAttribute(Attribute::StructRet) || 277 Arg.hasAttribute(Attribute::SwiftSelf) || 278 Arg.hasAttribute(Attribute::SwiftError) || 279 Arg.hasAttribute(Attribute::Nest) || VRegs[Idx].size() > 1) 280 return false; 281 282 ArgInfo OrigArg(VRegs[Idx], Arg.getType(), Idx); 283 setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F); 284 splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv()); 285 Idx++; 286 } 287 288 if (SplitArgs.empty()) 289 return true; 290 291 MachineBasicBlock &MBB = MIRBuilder.getMBB(); 292 if (!MBB.empty()) 293 MIRBuilder.setInstr(*MBB.begin()); 294 295 X86OutgoingValueAssigner Assigner(CC_X86); 296 FormalArgHandler Handler(MIRBuilder, MRI); 297 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 298 F.getCallingConv(), F.isVarArg())) 299 return false; 300 301 // Move back to the end of the basic block. 302 MIRBuilder.setMBB(MBB); 303 304 return true; 305 } 306 307 bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, 308 CallLoweringInfo &Info) const { 309 MachineFunction &MF = MIRBuilder.getMF(); 310 const Function &F = MF.getFunction(); 311 MachineRegisterInfo &MRI = MF.getRegInfo(); 312 const DataLayout &DL = F.getParent()->getDataLayout(); 313 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 314 const TargetInstrInfo &TII = *STI.getInstrInfo(); 315 const X86RegisterInfo *TRI = STI.getRegisterInfo(); 316 317 // Handle only Linux C, X86_64_SysV calling conventions for now. 318 if (!STI.isTargetLinux() || !(Info.CallConv == CallingConv::C || 319 Info.CallConv == CallingConv::X86_64_SysV)) 320 return false; 321 322 unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); 323 auto CallSeqStart = MIRBuilder.buildInstr(AdjStackDown); 324 325 // Create a temporarily-floating call instruction so we can add the implicit 326 // uses of arg registers. 327 bool Is64Bit = STI.is64Bit(); 328 unsigned CallOpc = Info.Callee.isReg() 329 ? (Is64Bit ? X86::CALL64r : X86::CALL32r) 330 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32); 331 332 auto MIB = MIRBuilder.buildInstrNoInsert(CallOpc) 333 .add(Info.Callee) 334 .addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv)); 335 336 SmallVector<ArgInfo, 8> SplitArgs; 337 for (const auto &OrigArg : Info.OrigArgs) { 338 339 // TODO: handle not simple cases. 340 if (OrigArg.Flags[0].isByVal()) 341 return false; 342 343 if (OrigArg.Regs.size() > 1) 344 return false; 345 346 splitToValueTypes(OrigArg, SplitArgs, DL, Info.CallConv); 347 } 348 // Do the actual argument marshalling. 349 X86OutgoingValueAssigner Assigner(CC_X86); 350 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB); 351 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 352 Info.CallConv, Info.IsVarArg)) 353 return false; 354 355 bool IsFixed = Info.OrigArgs.empty() ? true : Info.OrigArgs.back().IsFixed; 356 if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) { 357 // From AMD64 ABI document: 358 // For calls that may call functions that use varargs or stdargs 359 // (prototype-less calls or calls to functions containing ellipsis (...) in 360 // the declaration) %al is used as hidden argument to specify the number 361 // of SSE registers used. The contents of %al do not need to match exactly 362 // the number of registers, but must be an ubound on the number of SSE 363 // registers used and is in the range 0 - 8 inclusive. 364 365 MIRBuilder.buildInstr(X86::MOV8ri) 366 .addDef(X86::AL) 367 .addImm(Assigner.getNumXmmRegs()); 368 MIB.addUse(X86::AL, RegState::Implicit); 369 } 370 371 // Now we can add the actual call instruction to the correct basic block. 372 MIRBuilder.insertInstr(MIB); 373 374 // If Callee is a reg, since it is used by a target specific 375 // instruction, it must have a register class matching the 376 // constraint of that instruction. 377 if (Info.Callee.isReg()) 378 MIB->getOperand(0).setReg(constrainOperandRegClass( 379 MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(), 380 *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee, 381 0)); 382 383 // Finally we can copy the returned value back into its virtual-register. In 384 // symmetry with the arguments, the physical register must be an 385 // implicit-define of the call instruction. 386 387 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) { 388 if (Info.OrigRet.Regs.size() > 1) 389 return false; 390 391 SplitArgs.clear(); 392 SmallVector<Register, 8> NewRegs; 393 394 splitToValueTypes(Info.OrigRet, SplitArgs, DL, Info.CallConv); 395 396 X86OutgoingValueAssigner Assigner(RetCC_X86); 397 CallReturnHandler Handler(MIRBuilder, MRI, MIB); 398 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder, 399 Info.CallConv, Info.IsVarArg)) 400 return false; 401 402 if (!NewRegs.empty()) 403 MIRBuilder.buildMergeLikeInstr(Info.OrigRet.Regs[0], NewRegs); 404 } 405 406 CallSeqStart.addImm(Assigner.getStackSize()) 407 .addImm(0 /* see getFrameTotalSize */) 408 .addImm(0 /* see getFrameAdjustment */); 409 410 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); 411 MIRBuilder.buildInstr(AdjStackUp) 412 .addImm(Assigner.getStackSize()) 413 .addImm(0 /* NumBytesForCalleeToPop */); 414 415 if (!Info.CanLowerReturn) 416 insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs, 417 Info.DemoteRegister, Info.DemoteStackIndex); 418 419 return true; 420 } 421