1 //===-- SystemZFrameLowering.cpp - Frame lowering for SystemZ -------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "SystemZFrameLowering.h" 10 #include "SystemZCallingConv.h" 11 #include "SystemZInstrBuilder.h" 12 #include "SystemZInstrInfo.h" 13 #include "SystemZMachineFunctionInfo.h" 14 #include "SystemZRegisterInfo.h" 15 #include "SystemZSubtarget.h" 16 #include "llvm/CodeGen/MachineModuleInfo.h" 17 #include "llvm/CodeGen/MachineRegisterInfo.h" 18 #include "llvm/CodeGen/RegisterScavenging.h" 19 #include "llvm/IR/Function.h" 20 #include "llvm/Target/TargetMachine.h" 21 22 using namespace llvm; 23 24 namespace { 25 // The ABI-defined register save slots, relative to the CFA (i.e. 26 // incoming stack pointer + SystemZMC::CallFrameSize). 27 static const TargetFrameLowering::SpillSlot SpillOffsetTable[] = { 28 { SystemZ::R2D, 0x10 }, 29 { SystemZ::R3D, 0x18 }, 30 { SystemZ::R4D, 0x20 }, 31 { SystemZ::R5D, 0x28 }, 32 { SystemZ::R6D, 0x30 }, 33 { SystemZ::R7D, 0x38 }, 34 { SystemZ::R8D, 0x40 }, 35 { SystemZ::R9D, 0x48 }, 36 { SystemZ::R10D, 0x50 }, 37 { SystemZ::R11D, 0x58 }, 38 { SystemZ::R12D, 0x60 }, 39 { SystemZ::R13D, 0x68 }, 40 { SystemZ::R14D, 0x70 }, 41 { SystemZ::R15D, 0x78 }, 42 { SystemZ::F0D, 0x80 }, 43 { SystemZ::F2D, 0x88 }, 44 { SystemZ::F4D, 0x90 }, 45 { SystemZ::F6D, 0x98 } 46 }; 47 } // end anonymous namespace 48 49 SystemZFrameLowering::SystemZFrameLowering() 50 : TargetFrameLowering(TargetFrameLowering::StackGrowsDown, Align(8), 51 0, Align(8), false /* StackRealignable */), 52 RegSpillOffsets(0) { 53 // Due to the SystemZ ABI, the DWARF CFA (Canonical Frame Address) is not 54 // equal to the incoming stack pointer, but to incoming stack pointer plus 55 // 160. Instead of using a Local Area Offset, the Register save area will 56 // be occupied by fixed frame objects, and all offsets are actually 57 // relative to CFA. 58 59 // Create a mapping from register number to save slot offset. 60 // These offsets are relative to the start of the register save area. 61 RegSpillOffsets.grow(SystemZ::NUM_TARGET_REGS); 62 for (unsigned I = 0, E = array_lengthof(SpillOffsetTable); I != E; ++I) 63 RegSpillOffsets[SpillOffsetTable[I].Reg] = SpillOffsetTable[I].Offset; 64 } 65 66 bool SystemZFrameLowering:: 67 assignCalleeSavedSpillSlots(MachineFunction &MF, 68 const TargetRegisterInfo *TRI, 69 std::vector<CalleeSavedInfo> &CSI) const { 70 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>(); 71 MachineFrameInfo &MFFrame = MF.getFrameInfo(); 72 bool IsVarArg = MF.getFunction().isVarArg(); 73 if (CSI.empty()) 74 return true; // Early exit if no callee saved registers are modified! 75 76 unsigned LowGPR = 0; 77 unsigned HighGPR = SystemZ::R15D; 78 int StartSPOffset = SystemZMC::CallFrameSize; 79 for (auto &CS : CSI) { 80 unsigned Reg = CS.getReg(); 81 int Offset = getRegSpillOffset(MF, Reg); 82 if (Offset) { 83 if (SystemZ::GR64BitRegClass.contains(Reg) && StartSPOffset > Offset) { 84 LowGPR = Reg; 85 StartSPOffset = Offset; 86 } 87 Offset -= SystemZMC::CallFrameSize; 88 int FrameIdx = MFFrame.CreateFixedSpillStackObject(8, Offset); 89 CS.setFrameIdx(FrameIdx); 90 } else 91 CS.setFrameIdx(INT32_MAX); 92 } 93 94 // Save the range of call-saved registers, for use by the 95 // prologue/epilogue inserters. 96 ZFI->setRestoreGPRRegs(LowGPR, HighGPR, StartSPOffset); 97 if (IsVarArg) { 98 // Also save the GPR varargs, if any. R6D is call-saved, so would 99 // already be included, but we also need to handle the call-clobbered 100 // argument registers. 101 unsigned FirstGPR = ZFI->getVarArgsFirstGPR(); 102 if (FirstGPR < SystemZ::NumArgGPRs) { 103 unsigned Reg = SystemZ::ArgGPRs[FirstGPR]; 104 int Offset = getRegSpillOffset(MF, Reg); 105 if (StartSPOffset > Offset) { 106 LowGPR = Reg; StartSPOffset = Offset; 107 } 108 } 109 } 110 ZFI->setSpillGPRRegs(LowGPR, HighGPR, StartSPOffset); 111 112 // Create fixed stack objects for the remaining registers. 113 int CurrOffset = -SystemZMC::CallFrameSize; 114 if (usePackedStack(MF)) 115 CurrOffset += StartSPOffset; 116 117 for (auto &CS : CSI) { 118 if (CS.getFrameIdx() != INT32_MAX) 119 continue; 120 unsigned Reg = CS.getReg(); 121 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); 122 unsigned Size = TRI->getSpillSize(*RC); 123 CurrOffset -= Size; 124 assert(CurrOffset % 8 == 0 && 125 "8-byte alignment required for for all register save slots"); 126 int FrameIdx = MFFrame.CreateFixedSpillStackObject(Size, CurrOffset); 127 CS.setFrameIdx(FrameIdx); 128 } 129 130 return true; 131 } 132 133 void SystemZFrameLowering::determineCalleeSaves(MachineFunction &MF, 134 BitVector &SavedRegs, 135 RegScavenger *RS) const { 136 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS); 137 138 MachineFrameInfo &MFFrame = MF.getFrameInfo(); 139 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 140 bool HasFP = hasFP(MF); 141 SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>(); 142 bool IsVarArg = MF.getFunction().isVarArg(); 143 144 // va_start stores incoming FPR varargs in the normal way, but delegates 145 // the saving of incoming GPR varargs to spillCalleeSavedRegisters(). 146 // Record these pending uses, which typically include the call-saved 147 // argument register R6D. 148 if (IsVarArg) 149 for (unsigned I = MFI->getVarArgsFirstGPR(); I < SystemZ::NumArgGPRs; ++I) 150 SavedRegs.set(SystemZ::ArgGPRs[I]); 151 152 // If there are any landing pads, entering them will modify r6/r7. 153 if (!MF.getLandingPads().empty()) { 154 SavedRegs.set(SystemZ::R6D); 155 SavedRegs.set(SystemZ::R7D); 156 } 157 158 // If the function requires a frame pointer, record that the hard 159 // frame pointer will be clobbered. 160 if (HasFP) 161 SavedRegs.set(SystemZ::R11D); 162 163 // If the function calls other functions, record that the return 164 // address register will be clobbered. 165 if (MFFrame.hasCalls()) 166 SavedRegs.set(SystemZ::R14D); 167 168 // If we are saving GPRs other than the stack pointer, we might as well 169 // save and restore the stack pointer at the same time, via STMG and LMG. 170 // This allows the deallocation to be done by the LMG, rather than needing 171 // a separate %r15 addition. 172 const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF); 173 for (unsigned I = 0; CSRegs[I]; ++I) { 174 unsigned Reg = CSRegs[I]; 175 if (SystemZ::GR64BitRegClass.contains(Reg) && SavedRegs.test(Reg)) { 176 SavedRegs.set(SystemZ::R15D); 177 break; 178 } 179 } 180 } 181 182 // Add GPR64 to the save instruction being built by MIB, which is in basic 183 // block MBB. IsImplicit says whether this is an explicit operand to the 184 // instruction, or an implicit one that comes between the explicit start 185 // and end registers. 186 static void addSavedGPR(MachineBasicBlock &MBB, MachineInstrBuilder &MIB, 187 unsigned GPR64, bool IsImplicit) { 188 const TargetRegisterInfo *RI = 189 MBB.getParent()->getSubtarget().getRegisterInfo(); 190 Register GPR32 = RI->getSubReg(GPR64, SystemZ::subreg_l32); 191 bool IsLive = MBB.isLiveIn(GPR64) || MBB.isLiveIn(GPR32); 192 if (!IsLive || !IsImplicit) { 193 MIB.addReg(GPR64, getImplRegState(IsImplicit) | getKillRegState(!IsLive)); 194 if (!IsLive) 195 MBB.addLiveIn(GPR64); 196 } 197 } 198 199 bool SystemZFrameLowering::spillCalleeSavedRegisters( 200 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 201 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const { 202 if (CSI.empty()) 203 return false; 204 205 MachineFunction &MF = *MBB.getParent(); 206 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); 207 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>(); 208 bool IsVarArg = MF.getFunction().isVarArg(); 209 DebugLoc DL; 210 211 // Save GPRs 212 SystemZ::GPRRegs SpillGPRs = ZFI->getSpillGPRRegs(); 213 if (SpillGPRs.LowGPR) { 214 assert(SpillGPRs.LowGPR != SpillGPRs.HighGPR && 215 "Should be saving %r15 and something else"); 216 217 // Build an STMG instruction. 218 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(SystemZ::STMG)); 219 220 // Add the explicit register operands. 221 addSavedGPR(MBB, MIB, SpillGPRs.LowGPR, false); 222 addSavedGPR(MBB, MIB, SpillGPRs.HighGPR, false); 223 224 // Add the address. 225 MIB.addReg(SystemZ::R15D).addImm(SpillGPRs.GPROffset); 226 227 // Make sure all call-saved GPRs are included as operands and are 228 // marked as live on entry. 229 for (unsigned I = 0, E = CSI.size(); I != E; ++I) { 230 unsigned Reg = CSI[I].getReg(); 231 if (SystemZ::GR64BitRegClass.contains(Reg)) 232 addSavedGPR(MBB, MIB, Reg, true); 233 } 234 235 // ...likewise GPR varargs. 236 if (IsVarArg) 237 for (unsigned I = ZFI->getVarArgsFirstGPR(); I < SystemZ::NumArgGPRs; ++I) 238 addSavedGPR(MBB, MIB, SystemZ::ArgGPRs[I], true); 239 } 240 241 // Save FPRs/VRs in the normal TargetInstrInfo way. 242 for (unsigned I = 0, E = CSI.size(); I != E; ++I) { 243 unsigned Reg = CSI[I].getReg(); 244 if (SystemZ::FP64BitRegClass.contains(Reg)) { 245 MBB.addLiveIn(Reg); 246 TII->storeRegToStackSlot(MBB, MBBI, Reg, true, CSI[I].getFrameIdx(), 247 &SystemZ::FP64BitRegClass, TRI); 248 } 249 if (SystemZ::VR128BitRegClass.contains(Reg)) { 250 MBB.addLiveIn(Reg); 251 TII->storeRegToStackSlot(MBB, MBBI, Reg, true, CSI[I].getFrameIdx(), 252 &SystemZ::VR128BitRegClass, TRI); 253 } 254 } 255 256 return true; 257 } 258 259 bool SystemZFrameLowering::restoreCalleeSavedRegisters( 260 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 261 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const { 262 if (CSI.empty()) 263 return false; 264 265 MachineFunction &MF = *MBB.getParent(); 266 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); 267 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>(); 268 bool HasFP = hasFP(MF); 269 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 270 271 // Restore FPRs/VRs in the normal TargetInstrInfo way. 272 for (unsigned I = 0, E = CSI.size(); I != E; ++I) { 273 unsigned Reg = CSI[I].getReg(); 274 if (SystemZ::FP64BitRegClass.contains(Reg)) 275 TII->loadRegFromStackSlot(MBB, MBBI, Reg, CSI[I].getFrameIdx(), 276 &SystemZ::FP64BitRegClass, TRI); 277 if (SystemZ::VR128BitRegClass.contains(Reg)) 278 TII->loadRegFromStackSlot(MBB, MBBI, Reg, CSI[I].getFrameIdx(), 279 &SystemZ::VR128BitRegClass, TRI); 280 } 281 282 // Restore call-saved GPRs (but not call-clobbered varargs, which at 283 // this point might hold return values). 284 SystemZ::GPRRegs RestoreGPRs = ZFI->getRestoreGPRRegs(); 285 if (RestoreGPRs.LowGPR) { 286 // If we saved any of %r2-%r5 as varargs, we should also be saving 287 // and restoring %r6. If we're saving %r6 or above, we should be 288 // restoring it too. 289 assert(RestoreGPRs.LowGPR != RestoreGPRs.HighGPR && 290 "Should be loading %r15 and something else"); 291 292 // Build an LMG instruction. 293 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(SystemZ::LMG)); 294 295 // Add the explicit register operands. 296 MIB.addReg(RestoreGPRs.LowGPR, RegState::Define); 297 MIB.addReg(RestoreGPRs.HighGPR, RegState::Define); 298 299 // Add the address. 300 MIB.addReg(HasFP ? SystemZ::R11D : SystemZ::R15D); 301 MIB.addImm(RestoreGPRs.GPROffset); 302 303 // Do a second scan adding regs as being defined by instruction 304 for (unsigned I = 0, E = CSI.size(); I != E; ++I) { 305 unsigned Reg = CSI[I].getReg(); 306 if (Reg != RestoreGPRs.LowGPR && Reg != RestoreGPRs.HighGPR && 307 SystemZ::GR64BitRegClass.contains(Reg)) 308 MIB.addReg(Reg, RegState::ImplicitDefine); 309 } 310 } 311 312 return true; 313 } 314 315 void SystemZFrameLowering:: 316 processFunctionBeforeFrameFinalized(MachineFunction &MF, 317 RegScavenger *RS) const { 318 MachineFrameInfo &MFFrame = MF.getFrameInfo(); 319 bool BackChain = MF.getFunction().hasFnAttribute("backchain"); 320 321 if (!usePackedStack(MF) || BackChain) 322 // Create the incoming register save area. 323 getOrCreateFramePointerSaveIndex(MF); 324 325 // Get the size of our stack frame to be allocated ... 326 uint64_t StackSize = (MFFrame.estimateStackSize(MF) + 327 SystemZMC::CallFrameSize); 328 // ... and the maximum offset we may need to reach into the 329 // caller's frame to access the save area or stack arguments. 330 int64_t MaxArgOffset = 0; 331 for (int I = MFFrame.getObjectIndexBegin(); I != 0; ++I) 332 if (MFFrame.getObjectOffset(I) >= 0) { 333 int64_t ArgOffset = MFFrame.getObjectOffset(I) + 334 MFFrame.getObjectSize(I); 335 MaxArgOffset = std::max(MaxArgOffset, ArgOffset); 336 } 337 338 uint64_t MaxReach = StackSize + MaxArgOffset; 339 if (!isUInt<12>(MaxReach)) { 340 // We may need register scavenging slots if some parts of the frame 341 // are outside the reach of an unsigned 12-bit displacement. 342 // Create 2 for the case where both addresses in an MVC are 343 // out of range. 344 RS->addScavengingFrameIndex(MFFrame.CreateStackObject(8, Align(8), false)); 345 RS->addScavengingFrameIndex(MFFrame.CreateStackObject(8, Align(8), false)); 346 } 347 } 348 349 // Emit instructions before MBBI (in MBB) to add NumBytes to Reg. 350 static void emitIncrement(MachineBasicBlock &MBB, 351 MachineBasicBlock::iterator &MBBI, const DebugLoc &DL, 352 Register Reg, int64_t NumBytes, 353 const TargetInstrInfo *TII) { 354 while (NumBytes) { 355 unsigned Opcode; 356 int64_t ThisVal = NumBytes; 357 if (isInt<16>(NumBytes)) 358 Opcode = SystemZ::AGHI; 359 else { 360 Opcode = SystemZ::AGFI; 361 // Make sure we maintain 8-byte stack alignment. 362 int64_t MinVal = -uint64_t(1) << 31; 363 int64_t MaxVal = (int64_t(1) << 31) - 8; 364 if (ThisVal < MinVal) 365 ThisVal = MinVal; 366 else if (ThisVal > MaxVal) 367 ThisVal = MaxVal; 368 } 369 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII->get(Opcode), Reg) 370 .addReg(Reg).addImm(ThisVal); 371 // The CC implicit def is dead. 372 MI->getOperand(3).setIsDead(); 373 NumBytes -= ThisVal; 374 } 375 } 376 377 // Add CFI for the new CFA offset. 378 static void buildCFAOffs(MachineBasicBlock &MBB, 379 MachineBasicBlock::iterator MBBI, 380 const DebugLoc &DL, int Offset, 381 const SystemZInstrInfo *ZII) { 382 unsigned CFIIndex = MBB.getParent()->addFrameInst( 383 MCCFIInstruction::cfiDefCfaOffset(nullptr, -Offset)); 384 BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION)) 385 .addCFIIndex(CFIIndex); 386 } 387 388 // Add CFI for the new frame location. 389 static void buildDefCFAReg(MachineBasicBlock &MBB, 390 MachineBasicBlock::iterator MBBI, 391 const DebugLoc &DL, unsigned Reg, 392 const SystemZInstrInfo *ZII) { 393 MachineFunction &MF = *MBB.getParent(); 394 MachineModuleInfo &MMI = MF.getMMI(); 395 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo(); 396 unsigned RegNum = MRI->getDwarfRegNum(Reg, true); 397 unsigned CFIIndex = MF.addFrameInst( 398 MCCFIInstruction::createDefCfaRegister(nullptr, RegNum)); 399 BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION)) 400 .addCFIIndex(CFIIndex); 401 } 402 403 void SystemZFrameLowering::emitPrologue(MachineFunction &MF, 404 MachineBasicBlock &MBB) const { 405 assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); 406 const SystemZSubtarget &STI = MF.getSubtarget<SystemZSubtarget>(); 407 const SystemZTargetLowering &TLI = *STI.getTargetLowering(); 408 MachineFrameInfo &MFFrame = MF.getFrameInfo(); 409 auto *ZII = static_cast<const SystemZInstrInfo *>(STI.getInstrInfo()); 410 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>(); 411 MachineBasicBlock::iterator MBBI = MBB.begin(); 412 MachineModuleInfo &MMI = MF.getMMI(); 413 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo(); 414 const std::vector<CalleeSavedInfo> &CSI = MFFrame.getCalleeSavedInfo(); 415 bool HasFP = hasFP(MF); 416 417 // In GHC calling convention C stack space, including the ABI-defined 418 // 160-byte base area, is (de)allocated by GHC itself. This stack space may 419 // be used by LLVM as spill slots for the tail recursive GHC functions. Thus 420 // do not allocate stack space here, too. 421 if (MF.getFunction().getCallingConv() == CallingConv::GHC) { 422 if (MFFrame.getStackSize() > 2048 * sizeof(long)) { 423 report_fatal_error( 424 "Pre allocated stack space for GHC function is too small"); 425 } 426 if (HasFP) { 427 report_fatal_error( 428 "In GHC calling convention a frame pointer is not supported"); 429 } 430 MFFrame.setStackSize(MFFrame.getStackSize() + SystemZMC::CallFrameSize); 431 return; 432 } 433 434 // Debug location must be unknown since the first debug location is used 435 // to determine the end of the prologue. 436 DebugLoc DL; 437 438 // The current offset of the stack pointer from the CFA. 439 int64_t SPOffsetFromCFA = -SystemZMC::CFAOffsetFromInitialSP; 440 441 if (ZFI->getSpillGPRRegs().LowGPR) { 442 // Skip over the GPR saves. 443 if (MBBI != MBB.end() && MBBI->getOpcode() == SystemZ::STMG) 444 ++MBBI; 445 else 446 llvm_unreachable("Couldn't skip over GPR saves"); 447 448 // Add CFI for the GPR saves. 449 for (auto &Save : CSI) { 450 unsigned Reg = Save.getReg(); 451 if (SystemZ::GR64BitRegClass.contains(Reg)) { 452 int FI = Save.getFrameIdx(); 453 int64_t Offset = MFFrame.getObjectOffset(FI); 454 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset( 455 nullptr, MRI->getDwarfRegNum(Reg, true), Offset)); 456 BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION)) 457 .addCFIIndex(CFIIndex); 458 } 459 } 460 } 461 462 uint64_t StackSize = MFFrame.getStackSize(); 463 // We need to allocate the ABI-defined 160-byte base area whenever 464 // we allocate stack space for our own use and whenever we call another 465 // function. 466 bool HasStackObject = false; 467 for (unsigned i = 0, e = MFFrame.getObjectIndexEnd(); i != e; ++i) 468 if (!MFFrame.isDeadObjectIndex(i)) { 469 HasStackObject = true; 470 break; 471 } 472 if (HasStackObject || MFFrame.hasCalls()) 473 StackSize += SystemZMC::CallFrameSize; 474 // Don't allocate the incoming reg save area. 475 StackSize = StackSize > SystemZMC::CallFrameSize 476 ? StackSize - SystemZMC::CallFrameSize 477 : 0; 478 MFFrame.setStackSize(StackSize); 479 480 if (StackSize) { 481 // Determine if we want to store a backchain. 482 bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain"); 483 484 // If we need backchain, save current stack pointer. R1 is free at this 485 // point. 486 if (StoreBackchain) 487 BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::LGR)) 488 .addReg(SystemZ::R1D, RegState::Define).addReg(SystemZ::R15D); 489 490 // Allocate StackSize bytes. 491 int64_t Delta = -int64_t(StackSize); 492 const unsigned ProbeSize = TLI.getStackProbeSize(MF); 493 bool FreeProbe = (ZFI->getSpillGPRRegs().GPROffset && 494 (ZFI->getSpillGPRRegs().GPROffset + StackSize) < ProbeSize); 495 if (!FreeProbe && 496 MF.getSubtarget().getTargetLowering()->hasInlineStackProbe(MF)) { 497 // Stack probing may involve looping, but splitting the prologue block 498 // is not possible at this point since it would invalidate the 499 // SaveBlocks / RestoreBlocks sets of PEI in the single block function 500 // case. Build a pseudo to be handled later by inlineStackProbe(). 501 BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::PROBED_STACKALLOC)) 502 .addImm(StackSize); 503 } 504 else { 505 emitIncrement(MBB, MBBI, DL, SystemZ::R15D, Delta, ZII); 506 buildCFAOffs(MBB, MBBI, DL, SPOffsetFromCFA + Delta, ZII); 507 } 508 SPOffsetFromCFA += Delta; 509 510 if (StoreBackchain) { 511 // The back chain is stored topmost with packed-stack. 512 int Offset = usePackedStack(MF) ? SystemZMC::CallFrameSize - 8 : 0; 513 BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::STG)) 514 .addReg(SystemZ::R1D, RegState::Kill).addReg(SystemZ::R15D) 515 .addImm(Offset).addReg(0); 516 } 517 } 518 519 if (HasFP) { 520 // Copy the base of the frame to R11. 521 BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::LGR), SystemZ::R11D) 522 .addReg(SystemZ::R15D); 523 524 // Add CFI for the new frame location. 525 buildDefCFAReg(MBB, MBBI, DL, SystemZ::R11D, ZII); 526 527 // Mark the FramePtr as live at the beginning of every block except 528 // the entry block. (We'll have marked R11 as live on entry when 529 // saving the GPRs.) 530 for (auto I = std::next(MF.begin()), E = MF.end(); I != E; ++I) 531 I->addLiveIn(SystemZ::R11D); 532 } 533 534 // Skip over the FPR/VR saves. 535 SmallVector<unsigned, 8> CFIIndexes; 536 for (auto &Save : CSI) { 537 unsigned Reg = Save.getReg(); 538 if (SystemZ::FP64BitRegClass.contains(Reg)) { 539 if (MBBI != MBB.end() && 540 (MBBI->getOpcode() == SystemZ::STD || 541 MBBI->getOpcode() == SystemZ::STDY)) 542 ++MBBI; 543 else 544 llvm_unreachable("Couldn't skip over FPR save"); 545 } else if (SystemZ::VR128BitRegClass.contains(Reg)) { 546 if (MBBI != MBB.end() && 547 MBBI->getOpcode() == SystemZ::VST) 548 ++MBBI; 549 else 550 llvm_unreachable("Couldn't skip over VR save"); 551 } else 552 continue; 553 554 // Add CFI for the this save. 555 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); 556 Register IgnoredFrameReg; 557 int64_t Offset = 558 getFrameIndexReference(MF, Save.getFrameIdx(), IgnoredFrameReg); 559 560 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset( 561 nullptr, DwarfReg, SPOffsetFromCFA + Offset)); 562 CFIIndexes.push_back(CFIIndex); 563 } 564 // Complete the CFI for the FPR/VR saves, modelling them as taking effect 565 // after the last save. 566 for (auto CFIIndex : CFIIndexes) { 567 BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION)) 568 .addCFIIndex(CFIIndex); 569 } 570 } 571 572 void SystemZFrameLowering::emitEpilogue(MachineFunction &MF, 573 MachineBasicBlock &MBB) const { 574 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr(); 575 auto *ZII = 576 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo()); 577 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>(); 578 MachineFrameInfo &MFFrame = MF.getFrameInfo(); 579 580 // See SystemZFrameLowering::emitPrologue 581 if (MF.getFunction().getCallingConv() == CallingConv::GHC) 582 return; 583 584 // Skip the return instruction. 585 assert(MBBI->isReturn() && "Can only insert epilogue into returning blocks"); 586 587 uint64_t StackSize = MFFrame.getStackSize(); 588 if (ZFI->getRestoreGPRRegs().LowGPR) { 589 --MBBI; 590 unsigned Opcode = MBBI->getOpcode(); 591 if (Opcode != SystemZ::LMG) 592 llvm_unreachable("Expected to see callee-save register restore code"); 593 594 unsigned AddrOpNo = 2; 595 DebugLoc DL = MBBI->getDebugLoc(); 596 uint64_t Offset = StackSize + MBBI->getOperand(AddrOpNo + 1).getImm(); 597 unsigned NewOpcode = ZII->getOpcodeForOffset(Opcode, Offset); 598 599 // If the offset is too large, use the largest stack-aligned offset 600 // and add the rest to the base register (the stack or frame pointer). 601 if (!NewOpcode) { 602 uint64_t NumBytes = Offset - 0x7fff8; 603 emitIncrement(MBB, MBBI, DL, MBBI->getOperand(AddrOpNo).getReg(), 604 NumBytes, ZII); 605 Offset -= NumBytes; 606 NewOpcode = ZII->getOpcodeForOffset(Opcode, Offset); 607 assert(NewOpcode && "No restore instruction available"); 608 } 609 610 MBBI->setDesc(ZII->get(NewOpcode)); 611 MBBI->getOperand(AddrOpNo + 1).ChangeToImmediate(Offset); 612 } else if (StackSize) { 613 DebugLoc DL = MBBI->getDebugLoc(); 614 emitIncrement(MBB, MBBI, DL, SystemZ::R15D, StackSize, ZII); 615 } 616 } 617 618 void SystemZFrameLowering::inlineStackProbe(MachineFunction &MF, 619 MachineBasicBlock &PrologMBB) const { 620 auto *ZII = 621 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo()); 622 const SystemZSubtarget &STI = MF.getSubtarget<SystemZSubtarget>(); 623 const SystemZTargetLowering &TLI = *STI.getTargetLowering(); 624 625 MachineInstr *StackAllocMI = nullptr; 626 for (MachineInstr &MI : PrologMBB) 627 if (MI.getOpcode() == SystemZ::PROBED_STACKALLOC) { 628 StackAllocMI = &MI; 629 break; 630 } 631 if (StackAllocMI == nullptr) 632 return; 633 uint64_t StackSize = StackAllocMI->getOperand(0).getImm(); 634 const unsigned ProbeSize = TLI.getStackProbeSize(MF); 635 uint64_t NumFullBlocks = StackSize / ProbeSize; 636 uint64_t Residual = StackSize % ProbeSize; 637 int64_t SPOffsetFromCFA = -SystemZMC::CFAOffsetFromInitialSP; 638 MachineBasicBlock *MBB = &PrologMBB; 639 MachineBasicBlock::iterator MBBI = StackAllocMI; 640 const DebugLoc DL = StackAllocMI->getDebugLoc(); 641 642 // Allocate a block of Size bytes on the stack and probe it. 643 auto allocateAndProbe = [&](MachineBasicBlock &InsMBB, 644 MachineBasicBlock::iterator InsPt, unsigned Size, 645 bool EmitCFI) -> void { 646 emitIncrement(InsMBB, InsPt, DL, SystemZ::R15D, -int64_t(Size), ZII); 647 if (EmitCFI) { 648 SPOffsetFromCFA -= Size; 649 buildCFAOffs(InsMBB, InsPt, DL, SPOffsetFromCFA, ZII); 650 } 651 // Probe by means of a volatile compare. 652 MachineMemOperand *MMO = MF.getMachineMemOperand(MachinePointerInfo(), 653 MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, 8, Align(1)); 654 BuildMI(InsMBB, InsPt, DL, ZII->get(SystemZ::CG)) 655 .addReg(SystemZ::R0D, RegState::Undef) 656 .addReg(SystemZ::R15D).addImm(Size - 8).addReg(0) 657 .addMemOperand(MMO); 658 }; 659 660 if (NumFullBlocks < 3) { 661 // Emit unrolled probe statements. 662 for (unsigned int i = 0; i < NumFullBlocks; i++) 663 allocateAndProbe(*MBB, MBBI, ProbeSize, true/*EmitCFI*/); 664 } else { 665 // Emit a loop probing the pages. 666 uint64_t LoopAlloc = ProbeSize * NumFullBlocks; 667 SPOffsetFromCFA -= LoopAlloc; 668 669 BuildMI(*MBB, MBBI, DL, ZII->get(SystemZ::LGR), SystemZ::R1D) 670 .addReg(SystemZ::R15D); 671 buildDefCFAReg(*MBB, MBBI, DL, SystemZ::R1D, ZII); 672 emitIncrement(*MBB, MBBI, DL, SystemZ::R1D, -int64_t(LoopAlloc), ZII); 673 buildCFAOffs(*MBB, MBBI, DL, -int64_t(SystemZMC::CallFrameSize + LoopAlloc), 674 ZII); 675 676 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MBBI, MBB); 677 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB); 678 MBB->addSuccessor(LoopMBB); 679 LoopMBB->addSuccessor(LoopMBB); 680 LoopMBB->addSuccessor(DoneMBB); 681 682 MBB = LoopMBB; 683 allocateAndProbe(*MBB, MBB->end(), ProbeSize, false/*EmitCFI*/); 684 BuildMI(*MBB, MBB->end(), DL, ZII->get(SystemZ::CLGR)) 685 .addReg(SystemZ::R15D).addReg(SystemZ::R1D); 686 BuildMI(*MBB, MBB->end(), DL, ZII->get(SystemZ::BRC)) 687 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_GT).addMBB(MBB); 688 689 MBB = DoneMBB; 690 MBBI = DoneMBB->begin(); 691 buildDefCFAReg(*MBB, MBBI, DL, SystemZ::R15D, ZII); 692 693 recomputeLiveIns(*DoneMBB); 694 recomputeLiveIns(*LoopMBB); 695 } 696 697 if (Residual) 698 allocateAndProbe(*MBB, MBBI, Residual, true/*EmitCFI*/); 699 700 StackAllocMI->eraseFromParent(); 701 } 702 703 bool SystemZFrameLowering::hasFP(const MachineFunction &MF) const { 704 return (MF.getTarget().Options.DisableFramePointerElim(MF) || 705 MF.getFrameInfo().hasVarSizedObjects() || 706 MF.getInfo<SystemZMachineFunctionInfo>()->getManipulatesSP()); 707 } 708 709 bool 710 SystemZFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const { 711 // The ABI requires us to allocate 160 bytes of stack space for the callee, 712 // with any outgoing stack arguments being placed above that. It seems 713 // better to make that area a permanent feature of the frame even if 714 // we're using a frame pointer. 715 return true; 716 } 717 718 int SystemZFrameLowering::getFrameIndexReference(const MachineFunction &MF, 719 int FI, 720 Register &FrameReg) const { 721 // Our incoming SP is actually SystemZMC::CallFrameSize below the CFA, so 722 // add that difference here. 723 int64_t Offset = 724 TargetFrameLowering::getFrameIndexReference(MF, FI, FrameReg); 725 return Offset + SystemZMC::CallFrameSize; 726 } 727 728 MachineBasicBlock::iterator SystemZFrameLowering:: 729 eliminateCallFramePseudoInstr(MachineFunction &MF, 730 MachineBasicBlock &MBB, 731 MachineBasicBlock::iterator MI) const { 732 switch (MI->getOpcode()) { 733 case SystemZ::ADJCALLSTACKDOWN: 734 case SystemZ::ADJCALLSTACKUP: 735 assert(hasReservedCallFrame(MF) && 736 "ADJSTACKDOWN and ADJSTACKUP should be no-ops"); 737 return MBB.erase(MI); 738 break; 739 740 default: 741 llvm_unreachable("Unexpected call frame instruction"); 742 } 743 } 744 745 unsigned SystemZFrameLowering::getRegSpillOffset(MachineFunction &MF, 746 Register Reg) const { 747 bool IsVarArg = MF.getFunction().isVarArg(); 748 bool BackChain = MF.getFunction().hasFnAttribute("backchain"); 749 bool SoftFloat = MF.getSubtarget<SystemZSubtarget>().hasSoftFloat(); 750 unsigned Offset = RegSpillOffsets[Reg]; 751 if (usePackedStack(MF) && !(IsVarArg && !SoftFloat)) { 752 if (SystemZ::GR64BitRegClass.contains(Reg)) 753 // Put all GPRs at the top of the Register save area with packed 754 // stack. Make room for the backchain if needed. 755 Offset += BackChain ? 24 : 32; 756 else 757 Offset = 0; 758 } 759 return Offset; 760 } 761 762 int SystemZFrameLowering:: 763 getOrCreateFramePointerSaveIndex(MachineFunction &MF) const { 764 SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>(); 765 int FI = ZFI->getFramePointerSaveIndex(); 766 if (!FI) { 767 MachineFrameInfo &MFFrame = MF.getFrameInfo(); 768 // The back chain is stored topmost with packed-stack. 769 int Offset = usePackedStack(MF) ? -8 : -SystemZMC::CallFrameSize; 770 FI = MFFrame.CreateFixedObject(8, Offset, false); 771 ZFI->setFramePointerSaveIndex(FI); 772 } 773 return FI; 774 } 775 776 bool SystemZFrameLowering::usePackedStack(MachineFunction &MF) const { 777 bool HasPackedStackAttr = MF.getFunction().hasFnAttribute("packed-stack"); 778 bool BackChain = MF.getFunction().hasFnAttribute("backchain"); 779 bool SoftFloat = MF.getSubtarget<SystemZSubtarget>().hasSoftFloat(); 780 if (HasPackedStackAttr && BackChain && !SoftFloat) 781 report_fatal_error("packed-stack + backchain + hard-float is unsupported."); 782 bool CallConv = MF.getFunction().getCallingConv() != CallingConv::GHC; 783 return HasPackedStackAttr && CallConv; 784 } 785