1 //===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the X86 implementation of TargetFrameLowering class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "X86FrameLowering.h" 14 #include "X86InstrBuilder.h" 15 #include "X86InstrInfo.h" 16 #include "X86MachineFunctionInfo.h" 17 #include "X86Subtarget.h" 18 #include "X86TargetMachine.h" 19 #include "llvm/ADT/SmallSet.h" 20 #include "llvm/Analysis/EHPersonalities.h" 21 #include "llvm/CodeGen/MachineFrameInfo.h" 22 #include "llvm/CodeGen/MachineFunction.h" 23 #include "llvm/CodeGen/MachineInstrBuilder.h" 24 #include "llvm/CodeGen/MachineModuleInfo.h" 25 #include "llvm/CodeGen/MachineRegisterInfo.h" 26 #include "llvm/CodeGen/WinEHFuncInfo.h" 27 #include "llvm/IR/DataLayout.h" 28 #include "llvm/IR/Function.h" 29 #include "llvm/MC/MCAsmInfo.h" 30 #include "llvm/MC/MCSymbol.h" 31 #include "llvm/Support/Debug.h" 32 #include "llvm/Target/TargetOptions.h" 33 #include <cstdlib> 34 35 using namespace llvm; 36 37 X86FrameLowering::X86FrameLowering(const X86Subtarget &STI, 38 unsigned StackAlignOverride) 39 : TargetFrameLowering(StackGrowsDown, StackAlignOverride, 40 STI.is64Bit() ? -8 : -4), 41 STI(STI), TII(*STI.getInstrInfo()), TRI(STI.getRegisterInfo()) { 42 // Cache a bunch of frame-related predicates for this subtarget. 43 SlotSize = TRI->getSlotSize(); 44 Is64Bit = STI.is64Bit(); 45 IsLP64 = STI.isTarget64BitLP64(); 46 // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit. 47 Uses64BitFramePtr = STI.isTarget64BitLP64() || STI.isTargetNaCl64(); 48 StackPtr = TRI->getStackRegister(); 49 } 50 51 bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const { 52 return !MF.getFrameInfo().hasVarSizedObjects() && 53 !MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences(); 54 } 55 56 /// canSimplifyCallFramePseudos - If there is a reserved call frame, the 57 /// call frame pseudos can be simplified. Having a FP, as in the default 58 /// implementation, is not sufficient here since we can't always use it. 59 /// Use a more nuanced condition. 60 bool 61 X86FrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const { 62 return hasReservedCallFrame(MF) || 63 (hasFP(MF) && !TRI->needsStackRealignment(MF)) || 64 TRI->hasBasePointer(MF); 65 } 66 67 // needsFrameIndexResolution - Do we need to perform FI resolution for 68 // this function. Normally, this is required only when the function 69 // has any stack objects. However, FI resolution actually has another job, 70 // not apparent from the title - it resolves callframesetup/destroy 71 // that were not simplified earlier. 72 // So, this is required for x86 functions that have push sequences even 73 // when there are no stack objects. 74 bool 75 X86FrameLowering::needsFrameIndexResolution(const MachineFunction &MF) const { 76 return MF.getFrameInfo().hasStackObjects() || 77 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences(); 78 } 79 80 /// hasFP - Return true if the specified function should have a dedicated frame 81 /// pointer register. This is true if the function has variable sized allocas 82 /// or if frame pointer elimination is disabled. 83 bool X86FrameLowering::hasFP(const MachineFunction &MF) const { 84 const MachineFrameInfo &MFI = MF.getFrameInfo(); 85 return (MF.getTarget().Options.DisableFramePointerElim(MF) || 86 TRI->needsStackRealignment(MF) || 87 MFI.hasVarSizedObjects() || 88 MFI.isFrameAddressTaken() || MFI.hasOpaqueSPAdjustment() || 89 MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() || 90 MF.callsUnwindInit() || MF.hasEHFunclets() || MF.callsEHReturn() || 91 MFI.hasStackMap() || MFI.hasPatchPoint() || 92 MFI.hasCopyImplyingStackAdjustment()); 93 } 94 95 static unsigned getSUBriOpcode(unsigned IsLP64, int64_t Imm) { 96 if (IsLP64) { 97 if (isInt<8>(Imm)) 98 return X86::SUB64ri8; 99 return X86::SUB64ri32; 100 } else { 101 if (isInt<8>(Imm)) 102 return X86::SUB32ri8; 103 return X86::SUB32ri; 104 } 105 } 106 107 static unsigned getADDriOpcode(unsigned IsLP64, int64_t Imm) { 108 if (IsLP64) { 109 if (isInt<8>(Imm)) 110 return X86::ADD64ri8; 111 return X86::ADD64ri32; 112 } else { 113 if (isInt<8>(Imm)) 114 return X86::ADD32ri8; 115 return X86::ADD32ri; 116 } 117 } 118 119 static unsigned getSUBrrOpcode(unsigned isLP64) { 120 return isLP64 ? X86::SUB64rr : X86::SUB32rr; 121 } 122 123 static unsigned getADDrrOpcode(unsigned isLP64) { 124 return isLP64 ? X86::ADD64rr : X86::ADD32rr; 125 } 126 127 static unsigned getANDriOpcode(bool IsLP64, int64_t Imm) { 128 if (IsLP64) { 129 if (isInt<8>(Imm)) 130 return X86::AND64ri8; 131 return X86::AND64ri32; 132 } 133 if (isInt<8>(Imm)) 134 return X86::AND32ri8; 135 return X86::AND32ri; 136 } 137 138 static unsigned getLEArOpcode(unsigned IsLP64) { 139 return IsLP64 ? X86::LEA64r : X86::LEA32r; 140 } 141 142 /// findDeadCallerSavedReg - Return a caller-saved register that isn't live 143 /// when it reaches the "return" instruction. We can then pop a stack object 144 /// to this register without worry about clobbering it. 145 static unsigned findDeadCallerSavedReg(MachineBasicBlock &MBB, 146 MachineBasicBlock::iterator &MBBI, 147 const X86RegisterInfo *TRI, 148 bool Is64Bit) { 149 const MachineFunction *MF = MBB.getParent(); 150 if (MF->callsEHReturn()) 151 return 0; 152 153 const TargetRegisterClass &AvailableRegs = *TRI->getGPRsForTailCall(*MF); 154 155 if (MBBI == MBB.end()) 156 return 0; 157 158 switch (MBBI->getOpcode()) { 159 default: return 0; 160 case TargetOpcode::PATCHABLE_RET: 161 case X86::RET: 162 case X86::RETL: 163 case X86::RETQ: 164 case X86::RETIL: 165 case X86::RETIQ: 166 case X86::TCRETURNdi: 167 case X86::TCRETURNri: 168 case X86::TCRETURNmi: 169 case X86::TCRETURNdi64: 170 case X86::TCRETURNri64: 171 case X86::TCRETURNmi64: 172 case X86::EH_RETURN: 173 case X86::EH_RETURN64: { 174 SmallSet<uint16_t, 8> Uses; 175 for (unsigned i = 0, e = MBBI->getNumOperands(); i != e; ++i) { 176 MachineOperand &MO = MBBI->getOperand(i); 177 if (!MO.isReg() || MO.isDef()) 178 continue; 179 unsigned Reg = MO.getReg(); 180 if (!Reg) 181 continue; 182 for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) 183 Uses.insert(*AI); 184 } 185 186 for (auto CS : AvailableRegs) 187 if (!Uses.count(CS) && CS != X86::RIP && CS != X86::RSP && 188 CS != X86::ESP) 189 return CS; 190 } 191 } 192 193 return 0; 194 } 195 196 static bool isEAXLiveIn(MachineBasicBlock &MBB) { 197 for (MachineBasicBlock::RegisterMaskPair RegMask : MBB.liveins()) { 198 unsigned Reg = RegMask.PhysReg; 199 200 if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX || 201 Reg == X86::AH || Reg == X86::AL) 202 return true; 203 } 204 205 return false; 206 } 207 208 /// Check if the flags need to be preserved before the terminators. 209 /// This would be the case, if the eflags is live-in of the region 210 /// composed by the terminators or live-out of that region, without 211 /// being defined by a terminator. 212 static bool 213 flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB) { 214 for (const MachineInstr &MI : MBB.terminators()) { 215 bool BreakNext = false; 216 for (const MachineOperand &MO : MI.operands()) { 217 if (!MO.isReg()) 218 continue; 219 unsigned Reg = MO.getReg(); 220 if (Reg != X86::EFLAGS) 221 continue; 222 223 // This terminator needs an eflags that is not defined 224 // by a previous another terminator: 225 // EFLAGS is live-in of the region composed by the terminators. 226 if (!MO.isDef()) 227 return true; 228 // This terminator defines the eflags, i.e., we don't need to preserve it. 229 // However, we still need to check this specific terminator does not 230 // read a live-in value. 231 BreakNext = true; 232 } 233 // We found a definition of the eflags, no need to preserve them. 234 if (BreakNext) 235 return false; 236 } 237 238 // None of the terminators use or define the eflags. 239 // Check if they are live-out, that would imply we need to preserve them. 240 for (const MachineBasicBlock *Succ : MBB.successors()) 241 if (Succ->isLiveIn(X86::EFLAGS)) 242 return true; 243 244 return false; 245 } 246 247 /// emitSPUpdate - Emit a series of instructions to increment / decrement the 248 /// stack pointer by a constant value. 249 void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB, 250 MachineBasicBlock::iterator &MBBI, 251 const DebugLoc &DL, 252 int64_t NumBytes, bool InEpilogue) const { 253 bool isSub = NumBytes < 0; 254 uint64_t Offset = isSub ? -NumBytes : NumBytes; 255 MachineInstr::MIFlag Flag = 256 isSub ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy; 257 258 uint64_t Chunk = (1LL << 31) - 1; 259 260 if (Offset > Chunk) { 261 // Rather than emit a long series of instructions for large offsets, 262 // load the offset into a register and do one sub/add 263 unsigned Reg = 0; 264 unsigned Rax = (unsigned)(Is64Bit ? X86::RAX : X86::EAX); 265 266 if (isSub && !isEAXLiveIn(MBB)) 267 Reg = Rax; 268 else 269 Reg = findDeadCallerSavedReg(MBB, MBBI, TRI, Is64Bit); 270 271 unsigned MovRIOpc = Is64Bit ? X86::MOV64ri : X86::MOV32ri; 272 unsigned AddSubRROpc = 273 isSub ? getSUBrrOpcode(Is64Bit) : getADDrrOpcode(Is64Bit); 274 if (Reg) { 275 BuildMI(MBB, MBBI, DL, TII.get(MovRIOpc), Reg) 276 .addImm(Offset) 277 .setMIFlag(Flag); 278 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AddSubRROpc), StackPtr) 279 .addReg(StackPtr) 280 .addReg(Reg); 281 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. 282 return; 283 } else if (Offset > 8 * Chunk) { 284 // If we would need more than 8 add or sub instructions (a >16GB stack 285 // frame), it's worth spilling RAX to materialize this immediate. 286 // pushq %rax 287 // movabsq +-$Offset+-SlotSize, %rax 288 // addq %rsp, %rax 289 // xchg %rax, (%rsp) 290 // movq (%rsp), %rsp 291 assert(Is64Bit && "can't have 32-bit 16GB stack frame"); 292 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r)) 293 .addReg(Rax, RegState::Kill) 294 .setMIFlag(Flag); 295 // Subtract is not commutative, so negate the offset and always use add. 296 // Subtract 8 less and add 8 more to account for the PUSH we just did. 297 if (isSub) 298 Offset = -(Offset - SlotSize); 299 else 300 Offset = Offset + SlotSize; 301 BuildMI(MBB, MBBI, DL, TII.get(MovRIOpc), Rax) 302 .addImm(Offset) 303 .setMIFlag(Flag); 304 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(X86::ADD64rr), Rax) 305 .addReg(Rax) 306 .addReg(StackPtr); 307 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. 308 // Exchange the new SP in RAX with the top of the stack. 309 addRegOffset( 310 BuildMI(MBB, MBBI, DL, TII.get(X86::XCHG64rm), Rax).addReg(Rax), 311 StackPtr, false, 0); 312 // Load new SP from the top of the stack into RSP. 313 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), StackPtr), 314 StackPtr, false, 0); 315 return; 316 } 317 } 318 319 while (Offset) { 320 uint64_t ThisVal = std::min(Offset, Chunk); 321 if (ThisVal == SlotSize) { 322 // Use push / pop for slot sized adjustments as a size optimization. We 323 // need to find a dead register when using pop. 324 unsigned Reg = isSub 325 ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX) 326 : findDeadCallerSavedReg(MBB, MBBI, TRI, Is64Bit); 327 if (Reg) { 328 unsigned Opc = isSub 329 ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r) 330 : (Is64Bit ? X86::POP64r : X86::POP32r); 331 BuildMI(MBB, MBBI, DL, TII.get(Opc)) 332 .addReg(Reg, getDefRegState(!isSub) | getUndefRegState(isSub)) 333 .setMIFlag(Flag); 334 Offset -= ThisVal; 335 continue; 336 } 337 } 338 339 BuildStackAdjustment(MBB, MBBI, DL, isSub ? -ThisVal : ThisVal, InEpilogue) 340 .setMIFlag(Flag); 341 342 Offset -= ThisVal; 343 } 344 } 345 346 MachineInstrBuilder X86FrameLowering::BuildStackAdjustment( 347 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 348 const DebugLoc &DL, int64_t Offset, bool InEpilogue) const { 349 assert(Offset != 0 && "zero offset stack adjustment requested"); 350 351 // On Atom, using LEA to adjust SP is preferred, but using it in the epilogue 352 // is tricky. 353 bool UseLEA; 354 if (!InEpilogue) { 355 // Check if inserting the prologue at the beginning 356 // of MBB would require to use LEA operations. 357 // We need to use LEA operations if EFLAGS is live in, because 358 // it means an instruction will read it before it gets defined. 359 UseLEA = STI.useLeaForSP() || MBB.isLiveIn(X86::EFLAGS); 360 } else { 361 // If we can use LEA for SP but we shouldn't, check that none 362 // of the terminators uses the eflags. Otherwise we will insert 363 // a ADD that will redefine the eflags and break the condition. 364 // Alternatively, we could move the ADD, but this may not be possible 365 // and is an optimization anyway. 366 UseLEA = canUseLEAForSPInEpilogue(*MBB.getParent()); 367 if (UseLEA && !STI.useLeaForSP()) 368 UseLEA = flagsNeedToBePreservedBeforeTheTerminators(MBB); 369 // If that assert breaks, that means we do not do the right thing 370 // in canUseAsEpilogue. 371 assert((UseLEA || !flagsNeedToBePreservedBeforeTheTerminators(MBB)) && 372 "We shouldn't have allowed this insertion point"); 373 } 374 375 MachineInstrBuilder MI; 376 if (UseLEA) { 377 MI = addRegOffset(BuildMI(MBB, MBBI, DL, 378 TII.get(getLEArOpcode(Uses64BitFramePtr)), 379 StackPtr), 380 StackPtr, false, Offset); 381 } else { 382 bool IsSub = Offset < 0; 383 uint64_t AbsOffset = IsSub ? -Offset : Offset; 384 unsigned Opc = IsSub ? getSUBriOpcode(Uses64BitFramePtr, AbsOffset) 385 : getADDriOpcode(Uses64BitFramePtr, AbsOffset); 386 MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr) 387 .addReg(StackPtr) 388 .addImm(AbsOffset); 389 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. 390 } 391 return MI; 392 } 393 394 int X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB, 395 MachineBasicBlock::iterator &MBBI, 396 bool doMergeWithPrevious) const { 397 if ((doMergeWithPrevious && MBBI == MBB.begin()) || 398 (!doMergeWithPrevious && MBBI == MBB.end())) 399 return 0; 400 401 MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(MBBI) : MBBI; 402 403 PI = skipDebugInstructionsBackward(PI, MBB.begin()); 404 // It is assumed that ADD/SUB/LEA instruction is succeded by one CFI 405 // instruction, and that there are no DBG_VALUE or other instructions between 406 // ADD/SUB/LEA and its corresponding CFI instruction. 407 /* TODO: Add support for the case where there are multiple CFI instructions 408 below the ADD/SUB/LEA, e.g.: 409 ... 410 add 411 cfi_def_cfa_offset 412 cfi_offset 413 ... 414 */ 415 if (doMergeWithPrevious && PI != MBB.begin() && PI->isCFIInstruction()) 416 PI = std::prev(PI); 417 418 unsigned Opc = PI->getOpcode(); 419 int Offset = 0; 420 421 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 || 422 Opc == X86::ADD32ri || Opc == X86::ADD32ri8) && 423 PI->getOperand(0).getReg() == StackPtr){ 424 assert(PI->getOperand(1).getReg() == StackPtr); 425 Offset = PI->getOperand(2).getImm(); 426 } else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) && 427 PI->getOperand(0).getReg() == StackPtr && 428 PI->getOperand(1).getReg() == StackPtr && 429 PI->getOperand(2).getImm() == 1 && 430 PI->getOperand(3).getReg() == X86::NoRegister && 431 PI->getOperand(5).getReg() == X86::NoRegister) { 432 // For LEAs we have: def = lea SP, FI, noreg, Offset, noreg. 433 Offset = PI->getOperand(4).getImm(); 434 } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB64ri8 || 435 Opc == X86::SUB32ri || Opc == X86::SUB32ri8) && 436 PI->getOperand(0).getReg() == StackPtr) { 437 assert(PI->getOperand(1).getReg() == StackPtr); 438 Offset = -PI->getOperand(2).getImm(); 439 } else 440 return 0; 441 442 PI = MBB.erase(PI); 443 if (PI != MBB.end() && PI->isCFIInstruction()) PI = MBB.erase(PI); 444 if (!doMergeWithPrevious) 445 MBBI = skipDebugInstructionsForward(PI, MBB.end()); 446 447 return Offset; 448 } 449 450 void X86FrameLowering::BuildCFI(MachineBasicBlock &MBB, 451 MachineBasicBlock::iterator MBBI, 452 const DebugLoc &DL, 453 const MCCFIInstruction &CFIInst) const { 454 MachineFunction &MF = *MBB.getParent(); 455 unsigned CFIIndex = MF.addFrameInst(CFIInst); 456 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION)) 457 .addCFIIndex(CFIIndex); 458 } 459 460 void X86FrameLowering::emitCalleeSavedFrameMoves( 461 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 462 const DebugLoc &DL) const { 463 MachineFunction &MF = *MBB.getParent(); 464 MachineFrameInfo &MFI = MF.getFrameInfo(); 465 MachineModuleInfo &MMI = MF.getMMI(); 466 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo(); 467 468 // Add callee saved registers to move list. 469 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); 470 if (CSI.empty()) return; 471 472 // Calculate offsets. 473 for (std::vector<CalleeSavedInfo>::const_iterator 474 I = CSI.begin(), E = CSI.end(); I != E; ++I) { 475 int64_t Offset = MFI.getObjectOffset(I->getFrameIdx()); 476 unsigned Reg = I->getReg(); 477 478 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); 479 BuildCFI(MBB, MBBI, DL, 480 MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset)); 481 } 482 } 483 484 void X86FrameLowering::emitStackProbe(MachineFunction &MF, 485 MachineBasicBlock &MBB, 486 MachineBasicBlock::iterator MBBI, 487 const DebugLoc &DL, bool InProlog) const { 488 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 489 if (STI.isTargetWindowsCoreCLR()) { 490 if (InProlog) { 491 emitStackProbeInlineStub(MF, MBB, MBBI, DL, true); 492 } else { 493 emitStackProbeInline(MF, MBB, MBBI, DL, false); 494 } 495 } else { 496 emitStackProbeCall(MF, MBB, MBBI, DL, InProlog); 497 } 498 } 499 500 void X86FrameLowering::inlineStackProbe(MachineFunction &MF, 501 MachineBasicBlock &PrologMBB) const { 502 const StringRef ChkStkStubSymbol = "__chkstk_stub"; 503 MachineInstr *ChkStkStub = nullptr; 504 505 for (MachineInstr &MI : PrologMBB) { 506 if (MI.isCall() && MI.getOperand(0).isSymbol() && 507 ChkStkStubSymbol == MI.getOperand(0).getSymbolName()) { 508 ChkStkStub = &MI; 509 break; 510 } 511 } 512 513 if (ChkStkStub != nullptr) { 514 assert(!ChkStkStub->isBundled() && 515 "Not expecting bundled instructions here"); 516 MachineBasicBlock::iterator MBBI = std::next(ChkStkStub->getIterator()); 517 assert(std::prev(MBBI) == ChkStkStub && 518 "MBBI expected after __chkstk_stub."); 519 DebugLoc DL = PrologMBB.findDebugLoc(MBBI); 520 emitStackProbeInline(MF, PrologMBB, MBBI, DL, true); 521 ChkStkStub->eraseFromParent(); 522 } 523 } 524 525 void X86FrameLowering::emitStackProbeInline(MachineFunction &MF, 526 MachineBasicBlock &MBB, 527 MachineBasicBlock::iterator MBBI, 528 const DebugLoc &DL, 529 bool InProlog) const { 530 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 531 assert(STI.is64Bit() && "different expansion needed for 32 bit"); 532 assert(STI.isTargetWindowsCoreCLR() && "custom expansion expects CoreCLR"); 533 const TargetInstrInfo &TII = *STI.getInstrInfo(); 534 const BasicBlock *LLVM_BB = MBB.getBasicBlock(); 535 536 // RAX contains the number of bytes of desired stack adjustment. 537 // The handling here assumes this value has already been updated so as to 538 // maintain stack alignment. 539 // 540 // We need to exit with RSP modified by this amount and execute suitable 541 // page touches to notify the OS that we're growing the stack responsibly. 542 // All stack probing must be done without modifying RSP. 543 // 544 // MBB: 545 // SizeReg = RAX; 546 // ZeroReg = 0 547 // CopyReg = RSP 548 // Flags, TestReg = CopyReg - SizeReg 549 // FinalReg = !Flags.Ovf ? TestReg : ZeroReg 550 // LimitReg = gs magic thread env access 551 // if FinalReg >= LimitReg goto ContinueMBB 552 // RoundBB: 553 // RoundReg = page address of FinalReg 554 // LoopMBB: 555 // LoopReg = PHI(LimitReg,ProbeReg) 556 // ProbeReg = LoopReg - PageSize 557 // [ProbeReg] = 0 558 // if (ProbeReg > RoundReg) goto LoopMBB 559 // ContinueMBB: 560 // RSP = RSP - RAX 561 // [rest of original MBB] 562 563 // Set up the new basic blocks 564 MachineBasicBlock *RoundMBB = MF.CreateMachineBasicBlock(LLVM_BB); 565 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB); 566 MachineBasicBlock *ContinueMBB = MF.CreateMachineBasicBlock(LLVM_BB); 567 568 MachineFunction::iterator MBBIter = std::next(MBB.getIterator()); 569 MF.insert(MBBIter, RoundMBB); 570 MF.insert(MBBIter, LoopMBB); 571 MF.insert(MBBIter, ContinueMBB); 572 573 // Split MBB and move the tail portion down to ContinueMBB. 574 MachineBasicBlock::iterator BeforeMBBI = std::prev(MBBI); 575 ContinueMBB->splice(ContinueMBB->begin(), &MBB, MBBI, MBB.end()); 576 ContinueMBB->transferSuccessorsAndUpdatePHIs(&MBB); 577 578 // Some useful constants 579 const int64_t ThreadEnvironmentStackLimit = 0x10; 580 const int64_t PageSize = 0x1000; 581 const int64_t PageMask = ~(PageSize - 1); 582 583 // Registers we need. For the normal case we use virtual 584 // registers. For the prolog expansion we use RAX, RCX and RDX. 585 MachineRegisterInfo &MRI = MF.getRegInfo(); 586 const TargetRegisterClass *RegClass = &X86::GR64RegClass; 587 const Register SizeReg = InProlog ? X86::RAX 588 : MRI.createVirtualRegister(RegClass), 589 ZeroReg = InProlog ? X86::RCX 590 : MRI.createVirtualRegister(RegClass), 591 CopyReg = InProlog ? X86::RDX 592 : MRI.createVirtualRegister(RegClass), 593 TestReg = InProlog ? X86::RDX 594 : MRI.createVirtualRegister(RegClass), 595 FinalReg = InProlog ? X86::RDX 596 : MRI.createVirtualRegister(RegClass), 597 RoundedReg = InProlog ? X86::RDX 598 : MRI.createVirtualRegister(RegClass), 599 LimitReg = InProlog ? X86::RCX 600 : MRI.createVirtualRegister(RegClass), 601 JoinReg = InProlog ? X86::RCX 602 : MRI.createVirtualRegister(RegClass), 603 ProbeReg = InProlog ? X86::RCX 604 : MRI.createVirtualRegister(RegClass); 605 606 // SP-relative offsets where we can save RCX and RDX. 607 int64_t RCXShadowSlot = 0; 608 int64_t RDXShadowSlot = 0; 609 610 // If inlining in the prolog, save RCX and RDX. 611 if (InProlog) { 612 // Compute the offsets. We need to account for things already 613 // pushed onto the stack at this point: return address, frame 614 // pointer (if used), and callee saves. 615 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 616 const int64_t CalleeSaveSize = X86FI->getCalleeSavedFrameSize(); 617 const bool HasFP = hasFP(MF); 618 619 // Check if we need to spill RCX and/or RDX. 620 // Here we assume that no earlier prologue instruction changes RCX and/or 621 // RDX, so checking the block live-ins is enough. 622 const bool IsRCXLiveIn = MBB.isLiveIn(X86::RCX); 623 const bool IsRDXLiveIn = MBB.isLiveIn(X86::RDX); 624 int64_t InitSlot = 8 + CalleeSaveSize + (HasFP ? 8 : 0); 625 // Assign the initial slot to both registers, then change RDX's slot if both 626 // need to be spilled. 627 if (IsRCXLiveIn) 628 RCXShadowSlot = InitSlot; 629 if (IsRDXLiveIn) 630 RDXShadowSlot = InitSlot; 631 if (IsRDXLiveIn && IsRCXLiveIn) 632 RDXShadowSlot += 8; 633 // Emit the saves if needed. 634 if (IsRCXLiveIn) 635 addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false, 636 RCXShadowSlot) 637 .addReg(X86::RCX); 638 if (IsRDXLiveIn) 639 addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false, 640 RDXShadowSlot) 641 .addReg(X86::RDX); 642 } else { 643 // Not in the prolog. Copy RAX to a virtual reg. 644 BuildMI(&MBB, DL, TII.get(X86::MOV64rr), SizeReg).addReg(X86::RAX); 645 } 646 647 // Add code to MBB to check for overflow and set the new target stack pointer 648 // to zero if so. 649 BuildMI(&MBB, DL, TII.get(X86::XOR64rr), ZeroReg) 650 .addReg(ZeroReg, RegState::Undef) 651 .addReg(ZeroReg, RegState::Undef); 652 BuildMI(&MBB, DL, TII.get(X86::MOV64rr), CopyReg).addReg(X86::RSP); 653 BuildMI(&MBB, DL, TII.get(X86::SUB64rr), TestReg) 654 .addReg(CopyReg) 655 .addReg(SizeReg); 656 BuildMI(&MBB, DL, TII.get(X86::CMOV64rr), FinalReg) 657 .addReg(TestReg) 658 .addReg(ZeroReg) 659 .addImm(X86::COND_B); 660 661 // FinalReg now holds final stack pointer value, or zero if 662 // allocation would overflow. Compare against the current stack 663 // limit from the thread environment block. Note this limit is the 664 // lowest touched page on the stack, not the point at which the OS 665 // will cause an overflow exception, so this is just an optimization 666 // to avoid unnecessarily touching pages that are below the current 667 // SP but already committed to the stack by the OS. 668 BuildMI(&MBB, DL, TII.get(X86::MOV64rm), LimitReg) 669 .addReg(0) 670 .addImm(1) 671 .addReg(0) 672 .addImm(ThreadEnvironmentStackLimit) 673 .addReg(X86::GS); 674 BuildMI(&MBB, DL, TII.get(X86::CMP64rr)).addReg(FinalReg).addReg(LimitReg); 675 // Jump if the desired stack pointer is at or above the stack limit. 676 BuildMI(&MBB, DL, TII.get(X86::JCC_1)).addMBB(ContinueMBB).addImm(X86::COND_AE); 677 678 // Add code to roundMBB to round the final stack pointer to a page boundary. 679 RoundMBB->addLiveIn(FinalReg); 680 BuildMI(RoundMBB, DL, TII.get(X86::AND64ri32), RoundedReg) 681 .addReg(FinalReg) 682 .addImm(PageMask); 683 BuildMI(RoundMBB, DL, TII.get(X86::JMP_1)).addMBB(LoopMBB); 684 685 // LimitReg now holds the current stack limit, RoundedReg page-rounded 686 // final RSP value. Add code to loopMBB to decrement LimitReg page-by-page 687 // and probe until we reach RoundedReg. 688 if (!InProlog) { 689 BuildMI(LoopMBB, DL, TII.get(X86::PHI), JoinReg) 690 .addReg(LimitReg) 691 .addMBB(RoundMBB) 692 .addReg(ProbeReg) 693 .addMBB(LoopMBB); 694 } 695 696 LoopMBB->addLiveIn(JoinReg); 697 addRegOffset(BuildMI(LoopMBB, DL, TII.get(X86::LEA64r), ProbeReg), JoinReg, 698 false, -PageSize); 699 700 // Probe by storing a byte onto the stack. 701 BuildMI(LoopMBB, DL, TII.get(X86::MOV8mi)) 702 .addReg(ProbeReg) 703 .addImm(1) 704 .addReg(0) 705 .addImm(0) 706 .addReg(0) 707 .addImm(0); 708 709 LoopMBB->addLiveIn(RoundedReg); 710 BuildMI(LoopMBB, DL, TII.get(X86::CMP64rr)) 711 .addReg(RoundedReg) 712 .addReg(ProbeReg); 713 BuildMI(LoopMBB, DL, TII.get(X86::JCC_1)).addMBB(LoopMBB).addImm(X86::COND_NE); 714 715 MachineBasicBlock::iterator ContinueMBBI = ContinueMBB->getFirstNonPHI(); 716 717 // If in prolog, restore RDX and RCX. 718 if (InProlog) { 719 if (RCXShadowSlot) // It means we spilled RCX in the prologue. 720 addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL, 721 TII.get(X86::MOV64rm), X86::RCX), 722 X86::RSP, false, RCXShadowSlot); 723 if (RDXShadowSlot) // It means we spilled RDX in the prologue. 724 addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL, 725 TII.get(X86::MOV64rm), X86::RDX), 726 X86::RSP, false, RDXShadowSlot); 727 } 728 729 // Now that the probing is done, add code to continueMBB to update 730 // the stack pointer for real. 731 ContinueMBB->addLiveIn(SizeReg); 732 BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(X86::SUB64rr), X86::RSP) 733 .addReg(X86::RSP) 734 .addReg(SizeReg); 735 736 // Add the control flow edges we need. 737 MBB.addSuccessor(ContinueMBB); 738 MBB.addSuccessor(RoundMBB); 739 RoundMBB->addSuccessor(LoopMBB); 740 LoopMBB->addSuccessor(ContinueMBB); 741 LoopMBB->addSuccessor(LoopMBB); 742 743 // Mark all the instructions added to the prolog as frame setup. 744 if (InProlog) { 745 for (++BeforeMBBI; BeforeMBBI != MBB.end(); ++BeforeMBBI) { 746 BeforeMBBI->setFlag(MachineInstr::FrameSetup); 747 } 748 for (MachineInstr &MI : *RoundMBB) { 749 MI.setFlag(MachineInstr::FrameSetup); 750 } 751 for (MachineInstr &MI : *LoopMBB) { 752 MI.setFlag(MachineInstr::FrameSetup); 753 } 754 for (MachineBasicBlock::iterator CMBBI = ContinueMBB->begin(); 755 CMBBI != ContinueMBBI; ++CMBBI) { 756 CMBBI->setFlag(MachineInstr::FrameSetup); 757 } 758 } 759 } 760 761 void X86FrameLowering::emitStackProbeCall(MachineFunction &MF, 762 MachineBasicBlock &MBB, 763 MachineBasicBlock::iterator MBBI, 764 const DebugLoc &DL, 765 bool InProlog) const { 766 bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large; 767 768 // FIXME: Add retpoline support and remove this. 769 if (Is64Bit && IsLargeCodeModel && STI.useRetpolineIndirectCalls()) 770 report_fatal_error("Emitting stack probe calls on 64-bit with the large " 771 "code model and retpoline not yet implemented."); 772 773 unsigned CallOp; 774 if (Is64Bit) 775 CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32; 776 else 777 CallOp = X86::CALLpcrel32; 778 779 StringRef Symbol = STI.getTargetLowering()->getStackProbeSymbolName(MF); 780 781 MachineInstrBuilder CI; 782 MachineBasicBlock::iterator ExpansionMBBI = std::prev(MBBI); 783 784 // All current stack probes take AX and SP as input, clobber flags, and 785 // preserve all registers. x86_64 probes leave RSP unmodified. 786 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) { 787 // For the large code model, we have to call through a register. Use R11, 788 // as it is scratch in all supported calling conventions. 789 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::R11) 790 .addExternalSymbol(MF.createExternalSymbolName(Symbol)); 791 CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)).addReg(X86::R11); 792 } else { 793 CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)) 794 .addExternalSymbol(MF.createExternalSymbolName(Symbol)); 795 } 796 797 unsigned AX = Uses64BitFramePtr ? X86::RAX : X86::EAX; 798 unsigned SP = Uses64BitFramePtr ? X86::RSP : X86::ESP; 799 CI.addReg(AX, RegState::Implicit) 800 .addReg(SP, RegState::Implicit) 801 .addReg(AX, RegState::Define | RegState::Implicit) 802 .addReg(SP, RegState::Define | RegState::Implicit) 803 .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit); 804 805 if (STI.isTargetWin64() || !STI.isOSWindows()) { 806 // MSVC x32's _chkstk and cygwin/mingw's _alloca adjust %esp themselves. 807 // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp 808 // themselves. They also does not clobber %rax so we can reuse it when 809 // adjusting %rsp. 810 // All other platforms do not specify a particular ABI for the stack probe 811 // function, so we arbitrarily define it to not adjust %esp/%rsp itself. 812 BuildMI(MBB, MBBI, DL, TII.get(getSUBrrOpcode(Uses64BitFramePtr)), SP) 813 .addReg(SP) 814 .addReg(AX); 815 } 816 817 if (InProlog) { 818 // Apply the frame setup flag to all inserted instrs. 819 for (++ExpansionMBBI; ExpansionMBBI != MBBI; ++ExpansionMBBI) 820 ExpansionMBBI->setFlag(MachineInstr::FrameSetup); 821 } 822 } 823 824 void X86FrameLowering::emitStackProbeInlineStub( 825 MachineFunction &MF, MachineBasicBlock &MBB, 826 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const { 827 828 assert(InProlog && "ChkStkStub called outside prolog!"); 829 830 BuildMI(MBB, MBBI, DL, TII.get(X86::CALLpcrel32)) 831 .addExternalSymbol("__chkstk_stub"); 832 } 833 834 static unsigned calculateSetFPREG(uint64_t SPAdjust) { 835 // Win64 ABI has a less restrictive limitation of 240; 128 works equally well 836 // and might require smaller successive adjustments. 837 const uint64_t Win64MaxSEHOffset = 128; 838 uint64_t SEHFrameOffset = std::min(SPAdjust, Win64MaxSEHOffset); 839 // Win64 ABI requires 16-byte alignment for the UWOP_SET_FPREG opcode. 840 return SEHFrameOffset & -16; 841 } 842 843 // If we're forcing a stack realignment we can't rely on just the frame 844 // info, we need to know the ABI stack alignment as well in case we 845 // have a call out. Otherwise just make sure we have some alignment - we'll 846 // go with the minimum SlotSize. 847 uint64_t X86FrameLowering::calculateMaxStackAlign(const MachineFunction &MF) const { 848 const MachineFrameInfo &MFI = MF.getFrameInfo(); 849 uint64_t MaxAlign = MFI.getMaxAlignment(); // Desired stack alignment. 850 unsigned StackAlign = getStackAlignment(); 851 if (MF.getFunction().hasFnAttribute("stackrealign")) { 852 if (MFI.hasCalls()) 853 MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign; 854 else if (MaxAlign < SlotSize) 855 MaxAlign = SlotSize; 856 } 857 return MaxAlign; 858 } 859 860 void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB, 861 MachineBasicBlock::iterator MBBI, 862 const DebugLoc &DL, unsigned Reg, 863 uint64_t MaxAlign) const { 864 uint64_t Val = -MaxAlign; 865 unsigned AndOp = getANDriOpcode(Uses64BitFramePtr, Val); 866 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AndOp), Reg) 867 .addReg(Reg) 868 .addImm(Val) 869 .setMIFlag(MachineInstr::FrameSetup); 870 871 // The EFLAGS implicit def is dead. 872 MI->getOperand(3).setIsDead(); 873 } 874 875 bool X86FrameLowering::has128ByteRedZone(const MachineFunction& MF) const { 876 // x86-64 (non Win64) has a 128 byte red zone which is guaranteed not to be 877 // clobbered by any interrupt handler. 878 assert(&STI == &MF.getSubtarget<X86Subtarget>() && 879 "MF used frame lowering for wrong subtarget"); 880 const Function &Fn = MF.getFunction(); 881 const bool IsWin64CC = STI.isCallingConvWin64(Fn.getCallingConv()); 882 return Is64Bit && !IsWin64CC && !Fn.hasFnAttribute(Attribute::NoRedZone); 883 } 884 885 886 /// emitPrologue - Push callee-saved registers onto the stack, which 887 /// automatically adjust the stack pointer. Adjust the stack pointer to allocate 888 /// space for local variables. Also emit labels used by the exception handler to 889 /// generate the exception handling frames. 890 891 /* 892 Here's a gist of what gets emitted: 893 894 ; Establish frame pointer, if needed 895 [if needs FP] 896 push %rbp 897 .cfi_def_cfa_offset 16 898 .cfi_offset %rbp, -16 899 .seh_pushreg %rpb 900 mov %rsp, %rbp 901 .cfi_def_cfa_register %rbp 902 903 ; Spill general-purpose registers 904 [for all callee-saved GPRs] 905 pushq %<reg> 906 [if not needs FP] 907 .cfi_def_cfa_offset (offset from RETADDR) 908 .seh_pushreg %<reg> 909 910 ; If the required stack alignment > default stack alignment 911 ; rsp needs to be re-aligned. This creates a "re-alignment gap" 912 ; of unknown size in the stack frame. 913 [if stack needs re-alignment] 914 and $MASK, %rsp 915 916 ; Allocate space for locals 917 [if target is Windows and allocated space > 4096 bytes] 918 ; Windows needs special care for allocations larger 919 ; than one page. 920 mov $NNN, %rax 921 call ___chkstk_ms/___chkstk 922 sub %rax, %rsp 923 [else] 924 sub $NNN, %rsp 925 926 [if needs FP] 927 .seh_stackalloc (size of XMM spill slots) 928 .seh_setframe %rbp, SEHFrameOffset ; = size of all spill slots 929 [else] 930 .seh_stackalloc NNN 931 932 ; Spill XMMs 933 ; Note, that while only Windows 64 ABI specifies XMMs as callee-preserved, 934 ; they may get spilled on any platform, if the current function 935 ; calls @llvm.eh.unwind.init 936 [if needs FP] 937 [for all callee-saved XMM registers] 938 movaps %<xmm reg>, -MMM(%rbp) 939 [for all callee-saved XMM registers] 940 .seh_savexmm %<xmm reg>, (-MMM + SEHFrameOffset) 941 ; i.e. the offset relative to (%rbp - SEHFrameOffset) 942 [else] 943 [for all callee-saved XMM registers] 944 movaps %<xmm reg>, KKK(%rsp) 945 [for all callee-saved XMM registers] 946 .seh_savexmm %<xmm reg>, KKK 947 948 .seh_endprologue 949 950 [if needs base pointer] 951 mov %rsp, %rbx 952 [if needs to restore base pointer] 953 mov %rsp, -MMM(%rbp) 954 955 ; Emit CFI info 956 [if needs FP] 957 [for all callee-saved registers] 958 .cfi_offset %<reg>, (offset from %rbp) 959 [else] 960 .cfi_def_cfa_offset (offset from RETADDR) 961 [for all callee-saved registers] 962 .cfi_offset %<reg>, (offset from %rsp) 963 964 Notes: 965 - .seh directives are emitted only for Windows 64 ABI 966 - .cv_fpo directives are emitted on win32 when emitting CodeView 967 - .cfi directives are emitted for all other ABIs 968 - for 32-bit code, substitute %e?? registers for %r?? 969 */ 970 971 void X86FrameLowering::emitPrologue(MachineFunction &MF, 972 MachineBasicBlock &MBB) const { 973 assert(&STI == &MF.getSubtarget<X86Subtarget>() && 974 "MF used frame lowering for wrong subtarget"); 975 MachineBasicBlock::iterator MBBI = MBB.begin(); 976 MachineFrameInfo &MFI = MF.getFrameInfo(); 977 const Function &Fn = MF.getFunction(); 978 MachineModuleInfo &MMI = MF.getMMI(); 979 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 980 uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment. 981 uint64_t StackSize = MFI.getStackSize(); // Number of bytes to allocate. 982 bool IsFunclet = MBB.isEHFuncletEntry(); 983 EHPersonality Personality = EHPersonality::Unknown; 984 if (Fn.hasPersonalityFn()) 985 Personality = classifyEHPersonality(Fn.getPersonalityFn()); 986 bool FnHasClrFunclet = 987 MF.hasEHFunclets() && Personality == EHPersonality::CoreCLR; 988 bool IsClrFunclet = IsFunclet && FnHasClrFunclet; 989 bool HasFP = hasFP(MF); 990 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 991 bool NeedsWin64CFI = IsWin64Prologue && Fn.needsUnwindTableEntry(); 992 // FIXME: Emit FPO data for EH funclets. 993 bool NeedsWinFPO = 994 !IsFunclet && STI.isTargetWin32() && MMI.getModule()->getCodeViewFlag(); 995 bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO; 996 bool NeedsDwarfCFI = 997 !IsWin64Prologue && (MMI.hasDebugInfo() || Fn.needsUnwindTableEntry()); 998 unsigned FramePtr = TRI->getFrameRegister(MF); 999 const unsigned MachineFramePtr = 1000 STI.isTarget64BitILP32() 1001 ? getX86SubSuperRegister(FramePtr, 64) : FramePtr; 1002 unsigned BasePtr = TRI->getBaseRegister(); 1003 bool HasWinCFI = false; 1004 1005 // Debug location must be unknown since the first debug location is used 1006 // to determine the end of the prologue. 1007 DebugLoc DL; 1008 1009 // Add RETADDR move area to callee saved frame size. 1010 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta(); 1011 if (TailCallReturnAddrDelta && IsWin64Prologue) 1012 report_fatal_error("Can't handle guaranteed tail call under win64 yet"); 1013 1014 if (TailCallReturnAddrDelta < 0) 1015 X86FI->setCalleeSavedFrameSize( 1016 X86FI->getCalleeSavedFrameSize() - TailCallReturnAddrDelta); 1017 1018 bool UseStackProbe = !STI.getTargetLowering()->getStackProbeSymbolName(MF).empty(); 1019 1020 // The default stack probe size is 4096 if the function has no stackprobesize 1021 // attribute. 1022 unsigned StackProbeSize = 4096; 1023 if (Fn.hasFnAttribute("stack-probe-size")) 1024 Fn.getFnAttribute("stack-probe-size") 1025 .getValueAsString() 1026 .getAsInteger(0, StackProbeSize); 1027 1028 // Re-align the stack on 64-bit if the x86-interrupt calling convention is 1029 // used and an error code was pushed, since the x86-64 ABI requires a 16-byte 1030 // stack alignment. 1031 if (Fn.getCallingConv() == CallingConv::X86_INTR && Is64Bit && 1032 Fn.arg_size() == 2) { 1033 StackSize += 8; 1034 MFI.setStackSize(StackSize); 1035 emitSPUpdate(MBB, MBBI, DL, -8, /*InEpilogue=*/false); 1036 } 1037 1038 // If this is x86-64 and the Red Zone is not disabled, if we are a leaf 1039 // function, and use up to 128 bytes of stack space, don't have a frame 1040 // pointer, calls, or dynamic alloca then we do not need to adjust the 1041 // stack pointer (we fit in the Red Zone). We also check that we don't 1042 // push and pop from the stack. 1043 if (has128ByteRedZone(MF) && 1044 !TRI->needsStackRealignment(MF) && 1045 !MFI.hasVarSizedObjects() && // No dynamic alloca. 1046 !MFI.adjustsStack() && // No calls. 1047 !UseStackProbe && // No stack probes. 1048 !MFI.hasCopyImplyingStackAdjustment() && // Don't push and pop. 1049 !MF.shouldSplitStack()) { // Regular stack 1050 uint64_t MinSize = X86FI->getCalleeSavedFrameSize(); 1051 if (HasFP) MinSize += SlotSize; 1052 X86FI->setUsesRedZone(MinSize > 0 || StackSize > 0); 1053 StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0); 1054 MFI.setStackSize(StackSize); 1055 } 1056 1057 // Insert stack pointer adjustment for later moving of return addr. Only 1058 // applies to tail call optimized functions where the callee argument stack 1059 // size is bigger than the callers. 1060 if (TailCallReturnAddrDelta < 0) { 1061 BuildStackAdjustment(MBB, MBBI, DL, TailCallReturnAddrDelta, 1062 /*InEpilogue=*/false) 1063 .setMIFlag(MachineInstr::FrameSetup); 1064 } 1065 1066 // Mapping for machine moves: 1067 // 1068 // DST: VirtualFP AND 1069 // SRC: VirtualFP => DW_CFA_def_cfa_offset 1070 // ELSE => DW_CFA_def_cfa 1071 // 1072 // SRC: VirtualFP AND 1073 // DST: Register => DW_CFA_def_cfa_register 1074 // 1075 // ELSE 1076 // OFFSET < 0 => DW_CFA_offset_extended_sf 1077 // REG < 64 => DW_CFA_offset + Reg 1078 // ELSE => DW_CFA_offset_extended 1079 1080 uint64_t NumBytes = 0; 1081 int stackGrowth = -SlotSize; 1082 1083 // Find the funclet establisher parameter 1084 unsigned Establisher = X86::NoRegister; 1085 if (IsClrFunclet) 1086 Establisher = Uses64BitFramePtr ? X86::RCX : X86::ECX; 1087 else if (IsFunclet) 1088 Establisher = Uses64BitFramePtr ? X86::RDX : X86::EDX; 1089 1090 if (IsWin64Prologue && IsFunclet && !IsClrFunclet) { 1091 // Immediately spill establisher into the home slot. 1092 // The runtime cares about this. 1093 // MOV64mr %rdx, 16(%rsp) 1094 unsigned MOVmr = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr; 1095 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MOVmr)), StackPtr, true, 16) 1096 .addReg(Establisher) 1097 .setMIFlag(MachineInstr::FrameSetup); 1098 MBB.addLiveIn(Establisher); 1099 } 1100 1101 if (HasFP) { 1102 assert(MF.getRegInfo().isReserved(MachineFramePtr) && "FP reserved"); 1103 1104 // Calculate required stack adjustment. 1105 uint64_t FrameSize = StackSize - SlotSize; 1106 // If required, include space for extra hidden slot for stashing base pointer. 1107 if (X86FI->getRestoreBasePointer()) 1108 FrameSize += SlotSize; 1109 1110 NumBytes = FrameSize - X86FI->getCalleeSavedFrameSize(); 1111 1112 // Callee-saved registers are pushed on stack before the stack is realigned. 1113 if (TRI->needsStackRealignment(MF) && !IsWin64Prologue) 1114 NumBytes = alignTo(NumBytes, MaxAlign); 1115 1116 // Save EBP/RBP into the appropriate stack slot. 1117 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64r : X86::PUSH32r)) 1118 .addReg(MachineFramePtr, RegState::Kill) 1119 .setMIFlag(MachineInstr::FrameSetup); 1120 1121 if (NeedsDwarfCFI) { 1122 // Mark the place where EBP/RBP was saved. 1123 // Define the current CFA rule to use the provided offset. 1124 assert(StackSize); 1125 BuildCFI(MBB, MBBI, DL, 1126 MCCFIInstruction::createDefCfaOffset(nullptr, 2 * stackGrowth)); 1127 1128 // Change the rule for the FramePtr to be an "offset" rule. 1129 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true); 1130 BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createOffset( 1131 nullptr, DwarfFramePtr, 2 * stackGrowth)); 1132 } 1133 1134 if (NeedsWinCFI) { 1135 HasWinCFI = true; 1136 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg)) 1137 .addImm(FramePtr) 1138 .setMIFlag(MachineInstr::FrameSetup); 1139 } 1140 1141 if (!IsWin64Prologue && !IsFunclet) { 1142 // Update EBP with the new base value. 1143 BuildMI(MBB, MBBI, DL, 1144 TII.get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), 1145 FramePtr) 1146 .addReg(StackPtr) 1147 .setMIFlag(MachineInstr::FrameSetup); 1148 1149 if (NeedsDwarfCFI) { 1150 // Mark effective beginning of when frame pointer becomes valid. 1151 // Define the current CFA to use the EBP/RBP register. 1152 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true); 1153 BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createDefCfaRegister( 1154 nullptr, DwarfFramePtr)); 1155 } 1156 1157 if (NeedsWinFPO) { 1158 // .cv_fpo_setframe $FramePtr 1159 HasWinCFI = true; 1160 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame)) 1161 .addImm(FramePtr) 1162 .addImm(0) 1163 .setMIFlag(MachineInstr::FrameSetup); 1164 } 1165 } 1166 } else { 1167 assert(!IsFunclet && "funclets without FPs not yet implemented"); 1168 NumBytes = StackSize - X86FI->getCalleeSavedFrameSize(); 1169 } 1170 1171 // Update the offset adjustment, which is mainly used by codeview to translate 1172 // from ESP to VFRAME relative local variable offsets. 1173 if (!IsFunclet) { 1174 if (HasFP && TRI->needsStackRealignment(MF)) 1175 MFI.setOffsetAdjustment(-NumBytes); 1176 else 1177 MFI.setOffsetAdjustment(-StackSize); 1178 } 1179 1180 // For EH funclets, only allocate enough space for outgoing calls. Save the 1181 // NumBytes value that we would've used for the parent frame. 1182 unsigned ParentFrameNumBytes = NumBytes; 1183 if (IsFunclet) 1184 NumBytes = getWinEHFuncletFrameSize(MF); 1185 1186 // Skip the callee-saved push instructions. 1187 bool PushedRegs = false; 1188 int StackOffset = 2 * stackGrowth; 1189 1190 while (MBBI != MBB.end() && 1191 MBBI->getFlag(MachineInstr::FrameSetup) && 1192 (MBBI->getOpcode() == X86::PUSH32r || 1193 MBBI->getOpcode() == X86::PUSH64r)) { 1194 PushedRegs = true; 1195 unsigned Reg = MBBI->getOperand(0).getReg(); 1196 ++MBBI; 1197 1198 if (!HasFP && NeedsDwarfCFI) { 1199 // Mark callee-saved push instruction. 1200 // Define the current CFA rule to use the provided offset. 1201 assert(StackSize); 1202 BuildCFI(MBB, MBBI, DL, 1203 MCCFIInstruction::createDefCfaOffset(nullptr, StackOffset)); 1204 StackOffset += stackGrowth; 1205 } 1206 1207 if (NeedsWinCFI) { 1208 HasWinCFI = true; 1209 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg)) 1210 .addImm(Reg) 1211 .setMIFlag(MachineInstr::FrameSetup); 1212 } 1213 } 1214 1215 // Realign stack after we pushed callee-saved registers (so that we'll be 1216 // able to calculate their offsets from the frame pointer). 1217 // Don't do this for Win64, it needs to realign the stack after the prologue. 1218 if (!IsWin64Prologue && !IsFunclet && TRI->needsStackRealignment(MF)) { 1219 assert(HasFP && "There should be a frame pointer if stack is realigned."); 1220 BuildStackAlignAND(MBB, MBBI, DL, StackPtr, MaxAlign); 1221 1222 if (NeedsWinCFI) { 1223 HasWinCFI = true; 1224 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlign)) 1225 .addImm(MaxAlign) 1226 .setMIFlag(MachineInstr::FrameSetup); 1227 } 1228 } 1229 1230 // If there is an SUB32ri of ESP immediately before this instruction, merge 1231 // the two. This can be the case when tail call elimination is enabled and 1232 // the callee has more arguments then the caller. 1233 NumBytes -= mergeSPUpdates(MBB, MBBI, true); 1234 1235 // Adjust stack pointer: ESP -= numbytes. 1236 1237 // Windows and cygwin/mingw require a prologue helper routine when allocating 1238 // more than 4K bytes on the stack. Windows uses __chkstk and cygwin/mingw 1239 // uses __alloca. __alloca and the 32-bit version of __chkstk will probe the 1240 // stack and adjust the stack pointer in one go. The 64-bit version of 1241 // __chkstk is only responsible for probing the stack. The 64-bit prologue is 1242 // responsible for adjusting the stack pointer. Touching the stack at 4K 1243 // increments is necessary to ensure that the guard pages used by the OS 1244 // virtual memory manager are allocated in correct sequence. 1245 uint64_t AlignedNumBytes = NumBytes; 1246 if (IsWin64Prologue && !IsFunclet && TRI->needsStackRealignment(MF)) 1247 AlignedNumBytes = alignTo(AlignedNumBytes, MaxAlign); 1248 if (AlignedNumBytes >= StackProbeSize && UseStackProbe) { 1249 assert(!X86FI->getUsesRedZone() && 1250 "The Red Zone is not accounted for in stack probes"); 1251 1252 // Check whether EAX is livein for this block. 1253 bool isEAXAlive = isEAXLiveIn(MBB); 1254 1255 if (isEAXAlive) { 1256 if (Is64Bit) { 1257 // Save RAX 1258 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r)) 1259 .addReg(X86::RAX, RegState::Kill) 1260 .setMIFlag(MachineInstr::FrameSetup); 1261 } else { 1262 // Save EAX 1263 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH32r)) 1264 .addReg(X86::EAX, RegState::Kill) 1265 .setMIFlag(MachineInstr::FrameSetup); 1266 } 1267 } 1268 1269 if (Is64Bit) { 1270 // Handle the 64-bit Windows ABI case where we need to call __chkstk. 1271 // Function prologue is responsible for adjusting the stack pointer. 1272 int Alloc = isEAXAlive ? NumBytes - 8 : NumBytes; 1273 if (isUInt<32>(Alloc)) { 1274 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX) 1275 .addImm(Alloc) 1276 .setMIFlag(MachineInstr::FrameSetup); 1277 } else if (isInt<32>(Alloc)) { 1278 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri32), X86::RAX) 1279 .addImm(Alloc) 1280 .setMIFlag(MachineInstr::FrameSetup); 1281 } else { 1282 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::RAX) 1283 .addImm(Alloc) 1284 .setMIFlag(MachineInstr::FrameSetup); 1285 } 1286 } else { 1287 // Allocate NumBytes-4 bytes on stack in case of isEAXAlive. 1288 // We'll also use 4 already allocated bytes for EAX. 1289 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX) 1290 .addImm(isEAXAlive ? NumBytes - 4 : NumBytes) 1291 .setMIFlag(MachineInstr::FrameSetup); 1292 } 1293 1294 // Call __chkstk, __chkstk_ms, or __alloca. 1295 emitStackProbe(MF, MBB, MBBI, DL, true); 1296 1297 if (isEAXAlive) { 1298 // Restore RAX/EAX 1299 MachineInstr *MI; 1300 if (Is64Bit) 1301 MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV64rm), X86::RAX), 1302 StackPtr, false, NumBytes - 8); 1303 else 1304 MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV32rm), X86::EAX), 1305 StackPtr, false, NumBytes - 4); 1306 MI->setFlag(MachineInstr::FrameSetup); 1307 MBB.insert(MBBI, MI); 1308 } 1309 } else if (NumBytes) { 1310 emitSPUpdate(MBB, MBBI, DL, -(int64_t)NumBytes, /*InEpilogue=*/false); 1311 } 1312 1313 if (NeedsWinCFI && NumBytes) { 1314 HasWinCFI = true; 1315 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlloc)) 1316 .addImm(NumBytes) 1317 .setMIFlag(MachineInstr::FrameSetup); 1318 } 1319 1320 int SEHFrameOffset = 0; 1321 unsigned SPOrEstablisher; 1322 if (IsFunclet) { 1323 if (IsClrFunclet) { 1324 // The establisher parameter passed to a CLR funclet is actually a pointer 1325 // to the (mostly empty) frame of its nearest enclosing funclet; we have 1326 // to find the root function establisher frame by loading the PSPSym from 1327 // the intermediate frame. 1328 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF); 1329 MachinePointerInfo NoInfo; 1330 MBB.addLiveIn(Establisher); 1331 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), Establisher), 1332 Establisher, false, PSPSlotOffset) 1333 .addMemOperand(MF.getMachineMemOperand( 1334 NoInfo, MachineMemOperand::MOLoad, SlotSize, SlotSize)); 1335 ; 1336 // Save the root establisher back into the current funclet's (mostly 1337 // empty) frame, in case a sub-funclet or the GC needs it. 1338 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr, 1339 false, PSPSlotOffset) 1340 .addReg(Establisher) 1341 .addMemOperand( 1342 MF.getMachineMemOperand(NoInfo, MachineMemOperand::MOStore | 1343 MachineMemOperand::MOVolatile, 1344 SlotSize, SlotSize)); 1345 } 1346 SPOrEstablisher = Establisher; 1347 } else { 1348 SPOrEstablisher = StackPtr; 1349 } 1350 1351 if (IsWin64Prologue && HasFP) { 1352 // Set RBP to a small fixed offset from RSP. In the funclet case, we base 1353 // this calculation on the incoming establisher, which holds the value of 1354 // RSP from the parent frame at the end of the prologue. 1355 SEHFrameOffset = calculateSetFPREG(ParentFrameNumBytes); 1356 if (SEHFrameOffset) 1357 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr), 1358 SPOrEstablisher, false, SEHFrameOffset); 1359 else 1360 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rr), FramePtr) 1361 .addReg(SPOrEstablisher); 1362 1363 // If this is not a funclet, emit the CFI describing our frame pointer. 1364 if (NeedsWinCFI && !IsFunclet) { 1365 assert(!NeedsWinFPO && "this setframe incompatible with FPO data"); 1366 HasWinCFI = true; 1367 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame)) 1368 .addImm(FramePtr) 1369 .addImm(SEHFrameOffset) 1370 .setMIFlag(MachineInstr::FrameSetup); 1371 if (isAsynchronousEHPersonality(Personality)) 1372 MF.getWinEHFuncInfo()->SEHSetFrameOffset = SEHFrameOffset; 1373 } 1374 } else if (IsFunclet && STI.is32Bit()) { 1375 // Reset EBP / ESI to something good for funclets. 1376 MBBI = restoreWin32EHStackPointers(MBB, MBBI, DL); 1377 // If we're a catch funclet, we can be returned to via catchret. Save ESP 1378 // into the registration node so that the runtime will restore it for us. 1379 if (!MBB.isCleanupFuncletEntry()) { 1380 assert(Personality == EHPersonality::MSVC_CXX); 1381 unsigned FrameReg; 1382 int FI = MF.getWinEHFuncInfo()->EHRegNodeFrameIndex; 1383 int64_t EHRegOffset = getFrameIndexReference(MF, FI, FrameReg); 1384 // ESP is the first field, so no extra displacement is needed. 1385 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32mr)), FrameReg, 1386 false, EHRegOffset) 1387 .addReg(X86::ESP); 1388 } 1389 } 1390 1391 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) { 1392 const MachineInstr &FrameInstr = *MBBI; 1393 ++MBBI; 1394 1395 if (NeedsWinCFI) { 1396 int FI; 1397 if (unsigned Reg = TII.isStoreToStackSlot(FrameInstr, FI)) { 1398 if (X86::FR64RegClass.contains(Reg)) { 1399 int Offset; 1400 unsigned IgnoredFrameReg; 1401 if (IsWin64Prologue && IsFunclet) 1402 Offset = getWin64EHFrameIndexRef(MF, FI, IgnoredFrameReg); 1403 else 1404 Offset = getFrameIndexReference(MF, FI, IgnoredFrameReg) + 1405 SEHFrameOffset; 1406 1407 HasWinCFI = true; 1408 assert(!NeedsWinFPO && "SEH_SaveXMM incompatible with FPO data"); 1409 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SaveXMM)) 1410 .addImm(Reg) 1411 .addImm(Offset) 1412 .setMIFlag(MachineInstr::FrameSetup); 1413 } 1414 } 1415 } 1416 } 1417 1418 if (NeedsWinCFI && HasWinCFI) 1419 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_EndPrologue)) 1420 .setMIFlag(MachineInstr::FrameSetup); 1421 1422 if (FnHasClrFunclet && !IsFunclet) { 1423 // Save the so-called Initial-SP (i.e. the value of the stack pointer 1424 // immediately after the prolog) into the PSPSlot so that funclets 1425 // and the GC can recover it. 1426 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF); 1427 auto PSPInfo = MachinePointerInfo::getFixedStack( 1428 MF, MF.getWinEHFuncInfo()->PSPSymFrameIdx); 1429 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr, false, 1430 PSPSlotOffset) 1431 .addReg(StackPtr) 1432 .addMemOperand(MF.getMachineMemOperand( 1433 PSPInfo, MachineMemOperand::MOStore | MachineMemOperand::MOVolatile, 1434 SlotSize, SlotSize)); 1435 } 1436 1437 // Realign stack after we spilled callee-saved registers (so that we'll be 1438 // able to calculate their offsets from the frame pointer). 1439 // Win64 requires aligning the stack after the prologue. 1440 if (IsWin64Prologue && TRI->needsStackRealignment(MF)) { 1441 assert(HasFP && "There should be a frame pointer if stack is realigned."); 1442 BuildStackAlignAND(MBB, MBBI, DL, SPOrEstablisher, MaxAlign); 1443 } 1444 1445 // We already dealt with stack realignment and funclets above. 1446 if (IsFunclet && STI.is32Bit()) 1447 return; 1448 1449 // If we need a base pointer, set it up here. It's whatever the value 1450 // of the stack pointer is at this point. Any variable size objects 1451 // will be allocated after this, so we can still use the base pointer 1452 // to reference locals. 1453 if (TRI->hasBasePointer(MF)) { 1454 // Update the base pointer with the current stack pointer. 1455 unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr; 1456 BuildMI(MBB, MBBI, DL, TII.get(Opc), BasePtr) 1457 .addReg(SPOrEstablisher) 1458 .setMIFlag(MachineInstr::FrameSetup); 1459 if (X86FI->getRestoreBasePointer()) { 1460 // Stash value of base pointer. Saving RSP instead of EBP shortens 1461 // dependence chain. Used by SjLj EH. 1462 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr; 1463 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), 1464 FramePtr, true, X86FI->getRestoreBasePointerOffset()) 1465 .addReg(SPOrEstablisher) 1466 .setMIFlag(MachineInstr::FrameSetup); 1467 } 1468 1469 if (X86FI->getHasSEHFramePtrSave() && !IsFunclet) { 1470 // Stash the value of the frame pointer relative to the base pointer for 1471 // Win32 EH. This supports Win32 EH, which does the inverse of the above: 1472 // it recovers the frame pointer from the base pointer rather than the 1473 // other way around. 1474 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr; 1475 unsigned UsedReg; 1476 int Offset = 1477 getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg); 1478 assert(UsedReg == BasePtr); 1479 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), UsedReg, true, Offset) 1480 .addReg(FramePtr) 1481 .setMIFlag(MachineInstr::FrameSetup); 1482 } 1483 } 1484 1485 if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) { 1486 // Mark end of stack pointer adjustment. 1487 if (!HasFP && NumBytes) { 1488 // Define the current CFA rule to use the provided offset. 1489 assert(StackSize); 1490 BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createDefCfaOffset( 1491 nullptr, -StackSize + stackGrowth)); 1492 } 1493 1494 // Emit DWARF info specifying the offsets of the callee-saved registers. 1495 emitCalleeSavedFrameMoves(MBB, MBBI, DL); 1496 } 1497 1498 // X86 Interrupt handling function cannot assume anything about the direction 1499 // flag (DF in EFLAGS register). Clear this flag by creating "cld" instruction 1500 // in each prologue of interrupt handler function. 1501 // 1502 // FIXME: Create "cld" instruction only in these cases: 1503 // 1. The interrupt handling function uses any of the "rep" instructions. 1504 // 2. Interrupt handling function calls another function. 1505 // 1506 if (Fn.getCallingConv() == CallingConv::X86_INTR) 1507 BuildMI(MBB, MBBI, DL, TII.get(X86::CLD)) 1508 .setMIFlag(MachineInstr::FrameSetup); 1509 1510 // At this point we know if the function has WinCFI or not. 1511 MF.setHasWinCFI(HasWinCFI); 1512 } 1513 1514 bool X86FrameLowering::canUseLEAForSPInEpilogue( 1515 const MachineFunction &MF) const { 1516 // We can't use LEA instructions for adjusting the stack pointer if we don't 1517 // have a frame pointer in the Win64 ABI. Only ADD instructions may be used 1518 // to deallocate the stack. 1519 // This means that we can use LEA for SP in two situations: 1520 // 1. We *aren't* using the Win64 ABI which means we are free to use LEA. 1521 // 2. We *have* a frame pointer which means we are permitted to use LEA. 1522 return !MF.getTarget().getMCAsmInfo()->usesWindowsCFI() || hasFP(MF); 1523 } 1524 1525 static bool isFuncletReturnInstr(MachineInstr &MI) { 1526 switch (MI.getOpcode()) { 1527 case X86::CATCHRET: 1528 case X86::CLEANUPRET: 1529 return true; 1530 default: 1531 return false; 1532 } 1533 llvm_unreachable("impossible"); 1534 } 1535 1536 // CLR funclets use a special "Previous Stack Pointer Symbol" slot on the 1537 // stack. It holds a pointer to the bottom of the root function frame. The 1538 // establisher frame pointer passed to a nested funclet may point to the 1539 // (mostly empty) frame of its parent funclet, but it will need to find 1540 // the frame of the root function to access locals. To facilitate this, 1541 // every funclet copies the pointer to the bottom of the root function 1542 // frame into a PSPSym slot in its own (mostly empty) stack frame. Using the 1543 // same offset for the PSPSym in the root function frame that's used in the 1544 // funclets' frames allows each funclet to dynamically accept any ancestor 1545 // frame as its establisher argument (the runtime doesn't guarantee the 1546 // immediate parent for some reason lost to history), and also allows the GC, 1547 // which uses the PSPSym for some bookkeeping, to find it in any funclet's 1548 // frame with only a single offset reported for the entire method. 1549 unsigned 1550 X86FrameLowering::getPSPSlotOffsetFromSP(const MachineFunction &MF) const { 1551 const WinEHFuncInfo &Info = *MF.getWinEHFuncInfo(); 1552 unsigned SPReg; 1553 int Offset = getFrameIndexReferencePreferSP(MF, Info.PSPSymFrameIdx, SPReg, 1554 /*IgnoreSPUpdates*/ true); 1555 assert(Offset >= 0 && SPReg == TRI->getStackRegister()); 1556 return static_cast<unsigned>(Offset); 1557 } 1558 1559 unsigned 1560 X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const { 1561 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 1562 // This is the size of the pushed CSRs. 1563 unsigned CSSize = X86FI->getCalleeSavedFrameSize(); 1564 // This is the size of callee saved XMMs. 1565 const auto& WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo(); 1566 unsigned XMMSize = WinEHXMMSlotInfo.size() * 1567 TRI->getSpillSize(X86::VR128RegClass); 1568 // This is the amount of stack a funclet needs to allocate. 1569 unsigned UsedSize; 1570 EHPersonality Personality = 1571 classifyEHPersonality(MF.getFunction().getPersonalityFn()); 1572 if (Personality == EHPersonality::CoreCLR) { 1573 // CLR funclets need to hold enough space to include the PSPSym, at the 1574 // same offset from the stack pointer (immediately after the prolog) as it 1575 // resides at in the main function. 1576 UsedSize = getPSPSlotOffsetFromSP(MF) + SlotSize; 1577 } else { 1578 // Other funclets just need enough stack for outgoing call arguments. 1579 UsedSize = MF.getFrameInfo().getMaxCallFrameSize(); 1580 } 1581 // RBP is not included in the callee saved register block. After pushing RBP, 1582 // everything is 16 byte aligned. Everything we allocate before an outgoing 1583 // call must also be 16 byte aligned. 1584 unsigned FrameSizeMinusRBP = alignTo(CSSize + UsedSize, getStackAlignment()); 1585 // Subtract out the size of the callee saved registers. This is how much stack 1586 // each funclet will allocate. 1587 return FrameSizeMinusRBP + XMMSize - CSSize; 1588 } 1589 1590 static bool isTailCallOpcode(unsigned Opc) { 1591 return Opc == X86::TCRETURNri || Opc == X86::TCRETURNdi || 1592 Opc == X86::TCRETURNmi || 1593 Opc == X86::TCRETURNri64 || Opc == X86::TCRETURNdi64 || 1594 Opc == X86::TCRETURNmi64; 1595 } 1596 1597 void X86FrameLowering::emitEpilogue(MachineFunction &MF, 1598 MachineBasicBlock &MBB) const { 1599 const MachineFrameInfo &MFI = MF.getFrameInfo(); 1600 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 1601 MachineBasicBlock::iterator Terminator = MBB.getFirstTerminator(); 1602 MachineBasicBlock::iterator MBBI = Terminator; 1603 DebugLoc DL; 1604 if (MBBI != MBB.end()) 1605 DL = MBBI->getDebugLoc(); 1606 // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit. 1607 const bool Is64BitILP32 = STI.isTarget64BitILP32(); 1608 unsigned FramePtr = TRI->getFrameRegister(MF); 1609 unsigned MachineFramePtr = 1610 Is64BitILP32 ? getX86SubSuperRegister(FramePtr, 64) : FramePtr; 1611 1612 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 1613 bool NeedsWin64CFI = 1614 IsWin64Prologue && MF.getFunction().needsUnwindTableEntry(); 1615 bool IsFunclet = MBBI == MBB.end() ? false : isFuncletReturnInstr(*MBBI); 1616 1617 // Get the number of bytes to allocate from the FrameInfo. 1618 uint64_t StackSize = MFI.getStackSize(); 1619 uint64_t MaxAlign = calculateMaxStackAlign(MF); 1620 unsigned CSSize = X86FI->getCalleeSavedFrameSize(); 1621 bool HasFP = hasFP(MF); 1622 uint64_t NumBytes = 0; 1623 1624 bool NeedsDwarfCFI = 1625 (!MF.getTarget().getTargetTriple().isOSDarwin() && 1626 !MF.getTarget().getTargetTriple().isOSWindows()) && 1627 (MF.getMMI().hasDebugInfo() || MF.getFunction().needsUnwindTableEntry()); 1628 1629 if (IsFunclet) { 1630 assert(HasFP && "EH funclets without FP not yet implemented"); 1631 NumBytes = getWinEHFuncletFrameSize(MF); 1632 } else if (HasFP) { 1633 // Calculate required stack adjustment. 1634 uint64_t FrameSize = StackSize - SlotSize; 1635 NumBytes = FrameSize - CSSize; 1636 1637 // Callee-saved registers were pushed on stack before the stack was 1638 // realigned. 1639 if (TRI->needsStackRealignment(MF) && !IsWin64Prologue) 1640 NumBytes = alignTo(FrameSize, MaxAlign); 1641 } else { 1642 NumBytes = StackSize - CSSize; 1643 } 1644 uint64_t SEHStackAllocAmt = NumBytes; 1645 1646 if (HasFP) { 1647 // Pop EBP. 1648 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::POP64r : X86::POP32r), 1649 MachineFramePtr) 1650 .setMIFlag(MachineInstr::FrameDestroy); 1651 if (NeedsDwarfCFI) { 1652 unsigned DwarfStackPtr = 1653 TRI->getDwarfRegNum(Is64Bit ? X86::RSP : X86::ESP, true); 1654 BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createDefCfa( 1655 nullptr, DwarfStackPtr, -SlotSize)); 1656 --MBBI; 1657 } 1658 } 1659 1660 MachineBasicBlock::iterator FirstCSPop = MBBI; 1661 // Skip the callee-saved pop instructions. 1662 while (MBBI != MBB.begin()) { 1663 MachineBasicBlock::iterator PI = std::prev(MBBI); 1664 unsigned Opc = PI->getOpcode(); 1665 1666 if (Opc != X86::DBG_VALUE && !PI->isTerminator()) { 1667 if ((Opc != X86::POP32r || !PI->getFlag(MachineInstr::FrameDestroy)) && 1668 (Opc != X86::POP64r || !PI->getFlag(MachineInstr::FrameDestroy))) 1669 break; 1670 FirstCSPop = PI; 1671 } 1672 1673 --MBBI; 1674 } 1675 MBBI = FirstCSPop; 1676 1677 if (IsFunclet && Terminator->getOpcode() == X86::CATCHRET) 1678 emitCatchRetReturnValue(MBB, FirstCSPop, &*Terminator); 1679 1680 if (MBBI != MBB.end()) 1681 DL = MBBI->getDebugLoc(); 1682 1683 // If there is an ADD32ri or SUB32ri of ESP immediately before this 1684 // instruction, merge the two instructions. 1685 if (NumBytes || MFI.hasVarSizedObjects()) 1686 NumBytes += mergeSPUpdates(MBB, MBBI, true); 1687 1688 // If dynamic alloca is used, then reset esp to point to the last callee-saved 1689 // slot before popping them off! Same applies for the case, when stack was 1690 // realigned. Don't do this if this was a funclet epilogue, since the funclets 1691 // will not do realignment or dynamic stack allocation. 1692 if ((TRI->needsStackRealignment(MF) || MFI.hasVarSizedObjects()) && 1693 !IsFunclet) { 1694 if (TRI->needsStackRealignment(MF)) 1695 MBBI = FirstCSPop; 1696 unsigned SEHFrameOffset = calculateSetFPREG(SEHStackAllocAmt); 1697 uint64_t LEAAmount = 1698 IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize; 1699 1700 // There are only two legal forms of epilogue: 1701 // - add SEHAllocationSize, %rsp 1702 // - lea SEHAllocationSize(%FramePtr), %rsp 1703 // 1704 // 'mov %FramePtr, %rsp' will not be recognized as an epilogue sequence. 1705 // However, we may use this sequence if we have a frame pointer because the 1706 // effects of the prologue can safely be undone. 1707 if (LEAAmount != 0) { 1708 unsigned Opc = getLEArOpcode(Uses64BitFramePtr); 1709 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr), 1710 FramePtr, false, LEAAmount); 1711 --MBBI; 1712 } else { 1713 unsigned Opc = (Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr); 1714 BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr) 1715 .addReg(FramePtr); 1716 --MBBI; 1717 } 1718 } else if (NumBytes) { 1719 // Adjust stack pointer back: ESP += numbytes. 1720 emitSPUpdate(MBB, MBBI, DL, NumBytes, /*InEpilogue=*/true); 1721 if (!hasFP(MF) && NeedsDwarfCFI) { 1722 // Define the current CFA rule to use the provided offset. 1723 BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createDefCfaOffset( 1724 nullptr, -CSSize - SlotSize)); 1725 } 1726 --MBBI; 1727 } 1728 1729 // Windows unwinder will not invoke function's exception handler if IP is 1730 // either in prologue or in epilogue. This behavior causes a problem when a 1731 // call immediately precedes an epilogue, because the return address points 1732 // into the epilogue. To cope with that, we insert an epilogue marker here, 1733 // then replace it with a 'nop' if it ends up immediately after a CALL in the 1734 // final emitted code. 1735 if (NeedsWin64CFI && MF.hasWinCFI()) 1736 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_Epilogue)); 1737 1738 if (!hasFP(MF) && NeedsDwarfCFI) { 1739 MBBI = FirstCSPop; 1740 int64_t Offset = -CSSize - SlotSize; 1741 // Mark callee-saved pop instruction. 1742 // Define the current CFA rule to use the provided offset. 1743 while (MBBI != MBB.end()) { 1744 MachineBasicBlock::iterator PI = MBBI; 1745 unsigned Opc = PI->getOpcode(); 1746 ++MBBI; 1747 if (Opc == X86::POP32r || Opc == X86::POP64r) { 1748 Offset += SlotSize; 1749 BuildCFI(MBB, MBBI, DL, 1750 MCCFIInstruction::createDefCfaOffset(nullptr, Offset)); 1751 } 1752 } 1753 } 1754 1755 if (Terminator == MBB.end() || !isTailCallOpcode(Terminator->getOpcode())) { 1756 // Add the return addr area delta back since we are not tail calling. 1757 int Offset = -1 * X86FI->getTCReturnAddrDelta(); 1758 assert(Offset >= 0 && "TCDelta should never be positive"); 1759 if (Offset) { 1760 // Check for possible merge with preceding ADD instruction. 1761 Offset += mergeSPUpdates(MBB, Terminator, true); 1762 emitSPUpdate(MBB, Terminator, DL, Offset, /*InEpilogue=*/true); 1763 } 1764 } 1765 } 1766 1767 int X86FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI, 1768 unsigned &FrameReg) const { 1769 const MachineFrameInfo &MFI = MF.getFrameInfo(); 1770 1771 bool IsFixed = MFI.isFixedObjectIndex(FI); 1772 // We can't calculate offset from frame pointer if the stack is realigned, 1773 // so enforce usage of stack/base pointer. The base pointer is used when we 1774 // have dynamic allocas in addition to dynamic realignment. 1775 if (TRI->hasBasePointer(MF)) 1776 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getBaseRegister(); 1777 else if (TRI->needsStackRealignment(MF)) 1778 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getStackRegister(); 1779 else 1780 FrameReg = TRI->getFrameRegister(MF); 1781 1782 // Offset will hold the offset from the stack pointer at function entry to the 1783 // object. 1784 // We need to factor in additional offsets applied during the prologue to the 1785 // frame, base, and stack pointer depending on which is used. 1786 int Offset = MFI.getObjectOffset(FI) - getOffsetOfLocalArea(); 1787 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 1788 unsigned CSSize = X86FI->getCalleeSavedFrameSize(); 1789 uint64_t StackSize = MFI.getStackSize(); 1790 bool HasFP = hasFP(MF); 1791 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 1792 int64_t FPDelta = 0; 1793 1794 // In an x86 interrupt, remove the offset we added to account for the return 1795 // address from any stack object allocated in the caller's frame. Interrupts 1796 // do not have a standard return address. Fixed objects in the current frame, 1797 // such as SSE register spills, should not get this treatment. 1798 if (MF.getFunction().getCallingConv() == CallingConv::X86_INTR && 1799 Offset >= 0) { 1800 Offset += getOffsetOfLocalArea(); 1801 } 1802 1803 if (IsWin64Prologue) { 1804 assert(!MFI.hasCalls() || (StackSize % 16) == 8); 1805 1806 // Calculate required stack adjustment. 1807 uint64_t FrameSize = StackSize - SlotSize; 1808 // If required, include space for extra hidden slot for stashing base pointer. 1809 if (X86FI->getRestoreBasePointer()) 1810 FrameSize += SlotSize; 1811 uint64_t NumBytes = FrameSize - CSSize; 1812 1813 uint64_t SEHFrameOffset = calculateSetFPREG(NumBytes); 1814 if (FI && FI == X86FI->getFAIndex()) 1815 return -SEHFrameOffset; 1816 1817 // FPDelta is the offset from the "traditional" FP location of the old base 1818 // pointer followed by return address and the location required by the 1819 // restricted Win64 prologue. 1820 // Add FPDelta to all offsets below that go through the frame pointer. 1821 FPDelta = FrameSize - SEHFrameOffset; 1822 assert((!MFI.hasCalls() || (FPDelta % 16) == 0) && 1823 "FPDelta isn't aligned per the Win64 ABI!"); 1824 } 1825 1826 1827 if (TRI->hasBasePointer(MF)) { 1828 assert(HasFP && "VLAs and dynamic stack realign, but no FP?!"); 1829 if (FI < 0) { 1830 // Skip the saved EBP. 1831 return Offset + SlotSize + FPDelta; 1832 } else { 1833 assert((-(Offset + StackSize)) % MFI.getObjectAlignment(FI) == 0); 1834 return Offset + StackSize; 1835 } 1836 } else if (TRI->needsStackRealignment(MF)) { 1837 if (FI < 0) { 1838 // Skip the saved EBP. 1839 return Offset + SlotSize + FPDelta; 1840 } else { 1841 assert((-(Offset + StackSize)) % MFI.getObjectAlignment(FI) == 0); 1842 return Offset + StackSize; 1843 } 1844 // FIXME: Support tail calls 1845 } else { 1846 if (!HasFP) 1847 return Offset + StackSize; 1848 1849 // Skip the saved EBP. 1850 Offset += SlotSize; 1851 1852 // Skip the RETADDR move area 1853 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta(); 1854 if (TailCallReturnAddrDelta < 0) 1855 Offset -= TailCallReturnAddrDelta; 1856 } 1857 1858 return Offset + FPDelta; 1859 } 1860 1861 int X86FrameLowering::getWin64EHFrameIndexRef(const MachineFunction &MF, 1862 int FI, unsigned &FrameReg) const { 1863 const MachineFrameInfo &MFI = MF.getFrameInfo(); 1864 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 1865 const auto& WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo(); 1866 const auto it = WinEHXMMSlotInfo.find(FI); 1867 1868 if (it == WinEHXMMSlotInfo.end()) 1869 return getFrameIndexReference(MF, FI, FrameReg); 1870 1871 FrameReg = TRI->getStackRegister(); 1872 return alignDown(MFI.getMaxCallFrameSize(), getStackAlignment()) + it->second; 1873 } 1874 1875 int X86FrameLowering::getFrameIndexReferenceSP(const MachineFunction &MF, 1876 int FI, unsigned &FrameReg, 1877 int Adjustment) const { 1878 const MachineFrameInfo &MFI = MF.getFrameInfo(); 1879 FrameReg = TRI->getStackRegister(); 1880 return MFI.getObjectOffset(FI) - getOffsetOfLocalArea() + Adjustment; 1881 } 1882 1883 int 1884 X86FrameLowering::getFrameIndexReferencePreferSP(const MachineFunction &MF, 1885 int FI, unsigned &FrameReg, 1886 bool IgnoreSPUpdates) const { 1887 1888 const MachineFrameInfo &MFI = MF.getFrameInfo(); 1889 // Does not include any dynamic realign. 1890 const uint64_t StackSize = MFI.getStackSize(); 1891 // LLVM arranges the stack as follows: 1892 // ... 1893 // ARG2 1894 // ARG1 1895 // RETADDR 1896 // PUSH RBP <-- RBP points here 1897 // PUSH CSRs 1898 // ~~~~~~~ <-- possible stack realignment (non-win64) 1899 // ... 1900 // STACK OBJECTS 1901 // ... <-- RSP after prologue points here 1902 // ~~~~~~~ <-- possible stack realignment (win64) 1903 // 1904 // if (hasVarSizedObjects()): 1905 // ... <-- "base pointer" (ESI/RBX) points here 1906 // DYNAMIC ALLOCAS 1907 // ... <-- RSP points here 1908 // 1909 // Case 1: In the simple case of no stack realignment and no dynamic 1910 // allocas, both "fixed" stack objects (arguments and CSRs) are addressable 1911 // with fixed offsets from RSP. 1912 // 1913 // Case 2: In the case of stack realignment with no dynamic allocas, fixed 1914 // stack objects are addressed with RBP and regular stack objects with RSP. 1915 // 1916 // Case 3: In the case of dynamic allocas and stack realignment, RSP is used 1917 // to address stack arguments for outgoing calls and nothing else. The "base 1918 // pointer" points to local variables, and RBP points to fixed objects. 1919 // 1920 // In cases 2 and 3, we can only answer for non-fixed stack objects, and the 1921 // answer we give is relative to the SP after the prologue, and not the 1922 // SP in the middle of the function. 1923 1924 if (MFI.isFixedObjectIndex(FI) && TRI->needsStackRealignment(MF) && 1925 !STI.isTargetWin64()) 1926 return getFrameIndexReference(MF, FI, FrameReg); 1927 1928 // If !hasReservedCallFrame the function might have SP adjustement in the 1929 // body. So, even though the offset is statically known, it depends on where 1930 // we are in the function. 1931 if (!IgnoreSPUpdates && !hasReservedCallFrame(MF)) 1932 return getFrameIndexReference(MF, FI, FrameReg); 1933 1934 // We don't handle tail calls, and shouldn't be seeing them either. 1935 assert(MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta() >= 0 && 1936 "we don't handle this case!"); 1937 1938 // This is how the math works out: 1939 // 1940 // %rsp grows (i.e. gets lower) left to right. Each box below is 1941 // one word (eight bytes). Obj0 is the stack slot we're trying to 1942 // get to. 1943 // 1944 // ---------------------------------- 1945 // | BP | Obj0 | Obj1 | ... | ObjN | 1946 // ---------------------------------- 1947 // ^ ^ ^ ^ 1948 // A B C E 1949 // 1950 // A is the incoming stack pointer. 1951 // (B - A) is the local area offset (-8 for x86-64) [1] 1952 // (C - A) is the Offset returned by MFI.getObjectOffset for Obj0 [2] 1953 // 1954 // |(E - B)| is the StackSize (absolute value, positive). For a 1955 // stack that grown down, this works out to be (B - E). [3] 1956 // 1957 // E is also the value of %rsp after stack has been set up, and we 1958 // want (C - E) -- the value we can add to %rsp to get to Obj0. Now 1959 // (C - E) == (C - A) - (B - A) + (B - E) 1960 // { Using [1], [2] and [3] above } 1961 // == getObjectOffset - LocalAreaOffset + StackSize 1962 1963 return getFrameIndexReferenceSP(MF, FI, FrameReg, StackSize); 1964 } 1965 1966 bool X86FrameLowering::assignCalleeSavedSpillSlots( 1967 MachineFunction &MF, const TargetRegisterInfo *TRI, 1968 std::vector<CalleeSavedInfo> &CSI) const { 1969 MachineFrameInfo &MFI = MF.getFrameInfo(); 1970 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 1971 1972 unsigned CalleeSavedFrameSize = 0; 1973 unsigned XMMCalleeSavedFrameSize = 0; 1974 auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo(); 1975 int SpillSlotOffset = getOffsetOfLocalArea() + X86FI->getTCReturnAddrDelta(); 1976 1977 int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta(); 1978 1979 if (TailCallReturnAddrDelta < 0) { 1980 // create RETURNADDR area 1981 // arg 1982 // arg 1983 // RETADDR 1984 // { ... 1985 // RETADDR area 1986 // ... 1987 // } 1988 // [EBP] 1989 MFI.CreateFixedObject(-TailCallReturnAddrDelta, 1990 TailCallReturnAddrDelta - SlotSize, true); 1991 } 1992 1993 // Spill the BasePtr if it's used. 1994 if (this->TRI->hasBasePointer(MF)) { 1995 // Allocate a spill slot for EBP if we have a base pointer and EH funclets. 1996 if (MF.hasEHFunclets()) { 1997 int FI = MFI.CreateSpillStackObject(SlotSize, SlotSize); 1998 X86FI->setHasSEHFramePtrSave(true); 1999 X86FI->setSEHFramePtrSaveIndex(FI); 2000 } 2001 } 2002 2003 if (hasFP(MF)) { 2004 // emitPrologue always spills frame register the first thing. 2005 SpillSlotOffset -= SlotSize; 2006 MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset); 2007 2008 // Since emitPrologue and emitEpilogue will handle spilling and restoring of 2009 // the frame register, we can delete it from CSI list and not have to worry 2010 // about avoiding it later. 2011 unsigned FPReg = TRI->getFrameRegister(MF); 2012 for (unsigned i = 0; i < CSI.size(); ++i) { 2013 if (TRI->regsOverlap(CSI[i].getReg(),FPReg)) { 2014 CSI.erase(CSI.begin() + i); 2015 break; 2016 } 2017 } 2018 } 2019 2020 // Assign slots for GPRs. It increases frame size. 2021 for (unsigned i = CSI.size(); i != 0; --i) { 2022 unsigned Reg = CSI[i - 1].getReg(); 2023 2024 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg)) 2025 continue; 2026 2027 SpillSlotOffset -= SlotSize; 2028 CalleeSavedFrameSize += SlotSize; 2029 2030 int SlotIndex = MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset); 2031 CSI[i - 1].setFrameIdx(SlotIndex); 2032 } 2033 2034 X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize); 2035 MFI.setCVBytesOfCalleeSavedRegisters(CalleeSavedFrameSize); 2036 2037 // Assign slots for XMMs. 2038 for (unsigned i = CSI.size(); i != 0; --i) { 2039 unsigned Reg = CSI[i - 1].getReg(); 2040 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg)) 2041 continue; 2042 2043 // If this is k-register make sure we lookup via the largest legal type. 2044 MVT VT = MVT::Other; 2045 if (X86::VK16RegClass.contains(Reg)) 2046 VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1; 2047 2048 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT); 2049 unsigned Size = TRI->getSpillSize(*RC); 2050 unsigned Align = TRI->getSpillAlignment(*RC); 2051 // ensure alignment 2052 assert(SpillSlotOffset < 0 && "SpillSlotOffset should always < 0 on X86"); 2053 SpillSlotOffset = -alignTo(-SpillSlotOffset, Align); 2054 2055 // spill into slot 2056 SpillSlotOffset -= Size; 2057 int SlotIndex = MFI.CreateFixedSpillStackObject(Size, SpillSlotOffset); 2058 CSI[i - 1].setFrameIdx(SlotIndex); 2059 MFI.ensureMaxAlignment(Align); 2060 2061 // Save the start offset and size of XMM in stack frame for funclets. 2062 if (X86::VR128RegClass.contains(Reg)) { 2063 WinEHXMMSlotInfo[SlotIndex] = XMMCalleeSavedFrameSize; 2064 XMMCalleeSavedFrameSize += Size; 2065 } 2066 } 2067 2068 return true; 2069 } 2070 2071 bool X86FrameLowering::spillCalleeSavedRegisters( 2072 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, 2073 const std::vector<CalleeSavedInfo> &CSI, 2074 const TargetRegisterInfo *TRI) const { 2075 DebugLoc DL = MBB.findDebugLoc(MI); 2076 2077 // Don't save CSRs in 32-bit EH funclets. The caller saves EBX, EBP, ESI, EDI 2078 // for us, and there are no XMM CSRs on Win32. 2079 if (MBB.isEHFuncletEntry() && STI.is32Bit() && STI.isOSWindows()) 2080 return true; 2081 2082 // Push GPRs. It increases frame size. 2083 const MachineFunction &MF = *MBB.getParent(); 2084 unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r; 2085 for (unsigned i = CSI.size(); i != 0; --i) { 2086 unsigned Reg = CSI[i - 1].getReg(); 2087 2088 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg)) 2089 continue; 2090 2091 const MachineRegisterInfo &MRI = MF.getRegInfo(); 2092 bool isLiveIn = MRI.isLiveIn(Reg); 2093 if (!isLiveIn) 2094 MBB.addLiveIn(Reg); 2095 2096 // Decide whether we can add a kill flag to the use. 2097 bool CanKill = !isLiveIn; 2098 // Check if any subregister is live-in 2099 if (CanKill) { 2100 for (MCRegAliasIterator AReg(Reg, TRI, false); AReg.isValid(); ++AReg) { 2101 if (MRI.isLiveIn(*AReg)) { 2102 CanKill = false; 2103 break; 2104 } 2105 } 2106 } 2107 2108 // Do not set a kill flag on values that are also marked as live-in. This 2109 // happens with the @llvm-returnaddress intrinsic and with arguments 2110 // passed in callee saved registers. 2111 // Omitting the kill flags is conservatively correct even if the live-in 2112 // is not used after all. 2113 BuildMI(MBB, MI, DL, TII.get(Opc)).addReg(Reg, getKillRegState(CanKill)) 2114 .setMIFlag(MachineInstr::FrameSetup); 2115 } 2116 2117 // Make XMM regs spilled. X86 does not have ability of push/pop XMM. 2118 // It can be done by spilling XMMs to stack frame. 2119 for (unsigned i = CSI.size(); i != 0; --i) { 2120 unsigned Reg = CSI[i-1].getReg(); 2121 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg)) 2122 continue; 2123 2124 // If this is k-register make sure we lookup via the largest legal type. 2125 MVT VT = MVT::Other; 2126 if (X86::VK16RegClass.contains(Reg)) 2127 VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1; 2128 2129 // Add the callee-saved register as live-in. It's killed at the spill. 2130 MBB.addLiveIn(Reg); 2131 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT); 2132 2133 TII.storeRegToStackSlot(MBB, MI, Reg, true, CSI[i - 1].getFrameIdx(), RC, 2134 TRI); 2135 --MI; 2136 MI->setFlag(MachineInstr::FrameSetup); 2137 ++MI; 2138 } 2139 2140 return true; 2141 } 2142 2143 void X86FrameLowering::emitCatchRetReturnValue(MachineBasicBlock &MBB, 2144 MachineBasicBlock::iterator MBBI, 2145 MachineInstr *CatchRet) const { 2146 // SEH shouldn't use catchret. 2147 assert(!isAsynchronousEHPersonality(classifyEHPersonality( 2148 MBB.getParent()->getFunction().getPersonalityFn())) && 2149 "SEH should not use CATCHRET"); 2150 DebugLoc DL = CatchRet->getDebugLoc(); 2151 MachineBasicBlock *CatchRetTarget = CatchRet->getOperand(0).getMBB(); 2152 2153 // Fill EAX/RAX with the address of the target block. 2154 if (STI.is64Bit()) { 2155 // LEA64r CatchRetTarget(%rip), %rax 2156 BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), X86::RAX) 2157 .addReg(X86::RIP) 2158 .addImm(0) 2159 .addReg(0) 2160 .addMBB(CatchRetTarget) 2161 .addReg(0); 2162 } else { 2163 // MOV32ri $CatchRetTarget, %eax 2164 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX) 2165 .addMBB(CatchRetTarget); 2166 } 2167 2168 // Record that we've taken the address of CatchRetTarget and no longer just 2169 // reference it in a terminator. 2170 CatchRetTarget->setHasAddressTaken(); 2171 } 2172 2173 bool X86FrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB, 2174 MachineBasicBlock::iterator MI, 2175 std::vector<CalleeSavedInfo> &CSI, 2176 const TargetRegisterInfo *TRI) const { 2177 if (CSI.empty()) 2178 return false; 2179 2180 if (MI != MBB.end() && isFuncletReturnInstr(*MI) && STI.isOSWindows()) { 2181 // Don't restore CSRs in 32-bit EH funclets. Matches 2182 // spillCalleeSavedRegisters. 2183 if (STI.is32Bit()) 2184 return true; 2185 // Don't restore CSRs before an SEH catchret. SEH except blocks do not form 2186 // funclets. emitEpilogue transforms these to normal jumps. 2187 if (MI->getOpcode() == X86::CATCHRET) { 2188 const Function &F = MBB.getParent()->getFunction(); 2189 bool IsSEH = isAsynchronousEHPersonality( 2190 classifyEHPersonality(F.getPersonalityFn())); 2191 if (IsSEH) 2192 return true; 2193 } 2194 } 2195 2196 DebugLoc DL = MBB.findDebugLoc(MI); 2197 2198 // Reload XMMs from stack frame. 2199 for (unsigned i = 0, e = CSI.size(); i != e; ++i) { 2200 unsigned Reg = CSI[i].getReg(); 2201 if (X86::GR64RegClass.contains(Reg) || 2202 X86::GR32RegClass.contains(Reg)) 2203 continue; 2204 2205 // If this is k-register make sure we lookup via the largest legal type. 2206 MVT VT = MVT::Other; 2207 if (X86::VK16RegClass.contains(Reg)) 2208 VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1; 2209 2210 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT); 2211 TII.loadRegFromStackSlot(MBB, MI, Reg, CSI[i].getFrameIdx(), RC, TRI); 2212 } 2213 2214 // POP GPRs. 2215 unsigned Opc = STI.is64Bit() ? X86::POP64r : X86::POP32r; 2216 for (unsigned i = 0, e = CSI.size(); i != e; ++i) { 2217 unsigned Reg = CSI[i].getReg(); 2218 if (!X86::GR64RegClass.contains(Reg) && 2219 !X86::GR32RegClass.contains(Reg)) 2220 continue; 2221 2222 BuildMI(MBB, MI, DL, TII.get(Opc), Reg) 2223 .setMIFlag(MachineInstr::FrameDestroy); 2224 } 2225 return true; 2226 } 2227 2228 void X86FrameLowering::determineCalleeSaves(MachineFunction &MF, 2229 BitVector &SavedRegs, 2230 RegScavenger *RS) const { 2231 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS); 2232 2233 // Spill the BasePtr if it's used. 2234 if (TRI->hasBasePointer(MF)){ 2235 unsigned BasePtr = TRI->getBaseRegister(); 2236 if (STI.isTarget64BitILP32()) 2237 BasePtr = getX86SubSuperRegister(BasePtr, 64); 2238 SavedRegs.set(BasePtr); 2239 } 2240 } 2241 2242 static bool 2243 HasNestArgument(const MachineFunction *MF) { 2244 const Function &F = MF->getFunction(); 2245 for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); 2246 I != E; I++) { 2247 if (I->hasNestAttr()) 2248 return true; 2249 } 2250 return false; 2251 } 2252 2253 /// GetScratchRegister - Get a temp register for performing work in the 2254 /// segmented stack and the Erlang/HiPE stack prologue. Depending on platform 2255 /// and the properties of the function either one or two registers will be 2256 /// needed. Set primary to true for the first register, false for the second. 2257 static unsigned 2258 GetScratchRegister(bool Is64Bit, bool IsLP64, const MachineFunction &MF, bool Primary) { 2259 CallingConv::ID CallingConvention = MF.getFunction().getCallingConv(); 2260 2261 // Erlang stuff. 2262 if (CallingConvention == CallingConv::HiPE) { 2263 if (Is64Bit) 2264 return Primary ? X86::R14 : X86::R13; 2265 else 2266 return Primary ? X86::EBX : X86::EDI; 2267 } 2268 2269 if (Is64Bit) { 2270 if (IsLP64) 2271 return Primary ? X86::R11 : X86::R12; 2272 else 2273 return Primary ? X86::R11D : X86::R12D; 2274 } 2275 2276 bool IsNested = HasNestArgument(&MF); 2277 2278 if (CallingConvention == CallingConv::X86_FastCall || 2279 CallingConvention == CallingConv::Fast) { 2280 if (IsNested) 2281 report_fatal_error("Segmented stacks does not support fastcall with " 2282 "nested function."); 2283 return Primary ? X86::EAX : X86::ECX; 2284 } 2285 if (IsNested) 2286 return Primary ? X86::EDX : X86::EAX; 2287 return Primary ? X86::ECX : X86::EAX; 2288 } 2289 2290 // The stack limit in the TCB is set to this many bytes above the actual stack 2291 // limit. 2292 static const uint64_t kSplitStackAvailable = 256; 2293 2294 void X86FrameLowering::adjustForSegmentedStacks( 2295 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const { 2296 MachineFrameInfo &MFI = MF.getFrameInfo(); 2297 uint64_t StackSize; 2298 unsigned TlsReg, TlsOffset; 2299 DebugLoc DL; 2300 2301 // To support shrink-wrapping we would need to insert the new blocks 2302 // at the right place and update the branches to PrologueMBB. 2303 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet"); 2304 2305 unsigned ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true); 2306 assert(!MF.getRegInfo().isLiveIn(ScratchReg) && 2307 "Scratch register is live-in"); 2308 2309 if (MF.getFunction().isVarArg()) 2310 report_fatal_error("Segmented stacks do not support vararg functions."); 2311 if (!STI.isTargetLinux() && !STI.isTargetDarwin() && !STI.isTargetWin32() && 2312 !STI.isTargetWin64() && !STI.isTargetFreeBSD() && 2313 !STI.isTargetDragonFly()) 2314 report_fatal_error("Segmented stacks not supported on this platform."); 2315 2316 // Eventually StackSize will be calculated by a link-time pass; which will 2317 // also decide whether checking code needs to be injected into this particular 2318 // prologue. 2319 StackSize = MFI.getStackSize(); 2320 2321 // Do not generate a prologue for leaf functions with a stack of size zero. 2322 // For non-leaf functions we have to allow for the possibility that the 2323 // callis to a non-split function, as in PR37807. This function could also 2324 // take the address of a non-split function. When the linker tries to adjust 2325 // its non-existent prologue, it would fail with an error. Mark the object 2326 // file so that such failures are not errors. See this Go language bug-report 2327 // https://go-review.googlesource.com/c/go/+/148819/ 2328 if (StackSize == 0 && !MFI.hasTailCall()) { 2329 MF.getMMI().setHasNosplitStack(true); 2330 return; 2331 } 2332 2333 MachineBasicBlock *allocMBB = MF.CreateMachineBasicBlock(); 2334 MachineBasicBlock *checkMBB = MF.CreateMachineBasicBlock(); 2335 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 2336 bool IsNested = false; 2337 2338 // We need to know if the function has a nest argument only in 64 bit mode. 2339 if (Is64Bit) 2340 IsNested = HasNestArgument(&MF); 2341 2342 // The MOV R10, RAX needs to be in a different block, since the RET we emit in 2343 // allocMBB needs to be last (terminating) instruction. 2344 2345 for (const auto &LI : PrologueMBB.liveins()) { 2346 allocMBB->addLiveIn(LI); 2347 checkMBB->addLiveIn(LI); 2348 } 2349 2350 if (IsNested) 2351 allocMBB->addLiveIn(IsLP64 ? X86::R10 : X86::R10D); 2352 2353 MF.push_front(allocMBB); 2354 MF.push_front(checkMBB); 2355 2356 // When the frame size is less than 256 we just compare the stack 2357 // boundary directly to the value of the stack pointer, per gcc. 2358 bool CompareStackPointer = StackSize < kSplitStackAvailable; 2359 2360 // Read the limit off the current stacklet off the stack_guard location. 2361 if (Is64Bit) { 2362 if (STI.isTargetLinux()) { 2363 TlsReg = X86::FS; 2364 TlsOffset = IsLP64 ? 0x70 : 0x40; 2365 } else if (STI.isTargetDarwin()) { 2366 TlsReg = X86::GS; 2367 TlsOffset = 0x60 + 90*8; // See pthread_machdep.h. Steal TLS slot 90. 2368 } else if (STI.isTargetWin64()) { 2369 TlsReg = X86::GS; 2370 TlsOffset = 0x28; // pvArbitrary, reserved for application use 2371 } else if (STI.isTargetFreeBSD()) { 2372 TlsReg = X86::FS; 2373 TlsOffset = 0x18; 2374 } else if (STI.isTargetDragonFly()) { 2375 TlsReg = X86::FS; 2376 TlsOffset = 0x20; // use tls_tcb.tcb_segstack 2377 } else { 2378 report_fatal_error("Segmented stacks not supported on this platform."); 2379 } 2380 2381 if (CompareStackPointer) 2382 ScratchReg = IsLP64 ? X86::RSP : X86::ESP; 2383 else 2384 BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::LEA64r : X86::LEA64_32r), ScratchReg).addReg(X86::RSP) 2385 .addImm(1).addReg(0).addImm(-StackSize).addReg(0); 2386 2387 BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::CMP64rm : X86::CMP32rm)).addReg(ScratchReg) 2388 .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg); 2389 } else { 2390 if (STI.isTargetLinux()) { 2391 TlsReg = X86::GS; 2392 TlsOffset = 0x30; 2393 } else if (STI.isTargetDarwin()) { 2394 TlsReg = X86::GS; 2395 TlsOffset = 0x48 + 90*4; 2396 } else if (STI.isTargetWin32()) { 2397 TlsReg = X86::FS; 2398 TlsOffset = 0x14; // pvArbitrary, reserved for application use 2399 } else if (STI.isTargetDragonFly()) { 2400 TlsReg = X86::FS; 2401 TlsOffset = 0x10; // use tls_tcb.tcb_segstack 2402 } else if (STI.isTargetFreeBSD()) { 2403 report_fatal_error("Segmented stacks not supported on FreeBSD i386."); 2404 } else { 2405 report_fatal_error("Segmented stacks not supported on this platform."); 2406 } 2407 2408 if (CompareStackPointer) 2409 ScratchReg = X86::ESP; 2410 else 2411 BuildMI(checkMBB, DL, TII.get(X86::LEA32r), ScratchReg).addReg(X86::ESP) 2412 .addImm(1).addReg(0).addImm(-StackSize).addReg(0); 2413 2414 if (STI.isTargetLinux() || STI.isTargetWin32() || STI.isTargetWin64() || 2415 STI.isTargetDragonFly()) { 2416 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm)).addReg(ScratchReg) 2417 .addReg(0).addImm(0).addReg(0).addImm(TlsOffset).addReg(TlsReg); 2418 } else if (STI.isTargetDarwin()) { 2419 2420 // TlsOffset doesn't fit into a mod r/m byte so we need an extra register. 2421 unsigned ScratchReg2; 2422 bool SaveScratch2; 2423 if (CompareStackPointer) { 2424 // The primary scratch register is available for holding the TLS offset. 2425 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, true); 2426 SaveScratch2 = false; 2427 } else { 2428 // Need to use a second register to hold the TLS offset 2429 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, false); 2430 2431 // Unfortunately, with fastcc the second scratch register may hold an 2432 // argument. 2433 SaveScratch2 = MF.getRegInfo().isLiveIn(ScratchReg2); 2434 } 2435 2436 // If Scratch2 is live-in then it needs to be saved. 2437 assert((!MF.getRegInfo().isLiveIn(ScratchReg2) || SaveScratch2) && 2438 "Scratch register is live-in and not saved"); 2439 2440 if (SaveScratch2) 2441 BuildMI(checkMBB, DL, TII.get(X86::PUSH32r)) 2442 .addReg(ScratchReg2, RegState::Kill); 2443 2444 BuildMI(checkMBB, DL, TII.get(X86::MOV32ri), ScratchReg2) 2445 .addImm(TlsOffset); 2446 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm)) 2447 .addReg(ScratchReg) 2448 .addReg(ScratchReg2).addImm(1).addReg(0) 2449 .addImm(0) 2450 .addReg(TlsReg); 2451 2452 if (SaveScratch2) 2453 BuildMI(checkMBB, DL, TII.get(X86::POP32r), ScratchReg2); 2454 } 2455 } 2456 2457 // This jump is taken if SP >= (Stacklet Limit + Stack Space required). 2458 // It jumps to normal execution of the function body. 2459 BuildMI(checkMBB, DL, TII.get(X86::JCC_1)).addMBB(&PrologueMBB).addImm(X86::COND_A); 2460 2461 // On 32 bit we first push the arguments size and then the frame size. On 64 2462 // bit, we pass the stack frame size in r10 and the argument size in r11. 2463 if (Is64Bit) { 2464 // Functions with nested arguments use R10, so it needs to be saved across 2465 // the call to _morestack 2466 2467 const unsigned RegAX = IsLP64 ? X86::RAX : X86::EAX; 2468 const unsigned Reg10 = IsLP64 ? X86::R10 : X86::R10D; 2469 const unsigned Reg11 = IsLP64 ? X86::R11 : X86::R11D; 2470 const unsigned MOVrr = IsLP64 ? X86::MOV64rr : X86::MOV32rr; 2471 const unsigned MOVri = IsLP64 ? X86::MOV64ri : X86::MOV32ri; 2472 2473 if (IsNested) 2474 BuildMI(allocMBB, DL, TII.get(MOVrr), RegAX).addReg(Reg10); 2475 2476 BuildMI(allocMBB, DL, TII.get(MOVri), Reg10) 2477 .addImm(StackSize); 2478 BuildMI(allocMBB, DL, TII.get(MOVri), Reg11) 2479 .addImm(X86FI->getArgumentStackSize()); 2480 } else { 2481 BuildMI(allocMBB, DL, TII.get(X86::PUSHi32)) 2482 .addImm(X86FI->getArgumentStackSize()); 2483 BuildMI(allocMBB, DL, TII.get(X86::PUSHi32)) 2484 .addImm(StackSize); 2485 } 2486 2487 // __morestack is in libgcc 2488 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) { 2489 // Under the large code model, we cannot assume that __morestack lives 2490 // within 2^31 bytes of the call site, so we cannot use pc-relative 2491 // addressing. We cannot perform the call via a temporary register, 2492 // as the rax register may be used to store the static chain, and all 2493 // other suitable registers may be either callee-save or used for 2494 // parameter passing. We cannot use the stack at this point either 2495 // because __morestack manipulates the stack directly. 2496 // 2497 // To avoid these issues, perform an indirect call via a read-only memory 2498 // location containing the address. 2499 // 2500 // This solution is not perfect, as it assumes that the .rodata section 2501 // is laid out within 2^31 bytes of each function body, but this seems 2502 // to be sufficient for JIT. 2503 // FIXME: Add retpoline support and remove the error here.. 2504 if (STI.useRetpolineIndirectCalls()) 2505 report_fatal_error("Emitting morestack calls on 64-bit with the large " 2506 "code model and retpoline not yet implemented."); 2507 BuildMI(allocMBB, DL, TII.get(X86::CALL64m)) 2508 .addReg(X86::RIP) 2509 .addImm(0) 2510 .addReg(0) 2511 .addExternalSymbol("__morestack_addr") 2512 .addReg(0); 2513 MF.getMMI().setUsesMorestackAddr(true); 2514 } else { 2515 if (Is64Bit) 2516 BuildMI(allocMBB, DL, TII.get(X86::CALL64pcrel32)) 2517 .addExternalSymbol("__morestack"); 2518 else 2519 BuildMI(allocMBB, DL, TII.get(X86::CALLpcrel32)) 2520 .addExternalSymbol("__morestack"); 2521 } 2522 2523 if (IsNested) 2524 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET_RESTORE_R10)); 2525 else 2526 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET)); 2527 2528 allocMBB->addSuccessor(&PrologueMBB); 2529 2530 checkMBB->addSuccessor(allocMBB, BranchProbability::getZero()); 2531 checkMBB->addSuccessor(&PrologueMBB, BranchProbability::getOne()); 2532 2533 #ifdef EXPENSIVE_CHECKS 2534 MF.verify(); 2535 #endif 2536 } 2537 2538 /// Lookup an ERTS parameter in the !hipe.literals named metadata node. 2539 /// HiPE provides Erlang Runtime System-internal parameters, such as PCB offsets 2540 /// to fields it needs, through a named metadata node "hipe.literals" containing 2541 /// name-value pairs. 2542 static unsigned getHiPELiteral( 2543 NamedMDNode *HiPELiteralsMD, const StringRef LiteralName) { 2544 for (int i = 0, e = HiPELiteralsMD->getNumOperands(); i != e; ++i) { 2545 MDNode *Node = HiPELiteralsMD->getOperand(i); 2546 if (Node->getNumOperands() != 2) continue; 2547 MDString *NodeName = dyn_cast<MDString>(Node->getOperand(0)); 2548 ValueAsMetadata *NodeVal = dyn_cast<ValueAsMetadata>(Node->getOperand(1)); 2549 if (!NodeName || !NodeVal) continue; 2550 ConstantInt *ValConst = dyn_cast_or_null<ConstantInt>(NodeVal->getValue()); 2551 if (ValConst && NodeName->getString() == LiteralName) { 2552 return ValConst->getZExtValue(); 2553 } 2554 } 2555 2556 report_fatal_error("HiPE literal " + LiteralName 2557 + " required but not provided"); 2558 } 2559 2560 /// Erlang programs may need a special prologue to handle the stack size they 2561 /// might need at runtime. That is because Erlang/OTP does not implement a C 2562 /// stack but uses a custom implementation of hybrid stack/heap architecture. 2563 /// (for more information see Eric Stenman's Ph.D. thesis: 2564 /// http://publications.uu.se/uu/fulltext/nbn_se_uu_diva-2688.pdf) 2565 /// 2566 /// CheckStack: 2567 /// temp0 = sp - MaxStack 2568 /// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart 2569 /// OldStart: 2570 /// ... 2571 /// IncStack: 2572 /// call inc_stack # doubles the stack space 2573 /// temp0 = sp - MaxStack 2574 /// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart 2575 void X86FrameLowering::adjustForHiPEPrologue( 2576 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const { 2577 MachineFrameInfo &MFI = MF.getFrameInfo(); 2578 DebugLoc DL; 2579 2580 // To support shrink-wrapping we would need to insert the new blocks 2581 // at the right place and update the branches to PrologueMBB. 2582 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet"); 2583 2584 // HiPE-specific values 2585 NamedMDNode *HiPELiteralsMD = MF.getMMI().getModule() 2586 ->getNamedMetadata("hipe.literals"); 2587 if (!HiPELiteralsMD) 2588 report_fatal_error( 2589 "Can't generate HiPE prologue without runtime parameters"); 2590 const unsigned HipeLeafWords 2591 = getHiPELiteral(HiPELiteralsMD, 2592 Is64Bit ? "AMD64_LEAF_WORDS" : "X86_LEAF_WORDS"); 2593 const unsigned CCRegisteredArgs = Is64Bit ? 6 : 5; 2594 const unsigned Guaranteed = HipeLeafWords * SlotSize; 2595 unsigned CallerStkArity = MF.getFunction().arg_size() > CCRegisteredArgs ? 2596 MF.getFunction().arg_size() - CCRegisteredArgs : 0; 2597 unsigned MaxStack = MFI.getStackSize() + CallerStkArity*SlotSize + SlotSize; 2598 2599 assert(STI.isTargetLinux() && 2600 "HiPE prologue is only supported on Linux operating systems."); 2601 2602 // Compute the largest caller's frame that is needed to fit the callees' 2603 // frames. This 'MaxStack' is computed from: 2604 // 2605 // a) the fixed frame size, which is the space needed for all spilled temps, 2606 // b) outgoing on-stack parameter areas, and 2607 // c) the minimum stack space this function needs to make available for the 2608 // functions it calls (a tunable ABI property). 2609 if (MFI.hasCalls()) { 2610 unsigned MoreStackForCalls = 0; 2611 2612 for (auto &MBB : MF) { 2613 for (auto &MI : MBB) { 2614 if (!MI.isCall()) 2615 continue; 2616 2617 // Get callee operand. 2618 const MachineOperand &MO = MI.getOperand(0); 2619 2620 // Only take account of global function calls (no closures etc.). 2621 if (!MO.isGlobal()) 2622 continue; 2623 2624 const Function *F = dyn_cast<Function>(MO.getGlobal()); 2625 if (!F) 2626 continue; 2627 2628 // Do not update 'MaxStack' for primitive and built-in functions 2629 // (encoded with names either starting with "erlang."/"bif_" or not 2630 // having a ".", such as a simple <Module>.<Function>.<Arity>, or an 2631 // "_", such as the BIF "suspend_0") as they are executed on another 2632 // stack. 2633 if (F->getName().find("erlang.") != StringRef::npos || 2634 F->getName().find("bif_") != StringRef::npos || 2635 F->getName().find_first_of("._") == StringRef::npos) 2636 continue; 2637 2638 unsigned CalleeStkArity = 2639 F->arg_size() > CCRegisteredArgs ? F->arg_size()-CCRegisteredArgs : 0; 2640 if (HipeLeafWords - 1 > CalleeStkArity) 2641 MoreStackForCalls = std::max(MoreStackForCalls, 2642 (HipeLeafWords - 1 - CalleeStkArity) * SlotSize); 2643 } 2644 } 2645 MaxStack += MoreStackForCalls; 2646 } 2647 2648 // If the stack frame needed is larger than the guaranteed then runtime checks 2649 // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue. 2650 if (MaxStack > Guaranteed) { 2651 MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock(); 2652 MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock(); 2653 2654 for (const auto &LI : PrologueMBB.liveins()) { 2655 stackCheckMBB->addLiveIn(LI); 2656 incStackMBB->addLiveIn(LI); 2657 } 2658 2659 MF.push_front(incStackMBB); 2660 MF.push_front(stackCheckMBB); 2661 2662 unsigned ScratchReg, SPReg, PReg, SPLimitOffset; 2663 unsigned LEAop, CMPop, CALLop; 2664 SPLimitOffset = getHiPELiteral(HiPELiteralsMD, "P_NSP_LIMIT"); 2665 if (Is64Bit) { 2666 SPReg = X86::RSP; 2667 PReg = X86::RBP; 2668 LEAop = X86::LEA64r; 2669 CMPop = X86::CMP64rm; 2670 CALLop = X86::CALL64pcrel32; 2671 } else { 2672 SPReg = X86::ESP; 2673 PReg = X86::EBP; 2674 LEAop = X86::LEA32r; 2675 CMPop = X86::CMP32rm; 2676 CALLop = X86::CALLpcrel32; 2677 } 2678 2679 ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true); 2680 assert(!MF.getRegInfo().isLiveIn(ScratchReg) && 2681 "HiPE prologue scratch register is live-in"); 2682 2683 // Create new MBB for StackCheck: 2684 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(LEAop), ScratchReg), 2685 SPReg, false, -MaxStack); 2686 // SPLimitOffset is in a fixed heap location (pointed by BP). 2687 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(CMPop)) 2688 .addReg(ScratchReg), PReg, false, SPLimitOffset); 2689 BuildMI(stackCheckMBB, DL, TII.get(X86::JCC_1)).addMBB(&PrologueMBB).addImm(X86::COND_AE); 2690 2691 // Create new MBB for IncStack: 2692 BuildMI(incStackMBB, DL, TII.get(CALLop)). 2693 addExternalSymbol("inc_stack_0"); 2694 addRegOffset(BuildMI(incStackMBB, DL, TII.get(LEAop), ScratchReg), 2695 SPReg, false, -MaxStack); 2696 addRegOffset(BuildMI(incStackMBB, DL, TII.get(CMPop)) 2697 .addReg(ScratchReg), PReg, false, SPLimitOffset); 2698 BuildMI(incStackMBB, DL, TII.get(X86::JCC_1)).addMBB(incStackMBB).addImm(X86::COND_LE); 2699 2700 stackCheckMBB->addSuccessor(&PrologueMBB, {99, 100}); 2701 stackCheckMBB->addSuccessor(incStackMBB, {1, 100}); 2702 incStackMBB->addSuccessor(&PrologueMBB, {99, 100}); 2703 incStackMBB->addSuccessor(incStackMBB, {1, 100}); 2704 } 2705 #ifdef EXPENSIVE_CHECKS 2706 MF.verify(); 2707 #endif 2708 } 2709 2710 bool X86FrameLowering::adjustStackWithPops(MachineBasicBlock &MBB, 2711 MachineBasicBlock::iterator MBBI, 2712 const DebugLoc &DL, 2713 int Offset) const { 2714 2715 if (Offset <= 0) 2716 return false; 2717 2718 if (Offset % SlotSize) 2719 return false; 2720 2721 int NumPops = Offset / SlotSize; 2722 // This is only worth it if we have at most 2 pops. 2723 if (NumPops != 1 && NumPops != 2) 2724 return false; 2725 2726 // Handle only the trivial case where the adjustment directly follows 2727 // a call. This is the most common one, anyway. 2728 if (MBBI == MBB.begin()) 2729 return false; 2730 MachineBasicBlock::iterator Prev = std::prev(MBBI); 2731 if (!Prev->isCall() || !Prev->getOperand(1).isRegMask()) 2732 return false; 2733 2734 unsigned Regs[2]; 2735 unsigned FoundRegs = 0; 2736 2737 auto &MRI = MBB.getParent()->getRegInfo(); 2738 auto RegMask = Prev->getOperand(1); 2739 2740 auto &RegClass = 2741 Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass; 2742 // Try to find up to NumPops free registers. 2743 for (auto Candidate : RegClass) { 2744 2745 // Poor man's liveness: 2746 // Since we're immediately after a call, any register that is clobbered 2747 // by the call and not defined by it can be considered dead. 2748 if (!RegMask.clobbersPhysReg(Candidate)) 2749 continue; 2750 2751 // Don't clobber reserved registers 2752 if (MRI.isReserved(Candidate)) 2753 continue; 2754 2755 bool IsDef = false; 2756 for (const MachineOperand &MO : Prev->implicit_operands()) { 2757 if (MO.isReg() && MO.isDef() && 2758 TRI->isSuperOrSubRegisterEq(MO.getReg(), Candidate)) { 2759 IsDef = true; 2760 break; 2761 } 2762 } 2763 2764 if (IsDef) 2765 continue; 2766 2767 Regs[FoundRegs++] = Candidate; 2768 if (FoundRegs == (unsigned)NumPops) 2769 break; 2770 } 2771 2772 if (FoundRegs == 0) 2773 return false; 2774 2775 // If we found only one free register, but need two, reuse the same one twice. 2776 while (FoundRegs < (unsigned)NumPops) 2777 Regs[FoundRegs++] = Regs[0]; 2778 2779 for (int i = 0; i < NumPops; ++i) 2780 BuildMI(MBB, MBBI, DL, 2781 TII.get(STI.is64Bit() ? X86::POP64r : X86::POP32r), Regs[i]); 2782 2783 return true; 2784 } 2785 2786 MachineBasicBlock::iterator X86FrameLowering:: 2787 eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, 2788 MachineBasicBlock::iterator I) const { 2789 bool reserveCallFrame = hasReservedCallFrame(MF); 2790 unsigned Opcode = I->getOpcode(); 2791 bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode(); 2792 DebugLoc DL = I->getDebugLoc(); 2793 uint64_t Amount = !reserveCallFrame ? TII.getFrameSize(*I) : 0; 2794 uint64_t InternalAmt = (isDestroy || Amount) ? TII.getFrameAdjustment(*I) : 0; 2795 I = MBB.erase(I); 2796 auto InsertPos = skipDebugInstructionsForward(I, MBB.end()); 2797 2798 if (!reserveCallFrame) { 2799 // If the stack pointer can be changed after prologue, turn the 2800 // adjcallstackup instruction into a 'sub ESP, <amt>' and the 2801 // adjcallstackdown instruction into 'add ESP, <amt>' 2802 2803 // We need to keep the stack aligned properly. To do this, we round the 2804 // amount of space needed for the outgoing arguments up to the next 2805 // alignment boundary. 2806 unsigned StackAlign = getStackAlignment(); 2807 Amount = alignTo(Amount, StackAlign); 2808 2809 MachineModuleInfo &MMI = MF.getMMI(); 2810 const Function &F = MF.getFunction(); 2811 bool WindowsCFI = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 2812 bool DwarfCFI = !WindowsCFI && 2813 (MMI.hasDebugInfo() || F.needsUnwindTableEntry()); 2814 2815 // If we have any exception handlers in this function, and we adjust 2816 // the SP before calls, we may need to indicate this to the unwinder 2817 // using GNU_ARGS_SIZE. Note that this may be necessary even when 2818 // Amount == 0, because the preceding function may have set a non-0 2819 // GNU_ARGS_SIZE. 2820 // TODO: We don't need to reset this between subsequent functions, 2821 // if it didn't change. 2822 bool HasDwarfEHHandlers = !WindowsCFI && !MF.getLandingPads().empty(); 2823 2824 if (HasDwarfEHHandlers && !isDestroy && 2825 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences()) 2826 BuildCFI(MBB, InsertPos, DL, 2827 MCCFIInstruction::createGnuArgsSize(nullptr, Amount)); 2828 2829 if (Amount == 0) 2830 return I; 2831 2832 // Factor out the amount that gets handled inside the sequence 2833 // (Pushes of argument for frame setup, callee pops for frame destroy) 2834 Amount -= InternalAmt; 2835 2836 // TODO: This is needed only if we require precise CFA. 2837 // If this is a callee-pop calling convention, emit a CFA adjust for 2838 // the amount the callee popped. 2839 if (isDestroy && InternalAmt && DwarfCFI && !hasFP(MF)) 2840 BuildCFI(MBB, InsertPos, DL, 2841 MCCFIInstruction::createAdjustCfaOffset(nullptr, -InternalAmt)); 2842 2843 // Add Amount to SP to destroy a frame, or subtract to setup. 2844 int64_t StackAdjustment = isDestroy ? Amount : -Amount; 2845 2846 if (StackAdjustment) { 2847 // Merge with any previous or following adjustment instruction. Note: the 2848 // instructions merged with here do not have CFI, so their stack 2849 // adjustments do not feed into CfaAdjustment. 2850 StackAdjustment += mergeSPUpdates(MBB, InsertPos, true); 2851 StackAdjustment += mergeSPUpdates(MBB, InsertPos, false); 2852 2853 if (StackAdjustment) { 2854 if (!(F.hasMinSize() && 2855 adjustStackWithPops(MBB, InsertPos, DL, StackAdjustment))) 2856 BuildStackAdjustment(MBB, InsertPos, DL, StackAdjustment, 2857 /*InEpilogue=*/false); 2858 } 2859 } 2860 2861 if (DwarfCFI && !hasFP(MF)) { 2862 // If we don't have FP, but need to generate unwind information, 2863 // we need to set the correct CFA offset after the stack adjustment. 2864 // How much we adjust the CFA offset depends on whether we're emitting 2865 // CFI only for EH purposes or for debugging. EH only requires the CFA 2866 // offset to be correct at each call site, while for debugging we want 2867 // it to be more precise. 2868 2869 int64_t CfaAdjustment = -StackAdjustment; 2870 // TODO: When not using precise CFA, we also need to adjust for the 2871 // InternalAmt here. 2872 if (CfaAdjustment) { 2873 BuildCFI(MBB, InsertPos, DL, 2874 MCCFIInstruction::createAdjustCfaOffset(nullptr, 2875 CfaAdjustment)); 2876 } 2877 } 2878 2879 return I; 2880 } 2881 2882 if (isDestroy && InternalAmt) { 2883 // If we are performing frame pointer elimination and if the callee pops 2884 // something off the stack pointer, add it back. We do this until we have 2885 // more advanced stack pointer tracking ability. 2886 // We are not tracking the stack pointer adjustment by the callee, so make 2887 // sure we restore the stack pointer immediately after the call, there may 2888 // be spill code inserted between the CALL and ADJCALLSTACKUP instructions. 2889 MachineBasicBlock::iterator CI = I; 2890 MachineBasicBlock::iterator B = MBB.begin(); 2891 while (CI != B && !std::prev(CI)->isCall()) 2892 --CI; 2893 BuildStackAdjustment(MBB, CI, DL, -InternalAmt, /*InEpilogue=*/false); 2894 } 2895 2896 return I; 2897 } 2898 2899 bool X86FrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const { 2900 assert(MBB.getParent() && "Block is not attached to a function!"); 2901 const MachineFunction &MF = *MBB.getParent(); 2902 return !TRI->needsStackRealignment(MF) || !MBB.isLiveIn(X86::EFLAGS); 2903 } 2904 2905 bool X86FrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const { 2906 assert(MBB.getParent() && "Block is not attached to a function!"); 2907 2908 // Win64 has strict requirements in terms of epilogue and we are 2909 // not taking a chance at messing with them. 2910 // I.e., unless this block is already an exit block, we can't use 2911 // it as an epilogue. 2912 if (STI.isTargetWin64() && !MBB.succ_empty() && !MBB.isReturnBlock()) 2913 return false; 2914 2915 if (canUseLEAForSPInEpilogue(*MBB.getParent())) 2916 return true; 2917 2918 // If we cannot use LEA to adjust SP, we may need to use ADD, which 2919 // clobbers the EFLAGS. Check that we do not need to preserve it, 2920 // otherwise, conservatively assume this is not 2921 // safe to insert the epilogue here. 2922 return !flagsNeedToBePreservedBeforeTheTerminators(MBB); 2923 } 2924 2925 bool X86FrameLowering::enableShrinkWrapping(const MachineFunction &MF) const { 2926 // If we may need to emit frameless compact unwind information, give 2927 // up as this is currently broken: PR25614. 2928 return (MF.getFunction().hasFnAttribute(Attribute::NoUnwind) || hasFP(MF)) && 2929 // The lowering of segmented stack and HiPE only support entry blocks 2930 // as prologue blocks: PR26107. 2931 // This limitation may be lifted if we fix: 2932 // - adjustForSegmentedStacks 2933 // - adjustForHiPEPrologue 2934 MF.getFunction().getCallingConv() != CallingConv::HiPE && 2935 !MF.shouldSplitStack(); 2936 } 2937 2938 MachineBasicBlock::iterator X86FrameLowering::restoreWin32EHStackPointers( 2939 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 2940 const DebugLoc &DL, bool RestoreSP) const { 2941 assert(STI.isTargetWindowsMSVC() && "funclets only supported in MSVC env"); 2942 assert(STI.isTargetWin32() && "EBP/ESI restoration only required on win32"); 2943 assert(STI.is32Bit() && !Uses64BitFramePtr && 2944 "restoring EBP/ESI on non-32-bit target"); 2945 2946 MachineFunction &MF = *MBB.getParent(); 2947 unsigned FramePtr = TRI->getFrameRegister(MF); 2948 unsigned BasePtr = TRI->getBaseRegister(); 2949 WinEHFuncInfo &FuncInfo = *MF.getWinEHFuncInfo(); 2950 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 2951 MachineFrameInfo &MFI = MF.getFrameInfo(); 2952 2953 // FIXME: Don't set FrameSetup flag in catchret case. 2954 2955 int FI = FuncInfo.EHRegNodeFrameIndex; 2956 int EHRegSize = MFI.getObjectSize(FI); 2957 2958 if (RestoreSP) { 2959 // MOV32rm -EHRegSize(%ebp), %esp 2960 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), X86::ESP), 2961 X86::EBP, true, -EHRegSize) 2962 .setMIFlag(MachineInstr::FrameSetup); 2963 } 2964 2965 unsigned UsedReg; 2966 int EHRegOffset = getFrameIndexReference(MF, FI, UsedReg); 2967 int EndOffset = -EHRegOffset - EHRegSize; 2968 FuncInfo.EHRegNodeEndOffset = EndOffset; 2969 2970 if (UsedReg == FramePtr) { 2971 // ADD $offset, %ebp 2972 unsigned ADDri = getADDriOpcode(false, EndOffset); 2973 BuildMI(MBB, MBBI, DL, TII.get(ADDri), FramePtr) 2974 .addReg(FramePtr) 2975 .addImm(EndOffset) 2976 .setMIFlag(MachineInstr::FrameSetup) 2977 ->getOperand(3) 2978 .setIsDead(); 2979 assert(EndOffset >= 0 && 2980 "end of registration object above normal EBP position!"); 2981 } else if (UsedReg == BasePtr) { 2982 // LEA offset(%ebp), %esi 2983 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA32r), BasePtr), 2984 FramePtr, false, EndOffset) 2985 .setMIFlag(MachineInstr::FrameSetup); 2986 // MOV32rm SavedEBPOffset(%esi), %ebp 2987 assert(X86FI->getHasSEHFramePtrSave()); 2988 int Offset = 2989 getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg); 2990 assert(UsedReg == BasePtr); 2991 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), FramePtr), 2992 UsedReg, true, Offset) 2993 .setMIFlag(MachineInstr::FrameSetup); 2994 } else { 2995 llvm_unreachable("32-bit frames with WinEH must use FramePtr or BasePtr"); 2996 } 2997 return MBBI; 2998 } 2999 3000 int X86FrameLowering::getInitialCFAOffset(const MachineFunction &MF) const { 3001 return TRI->getSlotSize(); 3002 } 3003 3004 unsigned X86FrameLowering::getInitialCFARegister(const MachineFunction &MF) 3005 const { 3006 return TRI->getDwarfRegNum(StackPtr, true); 3007 } 3008 3009 namespace { 3010 // Struct used by orderFrameObjects to help sort the stack objects. 3011 struct X86FrameSortingObject { 3012 bool IsValid = false; // true if we care about this Object. 3013 unsigned ObjectIndex = 0; // Index of Object into MFI list. 3014 unsigned ObjectSize = 0; // Size of Object in bytes. 3015 unsigned ObjectAlignment = 1; // Alignment of Object in bytes. 3016 unsigned ObjectNumUses = 0; // Object static number of uses. 3017 }; 3018 3019 // The comparison function we use for std::sort to order our local 3020 // stack symbols. The current algorithm is to use an estimated 3021 // "density". This takes into consideration the size and number of 3022 // uses each object has in order to roughly minimize code size. 3023 // So, for example, an object of size 16B that is referenced 5 times 3024 // will get higher priority than 4 4B objects referenced 1 time each. 3025 // It's not perfect and we may be able to squeeze a few more bytes out of 3026 // it (for example : 0(esp) requires fewer bytes, symbols allocated at the 3027 // fringe end can have special consideration, given their size is less 3028 // important, etc.), but the algorithmic complexity grows too much to be 3029 // worth the extra gains we get. This gets us pretty close. 3030 // The final order leaves us with objects with highest priority going 3031 // at the end of our list. 3032 struct X86FrameSortingComparator { 3033 inline bool operator()(const X86FrameSortingObject &A, 3034 const X86FrameSortingObject &B) { 3035 uint64_t DensityAScaled, DensityBScaled; 3036 3037 // For consistency in our comparison, all invalid objects are placed 3038 // at the end. This also allows us to stop walking when we hit the 3039 // first invalid item after it's all sorted. 3040 if (!A.IsValid) 3041 return false; 3042 if (!B.IsValid) 3043 return true; 3044 3045 // The density is calculated by doing : 3046 // (double)DensityA = A.ObjectNumUses / A.ObjectSize 3047 // (double)DensityB = B.ObjectNumUses / B.ObjectSize 3048 // Since this approach may cause inconsistencies in 3049 // the floating point <, >, == comparisons, depending on the floating 3050 // point model with which the compiler was built, we're going 3051 // to scale both sides by multiplying with 3052 // A.ObjectSize * B.ObjectSize. This ends up factoring away 3053 // the division and, with it, the need for any floating point 3054 // arithmetic. 3055 DensityAScaled = static_cast<uint64_t>(A.ObjectNumUses) * 3056 static_cast<uint64_t>(B.ObjectSize); 3057 DensityBScaled = static_cast<uint64_t>(B.ObjectNumUses) * 3058 static_cast<uint64_t>(A.ObjectSize); 3059 3060 // If the two densities are equal, prioritize highest alignment 3061 // objects. This allows for similar alignment objects 3062 // to be packed together (given the same density). 3063 // There's room for improvement here, also, since we can pack 3064 // similar alignment (different density) objects next to each 3065 // other to save padding. This will also require further 3066 // complexity/iterations, and the overall gain isn't worth it, 3067 // in general. Something to keep in mind, though. 3068 if (DensityAScaled == DensityBScaled) 3069 return A.ObjectAlignment < B.ObjectAlignment; 3070 3071 return DensityAScaled < DensityBScaled; 3072 } 3073 }; 3074 } // namespace 3075 3076 // Order the symbols in the local stack. 3077 // We want to place the local stack objects in some sort of sensible order. 3078 // The heuristic we use is to try and pack them according to static number 3079 // of uses and size of object in order to minimize code size. 3080 void X86FrameLowering::orderFrameObjects( 3081 const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const { 3082 const MachineFrameInfo &MFI = MF.getFrameInfo(); 3083 3084 // Don't waste time if there's nothing to do. 3085 if (ObjectsToAllocate.empty()) 3086 return; 3087 3088 // Create an array of all MFI objects. We won't need all of these 3089 // objects, but we're going to create a full array of them to make 3090 // it easier to index into when we're counting "uses" down below. 3091 // We want to be able to easily/cheaply access an object by simply 3092 // indexing into it, instead of having to search for it every time. 3093 std::vector<X86FrameSortingObject> SortingObjects(MFI.getObjectIndexEnd()); 3094 3095 // Walk the objects we care about and mark them as such in our working 3096 // struct. 3097 for (auto &Obj : ObjectsToAllocate) { 3098 SortingObjects[Obj].IsValid = true; 3099 SortingObjects[Obj].ObjectIndex = Obj; 3100 SortingObjects[Obj].ObjectAlignment = MFI.getObjectAlignment(Obj); 3101 // Set the size. 3102 int ObjectSize = MFI.getObjectSize(Obj); 3103 if (ObjectSize == 0) 3104 // Variable size. Just use 4. 3105 SortingObjects[Obj].ObjectSize = 4; 3106 else 3107 SortingObjects[Obj].ObjectSize = ObjectSize; 3108 } 3109 3110 // Count the number of uses for each object. 3111 for (auto &MBB : MF) { 3112 for (auto &MI : MBB) { 3113 if (MI.isDebugInstr()) 3114 continue; 3115 for (const MachineOperand &MO : MI.operands()) { 3116 // Check to see if it's a local stack symbol. 3117 if (!MO.isFI()) 3118 continue; 3119 int Index = MO.getIndex(); 3120 // Check to see if it falls within our range, and is tagged 3121 // to require ordering. 3122 if (Index >= 0 && Index < MFI.getObjectIndexEnd() && 3123 SortingObjects[Index].IsValid) 3124 SortingObjects[Index].ObjectNumUses++; 3125 } 3126 } 3127 } 3128 3129 // Sort the objects using X86FrameSortingAlgorithm (see its comment for 3130 // info). 3131 llvm::stable_sort(SortingObjects, X86FrameSortingComparator()); 3132 3133 // Now modify the original list to represent the final order that 3134 // we want. The order will depend on whether we're going to access them 3135 // from the stack pointer or the frame pointer. For SP, the list should 3136 // end up with the END containing objects that we want with smaller offsets. 3137 // For FP, it should be flipped. 3138 int i = 0; 3139 for (auto &Obj : SortingObjects) { 3140 // All invalid items are sorted at the end, so it's safe to stop. 3141 if (!Obj.IsValid) 3142 break; 3143 ObjectsToAllocate[i++] = Obj.ObjectIndex; 3144 } 3145 3146 // Flip it if we're accessing off of the FP. 3147 if (!TRI->needsStackRealignment(MF) && hasFP(MF)) 3148 std::reverse(ObjectsToAllocate.begin(), ObjectsToAllocate.end()); 3149 } 3150 3151 3152 unsigned X86FrameLowering::getWinEHParentFrameOffset(const MachineFunction &MF) const { 3153 // RDX, the parent frame pointer, is homed into 16(%rsp) in the prologue. 3154 unsigned Offset = 16; 3155 // RBP is immediately pushed. 3156 Offset += SlotSize; 3157 // All callee-saved registers are then pushed. 3158 Offset += MF.getInfo<X86MachineFunctionInfo>()->getCalleeSavedFrameSize(); 3159 // Every funclet allocates enough stack space for the largest outgoing call. 3160 Offset += getWinEHFuncletFrameSize(MF); 3161 return Offset; 3162 } 3163 3164 void X86FrameLowering::processFunctionBeforeFrameFinalized( 3165 MachineFunction &MF, RegScavenger *RS) const { 3166 // Mark the function as not having WinCFI. We will set it back to true in 3167 // emitPrologue if it gets called and emits CFI. 3168 MF.setHasWinCFI(false); 3169 3170 // If this function isn't doing Win64-style C++ EH, we don't need to do 3171 // anything. 3172 const Function &F = MF.getFunction(); 3173 if (!STI.is64Bit() || !MF.hasEHFunclets() || 3174 classifyEHPersonality(F.getPersonalityFn()) != EHPersonality::MSVC_CXX) 3175 return; 3176 3177 // Win64 C++ EH needs to allocate the UnwindHelp object at some fixed offset 3178 // relative to RSP after the prologue. Find the offset of the last fixed 3179 // object, so that we can allocate a slot immediately following it. If there 3180 // were no fixed objects, use offset -SlotSize, which is immediately after the 3181 // return address. Fixed objects have negative frame indices. 3182 MachineFrameInfo &MFI = MF.getFrameInfo(); 3183 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo(); 3184 int64_t MinFixedObjOffset = -SlotSize; 3185 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) 3186 MinFixedObjOffset = std::min(MinFixedObjOffset, MFI.getObjectOffset(I)); 3187 3188 for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) { 3189 for (WinEHHandlerType &H : TBME.HandlerArray) { 3190 int FrameIndex = H.CatchObj.FrameIndex; 3191 if (FrameIndex != INT_MAX) { 3192 // Ensure alignment. 3193 unsigned Align = MFI.getObjectAlignment(FrameIndex); 3194 MinFixedObjOffset -= std::abs(MinFixedObjOffset) % Align; 3195 MinFixedObjOffset -= MFI.getObjectSize(FrameIndex); 3196 MFI.setObjectOffset(FrameIndex, MinFixedObjOffset); 3197 } 3198 } 3199 } 3200 3201 // Ensure alignment. 3202 MinFixedObjOffset -= std::abs(MinFixedObjOffset) % 8; 3203 int64_t UnwindHelpOffset = MinFixedObjOffset - SlotSize; 3204 int UnwindHelpFI = 3205 MFI.CreateFixedObject(SlotSize, UnwindHelpOffset, /*IsImmutable=*/false); 3206 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI; 3207 3208 // Store -2 into UnwindHelp on function entry. We have to scan forwards past 3209 // other frame setup instructions. 3210 MachineBasicBlock &MBB = MF.front(); 3211 auto MBBI = MBB.begin(); 3212 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) 3213 ++MBBI; 3214 3215 DebugLoc DL = MBB.findDebugLoc(MBBI); 3216 addFrameReference(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mi32)), 3217 UnwindHelpFI) 3218 .addImm(-2); 3219 } 3220