xref: /freebsd/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp (revision e32fecd0c2c3ee37c47ee100f169e7eb0282a873)
1 //===-- RISCVFrameLowering.cpp - RISCV Frame Information ------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the RISCV implementation of TargetFrameLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVFrameLowering.h"
14 #include "RISCVMachineFunctionInfo.h"
15 #include "RISCVSubtarget.h"
16 #include "llvm/CodeGen/MachineFrameInfo.h"
17 #include "llvm/CodeGen/MachineFunction.h"
18 #include "llvm/CodeGen/MachineInstrBuilder.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/CodeGen/RegisterScavenging.h"
21 #include "llvm/IR/DiagnosticInfo.h"
22 #include "llvm/MC/MCDwarf.h"
23 
24 #include <algorithm>
25 
26 using namespace llvm;
27 
28 // For now we use x18, a.k.a s2, as pointer to shadow call stack.
29 // User should explicitly set -ffixed-x18 and not use x18 in their asm.
30 static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
31                             MachineBasicBlock::iterator MI,
32                             const DebugLoc &DL) {
33   if (!MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack))
34     return;
35 
36   const auto &STI = MF.getSubtarget<RISCVSubtarget>();
37   Register RAReg = STI.getRegisterInfo()->getRARegister();
38 
39   // Do not save RA to the SCS if it's not saved to the regular stack,
40   // i.e. RA is not at risk of being overwritten.
41   std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
42   if (std::none_of(CSI.begin(), CSI.end(),
43                    [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
44     return;
45 
46   Register SCSPReg = RISCVABI::getSCSPReg();
47 
48   auto &Ctx = MF.getFunction().getContext();
49   if (!STI.isRegisterReservedByUser(SCSPReg)) {
50     Ctx.diagnose(DiagnosticInfoUnsupported{
51         MF.getFunction(), "x18 not reserved by user for Shadow Call Stack."});
52     return;
53   }
54 
55   const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
56   if (RVFI->useSaveRestoreLibCalls(MF)) {
57     Ctx.diagnose(DiagnosticInfoUnsupported{
58         MF.getFunction(),
59         "Shadow Call Stack cannot be combined with Save/Restore LibCalls."});
60     return;
61   }
62 
63   const RISCVInstrInfo *TII = STI.getInstrInfo();
64   bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit);
65   int64_t SlotSize = STI.getXLen() / 8;
66   // Store return address to shadow call stack
67   // s[w|d]  ra, 0(s2)
68   // addi    s2, s2, [4|8]
69   BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
70       .addReg(RAReg)
71       .addReg(SCSPReg)
72       .addImm(0)
73       .setMIFlag(MachineInstr::FrameSetup);
74   BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI))
75       .addReg(SCSPReg, RegState::Define)
76       .addReg(SCSPReg)
77       .addImm(SlotSize)
78       .setMIFlag(MachineInstr::FrameSetup);
79 }
80 
81 static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
82                             MachineBasicBlock::iterator MI,
83                             const DebugLoc &DL) {
84   if (!MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack))
85     return;
86 
87   const auto &STI = MF.getSubtarget<RISCVSubtarget>();
88   Register RAReg = STI.getRegisterInfo()->getRARegister();
89 
90   // See emitSCSPrologue() above.
91   std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
92   if (std::none_of(CSI.begin(), CSI.end(),
93                    [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
94     return;
95 
96   Register SCSPReg = RISCVABI::getSCSPReg();
97 
98   auto &Ctx = MF.getFunction().getContext();
99   if (!STI.isRegisterReservedByUser(SCSPReg)) {
100     Ctx.diagnose(DiagnosticInfoUnsupported{
101         MF.getFunction(), "x18 not reserved by user for Shadow Call Stack."});
102     return;
103   }
104 
105   const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
106   if (RVFI->useSaveRestoreLibCalls(MF)) {
107     Ctx.diagnose(DiagnosticInfoUnsupported{
108         MF.getFunction(),
109         "Shadow Call Stack cannot be combined with Save/Restore LibCalls."});
110     return;
111   }
112 
113   const RISCVInstrInfo *TII = STI.getInstrInfo();
114   bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit);
115   int64_t SlotSize = STI.getXLen() / 8;
116   // Load return address from shadow call stack
117   // l[w|d]  ra, -[4|8](s2)
118   // addi    s2, s2, -[4|8]
119   BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::LD : RISCV::LW))
120       .addReg(RAReg, RegState::Define)
121       .addReg(SCSPReg)
122       .addImm(-SlotSize)
123       .setMIFlag(MachineInstr::FrameDestroy);
124   BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI))
125       .addReg(SCSPReg, RegState::Define)
126       .addReg(SCSPReg)
127       .addImm(-SlotSize)
128       .setMIFlag(MachineInstr::FrameDestroy);
129 }
130 
131 // Get the ID of the libcall used for spilling and restoring callee saved
132 // registers. The ID is representative of the number of registers saved or
133 // restored by the libcall, except it is zero-indexed - ID 0 corresponds to a
134 // single register.
135 static int getLibCallID(const MachineFunction &MF,
136                         const std::vector<CalleeSavedInfo> &CSI) {
137   const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
138 
139   if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF))
140     return -1;
141 
142   Register MaxReg = RISCV::NoRegister;
143   for (auto &CS : CSI)
144     // RISCVRegisterInfo::hasReservedSpillSlot assigns negative frame indexes to
145     // registers which can be saved by libcall.
146     if (CS.getFrameIdx() < 0)
147       MaxReg = std::max(MaxReg.id(), CS.getReg().id());
148 
149   if (MaxReg == RISCV::NoRegister)
150     return -1;
151 
152   switch (MaxReg) {
153   default:
154     llvm_unreachable("Something has gone wrong!");
155   case /*s11*/ RISCV::X27: return 12;
156   case /*s10*/ RISCV::X26: return 11;
157   case /*s9*/  RISCV::X25: return 10;
158   case /*s8*/  RISCV::X24: return 9;
159   case /*s7*/  RISCV::X23: return 8;
160   case /*s6*/  RISCV::X22: return 7;
161   case /*s5*/  RISCV::X21: return 6;
162   case /*s4*/  RISCV::X20: return 5;
163   case /*s3*/  RISCV::X19: return 4;
164   case /*s2*/  RISCV::X18: return 3;
165   case /*s1*/  RISCV::X9:  return 2;
166   case /*s0*/  RISCV::X8:  return 1;
167   case /*ra*/  RISCV::X1:  return 0;
168   }
169 }
170 
171 // Get the name of the libcall used for spilling callee saved registers.
172 // If this function will not use save/restore libcalls, then return a nullptr.
173 static const char *
174 getSpillLibCallName(const MachineFunction &MF,
175                     const std::vector<CalleeSavedInfo> &CSI) {
176   static const char *const SpillLibCalls[] = {
177     "__riscv_save_0",
178     "__riscv_save_1",
179     "__riscv_save_2",
180     "__riscv_save_3",
181     "__riscv_save_4",
182     "__riscv_save_5",
183     "__riscv_save_6",
184     "__riscv_save_7",
185     "__riscv_save_8",
186     "__riscv_save_9",
187     "__riscv_save_10",
188     "__riscv_save_11",
189     "__riscv_save_12"
190   };
191 
192   int LibCallID = getLibCallID(MF, CSI);
193   if (LibCallID == -1)
194     return nullptr;
195   return SpillLibCalls[LibCallID];
196 }
197 
198 // Get the name of the libcall used for restoring callee saved registers.
199 // If this function will not use save/restore libcalls, then return a nullptr.
200 static const char *
201 getRestoreLibCallName(const MachineFunction &MF,
202                       const std::vector<CalleeSavedInfo> &CSI) {
203   static const char *const RestoreLibCalls[] = {
204     "__riscv_restore_0",
205     "__riscv_restore_1",
206     "__riscv_restore_2",
207     "__riscv_restore_3",
208     "__riscv_restore_4",
209     "__riscv_restore_5",
210     "__riscv_restore_6",
211     "__riscv_restore_7",
212     "__riscv_restore_8",
213     "__riscv_restore_9",
214     "__riscv_restore_10",
215     "__riscv_restore_11",
216     "__riscv_restore_12"
217   };
218 
219   int LibCallID = getLibCallID(MF, CSI);
220   if (LibCallID == -1)
221     return nullptr;
222   return RestoreLibCalls[LibCallID];
223 }
224 
225 // Return true if the specified function should have a dedicated frame
226 // pointer register.  This is true if frame pointer elimination is
227 // disabled, if it needs dynamic stack realignment, if the function has
228 // variable sized allocas, or if the frame address is taken.
229 bool RISCVFrameLowering::hasFP(const MachineFunction &MF) const {
230   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
231 
232   const MachineFrameInfo &MFI = MF.getFrameInfo();
233   return MF.getTarget().Options.DisableFramePointerElim(MF) ||
234          RegInfo->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
235          MFI.isFrameAddressTaken();
236 }
237 
238 bool RISCVFrameLowering::hasBP(const MachineFunction &MF) const {
239   const MachineFrameInfo &MFI = MF.getFrameInfo();
240   const TargetRegisterInfo *TRI = STI.getRegisterInfo();
241 
242   // If we do not reserve stack space for outgoing arguments in prologue,
243   // we will adjust the stack pointer before call instruction. After the
244   // adjustment, we can not use SP to access the stack objects for the
245   // arguments. Instead, use BP to access these stack objects.
246   return (MFI.hasVarSizedObjects() ||
247           (!hasReservedCallFrame(MF) && (!MFI.isMaxCallFrameSizeComputed() ||
248                                          MFI.getMaxCallFrameSize() != 0))) &&
249          TRI->hasStackRealignment(MF);
250 }
251 
252 // Determines the size of the frame and maximum call frame size.
253 void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const {
254   MachineFrameInfo &MFI = MF.getFrameInfo();
255   auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
256 
257   // Get the number of bytes to allocate from the FrameInfo.
258   uint64_t FrameSize = MFI.getStackSize();
259 
260   // Get the alignment.
261   Align StackAlign = getStackAlign();
262 
263   // Make sure the frame is aligned.
264   FrameSize = alignTo(FrameSize, StackAlign);
265 
266   // Update frame info.
267   MFI.setStackSize(FrameSize);
268 
269   // When using SP or BP to access stack objects, we may require extra padding
270   // to ensure the bottom of the RVV stack is correctly aligned within the main
271   // stack. We calculate this as the amount required to align the scalar local
272   // variable section up to the RVV alignment.
273   const TargetRegisterInfo *TRI = STI.getRegisterInfo();
274   if (RVFI->getRVVStackSize() && (!hasFP(MF) || TRI->hasStackRealignment(MF))) {
275     int ScalarLocalVarSize = FrameSize - RVFI->getCalleeSavedStackSize() -
276                              RVFI->getVarArgsSaveSize();
277     if (auto RVVPadding =
278             offsetToAlignment(ScalarLocalVarSize, RVFI->getRVVStackAlign()))
279       RVFI->setRVVPadding(RVVPadding);
280   }
281 }
282 
283 // Returns the stack size including RVV padding (when required), rounded back
284 // up to the required stack alignment.
285 uint64_t RISCVFrameLowering::getStackSizeWithRVVPadding(
286     const MachineFunction &MF) const {
287   const MachineFrameInfo &MFI = MF.getFrameInfo();
288   auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
289   return alignTo(MFI.getStackSize() + RVFI->getRVVPadding(), getStackAlign());
290 }
291 
292 void RISCVFrameLowering::adjustReg(MachineBasicBlock &MBB,
293                                    MachineBasicBlock::iterator MBBI,
294                                    const DebugLoc &DL, Register DestReg,
295                                    Register SrcReg, int64_t Val,
296                                    MachineInstr::MIFlag Flag) const {
297   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
298   const RISCVInstrInfo *TII = STI.getInstrInfo();
299 
300   if (DestReg == SrcReg && Val == 0)
301     return;
302 
303   if (isInt<12>(Val)) {
304     BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), DestReg)
305         .addReg(SrcReg)
306         .addImm(Val)
307         .setMIFlag(Flag);
308     return;
309   }
310 
311   // Try to split the offset across two ADDIs. We need to keep the stack pointer
312   // aligned after each ADDI. We need to determine the maximum value we can put
313   // in each ADDI. In the negative direction, we can use -2048 which is always
314   // sufficiently aligned. In the positive direction, we need to find the
315   // largest 12-bit immediate that is aligned. Exclude -4096 since it can be
316   // created with LUI.
317   assert(getStackAlign().value() < 2048 && "Stack alignment too large");
318   int64_t MaxPosAdjStep = 2048 - getStackAlign().value();
319   if (Val > -4096 && Val <= (2 * MaxPosAdjStep)) {
320     int64_t FirstAdj = Val < 0 ? -2048 : MaxPosAdjStep;
321     Val -= FirstAdj;
322     BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), DestReg)
323         .addReg(SrcReg)
324         .addImm(FirstAdj)
325         .setMIFlag(Flag);
326     BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), DestReg)
327         .addReg(DestReg, RegState::Kill)
328         .addImm(Val)
329         .setMIFlag(Flag);
330     return;
331   }
332 
333   unsigned Opc = RISCV::ADD;
334   if (Val < 0) {
335     Val = -Val;
336     Opc = RISCV::SUB;
337   }
338 
339   Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
340   TII->movImm(MBB, MBBI, DL, ScratchReg, Val, Flag);
341   BuildMI(MBB, MBBI, DL, TII->get(Opc), DestReg)
342       .addReg(SrcReg)
343       .addReg(ScratchReg, RegState::Kill)
344       .setMIFlag(Flag);
345 }
346 
347 // Returns the register used to hold the frame pointer.
348 static Register getFPReg(const RISCVSubtarget &STI) { return RISCV::X8; }
349 
350 // Returns the register used to hold the stack pointer.
351 static Register getSPReg(const RISCVSubtarget &STI) { return RISCV::X2; }
352 
353 static SmallVector<CalleeSavedInfo, 8>
354 getNonLibcallCSI(const MachineFunction &MF,
355                  const std::vector<CalleeSavedInfo> &CSI) {
356   const MachineFrameInfo &MFI = MF.getFrameInfo();
357   SmallVector<CalleeSavedInfo, 8> NonLibcallCSI;
358 
359   for (auto &CS : CSI) {
360     int FI = CS.getFrameIdx();
361     if (FI >= 0 && MFI.getStackID(FI) == TargetStackID::Default)
362       NonLibcallCSI.push_back(CS);
363   }
364 
365   return NonLibcallCSI;
366 }
367 
368 void RISCVFrameLowering::adjustStackForRVV(MachineFunction &MF,
369                                            MachineBasicBlock &MBB,
370                                            MachineBasicBlock::iterator MBBI,
371                                            const DebugLoc &DL, int64_t Amount,
372                                            MachineInstr::MIFlag Flag) const {
373   assert(Amount != 0 && "Did not need to adjust stack pointer for RVV.");
374 
375   const RISCVInstrInfo *TII = STI.getInstrInfo();
376   Register SPReg = getSPReg(STI);
377   unsigned Opc = RISCV::ADD;
378   if (Amount < 0) {
379     Amount = -Amount;
380     Opc = RISCV::SUB;
381   }
382   // 1. Multiply the number of v-slots to the length of registers
383   Register FactorRegister =
384       TII->getVLENFactoredAmount(MF, MBB, MBBI, DL, Amount, Flag);
385   // 2. SP = SP - RVV stack size
386   BuildMI(MBB, MBBI, DL, TII->get(Opc), SPReg)
387       .addReg(SPReg)
388       .addReg(FactorRegister, RegState::Kill)
389       .setMIFlag(Flag);
390 }
391 
392 void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
393                                       MachineBasicBlock &MBB) const {
394   MachineFrameInfo &MFI = MF.getFrameInfo();
395   auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
396   const RISCVRegisterInfo *RI = STI.getRegisterInfo();
397   const RISCVInstrInfo *TII = STI.getInstrInfo();
398   MachineBasicBlock::iterator MBBI = MBB.begin();
399 
400   Register FPReg = getFPReg(STI);
401   Register SPReg = getSPReg(STI);
402   Register BPReg = RISCVABI::getBPReg();
403 
404   // Debug location must be unknown since the first debug location is used
405   // to determine the end of the prologue.
406   DebugLoc DL;
407 
408   // All calls are tail calls in GHC calling conv, and functions have no
409   // prologue/epilogue.
410   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
411     return;
412 
413   // Emit prologue for shadow call stack.
414   emitSCSPrologue(MF, MBB, MBBI, DL);
415 
416   // Since spillCalleeSavedRegisters may have inserted a libcall, skip past
417   // any instructions marked as FrameSetup
418   while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
419     ++MBBI;
420 
421   // Determine the correct frame layout
422   determineFrameLayout(MF);
423 
424   // If libcalls are used to spill and restore callee-saved registers, the frame
425   // has two sections; the opaque section managed by the libcalls, and the
426   // section managed by MachineFrameInfo which can also hold callee saved
427   // registers in fixed stack slots, both of which have negative frame indices.
428   // This gets even more complicated when incoming arguments are passed via the
429   // stack, as these too have negative frame indices. An example is detailed
430   // below:
431   //
432   //  | incoming arg | <- FI[-3]
433   //  | libcallspill |
434   //  | calleespill  | <- FI[-2]
435   //  | calleespill  | <- FI[-1]
436   //  | this_frame   | <- FI[0]
437   //
438   // For negative frame indices, the offset from the frame pointer will differ
439   // depending on which of these groups the frame index applies to.
440   // The following calculates the correct offset knowing the number of callee
441   // saved registers spilt by the two methods.
442   if (int LibCallRegs = getLibCallID(MF, MFI.getCalleeSavedInfo()) + 1) {
443     // Calculate the size of the frame managed by the libcall. The libcalls are
444     // implemented such that the stack will always be 16 byte aligned.
445     unsigned LibCallFrameSize = alignTo((STI.getXLen() / 8) * LibCallRegs, 16);
446     RVFI->setLibCallStackSize(LibCallFrameSize);
447   }
448 
449   // FIXME (note copied from Lanai): This appears to be overallocating.  Needs
450   // investigation. Get the number of bytes to allocate from the FrameInfo.
451   uint64_t StackSize = getStackSizeWithRVVPadding(MF);
452   uint64_t RealStackSize = StackSize + RVFI->getLibCallStackSize();
453   uint64_t RVVStackSize = RVFI->getRVVStackSize();
454 
455   // Early exit if there is no need to allocate on the stack
456   if (RealStackSize == 0 && !MFI.adjustsStack() && RVVStackSize == 0)
457     return;
458 
459   // If the stack pointer has been marked as reserved, then produce an error if
460   // the frame requires stack allocation
461   if (STI.isRegisterReservedByUser(SPReg))
462     MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
463         MF.getFunction(), "Stack pointer required, but has been reserved."});
464 
465   uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
466   // Split the SP adjustment to reduce the offsets of callee saved spill.
467   if (FirstSPAdjustAmount) {
468     StackSize = FirstSPAdjustAmount;
469     RealStackSize = FirstSPAdjustAmount;
470   }
471 
472   // Allocate space on the stack if necessary.
473   adjustReg(MBB, MBBI, DL, SPReg, SPReg, -StackSize, MachineInstr::FrameSetup);
474 
475   // Emit ".cfi_def_cfa_offset RealStackSize"
476   unsigned CFIIndex = MF.addFrameInst(
477       MCCFIInstruction::cfiDefCfaOffset(nullptr, RealStackSize));
478   BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
479       .addCFIIndex(CFIIndex)
480       .setMIFlag(MachineInstr::FrameSetup);
481 
482   const auto &CSI = MFI.getCalleeSavedInfo();
483 
484   // The frame pointer is callee-saved, and code has been generated for us to
485   // save it to the stack. We need to skip over the storing of callee-saved
486   // registers as the frame pointer must be modified after it has been saved
487   // to the stack, not before.
488   // FIXME: assumes exactly one instruction is used to save each callee-saved
489   // register.
490   std::advance(MBBI, getNonLibcallCSI(MF, CSI).size());
491 
492   // Iterate over list of callee-saved registers and emit .cfi_offset
493   // directives.
494   for (const auto &Entry : CSI) {
495     int FrameIdx = Entry.getFrameIdx();
496     int64_t Offset;
497     // Offsets for objects with fixed locations (IE: those saved by libcall) are
498     // simply calculated from the frame index.
499     if (FrameIdx < 0)
500       Offset = FrameIdx * (int64_t) STI.getXLen() / 8;
501     else
502       Offset = MFI.getObjectOffset(Entry.getFrameIdx()) -
503                RVFI->getLibCallStackSize();
504     Register Reg = Entry.getReg();
505     unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
506         nullptr, RI->getDwarfRegNum(Reg, true), Offset));
507     BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
508         .addCFIIndex(CFIIndex)
509         .setMIFlag(MachineInstr::FrameSetup);
510   }
511 
512   // Generate new FP.
513   if (hasFP(MF)) {
514     if (STI.isRegisterReservedByUser(FPReg))
515       MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
516           MF.getFunction(), "Frame pointer required, but has been reserved."});
517 
518     adjustReg(MBB, MBBI, DL, FPReg, SPReg,
519               RealStackSize - RVFI->getVarArgsSaveSize(),
520               MachineInstr::FrameSetup);
521 
522     // Emit ".cfi_def_cfa $fp, RVFI->getVarArgsSaveSize()"
523     unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
524         nullptr, RI->getDwarfRegNum(FPReg, true), RVFI->getVarArgsSaveSize()));
525     BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
526         .addCFIIndex(CFIIndex)
527         .setMIFlag(MachineInstr::FrameSetup);
528   }
529 
530   // Emit the second SP adjustment after saving callee saved registers.
531   if (FirstSPAdjustAmount) {
532     uint64_t SecondSPAdjustAmount =
533         getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
534     assert(SecondSPAdjustAmount > 0 &&
535            "SecondSPAdjustAmount should be greater than zero");
536     adjustReg(MBB, MBBI, DL, SPReg, SPReg, -SecondSPAdjustAmount,
537               MachineInstr::FrameSetup);
538 
539     // If we are using a frame-pointer, and thus emitted ".cfi_def_cfa fp, 0",
540     // don't emit an sp-based .cfi_def_cfa_offset
541     if (!hasFP(MF)) {
542       // Emit ".cfi_def_cfa_offset StackSize"
543       unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(
544           nullptr, getStackSizeWithRVVPadding(MF)));
545       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
546           .addCFIIndex(CFIIndex)
547           .setMIFlag(MachineInstr::FrameSetup);
548     }
549   }
550 
551   if (RVVStackSize)
552     adjustStackForRVV(MF, MBB, MBBI, DL, -RVVStackSize,
553                       MachineInstr::FrameSetup);
554 
555   if (hasFP(MF)) {
556     // Realign Stack
557     const RISCVRegisterInfo *RI = STI.getRegisterInfo();
558     if (RI->hasStackRealignment(MF)) {
559       Align MaxAlignment = MFI.getMaxAlign();
560 
561       const RISCVInstrInfo *TII = STI.getInstrInfo();
562       if (isInt<12>(-(int)MaxAlignment.value())) {
563         BuildMI(MBB, MBBI, DL, TII->get(RISCV::ANDI), SPReg)
564             .addReg(SPReg)
565             .addImm(-(int)MaxAlignment.value())
566             .setMIFlag(MachineInstr::FrameSetup);
567       } else {
568         unsigned ShiftAmount = Log2(MaxAlignment);
569         Register VR =
570             MF.getRegInfo().createVirtualRegister(&RISCV::GPRRegClass);
571         BuildMI(MBB, MBBI, DL, TII->get(RISCV::SRLI), VR)
572             .addReg(SPReg)
573             .addImm(ShiftAmount)
574             .setMIFlag(MachineInstr::FrameSetup);
575         BuildMI(MBB, MBBI, DL, TII->get(RISCV::SLLI), SPReg)
576             .addReg(VR)
577             .addImm(ShiftAmount)
578             .setMIFlag(MachineInstr::FrameSetup);
579       }
580       // FP will be used to restore the frame in the epilogue, so we need
581       // another base register BP to record SP after re-alignment. SP will
582       // track the current stack after allocating variable sized objects.
583       if (hasBP(MF)) {
584         // move BP, SP
585         BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), BPReg)
586             .addReg(SPReg)
587             .addImm(0)
588             .setMIFlag(MachineInstr::FrameSetup);
589       }
590     }
591   }
592 }
593 
594 void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
595                                       MachineBasicBlock &MBB) const {
596   const RISCVRegisterInfo *RI = STI.getRegisterInfo();
597   MachineFrameInfo &MFI = MF.getFrameInfo();
598   auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
599   Register FPReg = getFPReg(STI);
600   Register SPReg = getSPReg(STI);
601 
602   // All calls are tail calls in GHC calling conv, and functions have no
603   // prologue/epilogue.
604   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
605     return;
606 
607   // Get the insert location for the epilogue. If there were no terminators in
608   // the block, get the last instruction.
609   MachineBasicBlock::iterator MBBI = MBB.end();
610   DebugLoc DL;
611   if (!MBB.empty()) {
612     MBBI = MBB.getLastNonDebugInstr();
613     if (MBBI != MBB.end())
614       DL = MBBI->getDebugLoc();
615 
616     MBBI = MBB.getFirstTerminator();
617 
618     // If callee-saved registers are saved via libcall, place stack adjustment
619     // before this call.
620     while (MBBI != MBB.begin() &&
621            std::prev(MBBI)->getFlag(MachineInstr::FrameDestroy))
622       --MBBI;
623   }
624 
625   const auto &CSI = getNonLibcallCSI(MF, MFI.getCalleeSavedInfo());
626 
627   // Skip to before the restores of callee-saved registers
628   // FIXME: assumes exactly one instruction is used to restore each
629   // callee-saved register.
630   auto LastFrameDestroy = MBBI;
631   if (!CSI.empty())
632     LastFrameDestroy = std::prev(MBBI, CSI.size());
633 
634   uint64_t StackSize = getStackSizeWithRVVPadding(MF);
635   uint64_t RealStackSize = StackSize + RVFI->getLibCallStackSize();
636   uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize();
637   uint64_t RVVStackSize = RVFI->getRVVStackSize();
638 
639   // Restore the stack pointer using the value of the frame pointer. Only
640   // necessary if the stack pointer was modified, meaning the stack size is
641   // unknown.
642   //
643   // In order to make sure the stack point is right through the EH region,
644   // we also need to restore stack pointer from the frame pointer if we
645   // don't preserve stack space within prologue/epilogue for outgoing variables,
646   // normally it's just checking the variable sized object is present or not
647   // is enough, but we also don't preserve that at prologue/epilogue when
648   // have vector objects in stack.
649   if (RI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
650       !hasReservedCallFrame(MF)) {
651     assert(hasFP(MF) && "frame pointer should not have been eliminated");
652     adjustReg(MBB, LastFrameDestroy, DL, SPReg, FPReg, -FPOffset,
653               MachineInstr::FrameDestroy);
654   } else {
655     if (RVVStackSize)
656       adjustStackForRVV(MF, MBB, LastFrameDestroy, DL, RVVStackSize,
657                         MachineInstr::FrameDestroy);
658   }
659 
660   uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
661   if (FirstSPAdjustAmount) {
662     uint64_t SecondSPAdjustAmount =
663         getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
664     assert(SecondSPAdjustAmount > 0 &&
665            "SecondSPAdjustAmount should be greater than zero");
666 
667     adjustReg(MBB, LastFrameDestroy, DL, SPReg, SPReg, SecondSPAdjustAmount,
668               MachineInstr::FrameDestroy);
669   }
670 
671   if (FirstSPAdjustAmount)
672     StackSize = FirstSPAdjustAmount;
673 
674   // Deallocate stack
675   adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackSize, MachineInstr::FrameDestroy);
676 
677   // Emit epilogue for shadow call stack.
678   emitSCSEpilogue(MF, MBB, MBBI, DL);
679 }
680 
681 StackOffset
682 RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
683                                            Register &FrameReg) const {
684   const MachineFrameInfo &MFI = MF.getFrameInfo();
685   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
686   const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
687 
688   // Callee-saved registers should be referenced relative to the stack
689   // pointer (positive offset), otherwise use the frame pointer (negative
690   // offset).
691   const auto &CSI = getNonLibcallCSI(MF, MFI.getCalleeSavedInfo());
692   int MinCSFI = 0;
693   int MaxCSFI = -1;
694   StackOffset Offset;
695   auto StackID = MFI.getStackID(FI);
696 
697   assert((StackID == TargetStackID::Default ||
698           StackID == TargetStackID::ScalableVector) &&
699          "Unexpected stack ID for the frame object.");
700   if (StackID == TargetStackID::Default) {
701     Offset =
702         StackOffset::getFixed(MFI.getObjectOffset(FI) - getOffsetOfLocalArea() +
703                               MFI.getOffsetAdjustment());
704   } else if (StackID == TargetStackID::ScalableVector) {
705     Offset = StackOffset::getScalable(MFI.getObjectOffset(FI));
706   }
707 
708   uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
709 
710   if (CSI.size()) {
711     MinCSFI = CSI[0].getFrameIdx();
712     MaxCSFI = CSI[CSI.size() - 1].getFrameIdx();
713   }
714 
715   if (FI >= MinCSFI && FI <= MaxCSFI) {
716     FrameReg = RISCV::X2;
717 
718     if (FirstSPAdjustAmount)
719       Offset += StackOffset::getFixed(FirstSPAdjustAmount);
720     else
721       Offset += StackOffset::getFixed(getStackSizeWithRVVPadding(MF));
722     return Offset;
723   }
724 
725   if (RI->hasStackRealignment(MF) && !MFI.isFixedObjectIndex(FI)) {
726     // If the stack was realigned, the frame pointer is set in order to allow
727     // SP to be restored, so we need another base register to record the stack
728     // after realignment.
729     // |--------------------------| -- <-- FP
730     // | callee-allocated save    | | <----|
731     // | area for register varargs| |      |
732     // |--------------------------| |      |
733     // | callee-saved registers   | |      |
734     // |--------------------------| --     |
735     // | realignment (the size of | |      |
736     // | this area is not counted | |      |
737     // | in MFI.getStackSize())   | |      |
738     // |--------------------------| --     |-- MFI.getStackSize()
739     // | RVV alignment padding    | |      |
740     // | (not counted in          | |      |
741     // | MFI.getStackSize() but   | |      |
742     // | counted in               | |      |
743     // | RVFI.getRVVStackSize())  | |      |
744     // |--------------------------| --     |
745     // | RVV objects              | |      |
746     // | (not counted in          | |      |
747     // | MFI.getStackSize())      | |      |
748     // |--------------------------| --     |
749     // | padding before RVV       | |      |
750     // | (not counted in          | |      |
751     // | MFI.getStackSize() or in | |      |
752     // | RVFI.getRVVStackSize())  | |      |
753     // |--------------------------| --     |
754     // | scalar local variables   | | <----'
755     // |--------------------------| -- <-- BP (if var sized objects present)
756     // | VarSize objects          | |
757     // |--------------------------| -- <-- SP
758     if (hasBP(MF)) {
759       FrameReg = RISCVABI::getBPReg();
760     } else {
761       // VarSize objects must be empty in this case!
762       assert(!MFI.hasVarSizedObjects());
763       FrameReg = RISCV::X2;
764     }
765   } else {
766     FrameReg = RI->getFrameRegister(MF);
767   }
768 
769   if (FrameReg == getFPReg(STI)) {
770     Offset += StackOffset::getFixed(RVFI->getVarArgsSaveSize());
771     if (FI >= 0)
772       Offset -= StackOffset::getFixed(RVFI->getLibCallStackSize());
773     // When using FP to access scalable vector objects, we need to minus
774     // the frame size.
775     //
776     // |--------------------------| -- <-- FP
777     // | callee-allocated save    | |
778     // | area for register varargs| |
779     // |--------------------------| |
780     // | callee-saved registers   | |
781     // |--------------------------| | MFI.getStackSize()
782     // | scalar local variables   | |
783     // |--------------------------| -- (Offset of RVV objects is from here.)
784     // | RVV objects              |
785     // |--------------------------|
786     // | VarSize objects          |
787     // |--------------------------| <-- SP
788     if (MFI.getStackID(FI) == TargetStackID::ScalableVector) {
789       assert(!RI->hasStackRealignment(MF) &&
790              "Can't index across variable sized realign");
791       // We don't expect any extra RVV alignment padding, as the stack size
792       // and RVV object sections should be correct aligned in their own
793       // right.
794       assert(MFI.getStackSize() == getStackSizeWithRVVPadding(MF) &&
795              "Inconsistent stack layout");
796       Offset -= StackOffset::getFixed(MFI.getStackSize());
797     }
798     return Offset;
799   }
800 
801   // This case handles indexing off both SP and BP.
802   // If indexing off SP, there must not be any var sized objects
803   assert(FrameReg == RISCVABI::getBPReg() || !MFI.hasVarSizedObjects());
804 
805   // When using SP to access frame objects, we need to add RVV stack size.
806   //
807   // |--------------------------| -- <-- FP
808   // | callee-allocated save    | | <----|
809   // | area for register varargs| |      |
810   // |--------------------------| |      |
811   // | callee-saved registers   | |      |
812   // |--------------------------| --     |
813   // | RVV alignment padding    | |      |
814   // | (not counted in          | |      |
815   // | MFI.getStackSize() but   | |      |
816   // | counted in               | |      |
817   // | RVFI.getRVVStackSize())  | |      |
818   // |--------------------------| --     |
819   // | RVV objects              | |      |-- MFI.getStackSize()
820   // | (not counted in          | |      |
821   // | MFI.getStackSize())      | |      |
822   // |--------------------------| --     |
823   // | padding before RVV       | |      |
824   // | (not counted in          | |      |
825   // | MFI.getStackSize())      | |      |
826   // |--------------------------| --     |
827   // | scalar local variables   | | <----'
828   // |--------------------------| -- <-- BP (if var sized objects present)
829   // | VarSize objects          | |
830   // |--------------------------| -- <-- SP
831   //
832   // The total amount of padding surrounding RVV objects is described by
833   // RVV->getRVVPadding() and it can be zero. It allows us to align the RVV
834   // objects to the required alignment.
835   if (MFI.getStackID(FI) == TargetStackID::Default) {
836     if (MFI.isFixedObjectIndex(FI)) {
837       assert(!RI->hasStackRealignment(MF) &&
838              "Can't index across variable sized realign");
839       Offset += StackOffset::get(getStackSizeWithRVVPadding(MF) +
840                                  RVFI->getLibCallStackSize(),
841                                  RVFI->getRVVStackSize());
842     } else {
843       Offset += StackOffset::getFixed(MFI.getStackSize());
844     }
845   } else if (MFI.getStackID(FI) == TargetStackID::ScalableVector) {
846     // Ensure the base of the RVV stack is correctly aligned: add on the
847     // alignment padding.
848     int ScalarLocalVarSize =
849       MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
850       RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
851     Offset += StackOffset::get(ScalarLocalVarSize, RVFI->getRVVStackSize());
852   }
853   return Offset;
854 }
855 
856 void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
857                                               BitVector &SavedRegs,
858                                               RegScavenger *RS) const {
859   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
860   // Unconditionally spill RA and FP only if the function uses a frame
861   // pointer.
862   if (hasFP(MF)) {
863     SavedRegs.set(RISCV::X1);
864     SavedRegs.set(RISCV::X8);
865   }
866   // Mark BP as used if function has dedicated base pointer.
867   if (hasBP(MF))
868     SavedRegs.set(RISCVABI::getBPReg());
869 
870   // If interrupt is enabled and there are calls in the handler,
871   // unconditionally save all Caller-saved registers and
872   // all FP registers, regardless whether they are used.
873   MachineFrameInfo &MFI = MF.getFrameInfo();
874 
875   if (MF.getFunction().hasFnAttribute("interrupt") && MFI.hasCalls()) {
876 
877     static const MCPhysReg CSRegs[] = { RISCV::X1,      /* ra */
878       RISCV::X5, RISCV::X6, RISCV::X7,                  /* t0-t2 */
879       RISCV::X10, RISCV::X11,                           /* a0-a1, a2-a7 */
880       RISCV::X12, RISCV::X13, RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17,
881       RISCV::X28, RISCV::X29, RISCV::X30, RISCV::X31, 0 /* t3-t6 */
882     };
883 
884     for (unsigned i = 0; CSRegs[i]; ++i)
885       SavedRegs.set(CSRegs[i]);
886 
887     if (MF.getSubtarget<RISCVSubtarget>().hasStdExtF()) {
888 
889       // If interrupt is enabled, this list contains all FP registers.
890       const MCPhysReg * Regs = MF.getRegInfo().getCalleeSavedRegs();
891 
892       for (unsigned i = 0; Regs[i]; ++i)
893         if (RISCV::FPR16RegClass.contains(Regs[i]) ||
894             RISCV::FPR32RegClass.contains(Regs[i]) ||
895             RISCV::FPR64RegClass.contains(Regs[i]))
896           SavedRegs.set(Regs[i]);
897     }
898   }
899 }
900 
901 std::pair<int64_t, Align>
902 RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const {
903   MachineFrameInfo &MFI = MF.getFrameInfo();
904   // Create a buffer of RVV objects to allocate.
905   SmallVector<int, 8> ObjectsToAllocate;
906   for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
907     unsigned StackID = MFI.getStackID(I);
908     if (StackID != TargetStackID::ScalableVector)
909       continue;
910     if (MFI.isDeadObjectIndex(I))
911       continue;
912 
913     ObjectsToAllocate.push_back(I);
914   }
915 
916   // The minimum alignment is 16 bytes.
917   Align RVVStackAlign(16);
918   const auto &ST = MF.getSubtarget<RISCVSubtarget>();
919 
920   if (!ST.hasVInstructions()) {
921     assert(ObjectsToAllocate.empty() &&
922            "Can't allocate scalable-vector objects without V instructions");
923     return std::make_pair(0, RVVStackAlign);
924   }
925 
926   // Allocate all RVV locals and spills
927   int64_t Offset = 0;
928   for (int FI : ObjectsToAllocate) {
929     // ObjectSize in bytes.
930     int64_t ObjectSize = MFI.getObjectSize(FI);
931     auto ObjectAlign = std::max(Align(8), MFI.getObjectAlign(FI));
932     // If the data type is the fractional vector type, reserve one vector
933     // register for it.
934     if (ObjectSize < 8)
935       ObjectSize = 8;
936     Offset = alignTo(Offset + ObjectSize, ObjectAlign);
937     MFI.setObjectOffset(FI, -Offset);
938     // Update the maximum alignment of the RVV stack section
939     RVVStackAlign = std::max(RVVStackAlign, ObjectAlign);
940   }
941 
942   // Ensure the alignment of the RVV stack. Since we want the most-aligned
943   // object right at the bottom (i.e., any padding at the top of the frame),
944   // readjust all RVV objects down by the alignment padding.
945   uint64_t StackSize = Offset;
946   if (auto AlignmentPadding = offsetToAlignment(StackSize, RVVStackAlign)) {
947     StackSize += AlignmentPadding;
948     for (int FI : ObjectsToAllocate)
949       MFI.setObjectOffset(FI, MFI.getObjectOffset(FI) - AlignmentPadding);
950   }
951 
952   return std::make_pair(StackSize, RVVStackAlign);
953 }
954 
955 static unsigned getScavSlotsNumForRVV(MachineFunction &MF) {
956   // For RVV spill, scalable stack offsets computing requires up to two scratch
957   // registers
958   static constexpr unsigned ScavSlotsNumRVVSpillScalableObject = 2;
959 
960   // For RVV spill, non-scalable stack offsets computing requires up to one
961   // scratch register.
962   static constexpr unsigned ScavSlotsNumRVVSpillNonScalableObject = 1;
963 
964   // ADDI instruction's destination register can be used for computing
965   // offsets. So Scalable stack offsets require up to one scratch register.
966   static constexpr unsigned ScavSlotsADDIScalableObject = 1;
967 
968   static constexpr unsigned MaxScavSlotsNumKnown =
969       std::max({ScavSlotsADDIScalableObject, ScavSlotsNumRVVSpillScalableObject,
970                 ScavSlotsNumRVVSpillNonScalableObject});
971 
972   unsigned MaxScavSlotsNum = 0;
973   if (!MF.getSubtarget<RISCVSubtarget>().hasVInstructions())
974     return false;
975   for (const MachineBasicBlock &MBB : MF)
976     for (const MachineInstr &MI : MBB) {
977       bool IsRVVSpill = RISCV::isRVVSpill(MI);
978       for (auto &MO : MI.operands()) {
979         if (!MO.isFI())
980           continue;
981         bool IsScalableVectorID = MF.getFrameInfo().getStackID(MO.getIndex()) ==
982                                   TargetStackID::ScalableVector;
983         if (IsRVVSpill) {
984           MaxScavSlotsNum = std::max(
985               MaxScavSlotsNum, IsScalableVectorID
986                                    ? ScavSlotsNumRVVSpillScalableObject
987                                    : ScavSlotsNumRVVSpillNonScalableObject);
988         } else if (MI.getOpcode() == RISCV::ADDI && IsScalableVectorID) {
989           MaxScavSlotsNum =
990               std::max(MaxScavSlotsNum, ScavSlotsADDIScalableObject);
991         }
992       }
993       if (MaxScavSlotsNum == MaxScavSlotsNumKnown)
994         return MaxScavSlotsNumKnown;
995     }
996   return MaxScavSlotsNum;
997 }
998 
999 void RISCVFrameLowering::processFunctionBeforeFrameFinalized(
1000     MachineFunction &MF, RegScavenger *RS) const {
1001   const RISCVRegisterInfo *RegInfo =
1002       MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
1003   MachineFrameInfo &MFI = MF.getFrameInfo();
1004   const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1005   auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1006 
1007   int64_t RVVStackSize;
1008   Align RVVStackAlign;
1009   std::tie(RVVStackSize, RVVStackAlign) = assignRVVStackObjectOffsets(MF);
1010 
1011   RVFI->setRVVStackSize(RVVStackSize);
1012   RVFI->setRVVStackAlign(RVVStackAlign);
1013 
1014   // Ensure the entire stack is aligned to at least the RVV requirement: some
1015   // scalable-vector object alignments are not considered by the
1016   // target-independent code.
1017   MFI.ensureMaxAlignment(RVVStackAlign);
1018 
1019   // estimateStackSize has been observed to under-estimate the final stack
1020   // size, so give ourselves wiggle-room by checking for stack size
1021   // representable an 11-bit signed field rather than 12-bits.
1022   // FIXME: It may be possible to craft a function with a small stack that
1023   // still needs an emergency spill slot for branch relaxation. This case
1024   // would currently be missed.
1025   // RVV loads & stores have no capacity to hold the immediate address offsets
1026   // so we must always reserve an emergency spill slot if the MachineFunction
1027   // contains any RVV spills.
1028   unsigned ScavSlotsNum = 0;
1029   if (!isInt<11>(MFI.estimateStackSize(MF)))
1030     ScavSlotsNum = 1;
1031 
1032   ScavSlotsNum = std::max(ScavSlotsNum, getScavSlotsNumForRVV(MF));
1033   for (unsigned i = 0; i < ScavSlotsNum; i++) {
1034     RS->addScavengingFrameIndex(MFI.CreateStackObject(
1035         RegInfo->getSpillSize(*RC), RegInfo->getSpillAlign(*RC), false));
1036   }
1037 
1038   if (MFI.getCalleeSavedInfo().empty() || RVFI->useSaveRestoreLibCalls(MF)) {
1039     RVFI->setCalleeSavedStackSize(0);
1040     return;
1041   }
1042 
1043   unsigned Size = 0;
1044   for (const auto &Info : MFI.getCalleeSavedInfo()) {
1045     int FrameIdx = Info.getFrameIdx();
1046     if (MFI.getStackID(FrameIdx) != TargetStackID::Default)
1047       continue;
1048 
1049     Size += MFI.getObjectSize(FrameIdx);
1050   }
1051   RVFI->setCalleeSavedStackSize(Size);
1052 }
1053 
1054 static bool hasRVVFrameObject(const MachineFunction &MF) {
1055   // Originally, the function will scan all the stack objects to check whether
1056   // if there is any scalable vector object on the stack or not. However, it
1057   // causes errors in the register allocator. In issue 53016, it returns false
1058   // before RA because there is no RVV stack objects. After RA, it returns true
1059   // because there are spilling slots for RVV values during RA. It will not
1060   // reserve BP during register allocation and generate BP access in the PEI
1061   // pass due to the inconsistent behavior of the function.
1062   //
1063   // The function is changed to use hasVInstructions() as the return value. It
1064   // is not precise, but it can make the register allocation correct.
1065   //
1066   // FIXME: Find a better way to make the decision or revisit the solution in
1067   // D103622.
1068   //
1069   // Refer to https://github.com/llvm/llvm-project/issues/53016.
1070   return MF.getSubtarget<RISCVSubtarget>().hasVInstructions();
1071 }
1072 
1073 // Not preserve stack space within prologue for outgoing variables when the
1074 // function contains variable size objects or there are vector objects accessed
1075 // by the frame pointer.
1076 // Let eliminateCallFramePseudoInstr preserve stack space for it.
1077 bool RISCVFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
1078   return !MF.getFrameInfo().hasVarSizedObjects() &&
1079          !(hasFP(MF) && hasRVVFrameObject(MF));
1080 }
1081 
1082 // Eliminate ADJCALLSTACKDOWN, ADJCALLSTACKUP pseudo instructions.
1083 MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
1084     MachineFunction &MF, MachineBasicBlock &MBB,
1085     MachineBasicBlock::iterator MI) const {
1086   Register SPReg = RISCV::X2;
1087   DebugLoc DL = MI->getDebugLoc();
1088 
1089   if (!hasReservedCallFrame(MF)) {
1090     // If space has not been reserved for a call frame, ADJCALLSTACKDOWN and
1091     // ADJCALLSTACKUP must be converted to instructions manipulating the stack
1092     // pointer. This is necessary when there is a variable length stack
1093     // allocation (e.g. alloca), which means it's not possible to allocate
1094     // space for outgoing arguments from within the function prologue.
1095     int64_t Amount = MI->getOperand(0).getImm();
1096 
1097     if (Amount != 0) {
1098       // Ensure the stack remains aligned after adjustment.
1099       Amount = alignSPAdjust(Amount);
1100 
1101       if (MI->getOpcode() == RISCV::ADJCALLSTACKDOWN)
1102         Amount = -Amount;
1103 
1104       adjustReg(MBB, MI, DL, SPReg, SPReg, Amount, MachineInstr::NoFlags);
1105     }
1106   }
1107 
1108   return MBB.erase(MI);
1109 }
1110 
1111 // We would like to split the SP adjustment to reduce prologue/epilogue
1112 // as following instructions. In this way, the offset of the callee saved
1113 // register could fit in a single store.
1114 //   add     sp,sp,-2032
1115 //   sw      ra,2028(sp)
1116 //   sw      s0,2024(sp)
1117 //   sw      s1,2020(sp)
1118 //   sw      s3,2012(sp)
1119 //   sw      s4,2008(sp)
1120 //   add     sp,sp,-64
1121 uint64_t
1122 RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const {
1123   const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1124   const MachineFrameInfo &MFI = MF.getFrameInfo();
1125   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
1126   uint64_t StackSize = getStackSizeWithRVVPadding(MF);
1127 
1128   // Disable SplitSPAdjust if save-restore libcall is used. The callee-saved
1129   // registers will be pushed by the save-restore libcalls, so we don't have to
1130   // split the SP adjustment in this case.
1131   if (RVFI->getLibCallStackSize())
1132     return 0;
1133 
1134   // Return the FirstSPAdjustAmount if the StackSize can not fit in a signed
1135   // 12-bit and there exists a callee-saved register needing to be pushed.
1136   if (!isInt<12>(StackSize) && (CSI.size() > 0)) {
1137     // FirstSPAdjustAmount is chosen as (2048 - StackAlign) because 2048 will
1138     // cause sp = sp + 2048 in the epilogue to be split into multiple
1139     // instructions. Offsets smaller than 2048 can fit in a single load/store
1140     // instruction, and we have to stick with the stack alignment. 2048 has
1141     // 16-byte alignment. The stack alignment for RV32 and RV64 is 16 and for
1142     // RV32E it is 4. So (2048 - StackAlign) will satisfy the stack alignment.
1143     return 2048 - getStackAlign().value();
1144   }
1145   return 0;
1146 }
1147 
1148 bool RISCVFrameLowering::spillCalleeSavedRegisters(
1149     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1150     ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1151   if (CSI.empty())
1152     return true;
1153 
1154   MachineFunction *MF = MBB.getParent();
1155   const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
1156   DebugLoc DL;
1157   if (MI != MBB.end() && !MI->isDebugInstr())
1158     DL = MI->getDebugLoc();
1159 
1160   const char *SpillLibCall = getSpillLibCallName(*MF, CSI);
1161   if (SpillLibCall) {
1162     // Add spill libcall via non-callee-saved register t0.
1163     BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoCALLReg), RISCV::X5)
1164         .addExternalSymbol(SpillLibCall, RISCVII::MO_CALL)
1165         .setMIFlag(MachineInstr::FrameSetup);
1166 
1167     // Add registers spilled in libcall as liveins.
1168     for (auto &CS : CSI)
1169       MBB.addLiveIn(CS.getReg());
1170   }
1171 
1172   // Manually spill values not spilled by libcall.
1173   const auto &NonLibcallCSI = getNonLibcallCSI(*MF, CSI);
1174   for (auto &CS : NonLibcallCSI) {
1175     // Insert the spill to the stack frame.
1176     Register Reg = CS.getReg();
1177     const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1178     TII.storeRegToStackSlot(MBB, MI, Reg, !MBB.isLiveIn(Reg), CS.getFrameIdx(),
1179                             RC, TRI);
1180   }
1181 
1182   return true;
1183 }
1184 
1185 bool RISCVFrameLowering::restoreCalleeSavedRegisters(
1186     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1187     MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1188   if (CSI.empty())
1189     return true;
1190 
1191   MachineFunction *MF = MBB.getParent();
1192   const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
1193   DebugLoc DL;
1194   if (MI != MBB.end() && !MI->isDebugInstr())
1195     DL = MI->getDebugLoc();
1196 
1197   // Manually restore values not restored by libcall.
1198   // Keep the same order as in the prologue. There is no need to reverse the
1199   // order in the epilogue. In addition, the return address will be restored
1200   // first in the epilogue. It increases the opportunity to avoid the
1201   // load-to-use data hazard between loading RA and return by RA.
1202   // loadRegFromStackSlot can insert multiple instructions.
1203   const auto &NonLibcallCSI = getNonLibcallCSI(*MF, CSI);
1204   for (auto &CS : NonLibcallCSI) {
1205     Register Reg = CS.getReg();
1206     const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1207     TII.loadRegFromStackSlot(MBB, MI, Reg, CS.getFrameIdx(), RC, TRI);
1208     assert(MI != MBB.begin() && "loadRegFromStackSlot didn't insert any code!");
1209   }
1210 
1211   const char *RestoreLibCall = getRestoreLibCallName(*MF, CSI);
1212   if (RestoreLibCall) {
1213     // Add restore libcall via tail call.
1214     MachineBasicBlock::iterator NewMI =
1215         BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoTAIL))
1216             .addExternalSymbol(RestoreLibCall, RISCVII::MO_CALL)
1217             .setMIFlag(MachineInstr::FrameDestroy);
1218 
1219     // Remove trailing returns, since the terminator is now a tail call to the
1220     // restore function.
1221     if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) {
1222       NewMI->copyImplicitOps(*MF, *MI);
1223       MI->eraseFromParent();
1224     }
1225   }
1226 
1227   return true;
1228 }
1229 
1230 bool RISCVFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
1231   // Keep the conventional code flow when not optimizing.
1232   if (MF.getFunction().hasOptNone())
1233     return false;
1234 
1235   return true;
1236 }
1237 
1238 bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
1239   MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
1240   const MachineFunction *MF = MBB.getParent();
1241   const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1242 
1243   if (!RVFI->useSaveRestoreLibCalls(*MF))
1244     return true;
1245 
1246   // Inserting a call to a __riscv_save libcall requires the use of the register
1247   // t0 (X5) to hold the return address. Therefore if this register is already
1248   // used we can't insert the call.
1249 
1250   RegScavenger RS;
1251   RS.enterBasicBlock(*TmpMBB);
1252   return !RS.isRegUsed(RISCV::X5);
1253 }
1254 
1255 bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
1256   const MachineFunction *MF = MBB.getParent();
1257   MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
1258   const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1259 
1260   if (!RVFI->useSaveRestoreLibCalls(*MF))
1261     return true;
1262 
1263   // Using the __riscv_restore libcalls to restore CSRs requires a tail call.
1264   // This means if we still need to continue executing code within this function
1265   // the restore cannot take place in this basic block.
1266 
1267   if (MBB.succ_size() > 1)
1268     return false;
1269 
1270   MachineBasicBlock *SuccMBB =
1271       MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin();
1272 
1273   // Doing a tail call should be safe if there are no successors, because either
1274   // we have a returning block or the end of the block is unreachable, so the
1275   // restore will be eliminated regardless.
1276   if (!SuccMBB)
1277     return true;
1278 
1279   // The successor can only contain a return, since we would effectively be
1280   // replacing the successor with our own tail return at the end of our block.
1281   return SuccMBB->isReturnBlock() && SuccMBB->size() == 1;
1282 }
1283 
1284 bool RISCVFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
1285   switch (ID) {
1286   case TargetStackID::Default:
1287   case TargetStackID::ScalableVector:
1288     return true;
1289   case TargetStackID::NoAlloc:
1290   case TargetStackID::SGPRSpill:
1291   case TargetStackID::WasmLocal:
1292     return false;
1293   }
1294   llvm_unreachable("Invalid TargetStackID::Value");
1295 }
1296 
1297 TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const {
1298   return TargetStackID::ScalableVector;
1299 }
1300