xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp (revision 2e3507c25e42292b45a5482e116d278f5515d04d) 
1 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "SIMachineFunctionInfo.h"
10 #include "AMDGPUTargetMachine.h"
11 #include "AMDGPUSubtarget.h"
12 #include "SIRegisterInfo.h"
13 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "llvm/CodeGen/LiveIntervals.h"
16 #include "llvm/CodeGen/MachineBasicBlock.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/CodeGen/MIRParser/MIParser.h"
21 #include "llvm/IR/CallingConv.h"
22 #include "llvm/IR/DiagnosticInfo.h"
23 #include "llvm/IR/Function.h"
24 #include <cassert>
25 #include <optional>
26 #include <vector>
27 
28 #define MAX_LANES 64
29 
30 using namespace llvm;
31 
32 const GCNTargetMachine &getTM(const GCNSubtarget *STI) {
33   const SITargetLowering *TLI = STI->getTargetLowering();
34   return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine());
35 }
36 
37 SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F,
38                                              const GCNSubtarget *STI)
39   : AMDGPUMachineFunction(F, *STI),
40     Mode(F),
41     GWSResourcePSV(getTM(STI)),
42     PrivateSegmentBuffer(false),
43     DispatchPtr(false),
44     QueuePtr(false),
45     KernargSegmentPtr(false),
46     DispatchID(false),
47     FlatScratchInit(false),
48     WorkGroupIDX(false),
49     WorkGroupIDY(false),
50     WorkGroupIDZ(false),
51     WorkGroupInfo(false),
52     LDSKernelId(false),
53     PrivateSegmentWaveByteOffset(false),
54     WorkItemIDX(false),
55     WorkItemIDY(false),
56     WorkItemIDZ(false),
57     ImplicitBufferPtr(false),
58     ImplicitArgPtr(false),
59     GITPtrHigh(0xffffffff),
60     HighBitsOf32BitAddress(0) {
61   const GCNSubtarget &ST = *static_cast<const GCNSubtarget *>(STI);
62   FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
63   WavesPerEU = ST.getWavesPerEU(F);
64 
65   Occupancy = ST.computeOccupancy(F, getLDSSize());
66   CallingConv::ID CC = F.getCallingConv();
67 
68   VRegFlags.reserve(1024);
69 
70   // FIXME: Should have analysis or something rather than attribute to detect
71   // calls.
72   const bool HasCalls = F.hasFnAttribute("amdgpu-calls");
73 
74   const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL ||
75                         CC == CallingConv::SPIR_KERNEL;
76 
77   if (IsKernel) {
78     if (!F.arg_empty() || ST.getImplicitArgNumBytes(F) != 0)
79       KernargSegmentPtr = true;
80     WorkGroupIDX = true;
81     WorkItemIDX = true;
82   } else if (CC == CallingConv::AMDGPU_PS) {
83     PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
84   }
85 
86   MayNeedAGPRs = ST.hasMAIInsts();
87 
88   if (!isEntryFunction()) {
89     if (CC != CallingConv::AMDGPU_Gfx)
90       ArgInfo = AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;
91 
92     // TODO: Pick a high register, and shift down, similar to a kernel.
93     FrameOffsetReg = AMDGPU::SGPR33;
94     StackPtrOffsetReg = AMDGPU::SGPR32;
95 
96     if (!ST.enableFlatScratch()) {
97       // Non-entry functions have no special inputs for now, other registers
98       // required for scratch access.
99       ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
100 
101       ArgInfo.PrivateSegmentBuffer =
102         ArgDescriptor::createRegister(ScratchRSrcReg);
103     }
104 
105     if (!F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))
106       ImplicitArgPtr = true;
107   } else {
108     ImplicitArgPtr = false;
109     MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
110                                MaxKernArgAlign);
111 
112     if (ST.hasGFX90AInsts() &&
113         ST.getMaxNumVGPRs(F) <= AMDGPU::VGPR_32RegClass.getNumRegs() &&
114         !mayUseAGPRs(F))
115       MayNeedAGPRs = false; // We will select all MAI with VGPR operands.
116   }
117 
118   bool isAmdHsaOrMesa = ST.isAmdHsaOrMesa(F);
119   if (isAmdHsaOrMesa && !ST.enableFlatScratch())
120     PrivateSegmentBuffer = true;
121   else if (ST.isMesaGfxShader(F))
122     ImplicitBufferPtr = true;
123 
124   if (!AMDGPU::isGraphics(CC) ||
125       (CC == CallingConv::AMDGPU_CS && ST.hasArchitectedSGPRs())) {
126     if (IsKernel || !F.hasFnAttribute("amdgpu-no-workgroup-id-x"))
127       WorkGroupIDX = true;
128 
129     if (!F.hasFnAttribute("amdgpu-no-workgroup-id-y"))
130       WorkGroupIDY = true;
131 
132     if (!F.hasFnAttribute("amdgpu-no-workgroup-id-z"))
133       WorkGroupIDZ = true;
134   }
135 
136   if (!AMDGPU::isGraphics(CC)) {
137     if (IsKernel || !F.hasFnAttribute("amdgpu-no-workitem-id-x"))
138       WorkItemIDX = true;
139 
140     if (!F.hasFnAttribute("amdgpu-no-workitem-id-y") &&
141         ST.getMaxWorkitemID(F, 1) != 0)
142       WorkItemIDY = true;
143 
144     if (!F.hasFnAttribute("amdgpu-no-workitem-id-z") &&
145         ST.getMaxWorkitemID(F, 2) != 0)
146       WorkItemIDZ = true;
147 
148     if (!F.hasFnAttribute("amdgpu-no-dispatch-ptr"))
149       DispatchPtr = true;
150 
151     if (!F.hasFnAttribute("amdgpu-no-queue-ptr"))
152       QueuePtr = true;
153 
154     if (!F.hasFnAttribute("amdgpu-no-dispatch-id"))
155       DispatchID = true;
156 
157     if (!IsKernel && !F.hasFnAttribute("amdgpu-no-lds-kernel-id"))
158       LDSKernelId = true;
159   }
160 
161   // FIXME: This attribute is a hack, we just need an analysis on the function
162   // to look for allocas.
163   bool HasStackObjects = F.hasFnAttribute("amdgpu-stack-objects");
164 
165   // TODO: This could be refined a lot. The attribute is a poor way of
166   // detecting calls or stack objects that may require it before argument
167   // lowering.
168   if (ST.hasFlatAddressSpace() && isEntryFunction() &&
169       (isAmdHsaOrMesa || ST.enableFlatScratch()) &&
170       (HasCalls || HasStackObjects || ST.enableFlatScratch()) &&
171       !ST.flatScratchIsArchitected()) {
172     FlatScratchInit = true;
173   }
174 
175   if (isEntryFunction()) {
176     // X, XY, and XYZ are the only supported combinations, so make sure Y is
177     // enabled if Z is.
178     if (WorkItemIDZ)
179       WorkItemIDY = true;
180 
181     if (!ST.flatScratchIsArchitected()) {
182       PrivateSegmentWaveByteOffset = true;
183 
184       // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
185       if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
186           (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
187         ArgInfo.PrivateSegmentWaveByteOffset =
188             ArgDescriptor::createRegister(AMDGPU::SGPR5);
189     }
190   }
191 
192   Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");
193   StringRef S = A.getValueAsString();
194   if (!S.empty())
195     S.consumeInteger(0, GITPtrHigh);
196 
197   A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
198   S = A.getValueAsString();
199   if (!S.empty())
200     S.consumeInteger(0, HighBitsOf32BitAddress);
201 
202   // On GFX908, in order to guarantee copying between AGPRs, we need a scratch
203   // VGPR available at all times. For now, reserve highest available VGPR. After
204   // RA, shift it to the lowest available unused VGPR if the one exist.
205   if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
206     VGPRForAGPRCopy =
207         AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(F) - 1);
208   }
209 }
210 
211 MachineFunctionInfo *SIMachineFunctionInfo::clone(
212     BumpPtrAllocator &Allocator, MachineFunction &DestMF,
213     const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
214     const {
215   return DestMF.cloneInfo<SIMachineFunctionInfo>(*this);
216 }
217 
218 void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
219   limitOccupancy(getMaxWavesPerEU());
220   const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
221   limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
222                  MF.getFunction()));
223 }
224 
225 Register SIMachineFunctionInfo::addPrivateSegmentBuffer(
226   const SIRegisterInfo &TRI) {
227   ArgInfo.PrivateSegmentBuffer =
228     ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
229     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
230   NumUserSGPRs += 4;
231   return ArgInfo.PrivateSegmentBuffer.getRegister();
232 }
233 
234 Register SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
235   ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
236     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
237   NumUserSGPRs += 2;
238   return ArgInfo.DispatchPtr.getRegister();
239 }
240 
241 Register SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
242   ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
243     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
244   NumUserSGPRs += 2;
245   return ArgInfo.QueuePtr.getRegister();
246 }
247 
248 Register SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
249   ArgInfo.KernargSegmentPtr
250     = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
251     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
252   NumUserSGPRs += 2;
253   return ArgInfo.KernargSegmentPtr.getRegister();
254 }
255 
256 Register SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
257   ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
258     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
259   NumUserSGPRs += 2;
260   return ArgInfo.DispatchID.getRegister();
261 }
262 
263 Register SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
264   ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
265     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
266   NumUserSGPRs += 2;
267   return ArgInfo.FlatScratchInit.getRegister();
268 }
269 
270 Register SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
271   ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
272     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
273   NumUserSGPRs += 2;
274   return ArgInfo.ImplicitBufferPtr.getRegister();
275 }
276 
277 Register SIMachineFunctionInfo::addLDSKernelId() {
278   ArgInfo.LDSKernelId = ArgDescriptor::createRegister(getNextUserSGPR());
279   NumUserSGPRs += 1;
280   return ArgInfo.LDSKernelId.getRegister();
281 }
282 
283 void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR,
284                                              uint64_t Size, Align Alignment) {
285   // Skip if it is an entry function or the register is already added.
286   if (isEntryFunction() || WWMSpills.count(VGPR))
287     return;
288 
289   WWMSpills.insert(std::make_pair(
290       VGPR, MF.getFrameInfo().CreateSpillStackObject(Size, Alignment)));
291 }
292 
293 // Separate out the callee-saved and scratch registers.
294 void SIMachineFunctionInfo::splitWWMSpillRegisters(
295     MachineFunction &MF,
296     SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
297     SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const {
298   const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
299   for (auto &Reg : WWMSpills) {
300     if (isCalleeSavedReg(CSRegs, Reg.first))
301       CalleeSavedRegs.push_back(Reg);
302     else
303       ScratchRegs.push_back(Reg);
304   }
305 }
306 
307 bool SIMachineFunctionInfo::isCalleeSavedReg(const MCPhysReg *CSRegs,
308                                              MCPhysReg Reg) const {
309   for (unsigned I = 0; CSRegs[I]; ++I) {
310     if (CSRegs[I] == Reg)
311       return true;
312   }
313 
314   return false;
315 }
316 
317 bool SIMachineFunctionInfo::allocateVGPRForSGPRSpills(MachineFunction &MF,
318                                                       int FI,
319                                                       unsigned LaneIndex) {
320   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
321   const SIRegisterInfo *TRI = ST.getRegisterInfo();
322   MachineRegisterInfo &MRI = MF.getRegInfo();
323   Register LaneVGPR;
324   if (!LaneIndex) {
325     LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
326     if (LaneVGPR == AMDGPU::NoRegister) {
327       // We have no VGPRs left for spilling SGPRs. Reset because we will not
328       // partially spill the SGPR to VGPRs.
329       SGPRSpillToVGPRLanes.erase(FI);
330       return false;
331     }
332 
333     SpillVGPRs.push_back(LaneVGPR);
334     // Add this register as live-in to all blocks to avoid machine verifier
335     // complaining about use of an undefined physical register.
336     for (MachineBasicBlock &BB : MF)
337       BB.addLiveIn(LaneVGPR);
338   } else {
339     LaneVGPR = SpillVGPRs.back();
340   }
341 
342   SGPRSpillToVGPRLanes[FI].push_back(
343       SIRegisterInfo::SpilledReg(LaneVGPR, LaneIndex));
344   return true;
345 }
346 
347 bool SIMachineFunctionInfo::allocateVGPRForPrologEpilogSGPRSpills(
348     MachineFunction &MF, int FI, unsigned LaneIndex) {
349   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
350   const SIRegisterInfo *TRI = ST.getRegisterInfo();
351   MachineRegisterInfo &MRI = MF.getRegInfo();
352   Register LaneVGPR;
353   if (!LaneIndex) {
354     LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
355     if (LaneVGPR == AMDGPU::NoRegister) {
356       // We have no VGPRs left for spilling SGPRs. Reset because we will not
357       // partially spill the SGPR to VGPRs.
358       PrologEpilogSGPRSpillToVGPRLanes.erase(FI);
359       return false;
360     }
361 
362     allocateWWMSpill(MF, LaneVGPR);
363   } else {
364     LaneVGPR = WWMSpills.back().first;
365   }
366 
367   PrologEpilogSGPRSpillToVGPRLanes[FI].push_back(
368       SIRegisterInfo::SpilledReg(LaneVGPR, LaneIndex));
369   return true;
370 }
371 
372 bool SIMachineFunctionInfo::allocateSGPRSpillToVGPRLane(MachineFunction &MF,
373                                                         int FI,
374                                                         bool IsPrologEpilog) {
375   std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
376       IsPrologEpilog ? PrologEpilogSGPRSpillToVGPRLanes[FI]
377                      : SGPRSpillToVGPRLanes[FI];
378 
379   // This has already been allocated.
380   if (!SpillLanes.empty())
381     return true;
382 
383   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
384   MachineFrameInfo &FrameInfo = MF.getFrameInfo();
385   unsigned WaveSize = ST.getWavefrontSize();
386 
387   unsigned Size = FrameInfo.getObjectSize(FI);
388   unsigned NumLanes = Size / 4;
389 
390   if (NumLanes > WaveSize)
391     return false;
392 
393   assert(Size >= 4 && "invalid sgpr spill size");
394   assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
395          "not spilling SGPRs to VGPRs");
396 
397   unsigned &NumSpillLanes =
398       IsPrologEpilog ? NumVGPRPrologEpilogSpillLanes : NumVGPRSpillLanes;
399 
400   for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) {
401     unsigned LaneIndex = (NumSpillLanes % WaveSize);
402 
403     bool Allocated =
404         IsPrologEpilog
405             ? allocateVGPRForPrologEpilogSGPRSpills(MF, FI, LaneIndex)
406             : allocateVGPRForSGPRSpills(MF, FI, LaneIndex);
407     if (!Allocated) {
408       NumSpillLanes -= I;
409       return false;
410     }
411   }
412 
413   return true;
414 }
415 
416 /// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
417 /// Either AGPR is spilled to VGPR to vice versa.
418 /// Returns true if a \p FI can be eliminated completely.
419 bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF,
420                                                     int FI,
421                                                     bool isAGPRtoVGPR) {
422   MachineRegisterInfo &MRI = MF.getRegInfo();
423   MachineFrameInfo &FrameInfo = MF.getFrameInfo();
424   const GCNSubtarget &ST =  MF.getSubtarget<GCNSubtarget>();
425 
426   assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));
427 
428   auto &Spill = VGPRToAGPRSpills[FI];
429 
430   // This has already been allocated.
431   if (!Spill.Lanes.empty())
432     return Spill.FullyAllocated;
433 
434   unsigned Size = FrameInfo.getObjectSize(FI);
435   unsigned NumLanes = Size / 4;
436   Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
437 
438   const TargetRegisterClass &RC =
439       isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
440   auto Regs = RC.getRegisters();
441 
442   auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
443   const SIRegisterInfo *TRI = ST.getRegisterInfo();
444   Spill.FullyAllocated = true;
445 
446   // FIXME: Move allocation logic out of MachineFunctionInfo and initialize
447   // once.
448   BitVector OtherUsedRegs;
449   OtherUsedRegs.resize(TRI->getNumRegs());
450 
451   const uint32_t *CSRMask =
452       TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());
453   if (CSRMask)
454     OtherUsedRegs.setBitsInMask(CSRMask);
455 
456   // TODO: Should include register tuples, but doesn't matter with current
457   // usage.
458   for (MCPhysReg Reg : SpillAGPR)
459     OtherUsedRegs.set(Reg);
460   for (MCPhysReg Reg : SpillVGPR)
461     OtherUsedRegs.set(Reg);
462 
463   SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();
464   for (int I = NumLanes - 1; I >= 0; --I) {
465     NextSpillReg = std::find_if(
466         NextSpillReg, Regs.end(), [&MRI, &OtherUsedRegs](MCPhysReg Reg) {
467           return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
468                  !OtherUsedRegs[Reg];
469         });
470 
471     if (NextSpillReg == Regs.end()) { // Registers exhausted
472       Spill.FullyAllocated = false;
473       break;
474     }
475 
476     OtherUsedRegs.set(*NextSpillReg);
477     SpillRegs.push_back(*NextSpillReg);
478     MRI.reserveReg(*NextSpillReg, TRI);
479     Spill.Lanes[I] = *NextSpillReg++;
480   }
481 
482   return Spill.FullyAllocated;
483 }
484 
485 bool SIMachineFunctionInfo::removeDeadFrameIndices(
486     MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) {
487   // Remove dead frame indices from function frame. And also make sure to remove
488   // the frame indices from `SGPRSpillToVGPRLanes` data structure, otherwise, it
489   // could result in an unexpected side effect and bug, in case of any
490   // re-mapping of freed frame indices by later pass(es) like "stack slot
491   // coloring".
492   for (auto &R : make_early_inc_range(SGPRSpillToVGPRLanes)) {
493     MFI.RemoveStackObject(R.first);
494     SGPRSpillToVGPRLanes.erase(R.first);
495   }
496 
497   bool HaveSGPRToMemory = false;
498 
499   if (ResetSGPRSpillStackIDs) {
500     // All other SGPRs must be allocated on the default stack, so reset the
501     // stack ID.
502     for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E;
503          ++I) {
504       if (!checkIndexInPrologEpilogSGPRSpills(I)) {
505         if (MFI.getStackID(I) == TargetStackID::SGPRSpill) {
506           MFI.setStackID(I, TargetStackID::Default);
507           HaveSGPRToMemory = true;
508         }
509       }
510     }
511   }
512 
513   for (auto &R : VGPRToAGPRSpills) {
514     if (R.second.IsDead)
515       MFI.RemoveStackObject(R.first);
516   }
517 
518   return HaveSGPRToMemory;
519 }
520 
521 int SIMachineFunctionInfo::getScavengeFI(MachineFrameInfo &MFI,
522                                          const SIRegisterInfo &TRI) {
523   if (ScavengeFI)
524     return *ScavengeFI;
525   if (isEntryFunction()) {
526     ScavengeFI = MFI.CreateFixedObject(
527         TRI.getSpillSize(AMDGPU::SGPR_32RegClass), 0, false);
528   } else {
529     ScavengeFI = MFI.CreateStackObject(
530         TRI.getSpillSize(AMDGPU::SGPR_32RegClass),
531         TRI.getSpillAlign(AMDGPU::SGPR_32RegClass), false);
532   }
533   return *ScavengeFI;
534 }
535 
536 MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
537   assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
538   return AMDGPU::SGPR0 + NumUserSGPRs;
539 }
540 
541 MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
542   return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
543 }
544 
545 void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(Register Reg) {
546   VRegFlags.grow(Reg);
547 }
548 
549 void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(Register NewReg,
550                                                          Register SrcReg) {
551   VRegFlags.grow(NewReg);
552   VRegFlags[NewReg] = VRegFlags[SrcReg];
553 }
554 
555 Register
556 SIMachineFunctionInfo::getGITPtrLoReg(const MachineFunction &MF) const {
557   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
558   if (!ST.isAmdPalOS())
559     return Register();
560   Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in
561   if (ST.hasMergedShaders()) {
562     switch (MF.getFunction().getCallingConv()) {
563     case CallingConv::AMDGPU_HS:
564     case CallingConv::AMDGPU_GS:
565       // Low GIT address is passed in s8 rather than s0 for an LS+HS or
566       // ES+GS merged shader on gfx9+.
567       GitPtrLo = AMDGPU::SGPR8;
568       return GitPtrLo;
569     default:
570       return GitPtrLo;
571     }
572   }
573   return GitPtrLo;
574 }
575 
576 static yaml::StringValue regToString(Register Reg,
577                                      const TargetRegisterInfo &TRI) {
578   yaml::StringValue Dest;
579   {
580     raw_string_ostream OS(Dest.Value);
581     OS << printReg(Reg, &TRI);
582   }
583   return Dest;
584 }
585 
586 static std::optional<yaml::SIArgumentInfo>
587 convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
588                     const TargetRegisterInfo &TRI) {
589   yaml::SIArgumentInfo AI;
590 
591   auto convertArg = [&](std::optional<yaml::SIArgument> &A,
592                         const ArgDescriptor &Arg) {
593     if (!Arg)
594       return false;
595 
596     // Create a register or stack argument.
597     yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister());
598     if (Arg.isRegister()) {
599       raw_string_ostream OS(SA.RegisterName.Value);
600       OS << printReg(Arg.getRegister(), &TRI);
601     } else
602       SA.StackOffset = Arg.getStackOffset();
603     // Check and update the optional mask.
604     if (Arg.isMasked())
605       SA.Mask = Arg.getMask();
606 
607     A = SA;
608     return true;
609   };
610 
611   bool Any = false;
612   Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
613   Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
614   Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
615   Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
616   Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
617   Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
618   Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId);
619   Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
620   Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
621   Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
622   Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
623   Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
624   Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
625                     ArgInfo.PrivateSegmentWaveByteOffset);
626   Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
627   Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
628   Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
629   Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
630   Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
631 
632   if (Any)
633     return AI;
634 
635   return std::nullopt;
636 }
637 
638 yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
639     const llvm::SIMachineFunctionInfo &MFI, const TargetRegisterInfo &TRI,
640     const llvm::MachineFunction &MF)
641     : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
642       MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
643       GDSSize(MFI.getGDSSize()),
644       DynLDSAlign(MFI.getDynLDSAlign()), IsEntryFunction(MFI.isEntryFunction()),
645       NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
646       MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),
647       HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
648       HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
649       HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
650       Occupancy(MFI.getOccupancy()),
651       ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
652       FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
653       StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),
654       BytesInStackArgArea(MFI.getBytesInStackArgArea()),
655       ReturnsVoid(MFI.returnsVoid()),
656       ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)),
657       PSInputAddr(MFI.getPSInputAddr()),
658       PSInputEnable(MFI.getPSInputEnable()),
659       Mode(MFI.getMode()) {
660   for (Register Reg : MFI.getWWMReservedRegs())
661     WWMReservedRegs.push_back(regToString(Reg, TRI));
662 
663   if (MFI.getLongBranchReservedReg())
664     LongBranchReservedReg = regToString(MFI.getLongBranchReservedReg(), TRI);
665   if (MFI.getVGPRForAGPRCopy())
666     VGPRForAGPRCopy = regToString(MFI.getVGPRForAGPRCopy(), TRI);
667 
668   if (MFI.getSGPRForEXECCopy())
669     SGPRForEXECCopy = regToString(MFI.getSGPRForEXECCopy(), TRI);
670 
671   auto SFI = MFI.getOptionalScavengeFI();
672   if (SFI)
673     ScavengeFI = yaml::FrameIndex(*SFI, MF.getFrameInfo());
674 }
675 
676 void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
677   MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this);
678 }
679 
680 bool SIMachineFunctionInfo::initializeBaseYamlFields(
681     const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF,
682     PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) {
683   ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
684   MaxKernArgAlign = YamlMFI.MaxKernArgAlign;
685   LDSSize = YamlMFI.LDSSize;
686   GDSSize = YamlMFI.GDSSize;
687   DynLDSAlign = YamlMFI.DynLDSAlign;
688   PSInputAddr = YamlMFI.PSInputAddr;
689   PSInputEnable = YamlMFI.PSInputEnable;
690   HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress;
691   Occupancy = YamlMFI.Occupancy;
692   IsEntryFunction = YamlMFI.IsEntryFunction;
693   NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath;
694   MemoryBound = YamlMFI.MemoryBound;
695   WaveLimiter = YamlMFI.WaveLimiter;
696   HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
697   HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
698   BytesInStackArgArea = YamlMFI.BytesInStackArgArea;
699   ReturnsVoid = YamlMFI.ReturnsVoid;
700 
701   if (YamlMFI.ScavengeFI) {
702     auto FIOrErr = YamlMFI.ScavengeFI->getFI(MF.getFrameInfo());
703     if (!FIOrErr) {
704       // Create a diagnostic for a the frame index.
705       const MemoryBuffer &Buffer =
706           *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID());
707 
708       Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1,
709                            SourceMgr::DK_Error, toString(FIOrErr.takeError()),
710                            "", std::nullopt, std::nullopt);
711       SourceRange = YamlMFI.ScavengeFI->SourceRange;
712       return true;
713     }
714     ScavengeFI = *FIOrErr;
715   } else {
716     ScavengeFI = std::nullopt;
717   }
718   return false;
719 }
720 
721 bool SIMachineFunctionInfo::mayUseAGPRs(const Function &F) const {
722   for (const BasicBlock &BB : F) {
723     for (const Instruction &I : BB) {
724       const auto *CB = dyn_cast<CallBase>(&I);
725       if (!CB)
726         continue;
727 
728       if (CB->isInlineAsm()) {
729         const InlineAsm *IA = dyn_cast<InlineAsm>(CB->getCalledOperand());
730         for (const auto &CI : IA->ParseConstraints()) {
731           for (StringRef Code : CI.Codes) {
732             Code.consume_front("{");
733             if (Code.startswith("a"))
734               return true;
735           }
736         }
737         continue;
738       }
739 
740       const Function *Callee =
741           dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts());
742       if (!Callee)
743         return true;
744 
745       if (Callee->getIntrinsicID() == Intrinsic::not_intrinsic)
746         return true;
747     }
748   }
749 
750   return false;
751 }
752 
753 bool SIMachineFunctionInfo::usesAGPRs(const MachineFunction &MF) const {
754   if (UsesAGPRs)
755     return *UsesAGPRs;
756 
757   if (!mayNeedAGPRs()) {
758     UsesAGPRs = false;
759     return false;
760   }
761 
762   if (!AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv()) ||
763       MF.getFrameInfo().hasCalls()) {
764     UsesAGPRs = true;
765     return true;
766   }
767 
768   const MachineRegisterInfo &MRI = MF.getRegInfo();
769 
770   for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
771     const Register Reg = Register::index2VirtReg(I);
772     const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
773     if (RC && SIRegisterInfo::isAGPRClass(RC)) {
774       UsesAGPRs = true;
775       return true;
776     } else if (!RC && !MRI.use_empty(Reg) && MRI.getType(Reg).isValid()) {
777       // Defer caching UsesAGPRs, function might not yet been regbank selected.
778       return true;
779     }
780   }
781 
782   for (MCRegister Reg : AMDGPU::AGPR_32RegClass) {
783     if (MRI.isPhysRegUsed(Reg)) {
784       UsesAGPRs = true;
785       return true;
786     }
787   }
788 
789   UsesAGPRs = false;
790   return false;
791 }
792