1 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "SIMachineFunctionInfo.h"
10 #include "AMDGPUSubtarget.h"
11 #include "AMDGPUTargetMachine.h"
12 #include "GCNSubtarget.h"
13 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14 #include "SIRegisterInfo.h"
15 #include "Utils/AMDGPUBaseInfo.h"
16 #include "llvm/CodeGen/LiveIntervals.h"
17 #include "llvm/CodeGen/MIRParser/MIParser.h"
18 #include "llvm/CodeGen/MachineBasicBlock.h"
19 #include "llvm/CodeGen/MachineFrameInfo.h"
20 #include "llvm/CodeGen/MachineFunction.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/IR/CallingConv.h"
23 #include "llvm/IR/DiagnosticInfo.h"
24 #include "llvm/IR/Function.h"
25 #include <cassert>
26 #include <optional>
27 #include <vector>
28
29 enum { MAX_LANES = 64 };
30
31 using namespace llvm;
32
getTM(const GCNSubtarget * STI)33 const GCNTargetMachine &getTM(const GCNSubtarget *STI) {
34 const SITargetLowering *TLI = STI->getTargetLowering();
35 return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine());
36 }
37
SIMachineFunctionInfo(const Function & F,const GCNSubtarget * STI)38 SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F,
39 const GCNSubtarget *STI)
40 : AMDGPUMachineFunction(F, *STI), Mode(F, *STI), GWSResourcePSV(getTM(STI)),
41 UserSGPRInfo(F, *STI), WorkGroupIDX(false), WorkGroupIDY(false),
42 WorkGroupIDZ(false), WorkGroupInfo(false), LDSKernelId(false),
43 PrivateSegmentWaveByteOffset(false), WorkItemIDX(false),
44 WorkItemIDY(false), WorkItemIDZ(false), ImplicitArgPtr(false),
45 GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0) {
46 const GCNSubtarget &ST = *static_cast<const GCNSubtarget *>(STI);
47 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
48 WavesPerEU = ST.getWavesPerEU(F);
49 MaxNumWorkGroups = ST.getMaxNumWorkGroups(F);
50 assert(MaxNumWorkGroups.size() == 3);
51
52 Occupancy = ST.computeOccupancy(F, getLDSSize());
53 CallingConv::ID CC = F.getCallingConv();
54
55 VRegFlags.reserve(1024);
56
57 const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL ||
58 CC == CallingConv::SPIR_KERNEL;
59
60 if (IsKernel) {
61 WorkGroupIDX = true;
62 WorkItemIDX = true;
63 } else if (CC == CallingConv::AMDGPU_PS) {
64 PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
65 }
66
67 MayNeedAGPRs = ST.hasMAIInsts();
68
69 if (AMDGPU::isChainCC(CC)) {
70 // Chain functions don't receive an SP from their caller, but are free to
71 // set one up. For now, we can use s32 to match what amdgpu_gfx functions
72 // would use if called, but this can be revisited.
73 // FIXME: Only reserve this if we actually need it.
74 StackPtrOffsetReg = AMDGPU::SGPR32;
75
76 ScratchRSrcReg = AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51;
77
78 ArgInfo.PrivateSegmentBuffer =
79 ArgDescriptor::createRegister(ScratchRSrcReg);
80
81 ImplicitArgPtr = false;
82 } else if (!isEntryFunction()) {
83 if (CC != CallingConv::AMDGPU_Gfx)
84 ArgInfo = AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;
85
86 FrameOffsetReg = AMDGPU::SGPR33;
87 StackPtrOffsetReg = AMDGPU::SGPR32;
88
89 if (!ST.enableFlatScratch()) {
90 // Non-entry functions have no special inputs for now, other registers
91 // required for scratch access.
92 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
93
94 ArgInfo.PrivateSegmentBuffer =
95 ArgDescriptor::createRegister(ScratchRSrcReg);
96 }
97
98 if (!F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))
99 ImplicitArgPtr = true;
100 } else {
101 ImplicitArgPtr = false;
102 MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
103 MaxKernArgAlign);
104
105 if (ST.hasGFX90AInsts() &&
106 ST.getMaxNumVGPRs(F) <= AMDGPU::VGPR_32RegClass.getNumRegs() &&
107 !mayUseAGPRs(F))
108 MayNeedAGPRs = false; // We will select all MAI with VGPR operands.
109 }
110
111 if (!AMDGPU::isGraphics(CC) ||
112 ((CC == CallingConv::AMDGPU_CS || CC == CallingConv::AMDGPU_Gfx) &&
113 ST.hasArchitectedSGPRs())) {
114 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workgroup-id-x"))
115 WorkGroupIDX = true;
116
117 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-y"))
118 WorkGroupIDY = true;
119
120 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-z"))
121 WorkGroupIDZ = true;
122 }
123
124 if (!AMDGPU::isGraphics(CC)) {
125 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workitem-id-x"))
126 WorkItemIDX = true;
127
128 if (!F.hasFnAttribute("amdgpu-no-workitem-id-y") &&
129 ST.getMaxWorkitemID(F, 1) != 0)
130 WorkItemIDY = true;
131
132 if (!F.hasFnAttribute("amdgpu-no-workitem-id-z") &&
133 ST.getMaxWorkitemID(F, 2) != 0)
134 WorkItemIDZ = true;
135
136 if (!IsKernel && !F.hasFnAttribute("amdgpu-no-lds-kernel-id"))
137 LDSKernelId = true;
138 }
139
140 if (isEntryFunction()) {
141 // X, XY, and XYZ are the only supported combinations, so make sure Y is
142 // enabled if Z is.
143 if (WorkItemIDZ)
144 WorkItemIDY = true;
145
146 if (!ST.flatScratchIsArchitected()) {
147 PrivateSegmentWaveByteOffset = true;
148
149 // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
150 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
151 (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
152 ArgInfo.PrivateSegmentWaveByteOffset =
153 ArgDescriptor::createRegister(AMDGPU::SGPR5);
154 }
155 }
156
157 Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");
158 StringRef S = A.getValueAsString();
159 if (!S.empty())
160 S.consumeInteger(0, GITPtrHigh);
161
162 A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
163 S = A.getValueAsString();
164 if (!S.empty())
165 S.consumeInteger(0, HighBitsOf32BitAddress);
166
167 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch
168 // VGPR available at all times. For now, reserve highest available VGPR. After
169 // RA, shift it to the lowest available unused VGPR if the one exist.
170 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
171 VGPRForAGPRCopy =
172 AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(F) - 1);
173 }
174 }
175
clone(BumpPtrAllocator & Allocator,MachineFunction & DestMF,const DenseMap<MachineBasicBlock *,MachineBasicBlock * > & Src2DstMBB) const176 MachineFunctionInfo *SIMachineFunctionInfo::clone(
177 BumpPtrAllocator &Allocator, MachineFunction &DestMF,
178 const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
179 const {
180 return DestMF.cloneInfo<SIMachineFunctionInfo>(*this);
181 }
182
limitOccupancy(const MachineFunction & MF)183 void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
184 limitOccupancy(getMaxWavesPerEU());
185 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
186 limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
187 MF.getFunction()));
188 }
189
addPrivateSegmentBuffer(const SIRegisterInfo & TRI)190 Register SIMachineFunctionInfo::addPrivateSegmentBuffer(
191 const SIRegisterInfo &TRI) {
192 ArgInfo.PrivateSegmentBuffer =
193 ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
194 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
195 NumUserSGPRs += 4;
196 return ArgInfo.PrivateSegmentBuffer.getRegister();
197 }
198
addDispatchPtr(const SIRegisterInfo & TRI)199 Register SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
200 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
201 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
202 NumUserSGPRs += 2;
203 return ArgInfo.DispatchPtr.getRegister();
204 }
205
addQueuePtr(const SIRegisterInfo & TRI)206 Register SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
207 ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
208 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
209 NumUserSGPRs += 2;
210 return ArgInfo.QueuePtr.getRegister();
211 }
212
addKernargSegmentPtr(const SIRegisterInfo & TRI)213 Register SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
214 ArgInfo.KernargSegmentPtr
215 = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
216 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
217 NumUserSGPRs += 2;
218 return ArgInfo.KernargSegmentPtr.getRegister();
219 }
220
addDispatchID(const SIRegisterInfo & TRI)221 Register SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
222 ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
223 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
224 NumUserSGPRs += 2;
225 return ArgInfo.DispatchID.getRegister();
226 }
227
addFlatScratchInit(const SIRegisterInfo & TRI)228 Register SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
229 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
230 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
231 NumUserSGPRs += 2;
232 return ArgInfo.FlatScratchInit.getRegister();
233 }
234
addPrivateSegmentSize(const SIRegisterInfo & TRI)235 Register SIMachineFunctionInfo::addPrivateSegmentSize(const SIRegisterInfo &TRI) {
236 ArgInfo.PrivateSegmentSize = ArgDescriptor::createRegister(getNextUserSGPR());
237 NumUserSGPRs += 1;
238 return ArgInfo.PrivateSegmentSize.getRegister();
239 }
240
addImplicitBufferPtr(const SIRegisterInfo & TRI)241 Register SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
242 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
243 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
244 NumUserSGPRs += 2;
245 return ArgInfo.ImplicitBufferPtr.getRegister();
246 }
247
addLDSKernelId()248 Register SIMachineFunctionInfo::addLDSKernelId() {
249 ArgInfo.LDSKernelId = ArgDescriptor::createRegister(getNextUserSGPR());
250 NumUserSGPRs += 1;
251 return ArgInfo.LDSKernelId.getRegister();
252 }
253
addPreloadedKernArg(const SIRegisterInfo & TRI,const TargetRegisterClass * RC,unsigned AllocSizeDWord,int KernArgIdx,int PaddingSGPRs)254 SmallVectorImpl<MCRegister> *SIMachineFunctionInfo::addPreloadedKernArg(
255 const SIRegisterInfo &TRI, const TargetRegisterClass *RC,
256 unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs) {
257 assert(!ArgInfo.PreloadKernArgs.count(KernArgIdx) &&
258 "Preload kernel argument allocated twice.");
259 NumUserSGPRs += PaddingSGPRs;
260 // If the available register tuples are aligned with the kernarg to be
261 // preloaded use that register, otherwise we need to use a set of SGPRs and
262 // merge them.
263 Register PreloadReg =
264 TRI.getMatchingSuperReg(getNextUserSGPR(), AMDGPU::sub0, RC);
265 if (PreloadReg &&
266 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) {
267 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(PreloadReg);
268 NumUserSGPRs += AllocSizeDWord;
269 } else {
270 for (unsigned I = 0; I < AllocSizeDWord; ++I) {
271 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(getNextUserSGPR());
272 NumUserSGPRs++;
273 }
274 }
275
276 // Track the actual number of SGPRs that HW will preload to.
277 UserSGPRInfo.allocKernargPreloadSGPRs(AllocSizeDWord + PaddingSGPRs);
278 return &ArgInfo.PreloadKernArgs[KernArgIdx].Regs;
279 }
280
allocateWWMSpill(MachineFunction & MF,Register VGPR,uint64_t Size,Align Alignment)281 void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR,
282 uint64_t Size, Align Alignment) {
283 // Skip if it is an entry function or the register is already added.
284 if (isEntryFunction() || WWMSpills.count(VGPR))
285 return;
286
287 // Skip if this is a function with the amdgpu_cs_chain or
288 // amdgpu_cs_chain_preserve calling convention and this is a scratch register.
289 // We never need to allocate a spill for these because we don't even need to
290 // restore the inactive lanes for them (they're scratchier than the usual
291 // scratch registers).
292 if (isChainFunction() && SIRegisterInfo::isChainScratchRegister(VGPR))
293 return;
294
295 WWMSpills.insert(std::make_pair(
296 VGPR, MF.getFrameInfo().CreateSpillStackObject(Size, Alignment)));
297 }
298
299 // Separate out the callee-saved and scratch registers.
splitWWMSpillRegisters(MachineFunction & MF,SmallVectorImpl<std::pair<Register,int>> & CalleeSavedRegs,SmallVectorImpl<std::pair<Register,int>> & ScratchRegs) const300 void SIMachineFunctionInfo::splitWWMSpillRegisters(
301 MachineFunction &MF,
302 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
303 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const {
304 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
305 for (auto &Reg : WWMSpills) {
306 if (isCalleeSavedReg(CSRegs, Reg.first))
307 CalleeSavedRegs.push_back(Reg);
308 else
309 ScratchRegs.push_back(Reg);
310 }
311 }
312
isCalleeSavedReg(const MCPhysReg * CSRegs,MCPhysReg Reg) const313 bool SIMachineFunctionInfo::isCalleeSavedReg(const MCPhysReg *CSRegs,
314 MCPhysReg Reg) const {
315 for (unsigned I = 0; CSRegs[I]; ++I) {
316 if (CSRegs[I] == Reg)
317 return true;
318 }
319
320 return false;
321 }
322
shiftSpillPhysVGPRsToLowestRange(MachineFunction & MF)323 void SIMachineFunctionInfo::shiftSpillPhysVGPRsToLowestRange(
324 MachineFunction &MF) {
325 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
326 MachineRegisterInfo &MRI = MF.getRegInfo();
327 for (Register &Reg : SpillPhysVGPRs) {
328 Register NewReg =
329 TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
330 if (!NewReg || NewReg >= Reg)
331 break;
332
333 MRI.replaceRegWith(Reg, NewReg);
334
335 // Update various tables with the new VGPR.
336 WWMReservedRegs.remove(Reg);
337 WWMReservedRegs.insert(NewReg);
338 WWMSpills.insert(std::make_pair(NewReg, WWMSpills[Reg]));
339 WWMSpills.erase(Reg);
340
341 for (MachineBasicBlock &MBB : MF) {
342 MBB.removeLiveIn(Reg);
343 MBB.sortUniqueLiveIns();
344 }
345
346 Reg = NewReg;
347 }
348 }
349
allocateVirtualVGPRForSGPRSpills(MachineFunction & MF,int FI,unsigned LaneIndex)350 bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(
351 MachineFunction &MF, int FI, unsigned LaneIndex) {
352 MachineRegisterInfo &MRI = MF.getRegInfo();
353 Register LaneVGPR;
354 if (!LaneIndex) {
355 LaneVGPR = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
356 SpillVGPRs.push_back(LaneVGPR);
357 } else {
358 LaneVGPR = SpillVGPRs.back();
359 }
360
361 SGPRSpillsToVirtualVGPRLanes[FI].emplace_back(LaneVGPR, LaneIndex);
362 return true;
363 }
364
allocatePhysicalVGPRForSGPRSpills(MachineFunction & MF,int FI,unsigned LaneIndex,bool IsPrologEpilog)365 bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(
366 MachineFunction &MF, int FI, unsigned LaneIndex, bool IsPrologEpilog) {
367 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
368 const SIRegisterInfo *TRI = ST.getRegisterInfo();
369 MachineRegisterInfo &MRI = MF.getRegInfo();
370 Register LaneVGPR;
371 if (!LaneIndex) {
372 // Find the highest available register if called before RA to ensure the
373 // lowest registers are available for allocation. The LaneVGPR, in that
374 // case, will be shifted back to the lowest range after VGPR allocation.
375 LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF,
376 !IsPrologEpilog);
377 if (LaneVGPR == AMDGPU::NoRegister) {
378 // We have no VGPRs left for spilling SGPRs. Reset because we will not
379 // partially spill the SGPR to VGPRs.
380 SGPRSpillsToPhysicalVGPRLanes.erase(FI);
381 return false;
382 }
383
384 allocateWWMSpill(MF, LaneVGPR);
385 reserveWWMRegister(LaneVGPR);
386 for (MachineBasicBlock &MBB : MF) {
387 MBB.addLiveIn(LaneVGPR);
388 MBB.sortUniqueLiveIns();
389 }
390 SpillPhysVGPRs.push_back(LaneVGPR);
391 } else {
392 LaneVGPR = SpillPhysVGPRs.back();
393 }
394
395 SGPRSpillsToPhysicalVGPRLanes[FI].emplace_back(LaneVGPR, LaneIndex);
396 return true;
397 }
398
allocateSGPRSpillToVGPRLane(MachineFunction & MF,int FI,bool SpillToPhysVGPRLane,bool IsPrologEpilog)399 bool SIMachineFunctionInfo::allocateSGPRSpillToVGPRLane(
400 MachineFunction &MF, int FI, bool SpillToPhysVGPRLane,
401 bool IsPrologEpilog) {
402 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
403 SpillToPhysVGPRLane ? SGPRSpillsToPhysicalVGPRLanes[FI]
404 : SGPRSpillsToVirtualVGPRLanes[FI];
405
406 // This has already been allocated.
407 if (!SpillLanes.empty())
408 return true;
409
410 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
411 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
412 unsigned WaveSize = ST.getWavefrontSize();
413
414 unsigned Size = FrameInfo.getObjectSize(FI);
415 unsigned NumLanes = Size / 4;
416
417 if (NumLanes > WaveSize)
418 return false;
419
420 assert(Size >= 4 && "invalid sgpr spill size");
421 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
422 "not spilling SGPRs to VGPRs");
423
424 unsigned &NumSpillLanes = SpillToPhysVGPRLane ? NumPhysicalVGPRSpillLanes
425 : NumVirtualVGPRSpillLanes;
426
427 for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) {
428 unsigned LaneIndex = (NumSpillLanes % WaveSize);
429
430 bool Allocated = SpillToPhysVGPRLane
431 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex,
432 IsPrologEpilog)
433 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);
434 if (!Allocated) {
435 NumSpillLanes -= I;
436 return false;
437 }
438 }
439
440 return true;
441 }
442
443 /// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
444 /// Either AGPR is spilled to VGPR to vice versa.
445 /// Returns true if a \p FI can be eliminated completely.
allocateVGPRSpillToAGPR(MachineFunction & MF,int FI,bool isAGPRtoVGPR)446 bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF,
447 int FI,
448 bool isAGPRtoVGPR) {
449 MachineRegisterInfo &MRI = MF.getRegInfo();
450 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
451 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
452
453 assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));
454
455 auto &Spill = VGPRToAGPRSpills[FI];
456
457 // This has already been allocated.
458 if (!Spill.Lanes.empty())
459 return Spill.FullyAllocated;
460
461 unsigned Size = FrameInfo.getObjectSize(FI);
462 unsigned NumLanes = Size / 4;
463 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
464
465 const TargetRegisterClass &RC =
466 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
467 auto Regs = RC.getRegisters();
468
469 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
470 const SIRegisterInfo *TRI = ST.getRegisterInfo();
471 Spill.FullyAllocated = true;
472
473 // FIXME: Move allocation logic out of MachineFunctionInfo and initialize
474 // once.
475 BitVector OtherUsedRegs;
476 OtherUsedRegs.resize(TRI->getNumRegs());
477
478 const uint32_t *CSRMask =
479 TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());
480 if (CSRMask)
481 OtherUsedRegs.setBitsInMask(CSRMask);
482
483 // TODO: Should include register tuples, but doesn't matter with current
484 // usage.
485 for (MCPhysReg Reg : SpillAGPR)
486 OtherUsedRegs.set(Reg);
487 for (MCPhysReg Reg : SpillVGPR)
488 OtherUsedRegs.set(Reg);
489
490 SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();
491 for (int I = NumLanes - 1; I >= 0; --I) {
492 NextSpillReg = std::find_if(
493 NextSpillReg, Regs.end(), [&MRI, &OtherUsedRegs](MCPhysReg Reg) {
494 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
495 !OtherUsedRegs[Reg];
496 });
497
498 if (NextSpillReg == Regs.end()) { // Registers exhausted
499 Spill.FullyAllocated = false;
500 break;
501 }
502
503 OtherUsedRegs.set(*NextSpillReg);
504 SpillRegs.push_back(*NextSpillReg);
505 MRI.reserveReg(*NextSpillReg, TRI);
506 Spill.Lanes[I] = *NextSpillReg++;
507 }
508
509 return Spill.FullyAllocated;
510 }
511
removeDeadFrameIndices(MachineFrameInfo & MFI,bool ResetSGPRSpillStackIDs)512 bool SIMachineFunctionInfo::removeDeadFrameIndices(
513 MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) {
514 // Remove dead frame indices from function frame, however keep FP & BP since
515 // spills for them haven't been inserted yet. And also make sure to remove the
516 // frame indices from `SGPRSpillsToVirtualVGPRLanes` data structure,
517 // otherwise, it could result in an unexpected side effect and bug, in case of
518 // any re-mapping of freed frame indices by later pass(es) like "stack slot
519 // coloring".
520 for (auto &R : make_early_inc_range(SGPRSpillsToVirtualVGPRLanes)) {
521 MFI.RemoveStackObject(R.first);
522 SGPRSpillsToVirtualVGPRLanes.erase(R.first);
523 }
524
525 // Remove the dead frame indices of CSR SGPRs which are spilled to physical
526 // VGPR lanes during SILowerSGPRSpills pass.
527 if (!ResetSGPRSpillStackIDs) {
528 for (auto &R : make_early_inc_range(SGPRSpillsToPhysicalVGPRLanes)) {
529 MFI.RemoveStackObject(R.first);
530 SGPRSpillsToPhysicalVGPRLanes.erase(R.first);
531 }
532 }
533 bool HaveSGPRToMemory = false;
534
535 if (ResetSGPRSpillStackIDs) {
536 // All other SGPRs must be allocated on the default stack, so reset the
537 // stack ID.
538 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E;
539 ++I) {
540 if (!checkIndexInPrologEpilogSGPRSpills(I)) {
541 if (MFI.getStackID(I) == TargetStackID::SGPRSpill) {
542 MFI.setStackID(I, TargetStackID::Default);
543 HaveSGPRToMemory = true;
544 }
545 }
546 }
547 }
548
549 for (auto &R : VGPRToAGPRSpills) {
550 if (R.second.IsDead)
551 MFI.RemoveStackObject(R.first);
552 }
553
554 return HaveSGPRToMemory;
555 }
556
getScavengeFI(MachineFrameInfo & MFI,const SIRegisterInfo & TRI)557 int SIMachineFunctionInfo::getScavengeFI(MachineFrameInfo &MFI,
558 const SIRegisterInfo &TRI) {
559 if (ScavengeFI)
560 return *ScavengeFI;
561
562 ScavengeFI =
563 MFI.CreateStackObject(TRI.getSpillSize(AMDGPU::SGPR_32RegClass),
564 TRI.getSpillAlign(AMDGPU::SGPR_32RegClass), false);
565 return *ScavengeFI;
566 }
567
getNextUserSGPR() const568 MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
569 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
570 return AMDGPU::SGPR0 + NumUserSGPRs;
571 }
572
getNextSystemSGPR() const573 MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
574 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
575 }
576
MRI_NoteNewVirtualRegister(Register Reg)577 void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(Register Reg) {
578 VRegFlags.grow(Reg);
579 }
580
MRI_NoteCloneVirtualRegister(Register NewReg,Register SrcReg)581 void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(Register NewReg,
582 Register SrcReg) {
583 VRegFlags.grow(NewReg);
584 VRegFlags[NewReg] = VRegFlags[SrcReg];
585 }
586
587 Register
getGITPtrLoReg(const MachineFunction & MF) const588 SIMachineFunctionInfo::getGITPtrLoReg(const MachineFunction &MF) const {
589 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
590 if (!ST.isAmdPalOS())
591 return Register();
592 Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in
593 if (ST.hasMergedShaders()) {
594 switch (MF.getFunction().getCallingConv()) {
595 case CallingConv::AMDGPU_HS:
596 case CallingConv::AMDGPU_GS:
597 // Low GIT address is passed in s8 rather than s0 for an LS+HS or
598 // ES+GS merged shader on gfx9+.
599 GitPtrLo = AMDGPU::SGPR8;
600 return GitPtrLo;
601 default:
602 return GitPtrLo;
603 }
604 }
605 return GitPtrLo;
606 }
607
regToString(Register Reg,const TargetRegisterInfo & TRI)608 static yaml::StringValue regToString(Register Reg,
609 const TargetRegisterInfo &TRI) {
610 yaml::StringValue Dest;
611 {
612 raw_string_ostream OS(Dest.Value);
613 OS << printReg(Reg, &TRI);
614 }
615 return Dest;
616 }
617
618 static std::optional<yaml::SIArgumentInfo>
convertArgumentInfo(const AMDGPUFunctionArgInfo & ArgInfo,const TargetRegisterInfo & TRI)619 convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
620 const TargetRegisterInfo &TRI) {
621 yaml::SIArgumentInfo AI;
622
623 auto convertArg = [&](std::optional<yaml::SIArgument> &A,
624 const ArgDescriptor &Arg) {
625 if (!Arg)
626 return false;
627
628 // Create a register or stack argument.
629 yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister());
630 if (Arg.isRegister()) {
631 raw_string_ostream OS(SA.RegisterName.Value);
632 OS << printReg(Arg.getRegister(), &TRI);
633 } else
634 SA.StackOffset = Arg.getStackOffset();
635 // Check and update the optional mask.
636 if (Arg.isMasked())
637 SA.Mask = Arg.getMask();
638
639 A = SA;
640 return true;
641 };
642
643 // TODO: Need to serialize kernarg preloads.
644 bool Any = false;
645 Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
646 Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
647 Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
648 Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
649 Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
650 Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
651 Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId);
652 Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
653 Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
654 Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
655 Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
656 Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
657 Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
658 ArgInfo.PrivateSegmentWaveByteOffset);
659 Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
660 Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
661 Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
662 Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
663 Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
664
665 if (Any)
666 return AI;
667
668 return std::nullopt;
669 }
670
SIMachineFunctionInfo(const llvm::SIMachineFunctionInfo & MFI,const TargetRegisterInfo & TRI,const llvm::MachineFunction & MF)671 yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
672 const llvm::SIMachineFunctionInfo &MFI, const TargetRegisterInfo &TRI,
673 const llvm::MachineFunction &MF)
674 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
675 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
676 GDSSize(MFI.getGDSSize()),
677 DynLDSAlign(MFI.getDynLDSAlign()), IsEntryFunction(MFI.isEntryFunction()),
678 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
679 MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),
680 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
681 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
682 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
683 Occupancy(MFI.getOccupancy()),
684 ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
685 FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
686 StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),
687 BytesInStackArgArea(MFI.getBytesInStackArgArea()),
688 ReturnsVoid(MFI.returnsVoid()),
689 ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)),
690 PSInputAddr(MFI.getPSInputAddr()),
691 PSInputEnable(MFI.getPSInputEnable()),
692 Mode(MFI.getMode()) {
693 for (Register Reg : MFI.getWWMReservedRegs())
694 WWMReservedRegs.push_back(regToString(Reg, TRI));
695
696 if (MFI.getLongBranchReservedReg())
697 LongBranchReservedReg = regToString(MFI.getLongBranchReservedReg(), TRI);
698 if (MFI.getVGPRForAGPRCopy())
699 VGPRForAGPRCopy = regToString(MFI.getVGPRForAGPRCopy(), TRI);
700
701 if (MFI.getSGPRForEXECCopy())
702 SGPRForEXECCopy = regToString(MFI.getSGPRForEXECCopy(), TRI);
703
704 auto SFI = MFI.getOptionalScavengeFI();
705 if (SFI)
706 ScavengeFI = yaml::FrameIndex(*SFI, MF.getFrameInfo());
707 }
708
mappingImpl(yaml::IO & YamlIO)709 void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
710 MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this);
711 }
712
initializeBaseYamlFields(const yaml::SIMachineFunctionInfo & YamlMFI,const MachineFunction & MF,PerFunctionMIParsingState & PFS,SMDiagnostic & Error,SMRange & SourceRange)713 bool SIMachineFunctionInfo::initializeBaseYamlFields(
714 const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF,
715 PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) {
716 ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
717 MaxKernArgAlign = YamlMFI.MaxKernArgAlign;
718 LDSSize = YamlMFI.LDSSize;
719 GDSSize = YamlMFI.GDSSize;
720 DynLDSAlign = YamlMFI.DynLDSAlign;
721 PSInputAddr = YamlMFI.PSInputAddr;
722 PSInputEnable = YamlMFI.PSInputEnable;
723 HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress;
724 Occupancy = YamlMFI.Occupancy;
725 IsEntryFunction = YamlMFI.IsEntryFunction;
726 NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath;
727 MemoryBound = YamlMFI.MemoryBound;
728 WaveLimiter = YamlMFI.WaveLimiter;
729 HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
730 HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
731 BytesInStackArgArea = YamlMFI.BytesInStackArgArea;
732 ReturnsVoid = YamlMFI.ReturnsVoid;
733
734 if (YamlMFI.ScavengeFI) {
735 auto FIOrErr = YamlMFI.ScavengeFI->getFI(MF.getFrameInfo());
736 if (!FIOrErr) {
737 // Create a diagnostic for a the frame index.
738 const MemoryBuffer &Buffer =
739 *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID());
740
741 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1,
742 SourceMgr::DK_Error, toString(FIOrErr.takeError()),
743 "", std::nullopt, std::nullopt);
744 SourceRange = YamlMFI.ScavengeFI->SourceRange;
745 return true;
746 }
747 ScavengeFI = *FIOrErr;
748 } else {
749 ScavengeFI = std::nullopt;
750 }
751 return false;
752 }
753
mayUseAGPRs(const Function & F) const754 bool SIMachineFunctionInfo::mayUseAGPRs(const Function &F) const {
755 return !F.hasFnAttribute("amdgpu-no-agpr");
756 }
757
usesAGPRs(const MachineFunction & MF) const758 bool SIMachineFunctionInfo::usesAGPRs(const MachineFunction &MF) const {
759 if (UsesAGPRs)
760 return *UsesAGPRs;
761
762 if (!mayNeedAGPRs()) {
763 UsesAGPRs = false;
764 return false;
765 }
766
767 if (!AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv()) ||
768 MF.getFrameInfo().hasCalls()) {
769 UsesAGPRs = true;
770 return true;
771 }
772
773 const MachineRegisterInfo &MRI = MF.getRegInfo();
774
775 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
776 const Register Reg = Register::index2VirtReg(I);
777 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
778 if (RC && SIRegisterInfo::isAGPRClass(RC)) {
779 UsesAGPRs = true;
780 return true;
781 }
782 if (!RC && !MRI.use_empty(Reg) && MRI.getType(Reg).isValid()) {
783 // Defer caching UsesAGPRs, function might not yet been regbank selected.
784 return true;
785 }
786 }
787
788 for (MCRegister Reg : AMDGPU::AGPR_32RegClass) {
789 if (MRI.isPhysRegUsed(Reg)) {
790 UsesAGPRs = true;
791 return true;
792 }
793 }
794
795 UsesAGPRs = false;
796 return false;
797 }
798