Lines Matching +full:r +full:- +full:tile
1 //===-- X86FastPreTileConfig.cpp - Fast Tile Register Configure------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 /// \file Pass to preconfig the shape of physical tile registers
10 /// It inserts ldtilecfg ahead of each group of tile registers. The algorithm
11 /// walk each instruction of basic block in reverse order. All the tile
16 //===----------------------------------------------------------------------===//
53 int CfgSS = -1;
64 /// Has a bit set for tile virtual register for which it was determined
80 X86FastPreTileConfig() : MachineFunctionPass(ID), StackSlotForVirtReg(-1) {} in X86FastPreTileConfig()
84 return "Fast Tile Register Preconfigure"; in getPassName()
87 /// Perform tile register configure.
98 "Fast Tile Register Preconfigure", false, false)
100 "Fast Tile Register Preconfigure", false, false)
122 if (SS != -1) in getStackSpaceFor()
126 const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg); in getStackSpaceFor()
127 unsigned Size = TRI->getSpillSize(RC); in getStackSpaceFor()
128 Align Alignment = TRI->getSpillAlign(RC); in getStackSpaceFor()
129 int FrameIdx = MFI->CreateSpillStackObject(Size, Alignment); in getStackSpaceFor()
143 for (const MachineInstr &UseInst : MRI->use_nodbg_instructions(VirtReg)) { in mayLiveOut()
149 // The use and def are in the same MBB. If the tile register is in mayLiveOut()
151 // tile register. in mayLiveOut()
164 MachineBasicBlock &MBB = MF->front(); in InitializeTileConfigStackSpace()
167 if (ST->hasAVX512()) { in InitializeTileConfigStackSpace()
168 Register Zmm = MRI->createVirtualRegister(&X86::VR512RegClass); in InitializeTileConfigStackSpace()
169 BuildMI(MBB, MI, DL, TII->get(X86::AVX512_512_SET0), Zmm); in InitializeTileConfigStackSpace()
170 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr)), CfgSS) in InitializeTileConfigStackSpace()
172 } else if (ST->hasAVX2()) { in InitializeTileConfigStackSpace()
173 Register Ymm = MRI->createVirtualRegister(&X86::VR256RegClass); in InitializeTileConfigStackSpace()
174 BuildMI(MBB, MI, DL, TII->get(X86::AVX_SET0), Ymm); in InitializeTileConfigStackSpace()
175 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSYmr)), CfgSS) in InitializeTileConfigStackSpace()
177 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSYmr)), CfgSS, in InitializeTileConfigStackSpace()
181 assert(ST->hasSSE2() && "AMX should assume SSE2 enabled"); in InitializeTileConfigStackSpace()
182 unsigned StoreOpc = ST->hasAVX() ? X86::VMOVUPSmr : X86::MOVUPSmr; in InitializeTileConfigStackSpace()
183 Register Xmm = MRI->createVirtualRegister(&X86::VR128RegClass); in InitializeTileConfigStackSpace()
184 BuildMI(MBB, MI, DL, TII->get(X86::V_SET0), Xmm); in InitializeTileConfigStackSpace()
185 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS) in InitializeTileConfigStackSpace()
187 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS, 16) in InitializeTileConfigStackSpace()
189 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS, 32) in InitializeTileConfigStackSpace()
191 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS, 48) in InitializeTileConfigStackSpace()
195 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), CfgSS) in InitializeTileConfigStackSpace()
207 const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg); in spill()
208 // Don't need shape information for tile store, becasue it is adjacent to in spill()
209 // the tile def instruction. in spill()
210 TII->storeRegToStackSlot(*MBB, Before, VirtReg, Kill, FI, &RC, TRI, in spill()
222 const TargetRegisterClass &RC = *MRI->getRegClass(OrigReg); in reload()
230 // --> in reload()
232 if (UseMI->isCopy()) in reload()
233 TileReg = UseMI->getOperand(0).getReg(); in reload()
235 TileReg = MRI->createVirtualRegister(&RC); in reload()
236 // Can't use TII->loadRegFromStackSlot(), because we need the shape in reload()
240 Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass); in reload()
242 MachineInstr *NewMI = BuildMI(*UseMI->getParent(), UseMI, DebugLoc(), in reload()
243 TII->get(X86::MOV64ri), StrideReg) in reload()
246 BuildMI(*UseMI->getParent(), UseMI, DebugLoc(), TII->get(Opc), TileReg) in reload()
247 .addReg(RowMO->getReg()) in reload()
248 .addReg(ColMO->getReg()), in reload()
250 MachineOperand &MO = NewMI->getOperand(5); in reload()
253 RowMO->setIsKill(false); in reload()
254 ColMO->setIsKill(false); in reload()
256 if (UseMI->isCopy()) { in reload()
257 UseMI->eraseFromParent(); in reload()
260 for (auto &MO : UseMI->operands()) { in reload()
272 // The instruction must have 3 operands: tile def, row, col. in isTileDef()
282 MRI->getRegClass(Reg)->getID() == X86::TILERegClassID) in isTileDef()
292 MachineInstr *MI = MRI->getVRegDef(TileReg); in getShape()
294 MachineOperand *RowMO = &MI->getOperand(1); in getShape()
295 MachineOperand *ColMO = &MI->getOperand(2); in getShape()
297 } else if (MI->isCopy()) { in getShape()
298 TileReg = MI->getOperand(1).getReg(); in getShape()
304 assert(MI->isPHI() && "Unexpected PHI when get shape."); in getShape()
315 // -->
329 Register StackAddrReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass); in convertPHI()
331 TII->get(X86::PHI), StackAddrReg); in convertPHI()
332 Register RowReg = MRI->createVirtualRegister(&X86::GR16RegClass); in convertPHI()
334 TII->get(X86::PHI), RowReg); in convertPHI()
335 Register ColReg = MRI->createVirtualRegister(&X86::GR16RegClass); in convertPHI()
337 TII->get(X86::PHI), ColReg); in convertPHI()
342 // Get the 2 incoming value of tile register and MBB. in convertPHI()
350 MachineInstr *TileDefMI = MRI->getVRegDef(InTileReg); in convertPHI()
352 if (TileDefMI->isPHI()) { in convertPHI()
353 InsertPos = TileDefMI->getParent()->getFirstNonPHI(); in convertPHI()
357 // def t2 t3 = phi(t1, t4) <-- in convertPHI()
359 // t4 = phi(t2, t3)------------- in convertPHI()
373 convertPHI(TileDefMI->getParent(), *TileDefMI); in convertPHI()
376 MachineInstr *TileLoad = MRI->getVRegDef(InTileReg); in convertPHI()
377 assert(TileLoad && TileLoad->getOpcode() == X86::PTILELOADDV); in convertPHI()
378 Register InRowReg = TileLoad->getOperand(1).getReg(); in convertPHI()
379 Register InColReg = TileLoad->getOperand(2).getReg(); in convertPHI()
380 Register InStackAddrReg = TileLoad->getOperand(3).getReg(); in convertPHI()
386 InsertPos = TileDefMI->getIterator(); in convertPHI()
390 Shape.getRow()->setIsKill(false); in convertPHI()
391 Shape.getCol()->setIsKill(false); in convertPHI()
392 RowPHI.addReg(Shape.getRow()->getReg()).addMBB(InMBB); in convertPHI()
393 ColPHI.addReg(Shape.getCol()->getReg()).addMBB(InMBB); in convertPHI()
395 // The incoming tile register live out of its def BB, it would be spilled. in convertPHI()
396 // Create MI to get the spill stack slot address for the tile register in convertPHI()
399 MRI->createVirtualRegister(&X86::GR64_NOSPRegClass); in convertPHI()
400 addOffset(BuildMI(*TileDefMI->getParent(), InsertPos, DebugLoc(), in convertPHI()
401 TII->get(X86::LEA64r), InStackAddrReg) in convertPHI()
408 MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI(); in convertPHI()
409 Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass); in convertPHI()
410 BuildMI(*MBB, InsertPos, DebugLoc(), TII->get(X86::MOV64ri), StrideReg) in convertPHI()
414 BuildMI(*MBB, InsertPos, DebugLoc(), TII->get(X86::PTILELOADDV), TileReg) in convertPHI()
418 MachineOperand &MO = NewMI->getOperand(5); in convertPHI()
428 MRI->getRegClass(MO.getReg())->getID() == X86::TILERegClassID) in isTileRegDef()
443 // Canonicalize the phi node first. One tile phi may depeneds previous in canonicalizePHIs()
449 // --> in canonicalizePHIs()
460 for (unsigned I = 1, E = PHI->getNumOperands(); I != E; I += 2) { in canonicalizePHIs()
461 Register InTileReg = PHI->getOperand(I).getReg(); in canonicalizePHIs()
462 MachineBasicBlock *InMBB = PHI->getOperand(I + 1).getMBB(); in canonicalizePHIs()
463 DefMI = MRI->getVRegDef(InTileReg); in canonicalizePHIs()
464 if (InMBB != &MBB || !DefMI->isPHI()) in canonicalizePHIs()
467 InMO = &PHI->getOperand(I); in canonicalizePHIs()
477 for (unsigned I = 1, E = DefMI->getNumOperands(); I != E; I += 2) { in canonicalizePHIs()
478 MachineBasicBlock *InMBB = PHI->getOperand(I + 1).getMBB(); in canonicalizePHIs()
481 DefTileReg = DefMI->getOperand(I).getReg(); in canonicalizePHIs()
482 InMO->setReg(DefTileReg); in canonicalizePHIs()
504 // PreTileConfig should configure the tile registers based on basic
507 this->MBB = &MBB; in configBasicBlock()
514 if (CfgSS == -1) in configBasicBlock()
515 CfgSS = MFI->CreateStackObject(ST->getTileConfigSize(), in configBasicBlock()
516 ST->getTileConfigAlignment(), false); in configBasicBlock()
518 BuildMI(MBB, Before, DebugLoc(), TII->get(X86::PLDTILECFGV)), CfgSS); in configBasicBlock()
528 MRI->getRegClass(Reg)->getID() == X86::TILERegClassID) in configBasicBlock()
537 // Don't collect the shape of used tile, the tile should be defined in configBasicBlock()
538 // before the tile use. Spill and reload would happen if there is only in configBasicBlock()
539 // tile use after ldtilecfg, so the shape can be collected from reload. in configBasicBlock()
543 // tilestore %r, %c, %t in configBasicBlock()
544 // --> in configBasicBlock()
547 // %t = tileload %r, %c in configBasicBlock()
548 // tilestore %r, %c, %t in configBasicBlock()
551 // According to AMX ABI, all the tile registers including config register in configBasicBlock()
556 I = ++LastShapeMI->getIterator(); in configBasicBlock()
566 //--------------------------------------------------------------------- in configBasicBlock()
588 //--------------------------------------------------------------------- in configBasicBlock()
595 // tilezero(row0, col0) <- MI in configBasicBlock()
598 // ldtilecfg <- insert in configBasicBlock()
601 Config(*(++LastShapeMI->getIterator())); in configBasicBlock()
604 MachineInstr *RowMI = MRI->getVRegDef(RowMO->getReg()); in configBasicBlock()
605 MachineInstr *ColMI = MRI->getVRegDef(ColMO->getReg()); in configBasicBlock()
608 if (RowMI->getParent() == &MBB) { in configBasicBlock()
614 if (ColMI->getParent() == &MBB) { in configBasicBlock()
625 for (MachineInstr &UseMI : MRI->use_instructions(TileReg)) { in configBasicBlock()
641 // Configure tile registers at the head of the MBB in configBasicBlock()
644 if (LastShapeMI == nullptr || LastShapeMI->isPHI()) in configBasicBlock()
647 Before = &*(++LastShapeMI->getIterator()); in configBasicBlock()
657 // Early exit in the common case of non-AMX code. in runOnMachineFunction()
658 if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA) in runOnMachineFunction()
664 TII = ST->getInstrInfo(); in runOnMachineFunction()
666 TRI = ST->getRegisterInfo(); in runOnMachineFunction()
667 CfgSS = -1; in runOnMachineFunction()
669 unsigned NumVirtRegs = MRI->getNumVirtRegs(); in runOnMachineFunction()
678 assert(MRI->isSSA()); in runOnMachineFunction()
685 // ldtilecfg for tile registers. The reserse post order is to facilitate in runOnMachineFunction()