1 //===-- X86LowerTileCopy.cpp - Expand Tile Copy Instructions---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines the pass which lower AMX tile copy instructions. Since 10 // there is no tile copy instruction, we need store tile register to stack 11 // and load from stack to another tile register. We need extra GR to hold 12 // the stride, and we need stack slot to hold the tile data register. 13 // We would run this pass after copy propagation, so that we don't miss copy 14 // optimization. And we would run this pass before prolog/epilog insertion, 15 // so that we can allocate stack slot. 16 // 17 //===----------------------------------------------------------------------===// 18 19 #include "X86.h" 20 #include "X86InstrBuilder.h" 21 #include "X86InstrInfo.h" 22 #include "X86MachineFunctionInfo.h" 23 #include "X86Subtarget.h" 24 #include "llvm/CodeGen/LiveRegUnits.h" 25 #include "llvm/CodeGen/MachineBasicBlock.h" 26 #include "llvm/CodeGen/MachineFrameInfo.h" 27 #include "llvm/CodeGen/MachineFunction.h" 28 #include "llvm/CodeGen/MachineFunctionPass.h" 29 #include "llvm/CodeGen/MachineInstr.h" 30 #include "llvm/CodeGen/MachineInstrBuilder.h" 31 #include "llvm/CodeGen/MachineOperand.h" 32 #include "llvm/CodeGen/Passes.h" 33 #include "llvm/IR/DebugLoc.h" 34 #include "llvm/InitializePasses.h" 35 #include "llvm/Support/Debug.h" 36 37 using namespace llvm; 38 39 #define DEBUG_TYPE "x86-lower-tile-copy" 40 41 namespace { 42 43 class X86LowerTileCopy : public MachineFunctionPass { 44 public: 45 static char ID; 46 47 X86LowerTileCopy() : MachineFunctionPass(ID) {} 48 49 void getAnalysisUsage(AnalysisUsage &AU) const override; 50 51 bool runOnMachineFunction(MachineFunction &MF) override; 52 53 StringRef getPassName() const override { return "X86 Lower Tile Copy"; } 54 }; 55 56 } // namespace 57 58 char X86LowerTileCopy::ID = 0; 59 60 INITIALIZE_PASS_BEGIN(X86LowerTileCopy, "lowertilecopy", "Tile Copy Lowering", 61 false, false) 62 INITIALIZE_PASS_END(X86LowerTileCopy, "lowertilecopy", "Tile Copy Lowering", 63 false, false) 64 65 void X86LowerTileCopy::getAnalysisUsage(AnalysisUsage &AU) const { 66 AU.setPreservesAll(); 67 MachineFunctionPass::getAnalysisUsage(AU); 68 } 69 70 FunctionPass *llvm::createX86LowerTileCopyPass() { 71 return new X86LowerTileCopy(); 72 } 73 74 bool X86LowerTileCopy::runOnMachineFunction(MachineFunction &MF) { 75 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>(); 76 if (FuncInfo->getAMXProgModel() != AMXProgModelEnum::ManagedRA) 77 return false; 78 79 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>(); 80 const X86InstrInfo *TII = ST.getInstrInfo(); 81 const TargetRegisterInfo *TRI = ST.getRegisterInfo(); 82 BitVector GR64Regs = 83 TRI->getAllocatableSet(MF, TRI->getRegClass(X86::GR64RegClassID)); 84 BitVector TILERegs = 85 TRI->getAllocatableSet(MF, TRI->getRegClass(X86::TILERegClassID)); 86 bool Changed = false; 87 88 for (MachineBasicBlock &MBB : MF) { 89 LiveRegUnits UsedRegs(*TRI); 90 UsedRegs.addLiveOuts(MBB); 91 for (MachineInstr &MI : llvm::make_early_inc_range(reverse(MBB))) { 92 UsedRegs.stepBackward(MI); 93 if (!MI.isCopy()) 94 continue; 95 MachineOperand &DstMO = MI.getOperand(0); 96 MachineOperand &SrcMO = MI.getOperand(1); 97 Register SrcReg = SrcMO.getReg(); 98 Register DstReg = DstMO.getReg(); 99 if (!X86::TILERegClass.contains(DstReg, SrcReg)) 100 continue; 101 102 // Allocate stack slot for tile register 103 unsigned Size = TRI->getSpillSize(X86::TILERegClass); 104 Align Alignment = TRI->getSpillAlign(X86::TILERegClass); 105 int TileSS = MF.getFrameInfo().CreateSpillStackObject(Size, Alignment); 106 107 int StrideSS = 0; 108 109 // Pick a killed register to avoid a save/reload. 110 Register GR64Cand = X86::NoRegister; 111 for (auto RegT : GR64Regs.set_bits()) { 112 if (UsedRegs.available(RegT)) { 113 GR64Cand = RegT; 114 break; 115 } 116 } 117 118 const DebugLoc &DL = MI.getDebugLoc(); 119 if (GR64Cand) { 120 // mov 64 %reg 121 BuildMI(MBB, MI, DL, TII->get(X86::MOV64ri), GR64Cand).addImm(64); 122 } else { 123 // No available register? Save RAX and reload it after use. 124 125 // Allocate stack slot for stride register 126 Size = TRI->getSpillSize(X86::GR64RegClass); 127 Alignment = TRI->getSpillAlign(X86::GR64RegClass); 128 StrideSS = MF.getFrameInfo().CreateSpillStackObject(Size, Alignment); 129 130 // mov %reg (%sp) 131 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV64mr)), 132 StrideSS) 133 .addReg(X86::RAX); 134 // mov 64 %reg 135 BuildMI(MBB, MI, DL, TII->get(X86::MOV64ri), X86::RAX).addImm(64); 136 } 137 // tilestored %tmm, (%sp, %idx) 138 #define GET_EGPR_IF_ENABLED(OPC) (ST.hasEGPR() ? OPC##_EVEX : OPC) 139 unsigned Opc = GET_EGPR_IF_ENABLED(X86::TILESTORED); 140 MachineInstr *NewMI = 141 addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc)), TileSS) 142 .addReg(SrcReg, getKillRegState(SrcMO.isKill())); 143 MachineOperand &MO = NewMI->getOperand(2); 144 MO.setReg(GR64Cand ? GR64Cand : X86::RAX); 145 MO.setIsKill(true); 146 // tileloadd (%sp, %idx), %tmm 147 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD); 148 #undef GET_EGPR_IF_ENABLED 149 NewMI = addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc), DstReg), 150 TileSS); 151 if (!GR64Cand) { 152 // restore %rax 153 // mov (%sp) %rax 154 addFrameReference( 155 BuildMI(MBB, MI, DL, TII->get(X86::MOV64rm), X86::RAX), StrideSS); 156 } 157 MI.eraseFromParent(); 158 Changed = true; 159 } 160 } 161 return Changed; 162 } 163