1 //===-- SystemZInstrInfo.cpp - SystemZ instruction information ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the SystemZ implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "SystemZInstrInfo.h" 14 #include "MCTargetDesc/SystemZMCTargetDesc.h" 15 #include "SystemZ.h" 16 #include "SystemZInstrBuilder.h" 17 #include "SystemZSubtarget.h" 18 #include "llvm/ADT/Statistic.h" 19 #include "llvm/CodeGen/LiveInterval.h" 20 #include "llvm/CodeGen/LiveIntervals.h" 21 #include "llvm/CodeGen/LiveRegUnits.h" 22 #include "llvm/CodeGen/LiveVariables.h" 23 #include "llvm/CodeGen/MachineBasicBlock.h" 24 #include "llvm/CodeGen/MachineFrameInfo.h" 25 #include "llvm/CodeGen/MachineFunction.h" 26 #include "llvm/CodeGen/MachineInstr.h" 27 #include "llvm/CodeGen/MachineMemOperand.h" 28 #include "llvm/CodeGen/MachineOperand.h" 29 #include "llvm/CodeGen/MachineRegisterInfo.h" 30 #include "llvm/CodeGen/SlotIndexes.h" 31 #include "llvm/CodeGen/StackMaps.h" 32 #include "llvm/CodeGen/TargetInstrInfo.h" 33 #include "llvm/CodeGen/TargetSubtargetInfo.h" 34 #include "llvm/CodeGen/VirtRegMap.h" 35 #include "llvm/MC/MCInstrDesc.h" 36 #include "llvm/MC/MCRegisterInfo.h" 37 #include "llvm/Support/BranchProbability.h" 38 #include "llvm/Support/ErrorHandling.h" 39 #include "llvm/Support/MathExtras.h" 40 #include "llvm/Target/TargetMachine.h" 41 #include <cassert> 42 #include <cstdint> 43 #include <iterator> 44 45 using namespace llvm; 46 47 #define GET_INSTRINFO_CTOR_DTOR 48 #define GET_INSTRMAP_INFO 49 #include "SystemZGenInstrInfo.inc" 50 51 #define DEBUG_TYPE "systemz-II" 52 53 // Return a mask with Count low bits set. allOnes(unsigned int Count)54 static uint64_t allOnes(unsigned int Count) { 55 return Count == 0 ? 0 : (uint64_t(1) << (Count - 1) << 1) - 1; 56 } 57 58 // Pin the vtable to this file. anchor()59 void SystemZInstrInfo::anchor() {} 60 SystemZInstrInfo(SystemZSubtarget & sti)61 SystemZInstrInfo::SystemZInstrInfo(SystemZSubtarget &sti) 62 : SystemZGenInstrInfo(-1, -1), 63 RI(sti.getSpecialRegisters()->getReturnFunctionAddressRegister()), 64 STI(sti) {} 65 66 // MI is a 128-bit load or store. Split it into two 64-bit loads or stores, 67 // each having the opcode given by NewOpcode. splitMove(MachineBasicBlock::iterator MI,unsigned NewOpcode) const68 void SystemZInstrInfo::splitMove(MachineBasicBlock::iterator MI, 69 unsigned NewOpcode) const { 70 MachineBasicBlock *MBB = MI->getParent(); 71 MachineFunction &MF = *MBB->getParent(); 72 73 // Get two load or store instructions. Use the original instruction for 74 // one of them and create a clone for the other. 75 MachineInstr *HighPartMI = MF.CloneMachineInstr(&*MI); 76 MachineInstr *LowPartMI = &*MI; 77 MBB->insert(LowPartMI, HighPartMI); 78 79 // Set up the two 64-bit registers and remember super reg and its flags. 80 MachineOperand &HighRegOp = HighPartMI->getOperand(0); 81 MachineOperand &LowRegOp = LowPartMI->getOperand(0); 82 Register Reg128 = LowRegOp.getReg(); 83 unsigned Reg128Killed = getKillRegState(LowRegOp.isKill()); 84 unsigned Reg128Undef = getUndefRegState(LowRegOp.isUndef()); 85 HighRegOp.setReg(RI.getSubReg(HighRegOp.getReg(), SystemZ::subreg_h64)); 86 LowRegOp.setReg(RI.getSubReg(LowRegOp.getReg(), SystemZ::subreg_l64)); 87 88 // The address in the first (high) instruction is already correct. 89 // Adjust the offset in the second (low) instruction. 90 MachineOperand &HighOffsetOp = HighPartMI->getOperand(2); 91 MachineOperand &LowOffsetOp = LowPartMI->getOperand(2); 92 LowOffsetOp.setImm(LowOffsetOp.getImm() + 8); 93 94 // Set the opcodes. 95 unsigned HighOpcode = getOpcodeForOffset(NewOpcode, HighOffsetOp.getImm()); 96 unsigned LowOpcode = getOpcodeForOffset(NewOpcode, LowOffsetOp.getImm()); 97 assert(HighOpcode && LowOpcode && "Both offsets should be in range"); 98 HighPartMI->setDesc(get(HighOpcode)); 99 LowPartMI->setDesc(get(LowOpcode)); 100 101 MachineInstr *FirstMI = HighPartMI; 102 if (MI->mayStore()) { 103 FirstMI->getOperand(0).setIsKill(false); 104 // Add implicit uses of the super register in case one of the subregs is 105 // undefined. We could track liveness and skip storing an undefined 106 // subreg, but this is hopefully rare (discovered with llvm-stress). 107 // If Reg128 was killed, set kill flag on MI. 108 unsigned Reg128UndefImpl = (Reg128Undef | RegState::Implicit); 109 MachineInstrBuilder(MF, HighPartMI).addReg(Reg128, Reg128UndefImpl); 110 MachineInstrBuilder(MF, LowPartMI).addReg(Reg128, (Reg128UndefImpl | Reg128Killed)); 111 } else { 112 // If HighPartMI clobbers any of the address registers, it needs to come 113 // after LowPartMI. 114 auto overlapsAddressReg = [&](Register Reg) -> bool { 115 return RI.regsOverlap(Reg, MI->getOperand(1).getReg()) || 116 RI.regsOverlap(Reg, MI->getOperand(3).getReg()); 117 }; 118 if (overlapsAddressReg(HighRegOp.getReg())) { 119 assert(!overlapsAddressReg(LowRegOp.getReg()) && 120 "Both loads clobber address!"); 121 MBB->splice(HighPartMI, MBB, LowPartMI); 122 FirstMI = LowPartMI; 123 } 124 } 125 126 // Clear the kill flags on the address registers in the first instruction. 127 FirstMI->getOperand(1).setIsKill(false); 128 FirstMI->getOperand(3).setIsKill(false); 129 } 130 131 // Split ADJDYNALLOC instruction MI. splitAdjDynAlloc(MachineBasicBlock::iterator MI) const132 void SystemZInstrInfo::splitAdjDynAlloc(MachineBasicBlock::iterator MI) const { 133 MachineBasicBlock *MBB = MI->getParent(); 134 MachineFunction &MF = *MBB->getParent(); 135 MachineFrameInfo &MFFrame = MF.getFrameInfo(); 136 MachineOperand &OffsetMO = MI->getOperand(2); 137 SystemZCallingConventionRegisters *Regs = STI.getSpecialRegisters(); 138 139 uint64_t Offset = (MFFrame.getMaxCallFrameSize() + 140 Regs->getCallFrameSize() + 141 Regs->getStackPointerBias() + 142 OffsetMO.getImm()); 143 unsigned NewOpcode = getOpcodeForOffset(SystemZ::LA, Offset); 144 assert(NewOpcode && "No support for huge argument lists yet"); 145 MI->setDesc(get(NewOpcode)); 146 OffsetMO.setImm(Offset); 147 } 148 149 // MI is an RI-style pseudo instruction. Replace it with LowOpcode 150 // if the first operand is a low GR32 and HighOpcode if the first operand 151 // is a high GR32. ConvertHigh is true if LowOpcode takes a signed operand 152 // and HighOpcode takes an unsigned 32-bit operand. In those cases, 153 // MI has the same kind of operand as LowOpcode, so needs to be converted 154 // if HighOpcode is used. expandRIPseudo(MachineInstr & MI,unsigned LowOpcode,unsigned HighOpcode,bool ConvertHigh) const155 void SystemZInstrInfo::expandRIPseudo(MachineInstr &MI, unsigned LowOpcode, 156 unsigned HighOpcode, 157 bool ConvertHigh) const { 158 Register Reg = MI.getOperand(0).getReg(); 159 bool IsHigh = SystemZ::isHighReg(Reg); 160 MI.setDesc(get(IsHigh ? HighOpcode : LowOpcode)); 161 if (IsHigh && ConvertHigh) 162 MI.getOperand(1).setImm(uint32_t(MI.getOperand(1).getImm())); 163 } 164 165 // MI is a three-operand RIE-style pseudo instruction. Replace it with 166 // LowOpcodeK if the registers are both low GR32s, otherwise use a move 167 // followed by HighOpcode or LowOpcode, depending on whether the target 168 // is a high or low GR32. expandRIEPseudo(MachineInstr & MI,unsigned LowOpcode,unsigned LowOpcodeK,unsigned HighOpcode) const169 void SystemZInstrInfo::expandRIEPseudo(MachineInstr &MI, unsigned LowOpcode, 170 unsigned LowOpcodeK, 171 unsigned HighOpcode) const { 172 Register DestReg = MI.getOperand(0).getReg(); 173 Register SrcReg = MI.getOperand(1).getReg(); 174 bool DestIsHigh = SystemZ::isHighReg(DestReg); 175 bool SrcIsHigh = SystemZ::isHighReg(SrcReg); 176 if (!DestIsHigh && !SrcIsHigh) 177 MI.setDesc(get(LowOpcodeK)); 178 else { 179 if (DestReg != SrcReg) { 180 emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, SrcReg, 181 SystemZ::LR, 32, MI.getOperand(1).isKill(), 182 MI.getOperand(1).isUndef()); 183 MI.getOperand(1).setReg(DestReg); 184 } 185 MI.setDesc(get(DestIsHigh ? HighOpcode : LowOpcode)); 186 MI.tieOperands(0, 1); 187 } 188 } 189 190 // MI is an RXY-style pseudo instruction. Replace it with LowOpcode 191 // if the first operand is a low GR32 and HighOpcode if the first operand 192 // is a high GR32. expandRXYPseudo(MachineInstr & MI,unsigned LowOpcode,unsigned HighOpcode) const193 void SystemZInstrInfo::expandRXYPseudo(MachineInstr &MI, unsigned LowOpcode, 194 unsigned HighOpcode) const { 195 Register Reg = MI.getOperand(0).getReg(); 196 unsigned Opcode = getOpcodeForOffset( 197 SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode, 198 MI.getOperand(2).getImm()); 199 MI.setDesc(get(Opcode)); 200 } 201 202 // MI is a load-on-condition pseudo instruction with a single register 203 // (source or destination) operand. Replace it with LowOpcode if the 204 // register is a low GR32 and HighOpcode if the register is a high GR32. expandLOCPseudo(MachineInstr & MI,unsigned LowOpcode,unsigned HighOpcode) const205 void SystemZInstrInfo::expandLOCPseudo(MachineInstr &MI, unsigned LowOpcode, 206 unsigned HighOpcode) const { 207 Register Reg = MI.getOperand(0).getReg(); 208 unsigned Opcode = SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode; 209 MI.setDesc(get(Opcode)); 210 } 211 212 // MI is an RR-style pseudo instruction that zero-extends the low Size bits 213 // of one GRX32 into another. Replace it with LowOpcode if both operands 214 // are low registers, otherwise use RISB[LH]G. expandZExtPseudo(MachineInstr & MI,unsigned LowOpcode,unsigned Size) const215 void SystemZInstrInfo::expandZExtPseudo(MachineInstr &MI, unsigned LowOpcode, 216 unsigned Size) const { 217 MachineInstrBuilder MIB = 218 emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), 219 MI.getOperand(0).getReg(), MI.getOperand(1).getReg(), LowOpcode, 220 Size, MI.getOperand(1).isKill(), MI.getOperand(1).isUndef()); 221 222 // Keep the remaining operands as-is. 223 for (const MachineOperand &MO : llvm::drop_begin(MI.operands(), 2)) 224 MIB.add(MO); 225 226 MI.eraseFromParent(); 227 } 228 expandLoadStackGuard(MachineInstr * MI) const229 void SystemZInstrInfo::expandLoadStackGuard(MachineInstr *MI) const { 230 MachineBasicBlock *MBB = MI->getParent(); 231 MachineFunction &MF = *MBB->getParent(); 232 const Register Reg64 = MI->getOperand(0).getReg(); 233 const Register Reg32 = RI.getSubReg(Reg64, SystemZ::subreg_l32); 234 235 // EAR can only load the low subregister so us a shift for %a0 to produce 236 // the GR containing %a0 and %a1. 237 238 // ear <reg>, %a0 239 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32) 240 .addReg(SystemZ::A0) 241 .addReg(Reg64, RegState::ImplicitDefine); 242 243 // sllg <reg>, <reg>, 32 244 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::SLLG), Reg64) 245 .addReg(Reg64) 246 .addReg(0) 247 .addImm(32); 248 249 // ear <reg>, %a1 250 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32) 251 .addReg(SystemZ::A1); 252 253 // lg <reg>, 40(<reg>) 254 MI->setDesc(get(SystemZ::LG)); 255 MachineInstrBuilder(MF, MI).addReg(Reg64).addImm(40).addReg(0); 256 } 257 258 // Emit a zero-extending move from 32-bit GPR SrcReg to 32-bit GPR 259 // DestReg before MBBI in MBB. Use LowLowOpcode when both DestReg and SrcReg 260 // are low registers, otherwise use RISB[LH]G. Size is the number of bits 261 // taken from the low end of SrcReg (8 for LLCR, 16 for LLHR and 32 for LR). 262 // KillSrc is true if this move is the last use of SrcReg. 263 MachineInstrBuilder emitGRX32Move(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,unsigned DestReg,unsigned SrcReg,unsigned LowLowOpcode,unsigned Size,bool KillSrc,bool UndefSrc) const264 SystemZInstrInfo::emitGRX32Move(MachineBasicBlock &MBB, 265 MachineBasicBlock::iterator MBBI, 266 const DebugLoc &DL, unsigned DestReg, 267 unsigned SrcReg, unsigned LowLowOpcode, 268 unsigned Size, bool KillSrc, 269 bool UndefSrc) const { 270 unsigned Opcode; 271 bool DestIsHigh = SystemZ::isHighReg(DestReg); 272 bool SrcIsHigh = SystemZ::isHighReg(SrcReg); 273 if (DestIsHigh && SrcIsHigh) 274 Opcode = SystemZ::RISBHH; 275 else if (DestIsHigh && !SrcIsHigh) 276 Opcode = SystemZ::RISBHL; 277 else if (!DestIsHigh && SrcIsHigh) 278 Opcode = SystemZ::RISBLH; 279 else { 280 return BuildMI(MBB, MBBI, DL, get(LowLowOpcode), DestReg) 281 .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)); 282 } 283 unsigned Rotate = (DestIsHigh != SrcIsHigh ? 32 : 0); 284 return BuildMI(MBB, MBBI, DL, get(Opcode), DestReg) 285 .addReg(DestReg, RegState::Undef) 286 .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)) 287 .addImm(32 - Size).addImm(128 + 31).addImm(Rotate); 288 } 289 commuteInstructionImpl(MachineInstr & MI,bool NewMI,unsigned OpIdx1,unsigned OpIdx2) const290 MachineInstr *SystemZInstrInfo::commuteInstructionImpl(MachineInstr &MI, 291 bool NewMI, 292 unsigned OpIdx1, 293 unsigned OpIdx2) const { 294 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { 295 if (NewMI) 296 return *MI.getParent()->getParent()->CloneMachineInstr(&MI); 297 return MI; 298 }; 299 300 switch (MI.getOpcode()) { 301 case SystemZ::SELRMux: 302 case SystemZ::SELFHR: 303 case SystemZ::SELR: 304 case SystemZ::SELGR: 305 case SystemZ::LOCRMux: 306 case SystemZ::LOCFHR: 307 case SystemZ::LOCR: 308 case SystemZ::LOCGR: { 309 auto &WorkingMI = cloneIfNew(MI); 310 // Invert condition. 311 unsigned CCValid = WorkingMI.getOperand(3).getImm(); 312 unsigned CCMask = WorkingMI.getOperand(4).getImm(); 313 WorkingMI.getOperand(4).setImm(CCMask ^ CCValid); 314 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 315 OpIdx1, OpIdx2); 316 } 317 default: 318 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 319 } 320 } 321 322 // If MI is a simple load or store for a frame object, return the register 323 // it loads or stores and set FrameIndex to the index of the frame object. 324 // Return 0 otherwise. 325 // 326 // Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores. isSimpleMove(const MachineInstr & MI,int & FrameIndex,unsigned Flag)327 static int isSimpleMove(const MachineInstr &MI, int &FrameIndex, 328 unsigned Flag) { 329 const MCInstrDesc &MCID = MI.getDesc(); 330 if ((MCID.TSFlags & Flag) && MI.getOperand(1).isFI() && 331 MI.getOperand(2).getImm() == 0 && MI.getOperand(3).getReg() == 0) { 332 FrameIndex = MI.getOperand(1).getIndex(); 333 return MI.getOperand(0).getReg(); 334 } 335 return 0; 336 } 337 isLoadFromStackSlot(const MachineInstr & MI,int & FrameIndex) const338 Register SystemZInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 339 int &FrameIndex) const { 340 return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXLoad); 341 } 342 isStoreToStackSlot(const MachineInstr & MI,int & FrameIndex) const343 Register SystemZInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 344 int &FrameIndex) const { 345 return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXStore); 346 } 347 isStackSlotCopy(const MachineInstr & MI,int & DestFrameIndex,int & SrcFrameIndex) const348 bool SystemZInstrInfo::isStackSlotCopy(const MachineInstr &MI, 349 int &DestFrameIndex, 350 int &SrcFrameIndex) const { 351 // Check for MVC 0(Length,FI1),0(FI2) 352 const MachineFrameInfo &MFI = MI.getParent()->getParent()->getFrameInfo(); 353 if (MI.getOpcode() != SystemZ::MVC || !MI.getOperand(0).isFI() || 354 MI.getOperand(1).getImm() != 0 || !MI.getOperand(3).isFI() || 355 MI.getOperand(4).getImm() != 0) 356 return false; 357 358 // Check that Length covers the full slots. 359 int64_t Length = MI.getOperand(2).getImm(); 360 unsigned FI1 = MI.getOperand(0).getIndex(); 361 unsigned FI2 = MI.getOperand(3).getIndex(); 362 if (MFI.getObjectSize(FI1) != Length || 363 MFI.getObjectSize(FI2) != Length) 364 return false; 365 366 DestFrameIndex = FI1; 367 SrcFrameIndex = FI2; 368 return true; 369 } 370 analyzeBranch(MachineBasicBlock & MBB,MachineBasicBlock * & TBB,MachineBasicBlock * & FBB,SmallVectorImpl<MachineOperand> & Cond,bool AllowModify) const371 bool SystemZInstrInfo::analyzeBranch(MachineBasicBlock &MBB, 372 MachineBasicBlock *&TBB, 373 MachineBasicBlock *&FBB, 374 SmallVectorImpl<MachineOperand> &Cond, 375 bool AllowModify) const { 376 // Most of the code and comments here are boilerplate. 377 378 // Start from the bottom of the block and work up, examining the 379 // terminator instructions. 380 MachineBasicBlock::iterator I = MBB.end(); 381 while (I != MBB.begin()) { 382 --I; 383 if (I->isDebugInstr()) 384 continue; 385 386 // Working from the bottom, when we see a non-terminator instruction, we're 387 // done. 388 if (!isUnpredicatedTerminator(*I)) 389 break; 390 391 // A terminator that isn't a branch can't easily be handled by this 392 // analysis. 393 if (!I->isBranch()) 394 return true; 395 396 // Can't handle indirect branches. 397 SystemZII::Branch Branch(getBranchInfo(*I)); 398 if (!Branch.hasMBBTarget()) 399 return true; 400 401 // Punt on compound branches. 402 if (Branch.Type != SystemZII::BranchNormal) 403 return true; 404 405 if (Branch.CCMask == SystemZ::CCMASK_ANY) { 406 // Handle unconditional branches. 407 if (!AllowModify) { 408 TBB = Branch.getMBBTarget(); 409 continue; 410 } 411 412 // If the block has any instructions after a JMP, delete them. 413 MBB.erase(std::next(I), MBB.end()); 414 415 Cond.clear(); 416 FBB = nullptr; 417 418 // Delete the JMP if it's equivalent to a fall-through. 419 if (MBB.isLayoutSuccessor(Branch.getMBBTarget())) { 420 TBB = nullptr; 421 I->eraseFromParent(); 422 I = MBB.end(); 423 continue; 424 } 425 426 // TBB is used to indicate the unconditinal destination. 427 TBB = Branch.getMBBTarget(); 428 continue; 429 } 430 431 // Working from the bottom, handle the first conditional branch. 432 if (Cond.empty()) { 433 // FIXME: add X86-style branch swap 434 FBB = TBB; 435 TBB = Branch.getMBBTarget(); 436 Cond.push_back(MachineOperand::CreateImm(Branch.CCValid)); 437 Cond.push_back(MachineOperand::CreateImm(Branch.CCMask)); 438 continue; 439 } 440 441 // Handle subsequent conditional branches. 442 assert(Cond.size() == 2 && TBB && "Should have seen a conditional branch"); 443 444 // Only handle the case where all conditional branches branch to the same 445 // destination. 446 if (TBB != Branch.getMBBTarget()) 447 return true; 448 449 // If the conditions are the same, we can leave them alone. 450 unsigned OldCCValid = Cond[0].getImm(); 451 unsigned OldCCMask = Cond[1].getImm(); 452 if (OldCCValid == Branch.CCValid && OldCCMask == Branch.CCMask) 453 continue; 454 455 // FIXME: Try combining conditions like X86 does. Should be easy on Z! 456 return false; 457 } 458 459 return false; 460 } 461 removeBranch(MachineBasicBlock & MBB,int * BytesRemoved) const462 unsigned SystemZInstrInfo::removeBranch(MachineBasicBlock &MBB, 463 int *BytesRemoved) const { 464 assert(!BytesRemoved && "code size not handled"); 465 466 // Most of the code and comments here are boilerplate. 467 MachineBasicBlock::iterator I = MBB.end(); 468 unsigned Count = 0; 469 470 while (I != MBB.begin()) { 471 --I; 472 if (I->isDebugInstr()) 473 continue; 474 if (!I->isBranch()) 475 break; 476 if (!getBranchInfo(*I).hasMBBTarget()) 477 break; 478 // Remove the branch. 479 I->eraseFromParent(); 480 I = MBB.end(); 481 ++Count; 482 } 483 484 return Count; 485 } 486 487 bool SystemZInstrInfo:: reverseBranchCondition(SmallVectorImpl<MachineOperand> & Cond) const488 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { 489 assert(Cond.size() == 2 && "Invalid condition"); 490 Cond[1].setImm(Cond[1].getImm() ^ Cond[0].getImm()); 491 return false; 492 } 493 insertBranch(MachineBasicBlock & MBB,MachineBasicBlock * TBB,MachineBasicBlock * FBB,ArrayRef<MachineOperand> Cond,const DebugLoc & DL,int * BytesAdded) const494 unsigned SystemZInstrInfo::insertBranch(MachineBasicBlock &MBB, 495 MachineBasicBlock *TBB, 496 MachineBasicBlock *FBB, 497 ArrayRef<MachineOperand> Cond, 498 const DebugLoc &DL, 499 int *BytesAdded) const { 500 // In this function we output 32-bit branches, which should always 501 // have enough range. They can be shortened and relaxed by later code 502 // in the pipeline, if desired. 503 504 // Shouldn't be a fall through. 505 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 506 assert((Cond.size() == 2 || Cond.size() == 0) && 507 "SystemZ branch conditions have one component!"); 508 assert(!BytesAdded && "code size not handled"); 509 510 if (Cond.empty()) { 511 // Unconditional branch? 512 assert(!FBB && "Unconditional branch with multiple successors!"); 513 BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(TBB); 514 return 1; 515 } 516 517 // Conditional branch. 518 unsigned Count = 0; 519 unsigned CCValid = Cond[0].getImm(); 520 unsigned CCMask = Cond[1].getImm(); 521 BuildMI(&MBB, DL, get(SystemZ::BRC)) 522 .addImm(CCValid).addImm(CCMask).addMBB(TBB); 523 ++Count; 524 525 if (FBB) { 526 // Two-way Conditional branch. Insert the second branch. 527 BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(FBB); 528 ++Count; 529 } 530 return Count; 531 } 532 analyzeCompare(const MachineInstr & MI,Register & SrcReg,Register & SrcReg2,int64_t & Mask,int64_t & Value) const533 bool SystemZInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 534 Register &SrcReg2, int64_t &Mask, 535 int64_t &Value) const { 536 assert(MI.isCompare() && "Caller should have checked for a comparison"); 537 538 if (MI.getNumExplicitOperands() == 2 && MI.getOperand(0).isReg() && 539 MI.getOperand(1).isImm()) { 540 SrcReg = MI.getOperand(0).getReg(); 541 SrcReg2 = 0; 542 Value = MI.getOperand(1).getImm(); 543 Mask = ~0; 544 return true; 545 } 546 547 return false; 548 } 549 canInsertSelect(const MachineBasicBlock & MBB,ArrayRef<MachineOperand> Pred,Register DstReg,Register TrueReg,Register FalseReg,int & CondCycles,int & TrueCycles,int & FalseCycles) const550 bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 551 ArrayRef<MachineOperand> Pred, 552 Register DstReg, Register TrueReg, 553 Register FalseReg, int &CondCycles, 554 int &TrueCycles, 555 int &FalseCycles) const { 556 // Not all subtargets have LOCR instructions. 557 if (!STI.hasLoadStoreOnCond()) 558 return false; 559 if (Pred.size() != 2) 560 return false; 561 562 // Check register classes. 563 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 564 const TargetRegisterClass *RC = 565 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); 566 if (!RC) 567 return false; 568 569 // We have LOCR instructions for 32 and 64 bit general purpose registers. 570 if ((STI.hasLoadStoreOnCond2() && 571 SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) || 572 SystemZ::GR32BitRegClass.hasSubClassEq(RC) || 573 SystemZ::GR64BitRegClass.hasSubClassEq(RC)) { 574 CondCycles = 2; 575 TrueCycles = 2; 576 FalseCycles = 2; 577 return true; 578 } 579 580 // Can't do anything else. 581 return false; 582 } 583 insertSelect(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,const DebugLoc & DL,Register DstReg,ArrayRef<MachineOperand> Pred,Register TrueReg,Register FalseReg) const584 void SystemZInstrInfo::insertSelect(MachineBasicBlock &MBB, 585 MachineBasicBlock::iterator I, 586 const DebugLoc &DL, Register DstReg, 587 ArrayRef<MachineOperand> Pred, 588 Register TrueReg, 589 Register FalseReg) const { 590 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 591 const TargetRegisterClass *RC = MRI.getRegClass(DstReg); 592 593 assert(Pred.size() == 2 && "Invalid condition"); 594 unsigned CCValid = Pred[0].getImm(); 595 unsigned CCMask = Pred[1].getImm(); 596 597 unsigned Opc; 598 if (SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) { 599 if (STI.hasMiscellaneousExtensions3()) 600 Opc = SystemZ::SELRMux; 601 else if (STI.hasLoadStoreOnCond2()) 602 Opc = SystemZ::LOCRMux; 603 else { 604 Opc = SystemZ::LOCR; 605 MRI.constrainRegClass(DstReg, &SystemZ::GR32BitRegClass); 606 Register TReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); 607 Register FReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); 608 BuildMI(MBB, I, DL, get(TargetOpcode::COPY), TReg).addReg(TrueReg); 609 BuildMI(MBB, I, DL, get(TargetOpcode::COPY), FReg).addReg(FalseReg); 610 TrueReg = TReg; 611 FalseReg = FReg; 612 } 613 } else if (SystemZ::GR64BitRegClass.hasSubClassEq(RC)) { 614 if (STI.hasMiscellaneousExtensions3()) 615 Opc = SystemZ::SELGR; 616 else 617 Opc = SystemZ::LOCGR; 618 } else 619 llvm_unreachable("Invalid register class"); 620 621 BuildMI(MBB, I, DL, get(Opc), DstReg) 622 .addReg(FalseReg).addReg(TrueReg) 623 .addImm(CCValid).addImm(CCMask); 624 } 625 optimizeLoadInstr(MachineInstr & MI,const MachineRegisterInfo * MRI,Register & FoldAsLoadDefReg,MachineInstr * & DefMI) const626 MachineInstr *SystemZInstrInfo::optimizeLoadInstr(MachineInstr &MI, 627 const MachineRegisterInfo *MRI, 628 Register &FoldAsLoadDefReg, 629 MachineInstr *&DefMI) const { 630 // Check whether we can move the DefMI load, and that it only has one use. 631 DefMI = MRI->getVRegDef(FoldAsLoadDefReg); 632 assert(DefMI); 633 bool SawStore = false; 634 if (!DefMI->isSafeToMove(nullptr, SawStore) || 635 !MRI->hasOneNonDBGUse(FoldAsLoadDefReg)) 636 return nullptr; 637 638 int UseOpIdx = 639 MI.findRegisterUseOperandIdx(FoldAsLoadDefReg, /*TRI=*/nullptr); 640 assert(UseOpIdx != -1 && "Expected FoldAsLoadDefReg to be used by MI."); 641 642 // Check whether we can fold the load. 643 if (MachineInstr *FoldMI = 644 foldMemoryOperand(MI, {((unsigned)UseOpIdx)}, *DefMI)) { 645 FoldAsLoadDefReg = 0; 646 return FoldMI; 647 } 648 649 return nullptr; 650 } 651 foldImmediate(MachineInstr & UseMI,MachineInstr & DefMI,Register Reg,MachineRegisterInfo * MRI) const652 bool SystemZInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 653 Register Reg, 654 MachineRegisterInfo *MRI) const { 655 unsigned DefOpc = DefMI.getOpcode(); 656 657 if (DefOpc == SystemZ::VGBM) { 658 int64_t ImmVal = DefMI.getOperand(1).getImm(); 659 if (ImmVal != 0) // TODO: Handle other values 660 return false; 661 662 // Fold gr128 = COPY (vr128 VGBM imm) 663 // 664 // %tmp:gr64 = LGHI 0 665 // to gr128 = REG_SEQUENCE %tmp, %tmp 666 assert(DefMI.getOperand(0).getReg() == Reg); 667 668 if (!UseMI.isCopy()) 669 return false; 670 671 Register CopyDstReg = UseMI.getOperand(0).getReg(); 672 if (CopyDstReg.isVirtual() && 673 MRI->getRegClass(CopyDstReg) == &SystemZ::GR128BitRegClass && 674 MRI->hasOneNonDBGUse(Reg)) { 675 // TODO: Handle physical registers 676 // TODO: Handle gr64 uses with subregister indexes 677 // TODO: Should this multi-use cases? 678 Register TmpReg = MRI->createVirtualRegister(&SystemZ::GR64BitRegClass); 679 MachineBasicBlock &MBB = *UseMI.getParent(); 680 681 loadImmediate(MBB, UseMI.getIterator(), TmpReg, ImmVal); 682 683 UseMI.setDesc(get(SystemZ::REG_SEQUENCE)); 684 UseMI.getOperand(1).setReg(TmpReg); 685 MachineInstrBuilder(*MBB.getParent(), &UseMI) 686 .addImm(SystemZ::subreg_h64) 687 .addReg(TmpReg) 688 .addImm(SystemZ::subreg_l64); 689 690 if (MRI->use_nodbg_empty(Reg)) 691 DefMI.eraseFromParent(); 692 return true; 693 } 694 695 return false; 696 } 697 698 if (DefOpc != SystemZ::LHIMux && DefOpc != SystemZ::LHI && 699 DefOpc != SystemZ::LGHI) 700 return false; 701 if (DefMI.getOperand(0).getReg() != Reg) 702 return false; 703 int32_t ImmVal = (int32_t)DefMI.getOperand(1).getImm(); 704 705 unsigned UseOpc = UseMI.getOpcode(); 706 unsigned NewUseOpc; 707 unsigned UseIdx; 708 int CommuteIdx = -1; 709 bool TieOps = false; 710 switch (UseOpc) { 711 case SystemZ::SELRMux: 712 TieOps = true; 713 [[fallthrough]]; 714 case SystemZ::LOCRMux: 715 if (!STI.hasLoadStoreOnCond2()) 716 return false; 717 NewUseOpc = SystemZ::LOCHIMux; 718 if (UseMI.getOperand(2).getReg() == Reg) 719 UseIdx = 2; 720 else if (UseMI.getOperand(1).getReg() == Reg) 721 UseIdx = 2, CommuteIdx = 1; 722 else 723 return false; 724 break; 725 case SystemZ::SELGR: 726 TieOps = true; 727 [[fallthrough]]; 728 case SystemZ::LOCGR: 729 if (!STI.hasLoadStoreOnCond2()) 730 return false; 731 NewUseOpc = SystemZ::LOCGHI; 732 if (UseMI.getOperand(2).getReg() == Reg) 733 UseIdx = 2; 734 else if (UseMI.getOperand(1).getReg() == Reg) 735 UseIdx = 2, CommuteIdx = 1; 736 else 737 return false; 738 break; 739 default: 740 return false; 741 } 742 743 if (CommuteIdx != -1) 744 if (!commuteInstruction(UseMI, false, CommuteIdx, UseIdx)) 745 return false; 746 747 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 748 UseMI.setDesc(get(NewUseOpc)); 749 if (TieOps) 750 UseMI.tieOperands(0, 1); 751 UseMI.getOperand(UseIdx).ChangeToImmediate(ImmVal); 752 if (DeleteDef) 753 DefMI.eraseFromParent(); 754 755 return true; 756 } 757 isPredicable(const MachineInstr & MI) const758 bool SystemZInstrInfo::isPredicable(const MachineInstr &MI) const { 759 unsigned Opcode = MI.getOpcode(); 760 if (Opcode == SystemZ::Return || 761 Opcode == SystemZ::Return_XPLINK || 762 Opcode == SystemZ::Trap || 763 Opcode == SystemZ::CallJG || 764 Opcode == SystemZ::CallBR) 765 return true; 766 return false; 767 } 768 769 bool SystemZInstrInfo:: isProfitableToIfCvt(MachineBasicBlock & MBB,unsigned NumCycles,unsigned ExtraPredCycles,BranchProbability Probability) const770 isProfitableToIfCvt(MachineBasicBlock &MBB, 771 unsigned NumCycles, unsigned ExtraPredCycles, 772 BranchProbability Probability) const { 773 // Avoid using conditional returns at the end of a loop (since then 774 // we'd need to emit an unconditional branch to the beginning anyway, 775 // making the loop body longer). This doesn't apply for low-probability 776 // loops (eg. compare-and-swap retry), so just decide based on branch 777 // probability instead of looping structure. 778 // However, since Compare and Trap instructions cost the same as a regular 779 // Compare instruction, we should allow the if conversion to convert this 780 // into a Conditional Compare regardless of the branch probability. 781 if (MBB.getLastNonDebugInstr()->getOpcode() != SystemZ::Trap && 782 MBB.succ_empty() && Probability < BranchProbability(1, 8)) 783 return false; 784 // For now only convert single instructions. 785 return NumCycles == 1; 786 } 787 788 bool SystemZInstrInfo:: isProfitableToIfCvt(MachineBasicBlock & TMBB,unsigned NumCyclesT,unsigned ExtraPredCyclesT,MachineBasicBlock & FMBB,unsigned NumCyclesF,unsigned ExtraPredCyclesF,BranchProbability Probability) const789 isProfitableToIfCvt(MachineBasicBlock &TMBB, 790 unsigned NumCyclesT, unsigned ExtraPredCyclesT, 791 MachineBasicBlock &FMBB, 792 unsigned NumCyclesF, unsigned ExtraPredCyclesF, 793 BranchProbability Probability) const { 794 // For now avoid converting mutually-exclusive cases. 795 return false; 796 } 797 798 bool SystemZInstrInfo:: isProfitableToDupForIfCvt(MachineBasicBlock & MBB,unsigned NumCycles,BranchProbability Probability) const799 isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, 800 BranchProbability Probability) const { 801 // For now only duplicate single instructions. 802 return NumCycles == 1; 803 } 804 PredicateInstruction(MachineInstr & MI,ArrayRef<MachineOperand> Pred) const805 bool SystemZInstrInfo::PredicateInstruction( 806 MachineInstr &MI, ArrayRef<MachineOperand> Pred) const { 807 assert(Pred.size() == 2 && "Invalid condition"); 808 unsigned CCValid = Pred[0].getImm(); 809 unsigned CCMask = Pred[1].getImm(); 810 assert(CCMask > 0 && CCMask < 15 && "Invalid predicate"); 811 unsigned Opcode = MI.getOpcode(); 812 if (Opcode == SystemZ::Trap) { 813 MI.setDesc(get(SystemZ::CondTrap)); 814 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 815 .addImm(CCValid).addImm(CCMask) 816 .addReg(SystemZ::CC, RegState::Implicit); 817 return true; 818 } 819 if (Opcode == SystemZ::Return || Opcode == SystemZ::Return_XPLINK) { 820 MI.setDesc(get(Opcode == SystemZ::Return ? SystemZ::CondReturn 821 : SystemZ::CondReturn_XPLINK)); 822 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 823 .addImm(CCValid) 824 .addImm(CCMask) 825 .addReg(SystemZ::CC, RegState::Implicit); 826 return true; 827 } 828 if (Opcode == SystemZ::CallJG) { 829 MachineOperand FirstOp = MI.getOperand(0); 830 const uint32_t *RegMask = MI.getOperand(1).getRegMask(); 831 MI.removeOperand(1); 832 MI.removeOperand(0); 833 MI.setDesc(get(SystemZ::CallBRCL)); 834 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 835 .addImm(CCValid) 836 .addImm(CCMask) 837 .add(FirstOp) 838 .addRegMask(RegMask) 839 .addReg(SystemZ::CC, RegState::Implicit); 840 return true; 841 } 842 if (Opcode == SystemZ::CallBR) { 843 MachineOperand Target = MI.getOperand(0); 844 const uint32_t *RegMask = MI.getOperand(1).getRegMask(); 845 MI.removeOperand(1); 846 MI.removeOperand(0); 847 MI.setDesc(get(SystemZ::CallBCR)); 848 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 849 .addImm(CCValid).addImm(CCMask) 850 .add(Target) 851 .addRegMask(RegMask) 852 .addReg(SystemZ::CC, RegState::Implicit); 853 return true; 854 } 855 return false; 856 } 857 copyPhysReg(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,MCRegister DestReg,MCRegister SrcReg,bool KillSrc) const858 void SystemZInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 859 MachineBasicBlock::iterator MBBI, 860 const DebugLoc &DL, MCRegister DestReg, 861 MCRegister SrcReg, bool KillSrc) const { 862 // Split 128-bit GPR moves into two 64-bit moves. Add implicit uses of the 863 // super register in case one of the subregs is undefined. 864 // This handles ADDR128 too. 865 if (SystemZ::GR128BitRegClass.contains(DestReg, SrcReg)) { 866 copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_h64), 867 RI.getSubReg(SrcReg, SystemZ::subreg_h64), KillSrc); 868 MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI)) 869 .addReg(SrcReg, RegState::Implicit); 870 copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_l64), 871 RI.getSubReg(SrcReg, SystemZ::subreg_l64), KillSrc); 872 MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI)) 873 .addReg(SrcReg, (getKillRegState(KillSrc) | RegState::Implicit)); 874 return; 875 } 876 877 if (SystemZ::GRX32BitRegClass.contains(DestReg, SrcReg)) { 878 emitGRX32Move(MBB, MBBI, DL, DestReg, SrcReg, SystemZ::LR, 32, KillSrc, 879 false); 880 return; 881 } 882 883 // Move 128-bit floating-point values between VR128 and FP128. 884 if (SystemZ::VR128BitRegClass.contains(DestReg) && 885 SystemZ::FP128BitRegClass.contains(SrcReg)) { 886 MCRegister SrcRegHi = 887 RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_h64), 888 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 889 MCRegister SrcRegLo = 890 RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_l64), 891 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 892 893 BuildMI(MBB, MBBI, DL, get(SystemZ::VMRHG), DestReg) 894 .addReg(SrcRegHi, getKillRegState(KillSrc)) 895 .addReg(SrcRegLo, getKillRegState(KillSrc)); 896 return; 897 } 898 if (SystemZ::FP128BitRegClass.contains(DestReg) && 899 SystemZ::VR128BitRegClass.contains(SrcReg)) { 900 MCRegister DestRegHi = 901 RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_h64), 902 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 903 MCRegister DestRegLo = 904 RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_l64), 905 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 906 907 if (DestRegHi != SrcReg) 908 copyPhysReg(MBB, MBBI, DL, DestRegHi, SrcReg, false); 909 BuildMI(MBB, MBBI, DL, get(SystemZ::VREPG), DestRegLo) 910 .addReg(SrcReg, getKillRegState(KillSrc)).addImm(1); 911 return; 912 } 913 914 if (SystemZ::FP128BitRegClass.contains(DestReg) && 915 SystemZ::GR128BitRegClass.contains(SrcReg)) { 916 MCRegister DestRegHi = RI.getSubReg(DestReg, SystemZ::subreg_h64); 917 MCRegister DestRegLo = RI.getSubReg(DestReg, SystemZ::subreg_l64); 918 MCRegister SrcRegHi = RI.getSubReg(SrcReg, SystemZ::subreg_h64); 919 MCRegister SrcRegLo = RI.getSubReg(SrcReg, SystemZ::subreg_l64); 920 921 BuildMI(MBB, MBBI, DL, get(SystemZ::LDGR), DestRegHi) 922 .addReg(SrcRegHi) 923 .addReg(DestReg, RegState::ImplicitDefine); 924 925 BuildMI(MBB, MBBI, DL, get(SystemZ::LDGR), DestRegLo) 926 .addReg(SrcRegLo, getKillRegState(KillSrc)); 927 return; 928 } 929 930 // Move CC value from a GR32. 931 if (DestReg == SystemZ::CC) { 932 unsigned Opcode = 933 SystemZ::GR32BitRegClass.contains(SrcReg) ? SystemZ::TMLH : SystemZ::TMHH; 934 BuildMI(MBB, MBBI, DL, get(Opcode)) 935 .addReg(SrcReg, getKillRegState(KillSrc)) 936 .addImm(3 << (SystemZ::IPM_CC - 16)); 937 return; 938 } 939 940 if (SystemZ::GR128BitRegClass.contains(DestReg) && 941 SystemZ::VR128BitRegClass.contains(SrcReg)) { 942 MCRegister DestH64 = RI.getSubReg(DestReg, SystemZ::subreg_h64); 943 MCRegister DestL64 = RI.getSubReg(DestReg, SystemZ::subreg_l64); 944 945 BuildMI(MBB, MBBI, DL, get(SystemZ::VLGVG), DestH64) 946 .addReg(SrcReg) 947 .addReg(SystemZ::NoRegister) 948 .addImm(0) 949 .addDef(DestReg, RegState::Implicit); 950 BuildMI(MBB, MBBI, DL, get(SystemZ::VLGVG), DestL64) 951 .addReg(SrcReg, getKillRegState(KillSrc)) 952 .addReg(SystemZ::NoRegister) 953 .addImm(1); 954 return; 955 } 956 957 if (SystemZ::VR128BitRegClass.contains(DestReg) && 958 SystemZ::GR128BitRegClass.contains(SrcReg)) { 959 BuildMI(MBB, MBBI, DL, get(SystemZ::VLVGP), DestReg) 960 .addReg(RI.getSubReg(SrcReg, SystemZ::subreg_h64)) 961 .addReg(RI.getSubReg(SrcReg, SystemZ::subreg_l64)); 962 return; 963 } 964 965 // Everything else needs only one instruction. 966 unsigned Opcode; 967 if (SystemZ::GR64BitRegClass.contains(DestReg, SrcReg)) 968 Opcode = SystemZ::LGR; 969 else if (SystemZ::FP32BitRegClass.contains(DestReg, SrcReg)) 970 // For z13 we prefer LDR over LER to avoid partial register dependencies. 971 Opcode = STI.hasVector() ? SystemZ::LDR32 : SystemZ::LER; 972 else if (SystemZ::FP64BitRegClass.contains(DestReg, SrcReg)) 973 Opcode = SystemZ::LDR; 974 else if (SystemZ::FP128BitRegClass.contains(DestReg, SrcReg)) 975 Opcode = SystemZ::LXR; 976 else if (SystemZ::VR32BitRegClass.contains(DestReg, SrcReg)) 977 Opcode = SystemZ::VLR32; 978 else if (SystemZ::VR64BitRegClass.contains(DestReg, SrcReg)) 979 Opcode = SystemZ::VLR64; 980 else if (SystemZ::VR128BitRegClass.contains(DestReg, SrcReg)) 981 Opcode = SystemZ::VLR; 982 else if (SystemZ::AR32BitRegClass.contains(DestReg, SrcReg)) 983 Opcode = SystemZ::CPYA; 984 else 985 llvm_unreachable("Impossible reg-to-reg copy"); 986 987 BuildMI(MBB, MBBI, DL, get(Opcode), DestReg) 988 .addReg(SrcReg, getKillRegState(KillSrc)); 989 } 990 storeRegToStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,Register SrcReg,bool isKill,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI,Register VReg) const991 void SystemZInstrInfo::storeRegToStackSlot( 992 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, 993 bool isKill, int FrameIdx, const TargetRegisterClass *RC, 994 const TargetRegisterInfo *TRI, Register VReg) const { 995 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 996 997 // Callers may expect a single instruction, so keep 128-bit moves 998 // together for now and lower them after register allocation. 999 unsigned LoadOpcode, StoreOpcode; 1000 getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode); 1001 addFrameReference(BuildMI(MBB, MBBI, DL, get(StoreOpcode)) 1002 .addReg(SrcReg, getKillRegState(isKill)), 1003 FrameIdx); 1004 } 1005 loadRegFromStackSlot(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,Register DestReg,int FrameIdx,const TargetRegisterClass * RC,const TargetRegisterInfo * TRI,Register VReg) const1006 void SystemZInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 1007 MachineBasicBlock::iterator MBBI, 1008 Register DestReg, int FrameIdx, 1009 const TargetRegisterClass *RC, 1010 const TargetRegisterInfo *TRI, 1011 Register VReg) const { 1012 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 1013 1014 // Callers may expect a single instruction, so keep 128-bit moves 1015 // together for now and lower them after register allocation. 1016 unsigned LoadOpcode, StoreOpcode; 1017 getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode); 1018 addFrameReference(BuildMI(MBB, MBBI, DL, get(LoadOpcode), DestReg), 1019 FrameIdx); 1020 } 1021 1022 // Return true if MI is a simple load or store with a 12-bit displacement 1023 // and no index. Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores. isSimpleBD12Move(const MachineInstr * MI,unsigned Flag)1024 static bool isSimpleBD12Move(const MachineInstr *MI, unsigned Flag) { 1025 const MCInstrDesc &MCID = MI->getDesc(); 1026 return ((MCID.TSFlags & Flag) && 1027 isUInt<12>(MI->getOperand(2).getImm()) && 1028 MI->getOperand(3).getReg() == 0); 1029 } 1030 1031 namespace { 1032 1033 struct LogicOp { 1034 LogicOp() = default; LogicOp__anon758fef8f0311::LogicOp1035 LogicOp(unsigned regSize, unsigned immLSB, unsigned immSize) 1036 : RegSize(regSize), ImmLSB(immLSB), ImmSize(immSize) {} 1037 operator bool__anon758fef8f0311::LogicOp1038 explicit operator bool() const { return RegSize; } 1039 1040 unsigned RegSize = 0; 1041 unsigned ImmLSB = 0; 1042 unsigned ImmSize = 0; 1043 }; 1044 1045 } // end anonymous namespace 1046 interpretAndImmediate(unsigned Opcode)1047 static LogicOp interpretAndImmediate(unsigned Opcode) { 1048 switch (Opcode) { 1049 case SystemZ::NILMux: return LogicOp(32, 0, 16); 1050 case SystemZ::NIHMux: return LogicOp(32, 16, 16); 1051 case SystemZ::NILL64: return LogicOp(64, 0, 16); 1052 case SystemZ::NILH64: return LogicOp(64, 16, 16); 1053 case SystemZ::NIHL64: return LogicOp(64, 32, 16); 1054 case SystemZ::NIHH64: return LogicOp(64, 48, 16); 1055 case SystemZ::NIFMux: return LogicOp(32, 0, 32); 1056 case SystemZ::NILF64: return LogicOp(64, 0, 32); 1057 case SystemZ::NIHF64: return LogicOp(64, 32, 32); 1058 default: return LogicOp(); 1059 } 1060 } 1061 transferDeadCC(MachineInstr * OldMI,MachineInstr * NewMI)1062 static void transferDeadCC(MachineInstr *OldMI, MachineInstr *NewMI) { 1063 if (OldMI->registerDefIsDead(SystemZ::CC, /*TRI=*/nullptr)) { 1064 MachineOperand *CCDef = 1065 NewMI->findRegisterDefOperand(SystemZ::CC, /*TRI=*/nullptr); 1066 if (CCDef != nullptr) 1067 CCDef->setIsDead(true); 1068 } 1069 } 1070 transferMIFlag(MachineInstr * OldMI,MachineInstr * NewMI,MachineInstr::MIFlag Flag)1071 static void transferMIFlag(MachineInstr *OldMI, MachineInstr *NewMI, 1072 MachineInstr::MIFlag Flag) { 1073 if (OldMI->getFlag(Flag)) 1074 NewMI->setFlag(Flag); 1075 } 1076 1077 MachineInstr * convertToThreeAddress(MachineInstr & MI,LiveVariables * LV,LiveIntervals * LIS) const1078 SystemZInstrInfo::convertToThreeAddress(MachineInstr &MI, LiveVariables *LV, 1079 LiveIntervals *LIS) const { 1080 MachineBasicBlock *MBB = MI.getParent(); 1081 1082 // Try to convert an AND into an RISBG-type instruction. 1083 // TODO: It might be beneficial to select RISBG and shorten to AND instead. 1084 if (LogicOp And = interpretAndImmediate(MI.getOpcode())) { 1085 uint64_t Imm = MI.getOperand(2).getImm() << And.ImmLSB; 1086 // AND IMMEDIATE leaves the other bits of the register unchanged. 1087 Imm |= allOnes(And.RegSize) & ~(allOnes(And.ImmSize) << And.ImmLSB); 1088 unsigned Start, End; 1089 if (isRxSBGMask(Imm, And.RegSize, Start, End)) { 1090 unsigned NewOpcode; 1091 if (And.RegSize == 64) { 1092 NewOpcode = SystemZ::RISBG; 1093 // Prefer RISBGN if available, since it does not clobber CC. 1094 if (STI.hasMiscellaneousExtensions()) 1095 NewOpcode = SystemZ::RISBGN; 1096 } else { 1097 NewOpcode = SystemZ::RISBMux; 1098 Start &= 31; 1099 End &= 31; 1100 } 1101 MachineOperand &Dest = MI.getOperand(0); 1102 MachineOperand &Src = MI.getOperand(1); 1103 MachineInstrBuilder MIB = 1104 BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpcode)) 1105 .add(Dest) 1106 .addReg(0) 1107 .addReg(Src.getReg(), getKillRegState(Src.isKill()), 1108 Src.getSubReg()) 1109 .addImm(Start) 1110 .addImm(End + 128) 1111 .addImm(0); 1112 if (LV) { 1113 unsigned NumOps = MI.getNumOperands(); 1114 for (unsigned I = 1; I < NumOps; ++I) { 1115 MachineOperand &Op = MI.getOperand(I); 1116 if (Op.isReg() && Op.isKill()) 1117 LV->replaceKillInstruction(Op.getReg(), MI, *MIB); 1118 } 1119 } 1120 if (LIS) 1121 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 1122 transferDeadCC(&MI, MIB); 1123 return MIB; 1124 } 1125 } 1126 return nullptr; 1127 } 1128 isAssociativeAndCommutative(const MachineInstr & Inst,bool Invert) const1129 bool SystemZInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst, 1130 bool Invert) const { 1131 unsigned Opc = Inst.getOpcode(); 1132 if (Invert) { 1133 auto InverseOpcode = getInverseOpcode(Opc); 1134 if (!InverseOpcode) 1135 return false; 1136 Opc = *InverseOpcode; 1137 } 1138 1139 switch (Opc) { 1140 default: 1141 break; 1142 // Adds and multiplications. 1143 case SystemZ::WFADB: 1144 case SystemZ::WFASB: 1145 case SystemZ::WFAXB: 1146 case SystemZ::VFADB: 1147 case SystemZ::VFASB: 1148 case SystemZ::WFMDB: 1149 case SystemZ::WFMSB: 1150 case SystemZ::WFMXB: 1151 case SystemZ::VFMDB: 1152 case SystemZ::VFMSB: 1153 return (Inst.getFlag(MachineInstr::MIFlag::FmReassoc) && 1154 Inst.getFlag(MachineInstr::MIFlag::FmNsz)); 1155 } 1156 1157 return false; 1158 } 1159 1160 std::optional<unsigned> getInverseOpcode(unsigned Opcode) const1161 SystemZInstrInfo::getInverseOpcode(unsigned Opcode) const { 1162 // fadd => fsub 1163 switch (Opcode) { 1164 case SystemZ::WFADB: 1165 return SystemZ::WFSDB; 1166 case SystemZ::WFASB: 1167 return SystemZ::WFSSB; 1168 case SystemZ::WFAXB: 1169 return SystemZ::WFSXB; 1170 case SystemZ::VFADB: 1171 return SystemZ::VFSDB; 1172 case SystemZ::VFASB: 1173 return SystemZ::VFSSB; 1174 // fsub => fadd 1175 case SystemZ::WFSDB: 1176 return SystemZ::WFADB; 1177 case SystemZ::WFSSB: 1178 return SystemZ::WFASB; 1179 case SystemZ::WFSXB: 1180 return SystemZ::WFAXB; 1181 case SystemZ::VFSDB: 1182 return SystemZ::VFADB; 1183 case SystemZ::VFSSB: 1184 return SystemZ::VFASB; 1185 default: 1186 return std::nullopt; 1187 } 1188 } 1189 foldMemoryOperandImpl(MachineFunction & MF,MachineInstr & MI,ArrayRef<unsigned> Ops,MachineBasicBlock::iterator InsertPt,int FrameIndex,LiveIntervals * LIS,VirtRegMap * VRM) const1190 MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl( 1191 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 1192 MachineBasicBlock::iterator InsertPt, int FrameIndex, 1193 LiveIntervals *LIS, VirtRegMap *VRM) const { 1194 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 1195 MachineRegisterInfo &MRI = MF.getRegInfo(); 1196 const MachineFrameInfo &MFI = MF.getFrameInfo(); 1197 unsigned Size = MFI.getObjectSize(FrameIndex); 1198 unsigned Opcode = MI.getOpcode(); 1199 1200 // Check CC liveness if new instruction introduces a dead def of CC. 1201 SlotIndex MISlot = SlotIndex(); 1202 LiveRange *CCLiveRange = nullptr; 1203 bool CCLiveAtMI = true; 1204 if (LIS) { 1205 MISlot = LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot(); 1206 auto CCUnits = TRI->regunits(MCRegister::from(SystemZ::CC)); 1207 assert(range_size(CCUnits) == 1 && "CC only has one reg unit."); 1208 CCLiveRange = &LIS->getRegUnit(*CCUnits.begin()); 1209 CCLiveAtMI = CCLiveRange->liveAt(MISlot); 1210 } 1211 1212 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 1213 if (!CCLiveAtMI && (Opcode == SystemZ::LA || Opcode == SystemZ::LAY) && 1214 isInt<8>(MI.getOperand(2).getImm()) && !MI.getOperand(3).getReg()) { 1215 // LA(Y) %reg, CONST(%reg) -> AGSI %mem, CONST 1216 MachineInstr *BuiltMI = BuildMI(*InsertPt->getParent(), InsertPt, 1217 MI.getDebugLoc(), get(SystemZ::AGSI)) 1218 .addFrameIndex(FrameIndex) 1219 .addImm(0) 1220 .addImm(MI.getOperand(2).getImm()); 1221 BuiltMI->findRegisterDefOperand(SystemZ::CC, /*TRI=*/nullptr) 1222 ->setIsDead(true); 1223 CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator()); 1224 return BuiltMI; 1225 } 1226 return nullptr; 1227 } 1228 1229 // All other cases require a single operand. 1230 if (Ops.size() != 1) 1231 return nullptr; 1232 1233 unsigned OpNum = Ops[0]; 1234 assert(Size * 8 == 1235 TRI->getRegSizeInBits(*MF.getRegInfo() 1236 .getRegClass(MI.getOperand(OpNum).getReg())) && 1237 "Invalid size combination"); 1238 1239 if ((Opcode == SystemZ::AHI || Opcode == SystemZ::AGHI) && OpNum == 0 && 1240 isInt<8>(MI.getOperand(2).getImm())) { 1241 // A(G)HI %reg, CONST -> A(G)SI %mem, CONST 1242 Opcode = (Opcode == SystemZ::AHI ? SystemZ::ASI : SystemZ::AGSI); 1243 MachineInstr *BuiltMI = 1244 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1245 .addFrameIndex(FrameIndex) 1246 .addImm(0) 1247 .addImm(MI.getOperand(2).getImm()); 1248 transferDeadCC(&MI, BuiltMI); 1249 transferMIFlag(&MI, BuiltMI, MachineInstr::NoSWrap); 1250 return BuiltMI; 1251 } 1252 1253 if ((Opcode == SystemZ::ALFI && OpNum == 0 && 1254 isInt<8>((int32_t)MI.getOperand(2).getImm())) || 1255 (Opcode == SystemZ::ALGFI && OpNum == 0 && 1256 isInt<8>((int64_t)MI.getOperand(2).getImm()))) { 1257 // AL(G)FI %reg, CONST -> AL(G)SI %mem, CONST 1258 Opcode = (Opcode == SystemZ::ALFI ? SystemZ::ALSI : SystemZ::ALGSI); 1259 MachineInstr *BuiltMI = 1260 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1261 .addFrameIndex(FrameIndex) 1262 .addImm(0) 1263 .addImm((int8_t)MI.getOperand(2).getImm()); 1264 transferDeadCC(&MI, BuiltMI); 1265 return BuiltMI; 1266 } 1267 1268 if ((Opcode == SystemZ::SLFI && OpNum == 0 && 1269 isInt<8>((int32_t)-MI.getOperand(2).getImm())) || 1270 (Opcode == SystemZ::SLGFI && OpNum == 0 && 1271 isInt<8>((int64_t)-MI.getOperand(2).getImm()))) { 1272 // SL(G)FI %reg, CONST -> AL(G)SI %mem, -CONST 1273 Opcode = (Opcode == SystemZ::SLFI ? SystemZ::ALSI : SystemZ::ALGSI); 1274 MachineInstr *BuiltMI = 1275 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1276 .addFrameIndex(FrameIndex) 1277 .addImm(0) 1278 .addImm((int8_t)-MI.getOperand(2).getImm()); 1279 transferDeadCC(&MI, BuiltMI); 1280 return BuiltMI; 1281 } 1282 1283 unsigned MemImmOpc = 0; 1284 switch (Opcode) { 1285 case SystemZ::LHIMux: 1286 case SystemZ::LHI: MemImmOpc = SystemZ::MVHI; break; 1287 case SystemZ::LGHI: MemImmOpc = SystemZ::MVGHI; break; 1288 case SystemZ::CHIMux: 1289 case SystemZ::CHI: MemImmOpc = SystemZ::CHSI; break; 1290 case SystemZ::CGHI: MemImmOpc = SystemZ::CGHSI; break; 1291 case SystemZ::CLFIMux: 1292 case SystemZ::CLFI: 1293 if (isUInt<16>(MI.getOperand(1).getImm())) 1294 MemImmOpc = SystemZ::CLFHSI; 1295 break; 1296 case SystemZ::CLGFI: 1297 if (isUInt<16>(MI.getOperand(1).getImm())) 1298 MemImmOpc = SystemZ::CLGHSI; 1299 break; 1300 default: break; 1301 } 1302 if (MemImmOpc) 1303 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1304 get(MemImmOpc)) 1305 .addFrameIndex(FrameIndex) 1306 .addImm(0) 1307 .addImm(MI.getOperand(1).getImm()); 1308 1309 if (Opcode == SystemZ::LGDR || Opcode == SystemZ::LDGR) { 1310 bool Op0IsGPR = (Opcode == SystemZ::LGDR); 1311 bool Op1IsGPR = (Opcode == SystemZ::LDGR); 1312 // If we're spilling the destination of an LDGR or LGDR, store the 1313 // source register instead. 1314 if (OpNum == 0) { 1315 unsigned StoreOpcode = Op1IsGPR ? SystemZ::STG : SystemZ::STD; 1316 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1317 get(StoreOpcode)) 1318 .add(MI.getOperand(1)) 1319 .addFrameIndex(FrameIndex) 1320 .addImm(0) 1321 .addReg(0); 1322 } 1323 // If we're spilling the source of an LDGR or LGDR, load the 1324 // destination register instead. 1325 if (OpNum == 1) { 1326 unsigned LoadOpcode = Op0IsGPR ? SystemZ::LG : SystemZ::LD; 1327 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1328 get(LoadOpcode)) 1329 .add(MI.getOperand(0)) 1330 .addFrameIndex(FrameIndex) 1331 .addImm(0) 1332 .addReg(0); 1333 } 1334 } 1335 1336 // Look for cases where the source of a simple store or the destination 1337 // of a simple load is being spilled. Try to use MVC instead. 1338 // 1339 // Although MVC is in practice a fast choice in these cases, it is still 1340 // logically a bytewise copy. This means that we cannot use it if the 1341 // load or store is volatile. We also wouldn't be able to use MVC if 1342 // the two memories partially overlap, but that case cannot occur here, 1343 // because we know that one of the memories is a full frame index. 1344 // 1345 // For performance reasons, we also want to avoid using MVC if the addresses 1346 // might be equal. We don't worry about that case here, because spill slot 1347 // coloring happens later, and because we have special code to remove 1348 // MVCs that turn out to be redundant. 1349 if (OpNum == 0 && MI.hasOneMemOperand()) { 1350 MachineMemOperand *MMO = *MI.memoperands_begin(); 1351 if (MMO->getSize() == Size && !MMO->isVolatile() && !MMO->isAtomic()) { 1352 // Handle conversion of loads. 1353 if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXLoad)) { 1354 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1355 get(SystemZ::MVC)) 1356 .addFrameIndex(FrameIndex) 1357 .addImm(0) 1358 .addImm(Size) 1359 .add(MI.getOperand(1)) 1360 .addImm(MI.getOperand(2).getImm()) 1361 .addMemOperand(MMO); 1362 } 1363 // Handle conversion of stores. 1364 if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXStore)) { 1365 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1366 get(SystemZ::MVC)) 1367 .add(MI.getOperand(1)) 1368 .addImm(MI.getOperand(2).getImm()) 1369 .addImm(Size) 1370 .addFrameIndex(FrameIndex) 1371 .addImm(0) 1372 .addMemOperand(MMO); 1373 } 1374 } 1375 } 1376 1377 // If the spilled operand is the final one or the instruction is 1378 // commutable, try to change <INSN>R into <INSN>. Don't introduce a def of 1379 // CC if it is live and MI does not define it. 1380 unsigned NumOps = MI.getNumExplicitOperands(); 1381 int MemOpcode = SystemZ::getMemOpcode(Opcode); 1382 if (MemOpcode == -1 || 1383 (CCLiveAtMI && !MI.definesRegister(SystemZ::CC, /*TRI=*/nullptr) && 1384 get(MemOpcode).hasImplicitDefOfPhysReg(SystemZ::CC))) 1385 return nullptr; 1386 1387 // Check if all other vregs have a usable allocation in the case of vector 1388 // to FP conversion. 1389 const MCInstrDesc &MCID = MI.getDesc(); 1390 for (unsigned I = 0, E = MCID.getNumOperands(); I != E; ++I) { 1391 const MCOperandInfo &MCOI = MCID.operands()[I]; 1392 if (MCOI.OperandType != MCOI::OPERAND_REGISTER || I == OpNum) 1393 continue; 1394 const TargetRegisterClass *RC = TRI->getRegClass(MCOI.RegClass); 1395 if (RC == &SystemZ::VR32BitRegClass || RC == &SystemZ::VR64BitRegClass) { 1396 Register Reg = MI.getOperand(I).getReg(); 1397 Register PhysReg = Reg.isVirtual() 1398 ? (VRM ? Register(VRM->getPhys(Reg)) : Register()) 1399 : Reg; 1400 if (!PhysReg || 1401 !(SystemZ::FP32BitRegClass.contains(PhysReg) || 1402 SystemZ::FP64BitRegClass.contains(PhysReg) || 1403 SystemZ::VF128BitRegClass.contains(PhysReg))) 1404 return nullptr; 1405 } 1406 } 1407 // Fused multiply and add/sub need to have the same dst and accumulator reg. 1408 bool FusedFPOp = (Opcode == SystemZ::WFMADB || Opcode == SystemZ::WFMASB || 1409 Opcode == SystemZ::WFMSDB || Opcode == SystemZ::WFMSSB); 1410 if (FusedFPOp) { 1411 Register DstReg = VRM->getPhys(MI.getOperand(0).getReg()); 1412 Register AccReg = VRM->getPhys(MI.getOperand(3).getReg()); 1413 if (OpNum == 0 || OpNum == 3 || DstReg != AccReg) 1414 return nullptr; 1415 } 1416 1417 // Try to swap compare operands if possible. 1418 bool NeedsCommute = false; 1419 if ((MI.getOpcode() == SystemZ::CR || MI.getOpcode() == SystemZ::CGR || 1420 MI.getOpcode() == SystemZ::CLR || MI.getOpcode() == SystemZ::CLGR || 1421 MI.getOpcode() == SystemZ::WFCDB || MI.getOpcode() == SystemZ::WFCSB || 1422 MI.getOpcode() == SystemZ::WFKDB || MI.getOpcode() == SystemZ::WFKSB) && 1423 OpNum == 0 && prepareCompareSwapOperands(MI)) 1424 NeedsCommute = true; 1425 1426 bool CCOperands = false; 1427 if (MI.getOpcode() == SystemZ::LOCRMux || MI.getOpcode() == SystemZ::LOCGR || 1428 MI.getOpcode() == SystemZ::SELRMux || MI.getOpcode() == SystemZ::SELGR) { 1429 assert(MI.getNumOperands() == 6 && NumOps == 5 && 1430 "LOCR/SELR instruction operands corrupt?"); 1431 NumOps -= 2; 1432 CCOperands = true; 1433 } 1434 1435 // See if this is a 3-address instruction that is convertible to 2-address 1436 // and suitable for folding below. Only try this with virtual registers 1437 // and a provided VRM (during regalloc). 1438 if (NumOps == 3 && SystemZ::getTargetMemOpcode(MemOpcode) != -1) { 1439 if (VRM == nullptr) 1440 return nullptr; 1441 else { 1442 Register DstReg = MI.getOperand(0).getReg(); 1443 Register DstPhys = 1444 (DstReg.isVirtual() ? Register(VRM->getPhys(DstReg)) : DstReg); 1445 Register SrcReg = (OpNum == 2 ? MI.getOperand(1).getReg() 1446 : ((OpNum == 1 && MI.isCommutable()) 1447 ? MI.getOperand(2).getReg() 1448 : Register())); 1449 if (DstPhys && !SystemZ::GRH32BitRegClass.contains(DstPhys) && SrcReg && 1450 SrcReg.isVirtual() && DstPhys == VRM->getPhys(SrcReg)) 1451 NeedsCommute = (OpNum == 1); 1452 else 1453 return nullptr; 1454 } 1455 } 1456 1457 if ((OpNum == NumOps - 1) || NeedsCommute || FusedFPOp) { 1458 const MCInstrDesc &MemDesc = get(MemOpcode); 1459 uint64_t AccessBytes = SystemZII::getAccessSize(MemDesc.TSFlags); 1460 assert(AccessBytes != 0 && "Size of access should be known"); 1461 assert(AccessBytes <= Size && "Access outside the frame index"); 1462 uint64_t Offset = Size - AccessBytes; 1463 MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, 1464 MI.getDebugLoc(), get(MemOpcode)); 1465 if (MI.isCompare()) { 1466 assert(NumOps == 2 && "Expected 2 register operands for a compare."); 1467 MIB.add(MI.getOperand(NeedsCommute ? 1 : 0)); 1468 } 1469 else if (FusedFPOp) { 1470 MIB.add(MI.getOperand(0)); 1471 MIB.add(MI.getOperand(3)); 1472 MIB.add(MI.getOperand(OpNum == 1 ? 2 : 1)); 1473 } 1474 else { 1475 MIB.add(MI.getOperand(0)); 1476 if (NeedsCommute) 1477 MIB.add(MI.getOperand(2)); 1478 else 1479 for (unsigned I = 1; I < OpNum; ++I) 1480 MIB.add(MI.getOperand(I)); 1481 } 1482 MIB.addFrameIndex(FrameIndex).addImm(Offset); 1483 if (MemDesc.TSFlags & SystemZII::HasIndex) 1484 MIB.addReg(0); 1485 if (CCOperands) { 1486 unsigned CCValid = MI.getOperand(NumOps).getImm(); 1487 unsigned CCMask = MI.getOperand(NumOps + 1).getImm(); 1488 MIB.addImm(CCValid); 1489 MIB.addImm(NeedsCommute ? CCMask ^ CCValid : CCMask); 1490 } 1491 if (MIB->definesRegister(SystemZ::CC, /*TRI=*/nullptr) && 1492 (!MI.definesRegister(SystemZ::CC, /*TRI=*/nullptr) || 1493 MI.registerDefIsDead(SystemZ::CC, /*TRI=*/nullptr))) { 1494 MIB->addRegisterDead(SystemZ::CC, TRI); 1495 if (CCLiveRange) 1496 CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator()); 1497 } 1498 // Constrain the register classes if converted from a vector opcode. The 1499 // allocated regs are in an FP reg-class per previous check above. 1500 for (const MachineOperand &MO : MIB->operands()) 1501 if (MO.isReg() && MO.getReg().isVirtual()) { 1502 Register Reg = MO.getReg(); 1503 if (MRI.getRegClass(Reg) == &SystemZ::VR32BitRegClass) 1504 MRI.setRegClass(Reg, &SystemZ::FP32BitRegClass); 1505 else if (MRI.getRegClass(Reg) == &SystemZ::VR64BitRegClass) 1506 MRI.setRegClass(Reg, &SystemZ::FP64BitRegClass); 1507 else if (MRI.getRegClass(Reg) == &SystemZ::VR128BitRegClass) 1508 MRI.setRegClass(Reg, &SystemZ::VF128BitRegClass); 1509 } 1510 1511 transferDeadCC(&MI, MIB); 1512 transferMIFlag(&MI, MIB, MachineInstr::NoSWrap); 1513 transferMIFlag(&MI, MIB, MachineInstr::NoFPExcept); 1514 return MIB; 1515 } 1516 1517 return nullptr; 1518 } 1519 foldMemoryOperandImpl(MachineFunction & MF,MachineInstr & MI,ArrayRef<unsigned> Ops,MachineBasicBlock::iterator InsertPt,MachineInstr & LoadMI,LiveIntervals * LIS) const1520 MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl( 1521 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 1522 MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, 1523 LiveIntervals *LIS) const { 1524 MachineRegisterInfo *MRI = &MF.getRegInfo(); 1525 MachineBasicBlock *MBB = MI.getParent(); 1526 1527 // For reassociable FP operations, any loads have been purposefully left 1528 // unfolded so that MachineCombiner can do its work on reg/reg 1529 // opcodes. After that, as many loads as possible are now folded. 1530 // TODO: This may be beneficial with other opcodes as well as machine-sink 1531 // can move loads close to their user in a different MBB, which the isel 1532 // matcher did not see. 1533 unsigned LoadOpc = 0; 1534 unsigned RegMemOpcode = 0; 1535 const TargetRegisterClass *FPRC = nullptr; 1536 RegMemOpcode = MI.getOpcode() == SystemZ::WFADB ? SystemZ::ADB 1537 : MI.getOpcode() == SystemZ::WFSDB ? SystemZ::SDB 1538 : MI.getOpcode() == SystemZ::WFMDB ? SystemZ::MDB 1539 : 0; 1540 if (RegMemOpcode) { 1541 LoadOpc = SystemZ::VL64; 1542 FPRC = &SystemZ::FP64BitRegClass; 1543 } else { 1544 RegMemOpcode = MI.getOpcode() == SystemZ::WFASB ? SystemZ::AEB 1545 : MI.getOpcode() == SystemZ::WFSSB ? SystemZ::SEB 1546 : MI.getOpcode() == SystemZ::WFMSB ? SystemZ::MEEB 1547 : 0; 1548 if (RegMemOpcode) { 1549 LoadOpc = SystemZ::VL32; 1550 FPRC = &SystemZ::FP32BitRegClass; 1551 } 1552 } 1553 if (!RegMemOpcode || LoadMI.getOpcode() != LoadOpc) 1554 return nullptr; 1555 1556 // If RegMemOpcode clobbers CC, first make sure CC is not live at this point. 1557 if (get(RegMemOpcode).hasImplicitDefOfPhysReg(SystemZ::CC)) { 1558 assert(LoadMI.getParent() == MI.getParent() && "Assuming a local fold."); 1559 assert(LoadMI != InsertPt && "Assuming InsertPt not to be first in MBB."); 1560 for (MachineBasicBlock::iterator MII = std::prev(InsertPt);; 1561 --MII) { 1562 if (MII->definesRegister(SystemZ::CC, /*TRI=*/nullptr)) { 1563 if (!MII->registerDefIsDead(SystemZ::CC, /*TRI=*/nullptr)) 1564 return nullptr; 1565 break; 1566 } 1567 if (MII == MBB->begin()) { 1568 if (MBB->isLiveIn(SystemZ::CC)) 1569 return nullptr; 1570 break; 1571 } 1572 } 1573 } 1574 1575 Register FoldAsLoadDefReg = LoadMI.getOperand(0).getReg(); 1576 if (Ops.size() != 1 || FoldAsLoadDefReg != MI.getOperand(Ops[0]).getReg()) 1577 return nullptr; 1578 Register DstReg = MI.getOperand(0).getReg(); 1579 MachineOperand LHS = MI.getOperand(1); 1580 MachineOperand RHS = MI.getOperand(2); 1581 MachineOperand &RegMO = RHS.getReg() == FoldAsLoadDefReg ? LHS : RHS; 1582 if ((RegMemOpcode == SystemZ::SDB || RegMemOpcode == SystemZ::SEB) && 1583 FoldAsLoadDefReg != RHS.getReg()) 1584 return nullptr; 1585 1586 MachineOperand &Base = LoadMI.getOperand(1); 1587 MachineOperand &Disp = LoadMI.getOperand(2); 1588 MachineOperand &Indx = LoadMI.getOperand(3); 1589 MachineInstrBuilder MIB = 1590 BuildMI(*MI.getParent(), InsertPt, MI.getDebugLoc(), get(RegMemOpcode), DstReg) 1591 .add(RegMO) 1592 .add(Base) 1593 .add(Disp) 1594 .add(Indx); 1595 MIB->addRegisterDead(SystemZ::CC, &RI); 1596 MRI->setRegClass(DstReg, FPRC); 1597 MRI->setRegClass(RegMO.getReg(), FPRC); 1598 transferMIFlag(&MI, MIB, MachineInstr::NoFPExcept); 1599 1600 return MIB; 1601 } 1602 expandPostRAPseudo(MachineInstr & MI) const1603 bool SystemZInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 1604 switch (MI.getOpcode()) { 1605 case SystemZ::L128: 1606 splitMove(MI, SystemZ::LG); 1607 return true; 1608 1609 case SystemZ::ST128: 1610 splitMove(MI, SystemZ::STG); 1611 return true; 1612 1613 case SystemZ::LX: 1614 splitMove(MI, SystemZ::LD); 1615 return true; 1616 1617 case SystemZ::STX: 1618 splitMove(MI, SystemZ::STD); 1619 return true; 1620 1621 case SystemZ::LBMux: 1622 expandRXYPseudo(MI, SystemZ::LB, SystemZ::LBH); 1623 return true; 1624 1625 case SystemZ::LHMux: 1626 expandRXYPseudo(MI, SystemZ::LH, SystemZ::LHH); 1627 return true; 1628 1629 case SystemZ::LLCRMux: 1630 expandZExtPseudo(MI, SystemZ::LLCR, 8); 1631 return true; 1632 1633 case SystemZ::LLHRMux: 1634 expandZExtPseudo(MI, SystemZ::LLHR, 16); 1635 return true; 1636 1637 case SystemZ::LLCMux: 1638 expandRXYPseudo(MI, SystemZ::LLC, SystemZ::LLCH); 1639 return true; 1640 1641 case SystemZ::LLHMux: 1642 expandRXYPseudo(MI, SystemZ::LLH, SystemZ::LLHH); 1643 return true; 1644 1645 case SystemZ::LMux: 1646 expandRXYPseudo(MI, SystemZ::L, SystemZ::LFH); 1647 return true; 1648 1649 case SystemZ::LOCMux: 1650 expandLOCPseudo(MI, SystemZ::LOC, SystemZ::LOCFH); 1651 return true; 1652 1653 case SystemZ::LOCHIMux: 1654 expandLOCPseudo(MI, SystemZ::LOCHI, SystemZ::LOCHHI); 1655 return true; 1656 1657 case SystemZ::STCMux: 1658 expandRXYPseudo(MI, SystemZ::STC, SystemZ::STCH); 1659 return true; 1660 1661 case SystemZ::STHMux: 1662 expandRXYPseudo(MI, SystemZ::STH, SystemZ::STHH); 1663 return true; 1664 1665 case SystemZ::STMux: 1666 expandRXYPseudo(MI, SystemZ::ST, SystemZ::STFH); 1667 return true; 1668 1669 case SystemZ::STOCMux: 1670 expandLOCPseudo(MI, SystemZ::STOC, SystemZ::STOCFH); 1671 return true; 1672 1673 case SystemZ::LHIMux: 1674 expandRIPseudo(MI, SystemZ::LHI, SystemZ::IIHF, true); 1675 return true; 1676 1677 case SystemZ::IIFMux: 1678 expandRIPseudo(MI, SystemZ::IILF, SystemZ::IIHF, false); 1679 return true; 1680 1681 case SystemZ::IILMux: 1682 expandRIPseudo(MI, SystemZ::IILL, SystemZ::IIHL, false); 1683 return true; 1684 1685 case SystemZ::IIHMux: 1686 expandRIPseudo(MI, SystemZ::IILH, SystemZ::IIHH, false); 1687 return true; 1688 1689 case SystemZ::NIFMux: 1690 expandRIPseudo(MI, SystemZ::NILF, SystemZ::NIHF, false); 1691 return true; 1692 1693 case SystemZ::NILMux: 1694 expandRIPseudo(MI, SystemZ::NILL, SystemZ::NIHL, false); 1695 return true; 1696 1697 case SystemZ::NIHMux: 1698 expandRIPseudo(MI, SystemZ::NILH, SystemZ::NIHH, false); 1699 return true; 1700 1701 case SystemZ::OIFMux: 1702 expandRIPseudo(MI, SystemZ::OILF, SystemZ::OIHF, false); 1703 return true; 1704 1705 case SystemZ::OILMux: 1706 expandRIPseudo(MI, SystemZ::OILL, SystemZ::OIHL, false); 1707 return true; 1708 1709 case SystemZ::OIHMux: 1710 expandRIPseudo(MI, SystemZ::OILH, SystemZ::OIHH, false); 1711 return true; 1712 1713 case SystemZ::XIFMux: 1714 expandRIPseudo(MI, SystemZ::XILF, SystemZ::XIHF, false); 1715 return true; 1716 1717 case SystemZ::TMLMux: 1718 expandRIPseudo(MI, SystemZ::TMLL, SystemZ::TMHL, false); 1719 return true; 1720 1721 case SystemZ::TMHMux: 1722 expandRIPseudo(MI, SystemZ::TMLH, SystemZ::TMHH, false); 1723 return true; 1724 1725 case SystemZ::AHIMux: 1726 expandRIPseudo(MI, SystemZ::AHI, SystemZ::AIH, false); 1727 return true; 1728 1729 case SystemZ::AHIMuxK: 1730 expandRIEPseudo(MI, SystemZ::AHI, SystemZ::AHIK, SystemZ::AIH); 1731 return true; 1732 1733 case SystemZ::AFIMux: 1734 expandRIPseudo(MI, SystemZ::AFI, SystemZ::AIH, false); 1735 return true; 1736 1737 case SystemZ::CHIMux: 1738 expandRIPseudo(MI, SystemZ::CHI, SystemZ::CIH, false); 1739 return true; 1740 1741 case SystemZ::CFIMux: 1742 expandRIPseudo(MI, SystemZ::CFI, SystemZ::CIH, false); 1743 return true; 1744 1745 case SystemZ::CLFIMux: 1746 expandRIPseudo(MI, SystemZ::CLFI, SystemZ::CLIH, false); 1747 return true; 1748 1749 case SystemZ::CMux: 1750 expandRXYPseudo(MI, SystemZ::C, SystemZ::CHF); 1751 return true; 1752 1753 case SystemZ::CLMux: 1754 expandRXYPseudo(MI, SystemZ::CL, SystemZ::CLHF); 1755 return true; 1756 1757 case SystemZ::RISBMux: { 1758 bool DestIsHigh = SystemZ::isHighReg(MI.getOperand(0).getReg()); 1759 bool SrcIsHigh = SystemZ::isHighReg(MI.getOperand(2).getReg()); 1760 if (SrcIsHigh == DestIsHigh) 1761 MI.setDesc(get(DestIsHigh ? SystemZ::RISBHH : SystemZ::RISBLL)); 1762 else { 1763 MI.setDesc(get(DestIsHigh ? SystemZ::RISBHL : SystemZ::RISBLH)); 1764 MI.getOperand(5).setImm(MI.getOperand(5).getImm() ^ 32); 1765 } 1766 return true; 1767 } 1768 1769 case SystemZ::ADJDYNALLOC: 1770 splitAdjDynAlloc(MI); 1771 return true; 1772 1773 case TargetOpcode::LOAD_STACK_GUARD: 1774 expandLoadStackGuard(&MI); 1775 return true; 1776 1777 default: 1778 return false; 1779 } 1780 } 1781 getInstSizeInBytes(const MachineInstr & MI) const1782 unsigned SystemZInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 1783 if (MI.isInlineAsm()) { 1784 const MachineFunction *MF = MI.getParent()->getParent(); 1785 const char *AsmStr = MI.getOperand(0).getSymbolName(); 1786 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo()); 1787 } 1788 else if (MI.getOpcode() == SystemZ::PATCHPOINT) 1789 return PatchPointOpers(&MI).getNumPatchBytes(); 1790 else if (MI.getOpcode() == SystemZ::STACKMAP) 1791 return MI.getOperand(1).getImm(); 1792 else if (MI.getOpcode() == SystemZ::FENTRY_CALL) 1793 return 6; 1794 1795 return MI.getDesc().getSize(); 1796 } 1797 1798 SystemZII::Branch getBranchInfo(const MachineInstr & MI) const1799 SystemZInstrInfo::getBranchInfo(const MachineInstr &MI) const { 1800 switch (MI.getOpcode()) { 1801 case SystemZ::BR: 1802 case SystemZ::BI: 1803 case SystemZ::J: 1804 case SystemZ::JG: 1805 return SystemZII::Branch(SystemZII::BranchNormal, SystemZ::CCMASK_ANY, 1806 SystemZ::CCMASK_ANY, &MI.getOperand(0)); 1807 1808 case SystemZ::BRC: 1809 case SystemZ::BRCL: 1810 return SystemZII::Branch(SystemZII::BranchNormal, MI.getOperand(0).getImm(), 1811 MI.getOperand(1).getImm(), &MI.getOperand(2)); 1812 1813 case SystemZ::BRCT: 1814 case SystemZ::BRCTH: 1815 return SystemZII::Branch(SystemZII::BranchCT, SystemZ::CCMASK_ICMP, 1816 SystemZ::CCMASK_CMP_NE, &MI.getOperand(2)); 1817 1818 case SystemZ::BRCTG: 1819 return SystemZII::Branch(SystemZII::BranchCTG, SystemZ::CCMASK_ICMP, 1820 SystemZ::CCMASK_CMP_NE, &MI.getOperand(2)); 1821 1822 case SystemZ::CIJ: 1823 case SystemZ::CRJ: 1824 return SystemZII::Branch(SystemZII::BranchC, SystemZ::CCMASK_ICMP, 1825 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1826 1827 case SystemZ::CLIJ: 1828 case SystemZ::CLRJ: 1829 return SystemZII::Branch(SystemZII::BranchCL, SystemZ::CCMASK_ICMP, 1830 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1831 1832 case SystemZ::CGIJ: 1833 case SystemZ::CGRJ: 1834 return SystemZII::Branch(SystemZII::BranchCG, SystemZ::CCMASK_ICMP, 1835 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1836 1837 case SystemZ::CLGIJ: 1838 case SystemZ::CLGRJ: 1839 return SystemZII::Branch(SystemZII::BranchCLG, SystemZ::CCMASK_ICMP, 1840 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1841 1842 case SystemZ::INLINEASM_BR: 1843 // Don't try to analyze asm goto, so pass nullptr as branch target argument. 1844 return SystemZII::Branch(SystemZII::AsmGoto, 0, 0, nullptr); 1845 1846 default: 1847 llvm_unreachable("Unrecognized branch opcode"); 1848 } 1849 } 1850 getLoadStoreOpcodes(const TargetRegisterClass * RC,unsigned & LoadOpcode,unsigned & StoreOpcode) const1851 void SystemZInstrInfo::getLoadStoreOpcodes(const TargetRegisterClass *RC, 1852 unsigned &LoadOpcode, 1853 unsigned &StoreOpcode) const { 1854 if (RC == &SystemZ::GR32BitRegClass || RC == &SystemZ::ADDR32BitRegClass) { 1855 LoadOpcode = SystemZ::L; 1856 StoreOpcode = SystemZ::ST; 1857 } else if (RC == &SystemZ::GRH32BitRegClass) { 1858 LoadOpcode = SystemZ::LFH; 1859 StoreOpcode = SystemZ::STFH; 1860 } else if (RC == &SystemZ::GRX32BitRegClass) { 1861 LoadOpcode = SystemZ::LMux; 1862 StoreOpcode = SystemZ::STMux; 1863 } else if (RC == &SystemZ::GR64BitRegClass || 1864 RC == &SystemZ::ADDR64BitRegClass) { 1865 LoadOpcode = SystemZ::LG; 1866 StoreOpcode = SystemZ::STG; 1867 } else if (RC == &SystemZ::GR128BitRegClass || 1868 RC == &SystemZ::ADDR128BitRegClass) { 1869 LoadOpcode = SystemZ::L128; 1870 StoreOpcode = SystemZ::ST128; 1871 } else if (RC == &SystemZ::FP32BitRegClass) { 1872 LoadOpcode = SystemZ::LE; 1873 StoreOpcode = SystemZ::STE; 1874 } else if (RC == &SystemZ::FP64BitRegClass) { 1875 LoadOpcode = SystemZ::LD; 1876 StoreOpcode = SystemZ::STD; 1877 } else if (RC == &SystemZ::FP128BitRegClass) { 1878 LoadOpcode = SystemZ::LX; 1879 StoreOpcode = SystemZ::STX; 1880 } else if (RC == &SystemZ::VR32BitRegClass) { 1881 LoadOpcode = SystemZ::VL32; 1882 StoreOpcode = SystemZ::VST32; 1883 } else if (RC == &SystemZ::VR64BitRegClass) { 1884 LoadOpcode = SystemZ::VL64; 1885 StoreOpcode = SystemZ::VST64; 1886 } else if (RC == &SystemZ::VF128BitRegClass || 1887 RC == &SystemZ::VR128BitRegClass) { 1888 LoadOpcode = SystemZ::VL; 1889 StoreOpcode = SystemZ::VST; 1890 } else 1891 llvm_unreachable("Unsupported regclass to load or store"); 1892 } 1893 getOpcodeForOffset(unsigned Opcode,int64_t Offset,const MachineInstr * MI) const1894 unsigned SystemZInstrInfo::getOpcodeForOffset(unsigned Opcode, 1895 int64_t Offset, 1896 const MachineInstr *MI) const { 1897 const MCInstrDesc &MCID = get(Opcode); 1898 int64_t Offset2 = (MCID.TSFlags & SystemZII::Is128Bit ? Offset + 8 : Offset); 1899 if (isUInt<12>(Offset) && isUInt<12>(Offset2)) { 1900 // Get the instruction to use for unsigned 12-bit displacements. 1901 int Disp12Opcode = SystemZ::getDisp12Opcode(Opcode); 1902 if (Disp12Opcode >= 0) 1903 return Disp12Opcode; 1904 1905 // All address-related instructions can use unsigned 12-bit 1906 // displacements. 1907 return Opcode; 1908 } 1909 if (isInt<20>(Offset) && isInt<20>(Offset2)) { 1910 // Get the instruction to use for signed 20-bit displacements. 1911 int Disp20Opcode = SystemZ::getDisp20Opcode(Opcode); 1912 if (Disp20Opcode >= 0) 1913 return Disp20Opcode; 1914 1915 // Check whether Opcode allows signed 20-bit displacements. 1916 if (MCID.TSFlags & SystemZII::Has20BitOffset) 1917 return Opcode; 1918 1919 // If a VR32/VR64 reg ended up in an FP register, use the FP opcode. 1920 if (MI && MI->getOperand(0).isReg()) { 1921 Register Reg = MI->getOperand(0).getReg(); 1922 if (Reg.isPhysical() && SystemZMC::getFirstReg(Reg) < 16) { 1923 switch (Opcode) { 1924 case SystemZ::VL32: 1925 return SystemZ::LEY; 1926 case SystemZ::VST32: 1927 return SystemZ::STEY; 1928 case SystemZ::VL64: 1929 return SystemZ::LDY; 1930 case SystemZ::VST64: 1931 return SystemZ::STDY; 1932 default: break; 1933 } 1934 } 1935 } 1936 } 1937 return 0; 1938 } 1939 hasDisplacementPairInsn(unsigned Opcode) const1940 bool SystemZInstrInfo::hasDisplacementPairInsn(unsigned Opcode) const { 1941 const MCInstrDesc &MCID = get(Opcode); 1942 if (MCID.TSFlags & SystemZII::Has20BitOffset) 1943 return SystemZ::getDisp12Opcode(Opcode) >= 0; 1944 return SystemZ::getDisp20Opcode(Opcode) >= 0; 1945 } 1946 getLoadAndTest(unsigned Opcode) const1947 unsigned SystemZInstrInfo::getLoadAndTest(unsigned Opcode) const { 1948 switch (Opcode) { 1949 case SystemZ::L: return SystemZ::LT; 1950 case SystemZ::LY: return SystemZ::LT; 1951 case SystemZ::LG: return SystemZ::LTG; 1952 case SystemZ::LGF: return SystemZ::LTGF; 1953 case SystemZ::LR: return SystemZ::LTR; 1954 case SystemZ::LGFR: return SystemZ::LTGFR; 1955 case SystemZ::LGR: return SystemZ::LTGR; 1956 case SystemZ::LCDFR: return SystemZ::LCDBR; 1957 case SystemZ::LPDFR: return SystemZ::LPDBR; 1958 case SystemZ::LNDFR: return SystemZ::LNDBR; 1959 case SystemZ::LCDFR_32: return SystemZ::LCEBR; 1960 case SystemZ::LPDFR_32: return SystemZ::LPEBR; 1961 case SystemZ::LNDFR_32: return SystemZ::LNEBR; 1962 // On zEC12 we prefer to use RISBGN. But if there is a chance to 1963 // actually use the condition code, we may turn it back into RISGB. 1964 // Note that RISBG is not really a "load-and-test" instruction, 1965 // but sets the same condition code values, so is OK to use here. 1966 case SystemZ::RISBGN: return SystemZ::RISBG; 1967 default: return 0; 1968 } 1969 } 1970 isRxSBGMask(uint64_t Mask,unsigned BitSize,unsigned & Start,unsigned & End) const1971 bool SystemZInstrInfo::isRxSBGMask(uint64_t Mask, unsigned BitSize, 1972 unsigned &Start, unsigned &End) const { 1973 // Reject trivial all-zero masks. 1974 Mask &= allOnes(BitSize); 1975 if (Mask == 0) 1976 return false; 1977 1978 // Handle the 1+0+ or 0+1+0* cases. Start then specifies the index of 1979 // the msb and End specifies the index of the lsb. 1980 unsigned LSB, Length; 1981 if (isShiftedMask_64(Mask, LSB, Length)) { 1982 Start = 63 - (LSB + Length - 1); 1983 End = 63 - LSB; 1984 return true; 1985 } 1986 1987 // Handle the wrap-around 1+0+1+ cases. Start then specifies the msb 1988 // of the low 1s and End specifies the lsb of the high 1s. 1989 if (isShiftedMask_64(Mask ^ allOnes(BitSize), LSB, Length)) { 1990 assert(LSB > 0 && "Bottom bit must be set"); 1991 assert(LSB + Length < BitSize && "Top bit must be set"); 1992 Start = 63 - (LSB - 1); 1993 End = 63 - (LSB + Length); 1994 return true; 1995 } 1996 1997 return false; 1998 } 1999 getFusedCompare(unsigned Opcode,SystemZII::FusedCompareType Type,const MachineInstr * MI) const2000 unsigned SystemZInstrInfo::getFusedCompare(unsigned Opcode, 2001 SystemZII::FusedCompareType Type, 2002 const MachineInstr *MI) const { 2003 switch (Opcode) { 2004 case SystemZ::CHI: 2005 case SystemZ::CGHI: 2006 if (!(MI && isInt<8>(MI->getOperand(1).getImm()))) 2007 return 0; 2008 break; 2009 case SystemZ::CLFI: 2010 case SystemZ::CLGFI: 2011 if (!(MI && isUInt<8>(MI->getOperand(1).getImm()))) 2012 return 0; 2013 break; 2014 case SystemZ::CL: 2015 case SystemZ::CLG: 2016 if (!STI.hasMiscellaneousExtensions()) 2017 return 0; 2018 if (!(MI && MI->getOperand(3).getReg() == 0)) 2019 return 0; 2020 break; 2021 } 2022 switch (Type) { 2023 case SystemZII::CompareAndBranch: 2024 switch (Opcode) { 2025 case SystemZ::CR: 2026 return SystemZ::CRJ; 2027 case SystemZ::CGR: 2028 return SystemZ::CGRJ; 2029 case SystemZ::CHI: 2030 return SystemZ::CIJ; 2031 case SystemZ::CGHI: 2032 return SystemZ::CGIJ; 2033 case SystemZ::CLR: 2034 return SystemZ::CLRJ; 2035 case SystemZ::CLGR: 2036 return SystemZ::CLGRJ; 2037 case SystemZ::CLFI: 2038 return SystemZ::CLIJ; 2039 case SystemZ::CLGFI: 2040 return SystemZ::CLGIJ; 2041 default: 2042 return 0; 2043 } 2044 case SystemZII::CompareAndReturn: 2045 switch (Opcode) { 2046 case SystemZ::CR: 2047 return SystemZ::CRBReturn; 2048 case SystemZ::CGR: 2049 return SystemZ::CGRBReturn; 2050 case SystemZ::CHI: 2051 return SystemZ::CIBReturn; 2052 case SystemZ::CGHI: 2053 return SystemZ::CGIBReturn; 2054 case SystemZ::CLR: 2055 return SystemZ::CLRBReturn; 2056 case SystemZ::CLGR: 2057 return SystemZ::CLGRBReturn; 2058 case SystemZ::CLFI: 2059 return SystemZ::CLIBReturn; 2060 case SystemZ::CLGFI: 2061 return SystemZ::CLGIBReturn; 2062 default: 2063 return 0; 2064 } 2065 case SystemZII::CompareAndSibcall: 2066 switch (Opcode) { 2067 case SystemZ::CR: 2068 return SystemZ::CRBCall; 2069 case SystemZ::CGR: 2070 return SystemZ::CGRBCall; 2071 case SystemZ::CHI: 2072 return SystemZ::CIBCall; 2073 case SystemZ::CGHI: 2074 return SystemZ::CGIBCall; 2075 case SystemZ::CLR: 2076 return SystemZ::CLRBCall; 2077 case SystemZ::CLGR: 2078 return SystemZ::CLGRBCall; 2079 case SystemZ::CLFI: 2080 return SystemZ::CLIBCall; 2081 case SystemZ::CLGFI: 2082 return SystemZ::CLGIBCall; 2083 default: 2084 return 0; 2085 } 2086 case SystemZII::CompareAndTrap: 2087 switch (Opcode) { 2088 case SystemZ::CR: 2089 return SystemZ::CRT; 2090 case SystemZ::CGR: 2091 return SystemZ::CGRT; 2092 case SystemZ::CHI: 2093 return SystemZ::CIT; 2094 case SystemZ::CGHI: 2095 return SystemZ::CGIT; 2096 case SystemZ::CLR: 2097 return SystemZ::CLRT; 2098 case SystemZ::CLGR: 2099 return SystemZ::CLGRT; 2100 case SystemZ::CLFI: 2101 return SystemZ::CLFIT; 2102 case SystemZ::CLGFI: 2103 return SystemZ::CLGIT; 2104 case SystemZ::CL: 2105 return SystemZ::CLT; 2106 case SystemZ::CLG: 2107 return SystemZ::CLGT; 2108 default: 2109 return 0; 2110 } 2111 } 2112 return 0; 2113 } 2114 2115 bool SystemZInstrInfo:: prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const2116 prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const { 2117 assert(MBBI->isCompare() && MBBI->getOperand(0).isReg() && 2118 MBBI->getOperand(1).isReg() && !MBBI->mayLoad() && 2119 "Not a compare reg/reg."); 2120 2121 MachineBasicBlock *MBB = MBBI->getParent(); 2122 bool CCLive = true; 2123 SmallVector<MachineInstr *, 4> CCUsers; 2124 for (MachineInstr &MI : llvm::make_range(std::next(MBBI), MBB->end())) { 2125 if (MI.readsRegister(SystemZ::CC, /*TRI=*/nullptr)) { 2126 unsigned Flags = MI.getDesc().TSFlags; 2127 if ((Flags & SystemZII::CCMaskFirst) || (Flags & SystemZII::CCMaskLast)) 2128 CCUsers.push_back(&MI); 2129 else 2130 return false; 2131 } 2132 if (MI.definesRegister(SystemZ::CC, /*TRI=*/nullptr)) { 2133 CCLive = false; 2134 break; 2135 } 2136 } 2137 if (CCLive) { 2138 LiveRegUnits LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo()); 2139 LiveRegs.addLiveOuts(*MBB); 2140 if (!LiveRegs.available(SystemZ::CC)) 2141 return false; 2142 } 2143 2144 // Update all CC users. 2145 for (unsigned Idx = 0; Idx < CCUsers.size(); ++Idx) { 2146 unsigned Flags = CCUsers[Idx]->getDesc().TSFlags; 2147 unsigned FirstOpNum = ((Flags & SystemZII::CCMaskFirst) ? 2148 0 : CCUsers[Idx]->getNumExplicitOperands() - 2); 2149 MachineOperand &CCMaskMO = CCUsers[Idx]->getOperand(FirstOpNum + 1); 2150 unsigned NewCCMask = SystemZ::reverseCCMask(CCMaskMO.getImm()); 2151 CCMaskMO.setImm(NewCCMask); 2152 } 2153 2154 return true; 2155 } 2156 reverseCCMask(unsigned CCMask)2157 unsigned SystemZ::reverseCCMask(unsigned CCMask) { 2158 return ((CCMask & SystemZ::CCMASK_CMP_EQ) | 2159 ((CCMask & SystemZ::CCMASK_CMP_GT) ? SystemZ::CCMASK_CMP_LT : 0) | 2160 ((CCMask & SystemZ::CCMASK_CMP_LT) ? SystemZ::CCMASK_CMP_GT : 0) | 2161 (CCMask & SystemZ::CCMASK_CMP_UO)); 2162 } 2163 emitBlockAfter(MachineBasicBlock * MBB)2164 MachineBasicBlock *SystemZ::emitBlockAfter(MachineBasicBlock *MBB) { 2165 MachineFunction &MF = *MBB->getParent(); 2166 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 2167 MF.insert(std::next(MachineFunction::iterator(MBB)), NewMBB); 2168 return NewMBB; 2169 } 2170 splitBlockAfter(MachineBasicBlock::iterator MI,MachineBasicBlock * MBB)2171 MachineBasicBlock *SystemZ::splitBlockAfter(MachineBasicBlock::iterator MI, 2172 MachineBasicBlock *MBB) { 2173 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 2174 NewMBB->splice(NewMBB->begin(), MBB, 2175 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 2176 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 2177 return NewMBB; 2178 } 2179 splitBlockBefore(MachineBasicBlock::iterator MI,MachineBasicBlock * MBB)2180 MachineBasicBlock *SystemZ::splitBlockBefore(MachineBasicBlock::iterator MI, 2181 MachineBasicBlock *MBB) { 2182 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 2183 NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end()); 2184 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 2185 return NewMBB; 2186 } 2187 getLoadAndTrap(unsigned Opcode) const2188 unsigned SystemZInstrInfo::getLoadAndTrap(unsigned Opcode) const { 2189 if (!STI.hasLoadAndTrap()) 2190 return 0; 2191 switch (Opcode) { 2192 case SystemZ::L: 2193 case SystemZ::LY: 2194 return SystemZ::LAT; 2195 case SystemZ::LG: 2196 return SystemZ::LGAT; 2197 case SystemZ::LFH: 2198 return SystemZ::LFHAT; 2199 case SystemZ::LLGF: 2200 return SystemZ::LLGFAT; 2201 case SystemZ::LLGT: 2202 return SystemZ::LLGTAT; 2203 } 2204 return 0; 2205 } 2206 loadImmediate(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,unsigned Reg,uint64_t Value) const2207 void SystemZInstrInfo::loadImmediate(MachineBasicBlock &MBB, 2208 MachineBasicBlock::iterator MBBI, 2209 unsigned Reg, uint64_t Value) const { 2210 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 2211 unsigned Opcode = 0; 2212 if (isInt<16>(Value)) 2213 Opcode = SystemZ::LGHI; 2214 else if (SystemZ::isImmLL(Value)) 2215 Opcode = SystemZ::LLILL; 2216 else if (SystemZ::isImmLH(Value)) { 2217 Opcode = SystemZ::LLILH; 2218 Value >>= 16; 2219 } 2220 else if (isInt<32>(Value)) 2221 Opcode = SystemZ::LGFI; 2222 if (Opcode) { 2223 BuildMI(MBB, MBBI, DL, get(Opcode), Reg).addImm(Value); 2224 return; 2225 } 2226 2227 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2228 assert (MRI.isSSA() && "Huge values only handled before reg-alloc ."); 2229 Register Reg0 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 2230 Register Reg1 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 2231 BuildMI(MBB, MBBI, DL, get(SystemZ::IMPLICIT_DEF), Reg0); 2232 BuildMI(MBB, MBBI, DL, get(SystemZ::IIHF64), Reg1) 2233 .addReg(Reg0).addImm(Value >> 32); 2234 BuildMI(MBB, MBBI, DL, get(SystemZ::IILF64), Reg) 2235 .addReg(Reg1).addImm(Value & ((uint64_t(1) << 32) - 1)); 2236 } 2237 verifyInstruction(const MachineInstr & MI,StringRef & ErrInfo) const2238 bool SystemZInstrInfo::verifyInstruction(const MachineInstr &MI, 2239 StringRef &ErrInfo) const { 2240 const MCInstrDesc &MCID = MI.getDesc(); 2241 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { 2242 if (I >= MCID.getNumOperands()) 2243 break; 2244 const MachineOperand &Op = MI.getOperand(I); 2245 const MCOperandInfo &MCOI = MCID.operands()[I]; 2246 // Addressing modes have register and immediate operands. Op should be a 2247 // register (or frame index) operand if MCOI.RegClass contains a valid 2248 // register class, or an immediate otherwise. 2249 if (MCOI.OperandType == MCOI::OPERAND_MEMORY && 2250 ((MCOI.RegClass != -1 && !Op.isReg() && !Op.isFI()) || 2251 (MCOI.RegClass == -1 && !Op.isImm()))) { 2252 ErrInfo = "Addressing mode operands corrupt!"; 2253 return false; 2254 } 2255 } 2256 2257 return true; 2258 } 2259 2260 bool SystemZInstrInfo:: areMemAccessesTriviallyDisjoint(const MachineInstr & MIa,const MachineInstr & MIb) const2261 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, 2262 const MachineInstr &MIb) const { 2263 2264 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) 2265 return false; 2266 2267 // If mem-operands show that the same address Value is used by both 2268 // instructions, check for non-overlapping offsets and widths. Not 2269 // sure if a register based analysis would be an improvement... 2270 2271 MachineMemOperand *MMOa = *MIa.memoperands_begin(); 2272 MachineMemOperand *MMOb = *MIb.memoperands_begin(); 2273 const Value *VALa = MMOa->getValue(); 2274 const Value *VALb = MMOb->getValue(); 2275 bool SameVal = (VALa && VALb && (VALa == VALb)); 2276 if (!SameVal) { 2277 const PseudoSourceValue *PSVa = MMOa->getPseudoValue(); 2278 const PseudoSourceValue *PSVb = MMOb->getPseudoValue(); 2279 if (PSVa && PSVb && (PSVa == PSVb)) 2280 SameVal = true; 2281 } 2282 if (SameVal) { 2283 int OffsetA = MMOa->getOffset(), OffsetB = MMOb->getOffset(); 2284 LocationSize WidthA = MMOa->getSize(), WidthB = MMOb->getSize(); 2285 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; 2286 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; 2287 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 2288 if (LowWidth.hasValue() && 2289 LowOffset + (int)LowWidth.getValue() <= HighOffset) 2290 return true; 2291 } 2292 2293 return false; 2294 } 2295 getConstValDefinedInReg(const MachineInstr & MI,const Register Reg,int64_t & ImmVal) const2296 bool SystemZInstrInfo::getConstValDefinedInReg(const MachineInstr &MI, 2297 const Register Reg, 2298 int64_t &ImmVal) const { 2299 2300 if (MI.getOpcode() == SystemZ::VGBM && Reg == MI.getOperand(0).getReg()) { 2301 ImmVal = MI.getOperand(1).getImm(); 2302 // TODO: Handle non-0 values 2303 return ImmVal == 0; 2304 } 2305 2306 return false; 2307 } 2308