1 //===-- SystemZInstrInfo.cpp - SystemZ instruction information ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the SystemZ implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "SystemZInstrInfo.h" 14 #include "MCTargetDesc/SystemZMCTargetDesc.h" 15 #include "SystemZ.h" 16 #include "SystemZInstrBuilder.h" 17 #include "SystemZSubtarget.h" 18 #include "llvm/ADT/Statistic.h" 19 #include "llvm/CodeGen/LiveInterval.h" 20 #include "llvm/CodeGen/LiveIntervals.h" 21 #include "llvm/CodeGen/LiveVariables.h" 22 #include "llvm/CodeGen/MachineBasicBlock.h" 23 #include "llvm/CodeGen/MachineFrameInfo.h" 24 #include "llvm/CodeGen/MachineFunction.h" 25 #include "llvm/CodeGen/MachineInstr.h" 26 #include "llvm/CodeGen/MachineMemOperand.h" 27 #include "llvm/CodeGen/MachineOperand.h" 28 #include "llvm/CodeGen/MachineRegisterInfo.h" 29 #include "llvm/CodeGen/SlotIndexes.h" 30 #include "llvm/CodeGen/StackMaps.h" 31 #include "llvm/CodeGen/TargetInstrInfo.h" 32 #include "llvm/CodeGen/TargetSubtargetInfo.h" 33 #include "llvm/MC/MCInstrDesc.h" 34 #include "llvm/MC/MCRegisterInfo.h" 35 #include "llvm/Support/BranchProbability.h" 36 #include "llvm/Support/ErrorHandling.h" 37 #include "llvm/Support/MathExtras.h" 38 #include "llvm/Target/TargetMachine.h" 39 #include <cassert> 40 #include <cstdint> 41 #include <iterator> 42 43 using namespace llvm; 44 45 #define GET_INSTRINFO_CTOR_DTOR 46 #define GET_INSTRMAP_INFO 47 #include "SystemZGenInstrInfo.inc" 48 49 #define DEBUG_TYPE "systemz-II" 50 51 // Return a mask with Count low bits set. 52 static uint64_t allOnes(unsigned int Count) { 53 return Count == 0 ? 0 : (uint64_t(1) << (Count - 1) << 1) - 1; 54 } 55 56 // Pin the vtable to this file. 57 void SystemZInstrInfo::anchor() {} 58 59 SystemZInstrInfo::SystemZInstrInfo(SystemZSubtarget &sti) 60 : SystemZGenInstrInfo(SystemZ::ADJCALLSTACKDOWN, SystemZ::ADJCALLSTACKUP), 61 RI(sti.getSpecialRegisters()->getReturnFunctionAddressRegister()), 62 STI(sti) {} 63 64 // MI is a 128-bit load or store. Split it into two 64-bit loads or stores, 65 // each having the opcode given by NewOpcode. 66 void SystemZInstrInfo::splitMove(MachineBasicBlock::iterator MI, 67 unsigned NewOpcode) const { 68 MachineBasicBlock *MBB = MI->getParent(); 69 MachineFunction &MF = *MBB->getParent(); 70 71 // Get two load or store instructions. Use the original instruction for one 72 // of them (arbitrarily the second here) and create a clone for the other. 73 MachineInstr *EarlierMI = MF.CloneMachineInstr(&*MI); 74 MBB->insert(MI, EarlierMI); 75 76 // Set up the two 64-bit registers and remember super reg and its flags. 77 MachineOperand &HighRegOp = EarlierMI->getOperand(0); 78 MachineOperand &LowRegOp = MI->getOperand(0); 79 Register Reg128 = LowRegOp.getReg(); 80 unsigned Reg128Killed = getKillRegState(LowRegOp.isKill()); 81 unsigned Reg128Undef = getUndefRegState(LowRegOp.isUndef()); 82 HighRegOp.setReg(RI.getSubReg(HighRegOp.getReg(), SystemZ::subreg_h64)); 83 LowRegOp.setReg(RI.getSubReg(LowRegOp.getReg(), SystemZ::subreg_l64)); 84 85 if (MI->mayStore()) { 86 // Add implicit uses of the super register in case one of the subregs is 87 // undefined. We could track liveness and skip storing an undefined 88 // subreg, but this is hopefully rare (discovered with llvm-stress). 89 // If Reg128 was killed, set kill flag on MI. 90 unsigned Reg128UndefImpl = (Reg128Undef | RegState::Implicit); 91 MachineInstrBuilder(MF, EarlierMI).addReg(Reg128, Reg128UndefImpl); 92 MachineInstrBuilder(MF, MI).addReg(Reg128, (Reg128UndefImpl | Reg128Killed)); 93 } 94 95 // The address in the first (high) instruction is already correct. 96 // Adjust the offset in the second (low) instruction. 97 MachineOperand &HighOffsetOp = EarlierMI->getOperand(2); 98 MachineOperand &LowOffsetOp = MI->getOperand(2); 99 LowOffsetOp.setImm(LowOffsetOp.getImm() + 8); 100 101 // Clear the kill flags on the registers in the first instruction. 102 if (EarlierMI->getOperand(0).isReg() && EarlierMI->getOperand(0).isUse()) 103 EarlierMI->getOperand(0).setIsKill(false); 104 EarlierMI->getOperand(1).setIsKill(false); 105 EarlierMI->getOperand(3).setIsKill(false); 106 107 // Set the opcodes. 108 unsigned HighOpcode = getOpcodeForOffset(NewOpcode, HighOffsetOp.getImm()); 109 unsigned LowOpcode = getOpcodeForOffset(NewOpcode, LowOffsetOp.getImm()); 110 assert(HighOpcode && LowOpcode && "Both offsets should be in range"); 111 112 EarlierMI->setDesc(get(HighOpcode)); 113 MI->setDesc(get(LowOpcode)); 114 } 115 116 // Split ADJDYNALLOC instruction MI. 117 void SystemZInstrInfo::splitAdjDynAlloc(MachineBasicBlock::iterator MI) const { 118 MachineBasicBlock *MBB = MI->getParent(); 119 MachineFunction &MF = *MBB->getParent(); 120 MachineFrameInfo &MFFrame = MF.getFrameInfo(); 121 MachineOperand &OffsetMO = MI->getOperand(2); 122 123 uint64_t Offset = (MFFrame.getMaxCallFrameSize() + 124 SystemZMC::ELFCallFrameSize + 125 OffsetMO.getImm()); 126 unsigned NewOpcode = getOpcodeForOffset(SystemZ::LA, Offset); 127 assert(NewOpcode && "No support for huge argument lists yet"); 128 MI->setDesc(get(NewOpcode)); 129 OffsetMO.setImm(Offset); 130 } 131 132 // MI is an RI-style pseudo instruction. Replace it with LowOpcode 133 // if the first operand is a low GR32 and HighOpcode if the first operand 134 // is a high GR32. ConvertHigh is true if LowOpcode takes a signed operand 135 // and HighOpcode takes an unsigned 32-bit operand. In those cases, 136 // MI has the same kind of operand as LowOpcode, so needs to be converted 137 // if HighOpcode is used. 138 void SystemZInstrInfo::expandRIPseudo(MachineInstr &MI, unsigned LowOpcode, 139 unsigned HighOpcode, 140 bool ConvertHigh) const { 141 Register Reg = MI.getOperand(0).getReg(); 142 bool IsHigh = SystemZ::isHighReg(Reg); 143 MI.setDesc(get(IsHigh ? HighOpcode : LowOpcode)); 144 if (IsHigh && ConvertHigh) 145 MI.getOperand(1).setImm(uint32_t(MI.getOperand(1).getImm())); 146 } 147 148 // MI is a three-operand RIE-style pseudo instruction. Replace it with 149 // LowOpcodeK if the registers are both low GR32s, otherwise use a move 150 // followed by HighOpcode or LowOpcode, depending on whether the target 151 // is a high or low GR32. 152 void SystemZInstrInfo::expandRIEPseudo(MachineInstr &MI, unsigned LowOpcode, 153 unsigned LowOpcodeK, 154 unsigned HighOpcode) const { 155 Register DestReg = MI.getOperand(0).getReg(); 156 Register SrcReg = MI.getOperand(1).getReg(); 157 bool DestIsHigh = SystemZ::isHighReg(DestReg); 158 bool SrcIsHigh = SystemZ::isHighReg(SrcReg); 159 if (!DestIsHigh && !SrcIsHigh) 160 MI.setDesc(get(LowOpcodeK)); 161 else { 162 if (DestReg != SrcReg) { 163 emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, SrcReg, 164 SystemZ::LR, 32, MI.getOperand(1).isKill(), 165 MI.getOperand(1).isUndef()); 166 MI.getOperand(1).setReg(DestReg); 167 } 168 MI.setDesc(get(DestIsHigh ? HighOpcode : LowOpcode)); 169 MI.tieOperands(0, 1); 170 } 171 } 172 173 // MI is an RXY-style pseudo instruction. Replace it with LowOpcode 174 // if the first operand is a low GR32 and HighOpcode if the first operand 175 // is a high GR32. 176 void SystemZInstrInfo::expandRXYPseudo(MachineInstr &MI, unsigned LowOpcode, 177 unsigned HighOpcode) const { 178 Register Reg = MI.getOperand(0).getReg(); 179 unsigned Opcode = getOpcodeForOffset( 180 SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode, 181 MI.getOperand(2).getImm()); 182 MI.setDesc(get(Opcode)); 183 } 184 185 // MI is a load-on-condition pseudo instruction with a single register 186 // (source or destination) operand. Replace it with LowOpcode if the 187 // register is a low GR32 and HighOpcode if the register is a high GR32. 188 void SystemZInstrInfo::expandLOCPseudo(MachineInstr &MI, unsigned LowOpcode, 189 unsigned HighOpcode) const { 190 Register Reg = MI.getOperand(0).getReg(); 191 unsigned Opcode = SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode; 192 MI.setDesc(get(Opcode)); 193 } 194 195 // MI is an RR-style pseudo instruction that zero-extends the low Size bits 196 // of one GRX32 into another. Replace it with LowOpcode if both operands 197 // are low registers, otherwise use RISB[LH]G. 198 void SystemZInstrInfo::expandZExtPseudo(MachineInstr &MI, unsigned LowOpcode, 199 unsigned Size) const { 200 MachineInstrBuilder MIB = 201 emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), 202 MI.getOperand(0).getReg(), MI.getOperand(1).getReg(), LowOpcode, 203 Size, MI.getOperand(1).isKill(), MI.getOperand(1).isUndef()); 204 205 // Keep the remaining operands as-is. 206 for (const MachineOperand &MO : llvm::drop_begin(MI.operands(), 2)) 207 MIB.add(MO); 208 209 MI.eraseFromParent(); 210 } 211 212 void SystemZInstrInfo::expandLoadStackGuard(MachineInstr *MI) const { 213 MachineBasicBlock *MBB = MI->getParent(); 214 MachineFunction &MF = *MBB->getParent(); 215 const Register Reg64 = MI->getOperand(0).getReg(); 216 const Register Reg32 = RI.getSubReg(Reg64, SystemZ::subreg_l32); 217 218 // EAR can only load the low subregister so us a shift for %a0 to produce 219 // the GR containing %a0 and %a1. 220 221 // ear <reg>, %a0 222 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32) 223 .addReg(SystemZ::A0) 224 .addReg(Reg64, RegState::ImplicitDefine); 225 226 // sllg <reg>, <reg>, 32 227 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::SLLG), Reg64) 228 .addReg(Reg64) 229 .addReg(0) 230 .addImm(32); 231 232 // ear <reg>, %a1 233 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32) 234 .addReg(SystemZ::A1); 235 236 // lg <reg>, 40(<reg>) 237 MI->setDesc(get(SystemZ::LG)); 238 MachineInstrBuilder(MF, MI).addReg(Reg64).addImm(40).addReg(0); 239 } 240 241 // Emit a zero-extending move from 32-bit GPR SrcReg to 32-bit GPR 242 // DestReg before MBBI in MBB. Use LowLowOpcode when both DestReg and SrcReg 243 // are low registers, otherwise use RISB[LH]G. Size is the number of bits 244 // taken from the low end of SrcReg (8 for LLCR, 16 for LLHR and 32 for LR). 245 // KillSrc is true if this move is the last use of SrcReg. 246 MachineInstrBuilder 247 SystemZInstrInfo::emitGRX32Move(MachineBasicBlock &MBB, 248 MachineBasicBlock::iterator MBBI, 249 const DebugLoc &DL, unsigned DestReg, 250 unsigned SrcReg, unsigned LowLowOpcode, 251 unsigned Size, bool KillSrc, 252 bool UndefSrc) const { 253 unsigned Opcode; 254 bool DestIsHigh = SystemZ::isHighReg(DestReg); 255 bool SrcIsHigh = SystemZ::isHighReg(SrcReg); 256 if (DestIsHigh && SrcIsHigh) 257 Opcode = SystemZ::RISBHH; 258 else if (DestIsHigh && !SrcIsHigh) 259 Opcode = SystemZ::RISBHL; 260 else if (!DestIsHigh && SrcIsHigh) 261 Opcode = SystemZ::RISBLH; 262 else { 263 return BuildMI(MBB, MBBI, DL, get(LowLowOpcode), DestReg) 264 .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)); 265 } 266 unsigned Rotate = (DestIsHigh != SrcIsHigh ? 32 : 0); 267 return BuildMI(MBB, MBBI, DL, get(Opcode), DestReg) 268 .addReg(DestReg, RegState::Undef) 269 .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)) 270 .addImm(32 - Size).addImm(128 + 31).addImm(Rotate); 271 } 272 273 MachineInstr *SystemZInstrInfo::commuteInstructionImpl(MachineInstr &MI, 274 bool NewMI, 275 unsigned OpIdx1, 276 unsigned OpIdx2) const { 277 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { 278 if (NewMI) 279 return *MI.getParent()->getParent()->CloneMachineInstr(&MI); 280 return MI; 281 }; 282 283 switch (MI.getOpcode()) { 284 case SystemZ::SELRMux: 285 case SystemZ::SELFHR: 286 case SystemZ::SELR: 287 case SystemZ::SELGR: 288 case SystemZ::LOCRMux: 289 case SystemZ::LOCFHR: 290 case SystemZ::LOCR: 291 case SystemZ::LOCGR: { 292 auto &WorkingMI = cloneIfNew(MI); 293 // Invert condition. 294 unsigned CCValid = WorkingMI.getOperand(3).getImm(); 295 unsigned CCMask = WorkingMI.getOperand(4).getImm(); 296 WorkingMI.getOperand(4).setImm(CCMask ^ CCValid); 297 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 298 OpIdx1, OpIdx2); 299 } 300 default: 301 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 302 } 303 } 304 305 // If MI is a simple load or store for a frame object, return the register 306 // it loads or stores and set FrameIndex to the index of the frame object. 307 // Return 0 otherwise. 308 // 309 // Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores. 310 static int isSimpleMove(const MachineInstr &MI, int &FrameIndex, 311 unsigned Flag) { 312 const MCInstrDesc &MCID = MI.getDesc(); 313 if ((MCID.TSFlags & Flag) && MI.getOperand(1).isFI() && 314 MI.getOperand(2).getImm() == 0 && MI.getOperand(3).getReg() == 0) { 315 FrameIndex = MI.getOperand(1).getIndex(); 316 return MI.getOperand(0).getReg(); 317 } 318 return 0; 319 } 320 321 unsigned SystemZInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 322 int &FrameIndex) const { 323 return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXLoad); 324 } 325 326 unsigned SystemZInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 327 int &FrameIndex) const { 328 return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXStore); 329 } 330 331 bool SystemZInstrInfo::isStackSlotCopy(const MachineInstr &MI, 332 int &DestFrameIndex, 333 int &SrcFrameIndex) const { 334 // Check for MVC 0(Length,FI1),0(FI2) 335 const MachineFrameInfo &MFI = MI.getParent()->getParent()->getFrameInfo(); 336 if (MI.getOpcode() != SystemZ::MVC || !MI.getOperand(0).isFI() || 337 MI.getOperand(1).getImm() != 0 || !MI.getOperand(3).isFI() || 338 MI.getOperand(4).getImm() != 0) 339 return false; 340 341 // Check that Length covers the full slots. 342 int64_t Length = MI.getOperand(2).getImm(); 343 unsigned FI1 = MI.getOperand(0).getIndex(); 344 unsigned FI2 = MI.getOperand(3).getIndex(); 345 if (MFI.getObjectSize(FI1) != Length || 346 MFI.getObjectSize(FI2) != Length) 347 return false; 348 349 DestFrameIndex = FI1; 350 SrcFrameIndex = FI2; 351 return true; 352 } 353 354 bool SystemZInstrInfo::analyzeBranch(MachineBasicBlock &MBB, 355 MachineBasicBlock *&TBB, 356 MachineBasicBlock *&FBB, 357 SmallVectorImpl<MachineOperand> &Cond, 358 bool AllowModify) const { 359 // Most of the code and comments here are boilerplate. 360 361 // Start from the bottom of the block and work up, examining the 362 // terminator instructions. 363 MachineBasicBlock::iterator I = MBB.end(); 364 while (I != MBB.begin()) { 365 --I; 366 if (I->isDebugInstr()) 367 continue; 368 369 // Working from the bottom, when we see a non-terminator instruction, we're 370 // done. 371 if (!isUnpredicatedTerminator(*I)) 372 break; 373 374 // A terminator that isn't a branch can't easily be handled by this 375 // analysis. 376 if (!I->isBranch()) 377 return true; 378 379 // Can't handle indirect branches. 380 SystemZII::Branch Branch(getBranchInfo(*I)); 381 if (!Branch.hasMBBTarget()) 382 return true; 383 384 // Punt on compound branches. 385 if (Branch.Type != SystemZII::BranchNormal) 386 return true; 387 388 if (Branch.CCMask == SystemZ::CCMASK_ANY) { 389 // Handle unconditional branches. 390 if (!AllowModify) { 391 TBB = Branch.getMBBTarget(); 392 continue; 393 } 394 395 // If the block has any instructions after a JMP, delete them. 396 while (std::next(I) != MBB.end()) 397 std::next(I)->eraseFromParent(); 398 399 Cond.clear(); 400 FBB = nullptr; 401 402 // Delete the JMP if it's equivalent to a fall-through. 403 if (MBB.isLayoutSuccessor(Branch.getMBBTarget())) { 404 TBB = nullptr; 405 I->eraseFromParent(); 406 I = MBB.end(); 407 continue; 408 } 409 410 // TBB is used to indicate the unconditinal destination. 411 TBB = Branch.getMBBTarget(); 412 continue; 413 } 414 415 // Working from the bottom, handle the first conditional branch. 416 if (Cond.empty()) { 417 // FIXME: add X86-style branch swap 418 FBB = TBB; 419 TBB = Branch.getMBBTarget(); 420 Cond.push_back(MachineOperand::CreateImm(Branch.CCValid)); 421 Cond.push_back(MachineOperand::CreateImm(Branch.CCMask)); 422 continue; 423 } 424 425 // Handle subsequent conditional branches. 426 assert(Cond.size() == 2 && TBB && "Should have seen a conditional branch"); 427 428 // Only handle the case where all conditional branches branch to the same 429 // destination. 430 if (TBB != Branch.getMBBTarget()) 431 return true; 432 433 // If the conditions are the same, we can leave them alone. 434 unsigned OldCCValid = Cond[0].getImm(); 435 unsigned OldCCMask = Cond[1].getImm(); 436 if (OldCCValid == Branch.CCValid && OldCCMask == Branch.CCMask) 437 continue; 438 439 // FIXME: Try combining conditions like X86 does. Should be easy on Z! 440 return false; 441 } 442 443 return false; 444 } 445 446 unsigned SystemZInstrInfo::removeBranch(MachineBasicBlock &MBB, 447 int *BytesRemoved) const { 448 assert(!BytesRemoved && "code size not handled"); 449 450 // Most of the code and comments here are boilerplate. 451 MachineBasicBlock::iterator I = MBB.end(); 452 unsigned Count = 0; 453 454 while (I != MBB.begin()) { 455 --I; 456 if (I->isDebugInstr()) 457 continue; 458 if (!I->isBranch()) 459 break; 460 if (!getBranchInfo(*I).hasMBBTarget()) 461 break; 462 // Remove the branch. 463 I->eraseFromParent(); 464 I = MBB.end(); 465 ++Count; 466 } 467 468 return Count; 469 } 470 471 bool SystemZInstrInfo:: 472 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { 473 assert(Cond.size() == 2 && "Invalid condition"); 474 Cond[1].setImm(Cond[1].getImm() ^ Cond[0].getImm()); 475 return false; 476 } 477 478 unsigned SystemZInstrInfo::insertBranch(MachineBasicBlock &MBB, 479 MachineBasicBlock *TBB, 480 MachineBasicBlock *FBB, 481 ArrayRef<MachineOperand> Cond, 482 const DebugLoc &DL, 483 int *BytesAdded) const { 484 // In this function we output 32-bit branches, which should always 485 // have enough range. They can be shortened and relaxed by later code 486 // in the pipeline, if desired. 487 488 // Shouldn't be a fall through. 489 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 490 assert((Cond.size() == 2 || Cond.size() == 0) && 491 "SystemZ branch conditions have one component!"); 492 assert(!BytesAdded && "code size not handled"); 493 494 if (Cond.empty()) { 495 // Unconditional branch? 496 assert(!FBB && "Unconditional branch with multiple successors!"); 497 BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(TBB); 498 return 1; 499 } 500 501 // Conditional branch. 502 unsigned Count = 0; 503 unsigned CCValid = Cond[0].getImm(); 504 unsigned CCMask = Cond[1].getImm(); 505 BuildMI(&MBB, DL, get(SystemZ::BRC)) 506 .addImm(CCValid).addImm(CCMask).addMBB(TBB); 507 ++Count; 508 509 if (FBB) { 510 // Two-way Conditional branch. Insert the second branch. 511 BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(FBB); 512 ++Count; 513 } 514 return Count; 515 } 516 517 bool SystemZInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 518 Register &SrcReg2, int64_t &Mask, 519 int64_t &Value) const { 520 assert(MI.isCompare() && "Caller should have checked for a comparison"); 521 522 if (MI.getNumExplicitOperands() == 2 && MI.getOperand(0).isReg() && 523 MI.getOperand(1).isImm()) { 524 SrcReg = MI.getOperand(0).getReg(); 525 SrcReg2 = 0; 526 Value = MI.getOperand(1).getImm(); 527 Mask = ~0; 528 return true; 529 } 530 531 return false; 532 } 533 534 bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 535 ArrayRef<MachineOperand> Pred, 536 Register DstReg, Register TrueReg, 537 Register FalseReg, int &CondCycles, 538 int &TrueCycles, 539 int &FalseCycles) const { 540 // Not all subtargets have LOCR instructions. 541 if (!STI.hasLoadStoreOnCond()) 542 return false; 543 if (Pred.size() != 2) 544 return false; 545 546 // Check register classes. 547 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 548 const TargetRegisterClass *RC = 549 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); 550 if (!RC) 551 return false; 552 553 // We have LOCR instructions for 32 and 64 bit general purpose registers. 554 if ((STI.hasLoadStoreOnCond2() && 555 SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) || 556 SystemZ::GR32BitRegClass.hasSubClassEq(RC) || 557 SystemZ::GR64BitRegClass.hasSubClassEq(RC)) { 558 CondCycles = 2; 559 TrueCycles = 2; 560 FalseCycles = 2; 561 return true; 562 } 563 564 // Can't do anything else. 565 return false; 566 } 567 568 void SystemZInstrInfo::insertSelect(MachineBasicBlock &MBB, 569 MachineBasicBlock::iterator I, 570 const DebugLoc &DL, Register DstReg, 571 ArrayRef<MachineOperand> Pred, 572 Register TrueReg, 573 Register FalseReg) const { 574 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 575 const TargetRegisterClass *RC = MRI.getRegClass(DstReg); 576 577 assert(Pred.size() == 2 && "Invalid condition"); 578 unsigned CCValid = Pred[0].getImm(); 579 unsigned CCMask = Pred[1].getImm(); 580 581 unsigned Opc; 582 if (SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) { 583 if (STI.hasMiscellaneousExtensions3()) 584 Opc = SystemZ::SELRMux; 585 else if (STI.hasLoadStoreOnCond2()) 586 Opc = SystemZ::LOCRMux; 587 else { 588 Opc = SystemZ::LOCR; 589 MRI.constrainRegClass(DstReg, &SystemZ::GR32BitRegClass); 590 Register TReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); 591 Register FReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); 592 BuildMI(MBB, I, DL, get(TargetOpcode::COPY), TReg).addReg(TrueReg); 593 BuildMI(MBB, I, DL, get(TargetOpcode::COPY), FReg).addReg(FalseReg); 594 TrueReg = TReg; 595 FalseReg = FReg; 596 } 597 } else if (SystemZ::GR64BitRegClass.hasSubClassEq(RC)) { 598 if (STI.hasMiscellaneousExtensions3()) 599 Opc = SystemZ::SELGR; 600 else 601 Opc = SystemZ::LOCGR; 602 } else 603 llvm_unreachable("Invalid register class"); 604 605 BuildMI(MBB, I, DL, get(Opc), DstReg) 606 .addReg(FalseReg).addReg(TrueReg) 607 .addImm(CCValid).addImm(CCMask); 608 } 609 610 bool SystemZInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 611 Register Reg, 612 MachineRegisterInfo *MRI) const { 613 unsigned DefOpc = DefMI.getOpcode(); 614 if (DefOpc != SystemZ::LHIMux && DefOpc != SystemZ::LHI && 615 DefOpc != SystemZ::LGHI) 616 return false; 617 if (DefMI.getOperand(0).getReg() != Reg) 618 return false; 619 int32_t ImmVal = (int32_t)DefMI.getOperand(1).getImm(); 620 621 unsigned UseOpc = UseMI.getOpcode(); 622 unsigned NewUseOpc; 623 unsigned UseIdx; 624 int CommuteIdx = -1; 625 bool TieOps = false; 626 switch (UseOpc) { 627 case SystemZ::SELRMux: 628 TieOps = true; 629 LLVM_FALLTHROUGH; 630 case SystemZ::LOCRMux: 631 if (!STI.hasLoadStoreOnCond2()) 632 return false; 633 NewUseOpc = SystemZ::LOCHIMux; 634 if (UseMI.getOperand(2).getReg() == Reg) 635 UseIdx = 2; 636 else if (UseMI.getOperand(1).getReg() == Reg) 637 UseIdx = 2, CommuteIdx = 1; 638 else 639 return false; 640 break; 641 case SystemZ::SELGR: 642 TieOps = true; 643 LLVM_FALLTHROUGH; 644 case SystemZ::LOCGR: 645 if (!STI.hasLoadStoreOnCond2()) 646 return false; 647 NewUseOpc = SystemZ::LOCGHI; 648 if (UseMI.getOperand(2).getReg() == Reg) 649 UseIdx = 2; 650 else if (UseMI.getOperand(1).getReg() == Reg) 651 UseIdx = 2, CommuteIdx = 1; 652 else 653 return false; 654 break; 655 default: 656 return false; 657 } 658 659 if (CommuteIdx != -1) 660 if (!commuteInstruction(UseMI, false, CommuteIdx, UseIdx)) 661 return false; 662 663 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 664 UseMI.setDesc(get(NewUseOpc)); 665 if (TieOps) 666 UseMI.tieOperands(0, 1); 667 UseMI.getOperand(UseIdx).ChangeToImmediate(ImmVal); 668 if (DeleteDef) 669 DefMI.eraseFromParent(); 670 671 return true; 672 } 673 674 bool SystemZInstrInfo::isPredicable(const MachineInstr &MI) const { 675 unsigned Opcode = MI.getOpcode(); 676 if (Opcode == SystemZ::Return || 677 Opcode == SystemZ::Trap || 678 Opcode == SystemZ::CallJG || 679 Opcode == SystemZ::CallBR) 680 return true; 681 return false; 682 } 683 684 bool SystemZInstrInfo:: 685 isProfitableToIfCvt(MachineBasicBlock &MBB, 686 unsigned NumCycles, unsigned ExtraPredCycles, 687 BranchProbability Probability) const { 688 // Avoid using conditional returns at the end of a loop (since then 689 // we'd need to emit an unconditional branch to the beginning anyway, 690 // making the loop body longer). This doesn't apply for low-probability 691 // loops (eg. compare-and-swap retry), so just decide based on branch 692 // probability instead of looping structure. 693 // However, since Compare and Trap instructions cost the same as a regular 694 // Compare instruction, we should allow the if conversion to convert this 695 // into a Conditional Compare regardless of the branch probability. 696 if (MBB.getLastNonDebugInstr()->getOpcode() != SystemZ::Trap && 697 MBB.succ_empty() && Probability < BranchProbability(1, 8)) 698 return false; 699 // For now only convert single instructions. 700 return NumCycles == 1; 701 } 702 703 bool SystemZInstrInfo:: 704 isProfitableToIfCvt(MachineBasicBlock &TMBB, 705 unsigned NumCyclesT, unsigned ExtraPredCyclesT, 706 MachineBasicBlock &FMBB, 707 unsigned NumCyclesF, unsigned ExtraPredCyclesF, 708 BranchProbability Probability) const { 709 // For now avoid converting mutually-exclusive cases. 710 return false; 711 } 712 713 bool SystemZInstrInfo:: 714 isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, 715 BranchProbability Probability) const { 716 // For now only duplicate single instructions. 717 return NumCycles == 1; 718 } 719 720 bool SystemZInstrInfo::PredicateInstruction( 721 MachineInstr &MI, ArrayRef<MachineOperand> Pred) const { 722 assert(Pred.size() == 2 && "Invalid condition"); 723 unsigned CCValid = Pred[0].getImm(); 724 unsigned CCMask = Pred[1].getImm(); 725 assert(CCMask > 0 && CCMask < 15 && "Invalid predicate"); 726 unsigned Opcode = MI.getOpcode(); 727 if (Opcode == SystemZ::Trap) { 728 MI.setDesc(get(SystemZ::CondTrap)); 729 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 730 .addImm(CCValid).addImm(CCMask) 731 .addReg(SystemZ::CC, RegState::Implicit); 732 return true; 733 } 734 if (Opcode == SystemZ::Return) { 735 MI.setDesc(get(SystemZ::CondReturn)); 736 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 737 .addImm(CCValid).addImm(CCMask) 738 .addReg(SystemZ::CC, RegState::Implicit); 739 return true; 740 } 741 if (Opcode == SystemZ::CallJG) { 742 MachineOperand FirstOp = MI.getOperand(0); 743 const uint32_t *RegMask = MI.getOperand(1).getRegMask(); 744 MI.RemoveOperand(1); 745 MI.RemoveOperand(0); 746 MI.setDesc(get(SystemZ::CallBRCL)); 747 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 748 .addImm(CCValid) 749 .addImm(CCMask) 750 .add(FirstOp) 751 .addRegMask(RegMask) 752 .addReg(SystemZ::CC, RegState::Implicit); 753 return true; 754 } 755 if (Opcode == SystemZ::CallBR) { 756 MachineOperand Target = MI.getOperand(0); 757 const uint32_t *RegMask = MI.getOperand(1).getRegMask(); 758 MI.RemoveOperand(1); 759 MI.RemoveOperand(0); 760 MI.setDesc(get(SystemZ::CallBCR)); 761 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 762 .addImm(CCValid).addImm(CCMask) 763 .add(Target) 764 .addRegMask(RegMask) 765 .addReg(SystemZ::CC, RegState::Implicit); 766 return true; 767 } 768 return false; 769 } 770 771 void SystemZInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 772 MachineBasicBlock::iterator MBBI, 773 const DebugLoc &DL, MCRegister DestReg, 774 MCRegister SrcReg, bool KillSrc) const { 775 // Split 128-bit GPR moves into two 64-bit moves. Add implicit uses of the 776 // super register in case one of the subregs is undefined. 777 // This handles ADDR128 too. 778 if (SystemZ::GR128BitRegClass.contains(DestReg, SrcReg)) { 779 copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_h64), 780 RI.getSubReg(SrcReg, SystemZ::subreg_h64), KillSrc); 781 MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI)) 782 .addReg(SrcReg, RegState::Implicit); 783 copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_l64), 784 RI.getSubReg(SrcReg, SystemZ::subreg_l64), KillSrc); 785 MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI)) 786 .addReg(SrcReg, (getKillRegState(KillSrc) | RegState::Implicit)); 787 return; 788 } 789 790 if (SystemZ::GRX32BitRegClass.contains(DestReg, SrcReg)) { 791 emitGRX32Move(MBB, MBBI, DL, DestReg, SrcReg, SystemZ::LR, 32, KillSrc, 792 false); 793 return; 794 } 795 796 // Move 128-bit floating-point values between VR128 and FP128. 797 if (SystemZ::VR128BitRegClass.contains(DestReg) && 798 SystemZ::FP128BitRegClass.contains(SrcReg)) { 799 MCRegister SrcRegHi = 800 RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_h64), 801 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 802 MCRegister SrcRegLo = 803 RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_l64), 804 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 805 806 BuildMI(MBB, MBBI, DL, get(SystemZ::VMRHG), DestReg) 807 .addReg(SrcRegHi, getKillRegState(KillSrc)) 808 .addReg(SrcRegLo, getKillRegState(KillSrc)); 809 return; 810 } 811 if (SystemZ::FP128BitRegClass.contains(DestReg) && 812 SystemZ::VR128BitRegClass.contains(SrcReg)) { 813 MCRegister DestRegHi = 814 RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_h64), 815 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 816 MCRegister DestRegLo = 817 RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_l64), 818 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 819 820 if (DestRegHi != SrcReg) 821 copyPhysReg(MBB, MBBI, DL, DestRegHi, SrcReg, false); 822 BuildMI(MBB, MBBI, DL, get(SystemZ::VREPG), DestRegLo) 823 .addReg(SrcReg, getKillRegState(KillSrc)).addImm(1); 824 return; 825 } 826 827 // Move CC value from a GR32. 828 if (DestReg == SystemZ::CC) { 829 unsigned Opcode = 830 SystemZ::GR32BitRegClass.contains(SrcReg) ? SystemZ::TMLH : SystemZ::TMHH; 831 BuildMI(MBB, MBBI, DL, get(Opcode)) 832 .addReg(SrcReg, getKillRegState(KillSrc)) 833 .addImm(3 << (SystemZ::IPM_CC - 16)); 834 return; 835 } 836 837 // Everything else needs only one instruction. 838 unsigned Opcode; 839 if (SystemZ::GR64BitRegClass.contains(DestReg, SrcReg)) 840 Opcode = SystemZ::LGR; 841 else if (SystemZ::FP32BitRegClass.contains(DestReg, SrcReg)) 842 // For z13 we prefer LDR over LER to avoid partial register dependencies. 843 Opcode = STI.hasVector() ? SystemZ::LDR32 : SystemZ::LER; 844 else if (SystemZ::FP64BitRegClass.contains(DestReg, SrcReg)) 845 Opcode = SystemZ::LDR; 846 else if (SystemZ::FP128BitRegClass.contains(DestReg, SrcReg)) 847 Opcode = SystemZ::LXR; 848 else if (SystemZ::VR32BitRegClass.contains(DestReg, SrcReg)) 849 Opcode = SystemZ::VLR32; 850 else if (SystemZ::VR64BitRegClass.contains(DestReg, SrcReg)) 851 Opcode = SystemZ::VLR64; 852 else if (SystemZ::VR128BitRegClass.contains(DestReg, SrcReg)) 853 Opcode = SystemZ::VLR; 854 else if (SystemZ::AR32BitRegClass.contains(DestReg, SrcReg)) 855 Opcode = SystemZ::CPYA; 856 else 857 llvm_unreachable("Impossible reg-to-reg copy"); 858 859 BuildMI(MBB, MBBI, DL, get(Opcode), DestReg) 860 .addReg(SrcReg, getKillRegState(KillSrc)); 861 } 862 863 void SystemZInstrInfo::storeRegToStackSlot( 864 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, 865 bool isKill, int FrameIdx, const TargetRegisterClass *RC, 866 const TargetRegisterInfo *TRI) const { 867 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 868 869 // Callers may expect a single instruction, so keep 128-bit moves 870 // together for now and lower them after register allocation. 871 unsigned LoadOpcode, StoreOpcode; 872 getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode); 873 addFrameReference(BuildMI(MBB, MBBI, DL, get(StoreOpcode)) 874 .addReg(SrcReg, getKillRegState(isKill)), 875 FrameIdx); 876 } 877 878 void SystemZInstrInfo::loadRegFromStackSlot( 879 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, 880 int FrameIdx, const TargetRegisterClass *RC, 881 const TargetRegisterInfo *TRI) const { 882 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 883 884 // Callers may expect a single instruction, so keep 128-bit moves 885 // together for now and lower them after register allocation. 886 unsigned LoadOpcode, StoreOpcode; 887 getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode); 888 addFrameReference(BuildMI(MBB, MBBI, DL, get(LoadOpcode), DestReg), 889 FrameIdx); 890 } 891 892 // Return true if MI is a simple load or store with a 12-bit displacement 893 // and no index. Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores. 894 static bool isSimpleBD12Move(const MachineInstr *MI, unsigned Flag) { 895 const MCInstrDesc &MCID = MI->getDesc(); 896 return ((MCID.TSFlags & Flag) && 897 isUInt<12>(MI->getOperand(2).getImm()) && 898 MI->getOperand(3).getReg() == 0); 899 } 900 901 namespace { 902 903 struct LogicOp { 904 LogicOp() = default; 905 LogicOp(unsigned regSize, unsigned immLSB, unsigned immSize) 906 : RegSize(regSize), ImmLSB(immLSB), ImmSize(immSize) {} 907 908 explicit operator bool() const { return RegSize; } 909 910 unsigned RegSize = 0; 911 unsigned ImmLSB = 0; 912 unsigned ImmSize = 0; 913 }; 914 915 } // end anonymous namespace 916 917 static LogicOp interpretAndImmediate(unsigned Opcode) { 918 switch (Opcode) { 919 case SystemZ::NILMux: return LogicOp(32, 0, 16); 920 case SystemZ::NIHMux: return LogicOp(32, 16, 16); 921 case SystemZ::NILL64: return LogicOp(64, 0, 16); 922 case SystemZ::NILH64: return LogicOp(64, 16, 16); 923 case SystemZ::NIHL64: return LogicOp(64, 32, 16); 924 case SystemZ::NIHH64: return LogicOp(64, 48, 16); 925 case SystemZ::NIFMux: return LogicOp(32, 0, 32); 926 case SystemZ::NILF64: return LogicOp(64, 0, 32); 927 case SystemZ::NIHF64: return LogicOp(64, 32, 32); 928 default: return LogicOp(); 929 } 930 } 931 932 static void transferDeadCC(MachineInstr *OldMI, MachineInstr *NewMI) { 933 if (OldMI->registerDefIsDead(SystemZ::CC)) { 934 MachineOperand *CCDef = NewMI->findRegisterDefOperand(SystemZ::CC); 935 if (CCDef != nullptr) 936 CCDef->setIsDead(true); 937 } 938 } 939 940 static void transferMIFlag(MachineInstr *OldMI, MachineInstr *NewMI, 941 MachineInstr::MIFlag Flag) { 942 if (OldMI->getFlag(Flag)) 943 NewMI->setFlag(Flag); 944 } 945 946 MachineInstr * 947 SystemZInstrInfo::convertToThreeAddress(MachineInstr &MI, LiveVariables *LV, 948 LiveIntervals *LIS) const { 949 MachineBasicBlock *MBB = MI.getParent(); 950 951 // Try to convert an AND into an RISBG-type instruction. 952 // TODO: It might be beneficial to select RISBG and shorten to AND instead. 953 if (LogicOp And = interpretAndImmediate(MI.getOpcode())) { 954 uint64_t Imm = MI.getOperand(2).getImm() << And.ImmLSB; 955 // AND IMMEDIATE leaves the other bits of the register unchanged. 956 Imm |= allOnes(And.RegSize) & ~(allOnes(And.ImmSize) << And.ImmLSB); 957 unsigned Start, End; 958 if (isRxSBGMask(Imm, And.RegSize, Start, End)) { 959 unsigned NewOpcode; 960 if (And.RegSize == 64) { 961 NewOpcode = SystemZ::RISBG; 962 // Prefer RISBGN if available, since it does not clobber CC. 963 if (STI.hasMiscellaneousExtensions()) 964 NewOpcode = SystemZ::RISBGN; 965 } else { 966 NewOpcode = SystemZ::RISBMux; 967 Start &= 31; 968 End &= 31; 969 } 970 MachineOperand &Dest = MI.getOperand(0); 971 MachineOperand &Src = MI.getOperand(1); 972 MachineInstrBuilder MIB = 973 BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpcode)) 974 .add(Dest) 975 .addReg(0) 976 .addReg(Src.getReg(), getKillRegState(Src.isKill()), 977 Src.getSubReg()) 978 .addImm(Start) 979 .addImm(End + 128) 980 .addImm(0); 981 if (LV) { 982 unsigned NumOps = MI.getNumOperands(); 983 for (unsigned I = 1; I < NumOps; ++I) { 984 MachineOperand &Op = MI.getOperand(I); 985 if (Op.isReg() && Op.isKill()) 986 LV->replaceKillInstruction(Op.getReg(), MI, *MIB); 987 } 988 } 989 if (LIS) 990 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 991 transferDeadCC(&MI, MIB); 992 return MIB; 993 } 994 } 995 return nullptr; 996 } 997 998 MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl( 999 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 1000 MachineBasicBlock::iterator InsertPt, int FrameIndex, 1001 LiveIntervals *LIS, VirtRegMap *VRM) const { 1002 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 1003 MachineRegisterInfo &MRI = MF.getRegInfo(); 1004 const MachineFrameInfo &MFI = MF.getFrameInfo(); 1005 unsigned Size = MFI.getObjectSize(FrameIndex); 1006 unsigned Opcode = MI.getOpcode(); 1007 1008 // Check CC liveness if new instruction introduces a dead def of CC. 1009 MCRegUnitIterator CCUnit(MCRegister::from(SystemZ::CC), TRI); 1010 SlotIndex MISlot = SlotIndex(); 1011 LiveRange *CCLiveRange = nullptr; 1012 bool CCLiveAtMI = true; 1013 if (LIS) { 1014 MISlot = LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot(); 1015 CCLiveRange = &LIS->getRegUnit(*CCUnit); 1016 CCLiveAtMI = CCLiveRange->liveAt(MISlot); 1017 } 1018 ++CCUnit; 1019 assert(!CCUnit.isValid() && "CC only has one reg unit."); 1020 1021 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 1022 if (!CCLiveAtMI && (Opcode == SystemZ::LA || Opcode == SystemZ::LAY) && 1023 isInt<8>(MI.getOperand(2).getImm()) && !MI.getOperand(3).getReg()) { 1024 // LA(Y) %reg, CONST(%reg) -> AGSI %mem, CONST 1025 MachineInstr *BuiltMI = BuildMI(*InsertPt->getParent(), InsertPt, 1026 MI.getDebugLoc(), get(SystemZ::AGSI)) 1027 .addFrameIndex(FrameIndex) 1028 .addImm(0) 1029 .addImm(MI.getOperand(2).getImm()); 1030 BuiltMI->findRegisterDefOperand(SystemZ::CC)->setIsDead(true); 1031 CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator()); 1032 return BuiltMI; 1033 } 1034 return nullptr; 1035 } 1036 1037 // All other cases require a single operand. 1038 if (Ops.size() != 1) 1039 return nullptr; 1040 1041 unsigned OpNum = Ops[0]; 1042 assert(Size * 8 == 1043 TRI->getRegSizeInBits(*MF.getRegInfo() 1044 .getRegClass(MI.getOperand(OpNum).getReg())) && 1045 "Invalid size combination"); 1046 1047 if ((Opcode == SystemZ::AHI || Opcode == SystemZ::AGHI) && OpNum == 0 && 1048 isInt<8>(MI.getOperand(2).getImm())) { 1049 // A(G)HI %reg, CONST -> A(G)SI %mem, CONST 1050 Opcode = (Opcode == SystemZ::AHI ? SystemZ::ASI : SystemZ::AGSI); 1051 MachineInstr *BuiltMI = 1052 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1053 .addFrameIndex(FrameIndex) 1054 .addImm(0) 1055 .addImm(MI.getOperand(2).getImm()); 1056 transferDeadCC(&MI, BuiltMI); 1057 transferMIFlag(&MI, BuiltMI, MachineInstr::NoSWrap); 1058 return BuiltMI; 1059 } 1060 1061 if ((Opcode == SystemZ::ALFI && OpNum == 0 && 1062 isInt<8>((int32_t)MI.getOperand(2).getImm())) || 1063 (Opcode == SystemZ::ALGFI && OpNum == 0 && 1064 isInt<8>((int64_t)MI.getOperand(2).getImm()))) { 1065 // AL(G)FI %reg, CONST -> AL(G)SI %mem, CONST 1066 Opcode = (Opcode == SystemZ::ALFI ? SystemZ::ALSI : SystemZ::ALGSI); 1067 MachineInstr *BuiltMI = 1068 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1069 .addFrameIndex(FrameIndex) 1070 .addImm(0) 1071 .addImm((int8_t)MI.getOperand(2).getImm()); 1072 transferDeadCC(&MI, BuiltMI); 1073 return BuiltMI; 1074 } 1075 1076 if ((Opcode == SystemZ::SLFI && OpNum == 0 && 1077 isInt<8>((int32_t)-MI.getOperand(2).getImm())) || 1078 (Opcode == SystemZ::SLGFI && OpNum == 0 && 1079 isInt<8>((int64_t)-MI.getOperand(2).getImm()))) { 1080 // SL(G)FI %reg, CONST -> AL(G)SI %mem, -CONST 1081 Opcode = (Opcode == SystemZ::SLFI ? SystemZ::ALSI : SystemZ::ALGSI); 1082 MachineInstr *BuiltMI = 1083 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1084 .addFrameIndex(FrameIndex) 1085 .addImm(0) 1086 .addImm((int8_t)-MI.getOperand(2).getImm()); 1087 transferDeadCC(&MI, BuiltMI); 1088 return BuiltMI; 1089 } 1090 1091 unsigned MemImmOpc = 0; 1092 switch (Opcode) { 1093 case SystemZ::LHIMux: 1094 case SystemZ::LHI: MemImmOpc = SystemZ::MVHI; break; 1095 case SystemZ::LGHI: MemImmOpc = SystemZ::MVGHI; break; 1096 case SystemZ::CHIMux: 1097 case SystemZ::CHI: MemImmOpc = SystemZ::CHSI; break; 1098 case SystemZ::CGHI: MemImmOpc = SystemZ::CGHSI; break; 1099 case SystemZ::CLFIMux: 1100 case SystemZ::CLFI: 1101 if (isUInt<16>(MI.getOperand(1).getImm())) 1102 MemImmOpc = SystemZ::CLFHSI; 1103 break; 1104 case SystemZ::CLGFI: 1105 if (isUInt<16>(MI.getOperand(1).getImm())) 1106 MemImmOpc = SystemZ::CLGHSI; 1107 break; 1108 default: break; 1109 } 1110 if (MemImmOpc) 1111 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1112 get(MemImmOpc)) 1113 .addFrameIndex(FrameIndex) 1114 .addImm(0) 1115 .addImm(MI.getOperand(1).getImm()); 1116 1117 if (Opcode == SystemZ::LGDR || Opcode == SystemZ::LDGR) { 1118 bool Op0IsGPR = (Opcode == SystemZ::LGDR); 1119 bool Op1IsGPR = (Opcode == SystemZ::LDGR); 1120 // If we're spilling the destination of an LDGR or LGDR, store the 1121 // source register instead. 1122 if (OpNum == 0) { 1123 unsigned StoreOpcode = Op1IsGPR ? SystemZ::STG : SystemZ::STD; 1124 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1125 get(StoreOpcode)) 1126 .add(MI.getOperand(1)) 1127 .addFrameIndex(FrameIndex) 1128 .addImm(0) 1129 .addReg(0); 1130 } 1131 // If we're spilling the source of an LDGR or LGDR, load the 1132 // destination register instead. 1133 if (OpNum == 1) { 1134 unsigned LoadOpcode = Op0IsGPR ? SystemZ::LG : SystemZ::LD; 1135 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1136 get(LoadOpcode)) 1137 .add(MI.getOperand(0)) 1138 .addFrameIndex(FrameIndex) 1139 .addImm(0) 1140 .addReg(0); 1141 } 1142 } 1143 1144 // Look for cases where the source of a simple store or the destination 1145 // of a simple load is being spilled. Try to use MVC instead. 1146 // 1147 // Although MVC is in practice a fast choice in these cases, it is still 1148 // logically a bytewise copy. This means that we cannot use it if the 1149 // load or store is volatile. We also wouldn't be able to use MVC if 1150 // the two memories partially overlap, but that case cannot occur here, 1151 // because we know that one of the memories is a full frame index. 1152 // 1153 // For performance reasons, we also want to avoid using MVC if the addresses 1154 // might be equal. We don't worry about that case here, because spill slot 1155 // coloring happens later, and because we have special code to remove 1156 // MVCs that turn out to be redundant. 1157 if (OpNum == 0 && MI.hasOneMemOperand()) { 1158 MachineMemOperand *MMO = *MI.memoperands_begin(); 1159 if (MMO->getSize() == Size && !MMO->isVolatile() && !MMO->isAtomic()) { 1160 // Handle conversion of loads. 1161 if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXLoad)) { 1162 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1163 get(SystemZ::MVC)) 1164 .addFrameIndex(FrameIndex) 1165 .addImm(0) 1166 .addImm(Size) 1167 .add(MI.getOperand(1)) 1168 .addImm(MI.getOperand(2).getImm()) 1169 .addMemOperand(MMO); 1170 } 1171 // Handle conversion of stores. 1172 if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXStore)) { 1173 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1174 get(SystemZ::MVC)) 1175 .add(MI.getOperand(1)) 1176 .addImm(MI.getOperand(2).getImm()) 1177 .addImm(Size) 1178 .addFrameIndex(FrameIndex) 1179 .addImm(0) 1180 .addMemOperand(MMO); 1181 } 1182 } 1183 } 1184 1185 // If the spilled operand is the final one or the instruction is 1186 // commutable, try to change <INSN>R into <INSN>. Don't introduce a def of 1187 // CC if it is live and MI does not define it. 1188 unsigned NumOps = MI.getNumExplicitOperands(); 1189 int MemOpcode = SystemZ::getMemOpcode(Opcode); 1190 if (MemOpcode == -1 || 1191 (CCLiveAtMI && !MI.definesRegister(SystemZ::CC) && 1192 get(MemOpcode).hasImplicitDefOfPhysReg(SystemZ::CC))) 1193 return nullptr; 1194 1195 // Check if all other vregs have a usable allocation in the case of vector 1196 // to FP conversion. 1197 const MCInstrDesc &MCID = MI.getDesc(); 1198 for (unsigned I = 0, E = MCID.getNumOperands(); I != E; ++I) { 1199 const MCOperandInfo &MCOI = MCID.OpInfo[I]; 1200 if (MCOI.OperandType != MCOI::OPERAND_REGISTER || I == OpNum) 1201 continue; 1202 const TargetRegisterClass *RC = TRI->getRegClass(MCOI.RegClass); 1203 if (RC == &SystemZ::VR32BitRegClass || RC == &SystemZ::VR64BitRegClass) { 1204 Register Reg = MI.getOperand(I).getReg(); 1205 Register PhysReg = Register::isVirtualRegister(Reg) 1206 ? (VRM ? Register(VRM->getPhys(Reg)) : Register()) 1207 : Reg; 1208 if (!PhysReg || 1209 !(SystemZ::FP32BitRegClass.contains(PhysReg) || 1210 SystemZ::FP64BitRegClass.contains(PhysReg) || 1211 SystemZ::VF128BitRegClass.contains(PhysReg))) 1212 return nullptr; 1213 } 1214 } 1215 // Fused multiply and add/sub need to have the same dst and accumulator reg. 1216 bool FusedFPOp = (Opcode == SystemZ::WFMADB || Opcode == SystemZ::WFMASB || 1217 Opcode == SystemZ::WFMSDB || Opcode == SystemZ::WFMSSB); 1218 if (FusedFPOp) { 1219 Register DstReg = VRM->getPhys(MI.getOperand(0).getReg()); 1220 Register AccReg = VRM->getPhys(MI.getOperand(3).getReg()); 1221 if (OpNum == 0 || OpNum == 3 || DstReg != AccReg) 1222 return nullptr; 1223 } 1224 1225 // Try to swap compare operands if possible. 1226 bool NeedsCommute = false; 1227 if ((MI.getOpcode() == SystemZ::CR || MI.getOpcode() == SystemZ::CGR || 1228 MI.getOpcode() == SystemZ::CLR || MI.getOpcode() == SystemZ::CLGR || 1229 MI.getOpcode() == SystemZ::WFCDB || MI.getOpcode() == SystemZ::WFCSB || 1230 MI.getOpcode() == SystemZ::WFKDB || MI.getOpcode() == SystemZ::WFKSB) && 1231 OpNum == 0 && prepareCompareSwapOperands(MI)) 1232 NeedsCommute = true; 1233 1234 bool CCOperands = false; 1235 if (MI.getOpcode() == SystemZ::LOCRMux || MI.getOpcode() == SystemZ::LOCGR || 1236 MI.getOpcode() == SystemZ::SELRMux || MI.getOpcode() == SystemZ::SELGR) { 1237 assert(MI.getNumOperands() == 6 && NumOps == 5 && 1238 "LOCR/SELR instruction operands corrupt?"); 1239 NumOps -= 2; 1240 CCOperands = true; 1241 } 1242 1243 // See if this is a 3-address instruction that is convertible to 2-address 1244 // and suitable for folding below. Only try this with virtual registers 1245 // and a provided VRM (during regalloc). 1246 if (NumOps == 3 && SystemZ::getTargetMemOpcode(MemOpcode) != -1) { 1247 if (VRM == nullptr) 1248 return nullptr; 1249 else { 1250 Register DstReg = MI.getOperand(0).getReg(); 1251 Register DstPhys = 1252 (Register::isVirtualRegister(DstReg) ? Register(VRM->getPhys(DstReg)) 1253 : DstReg); 1254 Register SrcReg = (OpNum == 2 ? MI.getOperand(1).getReg() 1255 : ((OpNum == 1 && MI.isCommutable()) 1256 ? MI.getOperand(2).getReg() 1257 : Register())); 1258 if (DstPhys && !SystemZ::GRH32BitRegClass.contains(DstPhys) && SrcReg && 1259 Register::isVirtualRegister(SrcReg) && 1260 DstPhys == VRM->getPhys(SrcReg)) 1261 NeedsCommute = (OpNum == 1); 1262 else 1263 return nullptr; 1264 } 1265 } 1266 1267 if ((OpNum == NumOps - 1) || NeedsCommute || FusedFPOp) { 1268 const MCInstrDesc &MemDesc = get(MemOpcode); 1269 uint64_t AccessBytes = SystemZII::getAccessSize(MemDesc.TSFlags); 1270 assert(AccessBytes != 0 && "Size of access should be known"); 1271 assert(AccessBytes <= Size && "Access outside the frame index"); 1272 uint64_t Offset = Size - AccessBytes; 1273 MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, 1274 MI.getDebugLoc(), get(MemOpcode)); 1275 if (MI.isCompare()) { 1276 assert(NumOps == 2 && "Expected 2 register operands for a compare."); 1277 MIB.add(MI.getOperand(NeedsCommute ? 1 : 0)); 1278 } 1279 else if (FusedFPOp) { 1280 MIB.add(MI.getOperand(0)); 1281 MIB.add(MI.getOperand(3)); 1282 MIB.add(MI.getOperand(OpNum == 1 ? 2 : 1)); 1283 } 1284 else { 1285 MIB.add(MI.getOperand(0)); 1286 if (NeedsCommute) 1287 MIB.add(MI.getOperand(2)); 1288 else 1289 for (unsigned I = 1; I < OpNum; ++I) 1290 MIB.add(MI.getOperand(I)); 1291 } 1292 MIB.addFrameIndex(FrameIndex).addImm(Offset); 1293 if (MemDesc.TSFlags & SystemZII::HasIndex) 1294 MIB.addReg(0); 1295 if (CCOperands) { 1296 unsigned CCValid = MI.getOperand(NumOps).getImm(); 1297 unsigned CCMask = MI.getOperand(NumOps + 1).getImm(); 1298 MIB.addImm(CCValid); 1299 MIB.addImm(NeedsCommute ? CCMask ^ CCValid : CCMask); 1300 } 1301 if (MIB->definesRegister(SystemZ::CC) && 1302 (!MI.definesRegister(SystemZ::CC) || 1303 MI.registerDefIsDead(SystemZ::CC))) { 1304 MIB->addRegisterDead(SystemZ::CC, TRI); 1305 if (CCLiveRange) 1306 CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator()); 1307 } 1308 // Constrain the register classes if converted from a vector opcode. The 1309 // allocated regs are in an FP reg-class per previous check above. 1310 for (const MachineOperand &MO : MIB->operands()) 1311 if (MO.isReg() && Register::isVirtualRegister(MO.getReg())) { 1312 Register Reg = MO.getReg(); 1313 if (MRI.getRegClass(Reg) == &SystemZ::VR32BitRegClass) 1314 MRI.setRegClass(Reg, &SystemZ::FP32BitRegClass); 1315 else if (MRI.getRegClass(Reg) == &SystemZ::VR64BitRegClass) 1316 MRI.setRegClass(Reg, &SystemZ::FP64BitRegClass); 1317 else if (MRI.getRegClass(Reg) == &SystemZ::VR128BitRegClass) 1318 MRI.setRegClass(Reg, &SystemZ::VF128BitRegClass); 1319 } 1320 1321 transferDeadCC(&MI, MIB); 1322 transferMIFlag(&MI, MIB, MachineInstr::NoSWrap); 1323 transferMIFlag(&MI, MIB, MachineInstr::NoFPExcept); 1324 return MIB; 1325 } 1326 1327 return nullptr; 1328 } 1329 1330 MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl( 1331 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 1332 MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, 1333 LiveIntervals *LIS) const { 1334 return nullptr; 1335 } 1336 1337 bool SystemZInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 1338 switch (MI.getOpcode()) { 1339 case SystemZ::L128: 1340 splitMove(MI, SystemZ::LG); 1341 return true; 1342 1343 case SystemZ::ST128: 1344 splitMove(MI, SystemZ::STG); 1345 return true; 1346 1347 case SystemZ::LX: 1348 splitMove(MI, SystemZ::LD); 1349 return true; 1350 1351 case SystemZ::STX: 1352 splitMove(MI, SystemZ::STD); 1353 return true; 1354 1355 case SystemZ::LBMux: 1356 expandRXYPseudo(MI, SystemZ::LB, SystemZ::LBH); 1357 return true; 1358 1359 case SystemZ::LHMux: 1360 expandRXYPseudo(MI, SystemZ::LH, SystemZ::LHH); 1361 return true; 1362 1363 case SystemZ::LLCRMux: 1364 expandZExtPseudo(MI, SystemZ::LLCR, 8); 1365 return true; 1366 1367 case SystemZ::LLHRMux: 1368 expandZExtPseudo(MI, SystemZ::LLHR, 16); 1369 return true; 1370 1371 case SystemZ::LLCMux: 1372 expandRXYPseudo(MI, SystemZ::LLC, SystemZ::LLCH); 1373 return true; 1374 1375 case SystemZ::LLHMux: 1376 expandRXYPseudo(MI, SystemZ::LLH, SystemZ::LLHH); 1377 return true; 1378 1379 case SystemZ::LMux: 1380 expandRXYPseudo(MI, SystemZ::L, SystemZ::LFH); 1381 return true; 1382 1383 case SystemZ::LOCMux: 1384 expandLOCPseudo(MI, SystemZ::LOC, SystemZ::LOCFH); 1385 return true; 1386 1387 case SystemZ::LOCHIMux: 1388 expandLOCPseudo(MI, SystemZ::LOCHI, SystemZ::LOCHHI); 1389 return true; 1390 1391 case SystemZ::STCMux: 1392 expandRXYPseudo(MI, SystemZ::STC, SystemZ::STCH); 1393 return true; 1394 1395 case SystemZ::STHMux: 1396 expandRXYPseudo(MI, SystemZ::STH, SystemZ::STHH); 1397 return true; 1398 1399 case SystemZ::STMux: 1400 expandRXYPseudo(MI, SystemZ::ST, SystemZ::STFH); 1401 return true; 1402 1403 case SystemZ::STOCMux: 1404 expandLOCPseudo(MI, SystemZ::STOC, SystemZ::STOCFH); 1405 return true; 1406 1407 case SystemZ::LHIMux: 1408 expandRIPseudo(MI, SystemZ::LHI, SystemZ::IIHF, true); 1409 return true; 1410 1411 case SystemZ::IIFMux: 1412 expandRIPseudo(MI, SystemZ::IILF, SystemZ::IIHF, false); 1413 return true; 1414 1415 case SystemZ::IILMux: 1416 expandRIPseudo(MI, SystemZ::IILL, SystemZ::IIHL, false); 1417 return true; 1418 1419 case SystemZ::IIHMux: 1420 expandRIPseudo(MI, SystemZ::IILH, SystemZ::IIHH, false); 1421 return true; 1422 1423 case SystemZ::NIFMux: 1424 expandRIPseudo(MI, SystemZ::NILF, SystemZ::NIHF, false); 1425 return true; 1426 1427 case SystemZ::NILMux: 1428 expandRIPseudo(MI, SystemZ::NILL, SystemZ::NIHL, false); 1429 return true; 1430 1431 case SystemZ::NIHMux: 1432 expandRIPseudo(MI, SystemZ::NILH, SystemZ::NIHH, false); 1433 return true; 1434 1435 case SystemZ::OIFMux: 1436 expandRIPseudo(MI, SystemZ::OILF, SystemZ::OIHF, false); 1437 return true; 1438 1439 case SystemZ::OILMux: 1440 expandRIPseudo(MI, SystemZ::OILL, SystemZ::OIHL, false); 1441 return true; 1442 1443 case SystemZ::OIHMux: 1444 expandRIPseudo(MI, SystemZ::OILH, SystemZ::OIHH, false); 1445 return true; 1446 1447 case SystemZ::XIFMux: 1448 expandRIPseudo(MI, SystemZ::XILF, SystemZ::XIHF, false); 1449 return true; 1450 1451 case SystemZ::TMLMux: 1452 expandRIPseudo(MI, SystemZ::TMLL, SystemZ::TMHL, false); 1453 return true; 1454 1455 case SystemZ::TMHMux: 1456 expandRIPseudo(MI, SystemZ::TMLH, SystemZ::TMHH, false); 1457 return true; 1458 1459 case SystemZ::AHIMux: 1460 expandRIPseudo(MI, SystemZ::AHI, SystemZ::AIH, false); 1461 return true; 1462 1463 case SystemZ::AHIMuxK: 1464 expandRIEPseudo(MI, SystemZ::AHI, SystemZ::AHIK, SystemZ::AIH); 1465 return true; 1466 1467 case SystemZ::AFIMux: 1468 expandRIPseudo(MI, SystemZ::AFI, SystemZ::AIH, false); 1469 return true; 1470 1471 case SystemZ::CHIMux: 1472 expandRIPseudo(MI, SystemZ::CHI, SystemZ::CIH, false); 1473 return true; 1474 1475 case SystemZ::CFIMux: 1476 expandRIPseudo(MI, SystemZ::CFI, SystemZ::CIH, false); 1477 return true; 1478 1479 case SystemZ::CLFIMux: 1480 expandRIPseudo(MI, SystemZ::CLFI, SystemZ::CLIH, false); 1481 return true; 1482 1483 case SystemZ::CMux: 1484 expandRXYPseudo(MI, SystemZ::C, SystemZ::CHF); 1485 return true; 1486 1487 case SystemZ::CLMux: 1488 expandRXYPseudo(MI, SystemZ::CL, SystemZ::CLHF); 1489 return true; 1490 1491 case SystemZ::RISBMux: { 1492 bool DestIsHigh = SystemZ::isHighReg(MI.getOperand(0).getReg()); 1493 bool SrcIsHigh = SystemZ::isHighReg(MI.getOperand(2).getReg()); 1494 if (SrcIsHigh == DestIsHigh) 1495 MI.setDesc(get(DestIsHigh ? SystemZ::RISBHH : SystemZ::RISBLL)); 1496 else { 1497 MI.setDesc(get(DestIsHigh ? SystemZ::RISBHL : SystemZ::RISBLH)); 1498 MI.getOperand(5).setImm(MI.getOperand(5).getImm() ^ 32); 1499 } 1500 return true; 1501 } 1502 1503 case SystemZ::ADJDYNALLOC: 1504 splitAdjDynAlloc(MI); 1505 return true; 1506 1507 case TargetOpcode::LOAD_STACK_GUARD: 1508 expandLoadStackGuard(&MI); 1509 return true; 1510 1511 default: 1512 return false; 1513 } 1514 } 1515 1516 unsigned SystemZInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 1517 if (MI.isInlineAsm()) { 1518 const MachineFunction *MF = MI.getParent()->getParent(); 1519 const char *AsmStr = MI.getOperand(0).getSymbolName(); 1520 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo()); 1521 } 1522 else if (MI.getOpcode() == SystemZ::PATCHPOINT) 1523 return PatchPointOpers(&MI).getNumPatchBytes(); 1524 else if (MI.getOpcode() == SystemZ::STACKMAP) 1525 return MI.getOperand(1).getImm(); 1526 else if (MI.getOpcode() == SystemZ::FENTRY_CALL) 1527 return 6; 1528 1529 return MI.getDesc().getSize(); 1530 } 1531 1532 SystemZII::Branch 1533 SystemZInstrInfo::getBranchInfo(const MachineInstr &MI) const { 1534 switch (MI.getOpcode()) { 1535 case SystemZ::BR: 1536 case SystemZ::BI: 1537 case SystemZ::J: 1538 case SystemZ::JG: 1539 return SystemZII::Branch(SystemZII::BranchNormal, SystemZ::CCMASK_ANY, 1540 SystemZ::CCMASK_ANY, &MI.getOperand(0)); 1541 1542 case SystemZ::BRC: 1543 case SystemZ::BRCL: 1544 return SystemZII::Branch(SystemZII::BranchNormal, MI.getOperand(0).getImm(), 1545 MI.getOperand(1).getImm(), &MI.getOperand(2)); 1546 1547 case SystemZ::BRCT: 1548 case SystemZ::BRCTH: 1549 return SystemZII::Branch(SystemZII::BranchCT, SystemZ::CCMASK_ICMP, 1550 SystemZ::CCMASK_CMP_NE, &MI.getOperand(2)); 1551 1552 case SystemZ::BRCTG: 1553 return SystemZII::Branch(SystemZII::BranchCTG, SystemZ::CCMASK_ICMP, 1554 SystemZ::CCMASK_CMP_NE, &MI.getOperand(2)); 1555 1556 case SystemZ::CIJ: 1557 case SystemZ::CRJ: 1558 return SystemZII::Branch(SystemZII::BranchC, SystemZ::CCMASK_ICMP, 1559 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1560 1561 case SystemZ::CLIJ: 1562 case SystemZ::CLRJ: 1563 return SystemZII::Branch(SystemZII::BranchCL, SystemZ::CCMASK_ICMP, 1564 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1565 1566 case SystemZ::CGIJ: 1567 case SystemZ::CGRJ: 1568 return SystemZII::Branch(SystemZII::BranchCG, SystemZ::CCMASK_ICMP, 1569 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1570 1571 case SystemZ::CLGIJ: 1572 case SystemZ::CLGRJ: 1573 return SystemZII::Branch(SystemZII::BranchCLG, SystemZ::CCMASK_ICMP, 1574 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1575 1576 case SystemZ::INLINEASM_BR: 1577 // Don't try to analyze asm goto, so pass nullptr as branch target argument. 1578 return SystemZII::Branch(SystemZII::AsmGoto, 0, 0, nullptr); 1579 1580 default: 1581 llvm_unreachable("Unrecognized branch opcode"); 1582 } 1583 } 1584 1585 void SystemZInstrInfo::getLoadStoreOpcodes(const TargetRegisterClass *RC, 1586 unsigned &LoadOpcode, 1587 unsigned &StoreOpcode) const { 1588 if (RC == &SystemZ::GR32BitRegClass || RC == &SystemZ::ADDR32BitRegClass) { 1589 LoadOpcode = SystemZ::L; 1590 StoreOpcode = SystemZ::ST; 1591 } else if (RC == &SystemZ::GRH32BitRegClass) { 1592 LoadOpcode = SystemZ::LFH; 1593 StoreOpcode = SystemZ::STFH; 1594 } else if (RC == &SystemZ::GRX32BitRegClass) { 1595 LoadOpcode = SystemZ::LMux; 1596 StoreOpcode = SystemZ::STMux; 1597 } else if (RC == &SystemZ::GR64BitRegClass || 1598 RC == &SystemZ::ADDR64BitRegClass) { 1599 LoadOpcode = SystemZ::LG; 1600 StoreOpcode = SystemZ::STG; 1601 } else if (RC == &SystemZ::GR128BitRegClass || 1602 RC == &SystemZ::ADDR128BitRegClass) { 1603 LoadOpcode = SystemZ::L128; 1604 StoreOpcode = SystemZ::ST128; 1605 } else if (RC == &SystemZ::FP32BitRegClass) { 1606 LoadOpcode = SystemZ::LE; 1607 StoreOpcode = SystemZ::STE; 1608 } else if (RC == &SystemZ::FP64BitRegClass) { 1609 LoadOpcode = SystemZ::LD; 1610 StoreOpcode = SystemZ::STD; 1611 } else if (RC == &SystemZ::FP128BitRegClass) { 1612 LoadOpcode = SystemZ::LX; 1613 StoreOpcode = SystemZ::STX; 1614 } else if (RC == &SystemZ::VR32BitRegClass) { 1615 LoadOpcode = SystemZ::VL32; 1616 StoreOpcode = SystemZ::VST32; 1617 } else if (RC == &SystemZ::VR64BitRegClass) { 1618 LoadOpcode = SystemZ::VL64; 1619 StoreOpcode = SystemZ::VST64; 1620 } else if (RC == &SystemZ::VF128BitRegClass || 1621 RC == &SystemZ::VR128BitRegClass) { 1622 LoadOpcode = SystemZ::VL; 1623 StoreOpcode = SystemZ::VST; 1624 } else 1625 llvm_unreachable("Unsupported regclass to load or store"); 1626 } 1627 1628 unsigned SystemZInstrInfo::getOpcodeForOffset(unsigned Opcode, 1629 int64_t Offset) const { 1630 const MCInstrDesc &MCID = get(Opcode); 1631 int64_t Offset2 = (MCID.TSFlags & SystemZII::Is128Bit ? Offset + 8 : Offset); 1632 if (isUInt<12>(Offset) && isUInt<12>(Offset2)) { 1633 // Get the instruction to use for unsigned 12-bit displacements. 1634 int Disp12Opcode = SystemZ::getDisp12Opcode(Opcode); 1635 if (Disp12Opcode >= 0) 1636 return Disp12Opcode; 1637 1638 // All address-related instructions can use unsigned 12-bit 1639 // displacements. 1640 return Opcode; 1641 } 1642 if (isInt<20>(Offset) && isInt<20>(Offset2)) { 1643 // Get the instruction to use for signed 20-bit displacements. 1644 int Disp20Opcode = SystemZ::getDisp20Opcode(Opcode); 1645 if (Disp20Opcode >= 0) 1646 return Disp20Opcode; 1647 1648 // Check whether Opcode allows signed 20-bit displacements. 1649 if (MCID.TSFlags & SystemZII::Has20BitOffset) 1650 return Opcode; 1651 } 1652 return 0; 1653 } 1654 1655 bool SystemZInstrInfo::hasDisplacementPairInsn(unsigned Opcode) const { 1656 const MCInstrDesc &MCID = get(Opcode); 1657 if (MCID.TSFlags & SystemZII::Has20BitOffset) 1658 return SystemZ::getDisp12Opcode(Opcode) >= 0; 1659 return SystemZ::getDisp20Opcode(Opcode) >= 0; 1660 } 1661 1662 unsigned SystemZInstrInfo::getLoadAndTest(unsigned Opcode) const { 1663 switch (Opcode) { 1664 case SystemZ::L: return SystemZ::LT; 1665 case SystemZ::LY: return SystemZ::LT; 1666 case SystemZ::LG: return SystemZ::LTG; 1667 case SystemZ::LGF: return SystemZ::LTGF; 1668 case SystemZ::LR: return SystemZ::LTR; 1669 case SystemZ::LGFR: return SystemZ::LTGFR; 1670 case SystemZ::LGR: return SystemZ::LTGR; 1671 case SystemZ::LER: return SystemZ::LTEBR; 1672 case SystemZ::LDR: return SystemZ::LTDBR; 1673 case SystemZ::LXR: return SystemZ::LTXBR; 1674 case SystemZ::LCDFR: return SystemZ::LCDBR; 1675 case SystemZ::LPDFR: return SystemZ::LPDBR; 1676 case SystemZ::LNDFR: return SystemZ::LNDBR; 1677 case SystemZ::LCDFR_32: return SystemZ::LCEBR; 1678 case SystemZ::LPDFR_32: return SystemZ::LPEBR; 1679 case SystemZ::LNDFR_32: return SystemZ::LNEBR; 1680 // On zEC12 we prefer to use RISBGN. But if there is a chance to 1681 // actually use the condition code, we may turn it back into RISGB. 1682 // Note that RISBG is not really a "load-and-test" instruction, 1683 // but sets the same condition code values, so is OK to use here. 1684 case SystemZ::RISBGN: return SystemZ::RISBG; 1685 default: return 0; 1686 } 1687 } 1688 1689 // Return true if Mask matches the regexp 0*1+0*, given that zero masks 1690 // have already been filtered out. Store the first set bit in LSB and 1691 // the number of set bits in Length if so. 1692 static bool isStringOfOnes(uint64_t Mask, unsigned &LSB, unsigned &Length) { 1693 unsigned First = findFirstSet(Mask); 1694 uint64_t Top = (Mask >> First) + 1; 1695 if ((Top & -Top) == Top) { 1696 LSB = First; 1697 Length = findFirstSet(Top); 1698 return true; 1699 } 1700 return false; 1701 } 1702 1703 bool SystemZInstrInfo::isRxSBGMask(uint64_t Mask, unsigned BitSize, 1704 unsigned &Start, unsigned &End) const { 1705 // Reject trivial all-zero masks. 1706 Mask &= allOnes(BitSize); 1707 if (Mask == 0) 1708 return false; 1709 1710 // Handle the 1+0+ or 0+1+0* cases. Start then specifies the index of 1711 // the msb and End specifies the index of the lsb. 1712 unsigned LSB, Length; 1713 if (isStringOfOnes(Mask, LSB, Length)) { 1714 Start = 63 - (LSB + Length - 1); 1715 End = 63 - LSB; 1716 return true; 1717 } 1718 1719 // Handle the wrap-around 1+0+1+ cases. Start then specifies the msb 1720 // of the low 1s and End specifies the lsb of the high 1s. 1721 if (isStringOfOnes(Mask ^ allOnes(BitSize), LSB, Length)) { 1722 assert(LSB > 0 && "Bottom bit must be set"); 1723 assert(LSB + Length < BitSize && "Top bit must be set"); 1724 Start = 63 - (LSB - 1); 1725 End = 63 - (LSB + Length); 1726 return true; 1727 } 1728 1729 return false; 1730 } 1731 1732 unsigned SystemZInstrInfo::getFusedCompare(unsigned Opcode, 1733 SystemZII::FusedCompareType Type, 1734 const MachineInstr *MI) const { 1735 switch (Opcode) { 1736 case SystemZ::CHI: 1737 case SystemZ::CGHI: 1738 if (!(MI && isInt<8>(MI->getOperand(1).getImm()))) 1739 return 0; 1740 break; 1741 case SystemZ::CLFI: 1742 case SystemZ::CLGFI: 1743 if (!(MI && isUInt<8>(MI->getOperand(1).getImm()))) 1744 return 0; 1745 break; 1746 case SystemZ::CL: 1747 case SystemZ::CLG: 1748 if (!STI.hasMiscellaneousExtensions()) 1749 return 0; 1750 if (!(MI && MI->getOperand(3).getReg() == 0)) 1751 return 0; 1752 break; 1753 } 1754 switch (Type) { 1755 case SystemZII::CompareAndBranch: 1756 switch (Opcode) { 1757 case SystemZ::CR: 1758 return SystemZ::CRJ; 1759 case SystemZ::CGR: 1760 return SystemZ::CGRJ; 1761 case SystemZ::CHI: 1762 return SystemZ::CIJ; 1763 case SystemZ::CGHI: 1764 return SystemZ::CGIJ; 1765 case SystemZ::CLR: 1766 return SystemZ::CLRJ; 1767 case SystemZ::CLGR: 1768 return SystemZ::CLGRJ; 1769 case SystemZ::CLFI: 1770 return SystemZ::CLIJ; 1771 case SystemZ::CLGFI: 1772 return SystemZ::CLGIJ; 1773 default: 1774 return 0; 1775 } 1776 case SystemZII::CompareAndReturn: 1777 switch (Opcode) { 1778 case SystemZ::CR: 1779 return SystemZ::CRBReturn; 1780 case SystemZ::CGR: 1781 return SystemZ::CGRBReturn; 1782 case SystemZ::CHI: 1783 return SystemZ::CIBReturn; 1784 case SystemZ::CGHI: 1785 return SystemZ::CGIBReturn; 1786 case SystemZ::CLR: 1787 return SystemZ::CLRBReturn; 1788 case SystemZ::CLGR: 1789 return SystemZ::CLGRBReturn; 1790 case SystemZ::CLFI: 1791 return SystemZ::CLIBReturn; 1792 case SystemZ::CLGFI: 1793 return SystemZ::CLGIBReturn; 1794 default: 1795 return 0; 1796 } 1797 case SystemZII::CompareAndSibcall: 1798 switch (Opcode) { 1799 case SystemZ::CR: 1800 return SystemZ::CRBCall; 1801 case SystemZ::CGR: 1802 return SystemZ::CGRBCall; 1803 case SystemZ::CHI: 1804 return SystemZ::CIBCall; 1805 case SystemZ::CGHI: 1806 return SystemZ::CGIBCall; 1807 case SystemZ::CLR: 1808 return SystemZ::CLRBCall; 1809 case SystemZ::CLGR: 1810 return SystemZ::CLGRBCall; 1811 case SystemZ::CLFI: 1812 return SystemZ::CLIBCall; 1813 case SystemZ::CLGFI: 1814 return SystemZ::CLGIBCall; 1815 default: 1816 return 0; 1817 } 1818 case SystemZII::CompareAndTrap: 1819 switch (Opcode) { 1820 case SystemZ::CR: 1821 return SystemZ::CRT; 1822 case SystemZ::CGR: 1823 return SystemZ::CGRT; 1824 case SystemZ::CHI: 1825 return SystemZ::CIT; 1826 case SystemZ::CGHI: 1827 return SystemZ::CGIT; 1828 case SystemZ::CLR: 1829 return SystemZ::CLRT; 1830 case SystemZ::CLGR: 1831 return SystemZ::CLGRT; 1832 case SystemZ::CLFI: 1833 return SystemZ::CLFIT; 1834 case SystemZ::CLGFI: 1835 return SystemZ::CLGIT; 1836 case SystemZ::CL: 1837 return SystemZ::CLT; 1838 case SystemZ::CLG: 1839 return SystemZ::CLGT; 1840 default: 1841 return 0; 1842 } 1843 } 1844 return 0; 1845 } 1846 1847 bool SystemZInstrInfo:: 1848 prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const { 1849 assert(MBBI->isCompare() && MBBI->getOperand(0).isReg() && 1850 MBBI->getOperand(1).isReg() && !MBBI->mayLoad() && 1851 "Not a compare reg/reg."); 1852 1853 MachineBasicBlock *MBB = MBBI->getParent(); 1854 bool CCLive = true; 1855 SmallVector<MachineInstr *, 4> CCUsers; 1856 for (MachineBasicBlock::iterator Itr = std::next(MBBI); 1857 Itr != MBB->end(); ++Itr) { 1858 if (Itr->readsRegister(SystemZ::CC)) { 1859 unsigned Flags = Itr->getDesc().TSFlags; 1860 if ((Flags & SystemZII::CCMaskFirst) || (Flags & SystemZII::CCMaskLast)) 1861 CCUsers.push_back(&*Itr); 1862 else 1863 return false; 1864 } 1865 if (Itr->definesRegister(SystemZ::CC)) { 1866 CCLive = false; 1867 break; 1868 } 1869 } 1870 if (CCLive) { 1871 LivePhysRegs LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo()); 1872 LiveRegs.addLiveOuts(*MBB); 1873 if (LiveRegs.contains(SystemZ::CC)) 1874 return false; 1875 } 1876 1877 // Update all CC users. 1878 for (unsigned Idx = 0; Idx < CCUsers.size(); ++Idx) { 1879 unsigned Flags = CCUsers[Idx]->getDesc().TSFlags; 1880 unsigned FirstOpNum = ((Flags & SystemZII::CCMaskFirst) ? 1881 0 : CCUsers[Idx]->getNumExplicitOperands() - 2); 1882 MachineOperand &CCMaskMO = CCUsers[Idx]->getOperand(FirstOpNum + 1); 1883 unsigned NewCCMask = SystemZ::reverseCCMask(CCMaskMO.getImm()); 1884 CCMaskMO.setImm(NewCCMask); 1885 } 1886 1887 return true; 1888 } 1889 1890 unsigned SystemZ::reverseCCMask(unsigned CCMask) { 1891 return ((CCMask & SystemZ::CCMASK_CMP_EQ) | 1892 (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) | 1893 (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) | 1894 (CCMask & SystemZ::CCMASK_CMP_UO)); 1895 } 1896 1897 MachineBasicBlock *SystemZ::emitBlockAfter(MachineBasicBlock *MBB) { 1898 MachineFunction &MF = *MBB->getParent(); 1899 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 1900 MF.insert(std::next(MachineFunction::iterator(MBB)), NewMBB); 1901 return NewMBB; 1902 } 1903 1904 MachineBasicBlock *SystemZ::splitBlockAfter(MachineBasicBlock::iterator MI, 1905 MachineBasicBlock *MBB) { 1906 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 1907 NewMBB->splice(NewMBB->begin(), MBB, 1908 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 1909 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 1910 return NewMBB; 1911 } 1912 1913 MachineBasicBlock *SystemZ::splitBlockBefore(MachineBasicBlock::iterator MI, 1914 MachineBasicBlock *MBB) { 1915 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 1916 NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end()); 1917 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 1918 return NewMBB; 1919 } 1920 1921 unsigned SystemZInstrInfo::getLoadAndTrap(unsigned Opcode) const { 1922 if (!STI.hasLoadAndTrap()) 1923 return 0; 1924 switch (Opcode) { 1925 case SystemZ::L: 1926 case SystemZ::LY: 1927 return SystemZ::LAT; 1928 case SystemZ::LG: 1929 return SystemZ::LGAT; 1930 case SystemZ::LFH: 1931 return SystemZ::LFHAT; 1932 case SystemZ::LLGF: 1933 return SystemZ::LLGFAT; 1934 case SystemZ::LLGT: 1935 return SystemZ::LLGTAT; 1936 } 1937 return 0; 1938 } 1939 1940 void SystemZInstrInfo::loadImmediate(MachineBasicBlock &MBB, 1941 MachineBasicBlock::iterator MBBI, 1942 unsigned Reg, uint64_t Value) const { 1943 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 1944 unsigned Opcode = 0; 1945 if (isInt<16>(Value)) 1946 Opcode = SystemZ::LGHI; 1947 else if (SystemZ::isImmLL(Value)) 1948 Opcode = SystemZ::LLILL; 1949 else if (SystemZ::isImmLH(Value)) { 1950 Opcode = SystemZ::LLILH; 1951 Value >>= 16; 1952 } 1953 else if (isInt<32>(Value)) 1954 Opcode = SystemZ::LGFI; 1955 if (Opcode) { 1956 BuildMI(MBB, MBBI, DL, get(Opcode), Reg).addImm(Value); 1957 return; 1958 } 1959 1960 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1961 assert (MRI.isSSA() && "Huge values only handled before reg-alloc ."); 1962 Register Reg0 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 1963 Register Reg1 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 1964 BuildMI(MBB, MBBI, DL, get(SystemZ::IMPLICIT_DEF), Reg0); 1965 BuildMI(MBB, MBBI, DL, get(SystemZ::IIHF64), Reg1) 1966 .addReg(Reg0).addImm(Value >> 32); 1967 BuildMI(MBB, MBBI, DL, get(SystemZ::IILF64), Reg) 1968 .addReg(Reg1).addImm(Value & ((uint64_t(1) << 32) - 1)); 1969 } 1970 1971 bool SystemZInstrInfo::verifyInstruction(const MachineInstr &MI, 1972 StringRef &ErrInfo) const { 1973 const MCInstrDesc &MCID = MI.getDesc(); 1974 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { 1975 if (I >= MCID.getNumOperands()) 1976 break; 1977 const MachineOperand &Op = MI.getOperand(I); 1978 const MCOperandInfo &MCOI = MCID.OpInfo[I]; 1979 // Addressing modes have register and immediate operands. Op should be a 1980 // register (or frame index) operand if MCOI.RegClass contains a valid 1981 // register class, or an immediate otherwise. 1982 if (MCOI.OperandType == MCOI::OPERAND_MEMORY && 1983 ((MCOI.RegClass != -1 && !Op.isReg() && !Op.isFI()) || 1984 (MCOI.RegClass == -1 && !Op.isImm()))) { 1985 ErrInfo = "Addressing mode operands corrupt!"; 1986 return false; 1987 } 1988 } 1989 1990 return true; 1991 } 1992 1993 bool SystemZInstrInfo:: 1994 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, 1995 const MachineInstr &MIb) const { 1996 1997 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) 1998 return false; 1999 2000 // If mem-operands show that the same address Value is used by both 2001 // instructions, check for non-overlapping offsets and widths. Not 2002 // sure if a register based analysis would be an improvement... 2003 2004 MachineMemOperand *MMOa = *MIa.memoperands_begin(); 2005 MachineMemOperand *MMOb = *MIb.memoperands_begin(); 2006 const Value *VALa = MMOa->getValue(); 2007 const Value *VALb = MMOb->getValue(); 2008 bool SameVal = (VALa && VALb && (VALa == VALb)); 2009 if (!SameVal) { 2010 const PseudoSourceValue *PSVa = MMOa->getPseudoValue(); 2011 const PseudoSourceValue *PSVb = MMOb->getPseudoValue(); 2012 if (PSVa && PSVb && (PSVa == PSVb)) 2013 SameVal = true; 2014 } 2015 if (SameVal) { 2016 int OffsetA = MMOa->getOffset(), OffsetB = MMOb->getOffset(); 2017 int WidthA = MMOa->getSize(), WidthB = MMOb->getSize(); 2018 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; 2019 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; 2020 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 2021 if (LowOffset + LowWidth <= HighOffset) 2022 return true; 2023 } 2024 2025 return false; 2026 } 2027