1 //===-- SystemZInstrInfo.cpp - SystemZ instruction information ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the SystemZ implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "SystemZInstrInfo.h" 14 #include "MCTargetDesc/SystemZMCTargetDesc.h" 15 #include "SystemZ.h" 16 #include "SystemZInstrBuilder.h" 17 #include "SystemZSubtarget.h" 18 #include "llvm/ADT/Statistic.h" 19 #include "llvm/CodeGen/LiveInterval.h" 20 #include "llvm/CodeGen/LiveIntervals.h" 21 #include "llvm/CodeGen/LiveVariables.h" 22 #include "llvm/CodeGen/MachineBasicBlock.h" 23 #include "llvm/CodeGen/MachineFrameInfo.h" 24 #include "llvm/CodeGen/MachineFunction.h" 25 #include "llvm/CodeGen/MachineInstr.h" 26 #include "llvm/CodeGen/MachineMemOperand.h" 27 #include "llvm/CodeGen/MachineOperand.h" 28 #include "llvm/CodeGen/MachineRegisterInfo.h" 29 #include "llvm/CodeGen/SlotIndexes.h" 30 #include "llvm/CodeGen/TargetInstrInfo.h" 31 #include "llvm/CodeGen/TargetSubtargetInfo.h" 32 #include "llvm/MC/MCInstrDesc.h" 33 #include "llvm/MC/MCRegisterInfo.h" 34 #include "llvm/Support/BranchProbability.h" 35 #include "llvm/Support/ErrorHandling.h" 36 #include "llvm/Support/MathExtras.h" 37 #include "llvm/Target/TargetMachine.h" 38 #include <cassert> 39 #include <cstdint> 40 #include <iterator> 41 42 using namespace llvm; 43 44 #define GET_INSTRINFO_CTOR_DTOR 45 #define GET_INSTRMAP_INFO 46 #include "SystemZGenInstrInfo.inc" 47 48 #define DEBUG_TYPE "systemz-II" 49 50 // Return a mask with Count low bits set. 51 static uint64_t allOnes(unsigned int Count) { 52 return Count == 0 ? 0 : (uint64_t(1) << (Count - 1) << 1) - 1; 53 } 54 55 // Pin the vtable to this file. 56 void SystemZInstrInfo::anchor() {} 57 58 SystemZInstrInfo::SystemZInstrInfo(SystemZSubtarget &sti) 59 : SystemZGenInstrInfo(SystemZ::ADJCALLSTACKDOWN, SystemZ::ADJCALLSTACKUP), 60 RI(), STI(sti) { 61 } 62 63 // MI is a 128-bit load or store. Split it into two 64-bit loads or stores, 64 // each having the opcode given by NewOpcode. 65 void SystemZInstrInfo::splitMove(MachineBasicBlock::iterator MI, 66 unsigned NewOpcode) const { 67 MachineBasicBlock *MBB = MI->getParent(); 68 MachineFunction &MF = *MBB->getParent(); 69 70 // Get two load or store instructions. Use the original instruction for one 71 // of them (arbitrarily the second here) and create a clone for the other. 72 MachineInstr *EarlierMI = MF.CloneMachineInstr(&*MI); 73 MBB->insert(MI, EarlierMI); 74 75 // Set up the two 64-bit registers and remember super reg and its flags. 76 MachineOperand &HighRegOp = EarlierMI->getOperand(0); 77 MachineOperand &LowRegOp = MI->getOperand(0); 78 Register Reg128 = LowRegOp.getReg(); 79 unsigned Reg128Killed = getKillRegState(LowRegOp.isKill()); 80 unsigned Reg128Undef = getUndefRegState(LowRegOp.isUndef()); 81 HighRegOp.setReg(RI.getSubReg(HighRegOp.getReg(), SystemZ::subreg_h64)); 82 LowRegOp.setReg(RI.getSubReg(LowRegOp.getReg(), SystemZ::subreg_l64)); 83 84 if (MI->mayStore()) { 85 // Add implicit uses of the super register in case one of the subregs is 86 // undefined. We could track liveness and skip storing an undefined 87 // subreg, but this is hopefully rare (discovered with llvm-stress). 88 // If Reg128 was killed, set kill flag on MI. 89 unsigned Reg128UndefImpl = (Reg128Undef | RegState::Implicit); 90 MachineInstrBuilder(MF, EarlierMI).addReg(Reg128, Reg128UndefImpl); 91 MachineInstrBuilder(MF, MI).addReg(Reg128, (Reg128UndefImpl | Reg128Killed)); 92 } 93 94 // The address in the first (high) instruction is already correct. 95 // Adjust the offset in the second (low) instruction. 96 MachineOperand &HighOffsetOp = EarlierMI->getOperand(2); 97 MachineOperand &LowOffsetOp = MI->getOperand(2); 98 LowOffsetOp.setImm(LowOffsetOp.getImm() + 8); 99 100 // Clear the kill flags on the registers in the first instruction. 101 if (EarlierMI->getOperand(0).isReg() && EarlierMI->getOperand(0).isUse()) 102 EarlierMI->getOperand(0).setIsKill(false); 103 EarlierMI->getOperand(1).setIsKill(false); 104 EarlierMI->getOperand(3).setIsKill(false); 105 106 // Set the opcodes. 107 unsigned HighOpcode = getOpcodeForOffset(NewOpcode, HighOffsetOp.getImm()); 108 unsigned LowOpcode = getOpcodeForOffset(NewOpcode, LowOffsetOp.getImm()); 109 assert(HighOpcode && LowOpcode && "Both offsets should be in range"); 110 111 EarlierMI->setDesc(get(HighOpcode)); 112 MI->setDesc(get(LowOpcode)); 113 } 114 115 // Split ADJDYNALLOC instruction MI. 116 void SystemZInstrInfo::splitAdjDynAlloc(MachineBasicBlock::iterator MI) const { 117 MachineBasicBlock *MBB = MI->getParent(); 118 MachineFunction &MF = *MBB->getParent(); 119 MachineFrameInfo &MFFrame = MF.getFrameInfo(); 120 MachineOperand &OffsetMO = MI->getOperand(2); 121 122 uint64_t Offset = (MFFrame.getMaxCallFrameSize() + 123 SystemZMC::CallFrameSize + 124 OffsetMO.getImm()); 125 unsigned NewOpcode = getOpcodeForOffset(SystemZ::LA, Offset); 126 assert(NewOpcode && "No support for huge argument lists yet"); 127 MI->setDesc(get(NewOpcode)); 128 OffsetMO.setImm(Offset); 129 } 130 131 // MI is an RI-style pseudo instruction. Replace it with LowOpcode 132 // if the first operand is a low GR32 and HighOpcode if the first operand 133 // is a high GR32. ConvertHigh is true if LowOpcode takes a signed operand 134 // and HighOpcode takes an unsigned 32-bit operand. In those cases, 135 // MI has the same kind of operand as LowOpcode, so needs to be converted 136 // if HighOpcode is used. 137 void SystemZInstrInfo::expandRIPseudo(MachineInstr &MI, unsigned LowOpcode, 138 unsigned HighOpcode, 139 bool ConvertHigh) const { 140 Register Reg = MI.getOperand(0).getReg(); 141 bool IsHigh = SystemZ::isHighReg(Reg); 142 MI.setDesc(get(IsHigh ? HighOpcode : LowOpcode)); 143 if (IsHigh && ConvertHigh) 144 MI.getOperand(1).setImm(uint32_t(MI.getOperand(1).getImm())); 145 } 146 147 // MI is a three-operand RIE-style pseudo instruction. Replace it with 148 // LowOpcodeK if the registers are both low GR32s, otherwise use a move 149 // followed by HighOpcode or LowOpcode, depending on whether the target 150 // is a high or low GR32. 151 void SystemZInstrInfo::expandRIEPseudo(MachineInstr &MI, unsigned LowOpcode, 152 unsigned LowOpcodeK, 153 unsigned HighOpcode) const { 154 Register DestReg = MI.getOperand(0).getReg(); 155 Register SrcReg = MI.getOperand(1).getReg(); 156 bool DestIsHigh = SystemZ::isHighReg(DestReg); 157 bool SrcIsHigh = SystemZ::isHighReg(SrcReg); 158 if (!DestIsHigh && !SrcIsHigh) 159 MI.setDesc(get(LowOpcodeK)); 160 else { 161 if (DestReg != SrcReg) { 162 emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, SrcReg, 163 SystemZ::LR, 32, MI.getOperand(1).isKill(), 164 MI.getOperand(1).isUndef()); 165 MI.getOperand(1).setReg(DestReg); 166 } 167 MI.setDesc(get(DestIsHigh ? HighOpcode : LowOpcode)); 168 MI.tieOperands(0, 1); 169 } 170 } 171 172 // MI is an RXY-style pseudo instruction. Replace it with LowOpcode 173 // if the first operand is a low GR32 and HighOpcode if the first operand 174 // is a high GR32. 175 void SystemZInstrInfo::expandRXYPseudo(MachineInstr &MI, unsigned LowOpcode, 176 unsigned HighOpcode) const { 177 Register Reg = MI.getOperand(0).getReg(); 178 unsigned Opcode = getOpcodeForOffset( 179 SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode, 180 MI.getOperand(2).getImm()); 181 MI.setDesc(get(Opcode)); 182 } 183 184 // MI is a load-on-condition pseudo instruction with a single register 185 // (source or destination) operand. Replace it with LowOpcode if the 186 // register is a low GR32 and HighOpcode if the register is a high GR32. 187 void SystemZInstrInfo::expandLOCPseudo(MachineInstr &MI, unsigned LowOpcode, 188 unsigned HighOpcode) const { 189 Register Reg = MI.getOperand(0).getReg(); 190 unsigned Opcode = SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode; 191 MI.setDesc(get(Opcode)); 192 } 193 194 // MI is an RR-style pseudo instruction that zero-extends the low Size bits 195 // of one GRX32 into another. Replace it with LowOpcode if both operands 196 // are low registers, otherwise use RISB[LH]G. 197 void SystemZInstrInfo::expandZExtPseudo(MachineInstr &MI, unsigned LowOpcode, 198 unsigned Size) const { 199 MachineInstrBuilder MIB = 200 emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), 201 MI.getOperand(0).getReg(), MI.getOperand(1).getReg(), LowOpcode, 202 Size, MI.getOperand(1).isKill(), MI.getOperand(1).isUndef()); 203 204 // Keep the remaining operands as-is. 205 for (unsigned I = 2; I < MI.getNumOperands(); ++I) 206 MIB.add(MI.getOperand(I)); 207 208 MI.eraseFromParent(); 209 } 210 211 void SystemZInstrInfo::expandLoadStackGuard(MachineInstr *MI) const { 212 MachineBasicBlock *MBB = MI->getParent(); 213 MachineFunction &MF = *MBB->getParent(); 214 const Register Reg64 = MI->getOperand(0).getReg(); 215 const Register Reg32 = RI.getSubReg(Reg64, SystemZ::subreg_l32); 216 217 // EAR can only load the low subregister so us a shift for %a0 to produce 218 // the GR containing %a0 and %a1. 219 220 // ear <reg>, %a0 221 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32) 222 .addReg(SystemZ::A0) 223 .addReg(Reg64, RegState::ImplicitDefine); 224 225 // sllg <reg>, <reg>, 32 226 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::SLLG), Reg64) 227 .addReg(Reg64) 228 .addReg(0) 229 .addImm(32); 230 231 // ear <reg>, %a1 232 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32) 233 .addReg(SystemZ::A1); 234 235 // lg <reg>, 40(<reg>) 236 MI->setDesc(get(SystemZ::LG)); 237 MachineInstrBuilder(MF, MI).addReg(Reg64).addImm(40).addReg(0); 238 } 239 240 // Emit a zero-extending move from 32-bit GPR SrcReg to 32-bit GPR 241 // DestReg before MBBI in MBB. Use LowLowOpcode when both DestReg and SrcReg 242 // are low registers, otherwise use RISB[LH]G. Size is the number of bits 243 // taken from the low end of SrcReg (8 for LLCR, 16 for LLHR and 32 for LR). 244 // KillSrc is true if this move is the last use of SrcReg. 245 MachineInstrBuilder 246 SystemZInstrInfo::emitGRX32Move(MachineBasicBlock &MBB, 247 MachineBasicBlock::iterator MBBI, 248 const DebugLoc &DL, unsigned DestReg, 249 unsigned SrcReg, unsigned LowLowOpcode, 250 unsigned Size, bool KillSrc, 251 bool UndefSrc) const { 252 unsigned Opcode; 253 bool DestIsHigh = SystemZ::isHighReg(DestReg); 254 bool SrcIsHigh = SystemZ::isHighReg(SrcReg); 255 if (DestIsHigh && SrcIsHigh) 256 Opcode = SystemZ::RISBHH; 257 else if (DestIsHigh && !SrcIsHigh) 258 Opcode = SystemZ::RISBHL; 259 else if (!DestIsHigh && SrcIsHigh) 260 Opcode = SystemZ::RISBLH; 261 else { 262 return BuildMI(MBB, MBBI, DL, get(LowLowOpcode), DestReg) 263 .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)); 264 } 265 unsigned Rotate = (DestIsHigh != SrcIsHigh ? 32 : 0); 266 return BuildMI(MBB, MBBI, DL, get(Opcode), DestReg) 267 .addReg(DestReg, RegState::Undef) 268 .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)) 269 .addImm(32 - Size).addImm(128 + 31).addImm(Rotate); 270 } 271 272 MachineInstr *SystemZInstrInfo::commuteInstructionImpl(MachineInstr &MI, 273 bool NewMI, 274 unsigned OpIdx1, 275 unsigned OpIdx2) const { 276 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { 277 if (NewMI) 278 return *MI.getParent()->getParent()->CloneMachineInstr(&MI); 279 return MI; 280 }; 281 282 switch (MI.getOpcode()) { 283 case SystemZ::SELRMux: 284 case SystemZ::SELFHR: 285 case SystemZ::SELR: 286 case SystemZ::SELGR: 287 case SystemZ::LOCRMux: 288 case SystemZ::LOCFHR: 289 case SystemZ::LOCR: 290 case SystemZ::LOCGR: { 291 auto &WorkingMI = cloneIfNew(MI); 292 // Invert condition. 293 unsigned CCValid = WorkingMI.getOperand(3).getImm(); 294 unsigned CCMask = WorkingMI.getOperand(4).getImm(); 295 WorkingMI.getOperand(4).setImm(CCMask ^ CCValid); 296 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 297 OpIdx1, OpIdx2); 298 } 299 default: 300 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 301 } 302 } 303 304 // If MI is a simple load or store for a frame object, return the register 305 // it loads or stores and set FrameIndex to the index of the frame object. 306 // Return 0 otherwise. 307 // 308 // Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores. 309 static int isSimpleMove(const MachineInstr &MI, int &FrameIndex, 310 unsigned Flag) { 311 const MCInstrDesc &MCID = MI.getDesc(); 312 if ((MCID.TSFlags & Flag) && MI.getOperand(1).isFI() && 313 MI.getOperand(2).getImm() == 0 && MI.getOperand(3).getReg() == 0) { 314 FrameIndex = MI.getOperand(1).getIndex(); 315 return MI.getOperand(0).getReg(); 316 } 317 return 0; 318 } 319 320 unsigned SystemZInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 321 int &FrameIndex) const { 322 return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXLoad); 323 } 324 325 unsigned SystemZInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 326 int &FrameIndex) const { 327 return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXStore); 328 } 329 330 bool SystemZInstrInfo::isStackSlotCopy(const MachineInstr &MI, 331 int &DestFrameIndex, 332 int &SrcFrameIndex) const { 333 // Check for MVC 0(Length,FI1),0(FI2) 334 const MachineFrameInfo &MFI = MI.getParent()->getParent()->getFrameInfo(); 335 if (MI.getOpcode() != SystemZ::MVC || !MI.getOperand(0).isFI() || 336 MI.getOperand(1).getImm() != 0 || !MI.getOperand(3).isFI() || 337 MI.getOperand(4).getImm() != 0) 338 return false; 339 340 // Check that Length covers the full slots. 341 int64_t Length = MI.getOperand(2).getImm(); 342 unsigned FI1 = MI.getOperand(0).getIndex(); 343 unsigned FI2 = MI.getOperand(3).getIndex(); 344 if (MFI.getObjectSize(FI1) != Length || 345 MFI.getObjectSize(FI2) != Length) 346 return false; 347 348 DestFrameIndex = FI1; 349 SrcFrameIndex = FI2; 350 return true; 351 } 352 353 bool SystemZInstrInfo::analyzeBranch(MachineBasicBlock &MBB, 354 MachineBasicBlock *&TBB, 355 MachineBasicBlock *&FBB, 356 SmallVectorImpl<MachineOperand> &Cond, 357 bool AllowModify) const { 358 // Most of the code and comments here are boilerplate. 359 360 // Start from the bottom of the block and work up, examining the 361 // terminator instructions. 362 MachineBasicBlock::iterator I = MBB.end(); 363 while (I != MBB.begin()) { 364 --I; 365 if (I->isDebugInstr()) 366 continue; 367 368 // Working from the bottom, when we see a non-terminator instruction, we're 369 // done. 370 if (!isUnpredicatedTerminator(*I)) 371 break; 372 373 // A terminator that isn't a branch can't easily be handled by this 374 // analysis. 375 if (!I->isBranch()) 376 return true; 377 378 // Can't handle indirect branches. 379 SystemZII::Branch Branch(getBranchInfo(*I)); 380 if (!Branch.hasMBBTarget()) 381 return true; 382 383 // Punt on compound branches. 384 if (Branch.Type != SystemZII::BranchNormal) 385 return true; 386 387 if (Branch.CCMask == SystemZ::CCMASK_ANY) { 388 // Handle unconditional branches. 389 if (!AllowModify) { 390 TBB = Branch.getMBBTarget(); 391 continue; 392 } 393 394 // If the block has any instructions after a JMP, delete them. 395 while (std::next(I) != MBB.end()) 396 std::next(I)->eraseFromParent(); 397 398 Cond.clear(); 399 FBB = nullptr; 400 401 // Delete the JMP if it's equivalent to a fall-through. 402 if (MBB.isLayoutSuccessor(Branch.getMBBTarget())) { 403 TBB = nullptr; 404 I->eraseFromParent(); 405 I = MBB.end(); 406 continue; 407 } 408 409 // TBB is used to indicate the unconditinal destination. 410 TBB = Branch.getMBBTarget(); 411 continue; 412 } 413 414 // Working from the bottom, handle the first conditional branch. 415 if (Cond.empty()) { 416 // FIXME: add X86-style branch swap 417 FBB = TBB; 418 TBB = Branch.getMBBTarget(); 419 Cond.push_back(MachineOperand::CreateImm(Branch.CCValid)); 420 Cond.push_back(MachineOperand::CreateImm(Branch.CCMask)); 421 continue; 422 } 423 424 // Handle subsequent conditional branches. 425 assert(Cond.size() == 2 && TBB && "Should have seen a conditional branch"); 426 427 // Only handle the case where all conditional branches branch to the same 428 // destination. 429 if (TBB != Branch.getMBBTarget()) 430 return true; 431 432 // If the conditions are the same, we can leave them alone. 433 unsigned OldCCValid = Cond[0].getImm(); 434 unsigned OldCCMask = Cond[1].getImm(); 435 if (OldCCValid == Branch.CCValid && OldCCMask == Branch.CCMask) 436 continue; 437 438 // FIXME: Try combining conditions like X86 does. Should be easy on Z! 439 return false; 440 } 441 442 return false; 443 } 444 445 unsigned SystemZInstrInfo::removeBranch(MachineBasicBlock &MBB, 446 int *BytesRemoved) const { 447 assert(!BytesRemoved && "code size not handled"); 448 449 // Most of the code and comments here are boilerplate. 450 MachineBasicBlock::iterator I = MBB.end(); 451 unsigned Count = 0; 452 453 while (I != MBB.begin()) { 454 --I; 455 if (I->isDebugInstr()) 456 continue; 457 if (!I->isBranch()) 458 break; 459 if (!getBranchInfo(*I).hasMBBTarget()) 460 break; 461 // Remove the branch. 462 I->eraseFromParent(); 463 I = MBB.end(); 464 ++Count; 465 } 466 467 return Count; 468 } 469 470 bool SystemZInstrInfo:: 471 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { 472 assert(Cond.size() == 2 && "Invalid condition"); 473 Cond[1].setImm(Cond[1].getImm() ^ Cond[0].getImm()); 474 return false; 475 } 476 477 unsigned SystemZInstrInfo::insertBranch(MachineBasicBlock &MBB, 478 MachineBasicBlock *TBB, 479 MachineBasicBlock *FBB, 480 ArrayRef<MachineOperand> Cond, 481 const DebugLoc &DL, 482 int *BytesAdded) const { 483 // In this function we output 32-bit branches, which should always 484 // have enough range. They can be shortened and relaxed by later code 485 // in the pipeline, if desired. 486 487 // Shouldn't be a fall through. 488 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 489 assert((Cond.size() == 2 || Cond.size() == 0) && 490 "SystemZ branch conditions have one component!"); 491 assert(!BytesAdded && "code size not handled"); 492 493 if (Cond.empty()) { 494 // Unconditional branch? 495 assert(!FBB && "Unconditional branch with multiple successors!"); 496 BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(TBB); 497 return 1; 498 } 499 500 // Conditional branch. 501 unsigned Count = 0; 502 unsigned CCValid = Cond[0].getImm(); 503 unsigned CCMask = Cond[1].getImm(); 504 BuildMI(&MBB, DL, get(SystemZ::BRC)) 505 .addImm(CCValid).addImm(CCMask).addMBB(TBB); 506 ++Count; 507 508 if (FBB) { 509 // Two-way Conditional branch. Insert the second branch. 510 BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(FBB); 511 ++Count; 512 } 513 return Count; 514 } 515 516 bool SystemZInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 517 Register &SrcReg2, int &Mask, 518 int &Value) const { 519 assert(MI.isCompare() && "Caller should have checked for a comparison"); 520 521 if (MI.getNumExplicitOperands() == 2 && MI.getOperand(0).isReg() && 522 MI.getOperand(1).isImm()) { 523 SrcReg = MI.getOperand(0).getReg(); 524 SrcReg2 = 0; 525 Value = MI.getOperand(1).getImm(); 526 Mask = ~0; 527 return true; 528 } 529 530 return false; 531 } 532 533 bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 534 ArrayRef<MachineOperand> Pred, 535 Register DstReg, Register TrueReg, 536 Register FalseReg, int &CondCycles, 537 int &TrueCycles, 538 int &FalseCycles) const { 539 // Not all subtargets have LOCR instructions. 540 if (!STI.hasLoadStoreOnCond()) 541 return false; 542 if (Pred.size() != 2) 543 return false; 544 545 // Check register classes. 546 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 547 const TargetRegisterClass *RC = 548 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); 549 if (!RC) 550 return false; 551 552 // We have LOCR instructions for 32 and 64 bit general purpose registers. 553 if ((STI.hasLoadStoreOnCond2() && 554 SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) || 555 SystemZ::GR32BitRegClass.hasSubClassEq(RC) || 556 SystemZ::GR64BitRegClass.hasSubClassEq(RC)) { 557 CondCycles = 2; 558 TrueCycles = 2; 559 FalseCycles = 2; 560 return true; 561 } 562 563 // Can't do anything else. 564 return false; 565 } 566 567 void SystemZInstrInfo::insertSelect(MachineBasicBlock &MBB, 568 MachineBasicBlock::iterator I, 569 const DebugLoc &DL, Register DstReg, 570 ArrayRef<MachineOperand> Pred, 571 Register TrueReg, 572 Register FalseReg) const { 573 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 574 const TargetRegisterClass *RC = MRI.getRegClass(DstReg); 575 576 assert(Pred.size() == 2 && "Invalid condition"); 577 unsigned CCValid = Pred[0].getImm(); 578 unsigned CCMask = Pred[1].getImm(); 579 580 unsigned Opc; 581 if (SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) { 582 if (STI.hasMiscellaneousExtensions3()) 583 Opc = SystemZ::SELRMux; 584 else if (STI.hasLoadStoreOnCond2()) 585 Opc = SystemZ::LOCRMux; 586 else { 587 Opc = SystemZ::LOCR; 588 MRI.constrainRegClass(DstReg, &SystemZ::GR32BitRegClass); 589 Register TReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); 590 Register FReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); 591 BuildMI(MBB, I, DL, get(TargetOpcode::COPY), TReg).addReg(TrueReg); 592 BuildMI(MBB, I, DL, get(TargetOpcode::COPY), FReg).addReg(FalseReg); 593 TrueReg = TReg; 594 FalseReg = FReg; 595 } 596 } else if (SystemZ::GR64BitRegClass.hasSubClassEq(RC)) { 597 if (STI.hasMiscellaneousExtensions3()) 598 Opc = SystemZ::SELGR; 599 else 600 Opc = SystemZ::LOCGR; 601 } else 602 llvm_unreachable("Invalid register class"); 603 604 BuildMI(MBB, I, DL, get(Opc), DstReg) 605 .addReg(FalseReg).addReg(TrueReg) 606 .addImm(CCValid).addImm(CCMask); 607 } 608 609 bool SystemZInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 610 Register Reg, 611 MachineRegisterInfo *MRI) const { 612 unsigned DefOpc = DefMI.getOpcode(); 613 if (DefOpc != SystemZ::LHIMux && DefOpc != SystemZ::LHI && 614 DefOpc != SystemZ::LGHI) 615 return false; 616 if (DefMI.getOperand(0).getReg() != Reg) 617 return false; 618 int32_t ImmVal = (int32_t)DefMI.getOperand(1).getImm(); 619 620 unsigned UseOpc = UseMI.getOpcode(); 621 unsigned NewUseOpc; 622 unsigned UseIdx; 623 int CommuteIdx = -1; 624 bool TieOps = false; 625 switch (UseOpc) { 626 case SystemZ::SELRMux: 627 TieOps = true; 628 LLVM_FALLTHROUGH; 629 case SystemZ::LOCRMux: 630 if (!STI.hasLoadStoreOnCond2()) 631 return false; 632 NewUseOpc = SystemZ::LOCHIMux; 633 if (UseMI.getOperand(2).getReg() == Reg) 634 UseIdx = 2; 635 else if (UseMI.getOperand(1).getReg() == Reg) 636 UseIdx = 2, CommuteIdx = 1; 637 else 638 return false; 639 break; 640 case SystemZ::SELGR: 641 TieOps = true; 642 LLVM_FALLTHROUGH; 643 case SystemZ::LOCGR: 644 if (!STI.hasLoadStoreOnCond2()) 645 return false; 646 NewUseOpc = SystemZ::LOCGHI; 647 if (UseMI.getOperand(2).getReg() == Reg) 648 UseIdx = 2; 649 else if (UseMI.getOperand(1).getReg() == Reg) 650 UseIdx = 2, CommuteIdx = 1; 651 else 652 return false; 653 break; 654 default: 655 return false; 656 } 657 658 if (CommuteIdx != -1) 659 if (!commuteInstruction(UseMI, false, CommuteIdx, UseIdx)) 660 return false; 661 662 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 663 UseMI.setDesc(get(NewUseOpc)); 664 if (TieOps) 665 UseMI.tieOperands(0, 1); 666 UseMI.getOperand(UseIdx).ChangeToImmediate(ImmVal); 667 if (DeleteDef) 668 DefMI.eraseFromParent(); 669 670 return true; 671 } 672 673 bool SystemZInstrInfo::isPredicable(const MachineInstr &MI) const { 674 unsigned Opcode = MI.getOpcode(); 675 if (Opcode == SystemZ::Return || 676 Opcode == SystemZ::Trap || 677 Opcode == SystemZ::CallJG || 678 Opcode == SystemZ::CallBR) 679 return true; 680 return false; 681 } 682 683 bool SystemZInstrInfo:: 684 isProfitableToIfCvt(MachineBasicBlock &MBB, 685 unsigned NumCycles, unsigned ExtraPredCycles, 686 BranchProbability Probability) const { 687 // Avoid using conditional returns at the end of a loop (since then 688 // we'd need to emit an unconditional branch to the beginning anyway, 689 // making the loop body longer). This doesn't apply for low-probability 690 // loops (eg. compare-and-swap retry), so just decide based on branch 691 // probability instead of looping structure. 692 // However, since Compare and Trap instructions cost the same as a regular 693 // Compare instruction, we should allow the if conversion to convert this 694 // into a Conditional Compare regardless of the branch probability. 695 if (MBB.getLastNonDebugInstr()->getOpcode() != SystemZ::Trap && 696 MBB.succ_empty() && Probability < BranchProbability(1, 8)) 697 return false; 698 // For now only convert single instructions. 699 return NumCycles == 1; 700 } 701 702 bool SystemZInstrInfo:: 703 isProfitableToIfCvt(MachineBasicBlock &TMBB, 704 unsigned NumCyclesT, unsigned ExtraPredCyclesT, 705 MachineBasicBlock &FMBB, 706 unsigned NumCyclesF, unsigned ExtraPredCyclesF, 707 BranchProbability Probability) const { 708 // For now avoid converting mutually-exclusive cases. 709 return false; 710 } 711 712 bool SystemZInstrInfo:: 713 isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, 714 BranchProbability Probability) const { 715 // For now only duplicate single instructions. 716 return NumCycles == 1; 717 } 718 719 bool SystemZInstrInfo::PredicateInstruction( 720 MachineInstr &MI, ArrayRef<MachineOperand> Pred) const { 721 assert(Pred.size() == 2 && "Invalid condition"); 722 unsigned CCValid = Pred[0].getImm(); 723 unsigned CCMask = Pred[1].getImm(); 724 assert(CCMask > 0 && CCMask < 15 && "Invalid predicate"); 725 unsigned Opcode = MI.getOpcode(); 726 if (Opcode == SystemZ::Trap) { 727 MI.setDesc(get(SystemZ::CondTrap)); 728 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 729 .addImm(CCValid).addImm(CCMask) 730 .addReg(SystemZ::CC, RegState::Implicit); 731 return true; 732 } 733 if (Opcode == SystemZ::Return) { 734 MI.setDesc(get(SystemZ::CondReturn)); 735 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 736 .addImm(CCValid).addImm(CCMask) 737 .addReg(SystemZ::CC, RegState::Implicit); 738 return true; 739 } 740 if (Opcode == SystemZ::CallJG) { 741 MachineOperand FirstOp = MI.getOperand(0); 742 const uint32_t *RegMask = MI.getOperand(1).getRegMask(); 743 MI.RemoveOperand(1); 744 MI.RemoveOperand(0); 745 MI.setDesc(get(SystemZ::CallBRCL)); 746 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 747 .addImm(CCValid) 748 .addImm(CCMask) 749 .add(FirstOp) 750 .addRegMask(RegMask) 751 .addReg(SystemZ::CC, RegState::Implicit); 752 return true; 753 } 754 if (Opcode == SystemZ::CallBR) { 755 const uint32_t *RegMask = MI.getOperand(0).getRegMask(); 756 MI.RemoveOperand(0); 757 MI.setDesc(get(SystemZ::CallBCR)); 758 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 759 .addImm(CCValid).addImm(CCMask) 760 .addRegMask(RegMask) 761 .addReg(SystemZ::CC, RegState::Implicit); 762 return true; 763 } 764 return false; 765 } 766 767 void SystemZInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 768 MachineBasicBlock::iterator MBBI, 769 const DebugLoc &DL, MCRegister DestReg, 770 MCRegister SrcReg, bool KillSrc) const { 771 // Split 128-bit GPR moves into two 64-bit moves. Add implicit uses of the 772 // super register in case one of the subregs is undefined. 773 // This handles ADDR128 too. 774 if (SystemZ::GR128BitRegClass.contains(DestReg, SrcReg)) { 775 copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_h64), 776 RI.getSubReg(SrcReg, SystemZ::subreg_h64), KillSrc); 777 MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI)) 778 .addReg(SrcReg, RegState::Implicit); 779 copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_l64), 780 RI.getSubReg(SrcReg, SystemZ::subreg_l64), KillSrc); 781 MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI)) 782 .addReg(SrcReg, (getKillRegState(KillSrc) | RegState::Implicit)); 783 return; 784 } 785 786 if (SystemZ::GRX32BitRegClass.contains(DestReg, SrcReg)) { 787 emitGRX32Move(MBB, MBBI, DL, DestReg, SrcReg, SystemZ::LR, 32, KillSrc, 788 false); 789 return; 790 } 791 792 // Move 128-bit floating-point values between VR128 and FP128. 793 if (SystemZ::VR128BitRegClass.contains(DestReg) && 794 SystemZ::FP128BitRegClass.contains(SrcReg)) { 795 MCRegister SrcRegHi = 796 RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_h64), 797 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 798 MCRegister SrcRegLo = 799 RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_l64), 800 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 801 802 BuildMI(MBB, MBBI, DL, get(SystemZ::VMRHG), DestReg) 803 .addReg(SrcRegHi, getKillRegState(KillSrc)) 804 .addReg(SrcRegLo, getKillRegState(KillSrc)); 805 return; 806 } 807 if (SystemZ::FP128BitRegClass.contains(DestReg) && 808 SystemZ::VR128BitRegClass.contains(SrcReg)) { 809 MCRegister DestRegHi = 810 RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_h64), 811 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 812 MCRegister DestRegLo = 813 RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_l64), 814 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 815 816 if (DestRegHi != SrcReg) 817 copyPhysReg(MBB, MBBI, DL, DestRegHi, SrcReg, false); 818 BuildMI(MBB, MBBI, DL, get(SystemZ::VREPG), DestRegLo) 819 .addReg(SrcReg, getKillRegState(KillSrc)).addImm(1); 820 return; 821 } 822 823 // Move CC value from a GR32. 824 if (DestReg == SystemZ::CC) { 825 unsigned Opcode = 826 SystemZ::GR32BitRegClass.contains(SrcReg) ? SystemZ::TMLH : SystemZ::TMHH; 827 BuildMI(MBB, MBBI, DL, get(Opcode)) 828 .addReg(SrcReg, getKillRegState(KillSrc)) 829 .addImm(3 << (SystemZ::IPM_CC - 16)); 830 return; 831 } 832 833 // Everything else needs only one instruction. 834 unsigned Opcode; 835 if (SystemZ::GR64BitRegClass.contains(DestReg, SrcReg)) 836 Opcode = SystemZ::LGR; 837 else if (SystemZ::FP32BitRegClass.contains(DestReg, SrcReg)) 838 // For z13 we prefer LDR over LER to avoid partial register dependencies. 839 Opcode = STI.hasVector() ? SystemZ::LDR32 : SystemZ::LER; 840 else if (SystemZ::FP64BitRegClass.contains(DestReg, SrcReg)) 841 Opcode = SystemZ::LDR; 842 else if (SystemZ::FP128BitRegClass.contains(DestReg, SrcReg)) 843 Opcode = SystemZ::LXR; 844 else if (SystemZ::VR32BitRegClass.contains(DestReg, SrcReg)) 845 Opcode = SystemZ::VLR32; 846 else if (SystemZ::VR64BitRegClass.contains(DestReg, SrcReg)) 847 Opcode = SystemZ::VLR64; 848 else if (SystemZ::VR128BitRegClass.contains(DestReg, SrcReg)) 849 Opcode = SystemZ::VLR; 850 else if (SystemZ::AR32BitRegClass.contains(DestReg, SrcReg)) 851 Opcode = SystemZ::CPYA; 852 else 853 llvm_unreachable("Impossible reg-to-reg copy"); 854 855 BuildMI(MBB, MBBI, DL, get(Opcode), DestReg) 856 .addReg(SrcReg, getKillRegState(KillSrc)); 857 } 858 859 void SystemZInstrInfo::storeRegToStackSlot( 860 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, 861 bool isKill, int FrameIdx, const TargetRegisterClass *RC, 862 const TargetRegisterInfo *TRI) const { 863 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 864 865 // Callers may expect a single instruction, so keep 128-bit moves 866 // together for now and lower them after register allocation. 867 unsigned LoadOpcode, StoreOpcode; 868 getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode); 869 addFrameReference(BuildMI(MBB, MBBI, DL, get(StoreOpcode)) 870 .addReg(SrcReg, getKillRegState(isKill)), 871 FrameIdx); 872 } 873 874 void SystemZInstrInfo::loadRegFromStackSlot( 875 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, 876 int FrameIdx, const TargetRegisterClass *RC, 877 const TargetRegisterInfo *TRI) const { 878 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 879 880 // Callers may expect a single instruction, so keep 128-bit moves 881 // together for now and lower them after register allocation. 882 unsigned LoadOpcode, StoreOpcode; 883 getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode); 884 addFrameReference(BuildMI(MBB, MBBI, DL, get(LoadOpcode), DestReg), 885 FrameIdx); 886 } 887 888 // Return true if MI is a simple load or store with a 12-bit displacement 889 // and no index. Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores. 890 static bool isSimpleBD12Move(const MachineInstr *MI, unsigned Flag) { 891 const MCInstrDesc &MCID = MI->getDesc(); 892 return ((MCID.TSFlags & Flag) && 893 isUInt<12>(MI->getOperand(2).getImm()) && 894 MI->getOperand(3).getReg() == 0); 895 } 896 897 namespace { 898 899 struct LogicOp { 900 LogicOp() = default; 901 LogicOp(unsigned regSize, unsigned immLSB, unsigned immSize) 902 : RegSize(regSize), ImmLSB(immLSB), ImmSize(immSize) {} 903 904 explicit operator bool() const { return RegSize; } 905 906 unsigned RegSize = 0; 907 unsigned ImmLSB = 0; 908 unsigned ImmSize = 0; 909 }; 910 911 } // end anonymous namespace 912 913 static LogicOp interpretAndImmediate(unsigned Opcode) { 914 switch (Opcode) { 915 case SystemZ::NILMux: return LogicOp(32, 0, 16); 916 case SystemZ::NIHMux: return LogicOp(32, 16, 16); 917 case SystemZ::NILL64: return LogicOp(64, 0, 16); 918 case SystemZ::NILH64: return LogicOp(64, 16, 16); 919 case SystemZ::NIHL64: return LogicOp(64, 32, 16); 920 case SystemZ::NIHH64: return LogicOp(64, 48, 16); 921 case SystemZ::NIFMux: return LogicOp(32, 0, 32); 922 case SystemZ::NILF64: return LogicOp(64, 0, 32); 923 case SystemZ::NIHF64: return LogicOp(64, 32, 32); 924 default: return LogicOp(); 925 } 926 } 927 928 static void transferDeadCC(MachineInstr *OldMI, MachineInstr *NewMI) { 929 if (OldMI->registerDefIsDead(SystemZ::CC)) { 930 MachineOperand *CCDef = NewMI->findRegisterDefOperand(SystemZ::CC); 931 if (CCDef != nullptr) 932 CCDef->setIsDead(true); 933 } 934 } 935 936 static void transferMIFlag(MachineInstr *OldMI, MachineInstr *NewMI, 937 MachineInstr::MIFlag Flag) { 938 if (OldMI->getFlag(Flag)) 939 NewMI->setFlag(Flag); 940 } 941 942 MachineInstr *SystemZInstrInfo::convertToThreeAddress( 943 MachineFunction::iterator &MFI, MachineInstr &MI, LiveVariables *LV) const { 944 MachineBasicBlock *MBB = MI.getParent(); 945 946 // Try to convert an AND into an RISBG-type instruction. 947 // TODO: It might be beneficial to select RISBG and shorten to AND instead. 948 if (LogicOp And = interpretAndImmediate(MI.getOpcode())) { 949 uint64_t Imm = MI.getOperand(2).getImm() << And.ImmLSB; 950 // AND IMMEDIATE leaves the other bits of the register unchanged. 951 Imm |= allOnes(And.RegSize) & ~(allOnes(And.ImmSize) << And.ImmLSB); 952 unsigned Start, End; 953 if (isRxSBGMask(Imm, And.RegSize, Start, End)) { 954 unsigned NewOpcode; 955 if (And.RegSize == 64) { 956 NewOpcode = SystemZ::RISBG; 957 // Prefer RISBGN if available, since it does not clobber CC. 958 if (STI.hasMiscellaneousExtensions()) 959 NewOpcode = SystemZ::RISBGN; 960 } else { 961 NewOpcode = SystemZ::RISBMux; 962 Start &= 31; 963 End &= 31; 964 } 965 MachineOperand &Dest = MI.getOperand(0); 966 MachineOperand &Src = MI.getOperand(1); 967 MachineInstrBuilder MIB = 968 BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpcode)) 969 .add(Dest) 970 .addReg(0) 971 .addReg(Src.getReg(), getKillRegState(Src.isKill()), 972 Src.getSubReg()) 973 .addImm(Start) 974 .addImm(End + 128) 975 .addImm(0); 976 if (LV) { 977 unsigned NumOps = MI.getNumOperands(); 978 for (unsigned I = 1; I < NumOps; ++I) { 979 MachineOperand &Op = MI.getOperand(I); 980 if (Op.isReg() && Op.isKill()) 981 LV->replaceKillInstruction(Op.getReg(), MI, *MIB); 982 } 983 } 984 transferDeadCC(&MI, MIB); 985 return MIB; 986 } 987 } 988 return nullptr; 989 } 990 991 MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl( 992 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 993 MachineBasicBlock::iterator InsertPt, int FrameIndex, 994 LiveIntervals *LIS, VirtRegMap *VRM) const { 995 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 996 MachineRegisterInfo &MRI = MF.getRegInfo(); 997 const MachineFrameInfo &MFI = MF.getFrameInfo(); 998 unsigned Size = MFI.getObjectSize(FrameIndex); 999 unsigned Opcode = MI.getOpcode(); 1000 1001 // Check CC liveness if new instruction introduces a dead def of CC. 1002 MCRegUnitIterator CCUnit(SystemZ::CC, TRI); 1003 SlotIndex MISlot = SlotIndex(); 1004 LiveRange *CCLiveRange = nullptr; 1005 bool CCLiveAtMI = true; 1006 if (LIS) { 1007 MISlot = LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot(); 1008 CCLiveRange = &LIS->getRegUnit(*CCUnit); 1009 CCLiveAtMI = CCLiveRange->liveAt(MISlot); 1010 } 1011 ++CCUnit; 1012 assert(!CCUnit.isValid() && "CC only has one reg unit."); 1013 1014 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 1015 if (!CCLiveAtMI && (Opcode == SystemZ::LA || Opcode == SystemZ::LAY) && 1016 isInt<8>(MI.getOperand(2).getImm()) && !MI.getOperand(3).getReg()) { 1017 // LA(Y) %reg, CONST(%reg) -> AGSI %mem, CONST 1018 MachineInstr *BuiltMI = BuildMI(*InsertPt->getParent(), InsertPt, 1019 MI.getDebugLoc(), get(SystemZ::AGSI)) 1020 .addFrameIndex(FrameIndex) 1021 .addImm(0) 1022 .addImm(MI.getOperand(2).getImm()); 1023 BuiltMI->findRegisterDefOperand(SystemZ::CC)->setIsDead(true); 1024 CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator()); 1025 return BuiltMI; 1026 } 1027 return nullptr; 1028 } 1029 1030 // All other cases require a single operand. 1031 if (Ops.size() != 1) 1032 return nullptr; 1033 1034 unsigned OpNum = Ops[0]; 1035 assert(Size * 8 == 1036 TRI->getRegSizeInBits(*MF.getRegInfo() 1037 .getRegClass(MI.getOperand(OpNum).getReg())) && 1038 "Invalid size combination"); 1039 1040 if ((Opcode == SystemZ::AHI || Opcode == SystemZ::AGHI) && OpNum == 0 && 1041 isInt<8>(MI.getOperand(2).getImm())) { 1042 // A(G)HI %reg, CONST -> A(G)SI %mem, CONST 1043 Opcode = (Opcode == SystemZ::AHI ? SystemZ::ASI : SystemZ::AGSI); 1044 MachineInstr *BuiltMI = 1045 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1046 .addFrameIndex(FrameIndex) 1047 .addImm(0) 1048 .addImm(MI.getOperand(2).getImm()); 1049 transferDeadCC(&MI, BuiltMI); 1050 transferMIFlag(&MI, BuiltMI, MachineInstr::NoSWrap); 1051 return BuiltMI; 1052 } 1053 1054 if ((Opcode == SystemZ::ALFI && OpNum == 0 && 1055 isInt<8>((int32_t)MI.getOperand(2).getImm())) || 1056 (Opcode == SystemZ::ALGFI && OpNum == 0 && 1057 isInt<8>((int64_t)MI.getOperand(2).getImm()))) { 1058 // AL(G)FI %reg, CONST -> AL(G)SI %mem, CONST 1059 Opcode = (Opcode == SystemZ::ALFI ? SystemZ::ALSI : SystemZ::ALGSI); 1060 MachineInstr *BuiltMI = 1061 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1062 .addFrameIndex(FrameIndex) 1063 .addImm(0) 1064 .addImm((int8_t)MI.getOperand(2).getImm()); 1065 transferDeadCC(&MI, BuiltMI); 1066 return BuiltMI; 1067 } 1068 1069 if ((Opcode == SystemZ::SLFI && OpNum == 0 && 1070 isInt<8>((int32_t)-MI.getOperand(2).getImm())) || 1071 (Opcode == SystemZ::SLGFI && OpNum == 0 && 1072 isInt<8>((int64_t)-MI.getOperand(2).getImm()))) { 1073 // SL(G)FI %reg, CONST -> AL(G)SI %mem, -CONST 1074 Opcode = (Opcode == SystemZ::SLFI ? SystemZ::ALSI : SystemZ::ALGSI); 1075 MachineInstr *BuiltMI = 1076 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1077 .addFrameIndex(FrameIndex) 1078 .addImm(0) 1079 .addImm((int8_t)-MI.getOperand(2).getImm()); 1080 transferDeadCC(&MI, BuiltMI); 1081 return BuiltMI; 1082 } 1083 1084 unsigned MemImmOpc = 0; 1085 switch (Opcode) { 1086 case SystemZ::LHIMux: 1087 case SystemZ::LHI: MemImmOpc = SystemZ::MVHI; break; 1088 case SystemZ::LGHI: MemImmOpc = SystemZ::MVGHI; break; 1089 case SystemZ::CHIMux: 1090 case SystemZ::CHI: MemImmOpc = SystemZ::CHSI; break; 1091 case SystemZ::CGHI: MemImmOpc = SystemZ::CGHSI; break; 1092 case SystemZ::CLFIMux: 1093 case SystemZ::CLFI: 1094 if (isUInt<16>(MI.getOperand(1).getImm())) 1095 MemImmOpc = SystemZ::CLFHSI; 1096 break; 1097 case SystemZ::CLGFI: 1098 if (isUInt<16>(MI.getOperand(1).getImm())) 1099 MemImmOpc = SystemZ::CLGHSI; 1100 break; 1101 default: break; 1102 } 1103 if (MemImmOpc) 1104 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1105 get(MemImmOpc)) 1106 .addFrameIndex(FrameIndex) 1107 .addImm(0) 1108 .addImm(MI.getOperand(1).getImm()); 1109 1110 if (Opcode == SystemZ::LGDR || Opcode == SystemZ::LDGR) { 1111 bool Op0IsGPR = (Opcode == SystemZ::LGDR); 1112 bool Op1IsGPR = (Opcode == SystemZ::LDGR); 1113 // If we're spilling the destination of an LDGR or LGDR, store the 1114 // source register instead. 1115 if (OpNum == 0) { 1116 unsigned StoreOpcode = Op1IsGPR ? SystemZ::STG : SystemZ::STD; 1117 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1118 get(StoreOpcode)) 1119 .add(MI.getOperand(1)) 1120 .addFrameIndex(FrameIndex) 1121 .addImm(0) 1122 .addReg(0); 1123 } 1124 // If we're spilling the source of an LDGR or LGDR, load the 1125 // destination register instead. 1126 if (OpNum == 1) { 1127 unsigned LoadOpcode = Op0IsGPR ? SystemZ::LG : SystemZ::LD; 1128 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1129 get(LoadOpcode)) 1130 .add(MI.getOperand(0)) 1131 .addFrameIndex(FrameIndex) 1132 .addImm(0) 1133 .addReg(0); 1134 } 1135 } 1136 1137 // Look for cases where the source of a simple store or the destination 1138 // of a simple load is being spilled. Try to use MVC instead. 1139 // 1140 // Although MVC is in practice a fast choice in these cases, it is still 1141 // logically a bytewise copy. This means that we cannot use it if the 1142 // load or store is volatile. We also wouldn't be able to use MVC if 1143 // the two memories partially overlap, but that case cannot occur here, 1144 // because we know that one of the memories is a full frame index. 1145 // 1146 // For performance reasons, we also want to avoid using MVC if the addresses 1147 // might be equal. We don't worry about that case here, because spill slot 1148 // coloring happens later, and because we have special code to remove 1149 // MVCs that turn out to be redundant. 1150 if (OpNum == 0 && MI.hasOneMemOperand()) { 1151 MachineMemOperand *MMO = *MI.memoperands_begin(); 1152 if (MMO->getSize() == Size && !MMO->isVolatile() && !MMO->isAtomic()) { 1153 // Handle conversion of loads. 1154 if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXLoad)) { 1155 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1156 get(SystemZ::MVC)) 1157 .addFrameIndex(FrameIndex) 1158 .addImm(0) 1159 .addImm(Size) 1160 .add(MI.getOperand(1)) 1161 .addImm(MI.getOperand(2).getImm()) 1162 .addMemOperand(MMO); 1163 } 1164 // Handle conversion of stores. 1165 if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXStore)) { 1166 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1167 get(SystemZ::MVC)) 1168 .add(MI.getOperand(1)) 1169 .addImm(MI.getOperand(2).getImm()) 1170 .addImm(Size) 1171 .addFrameIndex(FrameIndex) 1172 .addImm(0) 1173 .addMemOperand(MMO); 1174 } 1175 } 1176 } 1177 1178 // If the spilled operand is the final one or the instruction is 1179 // commutable, try to change <INSN>R into <INSN>. Don't introduce a def of 1180 // CC if it is live and MI does not define it. 1181 unsigned NumOps = MI.getNumExplicitOperands(); 1182 int MemOpcode = SystemZ::getMemOpcode(Opcode); 1183 if (MemOpcode == -1 || 1184 (CCLiveAtMI && !MI.definesRegister(SystemZ::CC) && 1185 get(MemOpcode).hasImplicitDefOfPhysReg(SystemZ::CC))) 1186 return nullptr; 1187 1188 // Check if all other vregs have a usable allocation in the case of vector 1189 // to FP conversion. 1190 const MCInstrDesc &MCID = MI.getDesc(); 1191 for (unsigned I = 0, E = MCID.getNumOperands(); I != E; ++I) { 1192 const MCOperandInfo &MCOI = MCID.OpInfo[I]; 1193 if (MCOI.OperandType != MCOI::OPERAND_REGISTER || I == OpNum) 1194 continue; 1195 const TargetRegisterClass *RC = TRI->getRegClass(MCOI.RegClass); 1196 if (RC == &SystemZ::VR32BitRegClass || RC == &SystemZ::VR64BitRegClass) { 1197 Register Reg = MI.getOperand(I).getReg(); 1198 Register PhysReg = Register::isVirtualRegister(Reg) 1199 ? (VRM ? VRM->getPhys(Reg) : Register()) 1200 : Reg; 1201 if (!PhysReg || 1202 !(SystemZ::FP32BitRegClass.contains(PhysReg) || 1203 SystemZ::FP64BitRegClass.contains(PhysReg) || 1204 SystemZ::VF128BitRegClass.contains(PhysReg))) 1205 return nullptr; 1206 } 1207 } 1208 // Fused multiply and add/sub need to have the same dst and accumulator reg. 1209 bool FusedFPOp = (Opcode == SystemZ::WFMADB || Opcode == SystemZ::WFMASB || 1210 Opcode == SystemZ::WFMSDB || Opcode == SystemZ::WFMSSB); 1211 if (FusedFPOp) { 1212 Register DstReg = VRM->getPhys(MI.getOperand(0).getReg()); 1213 Register AccReg = VRM->getPhys(MI.getOperand(3).getReg()); 1214 if (OpNum == 0 || OpNum == 3 || DstReg != AccReg) 1215 return nullptr; 1216 } 1217 1218 // Try to swap compare operands if possible. 1219 bool NeedsCommute = false; 1220 if ((MI.getOpcode() == SystemZ::CR || MI.getOpcode() == SystemZ::CGR || 1221 MI.getOpcode() == SystemZ::CLR || MI.getOpcode() == SystemZ::CLGR || 1222 MI.getOpcode() == SystemZ::WFCDB || MI.getOpcode() == SystemZ::WFCSB || 1223 MI.getOpcode() == SystemZ::WFKDB || MI.getOpcode() == SystemZ::WFKSB) && 1224 OpNum == 0 && prepareCompareSwapOperands(MI)) 1225 NeedsCommute = true; 1226 1227 bool CCOperands = false; 1228 if (MI.getOpcode() == SystemZ::LOCRMux || MI.getOpcode() == SystemZ::LOCGR || 1229 MI.getOpcode() == SystemZ::SELRMux || MI.getOpcode() == SystemZ::SELGR) { 1230 assert(MI.getNumOperands() == 6 && NumOps == 5 && 1231 "LOCR/SELR instruction operands corrupt?"); 1232 NumOps -= 2; 1233 CCOperands = true; 1234 } 1235 1236 // See if this is a 3-address instruction that is convertible to 2-address 1237 // and suitable for folding below. Only try this with virtual registers 1238 // and a provided VRM (during regalloc). 1239 if (NumOps == 3 && SystemZ::getTargetMemOpcode(MemOpcode) != -1) { 1240 if (VRM == nullptr) 1241 return nullptr; 1242 else { 1243 Register DstReg = MI.getOperand(0).getReg(); 1244 Register DstPhys = 1245 (Register::isVirtualRegister(DstReg) ? VRM->getPhys(DstReg) : DstReg); 1246 Register SrcReg = (OpNum == 2 ? MI.getOperand(1).getReg() 1247 : ((OpNum == 1 && MI.isCommutable()) 1248 ? MI.getOperand(2).getReg() 1249 : Register())); 1250 if (DstPhys && !SystemZ::GRH32BitRegClass.contains(DstPhys) && SrcReg && 1251 Register::isVirtualRegister(SrcReg) && 1252 DstPhys == VRM->getPhys(SrcReg)) 1253 NeedsCommute = (OpNum == 1); 1254 else 1255 return nullptr; 1256 } 1257 } 1258 1259 if ((OpNum == NumOps - 1) || NeedsCommute || FusedFPOp) { 1260 const MCInstrDesc &MemDesc = get(MemOpcode); 1261 uint64_t AccessBytes = SystemZII::getAccessSize(MemDesc.TSFlags); 1262 assert(AccessBytes != 0 && "Size of access should be known"); 1263 assert(AccessBytes <= Size && "Access outside the frame index"); 1264 uint64_t Offset = Size - AccessBytes; 1265 MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, 1266 MI.getDebugLoc(), get(MemOpcode)); 1267 if (MI.isCompare()) { 1268 assert(NumOps == 2 && "Expected 2 register operands for a compare."); 1269 MIB.add(MI.getOperand(NeedsCommute ? 1 : 0)); 1270 } 1271 else if (FusedFPOp) { 1272 MIB.add(MI.getOperand(0)); 1273 MIB.add(MI.getOperand(3)); 1274 MIB.add(MI.getOperand(OpNum == 1 ? 2 : 1)); 1275 } 1276 else { 1277 MIB.add(MI.getOperand(0)); 1278 if (NeedsCommute) 1279 MIB.add(MI.getOperand(2)); 1280 else 1281 for (unsigned I = 1; I < OpNum; ++I) 1282 MIB.add(MI.getOperand(I)); 1283 } 1284 MIB.addFrameIndex(FrameIndex).addImm(Offset); 1285 if (MemDesc.TSFlags & SystemZII::HasIndex) 1286 MIB.addReg(0); 1287 if (CCOperands) { 1288 unsigned CCValid = MI.getOperand(NumOps).getImm(); 1289 unsigned CCMask = MI.getOperand(NumOps + 1).getImm(); 1290 MIB.addImm(CCValid); 1291 MIB.addImm(NeedsCommute ? CCMask ^ CCValid : CCMask); 1292 } 1293 if (MIB->definesRegister(SystemZ::CC) && 1294 (!MI.definesRegister(SystemZ::CC) || 1295 MI.registerDefIsDead(SystemZ::CC))) { 1296 MIB->addRegisterDead(SystemZ::CC, TRI); 1297 if (CCLiveRange) 1298 CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator()); 1299 } 1300 // Constrain the register classes if converted from a vector opcode. The 1301 // allocated regs are in an FP reg-class per previous check above. 1302 for (const MachineOperand &MO : MIB->operands()) 1303 if (MO.isReg() && Register::isVirtualRegister(MO.getReg())) { 1304 unsigned Reg = MO.getReg(); 1305 if (MRI.getRegClass(Reg) == &SystemZ::VR32BitRegClass) 1306 MRI.setRegClass(Reg, &SystemZ::FP32BitRegClass); 1307 else if (MRI.getRegClass(Reg) == &SystemZ::VR64BitRegClass) 1308 MRI.setRegClass(Reg, &SystemZ::FP64BitRegClass); 1309 else if (MRI.getRegClass(Reg) == &SystemZ::VR128BitRegClass) 1310 MRI.setRegClass(Reg, &SystemZ::VF128BitRegClass); 1311 } 1312 1313 transferDeadCC(&MI, MIB); 1314 transferMIFlag(&MI, MIB, MachineInstr::NoSWrap); 1315 transferMIFlag(&MI, MIB, MachineInstr::NoFPExcept); 1316 return MIB; 1317 } 1318 1319 return nullptr; 1320 } 1321 1322 MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl( 1323 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 1324 MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, 1325 LiveIntervals *LIS) const { 1326 return nullptr; 1327 } 1328 1329 bool SystemZInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 1330 switch (MI.getOpcode()) { 1331 case SystemZ::L128: 1332 splitMove(MI, SystemZ::LG); 1333 return true; 1334 1335 case SystemZ::ST128: 1336 splitMove(MI, SystemZ::STG); 1337 return true; 1338 1339 case SystemZ::LX: 1340 splitMove(MI, SystemZ::LD); 1341 return true; 1342 1343 case SystemZ::STX: 1344 splitMove(MI, SystemZ::STD); 1345 return true; 1346 1347 case SystemZ::LBMux: 1348 expandRXYPseudo(MI, SystemZ::LB, SystemZ::LBH); 1349 return true; 1350 1351 case SystemZ::LHMux: 1352 expandRXYPseudo(MI, SystemZ::LH, SystemZ::LHH); 1353 return true; 1354 1355 case SystemZ::LLCRMux: 1356 expandZExtPseudo(MI, SystemZ::LLCR, 8); 1357 return true; 1358 1359 case SystemZ::LLHRMux: 1360 expandZExtPseudo(MI, SystemZ::LLHR, 16); 1361 return true; 1362 1363 case SystemZ::LLCMux: 1364 expandRXYPseudo(MI, SystemZ::LLC, SystemZ::LLCH); 1365 return true; 1366 1367 case SystemZ::LLHMux: 1368 expandRXYPseudo(MI, SystemZ::LLH, SystemZ::LLHH); 1369 return true; 1370 1371 case SystemZ::LMux: 1372 expandRXYPseudo(MI, SystemZ::L, SystemZ::LFH); 1373 return true; 1374 1375 case SystemZ::LOCMux: 1376 expandLOCPseudo(MI, SystemZ::LOC, SystemZ::LOCFH); 1377 return true; 1378 1379 case SystemZ::LOCHIMux: 1380 expandLOCPseudo(MI, SystemZ::LOCHI, SystemZ::LOCHHI); 1381 return true; 1382 1383 case SystemZ::STCMux: 1384 expandRXYPseudo(MI, SystemZ::STC, SystemZ::STCH); 1385 return true; 1386 1387 case SystemZ::STHMux: 1388 expandRXYPseudo(MI, SystemZ::STH, SystemZ::STHH); 1389 return true; 1390 1391 case SystemZ::STMux: 1392 expandRXYPseudo(MI, SystemZ::ST, SystemZ::STFH); 1393 return true; 1394 1395 case SystemZ::STOCMux: 1396 expandLOCPseudo(MI, SystemZ::STOC, SystemZ::STOCFH); 1397 return true; 1398 1399 case SystemZ::LHIMux: 1400 expandRIPseudo(MI, SystemZ::LHI, SystemZ::IIHF, true); 1401 return true; 1402 1403 case SystemZ::IIFMux: 1404 expandRIPseudo(MI, SystemZ::IILF, SystemZ::IIHF, false); 1405 return true; 1406 1407 case SystemZ::IILMux: 1408 expandRIPseudo(MI, SystemZ::IILL, SystemZ::IIHL, false); 1409 return true; 1410 1411 case SystemZ::IIHMux: 1412 expandRIPseudo(MI, SystemZ::IILH, SystemZ::IIHH, false); 1413 return true; 1414 1415 case SystemZ::NIFMux: 1416 expandRIPseudo(MI, SystemZ::NILF, SystemZ::NIHF, false); 1417 return true; 1418 1419 case SystemZ::NILMux: 1420 expandRIPseudo(MI, SystemZ::NILL, SystemZ::NIHL, false); 1421 return true; 1422 1423 case SystemZ::NIHMux: 1424 expandRIPseudo(MI, SystemZ::NILH, SystemZ::NIHH, false); 1425 return true; 1426 1427 case SystemZ::OIFMux: 1428 expandRIPseudo(MI, SystemZ::OILF, SystemZ::OIHF, false); 1429 return true; 1430 1431 case SystemZ::OILMux: 1432 expandRIPseudo(MI, SystemZ::OILL, SystemZ::OIHL, false); 1433 return true; 1434 1435 case SystemZ::OIHMux: 1436 expandRIPseudo(MI, SystemZ::OILH, SystemZ::OIHH, false); 1437 return true; 1438 1439 case SystemZ::XIFMux: 1440 expandRIPseudo(MI, SystemZ::XILF, SystemZ::XIHF, false); 1441 return true; 1442 1443 case SystemZ::TMLMux: 1444 expandRIPseudo(MI, SystemZ::TMLL, SystemZ::TMHL, false); 1445 return true; 1446 1447 case SystemZ::TMHMux: 1448 expandRIPseudo(MI, SystemZ::TMLH, SystemZ::TMHH, false); 1449 return true; 1450 1451 case SystemZ::AHIMux: 1452 expandRIPseudo(MI, SystemZ::AHI, SystemZ::AIH, false); 1453 return true; 1454 1455 case SystemZ::AHIMuxK: 1456 expandRIEPseudo(MI, SystemZ::AHI, SystemZ::AHIK, SystemZ::AIH); 1457 return true; 1458 1459 case SystemZ::AFIMux: 1460 expandRIPseudo(MI, SystemZ::AFI, SystemZ::AIH, false); 1461 return true; 1462 1463 case SystemZ::CHIMux: 1464 expandRIPseudo(MI, SystemZ::CHI, SystemZ::CIH, false); 1465 return true; 1466 1467 case SystemZ::CFIMux: 1468 expandRIPseudo(MI, SystemZ::CFI, SystemZ::CIH, false); 1469 return true; 1470 1471 case SystemZ::CLFIMux: 1472 expandRIPseudo(MI, SystemZ::CLFI, SystemZ::CLIH, false); 1473 return true; 1474 1475 case SystemZ::CMux: 1476 expandRXYPseudo(MI, SystemZ::C, SystemZ::CHF); 1477 return true; 1478 1479 case SystemZ::CLMux: 1480 expandRXYPseudo(MI, SystemZ::CL, SystemZ::CLHF); 1481 return true; 1482 1483 case SystemZ::RISBMux: { 1484 bool DestIsHigh = SystemZ::isHighReg(MI.getOperand(0).getReg()); 1485 bool SrcIsHigh = SystemZ::isHighReg(MI.getOperand(2).getReg()); 1486 if (SrcIsHigh == DestIsHigh) 1487 MI.setDesc(get(DestIsHigh ? SystemZ::RISBHH : SystemZ::RISBLL)); 1488 else { 1489 MI.setDesc(get(DestIsHigh ? SystemZ::RISBHL : SystemZ::RISBLH)); 1490 MI.getOperand(5).setImm(MI.getOperand(5).getImm() ^ 32); 1491 } 1492 return true; 1493 } 1494 1495 case SystemZ::ADJDYNALLOC: 1496 splitAdjDynAlloc(MI); 1497 return true; 1498 1499 case TargetOpcode::LOAD_STACK_GUARD: 1500 expandLoadStackGuard(&MI); 1501 return true; 1502 1503 default: 1504 return false; 1505 } 1506 } 1507 1508 unsigned SystemZInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 1509 if (MI.isInlineAsm()) { 1510 const MachineFunction *MF = MI.getParent()->getParent(); 1511 const char *AsmStr = MI.getOperand(0).getSymbolName(); 1512 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo()); 1513 } 1514 return MI.getDesc().getSize(); 1515 } 1516 1517 SystemZII::Branch 1518 SystemZInstrInfo::getBranchInfo(const MachineInstr &MI) const { 1519 switch (MI.getOpcode()) { 1520 case SystemZ::BR: 1521 case SystemZ::BI: 1522 case SystemZ::J: 1523 case SystemZ::JG: 1524 return SystemZII::Branch(SystemZII::BranchNormal, SystemZ::CCMASK_ANY, 1525 SystemZ::CCMASK_ANY, &MI.getOperand(0)); 1526 1527 case SystemZ::BRC: 1528 case SystemZ::BRCL: 1529 return SystemZII::Branch(SystemZII::BranchNormal, MI.getOperand(0).getImm(), 1530 MI.getOperand(1).getImm(), &MI.getOperand(2)); 1531 1532 case SystemZ::BRCT: 1533 case SystemZ::BRCTH: 1534 return SystemZII::Branch(SystemZII::BranchCT, SystemZ::CCMASK_ICMP, 1535 SystemZ::CCMASK_CMP_NE, &MI.getOperand(2)); 1536 1537 case SystemZ::BRCTG: 1538 return SystemZII::Branch(SystemZII::BranchCTG, SystemZ::CCMASK_ICMP, 1539 SystemZ::CCMASK_CMP_NE, &MI.getOperand(2)); 1540 1541 case SystemZ::CIJ: 1542 case SystemZ::CRJ: 1543 return SystemZII::Branch(SystemZII::BranchC, SystemZ::CCMASK_ICMP, 1544 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1545 1546 case SystemZ::CLIJ: 1547 case SystemZ::CLRJ: 1548 return SystemZII::Branch(SystemZII::BranchCL, SystemZ::CCMASK_ICMP, 1549 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1550 1551 case SystemZ::CGIJ: 1552 case SystemZ::CGRJ: 1553 return SystemZII::Branch(SystemZII::BranchCG, SystemZ::CCMASK_ICMP, 1554 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1555 1556 case SystemZ::CLGIJ: 1557 case SystemZ::CLGRJ: 1558 return SystemZII::Branch(SystemZII::BranchCLG, SystemZ::CCMASK_ICMP, 1559 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1560 1561 case SystemZ::INLINEASM_BR: 1562 // Don't try to analyze asm goto, so pass nullptr as branch target argument. 1563 return SystemZII::Branch(SystemZII::AsmGoto, 0, 0, nullptr); 1564 1565 default: 1566 llvm_unreachable("Unrecognized branch opcode"); 1567 } 1568 } 1569 1570 void SystemZInstrInfo::getLoadStoreOpcodes(const TargetRegisterClass *RC, 1571 unsigned &LoadOpcode, 1572 unsigned &StoreOpcode) const { 1573 if (RC == &SystemZ::GR32BitRegClass || RC == &SystemZ::ADDR32BitRegClass) { 1574 LoadOpcode = SystemZ::L; 1575 StoreOpcode = SystemZ::ST; 1576 } else if (RC == &SystemZ::GRH32BitRegClass) { 1577 LoadOpcode = SystemZ::LFH; 1578 StoreOpcode = SystemZ::STFH; 1579 } else if (RC == &SystemZ::GRX32BitRegClass) { 1580 LoadOpcode = SystemZ::LMux; 1581 StoreOpcode = SystemZ::STMux; 1582 } else if (RC == &SystemZ::GR64BitRegClass || 1583 RC == &SystemZ::ADDR64BitRegClass) { 1584 LoadOpcode = SystemZ::LG; 1585 StoreOpcode = SystemZ::STG; 1586 } else if (RC == &SystemZ::GR128BitRegClass || 1587 RC == &SystemZ::ADDR128BitRegClass) { 1588 LoadOpcode = SystemZ::L128; 1589 StoreOpcode = SystemZ::ST128; 1590 } else if (RC == &SystemZ::FP32BitRegClass) { 1591 LoadOpcode = SystemZ::LE; 1592 StoreOpcode = SystemZ::STE; 1593 } else if (RC == &SystemZ::FP64BitRegClass) { 1594 LoadOpcode = SystemZ::LD; 1595 StoreOpcode = SystemZ::STD; 1596 } else if (RC == &SystemZ::FP128BitRegClass) { 1597 LoadOpcode = SystemZ::LX; 1598 StoreOpcode = SystemZ::STX; 1599 } else if (RC == &SystemZ::VR32BitRegClass) { 1600 LoadOpcode = SystemZ::VL32; 1601 StoreOpcode = SystemZ::VST32; 1602 } else if (RC == &SystemZ::VR64BitRegClass) { 1603 LoadOpcode = SystemZ::VL64; 1604 StoreOpcode = SystemZ::VST64; 1605 } else if (RC == &SystemZ::VF128BitRegClass || 1606 RC == &SystemZ::VR128BitRegClass) { 1607 LoadOpcode = SystemZ::VL; 1608 StoreOpcode = SystemZ::VST; 1609 } else 1610 llvm_unreachable("Unsupported regclass to load or store"); 1611 } 1612 1613 unsigned SystemZInstrInfo::getOpcodeForOffset(unsigned Opcode, 1614 int64_t Offset) const { 1615 const MCInstrDesc &MCID = get(Opcode); 1616 int64_t Offset2 = (MCID.TSFlags & SystemZII::Is128Bit ? Offset + 8 : Offset); 1617 if (isUInt<12>(Offset) && isUInt<12>(Offset2)) { 1618 // Get the instruction to use for unsigned 12-bit displacements. 1619 int Disp12Opcode = SystemZ::getDisp12Opcode(Opcode); 1620 if (Disp12Opcode >= 0) 1621 return Disp12Opcode; 1622 1623 // All address-related instructions can use unsigned 12-bit 1624 // displacements. 1625 return Opcode; 1626 } 1627 if (isInt<20>(Offset) && isInt<20>(Offset2)) { 1628 // Get the instruction to use for signed 20-bit displacements. 1629 int Disp20Opcode = SystemZ::getDisp20Opcode(Opcode); 1630 if (Disp20Opcode >= 0) 1631 return Disp20Opcode; 1632 1633 // Check whether Opcode allows signed 20-bit displacements. 1634 if (MCID.TSFlags & SystemZII::Has20BitOffset) 1635 return Opcode; 1636 } 1637 return 0; 1638 } 1639 1640 unsigned SystemZInstrInfo::getLoadAndTest(unsigned Opcode) const { 1641 switch (Opcode) { 1642 case SystemZ::L: return SystemZ::LT; 1643 case SystemZ::LY: return SystemZ::LT; 1644 case SystemZ::LG: return SystemZ::LTG; 1645 case SystemZ::LGF: return SystemZ::LTGF; 1646 case SystemZ::LR: return SystemZ::LTR; 1647 case SystemZ::LGFR: return SystemZ::LTGFR; 1648 case SystemZ::LGR: return SystemZ::LTGR; 1649 case SystemZ::LER: return SystemZ::LTEBR; 1650 case SystemZ::LDR: return SystemZ::LTDBR; 1651 case SystemZ::LXR: return SystemZ::LTXBR; 1652 case SystemZ::LCDFR: return SystemZ::LCDBR; 1653 case SystemZ::LPDFR: return SystemZ::LPDBR; 1654 case SystemZ::LNDFR: return SystemZ::LNDBR; 1655 case SystemZ::LCDFR_32: return SystemZ::LCEBR; 1656 case SystemZ::LPDFR_32: return SystemZ::LPEBR; 1657 case SystemZ::LNDFR_32: return SystemZ::LNEBR; 1658 // On zEC12 we prefer to use RISBGN. But if there is a chance to 1659 // actually use the condition code, we may turn it back into RISGB. 1660 // Note that RISBG is not really a "load-and-test" instruction, 1661 // but sets the same condition code values, so is OK to use here. 1662 case SystemZ::RISBGN: return SystemZ::RISBG; 1663 default: return 0; 1664 } 1665 } 1666 1667 // Return true if Mask matches the regexp 0*1+0*, given that zero masks 1668 // have already been filtered out. Store the first set bit in LSB and 1669 // the number of set bits in Length if so. 1670 static bool isStringOfOnes(uint64_t Mask, unsigned &LSB, unsigned &Length) { 1671 unsigned First = findFirstSet(Mask); 1672 uint64_t Top = (Mask >> First) + 1; 1673 if ((Top & -Top) == Top) { 1674 LSB = First; 1675 Length = findFirstSet(Top); 1676 return true; 1677 } 1678 return false; 1679 } 1680 1681 bool SystemZInstrInfo::isRxSBGMask(uint64_t Mask, unsigned BitSize, 1682 unsigned &Start, unsigned &End) const { 1683 // Reject trivial all-zero masks. 1684 Mask &= allOnes(BitSize); 1685 if (Mask == 0) 1686 return false; 1687 1688 // Handle the 1+0+ or 0+1+0* cases. Start then specifies the index of 1689 // the msb and End specifies the index of the lsb. 1690 unsigned LSB, Length; 1691 if (isStringOfOnes(Mask, LSB, Length)) { 1692 Start = 63 - (LSB + Length - 1); 1693 End = 63 - LSB; 1694 return true; 1695 } 1696 1697 // Handle the wrap-around 1+0+1+ cases. Start then specifies the msb 1698 // of the low 1s and End specifies the lsb of the high 1s. 1699 if (isStringOfOnes(Mask ^ allOnes(BitSize), LSB, Length)) { 1700 assert(LSB > 0 && "Bottom bit must be set"); 1701 assert(LSB + Length < BitSize && "Top bit must be set"); 1702 Start = 63 - (LSB - 1); 1703 End = 63 - (LSB + Length); 1704 return true; 1705 } 1706 1707 return false; 1708 } 1709 1710 unsigned SystemZInstrInfo::getFusedCompare(unsigned Opcode, 1711 SystemZII::FusedCompareType Type, 1712 const MachineInstr *MI) const { 1713 switch (Opcode) { 1714 case SystemZ::CHI: 1715 case SystemZ::CGHI: 1716 if (!(MI && isInt<8>(MI->getOperand(1).getImm()))) 1717 return 0; 1718 break; 1719 case SystemZ::CLFI: 1720 case SystemZ::CLGFI: 1721 if (!(MI && isUInt<8>(MI->getOperand(1).getImm()))) 1722 return 0; 1723 break; 1724 case SystemZ::CL: 1725 case SystemZ::CLG: 1726 if (!STI.hasMiscellaneousExtensions()) 1727 return 0; 1728 if (!(MI && MI->getOperand(3).getReg() == 0)) 1729 return 0; 1730 break; 1731 } 1732 switch (Type) { 1733 case SystemZII::CompareAndBranch: 1734 switch (Opcode) { 1735 case SystemZ::CR: 1736 return SystemZ::CRJ; 1737 case SystemZ::CGR: 1738 return SystemZ::CGRJ; 1739 case SystemZ::CHI: 1740 return SystemZ::CIJ; 1741 case SystemZ::CGHI: 1742 return SystemZ::CGIJ; 1743 case SystemZ::CLR: 1744 return SystemZ::CLRJ; 1745 case SystemZ::CLGR: 1746 return SystemZ::CLGRJ; 1747 case SystemZ::CLFI: 1748 return SystemZ::CLIJ; 1749 case SystemZ::CLGFI: 1750 return SystemZ::CLGIJ; 1751 default: 1752 return 0; 1753 } 1754 case SystemZII::CompareAndReturn: 1755 switch (Opcode) { 1756 case SystemZ::CR: 1757 return SystemZ::CRBReturn; 1758 case SystemZ::CGR: 1759 return SystemZ::CGRBReturn; 1760 case SystemZ::CHI: 1761 return SystemZ::CIBReturn; 1762 case SystemZ::CGHI: 1763 return SystemZ::CGIBReturn; 1764 case SystemZ::CLR: 1765 return SystemZ::CLRBReturn; 1766 case SystemZ::CLGR: 1767 return SystemZ::CLGRBReturn; 1768 case SystemZ::CLFI: 1769 return SystemZ::CLIBReturn; 1770 case SystemZ::CLGFI: 1771 return SystemZ::CLGIBReturn; 1772 default: 1773 return 0; 1774 } 1775 case SystemZII::CompareAndSibcall: 1776 switch (Opcode) { 1777 case SystemZ::CR: 1778 return SystemZ::CRBCall; 1779 case SystemZ::CGR: 1780 return SystemZ::CGRBCall; 1781 case SystemZ::CHI: 1782 return SystemZ::CIBCall; 1783 case SystemZ::CGHI: 1784 return SystemZ::CGIBCall; 1785 case SystemZ::CLR: 1786 return SystemZ::CLRBCall; 1787 case SystemZ::CLGR: 1788 return SystemZ::CLGRBCall; 1789 case SystemZ::CLFI: 1790 return SystemZ::CLIBCall; 1791 case SystemZ::CLGFI: 1792 return SystemZ::CLGIBCall; 1793 default: 1794 return 0; 1795 } 1796 case SystemZII::CompareAndTrap: 1797 switch (Opcode) { 1798 case SystemZ::CR: 1799 return SystemZ::CRT; 1800 case SystemZ::CGR: 1801 return SystemZ::CGRT; 1802 case SystemZ::CHI: 1803 return SystemZ::CIT; 1804 case SystemZ::CGHI: 1805 return SystemZ::CGIT; 1806 case SystemZ::CLR: 1807 return SystemZ::CLRT; 1808 case SystemZ::CLGR: 1809 return SystemZ::CLGRT; 1810 case SystemZ::CLFI: 1811 return SystemZ::CLFIT; 1812 case SystemZ::CLGFI: 1813 return SystemZ::CLGIT; 1814 case SystemZ::CL: 1815 return SystemZ::CLT; 1816 case SystemZ::CLG: 1817 return SystemZ::CLGT; 1818 default: 1819 return 0; 1820 } 1821 } 1822 return 0; 1823 } 1824 1825 bool SystemZInstrInfo:: 1826 prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const { 1827 assert(MBBI->isCompare() && MBBI->getOperand(0).isReg() && 1828 MBBI->getOperand(1).isReg() && !MBBI->mayLoad() && 1829 "Not a compare reg/reg."); 1830 1831 MachineBasicBlock *MBB = MBBI->getParent(); 1832 bool CCLive = true; 1833 SmallVector<MachineInstr *, 4> CCUsers; 1834 for (MachineBasicBlock::iterator Itr = std::next(MBBI); 1835 Itr != MBB->end(); ++Itr) { 1836 if (Itr->readsRegister(SystemZ::CC)) { 1837 unsigned Flags = Itr->getDesc().TSFlags; 1838 if ((Flags & SystemZII::CCMaskFirst) || (Flags & SystemZII::CCMaskLast)) 1839 CCUsers.push_back(&*Itr); 1840 else 1841 return false; 1842 } 1843 if (Itr->definesRegister(SystemZ::CC)) { 1844 CCLive = false; 1845 break; 1846 } 1847 } 1848 if (CCLive) { 1849 LivePhysRegs LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo()); 1850 LiveRegs.addLiveOuts(*MBB); 1851 if (LiveRegs.contains(SystemZ::CC)) 1852 return false; 1853 } 1854 1855 // Update all CC users. 1856 for (unsigned Idx = 0; Idx < CCUsers.size(); ++Idx) { 1857 unsigned Flags = CCUsers[Idx]->getDesc().TSFlags; 1858 unsigned FirstOpNum = ((Flags & SystemZII::CCMaskFirst) ? 1859 0 : CCUsers[Idx]->getNumExplicitOperands() - 2); 1860 MachineOperand &CCMaskMO = CCUsers[Idx]->getOperand(FirstOpNum + 1); 1861 unsigned NewCCMask = SystemZ::reverseCCMask(CCMaskMO.getImm()); 1862 CCMaskMO.setImm(NewCCMask); 1863 } 1864 1865 return true; 1866 } 1867 1868 unsigned SystemZ::reverseCCMask(unsigned CCMask) { 1869 return ((CCMask & SystemZ::CCMASK_CMP_EQ) | 1870 (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) | 1871 (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) | 1872 (CCMask & SystemZ::CCMASK_CMP_UO)); 1873 } 1874 1875 MachineBasicBlock *SystemZ::emitBlockAfter(MachineBasicBlock *MBB) { 1876 MachineFunction &MF = *MBB->getParent(); 1877 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 1878 MF.insert(std::next(MachineFunction::iterator(MBB)), NewMBB); 1879 return NewMBB; 1880 } 1881 1882 MachineBasicBlock *SystemZ::splitBlockAfter(MachineBasicBlock::iterator MI, 1883 MachineBasicBlock *MBB) { 1884 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 1885 NewMBB->splice(NewMBB->begin(), MBB, 1886 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 1887 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 1888 return NewMBB; 1889 } 1890 1891 MachineBasicBlock *SystemZ::splitBlockBefore(MachineBasicBlock::iterator MI, 1892 MachineBasicBlock *MBB) { 1893 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 1894 NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end()); 1895 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 1896 return NewMBB; 1897 } 1898 1899 unsigned SystemZInstrInfo::getLoadAndTrap(unsigned Opcode) const { 1900 if (!STI.hasLoadAndTrap()) 1901 return 0; 1902 switch (Opcode) { 1903 case SystemZ::L: 1904 case SystemZ::LY: 1905 return SystemZ::LAT; 1906 case SystemZ::LG: 1907 return SystemZ::LGAT; 1908 case SystemZ::LFH: 1909 return SystemZ::LFHAT; 1910 case SystemZ::LLGF: 1911 return SystemZ::LLGFAT; 1912 case SystemZ::LLGT: 1913 return SystemZ::LLGTAT; 1914 } 1915 return 0; 1916 } 1917 1918 void SystemZInstrInfo::loadImmediate(MachineBasicBlock &MBB, 1919 MachineBasicBlock::iterator MBBI, 1920 unsigned Reg, uint64_t Value) const { 1921 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 1922 unsigned Opcode; 1923 if (isInt<16>(Value)) 1924 Opcode = SystemZ::LGHI; 1925 else if (SystemZ::isImmLL(Value)) 1926 Opcode = SystemZ::LLILL; 1927 else if (SystemZ::isImmLH(Value)) { 1928 Opcode = SystemZ::LLILH; 1929 Value >>= 16; 1930 } else { 1931 assert(isInt<32>(Value) && "Huge values not handled yet"); 1932 Opcode = SystemZ::LGFI; 1933 } 1934 BuildMI(MBB, MBBI, DL, get(Opcode), Reg).addImm(Value); 1935 } 1936 1937 bool SystemZInstrInfo::verifyInstruction(const MachineInstr &MI, 1938 StringRef &ErrInfo) const { 1939 const MCInstrDesc &MCID = MI.getDesc(); 1940 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { 1941 if (I >= MCID.getNumOperands()) 1942 break; 1943 const MachineOperand &Op = MI.getOperand(I); 1944 const MCOperandInfo &MCOI = MCID.OpInfo[I]; 1945 // Addressing modes have register and immediate operands. Op should be a 1946 // register (or frame index) operand if MCOI.RegClass contains a valid 1947 // register class, or an immediate otherwise. 1948 if (MCOI.OperandType == MCOI::OPERAND_MEMORY && 1949 ((MCOI.RegClass != -1 && !Op.isReg() && !Op.isFI()) || 1950 (MCOI.RegClass == -1 && !Op.isImm()))) { 1951 ErrInfo = "Addressing mode operands corrupt!"; 1952 return false; 1953 } 1954 } 1955 1956 return true; 1957 } 1958 1959 bool SystemZInstrInfo:: 1960 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, 1961 const MachineInstr &MIb) const { 1962 1963 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) 1964 return false; 1965 1966 // If mem-operands show that the same address Value is used by both 1967 // instructions, check for non-overlapping offsets and widths. Not 1968 // sure if a register based analysis would be an improvement... 1969 1970 MachineMemOperand *MMOa = *MIa.memoperands_begin(); 1971 MachineMemOperand *MMOb = *MIb.memoperands_begin(); 1972 const Value *VALa = MMOa->getValue(); 1973 const Value *VALb = MMOb->getValue(); 1974 bool SameVal = (VALa && VALb && (VALa == VALb)); 1975 if (!SameVal) { 1976 const PseudoSourceValue *PSVa = MMOa->getPseudoValue(); 1977 const PseudoSourceValue *PSVb = MMOb->getPseudoValue(); 1978 if (PSVa && PSVb && (PSVa == PSVb)) 1979 SameVal = true; 1980 } 1981 if (SameVal) { 1982 int OffsetA = MMOa->getOffset(), OffsetB = MMOb->getOffset(); 1983 int WidthA = MMOa->getSize(), WidthB = MMOb->getSize(); 1984 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; 1985 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; 1986 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 1987 if (LowOffset + LowWidth <= HighOffset) 1988 return true; 1989 } 1990 1991 return false; 1992 } 1993