1 //===-- SystemZInstrInfo.cpp - SystemZ instruction information ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the SystemZ implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "SystemZInstrInfo.h" 14 #include "MCTargetDesc/SystemZMCTargetDesc.h" 15 #include "SystemZ.h" 16 #include "SystemZInstrBuilder.h" 17 #include "SystemZSubtarget.h" 18 #include "llvm/ADT/Statistic.h" 19 #include "llvm/CodeGen/LiveInterval.h" 20 #include "llvm/CodeGen/LiveIntervals.h" 21 #include "llvm/CodeGen/LiveVariables.h" 22 #include "llvm/CodeGen/MachineBasicBlock.h" 23 #include "llvm/CodeGen/MachineFrameInfo.h" 24 #include "llvm/CodeGen/MachineFunction.h" 25 #include "llvm/CodeGen/MachineInstr.h" 26 #include "llvm/CodeGen/MachineMemOperand.h" 27 #include "llvm/CodeGen/MachineOperand.h" 28 #include "llvm/CodeGen/MachineRegisterInfo.h" 29 #include "llvm/CodeGen/SlotIndexes.h" 30 #include "llvm/CodeGen/TargetInstrInfo.h" 31 #include "llvm/CodeGen/TargetSubtargetInfo.h" 32 #include "llvm/MC/MCInstrDesc.h" 33 #include "llvm/MC/MCRegisterInfo.h" 34 #include "llvm/Support/BranchProbability.h" 35 #include "llvm/Support/ErrorHandling.h" 36 #include "llvm/Support/MathExtras.h" 37 #include "llvm/Target/TargetMachine.h" 38 #include <cassert> 39 #include <cstdint> 40 #include <iterator> 41 42 using namespace llvm; 43 44 #define GET_INSTRINFO_CTOR_DTOR 45 #define GET_INSTRMAP_INFO 46 #include "SystemZGenInstrInfo.inc" 47 48 #define DEBUG_TYPE "systemz-II" 49 50 // Return a mask with Count low bits set. 51 static uint64_t allOnes(unsigned int Count) { 52 return Count == 0 ? 0 : (uint64_t(1) << (Count - 1) << 1) - 1; 53 } 54 55 // Pin the vtable to this file. 56 void SystemZInstrInfo::anchor() {} 57 58 SystemZInstrInfo::SystemZInstrInfo(SystemZSubtarget &sti) 59 : SystemZGenInstrInfo(SystemZ::ADJCALLSTACKDOWN, SystemZ::ADJCALLSTACKUP), 60 RI(), STI(sti) { 61 } 62 63 // MI is a 128-bit load or store. Split it into two 64-bit loads or stores, 64 // each having the opcode given by NewOpcode. 65 void SystemZInstrInfo::splitMove(MachineBasicBlock::iterator MI, 66 unsigned NewOpcode) const { 67 MachineBasicBlock *MBB = MI->getParent(); 68 MachineFunction &MF = *MBB->getParent(); 69 70 // Get two load or store instructions. Use the original instruction for one 71 // of them (arbitrarily the second here) and create a clone for the other. 72 MachineInstr *EarlierMI = MF.CloneMachineInstr(&*MI); 73 MBB->insert(MI, EarlierMI); 74 75 // Set up the two 64-bit registers and remember super reg and its flags. 76 MachineOperand &HighRegOp = EarlierMI->getOperand(0); 77 MachineOperand &LowRegOp = MI->getOperand(0); 78 Register Reg128 = LowRegOp.getReg(); 79 unsigned Reg128Killed = getKillRegState(LowRegOp.isKill()); 80 unsigned Reg128Undef = getUndefRegState(LowRegOp.isUndef()); 81 HighRegOp.setReg(RI.getSubReg(HighRegOp.getReg(), SystemZ::subreg_h64)); 82 LowRegOp.setReg(RI.getSubReg(LowRegOp.getReg(), SystemZ::subreg_l64)); 83 84 if (MI->mayStore()) { 85 // Add implicit uses of the super register in case one of the subregs is 86 // undefined. We could track liveness and skip storing an undefined 87 // subreg, but this is hopefully rare (discovered with llvm-stress). 88 // If Reg128 was killed, set kill flag on MI. 89 unsigned Reg128UndefImpl = (Reg128Undef | RegState::Implicit); 90 MachineInstrBuilder(MF, EarlierMI).addReg(Reg128, Reg128UndefImpl); 91 MachineInstrBuilder(MF, MI).addReg(Reg128, (Reg128UndefImpl | Reg128Killed)); 92 } 93 94 // The address in the first (high) instruction is already correct. 95 // Adjust the offset in the second (low) instruction. 96 MachineOperand &HighOffsetOp = EarlierMI->getOperand(2); 97 MachineOperand &LowOffsetOp = MI->getOperand(2); 98 LowOffsetOp.setImm(LowOffsetOp.getImm() + 8); 99 100 // Clear the kill flags on the registers in the first instruction. 101 if (EarlierMI->getOperand(0).isReg() && EarlierMI->getOperand(0).isUse()) 102 EarlierMI->getOperand(0).setIsKill(false); 103 EarlierMI->getOperand(1).setIsKill(false); 104 EarlierMI->getOperand(3).setIsKill(false); 105 106 // Set the opcodes. 107 unsigned HighOpcode = getOpcodeForOffset(NewOpcode, HighOffsetOp.getImm()); 108 unsigned LowOpcode = getOpcodeForOffset(NewOpcode, LowOffsetOp.getImm()); 109 assert(HighOpcode && LowOpcode && "Both offsets should be in range"); 110 111 EarlierMI->setDesc(get(HighOpcode)); 112 MI->setDesc(get(LowOpcode)); 113 } 114 115 // Split ADJDYNALLOC instruction MI. 116 void SystemZInstrInfo::splitAdjDynAlloc(MachineBasicBlock::iterator MI) const { 117 MachineBasicBlock *MBB = MI->getParent(); 118 MachineFunction &MF = *MBB->getParent(); 119 MachineFrameInfo &MFFrame = MF.getFrameInfo(); 120 MachineOperand &OffsetMO = MI->getOperand(2); 121 122 uint64_t Offset = (MFFrame.getMaxCallFrameSize() + 123 SystemZMC::CallFrameSize + 124 OffsetMO.getImm()); 125 unsigned NewOpcode = getOpcodeForOffset(SystemZ::LA, Offset); 126 assert(NewOpcode && "No support for huge argument lists yet"); 127 MI->setDesc(get(NewOpcode)); 128 OffsetMO.setImm(Offset); 129 } 130 131 // MI is an RI-style pseudo instruction. Replace it with LowOpcode 132 // if the first operand is a low GR32 and HighOpcode if the first operand 133 // is a high GR32. ConvertHigh is true if LowOpcode takes a signed operand 134 // and HighOpcode takes an unsigned 32-bit operand. In those cases, 135 // MI has the same kind of operand as LowOpcode, so needs to be converted 136 // if HighOpcode is used. 137 void SystemZInstrInfo::expandRIPseudo(MachineInstr &MI, unsigned LowOpcode, 138 unsigned HighOpcode, 139 bool ConvertHigh) const { 140 Register Reg = MI.getOperand(0).getReg(); 141 bool IsHigh = SystemZ::isHighReg(Reg); 142 MI.setDesc(get(IsHigh ? HighOpcode : LowOpcode)); 143 if (IsHigh && ConvertHigh) 144 MI.getOperand(1).setImm(uint32_t(MI.getOperand(1).getImm())); 145 } 146 147 // MI is a three-operand RIE-style pseudo instruction. Replace it with 148 // LowOpcodeK if the registers are both low GR32s, otherwise use a move 149 // followed by HighOpcode or LowOpcode, depending on whether the target 150 // is a high or low GR32. 151 void SystemZInstrInfo::expandRIEPseudo(MachineInstr &MI, unsigned LowOpcode, 152 unsigned LowOpcodeK, 153 unsigned HighOpcode) const { 154 Register DestReg = MI.getOperand(0).getReg(); 155 Register SrcReg = MI.getOperand(1).getReg(); 156 bool DestIsHigh = SystemZ::isHighReg(DestReg); 157 bool SrcIsHigh = SystemZ::isHighReg(SrcReg); 158 if (!DestIsHigh && !SrcIsHigh) 159 MI.setDesc(get(LowOpcodeK)); 160 else { 161 if (DestReg != SrcReg) { 162 emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, SrcReg, 163 SystemZ::LR, 32, MI.getOperand(1).isKill(), 164 MI.getOperand(1).isUndef()); 165 MI.getOperand(1).setReg(DestReg); 166 } 167 MI.setDesc(get(DestIsHigh ? HighOpcode : LowOpcode)); 168 MI.tieOperands(0, 1); 169 } 170 } 171 172 // MI is an RXY-style pseudo instruction. Replace it with LowOpcode 173 // if the first operand is a low GR32 and HighOpcode if the first operand 174 // is a high GR32. 175 void SystemZInstrInfo::expandRXYPseudo(MachineInstr &MI, unsigned LowOpcode, 176 unsigned HighOpcode) const { 177 Register Reg = MI.getOperand(0).getReg(); 178 unsigned Opcode = getOpcodeForOffset( 179 SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode, 180 MI.getOperand(2).getImm()); 181 MI.setDesc(get(Opcode)); 182 } 183 184 // MI is a load-on-condition pseudo instruction with a single register 185 // (source or destination) operand. Replace it with LowOpcode if the 186 // register is a low GR32 and HighOpcode if the register is a high GR32. 187 void SystemZInstrInfo::expandLOCPseudo(MachineInstr &MI, unsigned LowOpcode, 188 unsigned HighOpcode) const { 189 Register Reg = MI.getOperand(0).getReg(); 190 unsigned Opcode = SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode; 191 MI.setDesc(get(Opcode)); 192 } 193 194 // MI is an RR-style pseudo instruction that zero-extends the low Size bits 195 // of one GRX32 into another. Replace it with LowOpcode if both operands 196 // are low registers, otherwise use RISB[LH]G. 197 void SystemZInstrInfo::expandZExtPseudo(MachineInstr &MI, unsigned LowOpcode, 198 unsigned Size) const { 199 MachineInstrBuilder MIB = 200 emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), 201 MI.getOperand(0).getReg(), MI.getOperand(1).getReg(), LowOpcode, 202 Size, MI.getOperand(1).isKill(), MI.getOperand(1).isUndef()); 203 204 // Keep the remaining operands as-is. 205 for (unsigned I = 2; I < MI.getNumOperands(); ++I) 206 MIB.add(MI.getOperand(I)); 207 208 MI.eraseFromParent(); 209 } 210 211 void SystemZInstrInfo::expandLoadStackGuard(MachineInstr *MI) const { 212 MachineBasicBlock *MBB = MI->getParent(); 213 MachineFunction &MF = *MBB->getParent(); 214 const Register Reg64 = MI->getOperand(0).getReg(); 215 const Register Reg32 = RI.getSubReg(Reg64, SystemZ::subreg_l32); 216 217 // EAR can only load the low subregister so us a shift for %a0 to produce 218 // the GR containing %a0 and %a1. 219 220 // ear <reg>, %a0 221 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32) 222 .addReg(SystemZ::A0) 223 .addReg(Reg64, RegState::ImplicitDefine); 224 225 // sllg <reg>, <reg>, 32 226 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::SLLG), Reg64) 227 .addReg(Reg64) 228 .addReg(0) 229 .addImm(32); 230 231 // ear <reg>, %a1 232 BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32) 233 .addReg(SystemZ::A1); 234 235 // lg <reg>, 40(<reg>) 236 MI->setDesc(get(SystemZ::LG)); 237 MachineInstrBuilder(MF, MI).addReg(Reg64).addImm(40).addReg(0); 238 } 239 240 // Emit a zero-extending move from 32-bit GPR SrcReg to 32-bit GPR 241 // DestReg before MBBI in MBB. Use LowLowOpcode when both DestReg and SrcReg 242 // are low registers, otherwise use RISB[LH]G. Size is the number of bits 243 // taken from the low end of SrcReg (8 for LLCR, 16 for LLHR and 32 for LR). 244 // KillSrc is true if this move is the last use of SrcReg. 245 MachineInstrBuilder 246 SystemZInstrInfo::emitGRX32Move(MachineBasicBlock &MBB, 247 MachineBasicBlock::iterator MBBI, 248 const DebugLoc &DL, unsigned DestReg, 249 unsigned SrcReg, unsigned LowLowOpcode, 250 unsigned Size, bool KillSrc, 251 bool UndefSrc) const { 252 unsigned Opcode; 253 bool DestIsHigh = SystemZ::isHighReg(DestReg); 254 bool SrcIsHigh = SystemZ::isHighReg(SrcReg); 255 if (DestIsHigh && SrcIsHigh) 256 Opcode = SystemZ::RISBHH; 257 else if (DestIsHigh && !SrcIsHigh) 258 Opcode = SystemZ::RISBHL; 259 else if (!DestIsHigh && SrcIsHigh) 260 Opcode = SystemZ::RISBLH; 261 else { 262 return BuildMI(MBB, MBBI, DL, get(LowLowOpcode), DestReg) 263 .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)); 264 } 265 unsigned Rotate = (DestIsHigh != SrcIsHigh ? 32 : 0); 266 return BuildMI(MBB, MBBI, DL, get(Opcode), DestReg) 267 .addReg(DestReg, RegState::Undef) 268 .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc)) 269 .addImm(32 - Size).addImm(128 + 31).addImm(Rotate); 270 } 271 272 MachineInstr *SystemZInstrInfo::commuteInstructionImpl(MachineInstr &MI, 273 bool NewMI, 274 unsigned OpIdx1, 275 unsigned OpIdx2) const { 276 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { 277 if (NewMI) 278 return *MI.getParent()->getParent()->CloneMachineInstr(&MI); 279 return MI; 280 }; 281 282 switch (MI.getOpcode()) { 283 case SystemZ::SELRMux: 284 case SystemZ::SELFHR: 285 case SystemZ::SELR: 286 case SystemZ::SELGR: 287 case SystemZ::LOCRMux: 288 case SystemZ::LOCFHR: 289 case SystemZ::LOCR: 290 case SystemZ::LOCGR: { 291 auto &WorkingMI = cloneIfNew(MI); 292 // Invert condition. 293 unsigned CCValid = WorkingMI.getOperand(3).getImm(); 294 unsigned CCMask = WorkingMI.getOperand(4).getImm(); 295 WorkingMI.getOperand(4).setImm(CCMask ^ CCValid); 296 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 297 OpIdx1, OpIdx2); 298 } 299 default: 300 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 301 } 302 } 303 304 // If MI is a simple load or store for a frame object, return the register 305 // it loads or stores and set FrameIndex to the index of the frame object. 306 // Return 0 otherwise. 307 // 308 // Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores. 309 static int isSimpleMove(const MachineInstr &MI, int &FrameIndex, 310 unsigned Flag) { 311 const MCInstrDesc &MCID = MI.getDesc(); 312 if ((MCID.TSFlags & Flag) && MI.getOperand(1).isFI() && 313 MI.getOperand(2).getImm() == 0 && MI.getOperand(3).getReg() == 0) { 314 FrameIndex = MI.getOperand(1).getIndex(); 315 return MI.getOperand(0).getReg(); 316 } 317 return 0; 318 } 319 320 unsigned SystemZInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 321 int &FrameIndex) const { 322 return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXLoad); 323 } 324 325 unsigned SystemZInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 326 int &FrameIndex) const { 327 return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXStore); 328 } 329 330 bool SystemZInstrInfo::isStackSlotCopy(const MachineInstr &MI, 331 int &DestFrameIndex, 332 int &SrcFrameIndex) const { 333 // Check for MVC 0(Length,FI1),0(FI2) 334 const MachineFrameInfo &MFI = MI.getParent()->getParent()->getFrameInfo(); 335 if (MI.getOpcode() != SystemZ::MVC || !MI.getOperand(0).isFI() || 336 MI.getOperand(1).getImm() != 0 || !MI.getOperand(3).isFI() || 337 MI.getOperand(4).getImm() != 0) 338 return false; 339 340 // Check that Length covers the full slots. 341 int64_t Length = MI.getOperand(2).getImm(); 342 unsigned FI1 = MI.getOperand(0).getIndex(); 343 unsigned FI2 = MI.getOperand(3).getIndex(); 344 if (MFI.getObjectSize(FI1) != Length || 345 MFI.getObjectSize(FI2) != Length) 346 return false; 347 348 DestFrameIndex = FI1; 349 SrcFrameIndex = FI2; 350 return true; 351 } 352 353 bool SystemZInstrInfo::analyzeBranch(MachineBasicBlock &MBB, 354 MachineBasicBlock *&TBB, 355 MachineBasicBlock *&FBB, 356 SmallVectorImpl<MachineOperand> &Cond, 357 bool AllowModify) const { 358 // Most of the code and comments here are boilerplate. 359 360 // Start from the bottom of the block and work up, examining the 361 // terminator instructions. 362 MachineBasicBlock::iterator I = MBB.end(); 363 while (I != MBB.begin()) { 364 --I; 365 if (I->isDebugInstr()) 366 continue; 367 368 // Working from the bottom, when we see a non-terminator instruction, we're 369 // done. 370 if (!isUnpredicatedTerminator(*I)) 371 break; 372 373 // A terminator that isn't a branch can't easily be handled by this 374 // analysis. 375 if (!I->isBranch()) 376 return true; 377 378 // Can't handle indirect branches. 379 SystemZII::Branch Branch(getBranchInfo(*I)); 380 if (!Branch.hasMBBTarget()) 381 return true; 382 383 // Punt on compound branches. 384 if (Branch.Type != SystemZII::BranchNormal) 385 return true; 386 387 if (Branch.CCMask == SystemZ::CCMASK_ANY) { 388 // Handle unconditional branches. 389 if (!AllowModify) { 390 TBB = Branch.getMBBTarget(); 391 continue; 392 } 393 394 // If the block has any instructions after a JMP, delete them. 395 while (std::next(I) != MBB.end()) 396 std::next(I)->eraseFromParent(); 397 398 Cond.clear(); 399 FBB = nullptr; 400 401 // Delete the JMP if it's equivalent to a fall-through. 402 if (MBB.isLayoutSuccessor(Branch.getMBBTarget())) { 403 TBB = nullptr; 404 I->eraseFromParent(); 405 I = MBB.end(); 406 continue; 407 } 408 409 // TBB is used to indicate the unconditinal destination. 410 TBB = Branch.getMBBTarget(); 411 continue; 412 } 413 414 // Working from the bottom, handle the first conditional branch. 415 if (Cond.empty()) { 416 // FIXME: add X86-style branch swap 417 FBB = TBB; 418 TBB = Branch.getMBBTarget(); 419 Cond.push_back(MachineOperand::CreateImm(Branch.CCValid)); 420 Cond.push_back(MachineOperand::CreateImm(Branch.CCMask)); 421 continue; 422 } 423 424 // Handle subsequent conditional branches. 425 assert(Cond.size() == 2 && TBB && "Should have seen a conditional branch"); 426 427 // Only handle the case where all conditional branches branch to the same 428 // destination. 429 if (TBB != Branch.getMBBTarget()) 430 return true; 431 432 // If the conditions are the same, we can leave them alone. 433 unsigned OldCCValid = Cond[0].getImm(); 434 unsigned OldCCMask = Cond[1].getImm(); 435 if (OldCCValid == Branch.CCValid && OldCCMask == Branch.CCMask) 436 continue; 437 438 // FIXME: Try combining conditions like X86 does. Should be easy on Z! 439 return false; 440 } 441 442 return false; 443 } 444 445 unsigned SystemZInstrInfo::removeBranch(MachineBasicBlock &MBB, 446 int *BytesRemoved) const { 447 assert(!BytesRemoved && "code size not handled"); 448 449 // Most of the code and comments here are boilerplate. 450 MachineBasicBlock::iterator I = MBB.end(); 451 unsigned Count = 0; 452 453 while (I != MBB.begin()) { 454 --I; 455 if (I->isDebugInstr()) 456 continue; 457 if (!I->isBranch()) 458 break; 459 if (!getBranchInfo(*I).hasMBBTarget()) 460 break; 461 // Remove the branch. 462 I->eraseFromParent(); 463 I = MBB.end(); 464 ++Count; 465 } 466 467 return Count; 468 } 469 470 bool SystemZInstrInfo:: 471 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { 472 assert(Cond.size() == 2 && "Invalid condition"); 473 Cond[1].setImm(Cond[1].getImm() ^ Cond[0].getImm()); 474 return false; 475 } 476 477 unsigned SystemZInstrInfo::insertBranch(MachineBasicBlock &MBB, 478 MachineBasicBlock *TBB, 479 MachineBasicBlock *FBB, 480 ArrayRef<MachineOperand> Cond, 481 const DebugLoc &DL, 482 int *BytesAdded) const { 483 // In this function we output 32-bit branches, which should always 484 // have enough range. They can be shortened and relaxed by later code 485 // in the pipeline, if desired. 486 487 // Shouldn't be a fall through. 488 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 489 assert((Cond.size() == 2 || Cond.size() == 0) && 490 "SystemZ branch conditions have one component!"); 491 assert(!BytesAdded && "code size not handled"); 492 493 if (Cond.empty()) { 494 // Unconditional branch? 495 assert(!FBB && "Unconditional branch with multiple successors!"); 496 BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(TBB); 497 return 1; 498 } 499 500 // Conditional branch. 501 unsigned Count = 0; 502 unsigned CCValid = Cond[0].getImm(); 503 unsigned CCMask = Cond[1].getImm(); 504 BuildMI(&MBB, DL, get(SystemZ::BRC)) 505 .addImm(CCValid).addImm(CCMask).addMBB(TBB); 506 ++Count; 507 508 if (FBB) { 509 // Two-way Conditional branch. Insert the second branch. 510 BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(FBB); 511 ++Count; 512 } 513 return Count; 514 } 515 516 bool SystemZInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 517 Register &SrcReg2, int &Mask, 518 int &Value) const { 519 assert(MI.isCompare() && "Caller should have checked for a comparison"); 520 521 if (MI.getNumExplicitOperands() == 2 && MI.getOperand(0).isReg() && 522 MI.getOperand(1).isImm()) { 523 SrcReg = MI.getOperand(0).getReg(); 524 SrcReg2 = 0; 525 Value = MI.getOperand(1).getImm(); 526 Mask = ~0; 527 return true; 528 } 529 530 return false; 531 } 532 533 bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 534 ArrayRef<MachineOperand> Pred, 535 Register DstReg, Register TrueReg, 536 Register FalseReg, int &CondCycles, 537 int &TrueCycles, 538 int &FalseCycles) const { 539 // Not all subtargets have LOCR instructions. 540 if (!STI.hasLoadStoreOnCond()) 541 return false; 542 if (Pred.size() != 2) 543 return false; 544 545 // Check register classes. 546 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 547 const TargetRegisterClass *RC = 548 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); 549 if (!RC) 550 return false; 551 552 // We have LOCR instructions for 32 and 64 bit general purpose registers. 553 if ((STI.hasLoadStoreOnCond2() && 554 SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) || 555 SystemZ::GR32BitRegClass.hasSubClassEq(RC) || 556 SystemZ::GR64BitRegClass.hasSubClassEq(RC)) { 557 CondCycles = 2; 558 TrueCycles = 2; 559 FalseCycles = 2; 560 return true; 561 } 562 563 // Can't do anything else. 564 return false; 565 } 566 567 void SystemZInstrInfo::insertSelect(MachineBasicBlock &MBB, 568 MachineBasicBlock::iterator I, 569 const DebugLoc &DL, Register DstReg, 570 ArrayRef<MachineOperand> Pred, 571 Register TrueReg, 572 Register FalseReg) const { 573 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 574 const TargetRegisterClass *RC = MRI.getRegClass(DstReg); 575 576 assert(Pred.size() == 2 && "Invalid condition"); 577 unsigned CCValid = Pred[0].getImm(); 578 unsigned CCMask = Pred[1].getImm(); 579 580 unsigned Opc; 581 if (SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) { 582 if (STI.hasMiscellaneousExtensions3()) 583 Opc = SystemZ::SELRMux; 584 else if (STI.hasLoadStoreOnCond2()) 585 Opc = SystemZ::LOCRMux; 586 else { 587 Opc = SystemZ::LOCR; 588 MRI.constrainRegClass(DstReg, &SystemZ::GR32BitRegClass); 589 Register TReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); 590 Register FReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); 591 BuildMI(MBB, I, DL, get(TargetOpcode::COPY), TReg).addReg(TrueReg); 592 BuildMI(MBB, I, DL, get(TargetOpcode::COPY), FReg).addReg(FalseReg); 593 TrueReg = TReg; 594 FalseReg = FReg; 595 } 596 } else if (SystemZ::GR64BitRegClass.hasSubClassEq(RC)) { 597 if (STI.hasMiscellaneousExtensions3()) 598 Opc = SystemZ::SELGR; 599 else 600 Opc = SystemZ::LOCGR; 601 } else 602 llvm_unreachable("Invalid register class"); 603 604 BuildMI(MBB, I, DL, get(Opc), DstReg) 605 .addReg(FalseReg).addReg(TrueReg) 606 .addImm(CCValid).addImm(CCMask); 607 } 608 609 bool SystemZInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 610 Register Reg, 611 MachineRegisterInfo *MRI) const { 612 unsigned DefOpc = DefMI.getOpcode(); 613 if (DefOpc != SystemZ::LHIMux && DefOpc != SystemZ::LHI && 614 DefOpc != SystemZ::LGHI) 615 return false; 616 if (DefMI.getOperand(0).getReg() != Reg) 617 return false; 618 int32_t ImmVal = (int32_t)DefMI.getOperand(1).getImm(); 619 620 unsigned UseOpc = UseMI.getOpcode(); 621 unsigned NewUseOpc; 622 unsigned UseIdx; 623 int CommuteIdx = -1; 624 bool TieOps = false; 625 switch (UseOpc) { 626 case SystemZ::SELRMux: 627 TieOps = true; 628 LLVM_FALLTHROUGH; 629 case SystemZ::LOCRMux: 630 if (!STI.hasLoadStoreOnCond2()) 631 return false; 632 NewUseOpc = SystemZ::LOCHIMux; 633 if (UseMI.getOperand(2).getReg() == Reg) 634 UseIdx = 2; 635 else if (UseMI.getOperand(1).getReg() == Reg) 636 UseIdx = 2, CommuteIdx = 1; 637 else 638 return false; 639 break; 640 case SystemZ::SELGR: 641 TieOps = true; 642 LLVM_FALLTHROUGH; 643 case SystemZ::LOCGR: 644 if (!STI.hasLoadStoreOnCond2()) 645 return false; 646 NewUseOpc = SystemZ::LOCGHI; 647 if (UseMI.getOperand(2).getReg() == Reg) 648 UseIdx = 2; 649 else if (UseMI.getOperand(1).getReg() == Reg) 650 UseIdx = 2, CommuteIdx = 1; 651 else 652 return false; 653 break; 654 default: 655 return false; 656 } 657 658 if (CommuteIdx != -1) 659 if (!commuteInstruction(UseMI, false, CommuteIdx, UseIdx)) 660 return false; 661 662 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 663 UseMI.setDesc(get(NewUseOpc)); 664 if (TieOps) 665 UseMI.tieOperands(0, 1); 666 UseMI.getOperand(UseIdx).ChangeToImmediate(ImmVal); 667 if (DeleteDef) 668 DefMI.eraseFromParent(); 669 670 return true; 671 } 672 673 bool SystemZInstrInfo::isPredicable(const MachineInstr &MI) const { 674 unsigned Opcode = MI.getOpcode(); 675 if (Opcode == SystemZ::Return || 676 Opcode == SystemZ::Trap || 677 Opcode == SystemZ::CallJG || 678 Opcode == SystemZ::CallBR) 679 return true; 680 return false; 681 } 682 683 bool SystemZInstrInfo:: 684 isProfitableToIfCvt(MachineBasicBlock &MBB, 685 unsigned NumCycles, unsigned ExtraPredCycles, 686 BranchProbability Probability) const { 687 // Avoid using conditional returns at the end of a loop (since then 688 // we'd need to emit an unconditional branch to the beginning anyway, 689 // making the loop body longer). This doesn't apply for low-probability 690 // loops (eg. compare-and-swap retry), so just decide based on branch 691 // probability instead of looping structure. 692 // However, since Compare and Trap instructions cost the same as a regular 693 // Compare instruction, we should allow the if conversion to convert this 694 // into a Conditional Compare regardless of the branch probability. 695 if (MBB.getLastNonDebugInstr()->getOpcode() != SystemZ::Trap && 696 MBB.succ_empty() && Probability < BranchProbability(1, 8)) 697 return false; 698 // For now only convert single instructions. 699 return NumCycles == 1; 700 } 701 702 bool SystemZInstrInfo:: 703 isProfitableToIfCvt(MachineBasicBlock &TMBB, 704 unsigned NumCyclesT, unsigned ExtraPredCyclesT, 705 MachineBasicBlock &FMBB, 706 unsigned NumCyclesF, unsigned ExtraPredCyclesF, 707 BranchProbability Probability) const { 708 // For now avoid converting mutually-exclusive cases. 709 return false; 710 } 711 712 bool SystemZInstrInfo:: 713 isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, 714 BranchProbability Probability) const { 715 // For now only duplicate single instructions. 716 return NumCycles == 1; 717 } 718 719 bool SystemZInstrInfo::PredicateInstruction( 720 MachineInstr &MI, ArrayRef<MachineOperand> Pred) const { 721 assert(Pred.size() == 2 && "Invalid condition"); 722 unsigned CCValid = Pred[0].getImm(); 723 unsigned CCMask = Pred[1].getImm(); 724 assert(CCMask > 0 && CCMask < 15 && "Invalid predicate"); 725 unsigned Opcode = MI.getOpcode(); 726 if (Opcode == SystemZ::Trap) { 727 MI.setDesc(get(SystemZ::CondTrap)); 728 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 729 .addImm(CCValid).addImm(CCMask) 730 .addReg(SystemZ::CC, RegState::Implicit); 731 return true; 732 } 733 if (Opcode == SystemZ::Return) { 734 MI.setDesc(get(SystemZ::CondReturn)); 735 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 736 .addImm(CCValid).addImm(CCMask) 737 .addReg(SystemZ::CC, RegState::Implicit); 738 return true; 739 } 740 if (Opcode == SystemZ::CallJG) { 741 MachineOperand FirstOp = MI.getOperand(0); 742 const uint32_t *RegMask = MI.getOperand(1).getRegMask(); 743 MI.RemoveOperand(1); 744 MI.RemoveOperand(0); 745 MI.setDesc(get(SystemZ::CallBRCL)); 746 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 747 .addImm(CCValid) 748 .addImm(CCMask) 749 .add(FirstOp) 750 .addRegMask(RegMask) 751 .addReg(SystemZ::CC, RegState::Implicit); 752 return true; 753 } 754 if (Opcode == SystemZ::CallBR) { 755 MachineOperand Target = MI.getOperand(0); 756 const uint32_t *RegMask = MI.getOperand(1).getRegMask(); 757 MI.RemoveOperand(1); 758 MI.RemoveOperand(0); 759 MI.setDesc(get(SystemZ::CallBCR)); 760 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 761 .addImm(CCValid).addImm(CCMask) 762 .add(Target) 763 .addRegMask(RegMask) 764 .addReg(SystemZ::CC, RegState::Implicit); 765 return true; 766 } 767 return false; 768 } 769 770 void SystemZInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 771 MachineBasicBlock::iterator MBBI, 772 const DebugLoc &DL, MCRegister DestReg, 773 MCRegister SrcReg, bool KillSrc) const { 774 // Split 128-bit GPR moves into two 64-bit moves. Add implicit uses of the 775 // super register in case one of the subregs is undefined. 776 // This handles ADDR128 too. 777 if (SystemZ::GR128BitRegClass.contains(DestReg, SrcReg)) { 778 copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_h64), 779 RI.getSubReg(SrcReg, SystemZ::subreg_h64), KillSrc); 780 MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI)) 781 .addReg(SrcReg, RegState::Implicit); 782 copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_l64), 783 RI.getSubReg(SrcReg, SystemZ::subreg_l64), KillSrc); 784 MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI)) 785 .addReg(SrcReg, (getKillRegState(KillSrc) | RegState::Implicit)); 786 return; 787 } 788 789 if (SystemZ::GRX32BitRegClass.contains(DestReg, SrcReg)) { 790 emitGRX32Move(MBB, MBBI, DL, DestReg, SrcReg, SystemZ::LR, 32, KillSrc, 791 false); 792 return; 793 } 794 795 // Move 128-bit floating-point values between VR128 and FP128. 796 if (SystemZ::VR128BitRegClass.contains(DestReg) && 797 SystemZ::FP128BitRegClass.contains(SrcReg)) { 798 MCRegister SrcRegHi = 799 RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_h64), 800 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 801 MCRegister SrcRegLo = 802 RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_l64), 803 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 804 805 BuildMI(MBB, MBBI, DL, get(SystemZ::VMRHG), DestReg) 806 .addReg(SrcRegHi, getKillRegState(KillSrc)) 807 .addReg(SrcRegLo, getKillRegState(KillSrc)); 808 return; 809 } 810 if (SystemZ::FP128BitRegClass.contains(DestReg) && 811 SystemZ::VR128BitRegClass.contains(SrcReg)) { 812 MCRegister DestRegHi = 813 RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_h64), 814 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 815 MCRegister DestRegLo = 816 RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_l64), 817 SystemZ::subreg_h64, &SystemZ::VR128BitRegClass); 818 819 if (DestRegHi != SrcReg) 820 copyPhysReg(MBB, MBBI, DL, DestRegHi, SrcReg, false); 821 BuildMI(MBB, MBBI, DL, get(SystemZ::VREPG), DestRegLo) 822 .addReg(SrcReg, getKillRegState(KillSrc)).addImm(1); 823 return; 824 } 825 826 // Move CC value from a GR32. 827 if (DestReg == SystemZ::CC) { 828 unsigned Opcode = 829 SystemZ::GR32BitRegClass.contains(SrcReg) ? SystemZ::TMLH : SystemZ::TMHH; 830 BuildMI(MBB, MBBI, DL, get(Opcode)) 831 .addReg(SrcReg, getKillRegState(KillSrc)) 832 .addImm(3 << (SystemZ::IPM_CC - 16)); 833 return; 834 } 835 836 // Everything else needs only one instruction. 837 unsigned Opcode; 838 if (SystemZ::GR64BitRegClass.contains(DestReg, SrcReg)) 839 Opcode = SystemZ::LGR; 840 else if (SystemZ::FP32BitRegClass.contains(DestReg, SrcReg)) 841 // For z13 we prefer LDR over LER to avoid partial register dependencies. 842 Opcode = STI.hasVector() ? SystemZ::LDR32 : SystemZ::LER; 843 else if (SystemZ::FP64BitRegClass.contains(DestReg, SrcReg)) 844 Opcode = SystemZ::LDR; 845 else if (SystemZ::FP128BitRegClass.contains(DestReg, SrcReg)) 846 Opcode = SystemZ::LXR; 847 else if (SystemZ::VR32BitRegClass.contains(DestReg, SrcReg)) 848 Opcode = SystemZ::VLR32; 849 else if (SystemZ::VR64BitRegClass.contains(DestReg, SrcReg)) 850 Opcode = SystemZ::VLR64; 851 else if (SystemZ::VR128BitRegClass.contains(DestReg, SrcReg)) 852 Opcode = SystemZ::VLR; 853 else if (SystemZ::AR32BitRegClass.contains(DestReg, SrcReg)) 854 Opcode = SystemZ::CPYA; 855 else 856 llvm_unreachable("Impossible reg-to-reg copy"); 857 858 BuildMI(MBB, MBBI, DL, get(Opcode), DestReg) 859 .addReg(SrcReg, getKillRegState(KillSrc)); 860 } 861 862 void SystemZInstrInfo::storeRegToStackSlot( 863 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, 864 bool isKill, int FrameIdx, const TargetRegisterClass *RC, 865 const TargetRegisterInfo *TRI) const { 866 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 867 868 // Callers may expect a single instruction, so keep 128-bit moves 869 // together for now and lower them after register allocation. 870 unsigned LoadOpcode, StoreOpcode; 871 getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode); 872 addFrameReference(BuildMI(MBB, MBBI, DL, get(StoreOpcode)) 873 .addReg(SrcReg, getKillRegState(isKill)), 874 FrameIdx); 875 } 876 877 void SystemZInstrInfo::loadRegFromStackSlot( 878 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, 879 int FrameIdx, const TargetRegisterClass *RC, 880 const TargetRegisterInfo *TRI) const { 881 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 882 883 // Callers may expect a single instruction, so keep 128-bit moves 884 // together for now and lower them after register allocation. 885 unsigned LoadOpcode, StoreOpcode; 886 getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode); 887 addFrameReference(BuildMI(MBB, MBBI, DL, get(LoadOpcode), DestReg), 888 FrameIdx); 889 } 890 891 // Return true if MI is a simple load or store with a 12-bit displacement 892 // and no index. Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores. 893 static bool isSimpleBD12Move(const MachineInstr *MI, unsigned Flag) { 894 const MCInstrDesc &MCID = MI->getDesc(); 895 return ((MCID.TSFlags & Flag) && 896 isUInt<12>(MI->getOperand(2).getImm()) && 897 MI->getOperand(3).getReg() == 0); 898 } 899 900 namespace { 901 902 struct LogicOp { 903 LogicOp() = default; 904 LogicOp(unsigned regSize, unsigned immLSB, unsigned immSize) 905 : RegSize(regSize), ImmLSB(immLSB), ImmSize(immSize) {} 906 907 explicit operator bool() const { return RegSize; } 908 909 unsigned RegSize = 0; 910 unsigned ImmLSB = 0; 911 unsigned ImmSize = 0; 912 }; 913 914 } // end anonymous namespace 915 916 static LogicOp interpretAndImmediate(unsigned Opcode) { 917 switch (Opcode) { 918 case SystemZ::NILMux: return LogicOp(32, 0, 16); 919 case SystemZ::NIHMux: return LogicOp(32, 16, 16); 920 case SystemZ::NILL64: return LogicOp(64, 0, 16); 921 case SystemZ::NILH64: return LogicOp(64, 16, 16); 922 case SystemZ::NIHL64: return LogicOp(64, 32, 16); 923 case SystemZ::NIHH64: return LogicOp(64, 48, 16); 924 case SystemZ::NIFMux: return LogicOp(32, 0, 32); 925 case SystemZ::NILF64: return LogicOp(64, 0, 32); 926 case SystemZ::NIHF64: return LogicOp(64, 32, 32); 927 default: return LogicOp(); 928 } 929 } 930 931 static void transferDeadCC(MachineInstr *OldMI, MachineInstr *NewMI) { 932 if (OldMI->registerDefIsDead(SystemZ::CC)) { 933 MachineOperand *CCDef = NewMI->findRegisterDefOperand(SystemZ::CC); 934 if (CCDef != nullptr) 935 CCDef->setIsDead(true); 936 } 937 } 938 939 static void transferMIFlag(MachineInstr *OldMI, MachineInstr *NewMI, 940 MachineInstr::MIFlag Flag) { 941 if (OldMI->getFlag(Flag)) 942 NewMI->setFlag(Flag); 943 } 944 945 MachineInstr *SystemZInstrInfo::convertToThreeAddress( 946 MachineFunction::iterator &MFI, MachineInstr &MI, LiveVariables *LV) const { 947 MachineBasicBlock *MBB = MI.getParent(); 948 949 // Try to convert an AND into an RISBG-type instruction. 950 // TODO: It might be beneficial to select RISBG and shorten to AND instead. 951 if (LogicOp And = interpretAndImmediate(MI.getOpcode())) { 952 uint64_t Imm = MI.getOperand(2).getImm() << And.ImmLSB; 953 // AND IMMEDIATE leaves the other bits of the register unchanged. 954 Imm |= allOnes(And.RegSize) & ~(allOnes(And.ImmSize) << And.ImmLSB); 955 unsigned Start, End; 956 if (isRxSBGMask(Imm, And.RegSize, Start, End)) { 957 unsigned NewOpcode; 958 if (And.RegSize == 64) { 959 NewOpcode = SystemZ::RISBG; 960 // Prefer RISBGN if available, since it does not clobber CC. 961 if (STI.hasMiscellaneousExtensions()) 962 NewOpcode = SystemZ::RISBGN; 963 } else { 964 NewOpcode = SystemZ::RISBMux; 965 Start &= 31; 966 End &= 31; 967 } 968 MachineOperand &Dest = MI.getOperand(0); 969 MachineOperand &Src = MI.getOperand(1); 970 MachineInstrBuilder MIB = 971 BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpcode)) 972 .add(Dest) 973 .addReg(0) 974 .addReg(Src.getReg(), getKillRegState(Src.isKill()), 975 Src.getSubReg()) 976 .addImm(Start) 977 .addImm(End + 128) 978 .addImm(0); 979 if (LV) { 980 unsigned NumOps = MI.getNumOperands(); 981 for (unsigned I = 1; I < NumOps; ++I) { 982 MachineOperand &Op = MI.getOperand(I); 983 if (Op.isReg() && Op.isKill()) 984 LV->replaceKillInstruction(Op.getReg(), MI, *MIB); 985 } 986 } 987 transferDeadCC(&MI, MIB); 988 return MIB; 989 } 990 } 991 return nullptr; 992 } 993 994 MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl( 995 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 996 MachineBasicBlock::iterator InsertPt, int FrameIndex, 997 LiveIntervals *LIS, VirtRegMap *VRM) const { 998 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 999 MachineRegisterInfo &MRI = MF.getRegInfo(); 1000 const MachineFrameInfo &MFI = MF.getFrameInfo(); 1001 unsigned Size = MFI.getObjectSize(FrameIndex); 1002 unsigned Opcode = MI.getOpcode(); 1003 1004 // Check CC liveness if new instruction introduces a dead def of CC. 1005 MCRegUnitIterator CCUnit(MCRegister::from(SystemZ::CC), TRI); 1006 SlotIndex MISlot = SlotIndex(); 1007 LiveRange *CCLiveRange = nullptr; 1008 bool CCLiveAtMI = true; 1009 if (LIS) { 1010 MISlot = LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot(); 1011 CCLiveRange = &LIS->getRegUnit(*CCUnit); 1012 CCLiveAtMI = CCLiveRange->liveAt(MISlot); 1013 } 1014 ++CCUnit; 1015 assert(!CCUnit.isValid() && "CC only has one reg unit."); 1016 1017 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 1018 if (!CCLiveAtMI && (Opcode == SystemZ::LA || Opcode == SystemZ::LAY) && 1019 isInt<8>(MI.getOperand(2).getImm()) && !MI.getOperand(3).getReg()) { 1020 // LA(Y) %reg, CONST(%reg) -> AGSI %mem, CONST 1021 MachineInstr *BuiltMI = BuildMI(*InsertPt->getParent(), InsertPt, 1022 MI.getDebugLoc(), get(SystemZ::AGSI)) 1023 .addFrameIndex(FrameIndex) 1024 .addImm(0) 1025 .addImm(MI.getOperand(2).getImm()); 1026 BuiltMI->findRegisterDefOperand(SystemZ::CC)->setIsDead(true); 1027 CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator()); 1028 return BuiltMI; 1029 } 1030 return nullptr; 1031 } 1032 1033 // All other cases require a single operand. 1034 if (Ops.size() != 1) 1035 return nullptr; 1036 1037 unsigned OpNum = Ops[0]; 1038 assert(Size * 8 == 1039 TRI->getRegSizeInBits(*MF.getRegInfo() 1040 .getRegClass(MI.getOperand(OpNum).getReg())) && 1041 "Invalid size combination"); 1042 1043 if ((Opcode == SystemZ::AHI || Opcode == SystemZ::AGHI) && OpNum == 0 && 1044 isInt<8>(MI.getOperand(2).getImm())) { 1045 // A(G)HI %reg, CONST -> A(G)SI %mem, CONST 1046 Opcode = (Opcode == SystemZ::AHI ? SystemZ::ASI : SystemZ::AGSI); 1047 MachineInstr *BuiltMI = 1048 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1049 .addFrameIndex(FrameIndex) 1050 .addImm(0) 1051 .addImm(MI.getOperand(2).getImm()); 1052 transferDeadCC(&MI, BuiltMI); 1053 transferMIFlag(&MI, BuiltMI, MachineInstr::NoSWrap); 1054 return BuiltMI; 1055 } 1056 1057 if ((Opcode == SystemZ::ALFI && OpNum == 0 && 1058 isInt<8>((int32_t)MI.getOperand(2).getImm())) || 1059 (Opcode == SystemZ::ALGFI && OpNum == 0 && 1060 isInt<8>((int64_t)MI.getOperand(2).getImm()))) { 1061 // AL(G)FI %reg, CONST -> AL(G)SI %mem, CONST 1062 Opcode = (Opcode == SystemZ::ALFI ? SystemZ::ALSI : SystemZ::ALGSI); 1063 MachineInstr *BuiltMI = 1064 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1065 .addFrameIndex(FrameIndex) 1066 .addImm(0) 1067 .addImm((int8_t)MI.getOperand(2).getImm()); 1068 transferDeadCC(&MI, BuiltMI); 1069 return BuiltMI; 1070 } 1071 1072 if ((Opcode == SystemZ::SLFI && OpNum == 0 && 1073 isInt<8>((int32_t)-MI.getOperand(2).getImm())) || 1074 (Opcode == SystemZ::SLGFI && OpNum == 0 && 1075 isInt<8>((int64_t)-MI.getOperand(2).getImm()))) { 1076 // SL(G)FI %reg, CONST -> AL(G)SI %mem, -CONST 1077 Opcode = (Opcode == SystemZ::SLFI ? SystemZ::ALSI : SystemZ::ALGSI); 1078 MachineInstr *BuiltMI = 1079 BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode)) 1080 .addFrameIndex(FrameIndex) 1081 .addImm(0) 1082 .addImm((int8_t)-MI.getOperand(2).getImm()); 1083 transferDeadCC(&MI, BuiltMI); 1084 return BuiltMI; 1085 } 1086 1087 unsigned MemImmOpc = 0; 1088 switch (Opcode) { 1089 case SystemZ::LHIMux: 1090 case SystemZ::LHI: MemImmOpc = SystemZ::MVHI; break; 1091 case SystemZ::LGHI: MemImmOpc = SystemZ::MVGHI; break; 1092 case SystemZ::CHIMux: 1093 case SystemZ::CHI: MemImmOpc = SystemZ::CHSI; break; 1094 case SystemZ::CGHI: MemImmOpc = SystemZ::CGHSI; break; 1095 case SystemZ::CLFIMux: 1096 case SystemZ::CLFI: 1097 if (isUInt<16>(MI.getOperand(1).getImm())) 1098 MemImmOpc = SystemZ::CLFHSI; 1099 break; 1100 case SystemZ::CLGFI: 1101 if (isUInt<16>(MI.getOperand(1).getImm())) 1102 MemImmOpc = SystemZ::CLGHSI; 1103 break; 1104 default: break; 1105 } 1106 if (MemImmOpc) 1107 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1108 get(MemImmOpc)) 1109 .addFrameIndex(FrameIndex) 1110 .addImm(0) 1111 .addImm(MI.getOperand(1).getImm()); 1112 1113 if (Opcode == SystemZ::LGDR || Opcode == SystemZ::LDGR) { 1114 bool Op0IsGPR = (Opcode == SystemZ::LGDR); 1115 bool Op1IsGPR = (Opcode == SystemZ::LDGR); 1116 // If we're spilling the destination of an LDGR or LGDR, store the 1117 // source register instead. 1118 if (OpNum == 0) { 1119 unsigned StoreOpcode = Op1IsGPR ? SystemZ::STG : SystemZ::STD; 1120 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1121 get(StoreOpcode)) 1122 .add(MI.getOperand(1)) 1123 .addFrameIndex(FrameIndex) 1124 .addImm(0) 1125 .addReg(0); 1126 } 1127 // If we're spilling the source of an LDGR or LGDR, load the 1128 // destination register instead. 1129 if (OpNum == 1) { 1130 unsigned LoadOpcode = Op0IsGPR ? SystemZ::LG : SystemZ::LD; 1131 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1132 get(LoadOpcode)) 1133 .add(MI.getOperand(0)) 1134 .addFrameIndex(FrameIndex) 1135 .addImm(0) 1136 .addReg(0); 1137 } 1138 } 1139 1140 // Look for cases where the source of a simple store or the destination 1141 // of a simple load is being spilled. Try to use MVC instead. 1142 // 1143 // Although MVC is in practice a fast choice in these cases, it is still 1144 // logically a bytewise copy. This means that we cannot use it if the 1145 // load or store is volatile. We also wouldn't be able to use MVC if 1146 // the two memories partially overlap, but that case cannot occur here, 1147 // because we know that one of the memories is a full frame index. 1148 // 1149 // For performance reasons, we also want to avoid using MVC if the addresses 1150 // might be equal. We don't worry about that case here, because spill slot 1151 // coloring happens later, and because we have special code to remove 1152 // MVCs that turn out to be redundant. 1153 if (OpNum == 0 && MI.hasOneMemOperand()) { 1154 MachineMemOperand *MMO = *MI.memoperands_begin(); 1155 if (MMO->getSize() == Size && !MMO->isVolatile() && !MMO->isAtomic()) { 1156 // Handle conversion of loads. 1157 if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXLoad)) { 1158 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1159 get(SystemZ::MVC)) 1160 .addFrameIndex(FrameIndex) 1161 .addImm(0) 1162 .addImm(Size) 1163 .add(MI.getOperand(1)) 1164 .addImm(MI.getOperand(2).getImm()) 1165 .addMemOperand(MMO); 1166 } 1167 // Handle conversion of stores. 1168 if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXStore)) { 1169 return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), 1170 get(SystemZ::MVC)) 1171 .add(MI.getOperand(1)) 1172 .addImm(MI.getOperand(2).getImm()) 1173 .addImm(Size) 1174 .addFrameIndex(FrameIndex) 1175 .addImm(0) 1176 .addMemOperand(MMO); 1177 } 1178 } 1179 } 1180 1181 // If the spilled operand is the final one or the instruction is 1182 // commutable, try to change <INSN>R into <INSN>. Don't introduce a def of 1183 // CC if it is live and MI does not define it. 1184 unsigned NumOps = MI.getNumExplicitOperands(); 1185 int MemOpcode = SystemZ::getMemOpcode(Opcode); 1186 if (MemOpcode == -1 || 1187 (CCLiveAtMI && !MI.definesRegister(SystemZ::CC) && 1188 get(MemOpcode).hasImplicitDefOfPhysReg(SystemZ::CC))) 1189 return nullptr; 1190 1191 // Check if all other vregs have a usable allocation in the case of vector 1192 // to FP conversion. 1193 const MCInstrDesc &MCID = MI.getDesc(); 1194 for (unsigned I = 0, E = MCID.getNumOperands(); I != E; ++I) { 1195 const MCOperandInfo &MCOI = MCID.OpInfo[I]; 1196 if (MCOI.OperandType != MCOI::OPERAND_REGISTER || I == OpNum) 1197 continue; 1198 const TargetRegisterClass *RC = TRI->getRegClass(MCOI.RegClass); 1199 if (RC == &SystemZ::VR32BitRegClass || RC == &SystemZ::VR64BitRegClass) { 1200 Register Reg = MI.getOperand(I).getReg(); 1201 Register PhysReg = Register::isVirtualRegister(Reg) 1202 ? (VRM ? Register(VRM->getPhys(Reg)) : Register()) 1203 : Reg; 1204 if (!PhysReg || 1205 !(SystemZ::FP32BitRegClass.contains(PhysReg) || 1206 SystemZ::FP64BitRegClass.contains(PhysReg) || 1207 SystemZ::VF128BitRegClass.contains(PhysReg))) 1208 return nullptr; 1209 } 1210 } 1211 // Fused multiply and add/sub need to have the same dst and accumulator reg. 1212 bool FusedFPOp = (Opcode == SystemZ::WFMADB || Opcode == SystemZ::WFMASB || 1213 Opcode == SystemZ::WFMSDB || Opcode == SystemZ::WFMSSB); 1214 if (FusedFPOp) { 1215 Register DstReg = VRM->getPhys(MI.getOperand(0).getReg()); 1216 Register AccReg = VRM->getPhys(MI.getOperand(3).getReg()); 1217 if (OpNum == 0 || OpNum == 3 || DstReg != AccReg) 1218 return nullptr; 1219 } 1220 1221 // Try to swap compare operands if possible. 1222 bool NeedsCommute = false; 1223 if ((MI.getOpcode() == SystemZ::CR || MI.getOpcode() == SystemZ::CGR || 1224 MI.getOpcode() == SystemZ::CLR || MI.getOpcode() == SystemZ::CLGR || 1225 MI.getOpcode() == SystemZ::WFCDB || MI.getOpcode() == SystemZ::WFCSB || 1226 MI.getOpcode() == SystemZ::WFKDB || MI.getOpcode() == SystemZ::WFKSB) && 1227 OpNum == 0 && prepareCompareSwapOperands(MI)) 1228 NeedsCommute = true; 1229 1230 bool CCOperands = false; 1231 if (MI.getOpcode() == SystemZ::LOCRMux || MI.getOpcode() == SystemZ::LOCGR || 1232 MI.getOpcode() == SystemZ::SELRMux || MI.getOpcode() == SystemZ::SELGR) { 1233 assert(MI.getNumOperands() == 6 && NumOps == 5 && 1234 "LOCR/SELR instruction operands corrupt?"); 1235 NumOps -= 2; 1236 CCOperands = true; 1237 } 1238 1239 // See if this is a 3-address instruction that is convertible to 2-address 1240 // and suitable for folding below. Only try this with virtual registers 1241 // and a provided VRM (during regalloc). 1242 if (NumOps == 3 && SystemZ::getTargetMemOpcode(MemOpcode) != -1) { 1243 if (VRM == nullptr) 1244 return nullptr; 1245 else { 1246 Register DstReg = MI.getOperand(0).getReg(); 1247 Register DstPhys = 1248 (Register::isVirtualRegister(DstReg) ? Register(VRM->getPhys(DstReg)) 1249 : DstReg); 1250 Register SrcReg = (OpNum == 2 ? MI.getOperand(1).getReg() 1251 : ((OpNum == 1 && MI.isCommutable()) 1252 ? MI.getOperand(2).getReg() 1253 : Register())); 1254 if (DstPhys && !SystemZ::GRH32BitRegClass.contains(DstPhys) && SrcReg && 1255 Register::isVirtualRegister(SrcReg) && 1256 DstPhys == VRM->getPhys(SrcReg)) 1257 NeedsCommute = (OpNum == 1); 1258 else 1259 return nullptr; 1260 } 1261 } 1262 1263 if ((OpNum == NumOps - 1) || NeedsCommute || FusedFPOp) { 1264 const MCInstrDesc &MemDesc = get(MemOpcode); 1265 uint64_t AccessBytes = SystemZII::getAccessSize(MemDesc.TSFlags); 1266 assert(AccessBytes != 0 && "Size of access should be known"); 1267 assert(AccessBytes <= Size && "Access outside the frame index"); 1268 uint64_t Offset = Size - AccessBytes; 1269 MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, 1270 MI.getDebugLoc(), get(MemOpcode)); 1271 if (MI.isCompare()) { 1272 assert(NumOps == 2 && "Expected 2 register operands for a compare."); 1273 MIB.add(MI.getOperand(NeedsCommute ? 1 : 0)); 1274 } 1275 else if (FusedFPOp) { 1276 MIB.add(MI.getOperand(0)); 1277 MIB.add(MI.getOperand(3)); 1278 MIB.add(MI.getOperand(OpNum == 1 ? 2 : 1)); 1279 } 1280 else { 1281 MIB.add(MI.getOperand(0)); 1282 if (NeedsCommute) 1283 MIB.add(MI.getOperand(2)); 1284 else 1285 for (unsigned I = 1; I < OpNum; ++I) 1286 MIB.add(MI.getOperand(I)); 1287 } 1288 MIB.addFrameIndex(FrameIndex).addImm(Offset); 1289 if (MemDesc.TSFlags & SystemZII::HasIndex) 1290 MIB.addReg(0); 1291 if (CCOperands) { 1292 unsigned CCValid = MI.getOperand(NumOps).getImm(); 1293 unsigned CCMask = MI.getOperand(NumOps + 1).getImm(); 1294 MIB.addImm(CCValid); 1295 MIB.addImm(NeedsCommute ? CCMask ^ CCValid : CCMask); 1296 } 1297 if (MIB->definesRegister(SystemZ::CC) && 1298 (!MI.definesRegister(SystemZ::CC) || 1299 MI.registerDefIsDead(SystemZ::CC))) { 1300 MIB->addRegisterDead(SystemZ::CC, TRI); 1301 if (CCLiveRange) 1302 CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator()); 1303 } 1304 // Constrain the register classes if converted from a vector opcode. The 1305 // allocated regs are in an FP reg-class per previous check above. 1306 for (const MachineOperand &MO : MIB->operands()) 1307 if (MO.isReg() && Register::isVirtualRegister(MO.getReg())) { 1308 unsigned Reg = MO.getReg(); 1309 if (MRI.getRegClass(Reg) == &SystemZ::VR32BitRegClass) 1310 MRI.setRegClass(Reg, &SystemZ::FP32BitRegClass); 1311 else if (MRI.getRegClass(Reg) == &SystemZ::VR64BitRegClass) 1312 MRI.setRegClass(Reg, &SystemZ::FP64BitRegClass); 1313 else if (MRI.getRegClass(Reg) == &SystemZ::VR128BitRegClass) 1314 MRI.setRegClass(Reg, &SystemZ::VF128BitRegClass); 1315 } 1316 1317 transferDeadCC(&MI, MIB); 1318 transferMIFlag(&MI, MIB, MachineInstr::NoSWrap); 1319 transferMIFlag(&MI, MIB, MachineInstr::NoFPExcept); 1320 return MIB; 1321 } 1322 1323 return nullptr; 1324 } 1325 1326 MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl( 1327 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 1328 MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, 1329 LiveIntervals *LIS) const { 1330 return nullptr; 1331 } 1332 1333 bool SystemZInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 1334 switch (MI.getOpcode()) { 1335 case SystemZ::L128: 1336 splitMove(MI, SystemZ::LG); 1337 return true; 1338 1339 case SystemZ::ST128: 1340 splitMove(MI, SystemZ::STG); 1341 return true; 1342 1343 case SystemZ::LX: 1344 splitMove(MI, SystemZ::LD); 1345 return true; 1346 1347 case SystemZ::STX: 1348 splitMove(MI, SystemZ::STD); 1349 return true; 1350 1351 case SystemZ::LBMux: 1352 expandRXYPseudo(MI, SystemZ::LB, SystemZ::LBH); 1353 return true; 1354 1355 case SystemZ::LHMux: 1356 expandRXYPseudo(MI, SystemZ::LH, SystemZ::LHH); 1357 return true; 1358 1359 case SystemZ::LLCRMux: 1360 expandZExtPseudo(MI, SystemZ::LLCR, 8); 1361 return true; 1362 1363 case SystemZ::LLHRMux: 1364 expandZExtPseudo(MI, SystemZ::LLHR, 16); 1365 return true; 1366 1367 case SystemZ::LLCMux: 1368 expandRXYPseudo(MI, SystemZ::LLC, SystemZ::LLCH); 1369 return true; 1370 1371 case SystemZ::LLHMux: 1372 expandRXYPseudo(MI, SystemZ::LLH, SystemZ::LLHH); 1373 return true; 1374 1375 case SystemZ::LMux: 1376 expandRXYPseudo(MI, SystemZ::L, SystemZ::LFH); 1377 return true; 1378 1379 case SystemZ::LOCMux: 1380 expandLOCPseudo(MI, SystemZ::LOC, SystemZ::LOCFH); 1381 return true; 1382 1383 case SystemZ::LOCHIMux: 1384 expandLOCPseudo(MI, SystemZ::LOCHI, SystemZ::LOCHHI); 1385 return true; 1386 1387 case SystemZ::STCMux: 1388 expandRXYPseudo(MI, SystemZ::STC, SystemZ::STCH); 1389 return true; 1390 1391 case SystemZ::STHMux: 1392 expandRXYPseudo(MI, SystemZ::STH, SystemZ::STHH); 1393 return true; 1394 1395 case SystemZ::STMux: 1396 expandRXYPseudo(MI, SystemZ::ST, SystemZ::STFH); 1397 return true; 1398 1399 case SystemZ::STOCMux: 1400 expandLOCPseudo(MI, SystemZ::STOC, SystemZ::STOCFH); 1401 return true; 1402 1403 case SystemZ::LHIMux: 1404 expandRIPseudo(MI, SystemZ::LHI, SystemZ::IIHF, true); 1405 return true; 1406 1407 case SystemZ::IIFMux: 1408 expandRIPseudo(MI, SystemZ::IILF, SystemZ::IIHF, false); 1409 return true; 1410 1411 case SystemZ::IILMux: 1412 expandRIPseudo(MI, SystemZ::IILL, SystemZ::IIHL, false); 1413 return true; 1414 1415 case SystemZ::IIHMux: 1416 expandRIPseudo(MI, SystemZ::IILH, SystemZ::IIHH, false); 1417 return true; 1418 1419 case SystemZ::NIFMux: 1420 expandRIPseudo(MI, SystemZ::NILF, SystemZ::NIHF, false); 1421 return true; 1422 1423 case SystemZ::NILMux: 1424 expandRIPseudo(MI, SystemZ::NILL, SystemZ::NIHL, false); 1425 return true; 1426 1427 case SystemZ::NIHMux: 1428 expandRIPseudo(MI, SystemZ::NILH, SystemZ::NIHH, false); 1429 return true; 1430 1431 case SystemZ::OIFMux: 1432 expandRIPseudo(MI, SystemZ::OILF, SystemZ::OIHF, false); 1433 return true; 1434 1435 case SystemZ::OILMux: 1436 expandRIPseudo(MI, SystemZ::OILL, SystemZ::OIHL, false); 1437 return true; 1438 1439 case SystemZ::OIHMux: 1440 expandRIPseudo(MI, SystemZ::OILH, SystemZ::OIHH, false); 1441 return true; 1442 1443 case SystemZ::XIFMux: 1444 expandRIPseudo(MI, SystemZ::XILF, SystemZ::XIHF, false); 1445 return true; 1446 1447 case SystemZ::TMLMux: 1448 expandRIPseudo(MI, SystemZ::TMLL, SystemZ::TMHL, false); 1449 return true; 1450 1451 case SystemZ::TMHMux: 1452 expandRIPseudo(MI, SystemZ::TMLH, SystemZ::TMHH, false); 1453 return true; 1454 1455 case SystemZ::AHIMux: 1456 expandRIPseudo(MI, SystemZ::AHI, SystemZ::AIH, false); 1457 return true; 1458 1459 case SystemZ::AHIMuxK: 1460 expandRIEPseudo(MI, SystemZ::AHI, SystemZ::AHIK, SystemZ::AIH); 1461 return true; 1462 1463 case SystemZ::AFIMux: 1464 expandRIPseudo(MI, SystemZ::AFI, SystemZ::AIH, false); 1465 return true; 1466 1467 case SystemZ::CHIMux: 1468 expandRIPseudo(MI, SystemZ::CHI, SystemZ::CIH, false); 1469 return true; 1470 1471 case SystemZ::CFIMux: 1472 expandRIPseudo(MI, SystemZ::CFI, SystemZ::CIH, false); 1473 return true; 1474 1475 case SystemZ::CLFIMux: 1476 expandRIPseudo(MI, SystemZ::CLFI, SystemZ::CLIH, false); 1477 return true; 1478 1479 case SystemZ::CMux: 1480 expandRXYPseudo(MI, SystemZ::C, SystemZ::CHF); 1481 return true; 1482 1483 case SystemZ::CLMux: 1484 expandRXYPseudo(MI, SystemZ::CL, SystemZ::CLHF); 1485 return true; 1486 1487 case SystemZ::RISBMux: { 1488 bool DestIsHigh = SystemZ::isHighReg(MI.getOperand(0).getReg()); 1489 bool SrcIsHigh = SystemZ::isHighReg(MI.getOperand(2).getReg()); 1490 if (SrcIsHigh == DestIsHigh) 1491 MI.setDesc(get(DestIsHigh ? SystemZ::RISBHH : SystemZ::RISBLL)); 1492 else { 1493 MI.setDesc(get(DestIsHigh ? SystemZ::RISBHL : SystemZ::RISBLH)); 1494 MI.getOperand(5).setImm(MI.getOperand(5).getImm() ^ 32); 1495 } 1496 return true; 1497 } 1498 1499 case SystemZ::ADJDYNALLOC: 1500 splitAdjDynAlloc(MI); 1501 return true; 1502 1503 case TargetOpcode::LOAD_STACK_GUARD: 1504 expandLoadStackGuard(&MI); 1505 return true; 1506 1507 default: 1508 return false; 1509 } 1510 } 1511 1512 unsigned SystemZInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 1513 if (MI.isInlineAsm()) { 1514 const MachineFunction *MF = MI.getParent()->getParent(); 1515 const char *AsmStr = MI.getOperand(0).getSymbolName(); 1516 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo()); 1517 } 1518 return MI.getDesc().getSize(); 1519 } 1520 1521 SystemZII::Branch 1522 SystemZInstrInfo::getBranchInfo(const MachineInstr &MI) const { 1523 switch (MI.getOpcode()) { 1524 case SystemZ::BR: 1525 case SystemZ::BI: 1526 case SystemZ::J: 1527 case SystemZ::JG: 1528 return SystemZII::Branch(SystemZII::BranchNormal, SystemZ::CCMASK_ANY, 1529 SystemZ::CCMASK_ANY, &MI.getOperand(0)); 1530 1531 case SystemZ::BRC: 1532 case SystemZ::BRCL: 1533 return SystemZII::Branch(SystemZII::BranchNormal, MI.getOperand(0).getImm(), 1534 MI.getOperand(1).getImm(), &MI.getOperand(2)); 1535 1536 case SystemZ::BRCT: 1537 case SystemZ::BRCTH: 1538 return SystemZII::Branch(SystemZII::BranchCT, SystemZ::CCMASK_ICMP, 1539 SystemZ::CCMASK_CMP_NE, &MI.getOperand(2)); 1540 1541 case SystemZ::BRCTG: 1542 return SystemZII::Branch(SystemZII::BranchCTG, SystemZ::CCMASK_ICMP, 1543 SystemZ::CCMASK_CMP_NE, &MI.getOperand(2)); 1544 1545 case SystemZ::CIJ: 1546 case SystemZ::CRJ: 1547 return SystemZII::Branch(SystemZII::BranchC, SystemZ::CCMASK_ICMP, 1548 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1549 1550 case SystemZ::CLIJ: 1551 case SystemZ::CLRJ: 1552 return SystemZII::Branch(SystemZII::BranchCL, SystemZ::CCMASK_ICMP, 1553 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1554 1555 case SystemZ::CGIJ: 1556 case SystemZ::CGRJ: 1557 return SystemZII::Branch(SystemZII::BranchCG, SystemZ::CCMASK_ICMP, 1558 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1559 1560 case SystemZ::CLGIJ: 1561 case SystemZ::CLGRJ: 1562 return SystemZII::Branch(SystemZII::BranchCLG, SystemZ::CCMASK_ICMP, 1563 MI.getOperand(2).getImm(), &MI.getOperand(3)); 1564 1565 case SystemZ::INLINEASM_BR: 1566 // Don't try to analyze asm goto, so pass nullptr as branch target argument. 1567 return SystemZII::Branch(SystemZII::AsmGoto, 0, 0, nullptr); 1568 1569 default: 1570 llvm_unreachable("Unrecognized branch opcode"); 1571 } 1572 } 1573 1574 void SystemZInstrInfo::getLoadStoreOpcodes(const TargetRegisterClass *RC, 1575 unsigned &LoadOpcode, 1576 unsigned &StoreOpcode) const { 1577 if (RC == &SystemZ::GR32BitRegClass || RC == &SystemZ::ADDR32BitRegClass) { 1578 LoadOpcode = SystemZ::L; 1579 StoreOpcode = SystemZ::ST; 1580 } else if (RC == &SystemZ::GRH32BitRegClass) { 1581 LoadOpcode = SystemZ::LFH; 1582 StoreOpcode = SystemZ::STFH; 1583 } else if (RC == &SystemZ::GRX32BitRegClass) { 1584 LoadOpcode = SystemZ::LMux; 1585 StoreOpcode = SystemZ::STMux; 1586 } else if (RC == &SystemZ::GR64BitRegClass || 1587 RC == &SystemZ::ADDR64BitRegClass) { 1588 LoadOpcode = SystemZ::LG; 1589 StoreOpcode = SystemZ::STG; 1590 } else if (RC == &SystemZ::GR128BitRegClass || 1591 RC == &SystemZ::ADDR128BitRegClass) { 1592 LoadOpcode = SystemZ::L128; 1593 StoreOpcode = SystemZ::ST128; 1594 } else if (RC == &SystemZ::FP32BitRegClass) { 1595 LoadOpcode = SystemZ::LE; 1596 StoreOpcode = SystemZ::STE; 1597 } else if (RC == &SystemZ::FP64BitRegClass) { 1598 LoadOpcode = SystemZ::LD; 1599 StoreOpcode = SystemZ::STD; 1600 } else if (RC == &SystemZ::FP128BitRegClass) { 1601 LoadOpcode = SystemZ::LX; 1602 StoreOpcode = SystemZ::STX; 1603 } else if (RC == &SystemZ::VR32BitRegClass) { 1604 LoadOpcode = SystemZ::VL32; 1605 StoreOpcode = SystemZ::VST32; 1606 } else if (RC == &SystemZ::VR64BitRegClass) { 1607 LoadOpcode = SystemZ::VL64; 1608 StoreOpcode = SystemZ::VST64; 1609 } else if (RC == &SystemZ::VF128BitRegClass || 1610 RC == &SystemZ::VR128BitRegClass) { 1611 LoadOpcode = SystemZ::VL; 1612 StoreOpcode = SystemZ::VST; 1613 } else 1614 llvm_unreachable("Unsupported regclass to load or store"); 1615 } 1616 1617 unsigned SystemZInstrInfo::getOpcodeForOffset(unsigned Opcode, 1618 int64_t Offset) const { 1619 const MCInstrDesc &MCID = get(Opcode); 1620 int64_t Offset2 = (MCID.TSFlags & SystemZII::Is128Bit ? Offset + 8 : Offset); 1621 if (isUInt<12>(Offset) && isUInt<12>(Offset2)) { 1622 // Get the instruction to use for unsigned 12-bit displacements. 1623 int Disp12Opcode = SystemZ::getDisp12Opcode(Opcode); 1624 if (Disp12Opcode >= 0) 1625 return Disp12Opcode; 1626 1627 // All address-related instructions can use unsigned 12-bit 1628 // displacements. 1629 return Opcode; 1630 } 1631 if (isInt<20>(Offset) && isInt<20>(Offset2)) { 1632 // Get the instruction to use for signed 20-bit displacements. 1633 int Disp20Opcode = SystemZ::getDisp20Opcode(Opcode); 1634 if (Disp20Opcode >= 0) 1635 return Disp20Opcode; 1636 1637 // Check whether Opcode allows signed 20-bit displacements. 1638 if (MCID.TSFlags & SystemZII::Has20BitOffset) 1639 return Opcode; 1640 } 1641 return 0; 1642 } 1643 1644 unsigned SystemZInstrInfo::getLoadAndTest(unsigned Opcode) const { 1645 switch (Opcode) { 1646 case SystemZ::L: return SystemZ::LT; 1647 case SystemZ::LY: return SystemZ::LT; 1648 case SystemZ::LG: return SystemZ::LTG; 1649 case SystemZ::LGF: return SystemZ::LTGF; 1650 case SystemZ::LR: return SystemZ::LTR; 1651 case SystemZ::LGFR: return SystemZ::LTGFR; 1652 case SystemZ::LGR: return SystemZ::LTGR; 1653 case SystemZ::LER: return SystemZ::LTEBR; 1654 case SystemZ::LDR: return SystemZ::LTDBR; 1655 case SystemZ::LXR: return SystemZ::LTXBR; 1656 case SystemZ::LCDFR: return SystemZ::LCDBR; 1657 case SystemZ::LPDFR: return SystemZ::LPDBR; 1658 case SystemZ::LNDFR: return SystemZ::LNDBR; 1659 case SystemZ::LCDFR_32: return SystemZ::LCEBR; 1660 case SystemZ::LPDFR_32: return SystemZ::LPEBR; 1661 case SystemZ::LNDFR_32: return SystemZ::LNEBR; 1662 // On zEC12 we prefer to use RISBGN. But if there is a chance to 1663 // actually use the condition code, we may turn it back into RISGB. 1664 // Note that RISBG is not really a "load-and-test" instruction, 1665 // but sets the same condition code values, so is OK to use here. 1666 case SystemZ::RISBGN: return SystemZ::RISBG; 1667 default: return 0; 1668 } 1669 } 1670 1671 // Return true if Mask matches the regexp 0*1+0*, given that zero masks 1672 // have already been filtered out. Store the first set bit in LSB and 1673 // the number of set bits in Length if so. 1674 static bool isStringOfOnes(uint64_t Mask, unsigned &LSB, unsigned &Length) { 1675 unsigned First = findFirstSet(Mask); 1676 uint64_t Top = (Mask >> First) + 1; 1677 if ((Top & -Top) == Top) { 1678 LSB = First; 1679 Length = findFirstSet(Top); 1680 return true; 1681 } 1682 return false; 1683 } 1684 1685 bool SystemZInstrInfo::isRxSBGMask(uint64_t Mask, unsigned BitSize, 1686 unsigned &Start, unsigned &End) const { 1687 // Reject trivial all-zero masks. 1688 Mask &= allOnes(BitSize); 1689 if (Mask == 0) 1690 return false; 1691 1692 // Handle the 1+0+ or 0+1+0* cases. Start then specifies the index of 1693 // the msb and End specifies the index of the lsb. 1694 unsigned LSB, Length; 1695 if (isStringOfOnes(Mask, LSB, Length)) { 1696 Start = 63 - (LSB + Length - 1); 1697 End = 63 - LSB; 1698 return true; 1699 } 1700 1701 // Handle the wrap-around 1+0+1+ cases. Start then specifies the msb 1702 // of the low 1s and End specifies the lsb of the high 1s. 1703 if (isStringOfOnes(Mask ^ allOnes(BitSize), LSB, Length)) { 1704 assert(LSB > 0 && "Bottom bit must be set"); 1705 assert(LSB + Length < BitSize && "Top bit must be set"); 1706 Start = 63 - (LSB - 1); 1707 End = 63 - (LSB + Length); 1708 return true; 1709 } 1710 1711 return false; 1712 } 1713 1714 unsigned SystemZInstrInfo::getFusedCompare(unsigned Opcode, 1715 SystemZII::FusedCompareType Type, 1716 const MachineInstr *MI) const { 1717 switch (Opcode) { 1718 case SystemZ::CHI: 1719 case SystemZ::CGHI: 1720 if (!(MI && isInt<8>(MI->getOperand(1).getImm()))) 1721 return 0; 1722 break; 1723 case SystemZ::CLFI: 1724 case SystemZ::CLGFI: 1725 if (!(MI && isUInt<8>(MI->getOperand(1).getImm()))) 1726 return 0; 1727 break; 1728 case SystemZ::CL: 1729 case SystemZ::CLG: 1730 if (!STI.hasMiscellaneousExtensions()) 1731 return 0; 1732 if (!(MI && MI->getOperand(3).getReg() == 0)) 1733 return 0; 1734 break; 1735 } 1736 switch (Type) { 1737 case SystemZII::CompareAndBranch: 1738 switch (Opcode) { 1739 case SystemZ::CR: 1740 return SystemZ::CRJ; 1741 case SystemZ::CGR: 1742 return SystemZ::CGRJ; 1743 case SystemZ::CHI: 1744 return SystemZ::CIJ; 1745 case SystemZ::CGHI: 1746 return SystemZ::CGIJ; 1747 case SystemZ::CLR: 1748 return SystemZ::CLRJ; 1749 case SystemZ::CLGR: 1750 return SystemZ::CLGRJ; 1751 case SystemZ::CLFI: 1752 return SystemZ::CLIJ; 1753 case SystemZ::CLGFI: 1754 return SystemZ::CLGIJ; 1755 default: 1756 return 0; 1757 } 1758 case SystemZII::CompareAndReturn: 1759 switch (Opcode) { 1760 case SystemZ::CR: 1761 return SystemZ::CRBReturn; 1762 case SystemZ::CGR: 1763 return SystemZ::CGRBReturn; 1764 case SystemZ::CHI: 1765 return SystemZ::CIBReturn; 1766 case SystemZ::CGHI: 1767 return SystemZ::CGIBReturn; 1768 case SystemZ::CLR: 1769 return SystemZ::CLRBReturn; 1770 case SystemZ::CLGR: 1771 return SystemZ::CLGRBReturn; 1772 case SystemZ::CLFI: 1773 return SystemZ::CLIBReturn; 1774 case SystemZ::CLGFI: 1775 return SystemZ::CLGIBReturn; 1776 default: 1777 return 0; 1778 } 1779 case SystemZII::CompareAndSibcall: 1780 switch (Opcode) { 1781 case SystemZ::CR: 1782 return SystemZ::CRBCall; 1783 case SystemZ::CGR: 1784 return SystemZ::CGRBCall; 1785 case SystemZ::CHI: 1786 return SystemZ::CIBCall; 1787 case SystemZ::CGHI: 1788 return SystemZ::CGIBCall; 1789 case SystemZ::CLR: 1790 return SystemZ::CLRBCall; 1791 case SystemZ::CLGR: 1792 return SystemZ::CLGRBCall; 1793 case SystemZ::CLFI: 1794 return SystemZ::CLIBCall; 1795 case SystemZ::CLGFI: 1796 return SystemZ::CLGIBCall; 1797 default: 1798 return 0; 1799 } 1800 case SystemZII::CompareAndTrap: 1801 switch (Opcode) { 1802 case SystemZ::CR: 1803 return SystemZ::CRT; 1804 case SystemZ::CGR: 1805 return SystemZ::CGRT; 1806 case SystemZ::CHI: 1807 return SystemZ::CIT; 1808 case SystemZ::CGHI: 1809 return SystemZ::CGIT; 1810 case SystemZ::CLR: 1811 return SystemZ::CLRT; 1812 case SystemZ::CLGR: 1813 return SystemZ::CLGRT; 1814 case SystemZ::CLFI: 1815 return SystemZ::CLFIT; 1816 case SystemZ::CLGFI: 1817 return SystemZ::CLGIT; 1818 case SystemZ::CL: 1819 return SystemZ::CLT; 1820 case SystemZ::CLG: 1821 return SystemZ::CLGT; 1822 default: 1823 return 0; 1824 } 1825 } 1826 return 0; 1827 } 1828 1829 bool SystemZInstrInfo:: 1830 prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const { 1831 assert(MBBI->isCompare() && MBBI->getOperand(0).isReg() && 1832 MBBI->getOperand(1).isReg() && !MBBI->mayLoad() && 1833 "Not a compare reg/reg."); 1834 1835 MachineBasicBlock *MBB = MBBI->getParent(); 1836 bool CCLive = true; 1837 SmallVector<MachineInstr *, 4> CCUsers; 1838 for (MachineBasicBlock::iterator Itr = std::next(MBBI); 1839 Itr != MBB->end(); ++Itr) { 1840 if (Itr->readsRegister(SystemZ::CC)) { 1841 unsigned Flags = Itr->getDesc().TSFlags; 1842 if ((Flags & SystemZII::CCMaskFirst) || (Flags & SystemZII::CCMaskLast)) 1843 CCUsers.push_back(&*Itr); 1844 else 1845 return false; 1846 } 1847 if (Itr->definesRegister(SystemZ::CC)) { 1848 CCLive = false; 1849 break; 1850 } 1851 } 1852 if (CCLive) { 1853 LivePhysRegs LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo()); 1854 LiveRegs.addLiveOuts(*MBB); 1855 if (LiveRegs.contains(SystemZ::CC)) 1856 return false; 1857 } 1858 1859 // Update all CC users. 1860 for (unsigned Idx = 0; Idx < CCUsers.size(); ++Idx) { 1861 unsigned Flags = CCUsers[Idx]->getDesc().TSFlags; 1862 unsigned FirstOpNum = ((Flags & SystemZII::CCMaskFirst) ? 1863 0 : CCUsers[Idx]->getNumExplicitOperands() - 2); 1864 MachineOperand &CCMaskMO = CCUsers[Idx]->getOperand(FirstOpNum + 1); 1865 unsigned NewCCMask = SystemZ::reverseCCMask(CCMaskMO.getImm()); 1866 CCMaskMO.setImm(NewCCMask); 1867 } 1868 1869 return true; 1870 } 1871 1872 unsigned SystemZ::reverseCCMask(unsigned CCMask) { 1873 return ((CCMask & SystemZ::CCMASK_CMP_EQ) | 1874 (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) | 1875 (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) | 1876 (CCMask & SystemZ::CCMASK_CMP_UO)); 1877 } 1878 1879 MachineBasicBlock *SystemZ::emitBlockAfter(MachineBasicBlock *MBB) { 1880 MachineFunction &MF = *MBB->getParent(); 1881 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 1882 MF.insert(std::next(MachineFunction::iterator(MBB)), NewMBB); 1883 return NewMBB; 1884 } 1885 1886 MachineBasicBlock *SystemZ::splitBlockAfter(MachineBasicBlock::iterator MI, 1887 MachineBasicBlock *MBB) { 1888 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 1889 NewMBB->splice(NewMBB->begin(), MBB, 1890 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 1891 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 1892 return NewMBB; 1893 } 1894 1895 MachineBasicBlock *SystemZ::splitBlockBefore(MachineBasicBlock::iterator MI, 1896 MachineBasicBlock *MBB) { 1897 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 1898 NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end()); 1899 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 1900 return NewMBB; 1901 } 1902 1903 unsigned SystemZInstrInfo::getLoadAndTrap(unsigned Opcode) const { 1904 if (!STI.hasLoadAndTrap()) 1905 return 0; 1906 switch (Opcode) { 1907 case SystemZ::L: 1908 case SystemZ::LY: 1909 return SystemZ::LAT; 1910 case SystemZ::LG: 1911 return SystemZ::LGAT; 1912 case SystemZ::LFH: 1913 return SystemZ::LFHAT; 1914 case SystemZ::LLGF: 1915 return SystemZ::LLGFAT; 1916 case SystemZ::LLGT: 1917 return SystemZ::LLGTAT; 1918 } 1919 return 0; 1920 } 1921 1922 void SystemZInstrInfo::loadImmediate(MachineBasicBlock &MBB, 1923 MachineBasicBlock::iterator MBBI, 1924 unsigned Reg, uint64_t Value) const { 1925 DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); 1926 unsigned Opcode; 1927 if (isInt<16>(Value)) 1928 Opcode = SystemZ::LGHI; 1929 else if (SystemZ::isImmLL(Value)) 1930 Opcode = SystemZ::LLILL; 1931 else if (SystemZ::isImmLH(Value)) { 1932 Opcode = SystemZ::LLILH; 1933 Value >>= 16; 1934 } else { 1935 assert(isInt<32>(Value) && "Huge values not handled yet"); 1936 Opcode = SystemZ::LGFI; 1937 } 1938 BuildMI(MBB, MBBI, DL, get(Opcode), Reg).addImm(Value); 1939 } 1940 1941 bool SystemZInstrInfo::verifyInstruction(const MachineInstr &MI, 1942 StringRef &ErrInfo) const { 1943 const MCInstrDesc &MCID = MI.getDesc(); 1944 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { 1945 if (I >= MCID.getNumOperands()) 1946 break; 1947 const MachineOperand &Op = MI.getOperand(I); 1948 const MCOperandInfo &MCOI = MCID.OpInfo[I]; 1949 // Addressing modes have register and immediate operands. Op should be a 1950 // register (or frame index) operand if MCOI.RegClass contains a valid 1951 // register class, or an immediate otherwise. 1952 if (MCOI.OperandType == MCOI::OPERAND_MEMORY && 1953 ((MCOI.RegClass != -1 && !Op.isReg() && !Op.isFI()) || 1954 (MCOI.RegClass == -1 && !Op.isImm()))) { 1955 ErrInfo = "Addressing mode operands corrupt!"; 1956 return false; 1957 } 1958 } 1959 1960 return true; 1961 } 1962 1963 bool SystemZInstrInfo:: 1964 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, 1965 const MachineInstr &MIb) const { 1966 1967 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) 1968 return false; 1969 1970 // If mem-operands show that the same address Value is used by both 1971 // instructions, check for non-overlapping offsets and widths. Not 1972 // sure if a register based analysis would be an improvement... 1973 1974 MachineMemOperand *MMOa = *MIa.memoperands_begin(); 1975 MachineMemOperand *MMOb = *MIb.memoperands_begin(); 1976 const Value *VALa = MMOa->getValue(); 1977 const Value *VALb = MMOb->getValue(); 1978 bool SameVal = (VALa && VALb && (VALa == VALb)); 1979 if (!SameVal) { 1980 const PseudoSourceValue *PSVa = MMOa->getPseudoValue(); 1981 const PseudoSourceValue *PSVb = MMOb->getPseudoValue(); 1982 if (PSVa && PSVb && (PSVa == PSVb)) 1983 SameVal = true; 1984 } 1985 if (SameVal) { 1986 int OffsetA = MMOa->getOffset(), OffsetB = MMOb->getOffset(); 1987 int WidthA = MMOa->getSize(), WidthB = MMOb->getSize(); 1988 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; 1989 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; 1990 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 1991 if (LowOffset + LowWidth <= HighOffset) 1992 return true; 1993 } 1994 1995 return false; 1996 } 1997