1 //===-- TargetInstrInfo.cpp - Target Instruction Information --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/CodeGen/TargetInstrInfo.h" 14 #include "llvm/CodeGen/MachineFrameInfo.h" 15 #include "llvm/CodeGen/MachineInstrBuilder.h" 16 #include "llvm/CodeGen/MachineMemOperand.h" 17 #include "llvm/CodeGen/MachineRegisterInfo.h" 18 #include "llvm/CodeGen/PseudoSourceValue.h" 19 #include "llvm/CodeGen/ScoreboardHazardRecognizer.h" 20 #include "llvm/CodeGen/StackMaps.h" 21 #include "llvm/CodeGen/TargetFrameLowering.h" 22 #include "llvm/CodeGen/TargetLowering.h" 23 #include "llvm/CodeGen/TargetRegisterInfo.h" 24 #include "llvm/CodeGen/TargetSchedule.h" 25 #include "llvm/IR/DataLayout.h" 26 #include "llvm/MC/MCAsmInfo.h" 27 #include "llvm/MC/MCInstrItineraries.h" 28 #include "llvm/Support/CommandLine.h" 29 #include "llvm/Support/ErrorHandling.h" 30 #include "llvm/Support/raw_ostream.h" 31 #include "llvm/Target/TargetMachine.h" 32 #include <cctype> 33 34 using namespace llvm; 35 36 static cl::opt<bool> DisableHazardRecognizer( 37 "disable-sched-hazard", cl::Hidden, cl::init(false), 38 cl::desc("Disable hazard detection during preRA scheduling")); 39 40 TargetInstrInfo::~TargetInstrInfo() { 41 } 42 43 const TargetRegisterClass* 44 TargetInstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum, 45 const TargetRegisterInfo *TRI, 46 const MachineFunction &MF) const { 47 if (OpNum >= MCID.getNumOperands()) 48 return nullptr; 49 50 short RegClass = MCID.OpInfo[OpNum].RegClass; 51 if (MCID.OpInfo[OpNum].isLookupPtrRegClass()) 52 return TRI->getPointerRegClass(MF, RegClass); 53 54 // Instructions like INSERT_SUBREG do not have fixed register classes. 55 if (RegClass < 0) 56 return nullptr; 57 58 // Otherwise just look it up normally. 59 return TRI->getRegClass(RegClass); 60 } 61 62 /// insertNoop - Insert a noop into the instruction stream at the specified 63 /// point. 64 void TargetInstrInfo::insertNoop(MachineBasicBlock &MBB, 65 MachineBasicBlock::iterator MI) const { 66 llvm_unreachable("Target didn't implement insertNoop!"); 67 } 68 69 static bool isAsmComment(const char *Str, const MCAsmInfo &MAI) { 70 return strncmp(Str, MAI.getCommentString().data(), 71 MAI.getCommentString().size()) == 0; 72 } 73 74 /// Measure the specified inline asm to determine an approximation of its 75 /// length. 76 /// Comments (which run till the next SeparatorString or newline) do not 77 /// count as an instruction. 78 /// Any other non-whitespace text is considered an instruction, with 79 /// multiple instructions separated by SeparatorString or newlines. 80 /// Variable-length instructions are not handled here; this function 81 /// may be overloaded in the target code to do that. 82 /// We implement a special case of the .space directive which takes only a 83 /// single integer argument in base 10 that is the size in bytes. This is a 84 /// restricted form of the GAS directive in that we only interpret 85 /// simple--i.e. not a logical or arithmetic expression--size values without 86 /// the optional fill value. This is primarily used for creating arbitrary 87 /// sized inline asm blocks for testing purposes. 88 unsigned TargetInstrInfo::getInlineAsmLength( 89 const char *Str, 90 const MCAsmInfo &MAI, const TargetSubtargetInfo *STI) const { 91 // Count the number of instructions in the asm. 92 bool AtInsnStart = true; 93 unsigned Length = 0; 94 const unsigned MaxInstLength = MAI.getMaxInstLength(STI); 95 for (; *Str; ++Str) { 96 if (*Str == '\n' || strncmp(Str, MAI.getSeparatorString(), 97 strlen(MAI.getSeparatorString())) == 0) { 98 AtInsnStart = true; 99 } else if (isAsmComment(Str, MAI)) { 100 // Stop counting as an instruction after a comment until the next 101 // separator. 102 AtInsnStart = false; 103 } 104 105 if (AtInsnStart && !std::isspace(static_cast<unsigned char>(*Str))) { 106 unsigned AddLength = MaxInstLength; 107 if (strncmp(Str, ".space", 6) == 0) { 108 char *EStr; 109 int SpaceSize; 110 SpaceSize = strtol(Str + 6, &EStr, 10); 111 SpaceSize = SpaceSize < 0 ? 0 : SpaceSize; 112 while (*EStr != '\n' && std::isspace(static_cast<unsigned char>(*EStr))) 113 ++EStr; 114 if (*EStr == '\0' || *EStr == '\n' || 115 isAsmComment(EStr, MAI)) // Successfully parsed .space argument 116 AddLength = SpaceSize; 117 } 118 Length += AddLength; 119 AtInsnStart = false; 120 } 121 } 122 123 return Length; 124 } 125 126 /// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything 127 /// after it, replacing it with an unconditional branch to NewDest. 128 void 129 TargetInstrInfo::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, 130 MachineBasicBlock *NewDest) const { 131 MachineBasicBlock *MBB = Tail->getParent(); 132 133 // Remove all the old successors of MBB from the CFG. 134 while (!MBB->succ_empty()) 135 MBB->removeSuccessor(MBB->succ_begin()); 136 137 // Save off the debug loc before erasing the instruction. 138 DebugLoc DL = Tail->getDebugLoc(); 139 140 // Update call site info and remove all the dead instructions 141 // from the end of MBB. 142 while (Tail != MBB->end()) { 143 auto MI = Tail++; 144 if (MI->isCall()) 145 MBB->getParent()->updateCallSiteInfo(&*MI); 146 MBB->erase(MI); 147 } 148 149 // If MBB isn't immediately before MBB, insert a branch to it. 150 if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest)) 151 insertBranch(*MBB, NewDest, nullptr, SmallVector<MachineOperand, 0>(), DL); 152 MBB->addSuccessor(NewDest); 153 } 154 155 MachineInstr *TargetInstrInfo::commuteInstructionImpl(MachineInstr &MI, 156 bool NewMI, unsigned Idx1, 157 unsigned Idx2) const { 158 const MCInstrDesc &MCID = MI.getDesc(); 159 bool HasDef = MCID.getNumDefs(); 160 if (HasDef && !MI.getOperand(0).isReg()) 161 // No idea how to commute this instruction. Target should implement its own. 162 return nullptr; 163 164 unsigned CommutableOpIdx1 = Idx1; (void)CommutableOpIdx1; 165 unsigned CommutableOpIdx2 = Idx2; (void)CommutableOpIdx2; 166 assert(findCommutedOpIndices(MI, CommutableOpIdx1, CommutableOpIdx2) && 167 CommutableOpIdx1 == Idx1 && CommutableOpIdx2 == Idx2 && 168 "TargetInstrInfo::CommuteInstructionImpl(): not commutable operands."); 169 assert(MI.getOperand(Idx1).isReg() && MI.getOperand(Idx2).isReg() && 170 "This only knows how to commute register operands so far"); 171 172 Register Reg0 = HasDef ? MI.getOperand(0).getReg() : Register(); 173 Register Reg1 = MI.getOperand(Idx1).getReg(); 174 Register Reg2 = MI.getOperand(Idx2).getReg(); 175 unsigned SubReg0 = HasDef ? MI.getOperand(0).getSubReg() : 0; 176 unsigned SubReg1 = MI.getOperand(Idx1).getSubReg(); 177 unsigned SubReg2 = MI.getOperand(Idx2).getSubReg(); 178 bool Reg1IsKill = MI.getOperand(Idx1).isKill(); 179 bool Reg2IsKill = MI.getOperand(Idx2).isKill(); 180 bool Reg1IsUndef = MI.getOperand(Idx1).isUndef(); 181 bool Reg2IsUndef = MI.getOperand(Idx2).isUndef(); 182 bool Reg1IsInternal = MI.getOperand(Idx1).isInternalRead(); 183 bool Reg2IsInternal = MI.getOperand(Idx2).isInternalRead(); 184 // Avoid calling isRenamable for virtual registers since we assert that 185 // renamable property is only queried/set for physical registers. 186 bool Reg1IsRenamable = TargetRegisterInfo::isPhysicalRegister(Reg1) 187 ? MI.getOperand(Idx1).isRenamable() 188 : false; 189 bool Reg2IsRenamable = TargetRegisterInfo::isPhysicalRegister(Reg2) 190 ? MI.getOperand(Idx2).isRenamable() 191 : false; 192 // If destination is tied to either of the commuted source register, then 193 // it must be updated. 194 if (HasDef && Reg0 == Reg1 && 195 MI.getDesc().getOperandConstraint(Idx1, MCOI::TIED_TO) == 0) { 196 Reg2IsKill = false; 197 Reg0 = Reg2; 198 SubReg0 = SubReg2; 199 } else if (HasDef && Reg0 == Reg2 && 200 MI.getDesc().getOperandConstraint(Idx2, MCOI::TIED_TO) == 0) { 201 Reg1IsKill = false; 202 Reg0 = Reg1; 203 SubReg0 = SubReg1; 204 } 205 206 MachineInstr *CommutedMI = nullptr; 207 if (NewMI) { 208 // Create a new instruction. 209 MachineFunction &MF = *MI.getMF(); 210 CommutedMI = MF.CloneMachineInstr(&MI); 211 } else { 212 CommutedMI = &MI; 213 } 214 215 if (HasDef) { 216 CommutedMI->getOperand(0).setReg(Reg0); 217 CommutedMI->getOperand(0).setSubReg(SubReg0); 218 } 219 CommutedMI->getOperand(Idx2).setReg(Reg1); 220 CommutedMI->getOperand(Idx1).setReg(Reg2); 221 CommutedMI->getOperand(Idx2).setSubReg(SubReg1); 222 CommutedMI->getOperand(Idx1).setSubReg(SubReg2); 223 CommutedMI->getOperand(Idx2).setIsKill(Reg1IsKill); 224 CommutedMI->getOperand(Idx1).setIsKill(Reg2IsKill); 225 CommutedMI->getOperand(Idx2).setIsUndef(Reg1IsUndef); 226 CommutedMI->getOperand(Idx1).setIsUndef(Reg2IsUndef); 227 CommutedMI->getOperand(Idx2).setIsInternalRead(Reg1IsInternal); 228 CommutedMI->getOperand(Idx1).setIsInternalRead(Reg2IsInternal); 229 // Avoid calling setIsRenamable for virtual registers since we assert that 230 // renamable property is only queried/set for physical registers. 231 if (TargetRegisterInfo::isPhysicalRegister(Reg1)) 232 CommutedMI->getOperand(Idx2).setIsRenamable(Reg1IsRenamable); 233 if (TargetRegisterInfo::isPhysicalRegister(Reg2)) 234 CommutedMI->getOperand(Idx1).setIsRenamable(Reg2IsRenamable); 235 return CommutedMI; 236 } 237 238 MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr &MI, bool NewMI, 239 unsigned OpIdx1, 240 unsigned OpIdx2) const { 241 // If OpIdx1 or OpIdx2 is not specified, then this method is free to choose 242 // any commutable operand, which is done in findCommutedOpIndices() method 243 // called below. 244 if ((OpIdx1 == CommuteAnyOperandIndex || OpIdx2 == CommuteAnyOperandIndex) && 245 !findCommutedOpIndices(MI, OpIdx1, OpIdx2)) { 246 assert(MI.isCommutable() && 247 "Precondition violation: MI must be commutable."); 248 return nullptr; 249 } 250 return commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 251 } 252 253 bool TargetInstrInfo::fixCommutedOpIndices(unsigned &ResultIdx1, 254 unsigned &ResultIdx2, 255 unsigned CommutableOpIdx1, 256 unsigned CommutableOpIdx2) { 257 if (ResultIdx1 == CommuteAnyOperandIndex && 258 ResultIdx2 == CommuteAnyOperandIndex) { 259 ResultIdx1 = CommutableOpIdx1; 260 ResultIdx2 = CommutableOpIdx2; 261 } else if (ResultIdx1 == CommuteAnyOperandIndex) { 262 if (ResultIdx2 == CommutableOpIdx1) 263 ResultIdx1 = CommutableOpIdx2; 264 else if (ResultIdx2 == CommutableOpIdx2) 265 ResultIdx1 = CommutableOpIdx1; 266 else 267 return false; 268 } else if (ResultIdx2 == CommuteAnyOperandIndex) { 269 if (ResultIdx1 == CommutableOpIdx1) 270 ResultIdx2 = CommutableOpIdx2; 271 else if (ResultIdx1 == CommutableOpIdx2) 272 ResultIdx2 = CommutableOpIdx1; 273 else 274 return false; 275 } else 276 // Check that the result operand indices match the given commutable 277 // operand indices. 278 return (ResultIdx1 == CommutableOpIdx1 && ResultIdx2 == CommutableOpIdx2) || 279 (ResultIdx1 == CommutableOpIdx2 && ResultIdx2 == CommutableOpIdx1); 280 281 return true; 282 } 283 284 bool TargetInstrInfo::findCommutedOpIndices(MachineInstr &MI, 285 unsigned &SrcOpIdx1, 286 unsigned &SrcOpIdx2) const { 287 assert(!MI.isBundle() && 288 "TargetInstrInfo::findCommutedOpIndices() can't handle bundles"); 289 290 const MCInstrDesc &MCID = MI.getDesc(); 291 if (!MCID.isCommutable()) 292 return false; 293 294 // This assumes v0 = op v1, v2 and commuting would swap v1 and v2. If this 295 // is not true, then the target must implement this. 296 unsigned CommutableOpIdx1 = MCID.getNumDefs(); 297 unsigned CommutableOpIdx2 = CommutableOpIdx1 + 1; 298 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 299 CommutableOpIdx1, CommutableOpIdx2)) 300 return false; 301 302 if (!MI.getOperand(SrcOpIdx1).isReg() || !MI.getOperand(SrcOpIdx2).isReg()) 303 // No idea. 304 return false; 305 return true; 306 } 307 308 bool TargetInstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const { 309 if (!MI.isTerminator()) return false; 310 311 // Conditional branch is a special case. 312 if (MI.isBranch() && !MI.isBarrier()) 313 return true; 314 if (!MI.isPredicable()) 315 return true; 316 return !isPredicated(MI); 317 } 318 319 bool TargetInstrInfo::PredicateInstruction( 320 MachineInstr &MI, ArrayRef<MachineOperand> Pred) const { 321 bool MadeChange = false; 322 323 assert(!MI.isBundle() && 324 "TargetInstrInfo::PredicateInstruction() can't handle bundles"); 325 326 const MCInstrDesc &MCID = MI.getDesc(); 327 if (!MI.isPredicable()) 328 return false; 329 330 for (unsigned j = 0, i = 0, e = MI.getNumOperands(); i != e; ++i) { 331 if (MCID.OpInfo[i].isPredicate()) { 332 MachineOperand &MO = MI.getOperand(i); 333 if (MO.isReg()) { 334 MO.setReg(Pred[j].getReg()); 335 MadeChange = true; 336 } else if (MO.isImm()) { 337 MO.setImm(Pred[j].getImm()); 338 MadeChange = true; 339 } else if (MO.isMBB()) { 340 MO.setMBB(Pred[j].getMBB()); 341 MadeChange = true; 342 } 343 ++j; 344 } 345 } 346 return MadeChange; 347 } 348 349 bool TargetInstrInfo::hasLoadFromStackSlot( 350 const MachineInstr &MI, 351 SmallVectorImpl<const MachineMemOperand *> &Accesses) const { 352 size_t StartSize = Accesses.size(); 353 for (MachineInstr::mmo_iterator o = MI.memoperands_begin(), 354 oe = MI.memoperands_end(); 355 o != oe; ++o) { 356 if ((*o)->isLoad() && 357 dyn_cast_or_null<FixedStackPseudoSourceValue>((*o)->getPseudoValue())) 358 Accesses.push_back(*o); 359 } 360 return Accesses.size() != StartSize; 361 } 362 363 bool TargetInstrInfo::hasStoreToStackSlot( 364 const MachineInstr &MI, 365 SmallVectorImpl<const MachineMemOperand *> &Accesses) const { 366 size_t StartSize = Accesses.size(); 367 for (MachineInstr::mmo_iterator o = MI.memoperands_begin(), 368 oe = MI.memoperands_end(); 369 o != oe; ++o) { 370 if ((*o)->isStore() && 371 dyn_cast_or_null<FixedStackPseudoSourceValue>((*o)->getPseudoValue())) 372 Accesses.push_back(*o); 373 } 374 return Accesses.size() != StartSize; 375 } 376 377 bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC, 378 unsigned SubIdx, unsigned &Size, 379 unsigned &Offset, 380 const MachineFunction &MF) const { 381 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 382 if (!SubIdx) { 383 Size = TRI->getSpillSize(*RC); 384 Offset = 0; 385 return true; 386 } 387 unsigned BitSize = TRI->getSubRegIdxSize(SubIdx); 388 // Convert bit size to byte size. 389 if (BitSize % 8) 390 return false; 391 392 int BitOffset = TRI->getSubRegIdxOffset(SubIdx); 393 if (BitOffset < 0 || BitOffset % 8) 394 return false; 395 396 Size = BitSize /= 8; 397 Offset = (unsigned)BitOffset / 8; 398 399 assert(TRI->getSpillSize(*RC) >= (Offset + Size) && "bad subregister range"); 400 401 if (!MF.getDataLayout().isLittleEndian()) { 402 Offset = TRI->getSpillSize(*RC) - (Offset + Size); 403 } 404 return true; 405 } 406 407 void TargetInstrInfo::reMaterialize(MachineBasicBlock &MBB, 408 MachineBasicBlock::iterator I, 409 unsigned DestReg, unsigned SubIdx, 410 const MachineInstr &Orig, 411 const TargetRegisterInfo &TRI) const { 412 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig); 413 MI->substituteRegister(MI->getOperand(0).getReg(), DestReg, SubIdx, TRI); 414 MBB.insert(I, MI); 415 } 416 417 bool TargetInstrInfo::produceSameValue(const MachineInstr &MI0, 418 const MachineInstr &MI1, 419 const MachineRegisterInfo *MRI) const { 420 return MI0.isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs); 421 } 422 423 MachineInstr &TargetInstrInfo::duplicate(MachineBasicBlock &MBB, 424 MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig) const { 425 assert(!Orig.isNotDuplicable() && "Instruction cannot be duplicated"); 426 MachineFunction &MF = *MBB.getParent(); 427 return MF.CloneMachineInstrBundle(MBB, InsertBefore, Orig); 428 } 429 430 // If the COPY instruction in MI can be folded to a stack operation, return 431 // the register class to use. 432 static const TargetRegisterClass *canFoldCopy(const MachineInstr &MI, 433 unsigned FoldIdx) { 434 assert(MI.isCopy() && "MI must be a COPY instruction"); 435 if (MI.getNumOperands() != 2) 436 return nullptr; 437 assert(FoldIdx<2 && "FoldIdx refers no nonexistent operand"); 438 439 const MachineOperand &FoldOp = MI.getOperand(FoldIdx); 440 const MachineOperand &LiveOp = MI.getOperand(1 - FoldIdx); 441 442 if (FoldOp.getSubReg() || LiveOp.getSubReg()) 443 return nullptr; 444 445 unsigned FoldReg = FoldOp.getReg(); 446 unsigned LiveReg = LiveOp.getReg(); 447 448 assert(TargetRegisterInfo::isVirtualRegister(FoldReg) && 449 "Cannot fold physregs"); 450 451 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); 452 const TargetRegisterClass *RC = MRI.getRegClass(FoldReg); 453 454 if (TargetRegisterInfo::isPhysicalRegister(LiveOp.getReg())) 455 return RC->contains(LiveOp.getReg()) ? RC : nullptr; 456 457 if (RC->hasSubClassEq(MRI.getRegClass(LiveReg))) 458 return RC; 459 460 // FIXME: Allow folding when register classes are memory compatible. 461 return nullptr; 462 } 463 464 void TargetInstrInfo::getNoop(MCInst &NopInst) const { 465 llvm_unreachable("Not implemented"); 466 } 467 468 static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI, 469 ArrayRef<unsigned> Ops, int FrameIndex, 470 const TargetInstrInfo &TII) { 471 unsigned StartIdx = 0; 472 switch (MI.getOpcode()) { 473 case TargetOpcode::STACKMAP: { 474 // StackMapLiveValues are foldable 475 StartIdx = StackMapOpers(&MI).getVarIdx(); 476 break; 477 } 478 case TargetOpcode::PATCHPOINT: { 479 // For PatchPoint, the call args are not foldable (even if reported in the 480 // stackmap e.g. via anyregcc). 481 StartIdx = PatchPointOpers(&MI).getVarIdx(); 482 break; 483 } 484 case TargetOpcode::STATEPOINT: { 485 // For statepoints, fold deopt and gc arguments, but not call arguments. 486 StartIdx = StatepointOpers(&MI).getVarIdx(); 487 break; 488 } 489 default: 490 llvm_unreachable("unexpected stackmap opcode"); 491 } 492 493 // Return false if any operands requested for folding are not foldable (not 494 // part of the stackmap's live values). 495 for (unsigned Op : Ops) { 496 if (Op < StartIdx) 497 return nullptr; 498 } 499 500 MachineInstr *NewMI = 501 MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true); 502 MachineInstrBuilder MIB(MF, NewMI); 503 504 // No need to fold return, the meta data, and function arguments 505 for (unsigned i = 0; i < StartIdx; ++i) 506 MIB.add(MI.getOperand(i)); 507 508 for (unsigned i = StartIdx; i < MI.getNumOperands(); ++i) { 509 MachineOperand &MO = MI.getOperand(i); 510 if (is_contained(Ops, i)) { 511 unsigned SpillSize; 512 unsigned SpillOffset; 513 // Compute the spill slot size and offset. 514 const TargetRegisterClass *RC = 515 MF.getRegInfo().getRegClass(MO.getReg()); 516 bool Valid = 517 TII.getStackSlotRange(RC, MO.getSubReg(), SpillSize, SpillOffset, MF); 518 if (!Valid) 519 report_fatal_error("cannot spill patchpoint subregister operand"); 520 MIB.addImm(StackMaps::IndirectMemRefOp); 521 MIB.addImm(SpillSize); 522 MIB.addFrameIndex(FrameIndex); 523 MIB.addImm(SpillOffset); 524 } 525 else 526 MIB.add(MO); 527 } 528 return NewMI; 529 } 530 531 MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, 532 ArrayRef<unsigned> Ops, int FI, 533 LiveIntervals *LIS, 534 VirtRegMap *VRM) const { 535 auto Flags = MachineMemOperand::MONone; 536 for (unsigned OpIdx : Ops) 537 Flags |= MI.getOperand(OpIdx).isDef() ? MachineMemOperand::MOStore 538 : MachineMemOperand::MOLoad; 539 540 MachineBasicBlock *MBB = MI.getParent(); 541 assert(MBB && "foldMemoryOperand needs an inserted instruction"); 542 MachineFunction &MF = *MBB->getParent(); 543 544 // If we're not folding a load into a subreg, the size of the load is the 545 // size of the spill slot. But if we are, we need to figure out what the 546 // actual load size is. 547 int64_t MemSize = 0; 548 const MachineFrameInfo &MFI = MF.getFrameInfo(); 549 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 550 551 if (Flags & MachineMemOperand::MOStore) { 552 MemSize = MFI.getObjectSize(FI); 553 } else { 554 for (unsigned OpIdx : Ops) { 555 int64_t OpSize = MFI.getObjectSize(FI); 556 557 if (auto SubReg = MI.getOperand(OpIdx).getSubReg()) { 558 unsigned SubRegSize = TRI->getSubRegIdxSize(SubReg); 559 if (SubRegSize > 0 && !(SubRegSize % 8)) 560 OpSize = SubRegSize / 8; 561 } 562 563 MemSize = std::max(MemSize, OpSize); 564 } 565 } 566 567 assert(MemSize && "Did not expect a zero-sized stack slot"); 568 569 MachineInstr *NewMI = nullptr; 570 571 if (MI.getOpcode() == TargetOpcode::STACKMAP || 572 MI.getOpcode() == TargetOpcode::PATCHPOINT || 573 MI.getOpcode() == TargetOpcode::STATEPOINT) { 574 // Fold stackmap/patchpoint. 575 NewMI = foldPatchpoint(MF, MI, Ops, FI, *this); 576 if (NewMI) 577 MBB->insert(MI, NewMI); 578 } else { 579 // Ask the target to do the actual folding. 580 NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI, LIS, VRM); 581 } 582 583 if (NewMI) { 584 NewMI->setMemRefs(MF, MI.memoperands()); 585 // Add a memory operand, foldMemoryOperandImpl doesn't do that. 586 assert((!(Flags & MachineMemOperand::MOStore) || 587 NewMI->mayStore()) && 588 "Folded a def to a non-store!"); 589 assert((!(Flags & MachineMemOperand::MOLoad) || 590 NewMI->mayLoad()) && 591 "Folded a use to a non-load!"); 592 assert(MFI.getObjectOffset(FI) != -1); 593 MachineMemOperand *MMO = MF.getMachineMemOperand( 594 MachinePointerInfo::getFixedStack(MF, FI), Flags, MemSize, 595 MFI.getObjectAlignment(FI)); 596 NewMI->addMemOperand(MF, MMO); 597 598 return NewMI; 599 } 600 601 // Straight COPY may fold as load/store. 602 if (!MI.isCopy() || Ops.size() != 1) 603 return nullptr; 604 605 const TargetRegisterClass *RC = canFoldCopy(MI, Ops[0]); 606 if (!RC) 607 return nullptr; 608 609 const MachineOperand &MO = MI.getOperand(1 - Ops[0]); 610 MachineBasicBlock::iterator Pos = MI; 611 612 if (Flags == MachineMemOperand::MOStore) 613 storeRegToStackSlot(*MBB, Pos, MO.getReg(), MO.isKill(), FI, RC, TRI); 614 else 615 loadRegFromStackSlot(*MBB, Pos, MO.getReg(), FI, RC, TRI); 616 return &*--Pos; 617 } 618 619 MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, 620 ArrayRef<unsigned> Ops, 621 MachineInstr &LoadMI, 622 LiveIntervals *LIS) const { 623 assert(LoadMI.canFoldAsLoad() && "LoadMI isn't foldable!"); 624 #ifndef NDEBUG 625 for (unsigned OpIdx : Ops) 626 assert(MI.getOperand(OpIdx).isUse() && "Folding load into def!"); 627 #endif 628 629 MachineBasicBlock &MBB = *MI.getParent(); 630 MachineFunction &MF = *MBB.getParent(); 631 632 // Ask the target to do the actual folding. 633 MachineInstr *NewMI = nullptr; 634 int FrameIndex = 0; 635 636 if ((MI.getOpcode() == TargetOpcode::STACKMAP || 637 MI.getOpcode() == TargetOpcode::PATCHPOINT || 638 MI.getOpcode() == TargetOpcode::STATEPOINT) && 639 isLoadFromStackSlot(LoadMI, FrameIndex)) { 640 // Fold stackmap/patchpoint. 641 NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this); 642 if (NewMI) 643 NewMI = &*MBB.insert(MI, NewMI); 644 } else { 645 // Ask the target to do the actual folding. 646 NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, LoadMI, LIS); 647 } 648 649 if (!NewMI) 650 return nullptr; 651 652 // Copy the memoperands from the load to the folded instruction. 653 if (MI.memoperands_empty()) { 654 NewMI->setMemRefs(MF, LoadMI.memoperands()); 655 } else { 656 // Handle the rare case of folding multiple loads. 657 NewMI->setMemRefs(MF, MI.memoperands()); 658 for (MachineInstr::mmo_iterator I = LoadMI.memoperands_begin(), 659 E = LoadMI.memoperands_end(); 660 I != E; ++I) { 661 NewMI->addMemOperand(MF, *I); 662 } 663 } 664 return NewMI; 665 } 666 667 bool TargetInstrInfo::hasReassociableOperands( 668 const MachineInstr &Inst, const MachineBasicBlock *MBB) const { 669 const MachineOperand &Op1 = Inst.getOperand(1); 670 const MachineOperand &Op2 = Inst.getOperand(2); 671 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 672 673 // We need virtual register definitions for the operands that we will 674 // reassociate. 675 MachineInstr *MI1 = nullptr; 676 MachineInstr *MI2 = nullptr; 677 if (Op1.isReg() && TargetRegisterInfo::isVirtualRegister(Op1.getReg())) 678 MI1 = MRI.getUniqueVRegDef(Op1.getReg()); 679 if (Op2.isReg() && TargetRegisterInfo::isVirtualRegister(Op2.getReg())) 680 MI2 = MRI.getUniqueVRegDef(Op2.getReg()); 681 682 // And they need to be in the trace (otherwise, they won't have a depth). 683 return MI1 && MI2 && MI1->getParent() == MBB && MI2->getParent() == MBB; 684 } 685 686 bool TargetInstrInfo::hasReassociableSibling(const MachineInstr &Inst, 687 bool &Commuted) const { 688 const MachineBasicBlock *MBB = Inst.getParent(); 689 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 690 MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg()); 691 MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg()); 692 unsigned AssocOpcode = Inst.getOpcode(); 693 694 // If only one operand has the same opcode and it's the second source operand, 695 // the operands must be commuted. 696 Commuted = MI1->getOpcode() != AssocOpcode && MI2->getOpcode() == AssocOpcode; 697 if (Commuted) 698 std::swap(MI1, MI2); 699 700 // 1. The previous instruction must be the same type as Inst. 701 // 2. The previous instruction must have virtual register definitions for its 702 // operands in the same basic block as Inst. 703 // 3. The previous instruction's result must only be used by Inst. 704 return MI1->getOpcode() == AssocOpcode && 705 hasReassociableOperands(*MI1, MBB) && 706 MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg()); 707 } 708 709 // 1. The operation must be associative and commutative. 710 // 2. The instruction must have virtual register definitions for its 711 // operands in the same basic block. 712 // 3. The instruction must have a reassociable sibling. 713 bool TargetInstrInfo::isReassociationCandidate(const MachineInstr &Inst, 714 bool &Commuted) const { 715 return isAssociativeAndCommutative(Inst) && 716 hasReassociableOperands(Inst, Inst.getParent()) && 717 hasReassociableSibling(Inst, Commuted); 718 } 719 720 // The concept of the reassociation pass is that these operations can benefit 721 // from this kind of transformation: 722 // 723 // A = ? op ? 724 // B = A op X (Prev) 725 // C = B op Y (Root) 726 // --> 727 // A = ? op ? 728 // B = X op Y 729 // C = A op B 730 // 731 // breaking the dependency between A and B, allowing them to be executed in 732 // parallel (or back-to-back in a pipeline) instead of depending on each other. 733 734 // FIXME: This has the potential to be expensive (compile time) while not 735 // improving the code at all. Some ways to limit the overhead: 736 // 1. Track successful transforms; bail out if hit rate gets too low. 737 // 2. Only enable at -O3 or some other non-default optimization level. 738 // 3. Pre-screen pattern candidates here: if an operand of the previous 739 // instruction is known to not increase the critical path, then don't match 740 // that pattern. 741 bool TargetInstrInfo::getMachineCombinerPatterns( 742 MachineInstr &Root, 743 SmallVectorImpl<MachineCombinerPattern> &Patterns) const { 744 bool Commute; 745 if (isReassociationCandidate(Root, Commute)) { 746 // We found a sequence of instructions that may be suitable for a 747 // reassociation of operands to increase ILP. Specify each commutation 748 // possibility for the Prev instruction in the sequence and let the 749 // machine combiner decide if changing the operands is worthwhile. 750 if (Commute) { 751 Patterns.push_back(MachineCombinerPattern::REASSOC_AX_YB); 752 Patterns.push_back(MachineCombinerPattern::REASSOC_XA_YB); 753 } else { 754 Patterns.push_back(MachineCombinerPattern::REASSOC_AX_BY); 755 Patterns.push_back(MachineCombinerPattern::REASSOC_XA_BY); 756 } 757 return true; 758 } 759 760 return false; 761 } 762 763 /// Return true when a code sequence can improve loop throughput. 764 bool 765 TargetInstrInfo::isThroughputPattern(MachineCombinerPattern Pattern) const { 766 return false; 767 } 768 769 /// Attempt the reassociation transformation to reduce critical path length. 770 /// See the above comments before getMachineCombinerPatterns(). 771 void TargetInstrInfo::reassociateOps( 772 MachineInstr &Root, MachineInstr &Prev, 773 MachineCombinerPattern Pattern, 774 SmallVectorImpl<MachineInstr *> &InsInstrs, 775 SmallVectorImpl<MachineInstr *> &DelInstrs, 776 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const { 777 MachineFunction *MF = Root.getMF(); 778 MachineRegisterInfo &MRI = MF->getRegInfo(); 779 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); 780 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); 781 const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI); 782 783 // This array encodes the operand index for each parameter because the 784 // operands may be commuted. Each row corresponds to a pattern value, 785 // and each column specifies the index of A, B, X, Y. 786 unsigned OpIdx[4][4] = { 787 { 1, 1, 2, 2 }, 788 { 1, 2, 2, 1 }, 789 { 2, 1, 1, 2 }, 790 { 2, 2, 1, 1 } 791 }; 792 793 int Row; 794 switch (Pattern) { 795 case MachineCombinerPattern::REASSOC_AX_BY: Row = 0; break; 796 case MachineCombinerPattern::REASSOC_AX_YB: Row = 1; break; 797 case MachineCombinerPattern::REASSOC_XA_BY: Row = 2; break; 798 case MachineCombinerPattern::REASSOC_XA_YB: Row = 3; break; 799 default: llvm_unreachable("unexpected MachineCombinerPattern"); 800 } 801 802 MachineOperand &OpA = Prev.getOperand(OpIdx[Row][0]); 803 MachineOperand &OpB = Root.getOperand(OpIdx[Row][1]); 804 MachineOperand &OpX = Prev.getOperand(OpIdx[Row][2]); 805 MachineOperand &OpY = Root.getOperand(OpIdx[Row][3]); 806 MachineOperand &OpC = Root.getOperand(0); 807 808 unsigned RegA = OpA.getReg(); 809 unsigned RegB = OpB.getReg(); 810 unsigned RegX = OpX.getReg(); 811 unsigned RegY = OpY.getReg(); 812 unsigned RegC = OpC.getReg(); 813 814 if (TargetRegisterInfo::isVirtualRegister(RegA)) 815 MRI.constrainRegClass(RegA, RC); 816 if (TargetRegisterInfo::isVirtualRegister(RegB)) 817 MRI.constrainRegClass(RegB, RC); 818 if (TargetRegisterInfo::isVirtualRegister(RegX)) 819 MRI.constrainRegClass(RegX, RC); 820 if (TargetRegisterInfo::isVirtualRegister(RegY)) 821 MRI.constrainRegClass(RegY, RC); 822 if (TargetRegisterInfo::isVirtualRegister(RegC)) 823 MRI.constrainRegClass(RegC, RC); 824 825 // Create a new virtual register for the result of (X op Y) instead of 826 // recycling RegB because the MachineCombiner's computation of the critical 827 // path requires a new register definition rather than an existing one. 828 unsigned NewVR = MRI.createVirtualRegister(RC); 829 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0)); 830 831 unsigned Opcode = Root.getOpcode(); 832 bool KillA = OpA.isKill(); 833 bool KillX = OpX.isKill(); 834 bool KillY = OpY.isKill(); 835 836 // Create new instructions for insertion. 837 MachineInstrBuilder MIB1 = 838 BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR) 839 .addReg(RegX, getKillRegState(KillX)) 840 .addReg(RegY, getKillRegState(KillY)); 841 MachineInstrBuilder MIB2 = 842 BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC) 843 .addReg(RegA, getKillRegState(KillA)) 844 .addReg(NewVR, getKillRegState(true)); 845 846 setSpecialOperandAttr(Root, Prev, *MIB1, *MIB2); 847 848 // Record new instructions for insertion and old instructions for deletion. 849 InsInstrs.push_back(MIB1); 850 InsInstrs.push_back(MIB2); 851 DelInstrs.push_back(&Prev); 852 DelInstrs.push_back(&Root); 853 } 854 855 void TargetInstrInfo::genAlternativeCodeSequence( 856 MachineInstr &Root, MachineCombinerPattern Pattern, 857 SmallVectorImpl<MachineInstr *> &InsInstrs, 858 SmallVectorImpl<MachineInstr *> &DelInstrs, 859 DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const { 860 MachineRegisterInfo &MRI = Root.getMF()->getRegInfo(); 861 862 // Select the previous instruction in the sequence based on the input pattern. 863 MachineInstr *Prev = nullptr; 864 switch (Pattern) { 865 case MachineCombinerPattern::REASSOC_AX_BY: 866 case MachineCombinerPattern::REASSOC_XA_BY: 867 Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg()); 868 break; 869 case MachineCombinerPattern::REASSOC_AX_YB: 870 case MachineCombinerPattern::REASSOC_XA_YB: 871 Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg()); 872 break; 873 default: 874 break; 875 } 876 877 assert(Prev && "Unknown pattern for machine combiner"); 878 879 reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg); 880 } 881 882 bool TargetInstrInfo::isReallyTriviallyReMaterializableGeneric( 883 const MachineInstr &MI, AliasAnalysis *AA) const { 884 const MachineFunction &MF = *MI.getMF(); 885 const MachineRegisterInfo &MRI = MF.getRegInfo(); 886 887 // Remat clients assume operand 0 is the defined register. 888 if (!MI.getNumOperands() || !MI.getOperand(0).isReg()) 889 return false; 890 unsigned DefReg = MI.getOperand(0).getReg(); 891 892 // A sub-register definition can only be rematerialized if the instruction 893 // doesn't read the other parts of the register. Otherwise it is really a 894 // read-modify-write operation on the full virtual register which cannot be 895 // moved safely. 896 if (TargetRegisterInfo::isVirtualRegister(DefReg) && 897 MI.getOperand(0).getSubReg() && MI.readsVirtualRegister(DefReg)) 898 return false; 899 900 // A load from a fixed stack slot can be rematerialized. This may be 901 // redundant with subsequent checks, but it's target-independent, 902 // simple, and a common case. 903 int FrameIdx = 0; 904 if (isLoadFromStackSlot(MI, FrameIdx) && 905 MF.getFrameInfo().isImmutableObjectIndex(FrameIdx)) 906 return true; 907 908 // Avoid instructions obviously unsafe for remat. 909 if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() || 910 MI.hasUnmodeledSideEffects()) 911 return false; 912 913 // Don't remat inline asm. We have no idea how expensive it is 914 // even if it's side effect free. 915 if (MI.isInlineAsm()) 916 return false; 917 918 // Avoid instructions which load from potentially varying memory. 919 if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad(AA)) 920 return false; 921 922 // If any of the registers accessed are non-constant, conservatively assume 923 // the instruction is not rematerializable. 924 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 925 const MachineOperand &MO = MI.getOperand(i); 926 if (!MO.isReg()) continue; 927 unsigned Reg = MO.getReg(); 928 if (Reg == 0) 929 continue; 930 931 // Check for a well-behaved physical register. 932 if (TargetRegisterInfo::isPhysicalRegister(Reg)) { 933 if (MO.isUse()) { 934 // If the physreg has no defs anywhere, it's just an ambient register 935 // and we can freely move its uses. Alternatively, if it's allocatable, 936 // it could get allocated to something with a def during allocation. 937 if (!MRI.isConstantPhysReg(Reg)) 938 return false; 939 } else { 940 // A physreg def. We can't remat it. 941 return false; 942 } 943 continue; 944 } 945 946 // Only allow one virtual-register def. There may be multiple defs of the 947 // same virtual register, though. 948 if (MO.isDef() && Reg != DefReg) 949 return false; 950 951 // Don't allow any virtual-register uses. Rematting an instruction with 952 // virtual register uses would length the live ranges of the uses, which 953 // is not necessarily a good idea, certainly not "trivial". 954 if (MO.isUse()) 955 return false; 956 } 957 958 // Everything checked out. 959 return true; 960 } 961 962 int TargetInstrInfo::getSPAdjust(const MachineInstr &MI) const { 963 const MachineFunction *MF = MI.getMF(); 964 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); 965 bool StackGrowsDown = 966 TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown; 967 968 unsigned FrameSetupOpcode = getCallFrameSetupOpcode(); 969 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode(); 970 971 if (!isFrameInstr(MI)) 972 return 0; 973 974 int SPAdj = TFI->alignSPAdjust(getFrameSize(MI)); 975 976 if ((!StackGrowsDown && MI.getOpcode() == FrameSetupOpcode) || 977 (StackGrowsDown && MI.getOpcode() == FrameDestroyOpcode)) 978 SPAdj = -SPAdj; 979 980 return SPAdj; 981 } 982 983 /// isSchedulingBoundary - Test if the given instruction should be 984 /// considered a scheduling boundary. This primarily includes labels 985 /// and terminators. 986 bool TargetInstrInfo::isSchedulingBoundary(const MachineInstr &MI, 987 const MachineBasicBlock *MBB, 988 const MachineFunction &MF) const { 989 // Terminators and labels can't be scheduled around. 990 if (MI.isTerminator() || MI.isPosition()) 991 return true; 992 993 // Don't attempt to schedule around any instruction that defines 994 // a stack-oriented pointer, as it's unlikely to be profitable. This 995 // saves compile time, because it doesn't require every single 996 // stack slot reference to depend on the instruction that does the 997 // modification. 998 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering(); 999 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); 1000 return MI.modifiesRegister(TLI.getStackPointerRegisterToSaveRestore(), TRI); 1001 } 1002 1003 // Provide a global flag for disabling the PreRA hazard recognizer that targets 1004 // may choose to honor. 1005 bool TargetInstrInfo::usePreRAHazardRecognizer() const { 1006 return !DisableHazardRecognizer; 1007 } 1008 1009 // Default implementation of CreateTargetRAHazardRecognizer. 1010 ScheduleHazardRecognizer *TargetInstrInfo:: 1011 CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, 1012 const ScheduleDAG *DAG) const { 1013 // Dummy hazard recognizer allows all instructions to issue. 1014 return new ScheduleHazardRecognizer(); 1015 } 1016 1017 // Default implementation of CreateTargetMIHazardRecognizer. 1018 ScheduleHazardRecognizer *TargetInstrInfo:: 1019 CreateTargetMIHazardRecognizer(const InstrItineraryData *II, 1020 const ScheduleDAG *DAG) const { 1021 return (ScheduleHazardRecognizer *) 1022 new ScoreboardHazardRecognizer(II, DAG, "machine-scheduler"); 1023 } 1024 1025 // Default implementation of CreateTargetPostRAHazardRecognizer. 1026 ScheduleHazardRecognizer *TargetInstrInfo:: 1027 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, 1028 const ScheduleDAG *DAG) const { 1029 return (ScheduleHazardRecognizer *) 1030 new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched"); 1031 } 1032 1033 //===----------------------------------------------------------------------===// 1034 // SelectionDAG latency interface. 1035 //===----------------------------------------------------------------------===// 1036 1037 int 1038 TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, 1039 SDNode *DefNode, unsigned DefIdx, 1040 SDNode *UseNode, unsigned UseIdx) const { 1041 if (!ItinData || ItinData->isEmpty()) 1042 return -1; 1043 1044 if (!DefNode->isMachineOpcode()) 1045 return -1; 1046 1047 unsigned DefClass = get(DefNode->getMachineOpcode()).getSchedClass(); 1048 if (!UseNode->isMachineOpcode()) 1049 return ItinData->getOperandCycle(DefClass, DefIdx); 1050 unsigned UseClass = get(UseNode->getMachineOpcode()).getSchedClass(); 1051 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); 1052 } 1053 1054 int TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 1055 SDNode *N) const { 1056 if (!ItinData || ItinData->isEmpty()) 1057 return 1; 1058 1059 if (!N->isMachineOpcode()) 1060 return 1; 1061 1062 return ItinData->getStageLatency(get(N->getMachineOpcode()).getSchedClass()); 1063 } 1064 1065 //===----------------------------------------------------------------------===// 1066 // MachineInstr latency interface. 1067 //===----------------------------------------------------------------------===// 1068 1069 unsigned TargetInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData, 1070 const MachineInstr &MI) const { 1071 if (!ItinData || ItinData->isEmpty()) 1072 return 1; 1073 1074 unsigned Class = MI.getDesc().getSchedClass(); 1075 int UOps = ItinData->Itineraries[Class].NumMicroOps; 1076 if (UOps >= 0) 1077 return UOps; 1078 1079 // The # of u-ops is dynamically determined. The specific target should 1080 // override this function to return the right number. 1081 return 1; 1082 } 1083 1084 /// Return the default expected latency for a def based on it's opcode. 1085 unsigned TargetInstrInfo::defaultDefLatency(const MCSchedModel &SchedModel, 1086 const MachineInstr &DefMI) const { 1087 if (DefMI.isTransient()) 1088 return 0; 1089 if (DefMI.mayLoad()) 1090 return SchedModel.LoadLatency; 1091 if (isHighLatencyDef(DefMI.getOpcode())) 1092 return SchedModel.HighLatency; 1093 return 1; 1094 } 1095 1096 unsigned TargetInstrInfo::getPredicationCost(const MachineInstr &) const { 1097 return 0; 1098 } 1099 1100 unsigned TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 1101 const MachineInstr &MI, 1102 unsigned *PredCost) const { 1103 // Default to one cycle for no itinerary. However, an "empty" itinerary may 1104 // still have a MinLatency property, which getStageLatency checks. 1105 if (!ItinData) 1106 return MI.mayLoad() ? 2 : 1; 1107 1108 return ItinData->getStageLatency(MI.getDesc().getSchedClass()); 1109 } 1110 1111 bool TargetInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel, 1112 const MachineInstr &DefMI, 1113 unsigned DefIdx) const { 1114 const InstrItineraryData *ItinData = SchedModel.getInstrItineraries(); 1115 if (!ItinData || ItinData->isEmpty()) 1116 return false; 1117 1118 unsigned DefClass = DefMI.getDesc().getSchedClass(); 1119 int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx); 1120 return (DefCycle != -1 && DefCycle <= 1); 1121 } 1122 1123 /// Both DefMI and UseMI must be valid. By default, call directly to the 1124 /// itinerary. This may be overriden by the target. 1125 int TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, 1126 const MachineInstr &DefMI, 1127 unsigned DefIdx, 1128 const MachineInstr &UseMI, 1129 unsigned UseIdx) const { 1130 unsigned DefClass = DefMI.getDesc().getSchedClass(); 1131 unsigned UseClass = UseMI.getDesc().getSchedClass(); 1132 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); 1133 } 1134 1135 /// If we can determine the operand latency from the def only, without itinerary 1136 /// lookup, do so. Otherwise return -1. 1137 int TargetInstrInfo::computeDefOperandLatency( 1138 const InstrItineraryData *ItinData, const MachineInstr &DefMI) const { 1139 1140 // Let the target hook getInstrLatency handle missing itineraries. 1141 if (!ItinData) 1142 return getInstrLatency(ItinData, DefMI); 1143 1144 if(ItinData->isEmpty()) 1145 return defaultDefLatency(ItinData->SchedModel, DefMI); 1146 1147 // ...operand lookup required 1148 return -1; 1149 } 1150 1151 bool TargetInstrInfo::getRegSequenceInputs( 1152 const MachineInstr &MI, unsigned DefIdx, 1153 SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const { 1154 assert((MI.isRegSequence() || 1155 MI.isRegSequenceLike()) && "Instruction do not have the proper type"); 1156 1157 if (!MI.isRegSequence()) 1158 return getRegSequenceLikeInputs(MI, DefIdx, InputRegs); 1159 1160 // We are looking at: 1161 // Def = REG_SEQUENCE v0, sub0, v1, sub1, ... 1162 assert(DefIdx == 0 && "REG_SEQUENCE only has one def"); 1163 for (unsigned OpIdx = 1, EndOpIdx = MI.getNumOperands(); OpIdx != EndOpIdx; 1164 OpIdx += 2) { 1165 const MachineOperand &MOReg = MI.getOperand(OpIdx); 1166 if (MOReg.isUndef()) 1167 continue; 1168 const MachineOperand &MOSubIdx = MI.getOperand(OpIdx + 1); 1169 assert(MOSubIdx.isImm() && 1170 "One of the subindex of the reg_sequence is not an immediate"); 1171 // Record Reg:SubReg, SubIdx. 1172 InputRegs.push_back(RegSubRegPairAndIdx(MOReg.getReg(), MOReg.getSubReg(), 1173 (unsigned)MOSubIdx.getImm())); 1174 } 1175 return true; 1176 } 1177 1178 bool TargetInstrInfo::getExtractSubregInputs( 1179 const MachineInstr &MI, unsigned DefIdx, 1180 RegSubRegPairAndIdx &InputReg) const { 1181 assert((MI.isExtractSubreg() || 1182 MI.isExtractSubregLike()) && "Instruction do not have the proper type"); 1183 1184 if (!MI.isExtractSubreg()) 1185 return getExtractSubregLikeInputs(MI, DefIdx, InputReg); 1186 1187 // We are looking at: 1188 // Def = EXTRACT_SUBREG v0.sub1, sub0. 1189 assert(DefIdx == 0 && "EXTRACT_SUBREG only has one def"); 1190 const MachineOperand &MOReg = MI.getOperand(1); 1191 if (MOReg.isUndef()) 1192 return false; 1193 const MachineOperand &MOSubIdx = MI.getOperand(2); 1194 assert(MOSubIdx.isImm() && 1195 "The subindex of the extract_subreg is not an immediate"); 1196 1197 InputReg.Reg = MOReg.getReg(); 1198 InputReg.SubReg = MOReg.getSubReg(); 1199 InputReg.SubIdx = (unsigned)MOSubIdx.getImm(); 1200 return true; 1201 } 1202 1203 bool TargetInstrInfo::getInsertSubregInputs( 1204 const MachineInstr &MI, unsigned DefIdx, 1205 RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const { 1206 assert((MI.isInsertSubreg() || 1207 MI.isInsertSubregLike()) && "Instruction do not have the proper type"); 1208 1209 if (!MI.isInsertSubreg()) 1210 return getInsertSubregLikeInputs(MI, DefIdx, BaseReg, InsertedReg); 1211 1212 // We are looking at: 1213 // Def = INSERT_SEQUENCE v0, v1, sub0. 1214 assert(DefIdx == 0 && "INSERT_SUBREG only has one def"); 1215 const MachineOperand &MOBaseReg = MI.getOperand(1); 1216 const MachineOperand &MOInsertedReg = MI.getOperand(2); 1217 if (MOInsertedReg.isUndef()) 1218 return false; 1219 const MachineOperand &MOSubIdx = MI.getOperand(3); 1220 assert(MOSubIdx.isImm() && 1221 "One of the subindex of the reg_sequence is not an immediate"); 1222 BaseReg.Reg = MOBaseReg.getReg(); 1223 BaseReg.SubReg = MOBaseReg.getSubReg(); 1224 1225 InsertedReg.Reg = MOInsertedReg.getReg(); 1226 InsertedReg.SubReg = MOInsertedReg.getSubReg(); 1227 InsertedReg.SubIdx = (unsigned)MOSubIdx.getImm(); 1228 return true; 1229 } 1230