1 //===-- ARMBaseInstrInfo.cpp - ARM Instruction Information ----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the Base ARM implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "ARMBaseInstrInfo.h" 14 #include "ARMBaseRegisterInfo.h" 15 #include "ARMConstantPoolValue.h" 16 #include "ARMFeatures.h" 17 #include "ARMHazardRecognizer.h" 18 #include "ARMMachineFunctionInfo.h" 19 #include "ARMSubtarget.h" 20 #include "MCTargetDesc/ARMAddressingModes.h" 21 #include "MCTargetDesc/ARMBaseInfo.h" 22 #include "MVETailPredUtils.h" 23 #include "llvm/ADT/DenseMap.h" 24 #include "llvm/ADT/STLExtras.h" 25 #include "llvm/ADT/SmallSet.h" 26 #include "llvm/ADT/SmallVector.h" 27 #include "llvm/ADT/Triple.h" 28 #include "llvm/CodeGen/LiveVariables.h" 29 #include "llvm/CodeGen/MachineBasicBlock.h" 30 #include "llvm/CodeGen/MachineConstantPool.h" 31 #include "llvm/CodeGen/MachineFrameInfo.h" 32 #include "llvm/CodeGen/MachineFunction.h" 33 #include "llvm/CodeGen/MachineInstr.h" 34 #include "llvm/CodeGen/MachineInstrBuilder.h" 35 #include "llvm/CodeGen/MachineMemOperand.h" 36 #include "llvm/CodeGen/MachineModuleInfo.h" 37 #include "llvm/CodeGen/MachineOperand.h" 38 #include "llvm/CodeGen/MachineRegisterInfo.h" 39 #include "llvm/CodeGen/MachineScheduler.h" 40 #include "llvm/CodeGen/MultiHazardRecognizer.h" 41 #include "llvm/CodeGen/ScoreboardHazardRecognizer.h" 42 #include "llvm/CodeGen/SelectionDAGNodes.h" 43 #include "llvm/CodeGen/TargetInstrInfo.h" 44 #include "llvm/CodeGen/TargetRegisterInfo.h" 45 #include "llvm/CodeGen/TargetSchedule.h" 46 #include "llvm/IR/Attributes.h" 47 #include "llvm/IR/Constants.h" 48 #include "llvm/IR/DebugLoc.h" 49 #include "llvm/IR/Function.h" 50 #include "llvm/IR/GlobalValue.h" 51 #include "llvm/MC/MCAsmInfo.h" 52 #include "llvm/MC/MCInstrDesc.h" 53 #include "llvm/MC/MCInstrItineraries.h" 54 #include "llvm/Support/BranchProbability.h" 55 #include "llvm/Support/Casting.h" 56 #include "llvm/Support/CommandLine.h" 57 #include "llvm/Support/Compiler.h" 58 #include "llvm/Support/Debug.h" 59 #include "llvm/Support/ErrorHandling.h" 60 #include "llvm/Support/raw_ostream.h" 61 #include "llvm/Target/TargetMachine.h" 62 #include <algorithm> 63 #include <cassert> 64 #include <cstdint> 65 #include <iterator> 66 #include <new> 67 #include <utility> 68 #include <vector> 69 70 using namespace llvm; 71 72 #define DEBUG_TYPE "arm-instrinfo" 73 74 #define GET_INSTRINFO_CTOR_DTOR 75 #include "ARMGenInstrInfo.inc" 76 77 static cl::opt<bool> 78 EnableARM3Addr("enable-arm-3-addr-conv", cl::Hidden, 79 cl::desc("Enable ARM 2-addr to 3-addr conv")); 80 81 /// ARM_MLxEntry - Record information about MLA / MLS instructions. 82 struct ARM_MLxEntry { 83 uint16_t MLxOpc; // MLA / MLS opcode 84 uint16_t MulOpc; // Expanded multiplication opcode 85 uint16_t AddSubOpc; // Expanded add / sub opcode 86 bool NegAcc; // True if the acc is negated before the add / sub. 87 bool HasLane; // True if instruction has an extra "lane" operand. 88 }; 89 90 static const ARM_MLxEntry ARM_MLxTable[] = { 91 // MLxOpc, MulOpc, AddSubOpc, NegAcc, HasLane 92 // fp scalar ops 93 { ARM::VMLAS, ARM::VMULS, ARM::VADDS, false, false }, 94 { ARM::VMLSS, ARM::VMULS, ARM::VSUBS, false, false }, 95 { ARM::VMLAD, ARM::VMULD, ARM::VADDD, false, false }, 96 { ARM::VMLSD, ARM::VMULD, ARM::VSUBD, false, false }, 97 { ARM::VNMLAS, ARM::VNMULS, ARM::VSUBS, true, false }, 98 { ARM::VNMLSS, ARM::VMULS, ARM::VSUBS, true, false }, 99 { ARM::VNMLAD, ARM::VNMULD, ARM::VSUBD, true, false }, 100 { ARM::VNMLSD, ARM::VMULD, ARM::VSUBD, true, false }, 101 102 // fp SIMD ops 103 { ARM::VMLAfd, ARM::VMULfd, ARM::VADDfd, false, false }, 104 { ARM::VMLSfd, ARM::VMULfd, ARM::VSUBfd, false, false }, 105 { ARM::VMLAfq, ARM::VMULfq, ARM::VADDfq, false, false }, 106 { ARM::VMLSfq, ARM::VMULfq, ARM::VSUBfq, false, false }, 107 { ARM::VMLAslfd, ARM::VMULslfd, ARM::VADDfd, false, true }, 108 { ARM::VMLSslfd, ARM::VMULslfd, ARM::VSUBfd, false, true }, 109 { ARM::VMLAslfq, ARM::VMULslfq, ARM::VADDfq, false, true }, 110 { ARM::VMLSslfq, ARM::VMULslfq, ARM::VSUBfq, false, true }, 111 }; 112 113 ARMBaseInstrInfo::ARMBaseInstrInfo(const ARMSubtarget& STI) 114 : ARMGenInstrInfo(ARM::ADJCALLSTACKDOWN, ARM::ADJCALLSTACKUP), 115 Subtarget(STI) { 116 for (unsigned i = 0, e = array_lengthof(ARM_MLxTable); i != e; ++i) { 117 if (!MLxEntryMap.insert(std::make_pair(ARM_MLxTable[i].MLxOpc, i)).second) 118 llvm_unreachable("Duplicated entries?"); 119 MLxHazardOpcodes.insert(ARM_MLxTable[i].AddSubOpc); 120 MLxHazardOpcodes.insert(ARM_MLxTable[i].MulOpc); 121 } 122 } 123 124 // Use a ScoreboardHazardRecognizer for prepass ARM scheduling. TargetInstrImpl 125 // currently defaults to no prepass hazard recognizer. 126 ScheduleHazardRecognizer * 127 ARMBaseInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, 128 const ScheduleDAG *DAG) const { 129 if (usePreRAHazardRecognizer()) { 130 const InstrItineraryData *II = 131 static_cast<const ARMSubtarget *>(STI)->getInstrItineraryData(); 132 return new ScoreboardHazardRecognizer(II, DAG, "pre-RA-sched"); 133 } 134 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG); 135 } 136 137 // Called during: 138 // - pre-RA scheduling 139 // - post-RA scheduling when FeatureUseMISched is set 140 ScheduleHazardRecognizer *ARMBaseInstrInfo::CreateTargetMIHazardRecognizer( 141 const InstrItineraryData *II, const ScheduleDAGMI *DAG) const { 142 MultiHazardRecognizer *MHR = new MultiHazardRecognizer(); 143 144 // We would like to restrict this hazard recognizer to only 145 // post-RA scheduling; we can tell that we're post-RA because we don't 146 // track VRegLiveness. 147 // Cortex-M7: TRM indicates that there is a single ITCM bank and two DTCM 148 // banks banked on bit 2. Assume that TCMs are in use. 149 if (Subtarget.isCortexM7() && !DAG->hasVRegLiveness()) 150 MHR->AddHazardRecognizer( 151 std::make_unique<ARMBankConflictHazardRecognizer>(DAG, 0x4, true)); 152 153 // Not inserting ARMHazardRecognizerFPMLx because that would change 154 // legacy behavior 155 156 auto BHR = TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG); 157 MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR)); 158 return MHR; 159 } 160 161 // Called during post-RA scheduling when FeatureUseMISched is not set 162 ScheduleHazardRecognizer *ARMBaseInstrInfo:: 163 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, 164 const ScheduleDAG *DAG) const { 165 MultiHazardRecognizer *MHR = new MultiHazardRecognizer(); 166 167 if (Subtarget.isThumb2() || Subtarget.hasVFP2Base()) 168 MHR->AddHazardRecognizer(std::make_unique<ARMHazardRecognizerFPMLx>()); 169 170 auto BHR = TargetInstrInfo::CreateTargetPostRAHazardRecognizer(II, DAG); 171 if (BHR) 172 MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR)); 173 return MHR; 174 } 175 176 MachineInstr *ARMBaseInstrInfo::convertToThreeAddress( 177 MachineFunction::iterator &MFI, MachineInstr &MI, LiveVariables *LV) const { 178 // FIXME: Thumb2 support. 179 180 if (!EnableARM3Addr) 181 return nullptr; 182 183 MachineFunction &MF = *MI.getParent()->getParent(); 184 uint64_t TSFlags = MI.getDesc().TSFlags; 185 bool isPre = false; 186 switch ((TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift) { 187 default: return nullptr; 188 case ARMII::IndexModePre: 189 isPre = true; 190 break; 191 case ARMII::IndexModePost: 192 break; 193 } 194 195 // Try splitting an indexed load/store to an un-indexed one plus an add/sub 196 // operation. 197 unsigned MemOpc = getUnindexedOpcode(MI.getOpcode()); 198 if (MemOpc == 0) 199 return nullptr; 200 201 MachineInstr *UpdateMI = nullptr; 202 MachineInstr *MemMI = nullptr; 203 unsigned AddrMode = (TSFlags & ARMII::AddrModeMask); 204 const MCInstrDesc &MCID = MI.getDesc(); 205 unsigned NumOps = MCID.getNumOperands(); 206 bool isLoad = !MI.mayStore(); 207 const MachineOperand &WB = isLoad ? MI.getOperand(1) : MI.getOperand(0); 208 const MachineOperand &Base = MI.getOperand(2); 209 const MachineOperand &Offset = MI.getOperand(NumOps - 3); 210 Register WBReg = WB.getReg(); 211 Register BaseReg = Base.getReg(); 212 Register OffReg = Offset.getReg(); 213 unsigned OffImm = MI.getOperand(NumOps - 2).getImm(); 214 ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI.getOperand(NumOps - 1).getImm(); 215 switch (AddrMode) { 216 default: llvm_unreachable("Unknown indexed op!"); 217 case ARMII::AddrMode2: { 218 bool isSub = ARM_AM::getAM2Op(OffImm) == ARM_AM::sub; 219 unsigned Amt = ARM_AM::getAM2Offset(OffImm); 220 if (OffReg == 0) { 221 if (ARM_AM::getSOImmVal(Amt) == -1) 222 // Can't encode it in a so_imm operand. This transformation will 223 // add more than 1 instruction. Abandon! 224 return nullptr; 225 UpdateMI = BuildMI(MF, MI.getDebugLoc(), 226 get(isSub ? ARM::SUBri : ARM::ADDri), WBReg) 227 .addReg(BaseReg) 228 .addImm(Amt) 229 .add(predOps(Pred)) 230 .add(condCodeOp()); 231 } else if (Amt != 0) { 232 ARM_AM::ShiftOpc ShOpc = ARM_AM::getAM2ShiftOpc(OffImm); 233 unsigned SOOpc = ARM_AM::getSORegOpc(ShOpc, Amt); 234 UpdateMI = BuildMI(MF, MI.getDebugLoc(), 235 get(isSub ? ARM::SUBrsi : ARM::ADDrsi), WBReg) 236 .addReg(BaseReg) 237 .addReg(OffReg) 238 .addReg(0) 239 .addImm(SOOpc) 240 .add(predOps(Pred)) 241 .add(condCodeOp()); 242 } else 243 UpdateMI = BuildMI(MF, MI.getDebugLoc(), 244 get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg) 245 .addReg(BaseReg) 246 .addReg(OffReg) 247 .add(predOps(Pred)) 248 .add(condCodeOp()); 249 break; 250 } 251 case ARMII::AddrMode3 : { 252 bool isSub = ARM_AM::getAM3Op(OffImm) == ARM_AM::sub; 253 unsigned Amt = ARM_AM::getAM3Offset(OffImm); 254 if (OffReg == 0) 255 // Immediate is 8-bits. It's guaranteed to fit in a so_imm operand. 256 UpdateMI = BuildMI(MF, MI.getDebugLoc(), 257 get(isSub ? ARM::SUBri : ARM::ADDri), WBReg) 258 .addReg(BaseReg) 259 .addImm(Amt) 260 .add(predOps(Pred)) 261 .add(condCodeOp()); 262 else 263 UpdateMI = BuildMI(MF, MI.getDebugLoc(), 264 get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg) 265 .addReg(BaseReg) 266 .addReg(OffReg) 267 .add(predOps(Pred)) 268 .add(condCodeOp()); 269 break; 270 } 271 } 272 273 std::vector<MachineInstr*> NewMIs; 274 if (isPre) { 275 if (isLoad) 276 MemMI = 277 BuildMI(MF, MI.getDebugLoc(), get(MemOpc), MI.getOperand(0).getReg()) 278 .addReg(WBReg) 279 .addImm(0) 280 .addImm(Pred); 281 else 282 MemMI = BuildMI(MF, MI.getDebugLoc(), get(MemOpc)) 283 .addReg(MI.getOperand(1).getReg()) 284 .addReg(WBReg) 285 .addReg(0) 286 .addImm(0) 287 .addImm(Pred); 288 NewMIs.push_back(MemMI); 289 NewMIs.push_back(UpdateMI); 290 } else { 291 if (isLoad) 292 MemMI = 293 BuildMI(MF, MI.getDebugLoc(), get(MemOpc), MI.getOperand(0).getReg()) 294 .addReg(BaseReg) 295 .addImm(0) 296 .addImm(Pred); 297 else 298 MemMI = BuildMI(MF, MI.getDebugLoc(), get(MemOpc)) 299 .addReg(MI.getOperand(1).getReg()) 300 .addReg(BaseReg) 301 .addReg(0) 302 .addImm(0) 303 .addImm(Pred); 304 if (WB.isDead()) 305 UpdateMI->getOperand(0).setIsDead(); 306 NewMIs.push_back(UpdateMI); 307 NewMIs.push_back(MemMI); 308 } 309 310 // Transfer LiveVariables states, kill / dead info. 311 if (LV) { 312 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 313 MachineOperand &MO = MI.getOperand(i); 314 if (MO.isReg() && Register::isVirtualRegister(MO.getReg())) { 315 Register Reg = MO.getReg(); 316 317 LiveVariables::VarInfo &VI = LV->getVarInfo(Reg); 318 if (MO.isDef()) { 319 MachineInstr *NewMI = (Reg == WBReg) ? UpdateMI : MemMI; 320 if (MO.isDead()) 321 LV->addVirtualRegisterDead(Reg, *NewMI); 322 } 323 if (MO.isUse() && MO.isKill()) { 324 for (unsigned j = 0; j < 2; ++j) { 325 // Look at the two new MI's in reverse order. 326 MachineInstr *NewMI = NewMIs[j]; 327 if (!NewMI->readsRegister(Reg)) 328 continue; 329 LV->addVirtualRegisterKilled(Reg, *NewMI); 330 if (VI.removeKill(MI)) 331 VI.Kills.push_back(NewMI); 332 break; 333 } 334 } 335 } 336 } 337 } 338 339 MachineBasicBlock::iterator MBBI = MI.getIterator(); 340 MFI->insert(MBBI, NewMIs[1]); 341 MFI->insert(MBBI, NewMIs[0]); 342 return NewMIs[0]; 343 } 344 345 // Branch analysis. 346 bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB, 347 MachineBasicBlock *&TBB, 348 MachineBasicBlock *&FBB, 349 SmallVectorImpl<MachineOperand> &Cond, 350 bool AllowModify) const { 351 TBB = nullptr; 352 FBB = nullptr; 353 354 MachineBasicBlock::instr_iterator I = MBB.instr_end(); 355 if (I == MBB.instr_begin()) 356 return false; // Empty blocks are easy. 357 --I; 358 359 // Walk backwards from the end of the basic block until the branch is 360 // analyzed or we give up. 361 while (isPredicated(*I) || I->isTerminator() || I->isDebugValue()) { 362 // Flag to be raised on unanalyzeable instructions. This is useful in cases 363 // where we want to clean up on the end of the basic block before we bail 364 // out. 365 bool CantAnalyze = false; 366 367 // Skip over DEBUG values, predicated nonterminators and speculation 368 // barrier terminators. 369 while (I->isDebugInstr() || !I->isTerminator() || 370 isSpeculationBarrierEndBBOpcode(I->getOpcode()) || 371 I->getOpcode() == ARM::t2DoLoopStartTP){ 372 if (I == MBB.instr_begin()) 373 return false; 374 --I; 375 } 376 377 if (isIndirectBranchOpcode(I->getOpcode()) || 378 isJumpTableBranchOpcode(I->getOpcode())) { 379 // Indirect branches and jump tables can't be analyzed, but we still want 380 // to clean up any instructions at the tail of the basic block. 381 CantAnalyze = true; 382 } else if (isUncondBranchOpcode(I->getOpcode())) { 383 TBB = I->getOperand(0).getMBB(); 384 } else if (isCondBranchOpcode(I->getOpcode())) { 385 // Bail out if we encounter multiple conditional branches. 386 if (!Cond.empty()) 387 return true; 388 389 assert(!FBB && "FBB should have been null."); 390 FBB = TBB; 391 TBB = I->getOperand(0).getMBB(); 392 Cond.push_back(I->getOperand(1)); 393 Cond.push_back(I->getOperand(2)); 394 } else if (I->isReturn()) { 395 // Returns can't be analyzed, but we should run cleanup. 396 CantAnalyze = true; 397 } else { 398 // We encountered other unrecognized terminator. Bail out immediately. 399 return true; 400 } 401 402 // Cleanup code - to be run for unpredicated unconditional branches and 403 // returns. 404 if (!isPredicated(*I) && 405 (isUncondBranchOpcode(I->getOpcode()) || 406 isIndirectBranchOpcode(I->getOpcode()) || 407 isJumpTableBranchOpcode(I->getOpcode()) || 408 I->isReturn())) { 409 // Forget any previous condition branch information - it no longer applies. 410 Cond.clear(); 411 FBB = nullptr; 412 413 // If we can modify the function, delete everything below this 414 // unconditional branch. 415 if (AllowModify) { 416 MachineBasicBlock::iterator DI = std::next(I); 417 while (DI != MBB.instr_end()) { 418 MachineInstr &InstToDelete = *DI; 419 ++DI; 420 // Speculation barriers must not be deleted. 421 if (isSpeculationBarrierEndBBOpcode(InstToDelete.getOpcode())) 422 continue; 423 InstToDelete.eraseFromParent(); 424 } 425 } 426 } 427 428 if (CantAnalyze) { 429 // We may not be able to analyze the block, but we could still have 430 // an unconditional branch as the last instruction in the block, which 431 // just branches to layout successor. If this is the case, then just 432 // remove it if we're allowed to make modifications. 433 if (AllowModify && !isPredicated(MBB.back()) && 434 isUncondBranchOpcode(MBB.back().getOpcode()) && 435 TBB && MBB.isLayoutSuccessor(TBB)) 436 removeBranch(MBB); 437 return true; 438 } 439 440 if (I == MBB.instr_begin()) 441 return false; 442 443 --I; 444 } 445 446 // We made it past the terminators without bailing out - we must have 447 // analyzed this branch successfully. 448 return false; 449 } 450 451 unsigned ARMBaseInstrInfo::removeBranch(MachineBasicBlock &MBB, 452 int *BytesRemoved) const { 453 assert(!BytesRemoved && "code size not handled"); 454 455 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); 456 if (I == MBB.end()) 457 return 0; 458 459 if (!isUncondBranchOpcode(I->getOpcode()) && 460 !isCondBranchOpcode(I->getOpcode())) 461 return 0; 462 463 // Remove the branch. 464 I->eraseFromParent(); 465 466 I = MBB.end(); 467 468 if (I == MBB.begin()) return 1; 469 --I; 470 if (!isCondBranchOpcode(I->getOpcode())) 471 return 1; 472 473 // Remove the branch. 474 I->eraseFromParent(); 475 return 2; 476 } 477 478 unsigned ARMBaseInstrInfo::insertBranch(MachineBasicBlock &MBB, 479 MachineBasicBlock *TBB, 480 MachineBasicBlock *FBB, 481 ArrayRef<MachineOperand> Cond, 482 const DebugLoc &DL, 483 int *BytesAdded) const { 484 assert(!BytesAdded && "code size not handled"); 485 ARMFunctionInfo *AFI = MBB.getParent()->getInfo<ARMFunctionInfo>(); 486 int BOpc = !AFI->isThumbFunction() 487 ? ARM::B : (AFI->isThumb2Function() ? ARM::t2B : ARM::tB); 488 int BccOpc = !AFI->isThumbFunction() 489 ? ARM::Bcc : (AFI->isThumb2Function() ? ARM::t2Bcc : ARM::tBcc); 490 bool isThumb = AFI->isThumbFunction() || AFI->isThumb2Function(); 491 492 // Shouldn't be a fall through. 493 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 494 assert((Cond.size() == 2 || Cond.size() == 0) && 495 "ARM branch conditions have two components!"); 496 497 // For conditional branches, we use addOperand to preserve CPSR flags. 498 499 if (!FBB) { 500 if (Cond.empty()) { // Unconditional branch? 501 if (isThumb) 502 BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB).add(predOps(ARMCC::AL)); 503 else 504 BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB); 505 } else 506 BuildMI(&MBB, DL, get(BccOpc)) 507 .addMBB(TBB) 508 .addImm(Cond[0].getImm()) 509 .add(Cond[1]); 510 return 1; 511 } 512 513 // Two-way conditional branch. 514 BuildMI(&MBB, DL, get(BccOpc)) 515 .addMBB(TBB) 516 .addImm(Cond[0].getImm()) 517 .add(Cond[1]); 518 if (isThumb) 519 BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB).add(predOps(ARMCC::AL)); 520 else 521 BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB); 522 return 2; 523 } 524 525 bool ARMBaseInstrInfo:: 526 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { 527 ARMCC::CondCodes CC = (ARMCC::CondCodes)(int)Cond[0].getImm(); 528 Cond[0].setImm(ARMCC::getOppositeCondition(CC)); 529 return false; 530 } 531 532 bool ARMBaseInstrInfo::isPredicated(const MachineInstr &MI) const { 533 if (MI.isBundle()) { 534 MachineBasicBlock::const_instr_iterator I = MI.getIterator(); 535 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end(); 536 while (++I != E && I->isInsideBundle()) { 537 int PIdx = I->findFirstPredOperandIdx(); 538 if (PIdx != -1 && I->getOperand(PIdx).getImm() != ARMCC::AL) 539 return true; 540 } 541 return false; 542 } 543 544 int PIdx = MI.findFirstPredOperandIdx(); 545 return PIdx != -1 && MI.getOperand(PIdx).getImm() != ARMCC::AL; 546 } 547 548 std::string ARMBaseInstrInfo::createMIROperandComment( 549 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, 550 const TargetRegisterInfo *TRI) const { 551 552 // First, let's see if there is a generic comment for this operand 553 std::string GenericComment = 554 TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI); 555 if (!GenericComment.empty()) 556 return GenericComment; 557 558 // If not, check if we have an immediate operand. 559 if (Op.getType() != MachineOperand::MO_Immediate) 560 return std::string(); 561 562 // And print its corresponding condition code if the immediate is a 563 // predicate. 564 int FirstPredOp = MI.findFirstPredOperandIdx(); 565 if (FirstPredOp != (int) OpIdx) 566 return std::string(); 567 568 std::string CC = "CC::"; 569 CC += ARMCondCodeToString((ARMCC::CondCodes)Op.getImm()); 570 return CC; 571 } 572 573 bool ARMBaseInstrInfo::PredicateInstruction( 574 MachineInstr &MI, ArrayRef<MachineOperand> Pred) const { 575 unsigned Opc = MI.getOpcode(); 576 if (isUncondBranchOpcode(Opc)) { 577 MI.setDesc(get(getMatchingCondBranchOpcode(Opc))); 578 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 579 .addImm(Pred[0].getImm()) 580 .addReg(Pred[1].getReg()); 581 return true; 582 } 583 584 int PIdx = MI.findFirstPredOperandIdx(); 585 if (PIdx != -1) { 586 MachineOperand &PMO = MI.getOperand(PIdx); 587 PMO.setImm(Pred[0].getImm()); 588 MI.getOperand(PIdx+1).setReg(Pred[1].getReg()); 589 590 // Thumb 1 arithmetic instructions do not set CPSR when executed inside an 591 // IT block. This affects how they are printed. 592 const MCInstrDesc &MCID = MI.getDesc(); 593 if (MCID.TSFlags & ARMII::ThumbArithFlagSetting) { 594 assert(MCID.OpInfo[1].isOptionalDef() && "CPSR def isn't expected operand"); 595 assert((MI.getOperand(1).isDead() || 596 MI.getOperand(1).getReg() != ARM::CPSR) && 597 "if conversion tried to stop defining used CPSR"); 598 MI.getOperand(1).setReg(ARM::NoRegister); 599 } 600 601 return true; 602 } 603 return false; 604 } 605 606 bool ARMBaseInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1, 607 ArrayRef<MachineOperand> Pred2) const { 608 if (Pred1.size() > 2 || Pred2.size() > 2) 609 return false; 610 611 ARMCC::CondCodes CC1 = (ARMCC::CondCodes)Pred1[0].getImm(); 612 ARMCC::CondCodes CC2 = (ARMCC::CondCodes)Pred2[0].getImm(); 613 if (CC1 == CC2) 614 return true; 615 616 switch (CC1) { 617 default: 618 return false; 619 case ARMCC::AL: 620 return true; 621 case ARMCC::HS: 622 return CC2 == ARMCC::HI; 623 case ARMCC::LS: 624 return CC2 == ARMCC::LO || CC2 == ARMCC::EQ; 625 case ARMCC::GE: 626 return CC2 == ARMCC::GT; 627 case ARMCC::LE: 628 return CC2 == ARMCC::LT; 629 } 630 } 631 632 bool ARMBaseInstrInfo::ClobbersPredicate(MachineInstr &MI, 633 std::vector<MachineOperand> &Pred, 634 bool SkipDead) const { 635 bool Found = false; 636 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 637 const MachineOperand &MO = MI.getOperand(i); 638 bool ClobbersCPSR = MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR); 639 bool IsCPSR = MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR; 640 if (ClobbersCPSR || IsCPSR) { 641 642 // Filter out T1 instructions that have a dead CPSR, 643 // allowing IT blocks to be generated containing T1 instructions 644 const MCInstrDesc &MCID = MI.getDesc(); 645 if (MCID.TSFlags & ARMII::ThumbArithFlagSetting && MO.isDead() && 646 SkipDead) 647 continue; 648 649 Pred.push_back(MO); 650 Found = true; 651 } 652 } 653 654 return Found; 655 } 656 657 bool ARMBaseInstrInfo::isCPSRDefined(const MachineInstr &MI) { 658 for (const auto &MO : MI.operands()) 659 if (MO.isReg() && MO.getReg() == ARM::CPSR && MO.isDef() && !MO.isDead()) 660 return true; 661 return false; 662 } 663 664 static bool isEligibleForITBlock(const MachineInstr *MI) { 665 switch (MI->getOpcode()) { 666 default: return true; 667 case ARM::tADC: // ADC (register) T1 668 case ARM::tADDi3: // ADD (immediate) T1 669 case ARM::tADDi8: // ADD (immediate) T2 670 case ARM::tADDrr: // ADD (register) T1 671 case ARM::tAND: // AND (register) T1 672 case ARM::tASRri: // ASR (immediate) T1 673 case ARM::tASRrr: // ASR (register) T1 674 case ARM::tBIC: // BIC (register) T1 675 case ARM::tEOR: // EOR (register) T1 676 case ARM::tLSLri: // LSL (immediate) T1 677 case ARM::tLSLrr: // LSL (register) T1 678 case ARM::tLSRri: // LSR (immediate) T1 679 case ARM::tLSRrr: // LSR (register) T1 680 case ARM::tMUL: // MUL T1 681 case ARM::tMVN: // MVN (register) T1 682 case ARM::tORR: // ORR (register) T1 683 case ARM::tROR: // ROR (register) T1 684 case ARM::tRSB: // RSB (immediate) T1 685 case ARM::tSBC: // SBC (register) T1 686 case ARM::tSUBi3: // SUB (immediate) T1 687 case ARM::tSUBi8: // SUB (immediate) T2 688 case ARM::tSUBrr: // SUB (register) T1 689 return !ARMBaseInstrInfo::isCPSRDefined(*MI); 690 } 691 } 692 693 /// isPredicable - Return true if the specified instruction can be predicated. 694 /// By default, this returns true for every instruction with a 695 /// PredicateOperand. 696 bool ARMBaseInstrInfo::isPredicable(const MachineInstr &MI) const { 697 if (!MI.isPredicable()) 698 return false; 699 700 if (MI.isBundle()) 701 return false; 702 703 if (!isEligibleForITBlock(&MI)) 704 return false; 705 706 const MachineFunction *MF = MI.getParent()->getParent(); 707 const ARMFunctionInfo *AFI = 708 MF->getInfo<ARMFunctionInfo>(); 709 710 // Neon instructions in Thumb2 IT blocks are deprecated, see ARMARM. 711 // In their ARM encoding, they can't be encoded in a conditional form. 712 if ((MI.getDesc().TSFlags & ARMII::DomainMask) == ARMII::DomainNEON) 713 return false; 714 715 // Make indirect control flow changes unpredicable when SLS mitigation is 716 // enabled. 717 const ARMSubtarget &ST = MF->getSubtarget<ARMSubtarget>(); 718 if (ST.hardenSlsRetBr() && isIndirectControlFlowNotComingBack(MI)) 719 return false; 720 if (ST.hardenSlsBlr() && isIndirectCall(MI)) 721 return false; 722 723 if (AFI->isThumb2Function()) { 724 if (getSubtarget().restrictIT()) 725 return isV8EligibleForIT(&MI); 726 } 727 728 return true; 729 } 730 731 namespace llvm { 732 733 template <> bool IsCPSRDead<MachineInstr>(const MachineInstr *MI) { 734 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { 735 const MachineOperand &MO = MI->getOperand(i); 736 if (!MO.isReg() || MO.isUndef() || MO.isUse()) 737 continue; 738 if (MO.getReg() != ARM::CPSR) 739 continue; 740 if (!MO.isDead()) 741 return false; 742 } 743 // all definitions of CPSR are dead 744 return true; 745 } 746 747 } // end namespace llvm 748 749 /// GetInstSize - Return the size of the specified MachineInstr. 750 /// 751 unsigned ARMBaseInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 752 const MachineBasicBlock &MBB = *MI.getParent(); 753 const MachineFunction *MF = MBB.getParent(); 754 const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo(); 755 756 const MCInstrDesc &MCID = MI.getDesc(); 757 if (MCID.getSize()) 758 return MCID.getSize(); 759 760 switch (MI.getOpcode()) { 761 default: 762 // pseudo-instruction sizes are zero. 763 return 0; 764 case TargetOpcode::BUNDLE: 765 return getInstBundleLength(MI); 766 case ARM::MOVi16_ga_pcrel: 767 case ARM::MOVTi16_ga_pcrel: 768 case ARM::t2MOVi16_ga_pcrel: 769 case ARM::t2MOVTi16_ga_pcrel: 770 return 4; 771 case ARM::MOVi32imm: 772 case ARM::t2MOVi32imm: 773 return 8; 774 case ARM::CONSTPOOL_ENTRY: 775 case ARM::JUMPTABLE_INSTS: 776 case ARM::JUMPTABLE_ADDRS: 777 case ARM::JUMPTABLE_TBB: 778 case ARM::JUMPTABLE_TBH: 779 // If this machine instr is a constant pool entry, its size is recorded as 780 // operand #2. 781 return MI.getOperand(2).getImm(); 782 case ARM::Int_eh_sjlj_longjmp: 783 return 16; 784 case ARM::tInt_eh_sjlj_longjmp: 785 return 10; 786 case ARM::tInt_WIN_eh_sjlj_longjmp: 787 return 12; 788 case ARM::Int_eh_sjlj_setjmp: 789 case ARM::Int_eh_sjlj_setjmp_nofp: 790 return 20; 791 case ARM::tInt_eh_sjlj_setjmp: 792 case ARM::t2Int_eh_sjlj_setjmp: 793 case ARM::t2Int_eh_sjlj_setjmp_nofp: 794 return 12; 795 case ARM::SPACE: 796 return MI.getOperand(1).getImm(); 797 case ARM::INLINEASM: 798 case ARM::INLINEASM_BR: { 799 // If this machine instr is an inline asm, measure it. 800 unsigned Size = getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI); 801 if (!MF->getInfo<ARMFunctionInfo>()->isThumbFunction()) 802 Size = alignTo(Size, 4); 803 return Size; 804 } 805 case ARM::SpeculationBarrierISBDSBEndBB: 806 case ARM::t2SpeculationBarrierISBDSBEndBB: 807 // This gets lowered to 2 4-byte instructions. 808 return 8; 809 case ARM::SpeculationBarrierSBEndBB: 810 case ARM::t2SpeculationBarrierSBEndBB: 811 // This gets lowered to 1 4-byte instructions. 812 return 4; 813 } 814 } 815 816 unsigned ARMBaseInstrInfo::getInstBundleLength(const MachineInstr &MI) const { 817 unsigned Size = 0; 818 MachineBasicBlock::const_instr_iterator I = MI.getIterator(); 819 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end(); 820 while (++I != E && I->isInsideBundle()) { 821 assert(!I->isBundle() && "No nested bundle!"); 822 Size += getInstSizeInBytes(*I); 823 } 824 return Size; 825 } 826 827 void ARMBaseInstrInfo::copyFromCPSR(MachineBasicBlock &MBB, 828 MachineBasicBlock::iterator I, 829 unsigned DestReg, bool KillSrc, 830 const ARMSubtarget &Subtarget) const { 831 unsigned Opc = Subtarget.isThumb() 832 ? (Subtarget.isMClass() ? ARM::t2MRS_M : ARM::t2MRS_AR) 833 : ARM::MRS; 834 835 MachineInstrBuilder MIB = 836 BuildMI(MBB, I, I->getDebugLoc(), get(Opc), DestReg); 837 838 // There is only 1 A/R class MRS instruction, and it always refers to 839 // APSR. However, there are lots of other possibilities on M-class cores. 840 if (Subtarget.isMClass()) 841 MIB.addImm(0x800); 842 843 MIB.add(predOps(ARMCC::AL)) 844 .addReg(ARM::CPSR, RegState::Implicit | getKillRegState(KillSrc)); 845 } 846 847 void ARMBaseInstrInfo::copyToCPSR(MachineBasicBlock &MBB, 848 MachineBasicBlock::iterator I, 849 unsigned SrcReg, bool KillSrc, 850 const ARMSubtarget &Subtarget) const { 851 unsigned Opc = Subtarget.isThumb() 852 ? (Subtarget.isMClass() ? ARM::t2MSR_M : ARM::t2MSR_AR) 853 : ARM::MSR; 854 855 MachineInstrBuilder MIB = BuildMI(MBB, I, I->getDebugLoc(), get(Opc)); 856 857 if (Subtarget.isMClass()) 858 MIB.addImm(0x800); 859 else 860 MIB.addImm(8); 861 862 MIB.addReg(SrcReg, getKillRegState(KillSrc)) 863 .add(predOps(ARMCC::AL)) 864 .addReg(ARM::CPSR, RegState::Implicit | RegState::Define); 865 } 866 867 void llvm::addUnpredicatedMveVpredNOp(MachineInstrBuilder &MIB) { 868 MIB.addImm(ARMVCC::None); 869 MIB.addReg(0); 870 } 871 872 void llvm::addUnpredicatedMveVpredROp(MachineInstrBuilder &MIB, 873 Register DestReg) { 874 addUnpredicatedMveVpredNOp(MIB); 875 MIB.addReg(DestReg, RegState::Undef); 876 } 877 878 void llvm::addPredicatedMveVpredNOp(MachineInstrBuilder &MIB, unsigned Cond) { 879 MIB.addImm(Cond); 880 MIB.addReg(ARM::VPR, RegState::Implicit); 881 } 882 883 void llvm::addPredicatedMveVpredROp(MachineInstrBuilder &MIB, 884 unsigned Cond, unsigned Inactive) { 885 addPredicatedMveVpredNOp(MIB, Cond); 886 MIB.addReg(Inactive); 887 } 888 889 void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 890 MachineBasicBlock::iterator I, 891 const DebugLoc &DL, MCRegister DestReg, 892 MCRegister SrcReg, bool KillSrc) const { 893 bool GPRDest = ARM::GPRRegClass.contains(DestReg); 894 bool GPRSrc = ARM::GPRRegClass.contains(SrcReg); 895 896 if (GPRDest && GPRSrc) { 897 BuildMI(MBB, I, DL, get(ARM::MOVr), DestReg) 898 .addReg(SrcReg, getKillRegState(KillSrc)) 899 .add(predOps(ARMCC::AL)) 900 .add(condCodeOp()); 901 return; 902 } 903 904 bool SPRDest = ARM::SPRRegClass.contains(DestReg); 905 bool SPRSrc = ARM::SPRRegClass.contains(SrcReg); 906 907 unsigned Opc = 0; 908 if (SPRDest && SPRSrc) 909 Opc = ARM::VMOVS; 910 else if (GPRDest && SPRSrc) 911 Opc = ARM::VMOVRS; 912 else if (SPRDest && GPRSrc) 913 Opc = ARM::VMOVSR; 914 else if (ARM::DPRRegClass.contains(DestReg, SrcReg) && Subtarget.hasFP64()) 915 Opc = ARM::VMOVD; 916 else if (ARM::QPRRegClass.contains(DestReg, SrcReg)) 917 Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR; 918 919 if (Opc) { 920 MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opc), DestReg); 921 MIB.addReg(SrcReg, getKillRegState(KillSrc)); 922 if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR) 923 MIB.addReg(SrcReg, getKillRegState(KillSrc)); 924 if (Opc == ARM::MVE_VORR) 925 addUnpredicatedMveVpredROp(MIB, DestReg); 926 else 927 MIB.add(predOps(ARMCC::AL)); 928 return; 929 } 930 931 // Handle register classes that require multiple instructions. 932 unsigned BeginIdx = 0; 933 unsigned SubRegs = 0; 934 int Spacing = 1; 935 936 // Use VORRq when possible. 937 if (ARM::QQPRRegClass.contains(DestReg, SrcReg)) { 938 Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR; 939 BeginIdx = ARM::qsub_0; 940 SubRegs = 2; 941 } else if (ARM::QQQQPRRegClass.contains(DestReg, SrcReg)) { 942 Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR; 943 BeginIdx = ARM::qsub_0; 944 SubRegs = 4; 945 // Fall back to VMOVD. 946 } else if (ARM::DPairRegClass.contains(DestReg, SrcReg)) { 947 Opc = ARM::VMOVD; 948 BeginIdx = ARM::dsub_0; 949 SubRegs = 2; 950 } else if (ARM::DTripleRegClass.contains(DestReg, SrcReg)) { 951 Opc = ARM::VMOVD; 952 BeginIdx = ARM::dsub_0; 953 SubRegs = 3; 954 } else if (ARM::DQuadRegClass.contains(DestReg, SrcReg)) { 955 Opc = ARM::VMOVD; 956 BeginIdx = ARM::dsub_0; 957 SubRegs = 4; 958 } else if (ARM::GPRPairRegClass.contains(DestReg, SrcReg)) { 959 Opc = Subtarget.isThumb2() ? ARM::tMOVr : ARM::MOVr; 960 BeginIdx = ARM::gsub_0; 961 SubRegs = 2; 962 } else if (ARM::DPairSpcRegClass.contains(DestReg, SrcReg)) { 963 Opc = ARM::VMOVD; 964 BeginIdx = ARM::dsub_0; 965 SubRegs = 2; 966 Spacing = 2; 967 } else if (ARM::DTripleSpcRegClass.contains(DestReg, SrcReg)) { 968 Opc = ARM::VMOVD; 969 BeginIdx = ARM::dsub_0; 970 SubRegs = 3; 971 Spacing = 2; 972 } else if (ARM::DQuadSpcRegClass.contains(DestReg, SrcReg)) { 973 Opc = ARM::VMOVD; 974 BeginIdx = ARM::dsub_0; 975 SubRegs = 4; 976 Spacing = 2; 977 } else if (ARM::DPRRegClass.contains(DestReg, SrcReg) && 978 !Subtarget.hasFP64()) { 979 Opc = ARM::VMOVS; 980 BeginIdx = ARM::ssub_0; 981 SubRegs = 2; 982 } else if (SrcReg == ARM::CPSR) { 983 copyFromCPSR(MBB, I, DestReg, KillSrc, Subtarget); 984 return; 985 } else if (DestReg == ARM::CPSR) { 986 copyToCPSR(MBB, I, SrcReg, KillSrc, Subtarget); 987 return; 988 } else if (DestReg == ARM::VPR) { 989 assert(ARM::GPRRegClass.contains(SrcReg)); 990 BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMSR_P0), DestReg) 991 .addReg(SrcReg, getKillRegState(KillSrc)) 992 .add(predOps(ARMCC::AL)); 993 return; 994 } else if (SrcReg == ARM::VPR) { 995 assert(ARM::GPRRegClass.contains(DestReg)); 996 BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMRS_P0), DestReg) 997 .addReg(SrcReg, getKillRegState(KillSrc)) 998 .add(predOps(ARMCC::AL)); 999 return; 1000 } else if (DestReg == ARM::FPSCR_NZCV) { 1001 assert(ARM::GPRRegClass.contains(SrcReg)); 1002 BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMSR_FPSCR_NZCVQC), DestReg) 1003 .addReg(SrcReg, getKillRegState(KillSrc)) 1004 .add(predOps(ARMCC::AL)); 1005 return; 1006 } else if (SrcReg == ARM::FPSCR_NZCV) { 1007 assert(ARM::GPRRegClass.contains(DestReg)); 1008 BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMRS_FPSCR_NZCVQC), DestReg) 1009 .addReg(SrcReg, getKillRegState(KillSrc)) 1010 .add(predOps(ARMCC::AL)); 1011 return; 1012 } 1013 1014 assert(Opc && "Impossible reg-to-reg copy"); 1015 1016 const TargetRegisterInfo *TRI = &getRegisterInfo(); 1017 MachineInstrBuilder Mov; 1018 1019 // Copy register tuples backward when the first Dest reg overlaps with SrcReg. 1020 if (TRI->regsOverlap(SrcReg, TRI->getSubReg(DestReg, BeginIdx))) { 1021 BeginIdx = BeginIdx + ((SubRegs - 1) * Spacing); 1022 Spacing = -Spacing; 1023 } 1024 #ifndef NDEBUG 1025 SmallSet<unsigned, 4> DstRegs; 1026 #endif 1027 for (unsigned i = 0; i != SubRegs; ++i) { 1028 Register Dst = TRI->getSubReg(DestReg, BeginIdx + i * Spacing); 1029 Register Src = TRI->getSubReg(SrcReg, BeginIdx + i * Spacing); 1030 assert(Dst && Src && "Bad sub-register"); 1031 #ifndef NDEBUG 1032 assert(!DstRegs.count(Src) && "destructive vector copy"); 1033 DstRegs.insert(Dst); 1034 #endif 1035 Mov = BuildMI(MBB, I, I->getDebugLoc(), get(Opc), Dst).addReg(Src); 1036 // VORR (NEON or MVE) takes two source operands. 1037 if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR) { 1038 Mov.addReg(Src); 1039 } 1040 // MVE VORR takes predicate operands in place of an ordinary condition. 1041 if (Opc == ARM::MVE_VORR) 1042 addUnpredicatedMveVpredROp(Mov, Dst); 1043 else 1044 Mov = Mov.add(predOps(ARMCC::AL)); 1045 // MOVr can set CC. 1046 if (Opc == ARM::MOVr) 1047 Mov = Mov.add(condCodeOp()); 1048 } 1049 // Add implicit super-register defs and kills to the last instruction. 1050 Mov->addRegisterDefined(DestReg, TRI); 1051 if (KillSrc) 1052 Mov->addRegisterKilled(SrcReg, TRI); 1053 } 1054 1055 Optional<DestSourcePair> 1056 ARMBaseInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const { 1057 // VMOVRRD is also a copy instruction but it requires 1058 // special way of handling. It is more complex copy version 1059 // and since that we are not considering it. For recognition 1060 // of such instruction isExtractSubregLike MI interface fuction 1061 // could be used. 1062 // VORRq is considered as a move only if two inputs are 1063 // the same register. 1064 if (!MI.isMoveReg() || 1065 (MI.getOpcode() == ARM::VORRq && 1066 MI.getOperand(1).getReg() != MI.getOperand(2).getReg())) 1067 return None; 1068 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 1069 } 1070 1071 Optional<ParamLoadedValue> 1072 ARMBaseInstrInfo::describeLoadedValue(const MachineInstr &MI, 1073 Register Reg) const { 1074 if (auto DstSrcPair = isCopyInstrImpl(MI)) { 1075 Register DstReg = DstSrcPair->Destination->getReg(); 1076 1077 // TODO: We don't handle cases where the forwarding reg is narrower/wider 1078 // than the copy registers. Consider for example: 1079 // 1080 // s16 = VMOVS s0 1081 // s17 = VMOVS s1 1082 // call @callee(d0) 1083 // 1084 // We'd like to describe the call site value of d0 as d8, but this requires 1085 // gathering and merging the descriptions for the two VMOVS instructions. 1086 // 1087 // We also don't handle the reverse situation, where the forwarding reg is 1088 // narrower than the copy destination: 1089 // 1090 // d8 = VMOVD d0 1091 // call @callee(s1) 1092 // 1093 // We need to produce a fragment description (the call site value of s1 is 1094 // /not/ just d8). 1095 if (DstReg != Reg) 1096 return None; 1097 } 1098 return TargetInstrInfo::describeLoadedValue(MI, Reg); 1099 } 1100 1101 const MachineInstrBuilder & 1102 ARMBaseInstrInfo::AddDReg(MachineInstrBuilder &MIB, unsigned Reg, 1103 unsigned SubIdx, unsigned State, 1104 const TargetRegisterInfo *TRI) const { 1105 if (!SubIdx) 1106 return MIB.addReg(Reg, State); 1107 1108 if (Register::isPhysicalRegister(Reg)) 1109 return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State); 1110 return MIB.addReg(Reg, State, SubIdx); 1111 } 1112 1113 void ARMBaseInstrInfo:: 1114 storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, 1115 Register SrcReg, bool isKill, int FI, 1116 const TargetRegisterClass *RC, 1117 const TargetRegisterInfo *TRI) const { 1118 MachineFunction &MF = *MBB.getParent(); 1119 MachineFrameInfo &MFI = MF.getFrameInfo(); 1120 Align Alignment = MFI.getObjectAlign(FI); 1121 1122 MachineMemOperand *MMO = MF.getMachineMemOperand( 1123 MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOStore, 1124 MFI.getObjectSize(FI), Alignment); 1125 1126 switch (TRI->getSpillSize(*RC)) { 1127 case 2: 1128 if (ARM::HPRRegClass.hasSubClassEq(RC)) { 1129 BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRH)) 1130 .addReg(SrcReg, getKillRegState(isKill)) 1131 .addFrameIndex(FI) 1132 .addImm(0) 1133 .addMemOperand(MMO) 1134 .add(predOps(ARMCC::AL)); 1135 } else 1136 llvm_unreachable("Unknown reg class!"); 1137 break; 1138 case 4: 1139 if (ARM::GPRRegClass.hasSubClassEq(RC)) { 1140 BuildMI(MBB, I, DebugLoc(), get(ARM::STRi12)) 1141 .addReg(SrcReg, getKillRegState(isKill)) 1142 .addFrameIndex(FI) 1143 .addImm(0) 1144 .addMemOperand(MMO) 1145 .add(predOps(ARMCC::AL)); 1146 } else if (ARM::SPRRegClass.hasSubClassEq(RC)) { 1147 BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRS)) 1148 .addReg(SrcReg, getKillRegState(isKill)) 1149 .addFrameIndex(FI) 1150 .addImm(0) 1151 .addMemOperand(MMO) 1152 .add(predOps(ARMCC::AL)); 1153 } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) { 1154 BuildMI(MBB, I, DebugLoc(), get(ARM::VSTR_P0_off)) 1155 .addReg(SrcReg, getKillRegState(isKill)) 1156 .addFrameIndex(FI) 1157 .addImm(0) 1158 .addMemOperand(MMO) 1159 .add(predOps(ARMCC::AL)); 1160 } else 1161 llvm_unreachable("Unknown reg class!"); 1162 break; 1163 case 8: 1164 if (ARM::DPRRegClass.hasSubClassEq(RC)) { 1165 BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRD)) 1166 .addReg(SrcReg, getKillRegState(isKill)) 1167 .addFrameIndex(FI) 1168 .addImm(0) 1169 .addMemOperand(MMO) 1170 .add(predOps(ARMCC::AL)); 1171 } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) { 1172 if (Subtarget.hasV5TEOps()) { 1173 MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::STRD)); 1174 AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI); 1175 AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI); 1176 MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO) 1177 .add(predOps(ARMCC::AL)); 1178 } else { 1179 // Fallback to STM instruction, which has existed since the dawn of 1180 // time. 1181 MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::STMIA)) 1182 .addFrameIndex(FI) 1183 .addMemOperand(MMO) 1184 .add(predOps(ARMCC::AL)); 1185 AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI); 1186 AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI); 1187 } 1188 } else 1189 llvm_unreachable("Unknown reg class!"); 1190 break; 1191 case 16: 1192 if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) { 1193 // Use aligned spills if the stack can be realigned. 1194 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) { 1195 BuildMI(MBB, I, DebugLoc(), get(ARM::VST1q64)) 1196 .addFrameIndex(FI) 1197 .addImm(16) 1198 .addReg(SrcReg, getKillRegState(isKill)) 1199 .addMemOperand(MMO) 1200 .add(predOps(ARMCC::AL)); 1201 } else { 1202 BuildMI(MBB, I, DebugLoc(), get(ARM::VSTMQIA)) 1203 .addReg(SrcReg, getKillRegState(isKill)) 1204 .addFrameIndex(FI) 1205 .addMemOperand(MMO) 1206 .add(predOps(ARMCC::AL)); 1207 } 1208 } else if (ARM::QPRRegClass.hasSubClassEq(RC) && 1209 Subtarget.hasMVEIntegerOps()) { 1210 auto MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::MVE_VSTRWU32)); 1211 MIB.addReg(SrcReg, getKillRegState(isKill)) 1212 .addFrameIndex(FI) 1213 .addImm(0) 1214 .addMemOperand(MMO); 1215 addUnpredicatedMveVpredNOp(MIB); 1216 } else 1217 llvm_unreachable("Unknown reg class!"); 1218 break; 1219 case 24: 1220 if (ARM::DTripleRegClass.hasSubClassEq(RC)) { 1221 // Use aligned spills if the stack can be realigned. 1222 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) && 1223 Subtarget.hasNEON()) { 1224 BuildMI(MBB, I, DebugLoc(), get(ARM::VST1d64TPseudo)) 1225 .addFrameIndex(FI) 1226 .addImm(16) 1227 .addReg(SrcReg, getKillRegState(isKill)) 1228 .addMemOperand(MMO) 1229 .add(predOps(ARMCC::AL)); 1230 } else { 1231 MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), 1232 get(ARM::VSTMDIA)) 1233 .addFrameIndex(FI) 1234 .add(predOps(ARMCC::AL)) 1235 .addMemOperand(MMO); 1236 MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI); 1237 MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI); 1238 AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI); 1239 } 1240 } else 1241 llvm_unreachable("Unknown reg class!"); 1242 break; 1243 case 32: 1244 if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) { 1245 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) && 1246 Subtarget.hasNEON()) { 1247 // FIXME: It's possible to only store part of the QQ register if the 1248 // spilled def has a sub-register index. 1249 BuildMI(MBB, I, DebugLoc(), get(ARM::VST1d64QPseudo)) 1250 .addFrameIndex(FI) 1251 .addImm(16) 1252 .addReg(SrcReg, getKillRegState(isKill)) 1253 .addMemOperand(MMO) 1254 .add(predOps(ARMCC::AL)); 1255 } else { 1256 MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), 1257 get(ARM::VSTMDIA)) 1258 .addFrameIndex(FI) 1259 .add(predOps(ARMCC::AL)) 1260 .addMemOperand(MMO); 1261 MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI); 1262 MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI); 1263 MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI); 1264 AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI); 1265 } 1266 } else 1267 llvm_unreachable("Unknown reg class!"); 1268 break; 1269 case 64: 1270 if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) { 1271 MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::VSTMDIA)) 1272 .addFrameIndex(FI) 1273 .add(predOps(ARMCC::AL)) 1274 .addMemOperand(MMO); 1275 MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI); 1276 MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI); 1277 MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI); 1278 MIB = AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI); 1279 MIB = AddDReg(MIB, SrcReg, ARM::dsub_4, 0, TRI); 1280 MIB = AddDReg(MIB, SrcReg, ARM::dsub_5, 0, TRI); 1281 MIB = AddDReg(MIB, SrcReg, ARM::dsub_6, 0, TRI); 1282 AddDReg(MIB, SrcReg, ARM::dsub_7, 0, TRI); 1283 } else 1284 llvm_unreachable("Unknown reg class!"); 1285 break; 1286 default: 1287 llvm_unreachable("Unknown reg class!"); 1288 } 1289 } 1290 1291 unsigned ARMBaseInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 1292 int &FrameIndex) const { 1293 switch (MI.getOpcode()) { 1294 default: break; 1295 case ARM::STRrs: 1296 case ARM::t2STRs: // FIXME: don't use t2STRs to access frame. 1297 if (MI.getOperand(1).isFI() && MI.getOperand(2).isReg() && 1298 MI.getOperand(3).isImm() && MI.getOperand(2).getReg() == 0 && 1299 MI.getOperand(3).getImm() == 0) { 1300 FrameIndex = MI.getOperand(1).getIndex(); 1301 return MI.getOperand(0).getReg(); 1302 } 1303 break; 1304 case ARM::STRi12: 1305 case ARM::t2STRi12: 1306 case ARM::tSTRspi: 1307 case ARM::VSTRD: 1308 case ARM::VSTRS: 1309 case ARM::VSTR_P0_off: 1310 case ARM::MVE_VSTRWU32: 1311 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() && 1312 MI.getOperand(2).getImm() == 0) { 1313 FrameIndex = MI.getOperand(1).getIndex(); 1314 return MI.getOperand(0).getReg(); 1315 } 1316 break; 1317 case ARM::VST1q64: 1318 case ARM::VST1d64TPseudo: 1319 case ARM::VST1d64QPseudo: 1320 if (MI.getOperand(0).isFI() && MI.getOperand(2).getSubReg() == 0) { 1321 FrameIndex = MI.getOperand(0).getIndex(); 1322 return MI.getOperand(2).getReg(); 1323 } 1324 break; 1325 case ARM::VSTMQIA: 1326 if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) { 1327 FrameIndex = MI.getOperand(1).getIndex(); 1328 return MI.getOperand(0).getReg(); 1329 } 1330 break; 1331 } 1332 1333 return 0; 1334 } 1335 1336 unsigned ARMBaseInstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI, 1337 int &FrameIndex) const { 1338 SmallVector<const MachineMemOperand *, 1> Accesses; 1339 if (MI.mayStore() && hasStoreToStackSlot(MI, Accesses) && 1340 Accesses.size() == 1) { 1341 FrameIndex = 1342 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) 1343 ->getFrameIndex(); 1344 return true; 1345 } 1346 return false; 1347 } 1348 1349 void ARMBaseInstrInfo:: 1350 loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, 1351 Register DestReg, int FI, 1352 const TargetRegisterClass *RC, 1353 const TargetRegisterInfo *TRI) const { 1354 DebugLoc DL; 1355 if (I != MBB.end()) DL = I->getDebugLoc(); 1356 MachineFunction &MF = *MBB.getParent(); 1357 MachineFrameInfo &MFI = MF.getFrameInfo(); 1358 const Align Alignment = MFI.getObjectAlign(FI); 1359 MachineMemOperand *MMO = MF.getMachineMemOperand( 1360 MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOLoad, 1361 MFI.getObjectSize(FI), Alignment); 1362 1363 switch (TRI->getSpillSize(*RC)) { 1364 case 2: 1365 if (ARM::HPRRegClass.hasSubClassEq(RC)) { 1366 BuildMI(MBB, I, DL, get(ARM::VLDRH), DestReg) 1367 .addFrameIndex(FI) 1368 .addImm(0) 1369 .addMemOperand(MMO) 1370 .add(predOps(ARMCC::AL)); 1371 } else 1372 llvm_unreachable("Unknown reg class!"); 1373 break; 1374 case 4: 1375 if (ARM::GPRRegClass.hasSubClassEq(RC)) { 1376 BuildMI(MBB, I, DL, get(ARM::LDRi12), DestReg) 1377 .addFrameIndex(FI) 1378 .addImm(0) 1379 .addMemOperand(MMO) 1380 .add(predOps(ARMCC::AL)); 1381 } else if (ARM::SPRRegClass.hasSubClassEq(RC)) { 1382 BuildMI(MBB, I, DL, get(ARM::VLDRS), DestReg) 1383 .addFrameIndex(FI) 1384 .addImm(0) 1385 .addMemOperand(MMO) 1386 .add(predOps(ARMCC::AL)); 1387 } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) { 1388 BuildMI(MBB, I, DL, get(ARM::VLDR_P0_off), DestReg) 1389 .addFrameIndex(FI) 1390 .addImm(0) 1391 .addMemOperand(MMO) 1392 .add(predOps(ARMCC::AL)); 1393 } else 1394 llvm_unreachable("Unknown reg class!"); 1395 break; 1396 case 8: 1397 if (ARM::DPRRegClass.hasSubClassEq(RC)) { 1398 BuildMI(MBB, I, DL, get(ARM::VLDRD), DestReg) 1399 .addFrameIndex(FI) 1400 .addImm(0) 1401 .addMemOperand(MMO) 1402 .add(predOps(ARMCC::AL)); 1403 } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) { 1404 MachineInstrBuilder MIB; 1405 1406 if (Subtarget.hasV5TEOps()) { 1407 MIB = BuildMI(MBB, I, DL, get(ARM::LDRD)); 1408 AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI); 1409 AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI); 1410 MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO) 1411 .add(predOps(ARMCC::AL)); 1412 } else { 1413 // Fallback to LDM instruction, which has existed since the dawn of 1414 // time. 1415 MIB = BuildMI(MBB, I, DL, get(ARM::LDMIA)) 1416 .addFrameIndex(FI) 1417 .addMemOperand(MMO) 1418 .add(predOps(ARMCC::AL)); 1419 MIB = AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI); 1420 MIB = AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI); 1421 } 1422 1423 if (Register::isPhysicalRegister(DestReg)) 1424 MIB.addReg(DestReg, RegState::ImplicitDefine); 1425 } else 1426 llvm_unreachable("Unknown reg class!"); 1427 break; 1428 case 16: 1429 if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) { 1430 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) { 1431 BuildMI(MBB, I, DL, get(ARM::VLD1q64), DestReg) 1432 .addFrameIndex(FI) 1433 .addImm(16) 1434 .addMemOperand(MMO) 1435 .add(predOps(ARMCC::AL)); 1436 } else { 1437 BuildMI(MBB, I, DL, get(ARM::VLDMQIA), DestReg) 1438 .addFrameIndex(FI) 1439 .addMemOperand(MMO) 1440 .add(predOps(ARMCC::AL)); 1441 } 1442 } else if (ARM::QPRRegClass.hasSubClassEq(RC) && 1443 Subtarget.hasMVEIntegerOps()) { 1444 auto MIB = BuildMI(MBB, I, DL, get(ARM::MVE_VLDRWU32), DestReg); 1445 MIB.addFrameIndex(FI) 1446 .addImm(0) 1447 .addMemOperand(MMO); 1448 addUnpredicatedMveVpredNOp(MIB); 1449 } else 1450 llvm_unreachable("Unknown reg class!"); 1451 break; 1452 case 24: 1453 if (ARM::DTripleRegClass.hasSubClassEq(RC)) { 1454 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) && 1455 Subtarget.hasNEON()) { 1456 BuildMI(MBB, I, DL, get(ARM::VLD1d64TPseudo), DestReg) 1457 .addFrameIndex(FI) 1458 .addImm(16) 1459 .addMemOperand(MMO) 1460 .add(predOps(ARMCC::AL)); 1461 } else { 1462 MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA)) 1463 .addFrameIndex(FI) 1464 .addMemOperand(MMO) 1465 .add(predOps(ARMCC::AL)); 1466 MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI); 1467 MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI); 1468 MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI); 1469 if (Register::isPhysicalRegister(DestReg)) 1470 MIB.addReg(DestReg, RegState::ImplicitDefine); 1471 } 1472 } else 1473 llvm_unreachable("Unknown reg class!"); 1474 break; 1475 case 32: 1476 if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) { 1477 if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) && 1478 Subtarget.hasNEON()) { 1479 BuildMI(MBB, I, DL, get(ARM::VLD1d64QPseudo), DestReg) 1480 .addFrameIndex(FI) 1481 .addImm(16) 1482 .addMemOperand(MMO) 1483 .add(predOps(ARMCC::AL)); 1484 } else { 1485 MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA)) 1486 .addFrameIndex(FI) 1487 .add(predOps(ARMCC::AL)) 1488 .addMemOperand(MMO); 1489 MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI); 1490 MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI); 1491 MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI); 1492 MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI); 1493 if (Register::isPhysicalRegister(DestReg)) 1494 MIB.addReg(DestReg, RegState::ImplicitDefine); 1495 } 1496 } else 1497 llvm_unreachable("Unknown reg class!"); 1498 break; 1499 case 64: 1500 if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) { 1501 MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA)) 1502 .addFrameIndex(FI) 1503 .add(predOps(ARMCC::AL)) 1504 .addMemOperand(MMO); 1505 MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI); 1506 MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI); 1507 MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI); 1508 MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI); 1509 MIB = AddDReg(MIB, DestReg, ARM::dsub_4, RegState::DefineNoRead, TRI); 1510 MIB = AddDReg(MIB, DestReg, ARM::dsub_5, RegState::DefineNoRead, TRI); 1511 MIB = AddDReg(MIB, DestReg, ARM::dsub_6, RegState::DefineNoRead, TRI); 1512 MIB = AddDReg(MIB, DestReg, ARM::dsub_7, RegState::DefineNoRead, TRI); 1513 if (Register::isPhysicalRegister(DestReg)) 1514 MIB.addReg(DestReg, RegState::ImplicitDefine); 1515 } else 1516 llvm_unreachable("Unknown reg class!"); 1517 break; 1518 default: 1519 llvm_unreachable("Unknown regclass!"); 1520 } 1521 } 1522 1523 unsigned ARMBaseInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 1524 int &FrameIndex) const { 1525 switch (MI.getOpcode()) { 1526 default: break; 1527 case ARM::LDRrs: 1528 case ARM::t2LDRs: // FIXME: don't use t2LDRs to access frame. 1529 if (MI.getOperand(1).isFI() && MI.getOperand(2).isReg() && 1530 MI.getOperand(3).isImm() && MI.getOperand(2).getReg() == 0 && 1531 MI.getOperand(3).getImm() == 0) { 1532 FrameIndex = MI.getOperand(1).getIndex(); 1533 return MI.getOperand(0).getReg(); 1534 } 1535 break; 1536 case ARM::LDRi12: 1537 case ARM::t2LDRi12: 1538 case ARM::tLDRspi: 1539 case ARM::VLDRD: 1540 case ARM::VLDRS: 1541 case ARM::VLDR_P0_off: 1542 case ARM::MVE_VLDRWU32: 1543 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() && 1544 MI.getOperand(2).getImm() == 0) { 1545 FrameIndex = MI.getOperand(1).getIndex(); 1546 return MI.getOperand(0).getReg(); 1547 } 1548 break; 1549 case ARM::VLD1q64: 1550 case ARM::VLD1d8TPseudo: 1551 case ARM::VLD1d16TPseudo: 1552 case ARM::VLD1d32TPseudo: 1553 case ARM::VLD1d64TPseudo: 1554 case ARM::VLD1d8QPseudo: 1555 case ARM::VLD1d16QPseudo: 1556 case ARM::VLD1d32QPseudo: 1557 case ARM::VLD1d64QPseudo: 1558 if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) { 1559 FrameIndex = MI.getOperand(1).getIndex(); 1560 return MI.getOperand(0).getReg(); 1561 } 1562 break; 1563 case ARM::VLDMQIA: 1564 if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) { 1565 FrameIndex = MI.getOperand(1).getIndex(); 1566 return MI.getOperand(0).getReg(); 1567 } 1568 break; 1569 } 1570 1571 return 0; 1572 } 1573 1574 unsigned ARMBaseInstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI, 1575 int &FrameIndex) const { 1576 SmallVector<const MachineMemOperand *, 1> Accesses; 1577 if (MI.mayLoad() && hasLoadFromStackSlot(MI, Accesses) && 1578 Accesses.size() == 1) { 1579 FrameIndex = 1580 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue()) 1581 ->getFrameIndex(); 1582 return true; 1583 } 1584 return false; 1585 } 1586 1587 /// Expands MEMCPY to either LDMIA/STMIA or LDMIA_UPD/STMID_UPD 1588 /// depending on whether the result is used. 1589 void ARMBaseInstrInfo::expandMEMCPY(MachineBasicBlock::iterator MI) const { 1590 bool isThumb1 = Subtarget.isThumb1Only(); 1591 bool isThumb2 = Subtarget.isThumb2(); 1592 const ARMBaseInstrInfo *TII = Subtarget.getInstrInfo(); 1593 1594 DebugLoc dl = MI->getDebugLoc(); 1595 MachineBasicBlock *BB = MI->getParent(); 1596 1597 MachineInstrBuilder LDM, STM; 1598 if (isThumb1 || !MI->getOperand(1).isDead()) { 1599 MachineOperand LDWb(MI->getOperand(1)); 1600 LDM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2LDMIA_UPD 1601 : isThumb1 ? ARM::tLDMIA_UPD 1602 : ARM::LDMIA_UPD)) 1603 .add(LDWb); 1604 } else { 1605 LDM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2LDMIA : ARM::LDMIA)); 1606 } 1607 1608 if (isThumb1 || !MI->getOperand(0).isDead()) { 1609 MachineOperand STWb(MI->getOperand(0)); 1610 STM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2STMIA_UPD 1611 : isThumb1 ? ARM::tSTMIA_UPD 1612 : ARM::STMIA_UPD)) 1613 .add(STWb); 1614 } else { 1615 STM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2STMIA : ARM::STMIA)); 1616 } 1617 1618 MachineOperand LDBase(MI->getOperand(3)); 1619 LDM.add(LDBase).add(predOps(ARMCC::AL)); 1620 1621 MachineOperand STBase(MI->getOperand(2)); 1622 STM.add(STBase).add(predOps(ARMCC::AL)); 1623 1624 // Sort the scratch registers into ascending order. 1625 const TargetRegisterInfo &TRI = getRegisterInfo(); 1626 SmallVector<unsigned, 6> ScratchRegs; 1627 for(unsigned I = 5; I < MI->getNumOperands(); ++I) 1628 ScratchRegs.push_back(MI->getOperand(I).getReg()); 1629 llvm::sort(ScratchRegs, 1630 [&TRI](const unsigned &Reg1, const unsigned &Reg2) -> bool { 1631 return TRI.getEncodingValue(Reg1) < 1632 TRI.getEncodingValue(Reg2); 1633 }); 1634 1635 for (const auto &Reg : ScratchRegs) { 1636 LDM.addReg(Reg, RegState::Define); 1637 STM.addReg(Reg, RegState::Kill); 1638 } 1639 1640 BB->erase(MI); 1641 } 1642 1643 bool ARMBaseInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 1644 if (MI.getOpcode() == TargetOpcode::LOAD_STACK_GUARD) { 1645 assert(getSubtarget().getTargetTriple().isOSBinFormatMachO() && 1646 "LOAD_STACK_GUARD currently supported only for MachO."); 1647 expandLoadStackGuard(MI); 1648 MI.getParent()->erase(MI); 1649 return true; 1650 } 1651 1652 if (MI.getOpcode() == ARM::MEMCPY) { 1653 expandMEMCPY(MI); 1654 return true; 1655 } 1656 1657 // This hook gets to expand COPY instructions before they become 1658 // copyPhysReg() calls. Look for VMOVS instructions that can legally be 1659 // widened to VMOVD. We prefer the VMOVD when possible because it may be 1660 // changed into a VORR that can go down the NEON pipeline. 1661 if (!MI.isCopy() || Subtarget.dontWidenVMOVS() || !Subtarget.hasFP64()) 1662 return false; 1663 1664 // Look for a copy between even S-registers. That is where we keep floats 1665 // when using NEON v2f32 instructions for f32 arithmetic. 1666 Register DstRegS = MI.getOperand(0).getReg(); 1667 Register SrcRegS = MI.getOperand(1).getReg(); 1668 if (!ARM::SPRRegClass.contains(DstRegS, SrcRegS)) 1669 return false; 1670 1671 const TargetRegisterInfo *TRI = &getRegisterInfo(); 1672 unsigned DstRegD = TRI->getMatchingSuperReg(DstRegS, ARM::ssub_0, 1673 &ARM::DPRRegClass); 1674 unsigned SrcRegD = TRI->getMatchingSuperReg(SrcRegS, ARM::ssub_0, 1675 &ARM::DPRRegClass); 1676 if (!DstRegD || !SrcRegD) 1677 return false; 1678 1679 // We want to widen this into a DstRegD = VMOVD SrcRegD copy. This is only 1680 // legal if the COPY already defines the full DstRegD, and it isn't a 1681 // sub-register insertion. 1682 if (!MI.definesRegister(DstRegD, TRI) || MI.readsRegister(DstRegD, TRI)) 1683 return false; 1684 1685 // A dead copy shouldn't show up here, but reject it just in case. 1686 if (MI.getOperand(0).isDead()) 1687 return false; 1688 1689 // All clear, widen the COPY. 1690 LLVM_DEBUG(dbgs() << "widening: " << MI); 1691 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI); 1692 1693 // Get rid of the old implicit-def of DstRegD. Leave it if it defines a Q-reg 1694 // or some other super-register. 1695 int ImpDefIdx = MI.findRegisterDefOperandIdx(DstRegD); 1696 if (ImpDefIdx != -1) 1697 MI.RemoveOperand(ImpDefIdx); 1698 1699 // Change the opcode and operands. 1700 MI.setDesc(get(ARM::VMOVD)); 1701 MI.getOperand(0).setReg(DstRegD); 1702 MI.getOperand(1).setReg(SrcRegD); 1703 MIB.add(predOps(ARMCC::AL)); 1704 1705 // We are now reading SrcRegD instead of SrcRegS. This may upset the 1706 // register scavenger and machine verifier, so we need to indicate that we 1707 // are reading an undefined value from SrcRegD, but a proper value from 1708 // SrcRegS. 1709 MI.getOperand(1).setIsUndef(); 1710 MIB.addReg(SrcRegS, RegState::Implicit); 1711 1712 // SrcRegD may actually contain an unrelated value in the ssub_1 1713 // sub-register. Don't kill it. Only kill the ssub_0 sub-register. 1714 if (MI.getOperand(1).isKill()) { 1715 MI.getOperand(1).setIsKill(false); 1716 MI.addRegisterKilled(SrcRegS, TRI, true); 1717 } 1718 1719 LLVM_DEBUG(dbgs() << "replaced by: " << MI); 1720 return true; 1721 } 1722 1723 /// Create a copy of a const pool value. Update CPI to the new index and return 1724 /// the label UID. 1725 static unsigned duplicateCPV(MachineFunction &MF, unsigned &CPI) { 1726 MachineConstantPool *MCP = MF.getConstantPool(); 1727 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); 1728 1729 const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPI]; 1730 assert(MCPE.isMachineConstantPoolEntry() && 1731 "Expecting a machine constantpool entry!"); 1732 ARMConstantPoolValue *ACPV = 1733 static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal); 1734 1735 unsigned PCLabelId = AFI->createPICLabelUId(); 1736 ARMConstantPoolValue *NewCPV = nullptr; 1737 1738 // FIXME: The below assumes PIC relocation model and that the function 1739 // is Thumb mode (t1 or t2). PCAdjustment would be 8 for ARM mode PIC, and 1740 // zero for non-PIC in ARM or Thumb. The callers are all of thumb LDR 1741 // instructions, so that's probably OK, but is PIC always correct when 1742 // we get here? 1743 if (ACPV->isGlobalValue()) 1744 NewCPV = ARMConstantPoolConstant::Create( 1745 cast<ARMConstantPoolConstant>(ACPV)->getGV(), PCLabelId, ARMCP::CPValue, 1746 4, ACPV->getModifier(), ACPV->mustAddCurrentAddress()); 1747 else if (ACPV->isExtSymbol()) 1748 NewCPV = ARMConstantPoolSymbol:: 1749 Create(MF.getFunction().getContext(), 1750 cast<ARMConstantPoolSymbol>(ACPV)->getSymbol(), PCLabelId, 4); 1751 else if (ACPV->isBlockAddress()) 1752 NewCPV = ARMConstantPoolConstant:: 1753 Create(cast<ARMConstantPoolConstant>(ACPV)->getBlockAddress(), PCLabelId, 1754 ARMCP::CPBlockAddress, 4); 1755 else if (ACPV->isLSDA()) 1756 NewCPV = ARMConstantPoolConstant::Create(&MF.getFunction(), PCLabelId, 1757 ARMCP::CPLSDA, 4); 1758 else if (ACPV->isMachineBasicBlock()) 1759 NewCPV = ARMConstantPoolMBB:: 1760 Create(MF.getFunction().getContext(), 1761 cast<ARMConstantPoolMBB>(ACPV)->getMBB(), PCLabelId, 4); 1762 else 1763 llvm_unreachable("Unexpected ARM constantpool value type!!"); 1764 CPI = MCP->getConstantPoolIndex(NewCPV, MCPE.getAlign()); 1765 return PCLabelId; 1766 } 1767 1768 void ARMBaseInstrInfo::reMaterialize(MachineBasicBlock &MBB, 1769 MachineBasicBlock::iterator I, 1770 Register DestReg, unsigned SubIdx, 1771 const MachineInstr &Orig, 1772 const TargetRegisterInfo &TRI) const { 1773 unsigned Opcode = Orig.getOpcode(); 1774 switch (Opcode) { 1775 default: { 1776 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig); 1777 MI->substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI); 1778 MBB.insert(I, MI); 1779 break; 1780 } 1781 case ARM::tLDRpci_pic: 1782 case ARM::t2LDRpci_pic: { 1783 MachineFunction &MF = *MBB.getParent(); 1784 unsigned CPI = Orig.getOperand(1).getIndex(); 1785 unsigned PCLabelId = duplicateCPV(MF, CPI); 1786 BuildMI(MBB, I, Orig.getDebugLoc(), get(Opcode), DestReg) 1787 .addConstantPoolIndex(CPI) 1788 .addImm(PCLabelId) 1789 .cloneMemRefs(Orig); 1790 break; 1791 } 1792 } 1793 } 1794 1795 MachineInstr & 1796 ARMBaseInstrInfo::duplicate(MachineBasicBlock &MBB, 1797 MachineBasicBlock::iterator InsertBefore, 1798 const MachineInstr &Orig) const { 1799 MachineInstr &Cloned = TargetInstrInfo::duplicate(MBB, InsertBefore, Orig); 1800 MachineBasicBlock::instr_iterator I = Cloned.getIterator(); 1801 for (;;) { 1802 switch (I->getOpcode()) { 1803 case ARM::tLDRpci_pic: 1804 case ARM::t2LDRpci_pic: { 1805 MachineFunction &MF = *MBB.getParent(); 1806 unsigned CPI = I->getOperand(1).getIndex(); 1807 unsigned PCLabelId = duplicateCPV(MF, CPI); 1808 I->getOperand(1).setIndex(CPI); 1809 I->getOperand(2).setImm(PCLabelId); 1810 break; 1811 } 1812 } 1813 if (!I->isBundledWithSucc()) 1814 break; 1815 ++I; 1816 } 1817 return Cloned; 1818 } 1819 1820 bool ARMBaseInstrInfo::produceSameValue(const MachineInstr &MI0, 1821 const MachineInstr &MI1, 1822 const MachineRegisterInfo *MRI) const { 1823 unsigned Opcode = MI0.getOpcode(); 1824 if (Opcode == ARM::t2LDRpci || 1825 Opcode == ARM::t2LDRpci_pic || 1826 Opcode == ARM::tLDRpci || 1827 Opcode == ARM::tLDRpci_pic || 1828 Opcode == ARM::LDRLIT_ga_pcrel || 1829 Opcode == ARM::LDRLIT_ga_pcrel_ldr || 1830 Opcode == ARM::tLDRLIT_ga_pcrel || 1831 Opcode == ARM::MOV_ga_pcrel || 1832 Opcode == ARM::MOV_ga_pcrel_ldr || 1833 Opcode == ARM::t2MOV_ga_pcrel) { 1834 if (MI1.getOpcode() != Opcode) 1835 return false; 1836 if (MI0.getNumOperands() != MI1.getNumOperands()) 1837 return false; 1838 1839 const MachineOperand &MO0 = MI0.getOperand(1); 1840 const MachineOperand &MO1 = MI1.getOperand(1); 1841 if (MO0.getOffset() != MO1.getOffset()) 1842 return false; 1843 1844 if (Opcode == ARM::LDRLIT_ga_pcrel || 1845 Opcode == ARM::LDRLIT_ga_pcrel_ldr || 1846 Opcode == ARM::tLDRLIT_ga_pcrel || 1847 Opcode == ARM::MOV_ga_pcrel || 1848 Opcode == ARM::MOV_ga_pcrel_ldr || 1849 Opcode == ARM::t2MOV_ga_pcrel) 1850 // Ignore the PC labels. 1851 return MO0.getGlobal() == MO1.getGlobal(); 1852 1853 const MachineFunction *MF = MI0.getParent()->getParent(); 1854 const MachineConstantPool *MCP = MF->getConstantPool(); 1855 int CPI0 = MO0.getIndex(); 1856 int CPI1 = MO1.getIndex(); 1857 const MachineConstantPoolEntry &MCPE0 = MCP->getConstants()[CPI0]; 1858 const MachineConstantPoolEntry &MCPE1 = MCP->getConstants()[CPI1]; 1859 bool isARMCP0 = MCPE0.isMachineConstantPoolEntry(); 1860 bool isARMCP1 = MCPE1.isMachineConstantPoolEntry(); 1861 if (isARMCP0 && isARMCP1) { 1862 ARMConstantPoolValue *ACPV0 = 1863 static_cast<ARMConstantPoolValue*>(MCPE0.Val.MachineCPVal); 1864 ARMConstantPoolValue *ACPV1 = 1865 static_cast<ARMConstantPoolValue*>(MCPE1.Val.MachineCPVal); 1866 return ACPV0->hasSameValue(ACPV1); 1867 } else if (!isARMCP0 && !isARMCP1) { 1868 return MCPE0.Val.ConstVal == MCPE1.Val.ConstVal; 1869 } 1870 return false; 1871 } else if (Opcode == ARM::PICLDR) { 1872 if (MI1.getOpcode() != Opcode) 1873 return false; 1874 if (MI0.getNumOperands() != MI1.getNumOperands()) 1875 return false; 1876 1877 Register Addr0 = MI0.getOperand(1).getReg(); 1878 Register Addr1 = MI1.getOperand(1).getReg(); 1879 if (Addr0 != Addr1) { 1880 if (!MRI || !Register::isVirtualRegister(Addr0) || 1881 !Register::isVirtualRegister(Addr1)) 1882 return false; 1883 1884 // This assumes SSA form. 1885 MachineInstr *Def0 = MRI->getVRegDef(Addr0); 1886 MachineInstr *Def1 = MRI->getVRegDef(Addr1); 1887 // Check if the loaded value, e.g. a constantpool of a global address, are 1888 // the same. 1889 if (!produceSameValue(*Def0, *Def1, MRI)) 1890 return false; 1891 } 1892 1893 for (unsigned i = 3, e = MI0.getNumOperands(); i != e; ++i) { 1894 // %12 = PICLDR %11, 0, 14, %noreg 1895 const MachineOperand &MO0 = MI0.getOperand(i); 1896 const MachineOperand &MO1 = MI1.getOperand(i); 1897 if (!MO0.isIdenticalTo(MO1)) 1898 return false; 1899 } 1900 return true; 1901 } 1902 1903 return MI0.isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs); 1904 } 1905 1906 /// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler to 1907 /// determine if two loads are loading from the same base address. It should 1908 /// only return true if the base pointers are the same and the only differences 1909 /// between the two addresses is the offset. It also returns the offsets by 1910 /// reference. 1911 /// 1912 /// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched 1913 /// is permanently disabled. 1914 bool ARMBaseInstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, 1915 int64_t &Offset1, 1916 int64_t &Offset2) const { 1917 // Don't worry about Thumb: just ARM and Thumb2. 1918 if (Subtarget.isThumb1Only()) return false; 1919 1920 if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode()) 1921 return false; 1922 1923 switch (Load1->getMachineOpcode()) { 1924 default: 1925 return false; 1926 case ARM::LDRi12: 1927 case ARM::LDRBi12: 1928 case ARM::LDRD: 1929 case ARM::LDRH: 1930 case ARM::LDRSB: 1931 case ARM::LDRSH: 1932 case ARM::VLDRD: 1933 case ARM::VLDRS: 1934 case ARM::t2LDRi8: 1935 case ARM::t2LDRBi8: 1936 case ARM::t2LDRDi8: 1937 case ARM::t2LDRSHi8: 1938 case ARM::t2LDRi12: 1939 case ARM::t2LDRBi12: 1940 case ARM::t2LDRSHi12: 1941 break; 1942 } 1943 1944 switch (Load2->getMachineOpcode()) { 1945 default: 1946 return false; 1947 case ARM::LDRi12: 1948 case ARM::LDRBi12: 1949 case ARM::LDRD: 1950 case ARM::LDRH: 1951 case ARM::LDRSB: 1952 case ARM::LDRSH: 1953 case ARM::VLDRD: 1954 case ARM::VLDRS: 1955 case ARM::t2LDRi8: 1956 case ARM::t2LDRBi8: 1957 case ARM::t2LDRSHi8: 1958 case ARM::t2LDRi12: 1959 case ARM::t2LDRBi12: 1960 case ARM::t2LDRSHi12: 1961 break; 1962 } 1963 1964 // Check if base addresses and chain operands match. 1965 if (Load1->getOperand(0) != Load2->getOperand(0) || 1966 Load1->getOperand(4) != Load2->getOperand(4)) 1967 return false; 1968 1969 // Index should be Reg0. 1970 if (Load1->getOperand(3) != Load2->getOperand(3)) 1971 return false; 1972 1973 // Determine the offsets. 1974 if (isa<ConstantSDNode>(Load1->getOperand(1)) && 1975 isa<ConstantSDNode>(Load2->getOperand(1))) { 1976 Offset1 = cast<ConstantSDNode>(Load1->getOperand(1))->getSExtValue(); 1977 Offset2 = cast<ConstantSDNode>(Load2->getOperand(1))->getSExtValue(); 1978 return true; 1979 } 1980 1981 return false; 1982 } 1983 1984 /// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to 1985 /// determine (in conjunction with areLoadsFromSameBasePtr) if two loads should 1986 /// be scheduled togther. On some targets if two loads are loading from 1987 /// addresses in the same cache line, it's better if they are scheduled 1988 /// together. This function takes two integers that represent the load offsets 1989 /// from the common base address. It returns true if it decides it's desirable 1990 /// to schedule the two loads together. "NumLoads" is the number of loads that 1991 /// have already been scheduled after Load1. 1992 /// 1993 /// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched 1994 /// is permanently disabled. 1995 bool ARMBaseInstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, 1996 int64_t Offset1, int64_t Offset2, 1997 unsigned NumLoads) const { 1998 // Don't worry about Thumb: just ARM and Thumb2. 1999 if (Subtarget.isThumb1Only()) return false; 2000 2001 assert(Offset2 > Offset1); 2002 2003 if ((Offset2 - Offset1) / 8 > 64) 2004 return false; 2005 2006 // Check if the machine opcodes are different. If they are different 2007 // then we consider them to not be of the same base address, 2008 // EXCEPT in the case of Thumb2 byte loads where one is LDRBi8 and the other LDRBi12. 2009 // In this case, they are considered to be the same because they are different 2010 // encoding forms of the same basic instruction. 2011 if ((Load1->getMachineOpcode() != Load2->getMachineOpcode()) && 2012 !((Load1->getMachineOpcode() == ARM::t2LDRBi8 && 2013 Load2->getMachineOpcode() == ARM::t2LDRBi12) || 2014 (Load1->getMachineOpcode() == ARM::t2LDRBi12 && 2015 Load2->getMachineOpcode() == ARM::t2LDRBi8))) 2016 return false; // FIXME: overly conservative? 2017 2018 // Four loads in a row should be sufficient. 2019 if (NumLoads >= 3) 2020 return false; 2021 2022 return true; 2023 } 2024 2025 bool ARMBaseInstrInfo::isSchedulingBoundary(const MachineInstr &MI, 2026 const MachineBasicBlock *MBB, 2027 const MachineFunction &MF) const { 2028 // Debug info is never a scheduling boundary. It's necessary to be explicit 2029 // due to the special treatment of IT instructions below, otherwise a 2030 // dbg_value followed by an IT will result in the IT instruction being 2031 // considered a scheduling hazard, which is wrong. It should be the actual 2032 // instruction preceding the dbg_value instruction(s), just like it is 2033 // when debug info is not present. 2034 if (MI.isDebugInstr()) 2035 return false; 2036 2037 // Terminators and labels can't be scheduled around. 2038 if (MI.isTerminator() || MI.isPosition()) 2039 return true; 2040 2041 // INLINEASM_BR can jump to another block 2042 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR) 2043 return true; 2044 2045 // Treat the start of the IT block as a scheduling boundary, but schedule 2046 // t2IT along with all instructions following it. 2047 // FIXME: This is a big hammer. But the alternative is to add all potential 2048 // true and anti dependencies to IT block instructions as implicit operands 2049 // to the t2IT instruction. The added compile time and complexity does not 2050 // seem worth it. 2051 MachineBasicBlock::const_iterator I = MI; 2052 // Make sure to skip any debug instructions 2053 while (++I != MBB->end() && I->isDebugInstr()) 2054 ; 2055 if (I != MBB->end() && I->getOpcode() == ARM::t2IT) 2056 return true; 2057 2058 // Don't attempt to schedule around any instruction that defines 2059 // a stack-oriented pointer, as it's unlikely to be profitable. This 2060 // saves compile time, because it doesn't require every single 2061 // stack slot reference to depend on the instruction that does the 2062 // modification. 2063 // Calls don't actually change the stack pointer, even if they have imp-defs. 2064 // No ARM calling conventions change the stack pointer. (X86 calling 2065 // conventions sometimes do). 2066 if (!MI.isCall() && MI.definesRegister(ARM::SP)) 2067 return true; 2068 2069 return false; 2070 } 2071 2072 bool ARMBaseInstrInfo:: 2073 isProfitableToIfCvt(MachineBasicBlock &MBB, 2074 unsigned NumCycles, unsigned ExtraPredCycles, 2075 BranchProbability Probability) const { 2076 if (!NumCycles) 2077 return false; 2078 2079 // If we are optimizing for size, see if the branch in the predecessor can be 2080 // lowered to cbn?z by the constant island lowering pass, and return false if 2081 // so. This results in a shorter instruction sequence. 2082 if (MBB.getParent()->getFunction().hasOptSize()) { 2083 MachineBasicBlock *Pred = *MBB.pred_begin(); 2084 if (!Pred->empty()) { 2085 MachineInstr *LastMI = &*Pred->rbegin(); 2086 if (LastMI->getOpcode() == ARM::t2Bcc) { 2087 const TargetRegisterInfo *TRI = &getRegisterInfo(); 2088 MachineInstr *CmpMI = findCMPToFoldIntoCBZ(LastMI, TRI); 2089 if (CmpMI) 2090 return false; 2091 } 2092 } 2093 } 2094 return isProfitableToIfCvt(MBB, NumCycles, ExtraPredCycles, 2095 MBB, 0, 0, Probability); 2096 } 2097 2098 bool ARMBaseInstrInfo:: 2099 isProfitableToIfCvt(MachineBasicBlock &TBB, 2100 unsigned TCycles, unsigned TExtra, 2101 MachineBasicBlock &FBB, 2102 unsigned FCycles, unsigned FExtra, 2103 BranchProbability Probability) const { 2104 if (!TCycles) 2105 return false; 2106 2107 // In thumb code we often end up trading one branch for a IT block, and 2108 // if we are cloning the instruction can increase code size. Prevent 2109 // blocks with multiple predecesors from being ifcvted to prevent this 2110 // cloning. 2111 if (Subtarget.isThumb2() && TBB.getParent()->getFunction().hasMinSize()) { 2112 if (TBB.pred_size() != 1 || FBB.pred_size() != 1) 2113 return false; 2114 } 2115 2116 // Attempt to estimate the relative costs of predication versus branching. 2117 // Here we scale up each component of UnpredCost to avoid precision issue when 2118 // scaling TCycles/FCycles by Probability. 2119 const unsigned ScalingUpFactor = 1024; 2120 2121 unsigned PredCost = (TCycles + FCycles + TExtra + FExtra) * ScalingUpFactor; 2122 unsigned UnpredCost; 2123 if (!Subtarget.hasBranchPredictor()) { 2124 // When we don't have a branch predictor it's always cheaper to not take a 2125 // branch than take it, so we have to take that into account. 2126 unsigned NotTakenBranchCost = 1; 2127 unsigned TakenBranchCost = Subtarget.getMispredictionPenalty(); 2128 unsigned TUnpredCycles, FUnpredCycles; 2129 if (!FCycles) { 2130 // Triangle: TBB is the fallthrough 2131 TUnpredCycles = TCycles + NotTakenBranchCost; 2132 FUnpredCycles = TakenBranchCost; 2133 } else { 2134 // Diamond: TBB is the block that is branched to, FBB is the fallthrough 2135 TUnpredCycles = TCycles + TakenBranchCost; 2136 FUnpredCycles = FCycles + NotTakenBranchCost; 2137 // The branch at the end of FBB will disappear when it's predicated, so 2138 // discount it from PredCost. 2139 PredCost -= 1 * ScalingUpFactor; 2140 } 2141 // The total cost is the cost of each path scaled by their probabilites 2142 unsigned TUnpredCost = Probability.scale(TUnpredCycles * ScalingUpFactor); 2143 unsigned FUnpredCost = Probability.getCompl().scale(FUnpredCycles * ScalingUpFactor); 2144 UnpredCost = TUnpredCost + FUnpredCost; 2145 // When predicating assume that the first IT can be folded away but later 2146 // ones cost one cycle each 2147 if (Subtarget.isThumb2() && TCycles + FCycles > 4) { 2148 PredCost += ((TCycles + FCycles - 4) / 4) * ScalingUpFactor; 2149 } 2150 } else { 2151 unsigned TUnpredCost = Probability.scale(TCycles * ScalingUpFactor); 2152 unsigned FUnpredCost = 2153 Probability.getCompl().scale(FCycles * ScalingUpFactor); 2154 UnpredCost = TUnpredCost + FUnpredCost; 2155 UnpredCost += 1 * ScalingUpFactor; // The branch itself 2156 UnpredCost += Subtarget.getMispredictionPenalty() * ScalingUpFactor / 10; 2157 } 2158 2159 return PredCost <= UnpredCost; 2160 } 2161 2162 unsigned 2163 ARMBaseInstrInfo::extraSizeToPredicateInstructions(const MachineFunction &MF, 2164 unsigned NumInsts) const { 2165 // Thumb2 needs a 2-byte IT instruction to predicate up to 4 instructions. 2166 // ARM has a condition code field in every predicable instruction, using it 2167 // doesn't change code size. 2168 if (!Subtarget.isThumb2()) 2169 return 0; 2170 2171 // It's possible that the size of the IT is restricted to a single block. 2172 unsigned MaxInsts = Subtarget.restrictIT() ? 1 : 4; 2173 return divideCeil(NumInsts, MaxInsts) * 2; 2174 } 2175 2176 unsigned 2177 ARMBaseInstrInfo::predictBranchSizeForIfCvt(MachineInstr &MI) const { 2178 // If this branch is likely to be folded into the comparison to form a 2179 // CB(N)Z, then removing it won't reduce code size at all, because that will 2180 // just replace the CB(N)Z with a CMP. 2181 if (MI.getOpcode() == ARM::t2Bcc && 2182 findCMPToFoldIntoCBZ(&MI, &getRegisterInfo())) 2183 return 0; 2184 2185 unsigned Size = getInstSizeInBytes(MI); 2186 2187 // For Thumb2, all branches are 32-bit instructions during the if conversion 2188 // pass, but may be replaced with 16-bit instructions during size reduction. 2189 // Since the branches considered by if conversion tend to be forward branches 2190 // over small basic blocks, they are very likely to be in range for the 2191 // narrow instructions, so we assume the final code size will be half what it 2192 // currently is. 2193 if (Subtarget.isThumb2()) 2194 Size /= 2; 2195 2196 return Size; 2197 } 2198 2199 bool 2200 ARMBaseInstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB, 2201 MachineBasicBlock &FMBB) const { 2202 // Reduce false anti-dependencies to let the target's out-of-order execution 2203 // engine do its thing. 2204 return Subtarget.isProfitableToUnpredicate(); 2205 } 2206 2207 /// getInstrPredicate - If instruction is predicated, returns its predicate 2208 /// condition, otherwise returns AL. It also returns the condition code 2209 /// register by reference. 2210 ARMCC::CondCodes llvm::getInstrPredicate(const MachineInstr &MI, 2211 Register &PredReg) { 2212 int PIdx = MI.findFirstPredOperandIdx(); 2213 if (PIdx == -1) { 2214 PredReg = 0; 2215 return ARMCC::AL; 2216 } 2217 2218 PredReg = MI.getOperand(PIdx+1).getReg(); 2219 return (ARMCC::CondCodes)MI.getOperand(PIdx).getImm(); 2220 } 2221 2222 unsigned llvm::getMatchingCondBranchOpcode(unsigned Opc) { 2223 if (Opc == ARM::B) 2224 return ARM::Bcc; 2225 if (Opc == ARM::tB) 2226 return ARM::tBcc; 2227 if (Opc == ARM::t2B) 2228 return ARM::t2Bcc; 2229 2230 llvm_unreachable("Unknown unconditional branch opcode!"); 2231 } 2232 2233 MachineInstr *ARMBaseInstrInfo::commuteInstructionImpl(MachineInstr &MI, 2234 bool NewMI, 2235 unsigned OpIdx1, 2236 unsigned OpIdx2) const { 2237 switch (MI.getOpcode()) { 2238 case ARM::MOVCCr: 2239 case ARM::t2MOVCCr: { 2240 // MOVCC can be commuted by inverting the condition. 2241 Register PredReg; 2242 ARMCC::CondCodes CC = getInstrPredicate(MI, PredReg); 2243 // MOVCC AL can't be inverted. Shouldn't happen. 2244 if (CC == ARMCC::AL || PredReg != ARM::CPSR) 2245 return nullptr; 2246 MachineInstr *CommutedMI = 2247 TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 2248 if (!CommutedMI) 2249 return nullptr; 2250 // After swapping the MOVCC operands, also invert the condition. 2251 CommutedMI->getOperand(CommutedMI->findFirstPredOperandIdx()) 2252 .setImm(ARMCC::getOppositeCondition(CC)); 2253 return CommutedMI; 2254 } 2255 } 2256 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 2257 } 2258 2259 /// Identify instructions that can be folded into a MOVCC instruction, and 2260 /// return the defining instruction. 2261 MachineInstr * 2262 ARMBaseInstrInfo::canFoldIntoMOVCC(Register Reg, const MachineRegisterInfo &MRI, 2263 const TargetInstrInfo *TII) const { 2264 if (!Reg.isVirtual()) 2265 return nullptr; 2266 if (!MRI.hasOneNonDBGUse(Reg)) 2267 return nullptr; 2268 MachineInstr *MI = MRI.getVRegDef(Reg); 2269 if (!MI) 2270 return nullptr; 2271 // Check if MI can be predicated and folded into the MOVCC. 2272 if (!isPredicable(*MI)) 2273 return nullptr; 2274 // Check if MI has any non-dead defs or physreg uses. This also detects 2275 // predicated instructions which will be reading CPSR. 2276 for (unsigned i = 1, e = MI->getNumOperands(); i != e; ++i) { 2277 const MachineOperand &MO = MI->getOperand(i); 2278 // Reject frame index operands, PEI can't handle the predicated pseudos. 2279 if (MO.isFI() || MO.isCPI() || MO.isJTI()) 2280 return nullptr; 2281 if (!MO.isReg()) 2282 continue; 2283 // MI can't have any tied operands, that would conflict with predication. 2284 if (MO.isTied()) 2285 return nullptr; 2286 if (Register::isPhysicalRegister(MO.getReg())) 2287 return nullptr; 2288 if (MO.isDef() && !MO.isDead()) 2289 return nullptr; 2290 } 2291 bool DontMoveAcrossStores = true; 2292 if (!MI->isSafeToMove(/* AliasAnalysis = */ nullptr, DontMoveAcrossStores)) 2293 return nullptr; 2294 return MI; 2295 } 2296 2297 bool ARMBaseInstrInfo::analyzeSelect(const MachineInstr &MI, 2298 SmallVectorImpl<MachineOperand> &Cond, 2299 unsigned &TrueOp, unsigned &FalseOp, 2300 bool &Optimizable) const { 2301 assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) && 2302 "Unknown select instruction"); 2303 // MOVCC operands: 2304 // 0: Def. 2305 // 1: True use. 2306 // 2: False use. 2307 // 3: Condition code. 2308 // 4: CPSR use. 2309 TrueOp = 1; 2310 FalseOp = 2; 2311 Cond.push_back(MI.getOperand(3)); 2312 Cond.push_back(MI.getOperand(4)); 2313 // We can always fold a def. 2314 Optimizable = true; 2315 return false; 2316 } 2317 2318 MachineInstr * 2319 ARMBaseInstrInfo::optimizeSelect(MachineInstr &MI, 2320 SmallPtrSetImpl<MachineInstr *> &SeenMIs, 2321 bool PreferFalse) const { 2322 assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) && 2323 "Unknown select instruction"); 2324 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 2325 MachineInstr *DefMI = canFoldIntoMOVCC(MI.getOperand(2).getReg(), MRI, this); 2326 bool Invert = !DefMI; 2327 if (!DefMI) 2328 DefMI = canFoldIntoMOVCC(MI.getOperand(1).getReg(), MRI, this); 2329 if (!DefMI) 2330 return nullptr; 2331 2332 // Find new register class to use. 2333 MachineOperand FalseReg = MI.getOperand(Invert ? 2 : 1); 2334 Register DestReg = MI.getOperand(0).getReg(); 2335 const TargetRegisterClass *PreviousClass = MRI.getRegClass(FalseReg.getReg()); 2336 if (!MRI.constrainRegClass(DestReg, PreviousClass)) 2337 return nullptr; 2338 2339 // Create a new predicated version of DefMI. 2340 // Rfalse is the first use. 2341 MachineInstrBuilder NewMI = 2342 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), DefMI->getDesc(), DestReg); 2343 2344 // Copy all the DefMI operands, excluding its (null) predicate. 2345 const MCInstrDesc &DefDesc = DefMI->getDesc(); 2346 for (unsigned i = 1, e = DefDesc.getNumOperands(); 2347 i != e && !DefDesc.OpInfo[i].isPredicate(); ++i) 2348 NewMI.add(DefMI->getOperand(i)); 2349 2350 unsigned CondCode = MI.getOperand(3).getImm(); 2351 if (Invert) 2352 NewMI.addImm(ARMCC::getOppositeCondition(ARMCC::CondCodes(CondCode))); 2353 else 2354 NewMI.addImm(CondCode); 2355 NewMI.add(MI.getOperand(4)); 2356 2357 // DefMI is not the -S version that sets CPSR, so add an optional %noreg. 2358 if (NewMI->hasOptionalDef()) 2359 NewMI.add(condCodeOp()); 2360 2361 // The output register value when the predicate is false is an implicit 2362 // register operand tied to the first def. 2363 // The tie makes the register allocator ensure the FalseReg is allocated the 2364 // same register as operand 0. 2365 FalseReg.setImplicit(); 2366 NewMI.add(FalseReg); 2367 NewMI->tieOperands(0, NewMI->getNumOperands() - 1); 2368 2369 // Update SeenMIs set: register newly created MI and erase removed DefMI. 2370 SeenMIs.insert(NewMI); 2371 SeenMIs.erase(DefMI); 2372 2373 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on 2374 // DefMI would be invalid when tranferred inside the loop. Checking for a 2375 // loop is expensive, but at least remove kill flags if they are in different 2376 // BBs. 2377 if (DefMI->getParent() != MI.getParent()) 2378 NewMI->clearKillInfo(); 2379 2380 // The caller will erase MI, but not DefMI. 2381 DefMI->eraseFromParent(); 2382 return NewMI; 2383 } 2384 2385 /// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether the 2386 /// instruction is encoded with an 'S' bit is determined by the optional CPSR 2387 /// def operand. 2388 /// 2389 /// This will go away once we can teach tblgen how to set the optional CPSR def 2390 /// operand itself. 2391 struct AddSubFlagsOpcodePair { 2392 uint16_t PseudoOpc; 2393 uint16_t MachineOpc; 2394 }; 2395 2396 static const AddSubFlagsOpcodePair AddSubFlagsOpcodeMap[] = { 2397 {ARM::ADDSri, ARM::ADDri}, 2398 {ARM::ADDSrr, ARM::ADDrr}, 2399 {ARM::ADDSrsi, ARM::ADDrsi}, 2400 {ARM::ADDSrsr, ARM::ADDrsr}, 2401 2402 {ARM::SUBSri, ARM::SUBri}, 2403 {ARM::SUBSrr, ARM::SUBrr}, 2404 {ARM::SUBSrsi, ARM::SUBrsi}, 2405 {ARM::SUBSrsr, ARM::SUBrsr}, 2406 2407 {ARM::RSBSri, ARM::RSBri}, 2408 {ARM::RSBSrsi, ARM::RSBrsi}, 2409 {ARM::RSBSrsr, ARM::RSBrsr}, 2410 2411 {ARM::tADDSi3, ARM::tADDi3}, 2412 {ARM::tADDSi8, ARM::tADDi8}, 2413 {ARM::tADDSrr, ARM::tADDrr}, 2414 {ARM::tADCS, ARM::tADC}, 2415 2416 {ARM::tSUBSi3, ARM::tSUBi3}, 2417 {ARM::tSUBSi8, ARM::tSUBi8}, 2418 {ARM::tSUBSrr, ARM::tSUBrr}, 2419 {ARM::tSBCS, ARM::tSBC}, 2420 {ARM::tRSBS, ARM::tRSB}, 2421 {ARM::tLSLSri, ARM::tLSLri}, 2422 2423 {ARM::t2ADDSri, ARM::t2ADDri}, 2424 {ARM::t2ADDSrr, ARM::t2ADDrr}, 2425 {ARM::t2ADDSrs, ARM::t2ADDrs}, 2426 2427 {ARM::t2SUBSri, ARM::t2SUBri}, 2428 {ARM::t2SUBSrr, ARM::t2SUBrr}, 2429 {ARM::t2SUBSrs, ARM::t2SUBrs}, 2430 2431 {ARM::t2RSBSri, ARM::t2RSBri}, 2432 {ARM::t2RSBSrs, ARM::t2RSBrs}, 2433 }; 2434 2435 unsigned llvm::convertAddSubFlagsOpcode(unsigned OldOpc) { 2436 for (unsigned i = 0, e = array_lengthof(AddSubFlagsOpcodeMap); i != e; ++i) 2437 if (OldOpc == AddSubFlagsOpcodeMap[i].PseudoOpc) 2438 return AddSubFlagsOpcodeMap[i].MachineOpc; 2439 return 0; 2440 } 2441 2442 void llvm::emitARMRegPlusImmediate(MachineBasicBlock &MBB, 2443 MachineBasicBlock::iterator &MBBI, 2444 const DebugLoc &dl, Register DestReg, 2445 Register BaseReg, int NumBytes, 2446 ARMCC::CondCodes Pred, Register PredReg, 2447 const ARMBaseInstrInfo &TII, 2448 unsigned MIFlags) { 2449 if (NumBytes == 0 && DestReg != BaseReg) { 2450 BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), DestReg) 2451 .addReg(BaseReg, RegState::Kill) 2452 .add(predOps(Pred, PredReg)) 2453 .add(condCodeOp()) 2454 .setMIFlags(MIFlags); 2455 return; 2456 } 2457 2458 bool isSub = NumBytes < 0; 2459 if (isSub) NumBytes = -NumBytes; 2460 2461 while (NumBytes) { 2462 unsigned RotAmt = ARM_AM::getSOImmValRotate(NumBytes); 2463 unsigned ThisVal = NumBytes & ARM_AM::rotr32(0xFF, RotAmt); 2464 assert(ThisVal && "Didn't extract field correctly"); 2465 2466 // We will handle these bits from offset, clear them. 2467 NumBytes &= ~ThisVal; 2468 2469 assert(ARM_AM::getSOImmVal(ThisVal) != -1 && "Bit extraction didn't work?"); 2470 2471 // Build the new ADD / SUB. 2472 unsigned Opc = isSub ? ARM::SUBri : ARM::ADDri; 2473 BuildMI(MBB, MBBI, dl, TII.get(Opc), DestReg) 2474 .addReg(BaseReg, RegState::Kill) 2475 .addImm(ThisVal) 2476 .add(predOps(Pred, PredReg)) 2477 .add(condCodeOp()) 2478 .setMIFlags(MIFlags); 2479 BaseReg = DestReg; 2480 } 2481 } 2482 2483 bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget, 2484 MachineFunction &MF, MachineInstr *MI, 2485 unsigned NumBytes) { 2486 // This optimisation potentially adds lots of load and store 2487 // micro-operations, it's only really a great benefit to code-size. 2488 if (!Subtarget.hasMinSize()) 2489 return false; 2490 2491 // If only one register is pushed/popped, LLVM can use an LDR/STR 2492 // instead. We can't modify those so make sure we're dealing with an 2493 // instruction we understand. 2494 bool IsPop = isPopOpcode(MI->getOpcode()); 2495 bool IsPush = isPushOpcode(MI->getOpcode()); 2496 if (!IsPush && !IsPop) 2497 return false; 2498 2499 bool IsVFPPushPop = MI->getOpcode() == ARM::VSTMDDB_UPD || 2500 MI->getOpcode() == ARM::VLDMDIA_UPD; 2501 bool IsT1PushPop = MI->getOpcode() == ARM::tPUSH || 2502 MI->getOpcode() == ARM::tPOP || 2503 MI->getOpcode() == ARM::tPOP_RET; 2504 2505 assert((IsT1PushPop || (MI->getOperand(0).getReg() == ARM::SP && 2506 MI->getOperand(1).getReg() == ARM::SP)) && 2507 "trying to fold sp update into non-sp-updating push/pop"); 2508 2509 // The VFP push & pop act on D-registers, so we can only fold an adjustment 2510 // by a multiple of 8 bytes in correctly. Similarly rN is 4-bytes. Don't try 2511 // if this is violated. 2512 if (NumBytes % (IsVFPPushPop ? 8 : 4) != 0) 2513 return false; 2514 2515 // ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+ 2516 // pred) so the list starts at 4. Thumb1 starts after the predicate. 2517 int RegListIdx = IsT1PushPop ? 2 : 4; 2518 2519 // Calculate the space we'll need in terms of registers. 2520 unsigned RegsNeeded; 2521 const TargetRegisterClass *RegClass; 2522 if (IsVFPPushPop) { 2523 RegsNeeded = NumBytes / 8; 2524 RegClass = &ARM::DPRRegClass; 2525 } else { 2526 RegsNeeded = NumBytes / 4; 2527 RegClass = &ARM::GPRRegClass; 2528 } 2529 2530 // We're going to have to strip all list operands off before 2531 // re-adding them since the order matters, so save the existing ones 2532 // for later. 2533 SmallVector<MachineOperand, 4> RegList; 2534 2535 // We're also going to need the first register transferred by this 2536 // instruction, which won't necessarily be the first register in the list. 2537 unsigned FirstRegEnc = -1; 2538 2539 const TargetRegisterInfo *TRI = MF.getRegInfo().getTargetRegisterInfo(); 2540 for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i) { 2541 MachineOperand &MO = MI->getOperand(i); 2542 RegList.push_back(MO); 2543 2544 if (MO.isReg() && !MO.isImplicit() && 2545 TRI->getEncodingValue(MO.getReg()) < FirstRegEnc) 2546 FirstRegEnc = TRI->getEncodingValue(MO.getReg()); 2547 } 2548 2549 const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF); 2550 2551 // Now try to find enough space in the reglist to allocate NumBytes. 2552 for (int CurRegEnc = FirstRegEnc - 1; CurRegEnc >= 0 && RegsNeeded; 2553 --CurRegEnc) { 2554 unsigned CurReg = RegClass->getRegister(CurRegEnc); 2555 if (IsT1PushPop && CurRegEnc > TRI->getEncodingValue(ARM::R7)) 2556 continue; 2557 if (!IsPop) { 2558 // Pushing any register is completely harmless, mark the register involved 2559 // as undef since we don't care about its value and must not restore it 2560 // during stack unwinding. 2561 RegList.push_back(MachineOperand::CreateReg(CurReg, false, false, 2562 false, false, true)); 2563 --RegsNeeded; 2564 continue; 2565 } 2566 2567 // However, we can only pop an extra register if it's not live. For 2568 // registers live within the function we might clobber a return value 2569 // register; the other way a register can be live here is if it's 2570 // callee-saved. 2571 if (isCalleeSavedRegister(CurReg, CSRegs) || 2572 MI->getParent()->computeRegisterLiveness(TRI, CurReg, MI) != 2573 MachineBasicBlock::LQR_Dead) { 2574 // VFP pops don't allow holes in the register list, so any skip is fatal 2575 // for our transformation. GPR pops do, so we should just keep looking. 2576 if (IsVFPPushPop) 2577 return false; 2578 else 2579 continue; 2580 } 2581 2582 // Mark the unimportant registers as <def,dead> in the POP. 2583 RegList.push_back(MachineOperand::CreateReg(CurReg, true, false, false, 2584 true)); 2585 --RegsNeeded; 2586 } 2587 2588 if (RegsNeeded > 0) 2589 return false; 2590 2591 // Finally we know we can profitably perform the optimisation so go 2592 // ahead: strip all existing registers off and add them back again 2593 // in the right order. 2594 for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i) 2595 MI->RemoveOperand(i); 2596 2597 // Add the complete list back in. 2598 MachineInstrBuilder MIB(MF, &*MI); 2599 for (int i = RegList.size() - 1; i >= 0; --i) 2600 MIB.add(RegList[i]); 2601 2602 return true; 2603 } 2604 2605 bool llvm::rewriteARMFrameIndex(MachineInstr &MI, unsigned FrameRegIdx, 2606 Register FrameReg, int &Offset, 2607 const ARMBaseInstrInfo &TII) { 2608 unsigned Opcode = MI.getOpcode(); 2609 const MCInstrDesc &Desc = MI.getDesc(); 2610 unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask); 2611 bool isSub = false; 2612 2613 // Memory operands in inline assembly always use AddrMode2. 2614 if (Opcode == ARM::INLINEASM || Opcode == ARM::INLINEASM_BR) 2615 AddrMode = ARMII::AddrMode2; 2616 2617 if (Opcode == ARM::ADDri) { 2618 Offset += MI.getOperand(FrameRegIdx+1).getImm(); 2619 if (Offset == 0) { 2620 // Turn it into a move. 2621 MI.setDesc(TII.get(ARM::MOVr)); 2622 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false); 2623 MI.RemoveOperand(FrameRegIdx+1); 2624 Offset = 0; 2625 return true; 2626 } else if (Offset < 0) { 2627 Offset = -Offset; 2628 isSub = true; 2629 MI.setDesc(TII.get(ARM::SUBri)); 2630 } 2631 2632 // Common case: small offset, fits into instruction. 2633 if (ARM_AM::getSOImmVal(Offset) != -1) { 2634 // Replace the FrameIndex with sp / fp 2635 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false); 2636 MI.getOperand(FrameRegIdx+1).ChangeToImmediate(Offset); 2637 Offset = 0; 2638 return true; 2639 } 2640 2641 // Otherwise, pull as much of the immedidate into this ADDri/SUBri 2642 // as possible. 2643 unsigned RotAmt = ARM_AM::getSOImmValRotate(Offset); 2644 unsigned ThisImmVal = Offset & ARM_AM::rotr32(0xFF, RotAmt); 2645 2646 // We will handle these bits from offset, clear them. 2647 Offset &= ~ThisImmVal; 2648 2649 // Get the properly encoded SOImmVal field. 2650 assert(ARM_AM::getSOImmVal(ThisImmVal) != -1 && 2651 "Bit extraction didn't work?"); 2652 MI.getOperand(FrameRegIdx+1).ChangeToImmediate(ThisImmVal); 2653 } else { 2654 unsigned ImmIdx = 0; 2655 int InstrOffs = 0; 2656 unsigned NumBits = 0; 2657 unsigned Scale = 1; 2658 switch (AddrMode) { 2659 case ARMII::AddrMode_i12: 2660 ImmIdx = FrameRegIdx + 1; 2661 InstrOffs = MI.getOperand(ImmIdx).getImm(); 2662 NumBits = 12; 2663 break; 2664 case ARMII::AddrMode2: 2665 ImmIdx = FrameRegIdx+2; 2666 InstrOffs = ARM_AM::getAM2Offset(MI.getOperand(ImmIdx).getImm()); 2667 if (ARM_AM::getAM2Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub) 2668 InstrOffs *= -1; 2669 NumBits = 12; 2670 break; 2671 case ARMII::AddrMode3: 2672 ImmIdx = FrameRegIdx+2; 2673 InstrOffs = ARM_AM::getAM3Offset(MI.getOperand(ImmIdx).getImm()); 2674 if (ARM_AM::getAM3Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub) 2675 InstrOffs *= -1; 2676 NumBits = 8; 2677 break; 2678 case ARMII::AddrMode4: 2679 case ARMII::AddrMode6: 2680 // Can't fold any offset even if it's zero. 2681 return false; 2682 case ARMII::AddrMode5: 2683 ImmIdx = FrameRegIdx+1; 2684 InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm()); 2685 if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub) 2686 InstrOffs *= -1; 2687 NumBits = 8; 2688 Scale = 4; 2689 break; 2690 case ARMII::AddrMode5FP16: 2691 ImmIdx = FrameRegIdx+1; 2692 InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm()); 2693 if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub) 2694 InstrOffs *= -1; 2695 NumBits = 8; 2696 Scale = 2; 2697 break; 2698 case ARMII::AddrModeT2_i7: 2699 case ARMII::AddrModeT2_i7s2: 2700 case ARMII::AddrModeT2_i7s4: 2701 ImmIdx = FrameRegIdx+1; 2702 InstrOffs = MI.getOperand(ImmIdx).getImm(); 2703 NumBits = 7; 2704 Scale = (AddrMode == ARMII::AddrModeT2_i7s2 ? 2 : 2705 AddrMode == ARMII::AddrModeT2_i7s4 ? 4 : 1); 2706 break; 2707 default: 2708 llvm_unreachable("Unsupported addressing mode!"); 2709 } 2710 2711 Offset += InstrOffs * Scale; 2712 assert((Offset & (Scale-1)) == 0 && "Can't encode this offset!"); 2713 if (Offset < 0) { 2714 Offset = -Offset; 2715 isSub = true; 2716 } 2717 2718 // Attempt to fold address comp. if opcode has offset bits 2719 if (NumBits > 0) { 2720 // Common case: small offset, fits into instruction. 2721 MachineOperand &ImmOp = MI.getOperand(ImmIdx); 2722 int ImmedOffset = Offset / Scale; 2723 unsigned Mask = (1 << NumBits) - 1; 2724 if ((unsigned)Offset <= Mask * Scale) { 2725 // Replace the FrameIndex with sp 2726 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false); 2727 // FIXME: When addrmode2 goes away, this will simplify (like the 2728 // T2 version), as the LDR.i12 versions don't need the encoding 2729 // tricks for the offset value. 2730 if (isSub) { 2731 if (AddrMode == ARMII::AddrMode_i12) 2732 ImmedOffset = -ImmedOffset; 2733 else 2734 ImmedOffset |= 1 << NumBits; 2735 } 2736 ImmOp.ChangeToImmediate(ImmedOffset); 2737 Offset = 0; 2738 return true; 2739 } 2740 2741 // Otherwise, it didn't fit. Pull in what we can to simplify the immed. 2742 ImmedOffset = ImmedOffset & Mask; 2743 if (isSub) { 2744 if (AddrMode == ARMII::AddrMode_i12) 2745 ImmedOffset = -ImmedOffset; 2746 else 2747 ImmedOffset |= 1 << NumBits; 2748 } 2749 ImmOp.ChangeToImmediate(ImmedOffset); 2750 Offset &= ~(Mask*Scale); 2751 } 2752 } 2753 2754 Offset = (isSub) ? -Offset : Offset; 2755 return Offset == 0; 2756 } 2757 2758 /// analyzeCompare - For a comparison instruction, return the source registers 2759 /// in SrcReg and SrcReg2 if having two register operands, and the value it 2760 /// compares against in CmpValue. Return true if the comparison instruction 2761 /// can be analyzed. 2762 bool ARMBaseInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 2763 Register &SrcReg2, int &CmpMask, 2764 int &CmpValue) const { 2765 switch (MI.getOpcode()) { 2766 default: break; 2767 case ARM::CMPri: 2768 case ARM::t2CMPri: 2769 case ARM::tCMPi8: 2770 SrcReg = MI.getOperand(0).getReg(); 2771 SrcReg2 = 0; 2772 CmpMask = ~0; 2773 CmpValue = MI.getOperand(1).getImm(); 2774 return true; 2775 case ARM::CMPrr: 2776 case ARM::t2CMPrr: 2777 case ARM::tCMPr: 2778 SrcReg = MI.getOperand(0).getReg(); 2779 SrcReg2 = MI.getOperand(1).getReg(); 2780 CmpMask = ~0; 2781 CmpValue = 0; 2782 return true; 2783 case ARM::TSTri: 2784 case ARM::t2TSTri: 2785 SrcReg = MI.getOperand(0).getReg(); 2786 SrcReg2 = 0; 2787 CmpMask = MI.getOperand(1).getImm(); 2788 CmpValue = 0; 2789 return true; 2790 } 2791 2792 return false; 2793 } 2794 2795 /// isSuitableForMask - Identify a suitable 'and' instruction that 2796 /// operates on the given source register and applies the same mask 2797 /// as a 'tst' instruction. Provide a limited look-through for copies. 2798 /// When successful, MI will hold the found instruction. 2799 static bool isSuitableForMask(MachineInstr *&MI, Register SrcReg, 2800 int CmpMask, bool CommonUse) { 2801 switch (MI->getOpcode()) { 2802 case ARM::ANDri: 2803 case ARM::t2ANDri: 2804 if (CmpMask != MI->getOperand(2).getImm()) 2805 return false; 2806 if (SrcReg == MI->getOperand(CommonUse ? 1 : 0).getReg()) 2807 return true; 2808 break; 2809 } 2810 2811 return false; 2812 } 2813 2814 /// getCmpToAddCondition - assume the flags are set by CMP(a,b), return 2815 /// the condition code if we modify the instructions such that flags are 2816 /// set by ADD(a,b,X). 2817 inline static ARMCC::CondCodes getCmpToAddCondition(ARMCC::CondCodes CC) { 2818 switch (CC) { 2819 default: return ARMCC::AL; 2820 case ARMCC::HS: return ARMCC::LO; 2821 case ARMCC::LO: return ARMCC::HS; 2822 case ARMCC::VS: return ARMCC::VS; 2823 case ARMCC::VC: return ARMCC::VC; 2824 } 2825 } 2826 2827 /// isRedundantFlagInstr - check whether the first instruction, whose only 2828 /// purpose is to update flags, can be made redundant. 2829 /// CMPrr can be made redundant by SUBrr if the operands are the same. 2830 /// CMPri can be made redundant by SUBri if the operands are the same. 2831 /// CMPrr(r0, r1) can be made redundant by ADDr[ri](r0, r1, X). 2832 /// This function can be extended later on. 2833 inline static bool isRedundantFlagInstr(const MachineInstr *CmpI, 2834 Register SrcReg, Register SrcReg2, 2835 int ImmValue, const MachineInstr *OI, 2836 bool &IsThumb1) { 2837 if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) && 2838 (OI->getOpcode() == ARM::SUBrr || OI->getOpcode() == ARM::t2SUBrr) && 2839 ((OI->getOperand(1).getReg() == SrcReg && 2840 OI->getOperand(2).getReg() == SrcReg2) || 2841 (OI->getOperand(1).getReg() == SrcReg2 && 2842 OI->getOperand(2).getReg() == SrcReg))) { 2843 IsThumb1 = false; 2844 return true; 2845 } 2846 2847 if (CmpI->getOpcode() == ARM::tCMPr && OI->getOpcode() == ARM::tSUBrr && 2848 ((OI->getOperand(2).getReg() == SrcReg && 2849 OI->getOperand(3).getReg() == SrcReg2) || 2850 (OI->getOperand(2).getReg() == SrcReg2 && 2851 OI->getOperand(3).getReg() == SrcReg))) { 2852 IsThumb1 = true; 2853 return true; 2854 } 2855 2856 if ((CmpI->getOpcode() == ARM::CMPri || CmpI->getOpcode() == ARM::t2CMPri) && 2857 (OI->getOpcode() == ARM::SUBri || OI->getOpcode() == ARM::t2SUBri) && 2858 OI->getOperand(1).getReg() == SrcReg && 2859 OI->getOperand(2).getImm() == ImmValue) { 2860 IsThumb1 = false; 2861 return true; 2862 } 2863 2864 if (CmpI->getOpcode() == ARM::tCMPi8 && 2865 (OI->getOpcode() == ARM::tSUBi8 || OI->getOpcode() == ARM::tSUBi3) && 2866 OI->getOperand(2).getReg() == SrcReg && 2867 OI->getOperand(3).getImm() == ImmValue) { 2868 IsThumb1 = true; 2869 return true; 2870 } 2871 2872 if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) && 2873 (OI->getOpcode() == ARM::ADDrr || OI->getOpcode() == ARM::t2ADDrr || 2874 OI->getOpcode() == ARM::ADDri || OI->getOpcode() == ARM::t2ADDri) && 2875 OI->getOperand(0).isReg() && OI->getOperand(1).isReg() && 2876 OI->getOperand(0).getReg() == SrcReg && 2877 OI->getOperand(1).getReg() == SrcReg2) { 2878 IsThumb1 = false; 2879 return true; 2880 } 2881 2882 if (CmpI->getOpcode() == ARM::tCMPr && 2883 (OI->getOpcode() == ARM::tADDi3 || OI->getOpcode() == ARM::tADDi8 || 2884 OI->getOpcode() == ARM::tADDrr) && 2885 OI->getOperand(0).getReg() == SrcReg && 2886 OI->getOperand(2).getReg() == SrcReg2) { 2887 IsThumb1 = true; 2888 return true; 2889 } 2890 2891 return false; 2892 } 2893 2894 static bool isOptimizeCompareCandidate(MachineInstr *MI, bool &IsThumb1) { 2895 switch (MI->getOpcode()) { 2896 default: return false; 2897 case ARM::tLSLri: 2898 case ARM::tLSRri: 2899 case ARM::tLSLrr: 2900 case ARM::tLSRrr: 2901 case ARM::tSUBrr: 2902 case ARM::tADDrr: 2903 case ARM::tADDi3: 2904 case ARM::tADDi8: 2905 case ARM::tSUBi3: 2906 case ARM::tSUBi8: 2907 case ARM::tMUL: 2908 case ARM::tADC: 2909 case ARM::tSBC: 2910 case ARM::tRSB: 2911 case ARM::tAND: 2912 case ARM::tORR: 2913 case ARM::tEOR: 2914 case ARM::tBIC: 2915 case ARM::tMVN: 2916 case ARM::tASRri: 2917 case ARM::tASRrr: 2918 case ARM::tROR: 2919 IsThumb1 = true; 2920 LLVM_FALLTHROUGH; 2921 case ARM::RSBrr: 2922 case ARM::RSBri: 2923 case ARM::RSCrr: 2924 case ARM::RSCri: 2925 case ARM::ADDrr: 2926 case ARM::ADDri: 2927 case ARM::ADCrr: 2928 case ARM::ADCri: 2929 case ARM::SUBrr: 2930 case ARM::SUBri: 2931 case ARM::SBCrr: 2932 case ARM::SBCri: 2933 case ARM::t2RSBri: 2934 case ARM::t2ADDrr: 2935 case ARM::t2ADDri: 2936 case ARM::t2ADCrr: 2937 case ARM::t2ADCri: 2938 case ARM::t2SUBrr: 2939 case ARM::t2SUBri: 2940 case ARM::t2SBCrr: 2941 case ARM::t2SBCri: 2942 case ARM::ANDrr: 2943 case ARM::ANDri: 2944 case ARM::t2ANDrr: 2945 case ARM::t2ANDri: 2946 case ARM::ORRrr: 2947 case ARM::ORRri: 2948 case ARM::t2ORRrr: 2949 case ARM::t2ORRri: 2950 case ARM::EORrr: 2951 case ARM::EORri: 2952 case ARM::t2EORrr: 2953 case ARM::t2EORri: 2954 case ARM::t2LSRri: 2955 case ARM::t2LSRrr: 2956 case ARM::t2LSLri: 2957 case ARM::t2LSLrr: 2958 return true; 2959 } 2960 } 2961 2962 /// optimizeCompareInstr - Convert the instruction supplying the argument to the 2963 /// comparison into one that sets the zero bit in the flags register; 2964 /// Remove a redundant Compare instruction if an earlier instruction can set the 2965 /// flags in the same way as Compare. 2966 /// E.g. SUBrr(r1,r2) and CMPrr(r1,r2). We also handle the case where two 2967 /// operands are swapped: SUBrr(r1,r2) and CMPrr(r2,r1), by updating the 2968 /// condition code of instructions which use the flags. 2969 bool ARMBaseInstrInfo::optimizeCompareInstr( 2970 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int CmpMask, 2971 int CmpValue, const MachineRegisterInfo *MRI) const { 2972 // Get the unique definition of SrcReg. 2973 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg); 2974 if (!MI) return false; 2975 2976 // Masked compares sometimes use the same register as the corresponding 'and'. 2977 if (CmpMask != ~0) { 2978 if (!isSuitableForMask(MI, SrcReg, CmpMask, false) || isPredicated(*MI)) { 2979 MI = nullptr; 2980 for (MachineRegisterInfo::use_instr_iterator 2981 UI = MRI->use_instr_begin(SrcReg), UE = MRI->use_instr_end(); 2982 UI != UE; ++UI) { 2983 if (UI->getParent() != CmpInstr.getParent()) 2984 continue; 2985 MachineInstr *PotentialAND = &*UI; 2986 if (!isSuitableForMask(PotentialAND, SrcReg, CmpMask, true) || 2987 isPredicated(*PotentialAND)) 2988 continue; 2989 MI = PotentialAND; 2990 break; 2991 } 2992 if (!MI) return false; 2993 } 2994 } 2995 2996 // Get ready to iterate backward from CmpInstr. 2997 MachineBasicBlock::iterator I = CmpInstr, E = MI, 2998 B = CmpInstr.getParent()->begin(); 2999 3000 // Early exit if CmpInstr is at the beginning of the BB. 3001 if (I == B) return false; 3002 3003 // There are two possible candidates which can be changed to set CPSR: 3004 // One is MI, the other is a SUB or ADD instruction. 3005 // For CMPrr(r1,r2), we are looking for SUB(r1,r2), SUB(r2,r1), or 3006 // ADDr[ri](r1, r2, X). 3007 // For CMPri(r1, CmpValue), we are looking for SUBri(r1, CmpValue). 3008 MachineInstr *SubAdd = nullptr; 3009 if (SrcReg2 != 0) 3010 // MI is not a candidate for CMPrr. 3011 MI = nullptr; 3012 else if (MI->getParent() != CmpInstr.getParent() || CmpValue != 0) { 3013 // Conservatively refuse to convert an instruction which isn't in the same 3014 // BB as the comparison. 3015 // For CMPri w/ CmpValue != 0, a SubAdd may still be a candidate. 3016 // Thus we cannot return here. 3017 if (CmpInstr.getOpcode() == ARM::CMPri || 3018 CmpInstr.getOpcode() == ARM::t2CMPri || 3019 CmpInstr.getOpcode() == ARM::tCMPi8) 3020 MI = nullptr; 3021 else 3022 return false; 3023 } 3024 3025 bool IsThumb1 = false; 3026 if (MI && !isOptimizeCompareCandidate(MI, IsThumb1)) 3027 return false; 3028 3029 // We also want to do this peephole for cases like this: if (a*b == 0), 3030 // and optimise away the CMP instruction from the generated code sequence: 3031 // MULS, MOVS, MOVS, CMP. Here the MOVS instructions load the boolean values 3032 // resulting from the select instruction, but these MOVS instructions for 3033 // Thumb1 (V6M) are flag setting and are thus preventing this optimisation. 3034 // However, if we only have MOVS instructions in between the CMP and the 3035 // other instruction (the MULS in this example), then the CPSR is dead so we 3036 // can safely reorder the sequence into: MOVS, MOVS, MULS, CMP. We do this 3037 // reordering and then continue the analysis hoping we can eliminate the 3038 // CMP. This peephole works on the vregs, so is still in SSA form. As a 3039 // consequence, the movs won't redefine/kill the MUL operands which would 3040 // make this reordering illegal. 3041 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3042 if (MI && IsThumb1) { 3043 --I; 3044 if (I != E && !MI->readsRegister(ARM::CPSR, TRI)) { 3045 bool CanReorder = true; 3046 for (; I != E; --I) { 3047 if (I->getOpcode() != ARM::tMOVi8) { 3048 CanReorder = false; 3049 break; 3050 } 3051 } 3052 if (CanReorder) { 3053 MI = MI->removeFromParent(); 3054 E = CmpInstr; 3055 CmpInstr.getParent()->insert(E, MI); 3056 } 3057 } 3058 I = CmpInstr; 3059 E = MI; 3060 } 3061 3062 // Check that CPSR isn't set between the comparison instruction and the one we 3063 // want to change. At the same time, search for SubAdd. 3064 bool SubAddIsThumb1 = false; 3065 do { 3066 const MachineInstr &Instr = *--I; 3067 3068 // Check whether CmpInstr can be made redundant by the current instruction. 3069 if (isRedundantFlagInstr(&CmpInstr, SrcReg, SrcReg2, CmpValue, &Instr, 3070 SubAddIsThumb1)) { 3071 SubAdd = &*I; 3072 break; 3073 } 3074 3075 // Allow E (which was initially MI) to be SubAdd but do not search before E. 3076 if (I == E) 3077 break; 3078 3079 if (Instr.modifiesRegister(ARM::CPSR, TRI) || 3080 Instr.readsRegister(ARM::CPSR, TRI)) 3081 // This instruction modifies or uses CPSR after the one we want to 3082 // change. We can't do this transformation. 3083 return false; 3084 3085 if (I == B) { 3086 // In some cases, we scan the use-list of an instruction for an AND; 3087 // that AND is in the same BB, but may not be scheduled before the 3088 // corresponding TST. In that case, bail out. 3089 // 3090 // FIXME: We could try to reschedule the AND. 3091 return false; 3092 } 3093 } while (true); 3094 3095 // Return false if no candidates exist. 3096 if (!MI && !SubAdd) 3097 return false; 3098 3099 // If we found a SubAdd, use it as it will be closer to the CMP 3100 if (SubAdd) { 3101 MI = SubAdd; 3102 IsThumb1 = SubAddIsThumb1; 3103 } 3104 3105 // We can't use a predicated instruction - it doesn't always write the flags. 3106 if (isPredicated(*MI)) 3107 return false; 3108 3109 // Scan forward for the use of CPSR 3110 // When checking against MI: if it's a conditional code that requires 3111 // checking of the V bit or C bit, then this is not safe to do. 3112 // It is safe to remove CmpInstr if CPSR is redefined or killed. 3113 // If we are done with the basic block, we need to check whether CPSR is 3114 // live-out. 3115 SmallVector<std::pair<MachineOperand*, ARMCC::CondCodes>, 4> 3116 OperandsToUpdate; 3117 bool isSafe = false; 3118 I = CmpInstr; 3119 E = CmpInstr.getParent()->end(); 3120 while (!isSafe && ++I != E) { 3121 const MachineInstr &Instr = *I; 3122 for (unsigned IO = 0, EO = Instr.getNumOperands(); 3123 !isSafe && IO != EO; ++IO) { 3124 const MachineOperand &MO = Instr.getOperand(IO); 3125 if (MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR)) { 3126 isSafe = true; 3127 break; 3128 } 3129 if (!MO.isReg() || MO.getReg() != ARM::CPSR) 3130 continue; 3131 if (MO.isDef()) { 3132 isSafe = true; 3133 break; 3134 } 3135 // Condition code is after the operand before CPSR except for VSELs. 3136 ARMCC::CondCodes CC; 3137 bool IsInstrVSel = true; 3138 switch (Instr.getOpcode()) { 3139 default: 3140 IsInstrVSel = false; 3141 CC = (ARMCC::CondCodes)Instr.getOperand(IO - 1).getImm(); 3142 break; 3143 case ARM::VSELEQD: 3144 case ARM::VSELEQS: 3145 case ARM::VSELEQH: 3146 CC = ARMCC::EQ; 3147 break; 3148 case ARM::VSELGTD: 3149 case ARM::VSELGTS: 3150 case ARM::VSELGTH: 3151 CC = ARMCC::GT; 3152 break; 3153 case ARM::VSELGED: 3154 case ARM::VSELGES: 3155 case ARM::VSELGEH: 3156 CC = ARMCC::GE; 3157 break; 3158 case ARM::VSELVSD: 3159 case ARM::VSELVSS: 3160 case ARM::VSELVSH: 3161 CC = ARMCC::VS; 3162 break; 3163 } 3164 3165 if (SubAdd) { 3166 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based 3167 // on CMP needs to be updated to be based on SUB. 3168 // If we have ADD(r1, r2, X) and CMP(r1, r2), the condition code also 3169 // needs to be modified. 3170 // Push the condition code operands to OperandsToUpdate. 3171 // If it is safe to remove CmpInstr, the condition code of these 3172 // operands will be modified. 3173 unsigned Opc = SubAdd->getOpcode(); 3174 bool IsSub = Opc == ARM::SUBrr || Opc == ARM::t2SUBrr || 3175 Opc == ARM::SUBri || Opc == ARM::t2SUBri || 3176 Opc == ARM::tSUBrr || Opc == ARM::tSUBi3 || 3177 Opc == ARM::tSUBi8; 3178 unsigned OpI = Opc != ARM::tSUBrr ? 1 : 2; 3179 if (!IsSub || 3180 (SrcReg2 != 0 && SubAdd->getOperand(OpI).getReg() == SrcReg2 && 3181 SubAdd->getOperand(OpI + 1).getReg() == SrcReg)) { 3182 // VSel doesn't support condition code update. 3183 if (IsInstrVSel) 3184 return false; 3185 // Ensure we can swap the condition. 3186 ARMCC::CondCodes NewCC = (IsSub ? getSwappedCondition(CC) : getCmpToAddCondition(CC)); 3187 if (NewCC == ARMCC::AL) 3188 return false; 3189 OperandsToUpdate.push_back( 3190 std::make_pair(&((*I).getOperand(IO - 1)), NewCC)); 3191 } 3192 } else { 3193 // No SubAdd, so this is x = <op> y, z; cmp x, 0. 3194 switch (CC) { 3195 case ARMCC::EQ: // Z 3196 case ARMCC::NE: // Z 3197 case ARMCC::MI: // N 3198 case ARMCC::PL: // N 3199 case ARMCC::AL: // none 3200 // CPSR can be used multiple times, we should continue. 3201 break; 3202 case ARMCC::HS: // C 3203 case ARMCC::LO: // C 3204 case ARMCC::VS: // V 3205 case ARMCC::VC: // V 3206 case ARMCC::HI: // C Z 3207 case ARMCC::LS: // C Z 3208 case ARMCC::GE: // N V 3209 case ARMCC::LT: // N V 3210 case ARMCC::GT: // Z N V 3211 case ARMCC::LE: // Z N V 3212 // The instruction uses the V bit or C bit which is not safe. 3213 return false; 3214 } 3215 } 3216 } 3217 } 3218 3219 // If CPSR is not killed nor re-defined, we should check whether it is 3220 // live-out. If it is live-out, do not optimize. 3221 if (!isSafe) { 3222 MachineBasicBlock *MBB = CmpInstr.getParent(); 3223 for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(), 3224 SE = MBB->succ_end(); SI != SE; ++SI) 3225 if ((*SI)->isLiveIn(ARM::CPSR)) 3226 return false; 3227 } 3228 3229 // Toggle the optional operand to CPSR (if it exists - in Thumb1 we always 3230 // set CPSR so this is represented as an explicit output) 3231 if (!IsThumb1) { 3232 MI->getOperand(5).setReg(ARM::CPSR); 3233 MI->getOperand(5).setIsDef(true); 3234 } 3235 assert(!isPredicated(*MI) && "Can't use flags from predicated instruction"); 3236 CmpInstr.eraseFromParent(); 3237 3238 // Modify the condition code of operands in OperandsToUpdate. 3239 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to 3240 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc. 3241 for (unsigned i = 0, e = OperandsToUpdate.size(); i < e; i++) 3242 OperandsToUpdate[i].first->setImm(OperandsToUpdate[i].second); 3243 3244 MI->clearRegisterDeads(ARM::CPSR); 3245 3246 return true; 3247 } 3248 3249 bool ARMBaseInstrInfo::shouldSink(const MachineInstr &MI) const { 3250 // Do not sink MI if it might be used to optimize a redundant compare. 3251 // We heuristically only look at the instruction immediately following MI to 3252 // avoid potentially searching the entire basic block. 3253 if (isPredicated(MI)) 3254 return true; 3255 MachineBasicBlock::const_iterator Next = &MI; 3256 ++Next; 3257 Register SrcReg, SrcReg2; 3258 int CmpMask, CmpValue; 3259 bool IsThumb1; 3260 if (Next != MI.getParent()->end() && 3261 analyzeCompare(*Next, SrcReg, SrcReg2, CmpMask, CmpValue) && 3262 isRedundantFlagInstr(&*Next, SrcReg, SrcReg2, CmpValue, &MI, IsThumb1)) 3263 return false; 3264 return true; 3265 } 3266 3267 bool ARMBaseInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 3268 Register Reg, 3269 MachineRegisterInfo *MRI) const { 3270 // Fold large immediates into add, sub, or, xor. 3271 unsigned DefOpc = DefMI.getOpcode(); 3272 if (DefOpc != ARM::t2MOVi32imm && DefOpc != ARM::MOVi32imm) 3273 return false; 3274 if (!DefMI.getOperand(1).isImm()) 3275 // Could be t2MOVi32imm @xx 3276 return false; 3277 3278 if (!MRI->hasOneNonDBGUse(Reg)) 3279 return false; 3280 3281 const MCInstrDesc &DefMCID = DefMI.getDesc(); 3282 if (DefMCID.hasOptionalDef()) { 3283 unsigned NumOps = DefMCID.getNumOperands(); 3284 const MachineOperand &MO = DefMI.getOperand(NumOps - 1); 3285 if (MO.getReg() == ARM::CPSR && !MO.isDead()) 3286 // If DefMI defines CPSR and it is not dead, it's obviously not safe 3287 // to delete DefMI. 3288 return false; 3289 } 3290 3291 const MCInstrDesc &UseMCID = UseMI.getDesc(); 3292 if (UseMCID.hasOptionalDef()) { 3293 unsigned NumOps = UseMCID.getNumOperands(); 3294 if (UseMI.getOperand(NumOps - 1).getReg() == ARM::CPSR) 3295 // If the instruction sets the flag, do not attempt this optimization 3296 // since it may change the semantics of the code. 3297 return false; 3298 } 3299 3300 unsigned UseOpc = UseMI.getOpcode(); 3301 unsigned NewUseOpc = 0; 3302 uint32_t ImmVal = (uint32_t)DefMI.getOperand(1).getImm(); 3303 uint32_t SOImmValV1 = 0, SOImmValV2 = 0; 3304 bool Commute = false; 3305 switch (UseOpc) { 3306 default: return false; 3307 case ARM::SUBrr: 3308 case ARM::ADDrr: 3309 case ARM::ORRrr: 3310 case ARM::EORrr: 3311 case ARM::t2SUBrr: 3312 case ARM::t2ADDrr: 3313 case ARM::t2ORRrr: 3314 case ARM::t2EORrr: { 3315 Commute = UseMI.getOperand(2).getReg() != Reg; 3316 switch (UseOpc) { 3317 default: break; 3318 case ARM::ADDrr: 3319 case ARM::SUBrr: 3320 if (UseOpc == ARM::SUBrr && Commute) 3321 return false; 3322 3323 // ADD/SUB are special because they're essentially the same operation, so 3324 // we can handle a larger range of immediates. 3325 if (ARM_AM::isSOImmTwoPartVal(ImmVal)) 3326 NewUseOpc = UseOpc == ARM::ADDrr ? ARM::ADDri : ARM::SUBri; 3327 else if (ARM_AM::isSOImmTwoPartVal(-ImmVal)) { 3328 ImmVal = -ImmVal; 3329 NewUseOpc = UseOpc == ARM::ADDrr ? ARM::SUBri : ARM::ADDri; 3330 } else 3331 return false; 3332 SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal); 3333 SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal); 3334 break; 3335 case ARM::ORRrr: 3336 case ARM::EORrr: 3337 if (!ARM_AM::isSOImmTwoPartVal(ImmVal)) 3338 return false; 3339 SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal); 3340 SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal); 3341 switch (UseOpc) { 3342 default: break; 3343 case ARM::ORRrr: NewUseOpc = ARM::ORRri; break; 3344 case ARM::EORrr: NewUseOpc = ARM::EORri; break; 3345 } 3346 break; 3347 case ARM::t2ADDrr: 3348 case ARM::t2SUBrr: { 3349 if (UseOpc == ARM::t2SUBrr && Commute) 3350 return false; 3351 3352 // ADD/SUB are special because they're essentially the same operation, so 3353 // we can handle a larger range of immediates. 3354 const bool ToSP = DefMI.getOperand(0).getReg() == ARM::SP; 3355 const unsigned t2ADD = ToSP ? ARM::t2ADDspImm : ARM::t2ADDri; 3356 const unsigned t2SUB = ToSP ? ARM::t2SUBspImm : ARM::t2SUBri; 3357 if (ARM_AM::isT2SOImmTwoPartVal(ImmVal)) 3358 NewUseOpc = UseOpc == ARM::t2ADDrr ? t2ADD : t2SUB; 3359 else if (ARM_AM::isT2SOImmTwoPartVal(-ImmVal)) { 3360 ImmVal = -ImmVal; 3361 NewUseOpc = UseOpc == ARM::t2ADDrr ? t2SUB : t2ADD; 3362 } else 3363 return false; 3364 SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal); 3365 SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal); 3366 break; 3367 } 3368 case ARM::t2ORRrr: 3369 case ARM::t2EORrr: 3370 if (!ARM_AM::isT2SOImmTwoPartVal(ImmVal)) 3371 return false; 3372 SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal); 3373 SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal); 3374 switch (UseOpc) { 3375 default: break; 3376 case ARM::t2ORRrr: NewUseOpc = ARM::t2ORRri; break; 3377 case ARM::t2EORrr: NewUseOpc = ARM::t2EORri; break; 3378 } 3379 break; 3380 } 3381 } 3382 } 3383 3384 unsigned OpIdx = Commute ? 2 : 1; 3385 Register Reg1 = UseMI.getOperand(OpIdx).getReg(); 3386 bool isKill = UseMI.getOperand(OpIdx).isKill(); 3387 const TargetRegisterClass *TRC = MRI->getRegClass(Reg); 3388 Register NewReg = MRI->createVirtualRegister(TRC); 3389 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), get(NewUseOpc), 3390 NewReg) 3391 .addReg(Reg1, getKillRegState(isKill)) 3392 .addImm(SOImmValV1) 3393 .add(predOps(ARMCC::AL)) 3394 .add(condCodeOp()); 3395 UseMI.setDesc(get(NewUseOpc)); 3396 UseMI.getOperand(1).setReg(NewReg); 3397 UseMI.getOperand(1).setIsKill(); 3398 UseMI.getOperand(2).ChangeToImmediate(SOImmValV2); 3399 DefMI.eraseFromParent(); 3400 // FIXME: t2ADDrr should be split, as different rulles apply when writing to SP. 3401 // Just as t2ADDri, that was split to [t2ADDri, t2ADDspImm]. 3402 // Then the below code will not be needed, as the input/output register 3403 // classes will be rgpr or gprSP. 3404 // For now, we fix the UseMI operand explicitly here: 3405 switch(NewUseOpc){ 3406 case ARM::t2ADDspImm: 3407 case ARM::t2SUBspImm: 3408 case ARM::t2ADDri: 3409 case ARM::t2SUBri: 3410 MRI->constrainRegClass(UseMI.getOperand(0).getReg(), TRC); 3411 } 3412 return true; 3413 } 3414 3415 static unsigned getNumMicroOpsSwiftLdSt(const InstrItineraryData *ItinData, 3416 const MachineInstr &MI) { 3417 switch (MI.getOpcode()) { 3418 default: { 3419 const MCInstrDesc &Desc = MI.getDesc(); 3420 int UOps = ItinData->getNumMicroOps(Desc.getSchedClass()); 3421 assert(UOps >= 0 && "bad # UOps"); 3422 return UOps; 3423 } 3424 3425 case ARM::LDRrs: 3426 case ARM::LDRBrs: 3427 case ARM::STRrs: 3428 case ARM::STRBrs: { 3429 unsigned ShOpVal = MI.getOperand(3).getImm(); 3430 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 3431 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 3432 if (!isSub && 3433 (ShImm == 0 || 3434 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 3435 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 3436 return 1; 3437 return 2; 3438 } 3439 3440 case ARM::LDRH: 3441 case ARM::STRH: { 3442 if (!MI.getOperand(2).getReg()) 3443 return 1; 3444 3445 unsigned ShOpVal = MI.getOperand(3).getImm(); 3446 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 3447 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 3448 if (!isSub && 3449 (ShImm == 0 || 3450 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 3451 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 3452 return 1; 3453 return 2; 3454 } 3455 3456 case ARM::LDRSB: 3457 case ARM::LDRSH: 3458 return (ARM_AM::getAM3Op(MI.getOperand(3).getImm()) == ARM_AM::sub) ? 3 : 2; 3459 3460 case ARM::LDRSB_POST: 3461 case ARM::LDRSH_POST: { 3462 Register Rt = MI.getOperand(0).getReg(); 3463 Register Rm = MI.getOperand(3).getReg(); 3464 return (Rt == Rm) ? 4 : 3; 3465 } 3466 3467 case ARM::LDR_PRE_REG: 3468 case ARM::LDRB_PRE_REG: { 3469 Register Rt = MI.getOperand(0).getReg(); 3470 Register Rm = MI.getOperand(3).getReg(); 3471 if (Rt == Rm) 3472 return 3; 3473 unsigned ShOpVal = MI.getOperand(4).getImm(); 3474 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 3475 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 3476 if (!isSub && 3477 (ShImm == 0 || 3478 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 3479 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 3480 return 2; 3481 return 3; 3482 } 3483 3484 case ARM::STR_PRE_REG: 3485 case ARM::STRB_PRE_REG: { 3486 unsigned ShOpVal = MI.getOperand(4).getImm(); 3487 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 3488 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 3489 if (!isSub && 3490 (ShImm == 0 || 3491 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 3492 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 3493 return 2; 3494 return 3; 3495 } 3496 3497 case ARM::LDRH_PRE: 3498 case ARM::STRH_PRE: { 3499 Register Rt = MI.getOperand(0).getReg(); 3500 Register Rm = MI.getOperand(3).getReg(); 3501 if (!Rm) 3502 return 2; 3503 if (Rt == Rm) 3504 return 3; 3505 return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 3 : 2; 3506 } 3507 3508 case ARM::LDR_POST_REG: 3509 case ARM::LDRB_POST_REG: 3510 case ARM::LDRH_POST: { 3511 Register Rt = MI.getOperand(0).getReg(); 3512 Register Rm = MI.getOperand(3).getReg(); 3513 return (Rt == Rm) ? 3 : 2; 3514 } 3515 3516 case ARM::LDR_PRE_IMM: 3517 case ARM::LDRB_PRE_IMM: 3518 case ARM::LDR_POST_IMM: 3519 case ARM::LDRB_POST_IMM: 3520 case ARM::STRB_POST_IMM: 3521 case ARM::STRB_POST_REG: 3522 case ARM::STRB_PRE_IMM: 3523 case ARM::STRH_POST: 3524 case ARM::STR_POST_IMM: 3525 case ARM::STR_POST_REG: 3526 case ARM::STR_PRE_IMM: 3527 return 2; 3528 3529 case ARM::LDRSB_PRE: 3530 case ARM::LDRSH_PRE: { 3531 Register Rm = MI.getOperand(3).getReg(); 3532 if (Rm == 0) 3533 return 3; 3534 Register Rt = MI.getOperand(0).getReg(); 3535 if (Rt == Rm) 3536 return 4; 3537 unsigned ShOpVal = MI.getOperand(4).getImm(); 3538 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 3539 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 3540 if (!isSub && 3541 (ShImm == 0 || 3542 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 3543 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 3544 return 3; 3545 return 4; 3546 } 3547 3548 case ARM::LDRD: { 3549 Register Rt = MI.getOperand(0).getReg(); 3550 Register Rn = MI.getOperand(2).getReg(); 3551 Register Rm = MI.getOperand(3).getReg(); 3552 if (Rm) 3553 return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 4 3554 : 3; 3555 return (Rt == Rn) ? 3 : 2; 3556 } 3557 3558 case ARM::STRD: { 3559 Register Rm = MI.getOperand(3).getReg(); 3560 if (Rm) 3561 return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 4 3562 : 3; 3563 return 2; 3564 } 3565 3566 case ARM::LDRD_POST: 3567 case ARM::t2LDRD_POST: 3568 return 3; 3569 3570 case ARM::STRD_POST: 3571 case ARM::t2STRD_POST: 3572 return 4; 3573 3574 case ARM::LDRD_PRE: { 3575 Register Rt = MI.getOperand(0).getReg(); 3576 Register Rn = MI.getOperand(3).getReg(); 3577 Register Rm = MI.getOperand(4).getReg(); 3578 if (Rm) 3579 return (ARM_AM::getAM3Op(MI.getOperand(5).getImm()) == ARM_AM::sub) ? 5 3580 : 4; 3581 return (Rt == Rn) ? 4 : 3; 3582 } 3583 3584 case ARM::t2LDRD_PRE: { 3585 Register Rt = MI.getOperand(0).getReg(); 3586 Register Rn = MI.getOperand(3).getReg(); 3587 return (Rt == Rn) ? 4 : 3; 3588 } 3589 3590 case ARM::STRD_PRE: { 3591 Register Rm = MI.getOperand(4).getReg(); 3592 if (Rm) 3593 return (ARM_AM::getAM3Op(MI.getOperand(5).getImm()) == ARM_AM::sub) ? 5 3594 : 4; 3595 return 3; 3596 } 3597 3598 case ARM::t2STRD_PRE: 3599 return 3; 3600 3601 case ARM::t2LDR_POST: 3602 case ARM::t2LDRB_POST: 3603 case ARM::t2LDRB_PRE: 3604 case ARM::t2LDRSBi12: 3605 case ARM::t2LDRSBi8: 3606 case ARM::t2LDRSBpci: 3607 case ARM::t2LDRSBs: 3608 case ARM::t2LDRH_POST: 3609 case ARM::t2LDRH_PRE: 3610 case ARM::t2LDRSBT: 3611 case ARM::t2LDRSB_POST: 3612 case ARM::t2LDRSB_PRE: 3613 case ARM::t2LDRSH_POST: 3614 case ARM::t2LDRSH_PRE: 3615 case ARM::t2LDRSHi12: 3616 case ARM::t2LDRSHi8: 3617 case ARM::t2LDRSHpci: 3618 case ARM::t2LDRSHs: 3619 return 2; 3620 3621 case ARM::t2LDRDi8: { 3622 Register Rt = MI.getOperand(0).getReg(); 3623 Register Rn = MI.getOperand(2).getReg(); 3624 return (Rt == Rn) ? 3 : 2; 3625 } 3626 3627 case ARM::t2STRB_POST: 3628 case ARM::t2STRB_PRE: 3629 case ARM::t2STRBs: 3630 case ARM::t2STRDi8: 3631 case ARM::t2STRH_POST: 3632 case ARM::t2STRH_PRE: 3633 case ARM::t2STRHs: 3634 case ARM::t2STR_POST: 3635 case ARM::t2STR_PRE: 3636 case ARM::t2STRs: 3637 return 2; 3638 } 3639 } 3640 3641 // Return the number of 32-bit words loaded by LDM or stored by STM. If this 3642 // can't be easily determined return 0 (missing MachineMemOperand). 3643 // 3644 // FIXME: The current MachineInstr design does not support relying on machine 3645 // mem operands to determine the width of a memory access. Instead, we expect 3646 // the target to provide this information based on the instruction opcode and 3647 // operands. However, using MachineMemOperand is the best solution now for 3648 // two reasons: 3649 // 3650 // 1) getNumMicroOps tries to infer LDM memory width from the total number of MI 3651 // operands. This is much more dangerous than using the MachineMemOperand 3652 // sizes because CodeGen passes can insert/remove optional machine operands. In 3653 // fact, it's totally incorrect for preRA passes and appears to be wrong for 3654 // postRA passes as well. 3655 // 3656 // 2) getNumLDMAddresses is only used by the scheduling machine model and any 3657 // machine model that calls this should handle the unknown (zero size) case. 3658 // 3659 // Long term, we should require a target hook that verifies MachineMemOperand 3660 // sizes during MC lowering. That target hook should be local to MC lowering 3661 // because we can't ensure that it is aware of other MI forms. Doing this will 3662 // ensure that MachineMemOperands are correctly propagated through all passes. 3663 unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr &MI) const { 3664 unsigned Size = 0; 3665 for (MachineInstr::mmo_iterator I = MI.memoperands_begin(), 3666 E = MI.memoperands_end(); 3667 I != E; ++I) { 3668 Size += (*I)->getSize(); 3669 } 3670 // FIXME: The scheduler currently can't handle values larger than 16. But 3671 // the values can actually go up to 32 for floating-point load/store 3672 // multiple (VLDMIA etc.). Also, the way this code is reasoning about memory 3673 // operations isn't right; we could end up with "extra" memory operands for 3674 // various reasons, like tail merge merging two memory operations. 3675 return std::min(Size / 4, 16U); 3676 } 3677 3678 static unsigned getNumMicroOpsSingleIssuePlusExtras(unsigned Opc, 3679 unsigned NumRegs) { 3680 unsigned UOps = 1 + NumRegs; // 1 for address computation. 3681 switch (Opc) { 3682 default: 3683 break; 3684 case ARM::VLDMDIA_UPD: 3685 case ARM::VLDMDDB_UPD: 3686 case ARM::VLDMSIA_UPD: 3687 case ARM::VLDMSDB_UPD: 3688 case ARM::VSTMDIA_UPD: 3689 case ARM::VSTMDDB_UPD: 3690 case ARM::VSTMSIA_UPD: 3691 case ARM::VSTMSDB_UPD: 3692 case ARM::LDMIA_UPD: 3693 case ARM::LDMDA_UPD: 3694 case ARM::LDMDB_UPD: 3695 case ARM::LDMIB_UPD: 3696 case ARM::STMIA_UPD: 3697 case ARM::STMDA_UPD: 3698 case ARM::STMDB_UPD: 3699 case ARM::STMIB_UPD: 3700 case ARM::tLDMIA_UPD: 3701 case ARM::tSTMIA_UPD: 3702 case ARM::t2LDMIA_UPD: 3703 case ARM::t2LDMDB_UPD: 3704 case ARM::t2STMIA_UPD: 3705 case ARM::t2STMDB_UPD: 3706 ++UOps; // One for base register writeback. 3707 break; 3708 case ARM::LDMIA_RET: 3709 case ARM::tPOP_RET: 3710 case ARM::t2LDMIA_RET: 3711 UOps += 2; // One for base reg wb, one for write to pc. 3712 break; 3713 } 3714 return UOps; 3715 } 3716 3717 unsigned ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData, 3718 const MachineInstr &MI) const { 3719 if (!ItinData || ItinData->isEmpty()) 3720 return 1; 3721 3722 const MCInstrDesc &Desc = MI.getDesc(); 3723 unsigned Class = Desc.getSchedClass(); 3724 int ItinUOps = ItinData->getNumMicroOps(Class); 3725 if (ItinUOps >= 0) { 3726 if (Subtarget.isSwift() && (Desc.mayLoad() || Desc.mayStore())) 3727 return getNumMicroOpsSwiftLdSt(ItinData, MI); 3728 3729 return ItinUOps; 3730 } 3731 3732 unsigned Opc = MI.getOpcode(); 3733 switch (Opc) { 3734 default: 3735 llvm_unreachable("Unexpected multi-uops instruction!"); 3736 case ARM::VLDMQIA: 3737 case ARM::VSTMQIA: 3738 return 2; 3739 3740 // The number of uOps for load / store multiple are determined by the number 3741 // registers. 3742 // 3743 // On Cortex-A8, each pair of register loads / stores can be scheduled on the 3744 // same cycle. The scheduling for the first load / store must be done 3745 // separately by assuming the address is not 64-bit aligned. 3746 // 3747 // On Cortex-A9, the formula is simply (#reg / 2) + (#reg % 2). If the address 3748 // is not 64-bit aligned, then AGU would take an extra cycle. For VFP / NEON 3749 // load / store multiple, the formula is (#reg / 2) + (#reg % 2) + 1. 3750 case ARM::VLDMDIA: 3751 case ARM::VLDMDIA_UPD: 3752 case ARM::VLDMDDB_UPD: 3753 case ARM::VLDMSIA: 3754 case ARM::VLDMSIA_UPD: 3755 case ARM::VLDMSDB_UPD: 3756 case ARM::VSTMDIA: 3757 case ARM::VSTMDIA_UPD: 3758 case ARM::VSTMDDB_UPD: 3759 case ARM::VSTMSIA: 3760 case ARM::VSTMSIA_UPD: 3761 case ARM::VSTMSDB_UPD: { 3762 unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands(); 3763 return (NumRegs / 2) + (NumRegs % 2) + 1; 3764 } 3765 3766 case ARM::LDMIA_RET: 3767 case ARM::LDMIA: 3768 case ARM::LDMDA: 3769 case ARM::LDMDB: 3770 case ARM::LDMIB: 3771 case ARM::LDMIA_UPD: 3772 case ARM::LDMDA_UPD: 3773 case ARM::LDMDB_UPD: 3774 case ARM::LDMIB_UPD: 3775 case ARM::STMIA: 3776 case ARM::STMDA: 3777 case ARM::STMDB: 3778 case ARM::STMIB: 3779 case ARM::STMIA_UPD: 3780 case ARM::STMDA_UPD: 3781 case ARM::STMDB_UPD: 3782 case ARM::STMIB_UPD: 3783 case ARM::tLDMIA: 3784 case ARM::tLDMIA_UPD: 3785 case ARM::tSTMIA_UPD: 3786 case ARM::tPOP_RET: 3787 case ARM::tPOP: 3788 case ARM::tPUSH: 3789 case ARM::t2LDMIA_RET: 3790 case ARM::t2LDMIA: 3791 case ARM::t2LDMDB: 3792 case ARM::t2LDMIA_UPD: 3793 case ARM::t2LDMDB_UPD: 3794 case ARM::t2STMIA: 3795 case ARM::t2STMDB: 3796 case ARM::t2STMIA_UPD: 3797 case ARM::t2STMDB_UPD: { 3798 unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands() + 1; 3799 switch (Subtarget.getLdStMultipleTiming()) { 3800 case ARMSubtarget::SingleIssuePlusExtras: 3801 return getNumMicroOpsSingleIssuePlusExtras(Opc, NumRegs); 3802 case ARMSubtarget::SingleIssue: 3803 // Assume the worst. 3804 return NumRegs; 3805 case ARMSubtarget::DoubleIssue: { 3806 if (NumRegs < 4) 3807 return 2; 3808 // 4 registers would be issued: 2, 2. 3809 // 5 registers would be issued: 2, 2, 1. 3810 unsigned UOps = (NumRegs / 2); 3811 if (NumRegs % 2) 3812 ++UOps; 3813 return UOps; 3814 } 3815 case ARMSubtarget::DoubleIssueCheckUnalignedAccess: { 3816 unsigned UOps = (NumRegs / 2); 3817 // If there are odd number of registers or if it's not 64-bit aligned, 3818 // then it takes an extra AGU (Address Generation Unit) cycle. 3819 if ((NumRegs % 2) || !MI.hasOneMemOperand() || 3820 (*MI.memoperands_begin())->getAlign() < Align(8)) 3821 ++UOps; 3822 return UOps; 3823 } 3824 } 3825 } 3826 } 3827 llvm_unreachable("Didn't find the number of microops"); 3828 } 3829 3830 int 3831 ARMBaseInstrInfo::getVLDMDefCycle(const InstrItineraryData *ItinData, 3832 const MCInstrDesc &DefMCID, 3833 unsigned DefClass, 3834 unsigned DefIdx, unsigned DefAlign) const { 3835 int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1; 3836 if (RegNo <= 0) 3837 // Def is the address writeback. 3838 return ItinData->getOperandCycle(DefClass, DefIdx); 3839 3840 int DefCycle; 3841 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) { 3842 // (regno / 2) + (regno % 2) + 1 3843 DefCycle = RegNo / 2 + 1; 3844 if (RegNo % 2) 3845 ++DefCycle; 3846 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) { 3847 DefCycle = RegNo; 3848 bool isSLoad = false; 3849 3850 switch (DefMCID.getOpcode()) { 3851 default: break; 3852 case ARM::VLDMSIA: 3853 case ARM::VLDMSIA_UPD: 3854 case ARM::VLDMSDB_UPD: 3855 isSLoad = true; 3856 break; 3857 } 3858 3859 // If there are odd number of 'S' registers or if it's not 64-bit aligned, 3860 // then it takes an extra cycle. 3861 if ((isSLoad && (RegNo % 2)) || DefAlign < 8) 3862 ++DefCycle; 3863 } else { 3864 // Assume the worst. 3865 DefCycle = RegNo + 2; 3866 } 3867 3868 return DefCycle; 3869 } 3870 3871 int 3872 ARMBaseInstrInfo::getLDMDefCycle(const InstrItineraryData *ItinData, 3873 const MCInstrDesc &DefMCID, 3874 unsigned DefClass, 3875 unsigned DefIdx, unsigned DefAlign) const { 3876 int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1; 3877 if (RegNo <= 0) 3878 // Def is the address writeback. 3879 return ItinData->getOperandCycle(DefClass, DefIdx); 3880 3881 int DefCycle; 3882 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) { 3883 // 4 registers would be issued: 1, 2, 1. 3884 // 5 registers would be issued: 1, 2, 2. 3885 DefCycle = RegNo / 2; 3886 if (DefCycle < 1) 3887 DefCycle = 1; 3888 // Result latency is issue cycle + 2: E2. 3889 DefCycle += 2; 3890 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) { 3891 DefCycle = (RegNo / 2); 3892 // If there are odd number of registers or if it's not 64-bit aligned, 3893 // then it takes an extra AGU (Address Generation Unit) cycle. 3894 if ((RegNo % 2) || DefAlign < 8) 3895 ++DefCycle; 3896 // Result latency is AGU cycles + 2. 3897 DefCycle += 2; 3898 } else { 3899 // Assume the worst. 3900 DefCycle = RegNo + 2; 3901 } 3902 3903 return DefCycle; 3904 } 3905 3906 int 3907 ARMBaseInstrInfo::getVSTMUseCycle(const InstrItineraryData *ItinData, 3908 const MCInstrDesc &UseMCID, 3909 unsigned UseClass, 3910 unsigned UseIdx, unsigned UseAlign) const { 3911 int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1; 3912 if (RegNo <= 0) 3913 return ItinData->getOperandCycle(UseClass, UseIdx); 3914 3915 int UseCycle; 3916 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) { 3917 // (regno / 2) + (regno % 2) + 1 3918 UseCycle = RegNo / 2 + 1; 3919 if (RegNo % 2) 3920 ++UseCycle; 3921 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) { 3922 UseCycle = RegNo; 3923 bool isSStore = false; 3924 3925 switch (UseMCID.getOpcode()) { 3926 default: break; 3927 case ARM::VSTMSIA: 3928 case ARM::VSTMSIA_UPD: 3929 case ARM::VSTMSDB_UPD: 3930 isSStore = true; 3931 break; 3932 } 3933 3934 // If there are odd number of 'S' registers or if it's not 64-bit aligned, 3935 // then it takes an extra cycle. 3936 if ((isSStore && (RegNo % 2)) || UseAlign < 8) 3937 ++UseCycle; 3938 } else { 3939 // Assume the worst. 3940 UseCycle = RegNo + 2; 3941 } 3942 3943 return UseCycle; 3944 } 3945 3946 int 3947 ARMBaseInstrInfo::getSTMUseCycle(const InstrItineraryData *ItinData, 3948 const MCInstrDesc &UseMCID, 3949 unsigned UseClass, 3950 unsigned UseIdx, unsigned UseAlign) const { 3951 int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1; 3952 if (RegNo <= 0) 3953 return ItinData->getOperandCycle(UseClass, UseIdx); 3954 3955 int UseCycle; 3956 if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) { 3957 UseCycle = RegNo / 2; 3958 if (UseCycle < 2) 3959 UseCycle = 2; 3960 // Read in E3. 3961 UseCycle += 2; 3962 } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) { 3963 UseCycle = (RegNo / 2); 3964 // If there are odd number of registers or if it's not 64-bit aligned, 3965 // then it takes an extra AGU (Address Generation Unit) cycle. 3966 if ((RegNo % 2) || UseAlign < 8) 3967 ++UseCycle; 3968 } else { 3969 // Assume the worst. 3970 UseCycle = 1; 3971 } 3972 return UseCycle; 3973 } 3974 3975 int 3976 ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, 3977 const MCInstrDesc &DefMCID, 3978 unsigned DefIdx, unsigned DefAlign, 3979 const MCInstrDesc &UseMCID, 3980 unsigned UseIdx, unsigned UseAlign) const { 3981 unsigned DefClass = DefMCID.getSchedClass(); 3982 unsigned UseClass = UseMCID.getSchedClass(); 3983 3984 if (DefIdx < DefMCID.getNumDefs() && UseIdx < UseMCID.getNumOperands()) 3985 return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); 3986 3987 // This may be a def / use of a variable_ops instruction, the operand 3988 // latency might be determinable dynamically. Let the target try to 3989 // figure it out. 3990 int DefCycle = -1; 3991 bool LdmBypass = false; 3992 switch (DefMCID.getOpcode()) { 3993 default: 3994 DefCycle = ItinData->getOperandCycle(DefClass, DefIdx); 3995 break; 3996 3997 case ARM::VLDMDIA: 3998 case ARM::VLDMDIA_UPD: 3999 case ARM::VLDMDDB_UPD: 4000 case ARM::VLDMSIA: 4001 case ARM::VLDMSIA_UPD: 4002 case ARM::VLDMSDB_UPD: 4003 DefCycle = getVLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign); 4004 break; 4005 4006 case ARM::LDMIA_RET: 4007 case ARM::LDMIA: 4008 case ARM::LDMDA: 4009 case ARM::LDMDB: 4010 case ARM::LDMIB: 4011 case ARM::LDMIA_UPD: 4012 case ARM::LDMDA_UPD: 4013 case ARM::LDMDB_UPD: 4014 case ARM::LDMIB_UPD: 4015 case ARM::tLDMIA: 4016 case ARM::tLDMIA_UPD: 4017 case ARM::tPUSH: 4018 case ARM::t2LDMIA_RET: 4019 case ARM::t2LDMIA: 4020 case ARM::t2LDMDB: 4021 case ARM::t2LDMIA_UPD: 4022 case ARM::t2LDMDB_UPD: 4023 LdmBypass = true; 4024 DefCycle = getLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign); 4025 break; 4026 } 4027 4028 if (DefCycle == -1) 4029 // We can't seem to determine the result latency of the def, assume it's 2. 4030 DefCycle = 2; 4031 4032 int UseCycle = -1; 4033 switch (UseMCID.getOpcode()) { 4034 default: 4035 UseCycle = ItinData->getOperandCycle(UseClass, UseIdx); 4036 break; 4037 4038 case ARM::VSTMDIA: 4039 case ARM::VSTMDIA_UPD: 4040 case ARM::VSTMDDB_UPD: 4041 case ARM::VSTMSIA: 4042 case ARM::VSTMSIA_UPD: 4043 case ARM::VSTMSDB_UPD: 4044 UseCycle = getVSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign); 4045 break; 4046 4047 case ARM::STMIA: 4048 case ARM::STMDA: 4049 case ARM::STMDB: 4050 case ARM::STMIB: 4051 case ARM::STMIA_UPD: 4052 case ARM::STMDA_UPD: 4053 case ARM::STMDB_UPD: 4054 case ARM::STMIB_UPD: 4055 case ARM::tSTMIA_UPD: 4056 case ARM::tPOP_RET: 4057 case ARM::tPOP: 4058 case ARM::t2STMIA: 4059 case ARM::t2STMDB: 4060 case ARM::t2STMIA_UPD: 4061 case ARM::t2STMDB_UPD: 4062 UseCycle = getSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign); 4063 break; 4064 } 4065 4066 if (UseCycle == -1) 4067 // Assume it's read in the first stage. 4068 UseCycle = 1; 4069 4070 UseCycle = DefCycle - UseCycle + 1; 4071 if (UseCycle > 0) { 4072 if (LdmBypass) { 4073 // It's a variable_ops instruction so we can't use DefIdx here. Just use 4074 // first def operand. 4075 if (ItinData->hasPipelineForwarding(DefClass, DefMCID.getNumOperands()-1, 4076 UseClass, UseIdx)) 4077 --UseCycle; 4078 } else if (ItinData->hasPipelineForwarding(DefClass, DefIdx, 4079 UseClass, UseIdx)) { 4080 --UseCycle; 4081 } 4082 } 4083 4084 return UseCycle; 4085 } 4086 4087 static const MachineInstr *getBundledDefMI(const TargetRegisterInfo *TRI, 4088 const MachineInstr *MI, unsigned Reg, 4089 unsigned &DefIdx, unsigned &Dist) { 4090 Dist = 0; 4091 4092 MachineBasicBlock::const_iterator I = MI; ++I; 4093 MachineBasicBlock::const_instr_iterator II = std::prev(I.getInstrIterator()); 4094 assert(II->isInsideBundle() && "Empty bundle?"); 4095 4096 int Idx = -1; 4097 while (II->isInsideBundle()) { 4098 Idx = II->findRegisterDefOperandIdx(Reg, false, true, TRI); 4099 if (Idx != -1) 4100 break; 4101 --II; 4102 ++Dist; 4103 } 4104 4105 assert(Idx != -1 && "Cannot find bundled definition!"); 4106 DefIdx = Idx; 4107 return &*II; 4108 } 4109 4110 static const MachineInstr *getBundledUseMI(const TargetRegisterInfo *TRI, 4111 const MachineInstr &MI, unsigned Reg, 4112 unsigned &UseIdx, unsigned &Dist) { 4113 Dist = 0; 4114 4115 MachineBasicBlock::const_instr_iterator II = ++MI.getIterator(); 4116 assert(II->isInsideBundle() && "Empty bundle?"); 4117 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end(); 4118 4119 // FIXME: This doesn't properly handle multiple uses. 4120 int Idx = -1; 4121 while (II != E && II->isInsideBundle()) { 4122 Idx = II->findRegisterUseOperandIdx(Reg, false, TRI); 4123 if (Idx != -1) 4124 break; 4125 if (II->getOpcode() != ARM::t2IT) 4126 ++Dist; 4127 ++II; 4128 } 4129 4130 if (Idx == -1) { 4131 Dist = 0; 4132 return nullptr; 4133 } 4134 4135 UseIdx = Idx; 4136 return &*II; 4137 } 4138 4139 /// Return the number of cycles to add to (or subtract from) the static 4140 /// itinerary based on the def opcode and alignment. The caller will ensure that 4141 /// adjusted latency is at least one cycle. 4142 static int adjustDefLatency(const ARMSubtarget &Subtarget, 4143 const MachineInstr &DefMI, 4144 const MCInstrDesc &DefMCID, unsigned DefAlign) { 4145 int Adjust = 0; 4146 if (Subtarget.isCortexA8() || Subtarget.isLikeA9() || Subtarget.isCortexA7()) { 4147 // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2] 4148 // variants are one cycle cheaper. 4149 switch (DefMCID.getOpcode()) { 4150 default: break; 4151 case ARM::LDRrs: 4152 case ARM::LDRBrs: { 4153 unsigned ShOpVal = DefMI.getOperand(3).getImm(); 4154 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 4155 if (ShImm == 0 || 4156 (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)) 4157 --Adjust; 4158 break; 4159 } 4160 case ARM::t2LDRs: 4161 case ARM::t2LDRBs: 4162 case ARM::t2LDRHs: 4163 case ARM::t2LDRSHs: { 4164 // Thumb2 mode: lsl only. 4165 unsigned ShAmt = DefMI.getOperand(3).getImm(); 4166 if (ShAmt == 0 || ShAmt == 2) 4167 --Adjust; 4168 break; 4169 } 4170 } 4171 } else if (Subtarget.isSwift()) { 4172 // FIXME: Properly handle all of the latency adjustments for address 4173 // writeback. 4174 switch (DefMCID.getOpcode()) { 4175 default: break; 4176 case ARM::LDRrs: 4177 case ARM::LDRBrs: { 4178 unsigned ShOpVal = DefMI.getOperand(3).getImm(); 4179 bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub; 4180 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 4181 if (!isSub && 4182 (ShImm == 0 || 4183 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 4184 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))) 4185 Adjust -= 2; 4186 else if (!isSub && 4187 ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr) 4188 --Adjust; 4189 break; 4190 } 4191 case ARM::t2LDRs: 4192 case ARM::t2LDRBs: 4193 case ARM::t2LDRHs: 4194 case ARM::t2LDRSHs: { 4195 // Thumb2 mode: lsl only. 4196 unsigned ShAmt = DefMI.getOperand(3).getImm(); 4197 if (ShAmt == 0 || ShAmt == 1 || ShAmt == 2 || ShAmt == 3) 4198 Adjust -= 2; 4199 break; 4200 } 4201 } 4202 } 4203 4204 if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment()) { 4205 switch (DefMCID.getOpcode()) { 4206 default: break; 4207 case ARM::VLD1q8: 4208 case ARM::VLD1q16: 4209 case ARM::VLD1q32: 4210 case ARM::VLD1q64: 4211 case ARM::VLD1q8wb_fixed: 4212 case ARM::VLD1q16wb_fixed: 4213 case ARM::VLD1q32wb_fixed: 4214 case ARM::VLD1q64wb_fixed: 4215 case ARM::VLD1q8wb_register: 4216 case ARM::VLD1q16wb_register: 4217 case ARM::VLD1q32wb_register: 4218 case ARM::VLD1q64wb_register: 4219 case ARM::VLD2d8: 4220 case ARM::VLD2d16: 4221 case ARM::VLD2d32: 4222 case ARM::VLD2q8: 4223 case ARM::VLD2q16: 4224 case ARM::VLD2q32: 4225 case ARM::VLD2d8wb_fixed: 4226 case ARM::VLD2d16wb_fixed: 4227 case ARM::VLD2d32wb_fixed: 4228 case ARM::VLD2q8wb_fixed: 4229 case ARM::VLD2q16wb_fixed: 4230 case ARM::VLD2q32wb_fixed: 4231 case ARM::VLD2d8wb_register: 4232 case ARM::VLD2d16wb_register: 4233 case ARM::VLD2d32wb_register: 4234 case ARM::VLD2q8wb_register: 4235 case ARM::VLD2q16wb_register: 4236 case ARM::VLD2q32wb_register: 4237 case ARM::VLD3d8: 4238 case ARM::VLD3d16: 4239 case ARM::VLD3d32: 4240 case ARM::VLD1d64T: 4241 case ARM::VLD3d8_UPD: 4242 case ARM::VLD3d16_UPD: 4243 case ARM::VLD3d32_UPD: 4244 case ARM::VLD1d64Twb_fixed: 4245 case ARM::VLD1d64Twb_register: 4246 case ARM::VLD3q8_UPD: 4247 case ARM::VLD3q16_UPD: 4248 case ARM::VLD3q32_UPD: 4249 case ARM::VLD4d8: 4250 case ARM::VLD4d16: 4251 case ARM::VLD4d32: 4252 case ARM::VLD1d64Q: 4253 case ARM::VLD4d8_UPD: 4254 case ARM::VLD4d16_UPD: 4255 case ARM::VLD4d32_UPD: 4256 case ARM::VLD1d64Qwb_fixed: 4257 case ARM::VLD1d64Qwb_register: 4258 case ARM::VLD4q8_UPD: 4259 case ARM::VLD4q16_UPD: 4260 case ARM::VLD4q32_UPD: 4261 case ARM::VLD1DUPq8: 4262 case ARM::VLD1DUPq16: 4263 case ARM::VLD1DUPq32: 4264 case ARM::VLD1DUPq8wb_fixed: 4265 case ARM::VLD1DUPq16wb_fixed: 4266 case ARM::VLD1DUPq32wb_fixed: 4267 case ARM::VLD1DUPq8wb_register: 4268 case ARM::VLD1DUPq16wb_register: 4269 case ARM::VLD1DUPq32wb_register: 4270 case ARM::VLD2DUPd8: 4271 case ARM::VLD2DUPd16: 4272 case ARM::VLD2DUPd32: 4273 case ARM::VLD2DUPd8wb_fixed: 4274 case ARM::VLD2DUPd16wb_fixed: 4275 case ARM::VLD2DUPd32wb_fixed: 4276 case ARM::VLD2DUPd8wb_register: 4277 case ARM::VLD2DUPd16wb_register: 4278 case ARM::VLD2DUPd32wb_register: 4279 case ARM::VLD4DUPd8: 4280 case ARM::VLD4DUPd16: 4281 case ARM::VLD4DUPd32: 4282 case ARM::VLD4DUPd8_UPD: 4283 case ARM::VLD4DUPd16_UPD: 4284 case ARM::VLD4DUPd32_UPD: 4285 case ARM::VLD1LNd8: 4286 case ARM::VLD1LNd16: 4287 case ARM::VLD1LNd32: 4288 case ARM::VLD1LNd8_UPD: 4289 case ARM::VLD1LNd16_UPD: 4290 case ARM::VLD1LNd32_UPD: 4291 case ARM::VLD2LNd8: 4292 case ARM::VLD2LNd16: 4293 case ARM::VLD2LNd32: 4294 case ARM::VLD2LNq16: 4295 case ARM::VLD2LNq32: 4296 case ARM::VLD2LNd8_UPD: 4297 case ARM::VLD2LNd16_UPD: 4298 case ARM::VLD2LNd32_UPD: 4299 case ARM::VLD2LNq16_UPD: 4300 case ARM::VLD2LNq32_UPD: 4301 case ARM::VLD4LNd8: 4302 case ARM::VLD4LNd16: 4303 case ARM::VLD4LNd32: 4304 case ARM::VLD4LNq16: 4305 case ARM::VLD4LNq32: 4306 case ARM::VLD4LNd8_UPD: 4307 case ARM::VLD4LNd16_UPD: 4308 case ARM::VLD4LNd32_UPD: 4309 case ARM::VLD4LNq16_UPD: 4310 case ARM::VLD4LNq32_UPD: 4311 // If the address is not 64-bit aligned, the latencies of these 4312 // instructions increases by one. 4313 ++Adjust; 4314 break; 4315 } 4316 } 4317 return Adjust; 4318 } 4319 4320 int ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, 4321 const MachineInstr &DefMI, 4322 unsigned DefIdx, 4323 const MachineInstr &UseMI, 4324 unsigned UseIdx) const { 4325 // No operand latency. The caller may fall back to getInstrLatency. 4326 if (!ItinData || ItinData->isEmpty()) 4327 return -1; 4328 4329 const MachineOperand &DefMO = DefMI.getOperand(DefIdx); 4330 Register Reg = DefMO.getReg(); 4331 4332 const MachineInstr *ResolvedDefMI = &DefMI; 4333 unsigned DefAdj = 0; 4334 if (DefMI.isBundle()) 4335 ResolvedDefMI = 4336 getBundledDefMI(&getRegisterInfo(), &DefMI, Reg, DefIdx, DefAdj); 4337 if (ResolvedDefMI->isCopyLike() || ResolvedDefMI->isInsertSubreg() || 4338 ResolvedDefMI->isRegSequence() || ResolvedDefMI->isImplicitDef()) { 4339 return 1; 4340 } 4341 4342 const MachineInstr *ResolvedUseMI = &UseMI; 4343 unsigned UseAdj = 0; 4344 if (UseMI.isBundle()) { 4345 ResolvedUseMI = 4346 getBundledUseMI(&getRegisterInfo(), UseMI, Reg, UseIdx, UseAdj); 4347 if (!ResolvedUseMI) 4348 return -1; 4349 } 4350 4351 return getOperandLatencyImpl( 4352 ItinData, *ResolvedDefMI, DefIdx, ResolvedDefMI->getDesc(), DefAdj, DefMO, 4353 Reg, *ResolvedUseMI, UseIdx, ResolvedUseMI->getDesc(), UseAdj); 4354 } 4355 4356 int ARMBaseInstrInfo::getOperandLatencyImpl( 4357 const InstrItineraryData *ItinData, const MachineInstr &DefMI, 4358 unsigned DefIdx, const MCInstrDesc &DefMCID, unsigned DefAdj, 4359 const MachineOperand &DefMO, unsigned Reg, const MachineInstr &UseMI, 4360 unsigned UseIdx, const MCInstrDesc &UseMCID, unsigned UseAdj) const { 4361 if (Reg == ARM::CPSR) { 4362 if (DefMI.getOpcode() == ARM::FMSTAT) { 4363 // fpscr -> cpsr stalls over 20 cycles on A8 (and earlier?) 4364 return Subtarget.isLikeA9() ? 1 : 20; 4365 } 4366 4367 // CPSR set and branch can be paired in the same cycle. 4368 if (UseMI.isBranch()) 4369 return 0; 4370 4371 // Otherwise it takes the instruction latency (generally one). 4372 unsigned Latency = getInstrLatency(ItinData, DefMI); 4373 4374 // For Thumb2 and -Os, prefer scheduling CPSR setting instruction close to 4375 // its uses. Instructions which are otherwise scheduled between them may 4376 // incur a code size penalty (not able to use the CPSR setting 16-bit 4377 // instructions). 4378 if (Latency > 0 && Subtarget.isThumb2()) { 4379 const MachineFunction *MF = DefMI.getParent()->getParent(); 4380 // FIXME: Use Function::hasOptSize(). 4381 if (MF->getFunction().hasFnAttribute(Attribute::OptimizeForSize)) 4382 --Latency; 4383 } 4384 return Latency; 4385 } 4386 4387 if (DefMO.isImplicit() || UseMI.getOperand(UseIdx).isImplicit()) 4388 return -1; 4389 4390 unsigned DefAlign = DefMI.hasOneMemOperand() 4391 ? (*DefMI.memoperands_begin())->getAlign().value() 4392 : 0; 4393 unsigned UseAlign = UseMI.hasOneMemOperand() 4394 ? (*UseMI.memoperands_begin())->getAlign().value() 4395 : 0; 4396 4397 // Get the itinerary's latency if possible, and handle variable_ops. 4398 int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign, UseMCID, 4399 UseIdx, UseAlign); 4400 // Unable to find operand latency. The caller may resort to getInstrLatency. 4401 if (Latency < 0) 4402 return Latency; 4403 4404 // Adjust for IT block position. 4405 int Adj = DefAdj + UseAdj; 4406 4407 // Adjust for dynamic def-side opcode variants not captured by the itinerary. 4408 Adj += adjustDefLatency(Subtarget, DefMI, DefMCID, DefAlign); 4409 if (Adj >= 0 || (int)Latency > -Adj) { 4410 return Latency + Adj; 4411 } 4412 // Return the itinerary latency, which may be zero but not less than zero. 4413 return Latency; 4414 } 4415 4416 int 4417 ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, 4418 SDNode *DefNode, unsigned DefIdx, 4419 SDNode *UseNode, unsigned UseIdx) const { 4420 if (!DefNode->isMachineOpcode()) 4421 return 1; 4422 4423 const MCInstrDesc &DefMCID = get(DefNode->getMachineOpcode()); 4424 4425 if (isZeroCost(DefMCID.Opcode)) 4426 return 0; 4427 4428 if (!ItinData || ItinData->isEmpty()) 4429 return DefMCID.mayLoad() ? 3 : 1; 4430 4431 if (!UseNode->isMachineOpcode()) { 4432 int Latency = ItinData->getOperandCycle(DefMCID.getSchedClass(), DefIdx); 4433 int Adj = Subtarget.getPreISelOperandLatencyAdjustment(); 4434 int Threshold = 1 + Adj; 4435 return Latency <= Threshold ? 1 : Latency - Adj; 4436 } 4437 4438 const MCInstrDesc &UseMCID = get(UseNode->getMachineOpcode()); 4439 auto *DefMN = cast<MachineSDNode>(DefNode); 4440 unsigned DefAlign = !DefMN->memoperands_empty() 4441 ? (*DefMN->memoperands_begin())->getAlign().value() 4442 : 0; 4443 auto *UseMN = cast<MachineSDNode>(UseNode); 4444 unsigned UseAlign = !UseMN->memoperands_empty() 4445 ? (*UseMN->memoperands_begin())->getAlign().value() 4446 : 0; 4447 int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign, 4448 UseMCID, UseIdx, UseAlign); 4449 4450 if (Latency > 1 && 4451 (Subtarget.isCortexA8() || Subtarget.isLikeA9() || 4452 Subtarget.isCortexA7())) { 4453 // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2] 4454 // variants are one cycle cheaper. 4455 switch (DefMCID.getOpcode()) { 4456 default: break; 4457 case ARM::LDRrs: 4458 case ARM::LDRBrs: { 4459 unsigned ShOpVal = 4460 cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue(); 4461 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 4462 if (ShImm == 0 || 4463 (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)) 4464 --Latency; 4465 break; 4466 } 4467 case ARM::t2LDRs: 4468 case ARM::t2LDRBs: 4469 case ARM::t2LDRHs: 4470 case ARM::t2LDRSHs: { 4471 // Thumb2 mode: lsl only. 4472 unsigned ShAmt = 4473 cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue(); 4474 if (ShAmt == 0 || ShAmt == 2) 4475 --Latency; 4476 break; 4477 } 4478 } 4479 } else if (DefIdx == 0 && Latency > 2 && Subtarget.isSwift()) { 4480 // FIXME: Properly handle all of the latency adjustments for address 4481 // writeback. 4482 switch (DefMCID.getOpcode()) { 4483 default: break; 4484 case ARM::LDRrs: 4485 case ARM::LDRBrs: { 4486 unsigned ShOpVal = 4487 cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue(); 4488 unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal); 4489 if (ShImm == 0 || 4490 ((ShImm == 1 || ShImm == 2 || ShImm == 3) && 4491 ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)) 4492 Latency -= 2; 4493 else if (ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr) 4494 --Latency; 4495 break; 4496 } 4497 case ARM::t2LDRs: 4498 case ARM::t2LDRBs: 4499 case ARM::t2LDRHs: 4500 case ARM::t2LDRSHs: 4501 // Thumb2 mode: lsl 0-3 only. 4502 Latency -= 2; 4503 break; 4504 } 4505 } 4506 4507 if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment()) 4508 switch (DefMCID.getOpcode()) { 4509 default: break; 4510 case ARM::VLD1q8: 4511 case ARM::VLD1q16: 4512 case ARM::VLD1q32: 4513 case ARM::VLD1q64: 4514 case ARM::VLD1q8wb_register: 4515 case ARM::VLD1q16wb_register: 4516 case ARM::VLD1q32wb_register: 4517 case ARM::VLD1q64wb_register: 4518 case ARM::VLD1q8wb_fixed: 4519 case ARM::VLD1q16wb_fixed: 4520 case ARM::VLD1q32wb_fixed: 4521 case ARM::VLD1q64wb_fixed: 4522 case ARM::VLD2d8: 4523 case ARM::VLD2d16: 4524 case ARM::VLD2d32: 4525 case ARM::VLD2q8Pseudo: 4526 case ARM::VLD2q16Pseudo: 4527 case ARM::VLD2q32Pseudo: 4528 case ARM::VLD2d8wb_fixed: 4529 case ARM::VLD2d16wb_fixed: 4530 case ARM::VLD2d32wb_fixed: 4531 case ARM::VLD2q8PseudoWB_fixed: 4532 case ARM::VLD2q16PseudoWB_fixed: 4533 case ARM::VLD2q32PseudoWB_fixed: 4534 case ARM::VLD2d8wb_register: 4535 case ARM::VLD2d16wb_register: 4536 case ARM::VLD2d32wb_register: 4537 case ARM::VLD2q8PseudoWB_register: 4538 case ARM::VLD2q16PseudoWB_register: 4539 case ARM::VLD2q32PseudoWB_register: 4540 case ARM::VLD3d8Pseudo: 4541 case ARM::VLD3d16Pseudo: 4542 case ARM::VLD3d32Pseudo: 4543 case ARM::VLD1d8TPseudo: 4544 case ARM::VLD1d16TPseudo: 4545 case ARM::VLD1d32TPseudo: 4546 case ARM::VLD1d64TPseudo: 4547 case ARM::VLD1d64TPseudoWB_fixed: 4548 case ARM::VLD1d64TPseudoWB_register: 4549 case ARM::VLD3d8Pseudo_UPD: 4550 case ARM::VLD3d16Pseudo_UPD: 4551 case ARM::VLD3d32Pseudo_UPD: 4552 case ARM::VLD3q8Pseudo_UPD: 4553 case ARM::VLD3q16Pseudo_UPD: 4554 case ARM::VLD3q32Pseudo_UPD: 4555 case ARM::VLD3q8oddPseudo: 4556 case ARM::VLD3q16oddPseudo: 4557 case ARM::VLD3q32oddPseudo: 4558 case ARM::VLD3q8oddPseudo_UPD: 4559 case ARM::VLD3q16oddPseudo_UPD: 4560 case ARM::VLD3q32oddPseudo_UPD: 4561 case ARM::VLD4d8Pseudo: 4562 case ARM::VLD4d16Pseudo: 4563 case ARM::VLD4d32Pseudo: 4564 case ARM::VLD1d8QPseudo: 4565 case ARM::VLD1d16QPseudo: 4566 case ARM::VLD1d32QPseudo: 4567 case ARM::VLD1d64QPseudo: 4568 case ARM::VLD1d64QPseudoWB_fixed: 4569 case ARM::VLD1d64QPseudoWB_register: 4570 case ARM::VLD1q8HighQPseudo: 4571 case ARM::VLD1q8LowQPseudo_UPD: 4572 case ARM::VLD1q8HighTPseudo: 4573 case ARM::VLD1q8LowTPseudo_UPD: 4574 case ARM::VLD1q16HighQPseudo: 4575 case ARM::VLD1q16LowQPseudo_UPD: 4576 case ARM::VLD1q16HighTPseudo: 4577 case ARM::VLD1q16LowTPseudo_UPD: 4578 case ARM::VLD1q32HighQPseudo: 4579 case ARM::VLD1q32LowQPseudo_UPD: 4580 case ARM::VLD1q32HighTPseudo: 4581 case ARM::VLD1q32LowTPseudo_UPD: 4582 case ARM::VLD1q64HighQPseudo: 4583 case ARM::VLD1q64LowQPseudo_UPD: 4584 case ARM::VLD1q64HighTPseudo: 4585 case ARM::VLD1q64LowTPseudo_UPD: 4586 case ARM::VLD4d8Pseudo_UPD: 4587 case ARM::VLD4d16Pseudo_UPD: 4588 case ARM::VLD4d32Pseudo_UPD: 4589 case ARM::VLD4q8Pseudo_UPD: 4590 case ARM::VLD4q16Pseudo_UPD: 4591 case ARM::VLD4q32Pseudo_UPD: 4592 case ARM::VLD4q8oddPseudo: 4593 case ARM::VLD4q16oddPseudo: 4594 case ARM::VLD4q32oddPseudo: 4595 case ARM::VLD4q8oddPseudo_UPD: 4596 case ARM::VLD4q16oddPseudo_UPD: 4597 case ARM::VLD4q32oddPseudo_UPD: 4598 case ARM::VLD1DUPq8: 4599 case ARM::VLD1DUPq16: 4600 case ARM::VLD1DUPq32: 4601 case ARM::VLD1DUPq8wb_fixed: 4602 case ARM::VLD1DUPq16wb_fixed: 4603 case ARM::VLD1DUPq32wb_fixed: 4604 case ARM::VLD1DUPq8wb_register: 4605 case ARM::VLD1DUPq16wb_register: 4606 case ARM::VLD1DUPq32wb_register: 4607 case ARM::VLD2DUPd8: 4608 case ARM::VLD2DUPd16: 4609 case ARM::VLD2DUPd32: 4610 case ARM::VLD2DUPd8wb_fixed: 4611 case ARM::VLD2DUPd16wb_fixed: 4612 case ARM::VLD2DUPd32wb_fixed: 4613 case ARM::VLD2DUPd8wb_register: 4614 case ARM::VLD2DUPd16wb_register: 4615 case ARM::VLD2DUPd32wb_register: 4616 case ARM::VLD2DUPq8EvenPseudo: 4617 case ARM::VLD2DUPq8OddPseudo: 4618 case ARM::VLD2DUPq16EvenPseudo: 4619 case ARM::VLD2DUPq16OddPseudo: 4620 case ARM::VLD2DUPq32EvenPseudo: 4621 case ARM::VLD2DUPq32OddPseudo: 4622 case ARM::VLD3DUPq8EvenPseudo: 4623 case ARM::VLD3DUPq8OddPseudo: 4624 case ARM::VLD3DUPq16EvenPseudo: 4625 case ARM::VLD3DUPq16OddPseudo: 4626 case ARM::VLD3DUPq32EvenPseudo: 4627 case ARM::VLD3DUPq32OddPseudo: 4628 case ARM::VLD4DUPd8Pseudo: 4629 case ARM::VLD4DUPd16Pseudo: 4630 case ARM::VLD4DUPd32Pseudo: 4631 case ARM::VLD4DUPd8Pseudo_UPD: 4632 case ARM::VLD4DUPd16Pseudo_UPD: 4633 case ARM::VLD4DUPd32Pseudo_UPD: 4634 case ARM::VLD4DUPq8EvenPseudo: 4635 case ARM::VLD4DUPq8OddPseudo: 4636 case ARM::VLD4DUPq16EvenPseudo: 4637 case ARM::VLD4DUPq16OddPseudo: 4638 case ARM::VLD4DUPq32EvenPseudo: 4639 case ARM::VLD4DUPq32OddPseudo: 4640 case ARM::VLD1LNq8Pseudo: 4641 case ARM::VLD1LNq16Pseudo: 4642 case ARM::VLD1LNq32Pseudo: 4643 case ARM::VLD1LNq8Pseudo_UPD: 4644 case ARM::VLD1LNq16Pseudo_UPD: 4645 case ARM::VLD1LNq32Pseudo_UPD: 4646 case ARM::VLD2LNd8Pseudo: 4647 case ARM::VLD2LNd16Pseudo: 4648 case ARM::VLD2LNd32Pseudo: 4649 case ARM::VLD2LNq16Pseudo: 4650 case ARM::VLD2LNq32Pseudo: 4651 case ARM::VLD2LNd8Pseudo_UPD: 4652 case ARM::VLD2LNd16Pseudo_UPD: 4653 case ARM::VLD2LNd32Pseudo_UPD: 4654 case ARM::VLD2LNq16Pseudo_UPD: 4655 case ARM::VLD2LNq32Pseudo_UPD: 4656 case ARM::VLD4LNd8Pseudo: 4657 case ARM::VLD4LNd16Pseudo: 4658 case ARM::VLD4LNd32Pseudo: 4659 case ARM::VLD4LNq16Pseudo: 4660 case ARM::VLD4LNq32Pseudo: 4661 case ARM::VLD4LNd8Pseudo_UPD: 4662 case ARM::VLD4LNd16Pseudo_UPD: 4663 case ARM::VLD4LNd32Pseudo_UPD: 4664 case ARM::VLD4LNq16Pseudo_UPD: 4665 case ARM::VLD4LNq32Pseudo_UPD: 4666 // If the address is not 64-bit aligned, the latencies of these 4667 // instructions increases by one. 4668 ++Latency; 4669 break; 4670 } 4671 4672 return Latency; 4673 } 4674 4675 unsigned ARMBaseInstrInfo::getPredicationCost(const MachineInstr &MI) const { 4676 if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() || 4677 MI.isImplicitDef()) 4678 return 0; 4679 4680 if (MI.isBundle()) 4681 return 0; 4682 4683 const MCInstrDesc &MCID = MI.getDesc(); 4684 4685 if (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) && 4686 !Subtarget.cheapPredicableCPSRDef())) { 4687 // When predicated, CPSR is an additional source operand for CPSR updating 4688 // instructions, this apparently increases their latencies. 4689 return 1; 4690 } 4691 return 0; 4692 } 4693 4694 unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 4695 const MachineInstr &MI, 4696 unsigned *PredCost) const { 4697 if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() || 4698 MI.isImplicitDef()) 4699 return 1; 4700 4701 // An instruction scheduler typically runs on unbundled instructions, however 4702 // other passes may query the latency of a bundled instruction. 4703 if (MI.isBundle()) { 4704 unsigned Latency = 0; 4705 MachineBasicBlock::const_instr_iterator I = MI.getIterator(); 4706 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end(); 4707 while (++I != E && I->isInsideBundle()) { 4708 if (I->getOpcode() != ARM::t2IT) 4709 Latency += getInstrLatency(ItinData, *I, PredCost); 4710 } 4711 return Latency; 4712 } 4713 4714 const MCInstrDesc &MCID = MI.getDesc(); 4715 if (PredCost && (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) && 4716 !Subtarget.cheapPredicableCPSRDef()))) { 4717 // When predicated, CPSR is an additional source operand for CPSR updating 4718 // instructions, this apparently increases their latencies. 4719 *PredCost = 1; 4720 } 4721 // Be sure to call getStageLatency for an empty itinerary in case it has a 4722 // valid MinLatency property. 4723 if (!ItinData) 4724 return MI.mayLoad() ? 3 : 1; 4725 4726 unsigned Class = MCID.getSchedClass(); 4727 4728 // For instructions with variable uops, use uops as latency. 4729 if (!ItinData->isEmpty() && ItinData->getNumMicroOps(Class) < 0) 4730 return getNumMicroOps(ItinData, MI); 4731 4732 // For the common case, fall back on the itinerary's latency. 4733 unsigned Latency = ItinData->getStageLatency(Class); 4734 4735 // Adjust for dynamic def-side opcode variants not captured by the itinerary. 4736 unsigned DefAlign = 4737 MI.hasOneMemOperand() ? (*MI.memoperands_begin())->getAlign().value() : 0; 4738 int Adj = adjustDefLatency(Subtarget, MI, MCID, DefAlign); 4739 if (Adj >= 0 || (int)Latency > -Adj) { 4740 return Latency + Adj; 4741 } 4742 return Latency; 4743 } 4744 4745 int ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 4746 SDNode *Node) const { 4747 if (!Node->isMachineOpcode()) 4748 return 1; 4749 4750 if (!ItinData || ItinData->isEmpty()) 4751 return 1; 4752 4753 unsigned Opcode = Node->getMachineOpcode(); 4754 switch (Opcode) { 4755 default: 4756 return ItinData->getStageLatency(get(Opcode).getSchedClass()); 4757 case ARM::VLDMQIA: 4758 case ARM::VSTMQIA: 4759 return 2; 4760 } 4761 } 4762 4763 bool ARMBaseInstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel, 4764 const MachineRegisterInfo *MRI, 4765 const MachineInstr &DefMI, 4766 unsigned DefIdx, 4767 const MachineInstr &UseMI, 4768 unsigned UseIdx) const { 4769 unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask; 4770 unsigned UDomain = UseMI.getDesc().TSFlags & ARMII::DomainMask; 4771 if (Subtarget.nonpipelinedVFP() && 4772 (DDomain == ARMII::DomainVFP || UDomain == ARMII::DomainVFP)) 4773 return true; 4774 4775 // Hoist VFP / NEON instructions with 4 or higher latency. 4776 unsigned Latency = 4777 SchedModel.computeOperandLatency(&DefMI, DefIdx, &UseMI, UseIdx); 4778 if (Latency <= 3) 4779 return false; 4780 return DDomain == ARMII::DomainVFP || DDomain == ARMII::DomainNEON || 4781 UDomain == ARMII::DomainVFP || UDomain == ARMII::DomainNEON; 4782 } 4783 4784 bool ARMBaseInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel, 4785 const MachineInstr &DefMI, 4786 unsigned DefIdx) const { 4787 const InstrItineraryData *ItinData = SchedModel.getInstrItineraries(); 4788 if (!ItinData || ItinData->isEmpty()) 4789 return false; 4790 4791 unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask; 4792 if (DDomain == ARMII::DomainGeneral) { 4793 unsigned DefClass = DefMI.getDesc().getSchedClass(); 4794 int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx); 4795 return (DefCycle != -1 && DefCycle <= 2); 4796 } 4797 return false; 4798 } 4799 4800 bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr &MI, 4801 StringRef &ErrInfo) const { 4802 if (convertAddSubFlagsOpcode(MI.getOpcode())) { 4803 ErrInfo = "Pseudo flag setting opcodes only exist in Selection DAG"; 4804 return false; 4805 } 4806 if (MI.getOpcode() == ARM::tMOVr && !Subtarget.hasV6Ops()) { 4807 // Make sure we don't generate a lo-lo mov that isn't supported. 4808 if (!ARM::hGPRRegClass.contains(MI.getOperand(0).getReg()) && 4809 !ARM::hGPRRegClass.contains(MI.getOperand(1).getReg())) { 4810 ErrInfo = "Non-flag-setting Thumb1 mov is v6-only"; 4811 return false; 4812 } 4813 } 4814 if (MI.getOpcode() == ARM::tPUSH || 4815 MI.getOpcode() == ARM::tPOP || 4816 MI.getOpcode() == ARM::tPOP_RET) { 4817 for (int i = 2, e = MI.getNumOperands(); i < e; ++i) { 4818 if (MI.getOperand(i).isImplicit() || 4819 !MI.getOperand(i).isReg()) 4820 continue; 4821 Register Reg = MI.getOperand(i).getReg(); 4822 if (Reg < ARM::R0 || Reg > ARM::R7) { 4823 if (!(MI.getOpcode() == ARM::tPUSH && Reg == ARM::LR) && 4824 !(MI.getOpcode() == ARM::tPOP_RET && Reg == ARM::PC)) { 4825 ErrInfo = "Unsupported register in Thumb1 push/pop"; 4826 return false; 4827 } 4828 } 4829 } 4830 } 4831 if (MI.getOpcode() == ARM::MVE_VMOV_q_rr) { 4832 assert(MI.getOperand(4).isImm() && MI.getOperand(5).isImm()); 4833 if ((MI.getOperand(4).getImm() != 2 && MI.getOperand(4).getImm() != 3) || 4834 MI.getOperand(4).getImm() != MI.getOperand(5).getImm() + 2) { 4835 ErrInfo = "Incorrect array index for MVE_VMOV_q_rr"; 4836 return false; 4837 } 4838 } 4839 return true; 4840 } 4841 4842 // LoadStackGuard has so far only been implemented for MachO. Different code 4843 // sequence is needed for other targets. 4844 void ARMBaseInstrInfo::expandLoadStackGuardBase(MachineBasicBlock::iterator MI, 4845 unsigned LoadImmOpc, 4846 unsigned LoadOpc) const { 4847 assert(!Subtarget.isROPI() && !Subtarget.isRWPI() && 4848 "ROPI/RWPI not currently supported with stack guard"); 4849 4850 MachineBasicBlock &MBB = *MI->getParent(); 4851 DebugLoc DL = MI->getDebugLoc(); 4852 Register Reg = MI->getOperand(0).getReg(); 4853 const GlobalValue *GV = 4854 cast<GlobalValue>((*MI->memoperands_begin())->getValue()); 4855 MachineInstrBuilder MIB; 4856 4857 BuildMI(MBB, MI, DL, get(LoadImmOpc), Reg) 4858 .addGlobalAddress(GV, 0, ARMII::MO_NONLAZY); 4859 4860 if (Subtarget.isGVIndirectSymbol(GV)) { 4861 MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg); 4862 MIB.addReg(Reg, RegState::Kill).addImm(0); 4863 auto Flags = MachineMemOperand::MOLoad | 4864 MachineMemOperand::MODereferenceable | 4865 MachineMemOperand::MOInvariant; 4866 MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( 4867 MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, Align(4)); 4868 MIB.addMemOperand(MMO).add(predOps(ARMCC::AL)); 4869 } 4870 4871 MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg); 4872 MIB.addReg(Reg, RegState::Kill) 4873 .addImm(0) 4874 .cloneMemRefs(*MI) 4875 .add(predOps(ARMCC::AL)); 4876 } 4877 4878 bool 4879 ARMBaseInstrInfo::isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc, 4880 unsigned &AddSubOpc, 4881 bool &NegAcc, bool &HasLane) const { 4882 DenseMap<unsigned, unsigned>::const_iterator I = MLxEntryMap.find(Opcode); 4883 if (I == MLxEntryMap.end()) 4884 return false; 4885 4886 const ARM_MLxEntry &Entry = ARM_MLxTable[I->second]; 4887 MulOpc = Entry.MulOpc; 4888 AddSubOpc = Entry.AddSubOpc; 4889 NegAcc = Entry.NegAcc; 4890 HasLane = Entry.HasLane; 4891 return true; 4892 } 4893 4894 //===----------------------------------------------------------------------===// 4895 // Execution domains. 4896 //===----------------------------------------------------------------------===// 4897 // 4898 // Some instructions go down the NEON pipeline, some go down the VFP pipeline, 4899 // and some can go down both. The vmov instructions go down the VFP pipeline, 4900 // but they can be changed to vorr equivalents that are executed by the NEON 4901 // pipeline. 4902 // 4903 // We use the following execution domain numbering: 4904 // 4905 enum ARMExeDomain { 4906 ExeGeneric = 0, 4907 ExeVFP = 1, 4908 ExeNEON = 2 4909 }; 4910 4911 // 4912 // Also see ARMInstrFormats.td and Domain* enums in ARMBaseInfo.h 4913 // 4914 std::pair<uint16_t, uint16_t> 4915 ARMBaseInstrInfo::getExecutionDomain(const MachineInstr &MI) const { 4916 // If we don't have access to NEON instructions then we won't be able 4917 // to swizzle anything to the NEON domain. Check to make sure. 4918 if (Subtarget.hasNEON()) { 4919 // VMOVD, VMOVRS and VMOVSR are VFP instructions, but can be changed to NEON 4920 // if they are not predicated. 4921 if (MI.getOpcode() == ARM::VMOVD && !isPredicated(MI)) 4922 return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON)); 4923 4924 // CortexA9 is particularly picky about mixing the two and wants these 4925 // converted. 4926 if (Subtarget.useNEONForFPMovs() && !isPredicated(MI) && 4927 (MI.getOpcode() == ARM::VMOVRS || MI.getOpcode() == ARM::VMOVSR || 4928 MI.getOpcode() == ARM::VMOVS)) 4929 return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON)); 4930 } 4931 // No other instructions can be swizzled, so just determine their domain. 4932 unsigned Domain = MI.getDesc().TSFlags & ARMII::DomainMask; 4933 4934 if (Domain & ARMII::DomainNEON) 4935 return std::make_pair(ExeNEON, 0); 4936 4937 // Certain instructions can go either way on Cortex-A8. 4938 // Treat them as NEON instructions. 4939 if ((Domain & ARMII::DomainNEONA8) && Subtarget.isCortexA8()) 4940 return std::make_pair(ExeNEON, 0); 4941 4942 if (Domain & ARMII::DomainVFP) 4943 return std::make_pair(ExeVFP, 0); 4944 4945 return std::make_pair(ExeGeneric, 0); 4946 } 4947 4948 static unsigned getCorrespondingDRegAndLane(const TargetRegisterInfo *TRI, 4949 unsigned SReg, unsigned &Lane) { 4950 unsigned DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_0, &ARM::DPRRegClass); 4951 Lane = 0; 4952 4953 if (DReg != ARM::NoRegister) 4954 return DReg; 4955 4956 Lane = 1; 4957 DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_1, &ARM::DPRRegClass); 4958 4959 assert(DReg && "S-register with no D super-register?"); 4960 return DReg; 4961 } 4962 4963 /// getImplicitSPRUseForDPRUse - Given a use of a DPR register and lane, 4964 /// set ImplicitSReg to a register number that must be marked as implicit-use or 4965 /// zero if no register needs to be defined as implicit-use. 4966 /// 4967 /// If the function cannot determine if an SPR should be marked implicit use or 4968 /// not, it returns false. 4969 /// 4970 /// This function handles cases where an instruction is being modified from taking 4971 /// an SPR to a DPR[Lane]. A use of the DPR is being added, which may conflict 4972 /// with an earlier def of an SPR corresponding to DPR[Lane^1] (i.e. the other 4973 /// lane of the DPR). 4974 /// 4975 /// If the other SPR is defined, an implicit-use of it should be added. Else, 4976 /// (including the case where the DPR itself is defined), it should not. 4977 /// 4978 static bool getImplicitSPRUseForDPRUse(const TargetRegisterInfo *TRI, 4979 MachineInstr &MI, unsigned DReg, 4980 unsigned Lane, unsigned &ImplicitSReg) { 4981 // If the DPR is defined or used already, the other SPR lane will be chained 4982 // correctly, so there is nothing to be done. 4983 if (MI.definesRegister(DReg, TRI) || MI.readsRegister(DReg, TRI)) { 4984 ImplicitSReg = 0; 4985 return true; 4986 } 4987 4988 // Otherwise we need to go searching to see if the SPR is set explicitly. 4989 ImplicitSReg = TRI->getSubReg(DReg, 4990 (Lane & 1) ? ARM::ssub_0 : ARM::ssub_1); 4991 MachineBasicBlock::LivenessQueryResult LQR = 4992 MI.getParent()->computeRegisterLiveness(TRI, ImplicitSReg, MI); 4993 4994 if (LQR == MachineBasicBlock::LQR_Live) 4995 return true; 4996 else if (LQR == MachineBasicBlock::LQR_Unknown) 4997 return false; 4998 4999 // If the register is known not to be live, there is no need to add an 5000 // implicit-use. 5001 ImplicitSReg = 0; 5002 return true; 5003 } 5004 5005 void ARMBaseInstrInfo::setExecutionDomain(MachineInstr &MI, 5006 unsigned Domain) const { 5007 unsigned DstReg, SrcReg, DReg; 5008 unsigned Lane; 5009 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI); 5010 const TargetRegisterInfo *TRI = &getRegisterInfo(); 5011 switch (MI.getOpcode()) { 5012 default: 5013 llvm_unreachable("cannot handle opcode!"); 5014 break; 5015 case ARM::VMOVD: 5016 if (Domain != ExeNEON) 5017 break; 5018 5019 // Zap the predicate operands. 5020 assert(!isPredicated(MI) && "Cannot predicate a VORRd"); 5021 5022 // Make sure we've got NEON instructions. 5023 assert(Subtarget.hasNEON() && "VORRd requires NEON"); 5024 5025 // Source instruction is %DDst = VMOVD %DSrc, 14, %noreg (; implicits) 5026 DstReg = MI.getOperand(0).getReg(); 5027 SrcReg = MI.getOperand(1).getReg(); 5028 5029 for (unsigned i = MI.getDesc().getNumOperands(); i; --i) 5030 MI.RemoveOperand(i - 1); 5031 5032 // Change to a %DDst = VORRd %DSrc, %DSrc, 14, %noreg (; implicits) 5033 MI.setDesc(get(ARM::VORRd)); 5034 MIB.addReg(DstReg, RegState::Define) 5035 .addReg(SrcReg) 5036 .addReg(SrcReg) 5037 .add(predOps(ARMCC::AL)); 5038 break; 5039 case ARM::VMOVRS: 5040 if (Domain != ExeNEON) 5041 break; 5042 assert(!isPredicated(MI) && "Cannot predicate a VGETLN"); 5043 5044 // Source instruction is %RDst = VMOVRS %SSrc, 14, %noreg (; implicits) 5045 DstReg = MI.getOperand(0).getReg(); 5046 SrcReg = MI.getOperand(1).getReg(); 5047 5048 for (unsigned i = MI.getDesc().getNumOperands(); i; --i) 5049 MI.RemoveOperand(i - 1); 5050 5051 DReg = getCorrespondingDRegAndLane(TRI, SrcReg, Lane); 5052 5053 // Convert to %RDst = VGETLNi32 %DSrc, Lane, 14, %noreg (; imps) 5054 // Note that DSrc has been widened and the other lane may be undef, which 5055 // contaminates the entire register. 5056 MI.setDesc(get(ARM::VGETLNi32)); 5057 MIB.addReg(DstReg, RegState::Define) 5058 .addReg(DReg, RegState::Undef) 5059 .addImm(Lane) 5060 .add(predOps(ARMCC::AL)); 5061 5062 // The old source should be an implicit use, otherwise we might think it 5063 // was dead before here. 5064 MIB.addReg(SrcReg, RegState::Implicit); 5065 break; 5066 case ARM::VMOVSR: { 5067 if (Domain != ExeNEON) 5068 break; 5069 assert(!isPredicated(MI) && "Cannot predicate a VSETLN"); 5070 5071 // Source instruction is %SDst = VMOVSR %RSrc, 14, %noreg (; implicits) 5072 DstReg = MI.getOperand(0).getReg(); 5073 SrcReg = MI.getOperand(1).getReg(); 5074 5075 DReg = getCorrespondingDRegAndLane(TRI, DstReg, Lane); 5076 5077 unsigned ImplicitSReg; 5078 if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg, Lane, ImplicitSReg)) 5079 break; 5080 5081 for (unsigned i = MI.getDesc().getNumOperands(); i; --i) 5082 MI.RemoveOperand(i - 1); 5083 5084 // Convert to %DDst = VSETLNi32 %DDst, %RSrc, Lane, 14, %noreg (; imps) 5085 // Again DDst may be undefined at the beginning of this instruction. 5086 MI.setDesc(get(ARM::VSETLNi32)); 5087 MIB.addReg(DReg, RegState::Define) 5088 .addReg(DReg, getUndefRegState(!MI.readsRegister(DReg, TRI))) 5089 .addReg(SrcReg) 5090 .addImm(Lane) 5091 .add(predOps(ARMCC::AL)); 5092 5093 // The narrower destination must be marked as set to keep previous chains 5094 // in place. 5095 MIB.addReg(DstReg, RegState::Define | RegState::Implicit); 5096 if (ImplicitSReg != 0) 5097 MIB.addReg(ImplicitSReg, RegState::Implicit); 5098 break; 5099 } 5100 case ARM::VMOVS: { 5101 if (Domain != ExeNEON) 5102 break; 5103 5104 // Source instruction is %SDst = VMOVS %SSrc, 14, %noreg (; implicits) 5105 DstReg = MI.getOperand(0).getReg(); 5106 SrcReg = MI.getOperand(1).getReg(); 5107 5108 unsigned DstLane = 0, SrcLane = 0, DDst, DSrc; 5109 DDst = getCorrespondingDRegAndLane(TRI, DstReg, DstLane); 5110 DSrc = getCorrespondingDRegAndLane(TRI, SrcReg, SrcLane); 5111 5112 unsigned ImplicitSReg; 5113 if (!getImplicitSPRUseForDPRUse(TRI, MI, DSrc, SrcLane, ImplicitSReg)) 5114 break; 5115 5116 for (unsigned i = MI.getDesc().getNumOperands(); i; --i) 5117 MI.RemoveOperand(i - 1); 5118 5119 if (DSrc == DDst) { 5120 // Destination can be: 5121 // %DDst = VDUPLN32d %DDst, Lane, 14, %noreg (; implicits) 5122 MI.setDesc(get(ARM::VDUPLN32d)); 5123 MIB.addReg(DDst, RegState::Define) 5124 .addReg(DDst, getUndefRegState(!MI.readsRegister(DDst, TRI))) 5125 .addImm(SrcLane) 5126 .add(predOps(ARMCC::AL)); 5127 5128 // Neither the source or the destination are naturally represented any 5129 // more, so add them in manually. 5130 MIB.addReg(DstReg, RegState::Implicit | RegState::Define); 5131 MIB.addReg(SrcReg, RegState::Implicit); 5132 if (ImplicitSReg != 0) 5133 MIB.addReg(ImplicitSReg, RegState::Implicit); 5134 break; 5135 } 5136 5137 // In general there's no single instruction that can perform an S <-> S 5138 // move in NEON space, but a pair of VEXT instructions *can* do the 5139 // job. It turns out that the VEXTs needed will only use DSrc once, with 5140 // the position based purely on the combination of lane-0 and lane-1 5141 // involved. For example 5142 // vmov s0, s2 -> vext.32 d0, d0, d1, #1 vext.32 d0, d0, d0, #1 5143 // vmov s1, s3 -> vext.32 d0, d1, d0, #1 vext.32 d0, d0, d0, #1 5144 // vmov s0, s3 -> vext.32 d0, d0, d0, #1 vext.32 d0, d1, d0, #1 5145 // vmov s1, s2 -> vext.32 d0, d0, d0, #1 vext.32 d0, d0, d1, #1 5146 // 5147 // Pattern of the MachineInstrs is: 5148 // %DDst = VEXTd32 %DSrc1, %DSrc2, Lane, 14, %noreg (;implicits) 5149 MachineInstrBuilder NewMIB; 5150 NewMIB = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(ARM::VEXTd32), 5151 DDst); 5152 5153 // On the first instruction, both DSrc and DDst may be undef if present. 5154 // Specifically when the original instruction didn't have them as an 5155 // <imp-use>. 5156 unsigned CurReg = SrcLane == 1 && DstLane == 1 ? DSrc : DDst; 5157 bool CurUndef = !MI.readsRegister(CurReg, TRI); 5158 NewMIB.addReg(CurReg, getUndefRegState(CurUndef)); 5159 5160 CurReg = SrcLane == 0 && DstLane == 0 ? DSrc : DDst; 5161 CurUndef = !MI.readsRegister(CurReg, TRI); 5162 NewMIB.addReg(CurReg, getUndefRegState(CurUndef)) 5163 .addImm(1) 5164 .add(predOps(ARMCC::AL)); 5165 5166 if (SrcLane == DstLane) 5167 NewMIB.addReg(SrcReg, RegState::Implicit); 5168 5169 MI.setDesc(get(ARM::VEXTd32)); 5170 MIB.addReg(DDst, RegState::Define); 5171 5172 // On the second instruction, DDst has definitely been defined above, so 5173 // it is not undef. DSrc, if present, can be undef as above. 5174 CurReg = SrcLane == 1 && DstLane == 0 ? DSrc : DDst; 5175 CurUndef = CurReg == DSrc && !MI.readsRegister(CurReg, TRI); 5176 MIB.addReg(CurReg, getUndefRegState(CurUndef)); 5177 5178 CurReg = SrcLane == 0 && DstLane == 1 ? DSrc : DDst; 5179 CurUndef = CurReg == DSrc && !MI.readsRegister(CurReg, TRI); 5180 MIB.addReg(CurReg, getUndefRegState(CurUndef)) 5181 .addImm(1) 5182 .add(predOps(ARMCC::AL)); 5183 5184 if (SrcLane != DstLane) 5185 MIB.addReg(SrcReg, RegState::Implicit); 5186 5187 // As before, the original destination is no longer represented, add it 5188 // implicitly. 5189 MIB.addReg(DstReg, RegState::Define | RegState::Implicit); 5190 if (ImplicitSReg != 0) 5191 MIB.addReg(ImplicitSReg, RegState::Implicit); 5192 break; 5193 } 5194 } 5195 } 5196 5197 //===----------------------------------------------------------------------===// 5198 // Partial register updates 5199 //===----------------------------------------------------------------------===// 5200 // 5201 // Swift renames NEON registers with 64-bit granularity. That means any 5202 // instruction writing an S-reg implicitly reads the containing D-reg. The 5203 // problem is mostly avoided by translating f32 operations to v2f32 operations 5204 // on D-registers, but f32 loads are still a problem. 5205 // 5206 // These instructions can load an f32 into a NEON register: 5207 // 5208 // VLDRS - Only writes S, partial D update. 5209 // VLD1LNd32 - Writes all D-regs, explicit partial D update, 2 uops. 5210 // VLD1DUPd32 - Writes all D-regs, no partial reg update, 2 uops. 5211 // 5212 // FCONSTD can be used as a dependency-breaking instruction. 5213 unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance( 5214 const MachineInstr &MI, unsigned OpNum, 5215 const TargetRegisterInfo *TRI) const { 5216 auto PartialUpdateClearance = Subtarget.getPartialUpdateClearance(); 5217 if (!PartialUpdateClearance) 5218 return 0; 5219 5220 assert(TRI && "Need TRI instance"); 5221 5222 const MachineOperand &MO = MI.getOperand(OpNum); 5223 if (MO.readsReg()) 5224 return 0; 5225 Register Reg = MO.getReg(); 5226 int UseOp = -1; 5227 5228 switch (MI.getOpcode()) { 5229 // Normal instructions writing only an S-register. 5230 case ARM::VLDRS: 5231 case ARM::FCONSTS: 5232 case ARM::VMOVSR: 5233 case ARM::VMOVv8i8: 5234 case ARM::VMOVv4i16: 5235 case ARM::VMOVv2i32: 5236 case ARM::VMOVv2f32: 5237 case ARM::VMOVv1i64: 5238 UseOp = MI.findRegisterUseOperandIdx(Reg, false, TRI); 5239 break; 5240 5241 // Explicitly reads the dependency. 5242 case ARM::VLD1LNd32: 5243 UseOp = 3; 5244 break; 5245 default: 5246 return 0; 5247 } 5248 5249 // If this instruction actually reads a value from Reg, there is no unwanted 5250 // dependency. 5251 if (UseOp != -1 && MI.getOperand(UseOp).readsReg()) 5252 return 0; 5253 5254 // We must be able to clobber the whole D-reg. 5255 if (Register::isVirtualRegister(Reg)) { 5256 // Virtual register must be a def undef foo:ssub_0 operand. 5257 if (!MO.getSubReg() || MI.readsVirtualRegister(Reg)) 5258 return 0; 5259 } else if (ARM::SPRRegClass.contains(Reg)) { 5260 // Physical register: MI must define the full D-reg. 5261 unsigned DReg = TRI->getMatchingSuperReg(Reg, ARM::ssub_0, 5262 &ARM::DPRRegClass); 5263 if (!DReg || !MI.definesRegister(DReg, TRI)) 5264 return 0; 5265 } 5266 5267 // MI has an unwanted D-register dependency. 5268 // Avoid defs in the previous N instructrions. 5269 return PartialUpdateClearance; 5270 } 5271 5272 // Break a partial register dependency after getPartialRegUpdateClearance 5273 // returned non-zero. 5274 void ARMBaseInstrInfo::breakPartialRegDependency( 5275 MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const { 5276 assert(OpNum < MI.getDesc().getNumDefs() && "OpNum is not a def"); 5277 assert(TRI && "Need TRI instance"); 5278 5279 const MachineOperand &MO = MI.getOperand(OpNum); 5280 Register Reg = MO.getReg(); 5281 assert(Register::isPhysicalRegister(Reg) && 5282 "Can't break virtual register dependencies."); 5283 unsigned DReg = Reg; 5284 5285 // If MI defines an S-reg, find the corresponding D super-register. 5286 if (ARM::SPRRegClass.contains(Reg)) { 5287 DReg = ARM::D0 + (Reg - ARM::S0) / 2; 5288 assert(TRI->isSuperRegister(Reg, DReg) && "Register enums broken"); 5289 } 5290 5291 assert(ARM::DPRRegClass.contains(DReg) && "Can only break D-reg deps"); 5292 assert(MI.definesRegister(DReg, TRI) && "MI doesn't clobber full D-reg"); 5293 5294 // FIXME: In some cases, VLDRS can be changed to a VLD1DUPd32 which defines 5295 // the full D-register by loading the same value to both lanes. The 5296 // instruction is micro-coded with 2 uops, so don't do this until we can 5297 // properly schedule micro-coded instructions. The dispatcher stalls cause 5298 // too big regressions. 5299 5300 // Insert the dependency-breaking FCONSTD before MI. 5301 // 96 is the encoding of 0.5, but the actual value doesn't matter here. 5302 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(ARM::FCONSTD), DReg) 5303 .addImm(96) 5304 .add(predOps(ARMCC::AL)); 5305 MI.addRegisterKilled(DReg, TRI, true); 5306 } 5307 5308 bool ARMBaseInstrInfo::hasNOP() const { 5309 return Subtarget.getFeatureBits()[ARM::HasV6KOps]; 5310 } 5311 5312 bool ARMBaseInstrInfo::isSwiftFastImmShift(const MachineInstr *MI) const { 5313 if (MI->getNumOperands() < 4) 5314 return true; 5315 unsigned ShOpVal = MI->getOperand(3).getImm(); 5316 unsigned ShImm = ARM_AM::getSORegOffset(ShOpVal); 5317 // Swift supports faster shifts for: lsl 2, lsl 1, and lsr 1. 5318 if ((ShImm == 1 && ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsr) || 5319 ((ShImm == 1 || ShImm == 2) && 5320 ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsl)) 5321 return true; 5322 5323 return false; 5324 } 5325 5326 bool ARMBaseInstrInfo::getRegSequenceLikeInputs( 5327 const MachineInstr &MI, unsigned DefIdx, 5328 SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const { 5329 assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index"); 5330 assert(MI.isRegSequenceLike() && "Invalid kind of instruction"); 5331 5332 switch (MI.getOpcode()) { 5333 case ARM::VMOVDRR: 5334 // dX = VMOVDRR rY, rZ 5335 // is the same as: 5336 // dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1 5337 // Populate the InputRegs accordingly. 5338 // rY 5339 const MachineOperand *MOReg = &MI.getOperand(1); 5340 if (!MOReg->isUndef()) 5341 InputRegs.push_back(RegSubRegPairAndIdx(MOReg->getReg(), 5342 MOReg->getSubReg(), ARM::ssub_0)); 5343 // rZ 5344 MOReg = &MI.getOperand(2); 5345 if (!MOReg->isUndef()) 5346 InputRegs.push_back(RegSubRegPairAndIdx(MOReg->getReg(), 5347 MOReg->getSubReg(), ARM::ssub_1)); 5348 return true; 5349 } 5350 llvm_unreachable("Target dependent opcode missing"); 5351 } 5352 5353 bool ARMBaseInstrInfo::getExtractSubregLikeInputs( 5354 const MachineInstr &MI, unsigned DefIdx, 5355 RegSubRegPairAndIdx &InputReg) const { 5356 assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index"); 5357 assert(MI.isExtractSubregLike() && "Invalid kind of instruction"); 5358 5359 switch (MI.getOpcode()) { 5360 case ARM::VMOVRRD: 5361 // rX, rY = VMOVRRD dZ 5362 // is the same as: 5363 // rX = EXTRACT_SUBREG dZ, ssub_0 5364 // rY = EXTRACT_SUBREG dZ, ssub_1 5365 const MachineOperand &MOReg = MI.getOperand(2); 5366 if (MOReg.isUndef()) 5367 return false; 5368 InputReg.Reg = MOReg.getReg(); 5369 InputReg.SubReg = MOReg.getSubReg(); 5370 InputReg.SubIdx = DefIdx == 0 ? ARM::ssub_0 : ARM::ssub_1; 5371 return true; 5372 } 5373 llvm_unreachable("Target dependent opcode missing"); 5374 } 5375 5376 bool ARMBaseInstrInfo::getInsertSubregLikeInputs( 5377 const MachineInstr &MI, unsigned DefIdx, RegSubRegPair &BaseReg, 5378 RegSubRegPairAndIdx &InsertedReg) const { 5379 assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index"); 5380 assert(MI.isInsertSubregLike() && "Invalid kind of instruction"); 5381 5382 switch (MI.getOpcode()) { 5383 case ARM::VSETLNi32: 5384 case ARM::MVE_VMOV_to_lane_32: 5385 // dX = VSETLNi32 dY, rZ, imm 5386 // qX = MVE_VMOV_to_lane_32 qY, rZ, imm 5387 const MachineOperand &MOBaseReg = MI.getOperand(1); 5388 const MachineOperand &MOInsertedReg = MI.getOperand(2); 5389 if (MOInsertedReg.isUndef()) 5390 return false; 5391 const MachineOperand &MOIndex = MI.getOperand(3); 5392 BaseReg.Reg = MOBaseReg.getReg(); 5393 BaseReg.SubReg = MOBaseReg.getSubReg(); 5394 5395 InsertedReg.Reg = MOInsertedReg.getReg(); 5396 InsertedReg.SubReg = MOInsertedReg.getSubReg(); 5397 InsertedReg.SubIdx = ARM::ssub_0 + MOIndex.getImm(); 5398 return true; 5399 } 5400 llvm_unreachable("Target dependent opcode missing"); 5401 } 5402 5403 std::pair<unsigned, unsigned> 5404 ARMBaseInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 5405 const unsigned Mask = ARMII::MO_OPTION_MASK; 5406 return std::make_pair(TF & Mask, TF & ~Mask); 5407 } 5408 5409 ArrayRef<std::pair<unsigned, const char *>> 5410 ARMBaseInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 5411 using namespace ARMII; 5412 5413 static const std::pair<unsigned, const char *> TargetFlags[] = { 5414 {MO_LO16, "arm-lo16"}, {MO_HI16, "arm-hi16"}}; 5415 return makeArrayRef(TargetFlags); 5416 } 5417 5418 ArrayRef<std::pair<unsigned, const char *>> 5419 ARMBaseInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const { 5420 using namespace ARMII; 5421 5422 static const std::pair<unsigned, const char *> TargetFlags[] = { 5423 {MO_COFFSTUB, "arm-coffstub"}, 5424 {MO_GOT, "arm-got"}, 5425 {MO_SBREL, "arm-sbrel"}, 5426 {MO_DLLIMPORT, "arm-dllimport"}, 5427 {MO_SECREL, "arm-secrel"}, 5428 {MO_NONLAZY, "arm-nonlazy"}}; 5429 return makeArrayRef(TargetFlags); 5430 } 5431 5432 Optional<RegImmPair> ARMBaseInstrInfo::isAddImmediate(const MachineInstr &MI, 5433 Register Reg) const { 5434 int Sign = 1; 5435 unsigned Opcode = MI.getOpcode(); 5436 int64_t Offset = 0; 5437 5438 // TODO: Handle cases where Reg is a super- or sub-register of the 5439 // destination register. 5440 const MachineOperand &Op0 = MI.getOperand(0); 5441 if (!Op0.isReg() || Reg != Op0.getReg()) 5442 return None; 5443 5444 // We describe SUBri or ADDri instructions. 5445 if (Opcode == ARM::SUBri) 5446 Sign = -1; 5447 else if (Opcode != ARM::ADDri) 5448 return None; 5449 5450 // TODO: Third operand can be global address (usually some string). Since 5451 // strings can be relocated we cannot calculate their offsets for 5452 // now. 5453 if (!MI.getOperand(1).isReg() || !MI.getOperand(2).isImm()) 5454 return None; 5455 5456 Offset = MI.getOperand(2).getImm() * Sign; 5457 return RegImmPair{MI.getOperand(1).getReg(), Offset}; 5458 } 5459 5460 bool llvm::registerDefinedBetween(unsigned Reg, 5461 MachineBasicBlock::iterator From, 5462 MachineBasicBlock::iterator To, 5463 const TargetRegisterInfo *TRI) { 5464 for (auto I = From; I != To; ++I) 5465 if (I->modifiesRegister(Reg, TRI)) 5466 return true; 5467 return false; 5468 } 5469 5470 MachineInstr *llvm::findCMPToFoldIntoCBZ(MachineInstr *Br, 5471 const TargetRegisterInfo *TRI) { 5472 // Search backwards to the instruction that defines CSPR. This may or not 5473 // be a CMP, we check that after this loop. If we find another instruction 5474 // that reads cpsr, we return nullptr. 5475 MachineBasicBlock::iterator CmpMI = Br; 5476 while (CmpMI != Br->getParent()->begin()) { 5477 --CmpMI; 5478 if (CmpMI->modifiesRegister(ARM::CPSR, TRI)) 5479 break; 5480 if (CmpMI->readsRegister(ARM::CPSR, TRI)) 5481 break; 5482 } 5483 5484 // Check that this inst is a CMP r[0-7], #0 and that the register 5485 // is not redefined between the cmp and the br. 5486 if (CmpMI->getOpcode() != ARM::tCMPi8 && CmpMI->getOpcode() != ARM::t2CMPri) 5487 return nullptr; 5488 Register Reg = CmpMI->getOperand(0).getReg(); 5489 Register PredReg; 5490 ARMCC::CondCodes Pred = getInstrPredicate(*CmpMI, PredReg); 5491 if (Pred != ARMCC::AL || CmpMI->getOperand(1).getImm() != 0) 5492 return nullptr; 5493 if (!isARMLowRegister(Reg)) 5494 return nullptr; 5495 if (registerDefinedBetween(Reg, CmpMI->getNextNode(), Br, TRI)) 5496 return nullptr; 5497 5498 return &*CmpMI; 5499 } 5500 5501 unsigned llvm::ConstantMaterializationCost(unsigned Val, 5502 const ARMSubtarget *Subtarget, 5503 bool ForCodesize) { 5504 if (Subtarget->isThumb()) { 5505 if (Val <= 255) // MOV 5506 return ForCodesize ? 2 : 1; 5507 if (Subtarget->hasV6T2Ops() && (Val <= 0xffff || // MOV 5508 ARM_AM::getT2SOImmVal(Val) != -1 || // MOVW 5509 ARM_AM::getT2SOImmVal(~Val) != -1)) // MVN 5510 return ForCodesize ? 4 : 1; 5511 if (Val <= 510) // MOV + ADDi8 5512 return ForCodesize ? 4 : 2; 5513 if (~Val <= 255) // MOV + MVN 5514 return ForCodesize ? 4 : 2; 5515 if (ARM_AM::isThumbImmShiftedVal(Val)) // MOV + LSL 5516 return ForCodesize ? 4 : 2; 5517 } else { 5518 if (ARM_AM::getSOImmVal(Val) != -1) // MOV 5519 return ForCodesize ? 4 : 1; 5520 if (ARM_AM::getSOImmVal(~Val) != -1) // MVN 5521 return ForCodesize ? 4 : 1; 5522 if (Subtarget->hasV6T2Ops() && Val <= 0xffff) // MOVW 5523 return ForCodesize ? 4 : 1; 5524 if (ARM_AM::isSOImmTwoPartVal(Val)) // two instrs 5525 return ForCodesize ? 8 : 2; 5526 if (ARM_AM::isSOImmTwoPartValNeg(Val)) // two instrs 5527 return ForCodesize ? 8 : 2; 5528 } 5529 if (Subtarget->useMovt()) // MOVW + MOVT 5530 return ForCodesize ? 8 : 2; 5531 return ForCodesize ? 8 : 3; // Literal pool load 5532 } 5533 5534 bool llvm::HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, 5535 const ARMSubtarget *Subtarget, 5536 bool ForCodesize) { 5537 // Check with ForCodesize 5538 unsigned Cost1 = ConstantMaterializationCost(Val1, Subtarget, ForCodesize); 5539 unsigned Cost2 = ConstantMaterializationCost(Val2, Subtarget, ForCodesize); 5540 if (Cost1 < Cost2) 5541 return true; 5542 if (Cost1 > Cost2) 5543 return false; 5544 5545 // If they are equal, try with !ForCodesize 5546 return ConstantMaterializationCost(Val1, Subtarget, !ForCodesize) < 5547 ConstantMaterializationCost(Val2, Subtarget, !ForCodesize); 5548 } 5549 5550 /// Constants defining how certain sequences should be outlined. 5551 /// This encompasses how an outlined function should be called, and what kind of 5552 /// frame should be emitted for that outlined function. 5553 /// 5554 /// \p MachineOutlinerTailCall implies that the function is being created from 5555 /// a sequence of instructions ending in a return. 5556 /// 5557 /// That is, 5558 /// 5559 /// I1 OUTLINED_FUNCTION: 5560 /// I2 --> B OUTLINED_FUNCTION I1 5561 /// BX LR I2 5562 /// BX LR 5563 /// 5564 /// +-------------------------+--------+-----+ 5565 /// | | Thumb2 | ARM | 5566 /// +-------------------------+--------+-----+ 5567 /// | Call overhead in Bytes | 4 | 4 | 5568 /// | Frame overhead in Bytes | 0 | 0 | 5569 /// | Stack fixup required | No | No | 5570 /// +-------------------------+--------+-----+ 5571 /// 5572 /// \p MachineOutlinerThunk implies that the function is being created from 5573 /// a sequence of instructions ending in a call. The outlined function is 5574 /// called with a BL instruction, and the outlined function tail-calls the 5575 /// original call destination. 5576 /// 5577 /// That is, 5578 /// 5579 /// I1 OUTLINED_FUNCTION: 5580 /// I2 --> BL OUTLINED_FUNCTION I1 5581 /// BL f I2 5582 /// B f 5583 /// 5584 /// +-------------------------+--------+-----+ 5585 /// | | Thumb2 | ARM | 5586 /// +-------------------------+--------+-----+ 5587 /// | Call overhead in Bytes | 4 | 4 | 5588 /// | Frame overhead in Bytes | 0 | 0 | 5589 /// | Stack fixup required | No | No | 5590 /// +-------------------------+--------+-----+ 5591 /// 5592 /// \p MachineOutlinerNoLRSave implies that the function should be called using 5593 /// a BL instruction, but doesn't require LR to be saved and restored. This 5594 /// happens when LR is known to be dead. 5595 /// 5596 /// That is, 5597 /// 5598 /// I1 OUTLINED_FUNCTION: 5599 /// I2 --> BL OUTLINED_FUNCTION I1 5600 /// I3 I2 5601 /// I3 5602 /// BX LR 5603 /// 5604 /// +-------------------------+--------+-----+ 5605 /// | | Thumb2 | ARM | 5606 /// +-------------------------+--------+-----+ 5607 /// | Call overhead in Bytes | 4 | 4 | 5608 /// | Frame overhead in Bytes | 4 | 4 | 5609 /// | Stack fixup required | No | No | 5610 /// +-------------------------+--------+-----+ 5611 /// 5612 /// \p MachineOutlinerRegSave implies that the function should be called with a 5613 /// save and restore of LR to an available register. This allows us to avoid 5614 /// stack fixups. Note that this outlining variant is compatible with the 5615 /// NoLRSave case. 5616 /// 5617 /// That is, 5618 /// 5619 /// I1 Save LR OUTLINED_FUNCTION: 5620 /// I2 --> BL OUTLINED_FUNCTION I1 5621 /// I3 Restore LR I2 5622 /// I3 5623 /// BX LR 5624 /// 5625 /// +-------------------------+--------+-----+ 5626 /// | | Thumb2 | ARM | 5627 /// +-------------------------+--------+-----+ 5628 /// | Call overhead in Bytes | 8 | 12 | 5629 /// | Frame overhead in Bytes | 2 | 4 | 5630 /// | Stack fixup required | No | No | 5631 /// +-------------------------+--------+-----+ 5632 /// 5633 /// \p MachineOutlinerDefault implies that the function should be called with 5634 /// a save and restore of LR to the stack. 5635 /// 5636 /// That is, 5637 /// 5638 /// I1 Save LR OUTLINED_FUNCTION: 5639 /// I2 --> BL OUTLINED_FUNCTION I1 5640 /// I3 Restore LR I2 5641 /// I3 5642 /// BX LR 5643 /// 5644 /// +-------------------------+--------+-----+ 5645 /// | | Thumb2 | ARM | 5646 /// +-------------------------+--------+-----+ 5647 /// | Call overhead in Bytes | 8 | 12 | 5648 /// | Frame overhead in Bytes | 2 | 4 | 5649 /// | Stack fixup required | Yes | Yes | 5650 /// +-------------------------+--------+-----+ 5651 5652 enum MachineOutlinerClass { 5653 MachineOutlinerTailCall, 5654 MachineOutlinerThunk, 5655 MachineOutlinerNoLRSave, 5656 MachineOutlinerRegSave, 5657 MachineOutlinerDefault 5658 }; 5659 5660 enum MachineOutlinerMBBFlags { 5661 LRUnavailableSomewhere = 0x2, 5662 HasCalls = 0x4, 5663 UnsafeRegsDead = 0x8 5664 }; 5665 5666 struct OutlinerCosts { 5667 const int CallTailCall; 5668 const int FrameTailCall; 5669 const int CallThunk; 5670 const int FrameThunk; 5671 const int CallNoLRSave; 5672 const int FrameNoLRSave; 5673 const int CallRegSave; 5674 const int FrameRegSave; 5675 const int CallDefault; 5676 const int FrameDefault; 5677 const int SaveRestoreLROnStack; 5678 5679 OutlinerCosts(const ARMSubtarget &target) 5680 : CallTailCall(target.isThumb() ? 4 : 4), 5681 FrameTailCall(target.isThumb() ? 0 : 0), 5682 CallThunk(target.isThumb() ? 4 : 4), 5683 FrameThunk(target.isThumb() ? 0 : 0), 5684 CallNoLRSave(target.isThumb() ? 4 : 4), 5685 FrameNoLRSave(target.isThumb() ? 4 : 4), 5686 CallRegSave(target.isThumb() ? 8 : 12), 5687 FrameRegSave(target.isThumb() ? 2 : 4), 5688 CallDefault(target.isThumb() ? 8 : 12), 5689 FrameDefault(target.isThumb() ? 2 : 4), 5690 SaveRestoreLROnStack(target.isThumb() ? 8 : 8) {} 5691 }; 5692 5693 unsigned 5694 ARMBaseInstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const { 5695 assert(C.LRUWasSet && "LRU wasn't set?"); 5696 MachineFunction *MF = C.getMF(); 5697 const ARMBaseRegisterInfo *ARI = static_cast<const ARMBaseRegisterInfo *>( 5698 MF->getSubtarget().getRegisterInfo()); 5699 5700 BitVector regsReserved = ARI->getReservedRegs(*MF); 5701 // Check if there is an available register across the sequence that we can 5702 // use. 5703 for (unsigned Reg : ARM::rGPRRegClass) { 5704 if (!(Reg < regsReserved.size() && regsReserved.test(Reg)) && 5705 Reg != ARM::LR && // LR is not reserved, but don't use it. 5706 Reg != ARM::R12 && // R12 is not guaranteed to be preserved. 5707 C.LRU.available(Reg) && C.UsedInSequence.available(Reg)) 5708 return Reg; 5709 } 5710 5711 // No suitable register. Return 0. 5712 return 0u; 5713 } 5714 5715 // Compute liveness of LR at the point after the interval [I, E), which 5716 // denotes a *backward* iteration through instructions. Used only for return 5717 // basic blocks, which do not end with a tail call. 5718 static bool isLRAvailable(const TargetRegisterInfo &TRI, 5719 MachineBasicBlock::reverse_iterator I, 5720 MachineBasicBlock::reverse_iterator E) { 5721 // At the end of the function LR dead. 5722 bool Live = false; 5723 for (; I != E; ++I) { 5724 const MachineInstr &MI = *I; 5725 5726 // Check defs of LR. 5727 if (MI.modifiesRegister(ARM::LR, &TRI)) 5728 Live = false; 5729 5730 // Check uses of LR. 5731 unsigned Opcode = MI.getOpcode(); 5732 if (Opcode == ARM::BX_RET || Opcode == ARM::MOVPCLR || 5733 Opcode == ARM::SUBS_PC_LR || Opcode == ARM::tBX_RET || 5734 Opcode == ARM::tBXNS_RET) { 5735 // These instructions use LR, but it's not an (explicit or implicit) 5736 // operand. 5737 Live = true; 5738 continue; 5739 } 5740 if (MI.readsRegister(ARM::LR, &TRI)) 5741 Live = true; 5742 } 5743 return !Live; 5744 } 5745 5746 outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( 5747 std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { 5748 outliner::Candidate &FirstCand = RepeatedSequenceLocs[0]; 5749 unsigned SequenceSize = 5750 std::accumulate(FirstCand.front(), std::next(FirstCand.back()), 0, 5751 [this](unsigned Sum, const MachineInstr &MI) { 5752 return Sum + getInstSizeInBytes(MI); 5753 }); 5754 5755 // Properties about candidate MBBs that hold for all of them. 5756 unsigned FlagsSetInAll = 0xF; 5757 5758 // Compute liveness information for each candidate, and set FlagsSetInAll. 5759 const TargetRegisterInfo &TRI = getRegisterInfo(); 5760 std::for_each( 5761 RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(), 5762 [&FlagsSetInAll](outliner::Candidate &C) { FlagsSetInAll &= C.Flags; }); 5763 5764 // According to the ARM Procedure Call Standard, the following are 5765 // undefined on entry/exit from a function call: 5766 // 5767 // * Register R12(IP), 5768 // * Condition codes (and thus the CPSR register) 5769 // 5770 // Since we control the instructions which are part of the outlined regions 5771 // we don't need to be fully compliant with the AAPCS, but we have to 5772 // guarantee that if a veneer is inserted at link time the code is still 5773 // correct. Because of this, we can't outline any sequence of instructions 5774 // where one of these registers is live into/across it. Thus, we need to 5775 // delete those candidates. 5776 auto CantGuaranteeValueAcrossCall = [&TRI](outliner::Candidate &C) { 5777 // If the unsafe registers in this block are all dead, then we don't need 5778 // to compute liveness here. 5779 if (C.Flags & UnsafeRegsDead) 5780 return false; 5781 C.initLRU(TRI); 5782 LiveRegUnits LRU = C.LRU; 5783 return (!LRU.available(ARM::R12) || !LRU.available(ARM::CPSR)); 5784 }; 5785 5786 // Are there any candidates where those registers are live? 5787 if (!(FlagsSetInAll & UnsafeRegsDead)) { 5788 // Erase every candidate that violates the restrictions above. (It could be 5789 // true that we have viable candidates, so it's not worth bailing out in 5790 // the case that, say, 1 out of 20 candidates violate the restructions.) 5791 llvm::erase_if(RepeatedSequenceLocs, CantGuaranteeValueAcrossCall); 5792 5793 // If the sequence doesn't have enough candidates left, then we're done. 5794 if (RepeatedSequenceLocs.size() < 2) 5795 return outliner::OutlinedFunction(); 5796 } 5797 5798 // At this point, we have only "safe" candidates to outline. Figure out 5799 // frame + call instruction information. 5800 5801 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back()->getOpcode(); 5802 5803 // Helper lambda which sets call information for every candidate. 5804 auto SetCandidateCallInfo = 5805 [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) { 5806 for (outliner::Candidate &C : RepeatedSequenceLocs) 5807 C.setCallInfo(CallID, NumBytesForCall); 5808 }; 5809 5810 OutlinerCosts Costs(Subtarget); 5811 unsigned FrameID = MachineOutlinerDefault; 5812 unsigned NumBytesToCreateFrame = Costs.FrameDefault; 5813 5814 // If the last instruction in any candidate is a terminator, then we should 5815 // tail call all of the candidates. 5816 if (RepeatedSequenceLocs[0].back()->isTerminator()) { 5817 FrameID = MachineOutlinerTailCall; 5818 NumBytesToCreateFrame = Costs.FrameTailCall; 5819 SetCandidateCallInfo(MachineOutlinerTailCall, Costs.CallTailCall); 5820 } else if (LastInstrOpcode == ARM::BL || LastInstrOpcode == ARM::BLX || 5821 LastInstrOpcode == ARM::BLX_noip || LastInstrOpcode == ARM::tBL || 5822 LastInstrOpcode == ARM::tBLXr || 5823 LastInstrOpcode == ARM::tBLXr_noip || 5824 LastInstrOpcode == ARM::tBLXi) { 5825 FrameID = MachineOutlinerThunk; 5826 NumBytesToCreateFrame = Costs.FrameThunk; 5827 SetCandidateCallInfo(MachineOutlinerThunk, Costs.CallThunk); 5828 } else { 5829 // We need to decide how to emit calls + frames. We can always emit the same 5830 // frame if we don't need to save to the stack. If we have to save to the 5831 // stack, then we need a different frame. 5832 unsigned NumBytesNoStackCalls = 0; 5833 std::vector<outliner::Candidate> CandidatesWithoutStackFixups; 5834 5835 for (outliner::Candidate &C : RepeatedSequenceLocs) { 5836 C.initLRU(TRI); 5837 // LR liveness is overestimated in return blocks, unless they end with a 5838 // tail call. 5839 const auto Last = C.getMBB()->rbegin(); 5840 const bool LRIsAvailable = 5841 C.getMBB()->isReturnBlock() && !Last->isCall() 5842 ? isLRAvailable(TRI, Last, 5843 (MachineBasicBlock::reverse_iterator)C.front()) 5844 : C.LRU.available(ARM::LR); 5845 if (LRIsAvailable) { 5846 FrameID = MachineOutlinerNoLRSave; 5847 NumBytesNoStackCalls += Costs.CallNoLRSave; 5848 C.setCallInfo(MachineOutlinerNoLRSave, Costs.CallNoLRSave); 5849 CandidatesWithoutStackFixups.push_back(C); 5850 } 5851 5852 // Is an unused register available? If so, we won't modify the stack, so 5853 // we can outline with the same frame type as those that don't save LR. 5854 else if (findRegisterToSaveLRTo(C)) { 5855 FrameID = MachineOutlinerRegSave; 5856 NumBytesNoStackCalls += Costs.CallRegSave; 5857 C.setCallInfo(MachineOutlinerRegSave, Costs.CallRegSave); 5858 CandidatesWithoutStackFixups.push_back(C); 5859 } 5860 5861 // Is SP used in the sequence at all? If not, we don't have to modify 5862 // the stack, so we are guaranteed to get the same frame. 5863 else if (C.UsedInSequence.available(ARM::SP)) { 5864 NumBytesNoStackCalls += Costs.CallDefault; 5865 C.setCallInfo(MachineOutlinerDefault, Costs.CallDefault); 5866 CandidatesWithoutStackFixups.push_back(C); 5867 } 5868 5869 // If we outline this, we need to modify the stack. Pretend we don't 5870 // outline this by saving all of its bytes. 5871 else 5872 NumBytesNoStackCalls += SequenceSize; 5873 } 5874 5875 // If there are no places where we have to save LR, then note that we don't 5876 // have to update the stack. Otherwise, give every candidate the default 5877 // call type 5878 if (NumBytesNoStackCalls <= 5879 RepeatedSequenceLocs.size() * Costs.CallDefault) { 5880 RepeatedSequenceLocs = CandidatesWithoutStackFixups; 5881 FrameID = MachineOutlinerNoLRSave; 5882 } else 5883 SetCandidateCallInfo(MachineOutlinerDefault, Costs.CallDefault); 5884 } 5885 5886 // Does every candidate's MBB contain a call? If so, then we might have a 5887 // call in the range. 5888 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) { 5889 // check if the range contains a call. These require a save + restore of 5890 // the link register. 5891 if (std::any_of(FirstCand.front(), FirstCand.back(), 5892 [](const MachineInstr &MI) { return MI.isCall(); })) 5893 NumBytesToCreateFrame += Costs.SaveRestoreLROnStack; 5894 5895 // Handle the last instruction separately. If it is tail call, then the 5896 // last instruction is a call, we don't want to save + restore in this 5897 // case. However, it could be possible that the last instruction is a 5898 // call without it being valid to tail call this sequence. We should 5899 // consider this as well. 5900 else if (FrameID != MachineOutlinerThunk && 5901 FrameID != MachineOutlinerTailCall && FirstCand.back()->isCall()) 5902 NumBytesToCreateFrame += Costs.SaveRestoreLROnStack; 5903 } 5904 5905 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 5906 NumBytesToCreateFrame, FrameID); 5907 } 5908 5909 bool ARMBaseInstrInfo::checkAndUpdateStackOffset(MachineInstr *MI, 5910 int64_t Fixup, 5911 bool Updt) const { 5912 int SPIdx = MI->findRegisterUseOperandIdx(ARM::SP); 5913 unsigned AddrMode = (MI->getDesc().TSFlags & ARMII::AddrModeMask); 5914 if (SPIdx < 0) 5915 // No SP operand 5916 return true; 5917 else if (SPIdx != 1 && (AddrMode != ARMII::AddrModeT2_i8s4 || SPIdx != 2)) 5918 // If SP is not the base register we can't do much 5919 return false; 5920 5921 // Stack might be involved but addressing mode doesn't handle any offset. 5922 // Rq: AddrModeT1_[1|2|4] don't operate on SP 5923 if (AddrMode == ARMII::AddrMode1 // Arithmetic instructions 5924 || AddrMode == ARMII::AddrMode4 // Load/Store Multiple 5925 || AddrMode == ARMII::AddrMode6 // Neon Load/Store Multiple 5926 || AddrMode == ARMII::AddrModeT2_so // SP can't be used as based register 5927 || AddrMode == ARMII::AddrModeT2_pc // PCrel access 5928 || AddrMode == ARMII::AddrMode2 // Used by PRE and POST indexed LD/ST 5929 || AddrMode == ARMII::AddrModeT2_i7 // v8.1-M MVE 5930 || AddrMode == ARMII::AddrModeT2_i7s2 // v8.1-M MVE 5931 || AddrMode == ARMII::AddrModeT2_i7s4 // v8.1-M sys regs VLDR/VSTR 5932 || AddrMode == ARMII::AddrModeNone) 5933 return false; 5934 5935 unsigned NumOps = MI->getDesc().getNumOperands(); 5936 unsigned ImmIdx = NumOps - 3; 5937 5938 const MachineOperand &Offset = MI->getOperand(ImmIdx); 5939 assert(Offset.isImm() && "Is not an immediate"); 5940 int64_t OffVal = Offset.getImm(); 5941 5942 if (OffVal < 0) 5943 // Don't override data if the are below SP. 5944 return false; 5945 5946 unsigned NumBits = 0; 5947 unsigned Scale = 1; 5948 5949 switch (AddrMode) { 5950 case ARMII::AddrMode3: 5951 if (ARM_AM::getAM3Op(OffVal) == ARM_AM::sub) 5952 return false; 5953 OffVal = ARM_AM::getAM3Offset(OffVal); 5954 NumBits = 8; 5955 break; 5956 case ARMII::AddrMode5: 5957 if (ARM_AM::getAM5Op(OffVal) == ARM_AM::sub) 5958 return false; 5959 OffVal = ARM_AM::getAM5Offset(OffVal); 5960 NumBits = 8; 5961 Scale = 4; 5962 break; 5963 case ARMII::AddrMode5FP16: 5964 if (ARM_AM::getAM5FP16Op(OffVal) == ARM_AM::sub) 5965 return false; 5966 OffVal = ARM_AM::getAM5FP16Offset(OffVal); 5967 NumBits = 8; 5968 Scale = 2; 5969 break; 5970 case ARMII::AddrModeT2_i8: 5971 NumBits = 8; 5972 break; 5973 case ARMII::AddrModeT2_i8s4: 5974 // FIXME: Values are already scaled in this addressing mode. 5975 assert((Fixup & 3) == 0 && "Can't encode this offset!"); 5976 NumBits = 10; 5977 break; 5978 case ARMII::AddrModeT2_ldrex: 5979 NumBits = 8; 5980 Scale = 4; 5981 break; 5982 case ARMII::AddrModeT2_i12: 5983 case ARMII::AddrMode_i12: 5984 NumBits = 12; 5985 break; 5986 case ARMII::AddrModeT1_s: // SP-relative LD/ST 5987 NumBits = 8; 5988 Scale = 4; 5989 break; 5990 default: 5991 llvm_unreachable("Unsupported addressing mode!"); 5992 } 5993 // Make sure the offset is encodable for instructions that scale the 5994 // immediate. 5995 assert(((OffVal * Scale + Fixup) & (Scale - 1)) == 0 && 5996 "Can't encode this offset!"); 5997 OffVal += Fixup / Scale; 5998 5999 unsigned Mask = (1 << NumBits) - 1; 6000 6001 if (OffVal <= Mask) { 6002 if (Updt) 6003 MI->getOperand(ImmIdx).setImm(OffVal); 6004 return true; 6005 } 6006 6007 return false; 6008 6009 } 6010 6011 bool ARMBaseInstrInfo::isFunctionSafeToOutlineFrom( 6012 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const { 6013 const Function &F = MF.getFunction(); 6014 6015 // Can F be deduplicated by the linker? If it can, don't outline from it. 6016 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage()) 6017 return false; 6018 6019 // Don't outline from functions with section markings; the program could 6020 // expect that all the code is in the named section. 6021 // FIXME: Allow outlining from multiple functions with the same section 6022 // marking. 6023 if (F.hasSection()) 6024 return false; 6025 6026 // FIXME: Thumb1 outlining is not handled 6027 if (MF.getInfo<ARMFunctionInfo>()->isThumb1OnlyFunction()) 6028 return false; 6029 6030 // It's safe to outline from MF. 6031 return true; 6032 } 6033 6034 bool ARMBaseInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, 6035 unsigned &Flags) const { 6036 // Check if LR is available through all of the MBB. If it's not, then set 6037 // a flag. 6038 assert(MBB.getParent()->getRegInfo().tracksLiveness() && 6039 "Suitable Machine Function for outlining must track liveness"); 6040 6041 LiveRegUnits LRU(getRegisterInfo()); 6042 6043 std::for_each(MBB.rbegin(), MBB.rend(), 6044 [&LRU](MachineInstr &MI) { LRU.accumulate(MI); }); 6045 6046 // Check if each of the unsafe registers are available... 6047 bool R12AvailableInBlock = LRU.available(ARM::R12); 6048 bool CPSRAvailableInBlock = LRU.available(ARM::CPSR); 6049 6050 // If all of these are dead (and not live out), we know we don't have to check 6051 // them later. 6052 if (R12AvailableInBlock && CPSRAvailableInBlock) 6053 Flags |= MachineOutlinerMBBFlags::UnsafeRegsDead; 6054 6055 // Now, add the live outs to the set. 6056 LRU.addLiveOuts(MBB); 6057 6058 // If any of these registers is available in the MBB, but also a live out of 6059 // the block, then we know outlining is unsafe. 6060 if (R12AvailableInBlock && !LRU.available(ARM::R12)) 6061 return false; 6062 if (CPSRAvailableInBlock && !LRU.available(ARM::CPSR)) 6063 return false; 6064 6065 // Check if there's a call inside this MachineBasicBlock. If there is, then 6066 // set a flag. 6067 if (any_of(MBB, [](MachineInstr &MI) { return MI.isCall(); })) 6068 Flags |= MachineOutlinerMBBFlags::HasCalls; 6069 6070 // LR liveness is overestimated in return blocks. 6071 6072 bool LRIsAvailable = 6073 MBB.isReturnBlock() && !MBB.back().isCall() 6074 ? isLRAvailable(getRegisterInfo(), MBB.rbegin(), MBB.rend()) 6075 : LRU.available(ARM::LR); 6076 if (!LRIsAvailable) 6077 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere; 6078 6079 return true; 6080 } 6081 6082 outliner::InstrType 6083 ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, 6084 unsigned Flags) const { 6085 MachineInstr &MI = *MIT; 6086 const TargetRegisterInfo *TRI = &getRegisterInfo(); 6087 6088 // Be conservative with inline ASM 6089 if (MI.isInlineAsm()) 6090 return outliner::InstrType::Illegal; 6091 6092 // Don't allow debug values to impact outlining type. 6093 if (MI.isDebugInstr() || MI.isIndirectDebugValue()) 6094 return outliner::InstrType::Invisible; 6095 6096 // At this point, KILL or IMPLICIT_DEF instructions don't really tell us much 6097 // so we can go ahead and skip over them. 6098 if (MI.isKill() || MI.isImplicitDef()) 6099 return outliner::InstrType::Invisible; 6100 6101 // PIC instructions contain labels, outlining them would break offset 6102 // computing. unsigned Opc = MI.getOpcode(); 6103 unsigned Opc = MI.getOpcode(); 6104 if (Opc == ARM::tPICADD || Opc == ARM::PICADD || Opc == ARM::PICSTR || 6105 Opc == ARM::PICSTRB || Opc == ARM::PICSTRH || Opc == ARM::PICLDR || 6106 Opc == ARM::PICLDRB || Opc == ARM::PICLDRH || Opc == ARM::PICLDRSB || 6107 Opc == ARM::PICLDRSH || Opc == ARM::t2LDRpci_pic || 6108 Opc == ARM::t2MOVi16_ga_pcrel || Opc == ARM::t2MOVTi16_ga_pcrel || 6109 Opc == ARM::t2MOV_ga_pcrel) 6110 return outliner::InstrType::Illegal; 6111 6112 // Be conservative with ARMv8.1 MVE instructions. 6113 if (Opc == ARM::t2BF_LabelPseudo || Opc == ARM::t2DoLoopStart || 6114 Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart || 6115 Opc == ARM::t2WhileLoopStartLR || Opc == ARM::t2WhileLoopStartTP || 6116 Opc == ARM::t2LoopDec || Opc == ARM::t2LoopEnd || 6117 Opc == ARM::t2LoopEndDec) 6118 return outliner::InstrType::Illegal; 6119 6120 const MCInstrDesc &MCID = MI.getDesc(); 6121 uint64_t MIFlags = MCID.TSFlags; 6122 if ((MIFlags & ARMII::DomainMask) == ARMII::DomainMVE) 6123 return outliner::InstrType::Illegal; 6124 6125 // Is this a terminator for a basic block? 6126 if (MI.isTerminator()) { 6127 // Don't outline if the branch is not unconditional. 6128 if (isPredicated(MI)) 6129 return outliner::InstrType::Illegal; 6130 6131 // Is this the end of a function? 6132 if (MI.getParent()->succ_empty()) 6133 return outliner::InstrType::Legal; 6134 6135 // It's not, so don't outline it. 6136 return outliner::InstrType::Illegal; 6137 } 6138 6139 // Make sure none of the operands are un-outlinable. 6140 for (const MachineOperand &MOP : MI.operands()) { 6141 if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() || 6142 MOP.isTargetIndex()) 6143 return outliner::InstrType::Illegal; 6144 } 6145 6146 // Don't outline if link register or program counter value are used. 6147 if (MI.readsRegister(ARM::LR, TRI) || MI.readsRegister(ARM::PC, TRI)) 6148 return outliner::InstrType::Illegal; 6149 6150 if (MI.isCall()) { 6151 // Get the function associated with the call. Look at each operand and find 6152 // the one that represents the calle and get its name. 6153 const Function *Callee = nullptr; 6154 for (const MachineOperand &MOP : MI.operands()) { 6155 if (MOP.isGlobal()) { 6156 Callee = dyn_cast<Function>(MOP.getGlobal()); 6157 break; 6158 } 6159 } 6160 6161 // Dont't outline calls to "mcount" like functions, in particular Linux 6162 // kernel function tracing relies on it. 6163 if (Callee && 6164 (Callee->getName() == "\01__gnu_mcount_nc" || 6165 Callee->getName() == "\01mcount" || Callee->getName() == "__mcount")) 6166 return outliner::InstrType::Illegal; 6167 6168 // If we don't know anything about the callee, assume it depends on the 6169 // stack layout of the caller. In that case, it's only legal to outline 6170 // as a tail-call. Explicitly list the call instructions we know about so 6171 // we don't get unexpected results with call pseudo-instructions. 6172 auto UnknownCallOutlineType = outliner::InstrType::Illegal; 6173 if (Opc == ARM::BL || Opc == ARM::tBL || Opc == ARM::BLX || 6174 Opc == ARM::BLX_noip || Opc == ARM::tBLXr || Opc == ARM::tBLXr_noip || 6175 Opc == ARM::tBLXi) 6176 UnknownCallOutlineType = outliner::InstrType::LegalTerminator; 6177 6178 if (!Callee) 6179 return UnknownCallOutlineType; 6180 6181 // We have a function we have information about. Check if it's something we 6182 // can safely outline. 6183 MachineFunction *MF = MI.getParent()->getParent(); 6184 MachineFunction *CalleeMF = MF->getMMI().getMachineFunction(*Callee); 6185 6186 // We don't know what's going on with the callee at all. Don't touch it. 6187 if (!CalleeMF) 6188 return UnknownCallOutlineType; 6189 6190 // Check if we know anything about the callee saves on the function. If we 6191 // don't, then don't touch it, since that implies that we haven't computed 6192 // anything about its stack frame yet. 6193 MachineFrameInfo &MFI = CalleeMF->getFrameInfo(); 6194 if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 || 6195 MFI.getNumObjects() > 0) 6196 return UnknownCallOutlineType; 6197 6198 // At this point, we can say that CalleeMF ought to not pass anything on the 6199 // stack. Therefore, we can outline it. 6200 return outliner::InstrType::Legal; 6201 } 6202 6203 // Since calls are handled, don't touch LR or PC 6204 if (MI.modifiesRegister(ARM::LR, TRI) || MI.modifiesRegister(ARM::PC, TRI)) 6205 return outliner::InstrType::Illegal; 6206 6207 // Does this use the stack? 6208 if (MI.modifiesRegister(ARM::SP, TRI) || MI.readsRegister(ARM::SP, TRI)) { 6209 // True if there is no chance that any outlined candidate from this range 6210 // could require stack fixups. That is, both 6211 // * LR is available in the range (No save/restore around call) 6212 // * The range doesn't include calls (No save/restore in outlined frame) 6213 // are true. 6214 // FIXME: This is very restrictive; the flags check the whole block, 6215 // not just the bit we will try to outline. 6216 bool MightNeedStackFixUp = 6217 (Flags & (MachineOutlinerMBBFlags::LRUnavailableSomewhere | 6218 MachineOutlinerMBBFlags::HasCalls)); 6219 6220 if (!MightNeedStackFixUp) 6221 return outliner::InstrType::Legal; 6222 6223 // Any modification of SP will break our code to save/restore LR. 6224 // FIXME: We could handle some instructions which add a constant offset to 6225 // SP, with a bit more work. 6226 if (MI.modifiesRegister(ARM::SP, TRI)) 6227 return outliner::InstrType::Illegal; 6228 6229 // At this point, we have a stack instruction that we might need to fix up. 6230 // up. We'll handle it if it's a load or store. 6231 if (checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(), 6232 false)) 6233 return outliner::InstrType::Legal; 6234 6235 // We can't fix it up, so don't outline it. 6236 return outliner::InstrType::Illegal; 6237 } 6238 6239 // Be conservative with IT blocks. 6240 if (MI.readsRegister(ARM::ITSTATE, TRI) || 6241 MI.modifiesRegister(ARM::ITSTATE, TRI)) 6242 return outliner::InstrType::Illegal; 6243 6244 // Don't outline positions. 6245 if (MI.isPosition()) 6246 return outliner::InstrType::Illegal; 6247 6248 return outliner::InstrType::Legal; 6249 } 6250 6251 void ARMBaseInstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const { 6252 for (MachineInstr &MI : MBB) { 6253 checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(), true); 6254 } 6255 } 6256 6257 void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB, 6258 MachineBasicBlock::iterator It) const { 6259 unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM; 6260 int Align = -Subtarget.getStackAlignment().value(); 6261 BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP) 6262 .addReg(ARM::LR, RegState::Kill) 6263 .addReg(ARM::SP) 6264 .addImm(Align) 6265 .add(predOps(ARMCC::AL)); 6266 } 6267 6268 void ARMBaseInstrInfo::emitCFIForLRSaveOnStack( 6269 MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { 6270 MachineFunction &MF = *MBB.getParent(); 6271 const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); 6272 unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); 6273 int Align = Subtarget.getStackAlignment().value(); 6274 // Add a CFI saying the stack was moved down. 6275 int64_t StackPosEntry = 6276 MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Align)); 6277 BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) 6278 .addCFIIndex(StackPosEntry) 6279 .setMIFlags(MachineInstr::FrameSetup); 6280 6281 // Add a CFI saying that the LR that we want to find is now higher than 6282 // before. 6283 int64_t LRPosEntry = 6284 MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfLR, -Align)); 6285 BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) 6286 .addCFIIndex(LRPosEntry) 6287 .setMIFlags(MachineInstr::FrameSetup); 6288 } 6289 6290 void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB, 6291 MachineBasicBlock::iterator It, 6292 Register Reg) const { 6293 MachineFunction &MF = *MBB.getParent(); 6294 const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); 6295 unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); 6296 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); 6297 6298 int64_t LRPosEntry = MF.addFrameInst( 6299 MCCFIInstruction::createRegister(nullptr, DwarfLR, DwarfReg)); 6300 BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) 6301 .addCFIIndex(LRPosEntry) 6302 .setMIFlags(MachineInstr::FrameSetup); 6303 } 6304 6305 void ARMBaseInstrInfo::restoreLRFromStack( 6306 MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { 6307 unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM; 6308 MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR) 6309 .addReg(ARM::SP, RegState::Define) 6310 .addReg(ARM::SP); 6311 if (!Subtarget.isThumb()) 6312 MIB.addReg(0); 6313 MIB.addImm(Subtarget.getStackAlignment().value()).add(predOps(ARMCC::AL)); 6314 } 6315 6316 void ARMBaseInstrInfo::emitCFIForLRRestoreFromStack( 6317 MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { 6318 // Now stack has moved back up... 6319 MachineFunction &MF = *MBB.getParent(); 6320 const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); 6321 unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); 6322 int64_t StackPosEntry = 6323 MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0)); 6324 BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) 6325 .addCFIIndex(StackPosEntry) 6326 .setMIFlags(MachineInstr::FrameDestroy); 6327 6328 // ... and we have restored LR. 6329 int64_t LRPosEntry = 6330 MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR)); 6331 BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) 6332 .addCFIIndex(LRPosEntry) 6333 .setMIFlags(MachineInstr::FrameDestroy); 6334 } 6335 6336 void ARMBaseInstrInfo::emitCFIForLRRestoreFromReg( 6337 MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { 6338 MachineFunction &MF = *MBB.getParent(); 6339 const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); 6340 unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); 6341 6342 int64_t LRPosEntry = 6343 MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR)); 6344 BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) 6345 .addCFIIndex(LRPosEntry) 6346 .setMIFlags(MachineInstr::FrameDestroy); 6347 } 6348 6349 void ARMBaseInstrInfo::buildOutlinedFrame( 6350 MachineBasicBlock &MBB, MachineFunction &MF, 6351 const outliner::OutlinedFunction &OF) const { 6352 // For thunk outlining, rewrite the last instruction from a call to a 6353 // tail-call. 6354 if (OF.FrameConstructionID == MachineOutlinerThunk) { 6355 MachineInstr *Call = &*--MBB.instr_end(); 6356 bool isThumb = Subtarget.isThumb(); 6357 unsigned FuncOp = isThumb ? 2 : 0; 6358 unsigned Opc = Call->getOperand(FuncOp).isReg() 6359 ? isThumb ? ARM::tTAILJMPr : ARM::TAILJMPr 6360 : isThumb ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd 6361 : ARM::tTAILJMPdND 6362 : ARM::TAILJMPd; 6363 MachineInstrBuilder MIB = BuildMI(MBB, MBB.end(), DebugLoc(), get(Opc)) 6364 .add(Call->getOperand(FuncOp)); 6365 if (isThumb && !Call->getOperand(FuncOp).isReg()) 6366 MIB.add(predOps(ARMCC::AL)); 6367 Call->eraseFromParent(); 6368 } 6369 6370 // Is there a call in the outlined range? 6371 auto IsNonTailCall = [](MachineInstr &MI) { 6372 return MI.isCall() && !MI.isReturn(); 6373 }; 6374 if (llvm::any_of(MBB.instrs(), IsNonTailCall)) { 6375 MachineBasicBlock::iterator It = MBB.begin(); 6376 MachineBasicBlock::iterator Et = MBB.end(); 6377 6378 if (OF.FrameConstructionID == MachineOutlinerTailCall || 6379 OF.FrameConstructionID == MachineOutlinerThunk) 6380 Et = std::prev(MBB.end()); 6381 6382 // We have to save and restore LR, we need to add it to the liveins if it 6383 // is not already part of the set. This is suffient since outlined 6384 // functions only have one block. 6385 if (!MBB.isLiveIn(ARM::LR)) 6386 MBB.addLiveIn(ARM::LR); 6387 6388 // Insert a save before the outlined region 6389 saveLROnStack(MBB, It); 6390 emitCFIForLRSaveOnStack(MBB, It); 6391 6392 // Fix up the instructions in the range, since we're going to modify the 6393 // stack. 6394 assert(OF.FrameConstructionID != MachineOutlinerDefault && 6395 "Can only fix up stack references once"); 6396 fixupPostOutline(MBB); 6397 6398 // Insert a restore before the terminator for the function. Restore LR. 6399 restoreLRFromStack(MBB, Et); 6400 emitCFIForLRRestoreFromStack(MBB, Et); 6401 } 6402 6403 // If this is a tail call outlined function, then there's already a return. 6404 if (OF.FrameConstructionID == MachineOutlinerTailCall || 6405 OF.FrameConstructionID == MachineOutlinerThunk) 6406 return; 6407 6408 // Here we have to insert the return ourselves. Get the correct opcode from 6409 // current feature set. 6410 BuildMI(MBB, MBB.end(), DebugLoc(), get(Subtarget.getReturnOpcode())) 6411 .add(predOps(ARMCC::AL)); 6412 6413 // Did we have to modify the stack by saving the link register? 6414 if (OF.FrameConstructionID != MachineOutlinerDefault && 6415 OF.Candidates[0].CallConstructionID != MachineOutlinerDefault) 6416 return; 6417 6418 // We modified the stack. 6419 // Walk over the basic block and fix up all the stack accesses. 6420 fixupPostOutline(MBB); 6421 } 6422 6423 MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall( 6424 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, 6425 MachineFunction &MF, const outliner::Candidate &C) const { 6426 MachineInstrBuilder MIB; 6427 MachineBasicBlock::iterator CallPt; 6428 unsigned Opc; 6429 bool isThumb = Subtarget.isThumb(); 6430 6431 // Are we tail calling? 6432 if (C.CallConstructionID == MachineOutlinerTailCall) { 6433 // If yes, then we can just branch to the label. 6434 Opc = isThumb 6435 ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND 6436 : ARM::TAILJMPd; 6437 MIB = BuildMI(MF, DebugLoc(), get(Opc)) 6438 .addGlobalAddress(M.getNamedValue(MF.getName())); 6439 if (isThumb) 6440 MIB.add(predOps(ARMCC::AL)); 6441 It = MBB.insert(It, MIB); 6442 return It; 6443 } 6444 6445 // Create the call instruction. 6446 Opc = isThumb ? ARM::tBL : ARM::BL; 6447 MachineInstrBuilder CallMIB = BuildMI(MF, DebugLoc(), get(Opc)); 6448 if (isThumb) 6449 CallMIB.add(predOps(ARMCC::AL)); 6450 CallMIB.addGlobalAddress(M.getNamedValue(MF.getName())); 6451 6452 if (C.CallConstructionID == MachineOutlinerNoLRSave || 6453 C.CallConstructionID == MachineOutlinerThunk) { 6454 // No, so just insert the call. 6455 It = MBB.insert(It, CallMIB); 6456 return It; 6457 } 6458 6459 const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>(); 6460 // Can we save to a register? 6461 if (C.CallConstructionID == MachineOutlinerRegSave) { 6462 unsigned Reg = findRegisterToSaveLRTo(C); 6463 assert(Reg != 0 && "No callee-saved register available?"); 6464 6465 // Save and restore LR from that register. 6466 copyPhysReg(MBB, It, DebugLoc(), Reg, ARM::LR, true); 6467 if (!AFI.isLRSpilled()) 6468 emitCFIForLRSaveToReg(MBB, It, Reg); 6469 CallPt = MBB.insert(It, CallMIB); 6470 copyPhysReg(MBB, It, DebugLoc(), ARM::LR, Reg, true); 6471 if (!AFI.isLRSpilled()) 6472 emitCFIForLRRestoreFromReg(MBB, It); 6473 It--; 6474 return CallPt; 6475 } 6476 // We have the default case. Save and restore from SP. 6477 if (!MBB.isLiveIn(ARM::LR)) 6478 MBB.addLiveIn(ARM::LR); 6479 saveLROnStack(MBB, It); 6480 if (!AFI.isLRSpilled()) 6481 emitCFIForLRSaveOnStack(MBB, It); 6482 CallPt = MBB.insert(It, CallMIB); 6483 restoreLRFromStack(MBB, It); 6484 if (!AFI.isLRSpilled()) 6485 emitCFIForLRRestoreFromStack(MBB, It); 6486 It--; 6487 return CallPt; 6488 } 6489 6490 bool ARMBaseInstrInfo::shouldOutlineFromFunctionByDefault( 6491 MachineFunction &MF) const { 6492 return Subtarget.isMClass() && MF.getFunction().hasMinSize(); 6493 } 6494 6495 bool ARMBaseInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI, 6496 AAResults *AA) const { 6497 // Try hard to rematerialize any VCTPs because if we spill P0, it will block 6498 // the tail predication conversion. This means that the element count 6499 // register has to be live for longer, but that has to be better than 6500 // spill/restore and VPT predication. 6501 return isVCTP(&MI) && !isPredicated(MI); 6502 } 6503 6504 unsigned llvm::getBLXOpcode(const MachineFunction &MF) { 6505 return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_noip 6506 : ARM::BLX; 6507 } 6508 6509 unsigned llvm::gettBLXrOpcode(const MachineFunction &MF) { 6510 return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::tBLXr_noip 6511 : ARM::tBLXr; 6512 } 6513 6514 unsigned llvm::getBLXpredOpcode(const MachineFunction &MF) { 6515 return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_pred_noip 6516 : ARM::BLX_pred; 6517 } 6518 6519