1 //===-- ARMHazardRecognizer.cpp - ARM postra hazard recognizer ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "ARMHazardRecognizer.h" 10 #include "ARMBaseInstrInfo.h" 11 #include "ARMBaseRegisterInfo.h" 12 #include "ARMSubtarget.h" 13 #include "llvm/Analysis/ValueTracking.h" 14 #include "llvm/CodeGen/MachineInstr.h" 15 #include "llvm/CodeGen/ScheduleDAG.h" 16 #include "llvm/CodeGen/TargetRegisterInfo.h" 17 #include "llvm/Support/CommandLine.h" 18 19 using namespace llvm; 20 21 static cl::opt<int> DataBankMask("arm-data-bank-mask", cl::init(-1), 22 cl::Hidden); 23 static cl::opt<bool> AssumeITCMConflict("arm-assume-itcm-bankconflict", 24 cl::init(false), cl::Hidden); 25 26 static bool hasRAWHazard(MachineInstr *DefMI, MachineInstr *MI, 27 const TargetRegisterInfo &TRI) { 28 // FIXME: Detect integer instructions properly. 29 const MCInstrDesc &MCID = MI->getDesc(); 30 unsigned Domain = MCID.TSFlags & ARMII::DomainMask; 31 if (MI->mayStore()) 32 return false; 33 unsigned Opcode = MCID.getOpcode(); 34 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD) 35 return false; 36 if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON)) 37 return MI->readsRegister(DefMI->getOperand(0).getReg(), &TRI); 38 return false; 39 } 40 41 ScheduleHazardRecognizer::HazardType 42 ARMHazardRecognizerFPMLx::getHazardType(SUnit *SU, int Stalls) { 43 assert(Stalls == 0 && "ARM hazards don't support scoreboard lookahead"); 44 45 MachineInstr *MI = SU->getInstr(); 46 47 if (!MI->isDebugInstr()) { 48 // Look for special VMLA / VMLS hazards. A VMUL / VADD / VSUB following 49 // a VMLA / VMLS will cause 4 cycle stall. 50 const MCInstrDesc &MCID = MI->getDesc(); 51 if (LastMI && (MCID.TSFlags & ARMII::DomainMask) != ARMII::DomainGeneral) { 52 MachineInstr *DefMI = LastMI; 53 const MCInstrDesc &LastMCID = LastMI->getDesc(); 54 const MachineFunction *MF = MI->getParent()->getParent(); 55 const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>( 56 MF->getSubtarget().getInstrInfo()); 57 58 // Skip over one non-VFP / NEON instruction. 59 if (!LastMI->isBarrier() && 60 !(TII.getSubtarget().hasMuxedUnits() && LastMI->mayLoadOrStore()) && 61 (LastMCID.TSFlags & ARMII::DomainMask) == ARMII::DomainGeneral) { 62 MachineBasicBlock::iterator I = LastMI; 63 if (I != LastMI->getParent()->begin()) { 64 I = std::prev(I); 65 DefMI = &*I; 66 } 67 } 68 69 if (TII.isFpMLxInstruction(DefMI->getOpcode()) && 70 (TII.canCauseFpMLxStall(MI->getOpcode()) || 71 hasRAWHazard(DefMI, MI, TII.getRegisterInfo()))) { 72 // Try to schedule another instruction for the next 4 cycles. 73 if (FpMLxStalls == 0) 74 FpMLxStalls = 4; 75 return Hazard; 76 } 77 } 78 } 79 return NoHazard; 80 } 81 82 void ARMHazardRecognizerFPMLx::Reset() { 83 LastMI = nullptr; 84 FpMLxStalls = 0; 85 } 86 87 void ARMHazardRecognizerFPMLx::EmitInstruction(SUnit *SU) { 88 MachineInstr *MI = SU->getInstr(); 89 if (!MI->isDebugInstr()) { 90 LastMI = MI; 91 FpMLxStalls = 0; 92 } 93 } 94 95 void ARMHazardRecognizerFPMLx::AdvanceCycle() { 96 if (FpMLxStalls && --FpMLxStalls == 0) 97 // Stalled for 4 cycles but still can't schedule any other instructions. 98 LastMI = nullptr; 99 } 100 101 void ARMHazardRecognizerFPMLx::RecedeCycle() { 102 llvm_unreachable("reverse ARM hazard checking unsupported"); 103 } 104 105 ///////// Bank conflicts handled as hazards ////////////// 106 107 static bool getBaseOffset(const MachineInstr &MI, const MachineOperand *&BaseOp, 108 int64_t &Offset) { 109 110 uint64_t TSFlags = MI.getDesc().TSFlags; 111 unsigned AddrMode = (TSFlags & ARMII::AddrModeMask); 112 unsigned IndexMode = 113 (TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift; 114 115 // Address mode tells us what we want to know about operands for T2 116 // instructions (but not size). It tells us size (but not about operands) 117 // for T1 instructions. 118 switch (AddrMode) { 119 default: 120 return false; 121 case ARMII::AddrModeT2_i8: 122 // t2LDRBT, t2LDRB_POST, t2LDRB_PRE, t2LDRBi8, 123 // t2LDRHT, t2LDRH_POST, t2LDRH_PRE, t2LDRHi8, 124 // t2LDRSBT, t2LDRSB_POST, t2LDRSB_PRE, t2LDRSBi8, 125 // t2LDRSHT, t2LDRSH_POST, t2LDRSH_PRE, t2LDRSHi8, 126 // t2LDRT, t2LDR_POST, t2LDR_PRE, t2LDRi8 127 BaseOp = &MI.getOperand(1); 128 Offset = (IndexMode == ARMII::IndexModePost) 129 ? 0 130 : (IndexMode == ARMII::IndexModePre || 131 IndexMode == ARMII::IndexModeUpd) 132 ? MI.getOperand(3).getImm() 133 : MI.getOperand(2).getImm(); 134 return true; 135 case ARMII::AddrModeT2_i12: 136 // t2LDRBi12, t2LDRHi12 137 // t2LDRSBi12, t2LDRSHi12 138 // t2LDRi12 139 BaseOp = &MI.getOperand(1); 140 Offset = MI.getOperand(2).getImm(); 141 return true; 142 case ARMII::AddrModeT2_i8s4: 143 // t2LDRD_POST, t2LDRD_PRE, t2LDRDi8 144 BaseOp = &MI.getOperand(2); 145 Offset = (IndexMode == ARMII::IndexModePost) 146 ? 0 147 : (IndexMode == ARMII::IndexModePre || 148 IndexMode == ARMII::IndexModeUpd) 149 ? MI.getOperand(4).getImm() 150 : MI.getOperand(3).getImm(); 151 return true; 152 case ARMII::AddrModeT1_1: 153 // tLDRBi, tLDRBr (watch out!), TLDRSB 154 case ARMII::AddrModeT1_2: 155 // tLDRHi, tLDRHr (watch out!), TLDRSH 156 case ARMII::AddrModeT1_4: 157 // tLDRi, tLDRr (watch out!) 158 BaseOp = &MI.getOperand(1); 159 Offset = MI.getOperand(2).isImm() ? MI.getOperand(2).getImm() : 0; 160 return MI.getOperand(2).isImm(); 161 } 162 return false; 163 } 164 165 ARMBankConflictHazardRecognizer::ARMBankConflictHazardRecognizer( 166 const ScheduleDAG *DAG, int64_t CPUBankMask, bool CPUAssumeITCMConflict) 167 : MF(DAG->MF), DL(DAG->MF.getDataLayout()), 168 DataMask(DataBankMask.getNumOccurrences() ? int64_t(DataBankMask) 169 : CPUBankMask), 170 AssumeITCMBankConflict(AssumeITCMConflict.getNumOccurrences() 171 ? AssumeITCMConflict 172 : CPUAssumeITCMConflict) { 173 MaxLookAhead = 1; 174 } 175 176 ScheduleHazardRecognizer::HazardType 177 ARMBankConflictHazardRecognizer::CheckOffsets(unsigned O0, unsigned O1) { 178 return (((O0 ^ O1) & DataMask) != 0) ? NoHazard : Hazard; 179 } 180 181 ScheduleHazardRecognizer::HazardType 182 ARMBankConflictHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { 183 MachineInstr &L0 = *SU->getInstr(); 184 if (!L0.mayLoad() || L0.mayStore() || L0.getNumMemOperands() != 1) 185 return NoHazard; 186 187 auto MO0 = *L0.memoperands().begin(); 188 auto BaseVal0 = MO0->getValue(); 189 auto BasePseudoVal0 = MO0->getPseudoValue(); 190 int64_t Offset0 = 0; 191 192 if (MO0->getSize() > 4) 193 return NoHazard; 194 195 bool SPvalid = false; 196 const MachineOperand *SP = nullptr; 197 int64_t SPOffset0 = 0; 198 199 for (auto L1 : Accesses) { 200 auto MO1 = *L1->memoperands().begin(); 201 auto BaseVal1 = MO1->getValue(); 202 auto BasePseudoVal1 = MO1->getPseudoValue(); 203 int64_t Offset1 = 0; 204 205 // Pointers to the same object 206 if (BaseVal0 && BaseVal1) { 207 const Value *Ptr0, *Ptr1; 208 Ptr0 = GetPointerBaseWithConstantOffset(BaseVal0, Offset0, DL, true); 209 Ptr1 = GetPointerBaseWithConstantOffset(BaseVal1, Offset1, DL, true); 210 if (Ptr0 == Ptr1 && Ptr0) 211 return CheckOffsets(Offset0, Offset1); 212 } 213 214 if (BasePseudoVal0 && BasePseudoVal1 && 215 BasePseudoVal0->kind() == BasePseudoVal1->kind() && 216 BasePseudoVal0->kind() == PseudoSourceValue::FixedStack) { 217 // Spills/fills 218 auto FS0 = cast<FixedStackPseudoSourceValue>(BasePseudoVal0); 219 auto FS1 = cast<FixedStackPseudoSourceValue>(BasePseudoVal1); 220 Offset0 = MF.getFrameInfo().getObjectOffset(FS0->getFrameIndex()); 221 Offset1 = MF.getFrameInfo().getObjectOffset(FS1->getFrameIndex()); 222 return CheckOffsets(Offset0, Offset1); 223 } 224 225 // Constant pools (likely in ITCM) 226 if (BasePseudoVal0 && BasePseudoVal1 && 227 BasePseudoVal0->kind() == BasePseudoVal1->kind() && 228 BasePseudoVal0->isConstantPool() && AssumeITCMBankConflict) 229 return Hazard; 230 231 // Is this a stack pointer-relative access? We could in general try to 232 // use "is this the same register and is it unchanged?", but the 233 // memory operand tracking is highly likely to have already found that. 234 // What we're after here is bank conflicts between different objects in 235 // the stack frame. 236 if (!SPvalid) { // set up SP 237 if (!getBaseOffset(L0, SP, SPOffset0) || SP->getReg().id() != ARM::SP) 238 SP = nullptr; 239 SPvalid = true; 240 } 241 if (SP) { 242 int64_t SPOffset1; 243 const MachineOperand *SP1; 244 if (getBaseOffset(*L1, SP1, SPOffset1) && SP1->getReg().id() == ARM::SP) 245 return CheckOffsets(SPOffset0, SPOffset1); 246 } 247 } 248 249 return NoHazard; 250 } 251 252 void ARMBankConflictHazardRecognizer::Reset() { Accesses.clear(); } 253 254 void ARMBankConflictHazardRecognizer::EmitInstruction(SUnit *SU) { 255 MachineInstr &MI = *SU->getInstr(); 256 if (!MI.mayLoad() || MI.mayStore() || MI.getNumMemOperands() != 1) 257 return; 258 259 auto MO = *MI.memoperands().begin(); 260 uint64_t Size1 = MO->getSize(); 261 if (Size1 > 4) 262 return; 263 Accesses.push_back(&MI); 264 } 265 266 void ARMBankConflictHazardRecognizer::AdvanceCycle() { Accesses.clear(); } 267 268 void ARMBankConflictHazardRecognizer::RecedeCycle() { Accesses.clear(); } 269