1 //===-- ARMHazardRecognizer.cpp - ARM postra hazard recognizer ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "ARMHazardRecognizer.h" 10 #include "ARMBaseInstrInfo.h" 11 #include "ARMBaseRegisterInfo.h" 12 #include "ARMSubtarget.h" 13 #include "llvm/Analysis/ValueTracking.h" 14 #include "llvm/CodeGen/MachineFrameInfo.h" 15 #include "llvm/CodeGen/MachineFunctionPass.h" 16 #include "llvm/CodeGen/MachineInstr.h" 17 #include "llvm/CodeGen/ScheduleDAG.h" 18 #include "llvm/CodeGen/TargetRegisterInfo.h" 19 #include "llvm/Support/CommandLine.h" 20 21 using namespace llvm; 22 23 static cl::opt<int> DataBankMask("arm-data-bank-mask", cl::init(-1), 24 cl::Hidden); 25 static cl::opt<bool> AssumeITCMConflict("arm-assume-itcm-bankconflict", 26 cl::init(false), cl::Hidden); 27 28 static bool hasRAWHazard(MachineInstr *DefMI, MachineInstr *MI, 29 const TargetRegisterInfo &TRI) { 30 // FIXME: Detect integer instructions properly. 31 const MCInstrDesc &MCID = MI->getDesc(); 32 unsigned Domain = MCID.TSFlags & ARMII::DomainMask; 33 if (MI->mayStore()) 34 return false; 35 unsigned Opcode = MCID.getOpcode(); 36 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD) 37 return false; 38 if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON)) 39 return MI->readsRegister(DefMI->getOperand(0).getReg(), &TRI); 40 return false; 41 } 42 43 ScheduleHazardRecognizer::HazardType 44 ARMHazardRecognizerFPMLx::getHazardType(SUnit *SU, int Stalls) { 45 assert(Stalls == 0 && "ARM hazards don't support scoreboard lookahead"); 46 47 MachineInstr *MI = SU->getInstr(); 48 49 if (!MI->isDebugInstr()) { 50 // Look for special VMLA / VMLS hazards. A VMUL / VADD / VSUB following 51 // a VMLA / VMLS will cause 4 cycle stall. 52 const MCInstrDesc &MCID = MI->getDesc(); 53 if (LastMI && (MCID.TSFlags & ARMII::DomainMask) != ARMII::DomainGeneral) { 54 MachineInstr *DefMI = LastMI; 55 const MCInstrDesc &LastMCID = LastMI->getDesc(); 56 const MachineFunction *MF = MI->getParent()->getParent(); 57 const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>( 58 MF->getSubtarget().getInstrInfo()); 59 60 // Skip over one non-VFP / NEON instruction. 61 if (!LastMI->isBarrier() && 62 !(TII.getSubtarget().hasMuxedUnits() && LastMI->mayLoadOrStore()) && 63 (LastMCID.TSFlags & ARMII::DomainMask) == ARMII::DomainGeneral) { 64 MachineBasicBlock::iterator I = LastMI; 65 if (I != LastMI->getParent()->begin()) { 66 I = std::prev(I); 67 DefMI = &*I; 68 } 69 } 70 71 if (TII.isFpMLxInstruction(DefMI->getOpcode()) && 72 (TII.canCauseFpMLxStall(MI->getOpcode()) || 73 hasRAWHazard(DefMI, MI, TII.getRegisterInfo()))) { 74 // Try to schedule another instruction for the next 4 cycles. 75 if (FpMLxStalls == 0) 76 FpMLxStalls = 4; 77 return Hazard; 78 } 79 } 80 } 81 return NoHazard; 82 } 83 84 void ARMHazardRecognizerFPMLx::Reset() { 85 LastMI = nullptr; 86 FpMLxStalls = 0; 87 } 88 89 void ARMHazardRecognizerFPMLx::EmitInstruction(SUnit *SU) { 90 MachineInstr *MI = SU->getInstr(); 91 if (!MI->isDebugInstr()) { 92 LastMI = MI; 93 FpMLxStalls = 0; 94 } 95 } 96 97 void ARMHazardRecognizerFPMLx::AdvanceCycle() { 98 if (FpMLxStalls && --FpMLxStalls == 0) 99 // Stalled for 4 cycles but still can't schedule any other instructions. 100 LastMI = nullptr; 101 } 102 103 void ARMHazardRecognizerFPMLx::RecedeCycle() { 104 llvm_unreachable("reverse ARM hazard checking unsupported"); 105 } 106 107 ///////// Bank conflicts handled as hazards ////////////// 108 109 static bool getBaseOffset(const MachineInstr &MI, const MachineOperand *&BaseOp, 110 int64_t &Offset) { 111 112 uint64_t TSFlags = MI.getDesc().TSFlags; 113 unsigned AddrMode = (TSFlags & ARMII::AddrModeMask); 114 unsigned IndexMode = 115 (TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift; 116 117 // Address mode tells us what we want to know about operands for T2 118 // instructions (but not size). It tells us size (but not about operands) 119 // for T1 instructions. 120 switch (AddrMode) { 121 default: 122 return false; 123 case ARMII::AddrModeT2_i8: 124 // t2LDRBT, t2LDRB_POST, t2LDRB_PRE, t2LDRBi8, 125 // t2LDRHT, t2LDRH_POST, t2LDRH_PRE, t2LDRHi8, 126 // t2LDRSBT, t2LDRSB_POST, t2LDRSB_PRE, t2LDRSBi8, 127 // t2LDRSHT, t2LDRSH_POST, t2LDRSH_PRE, t2LDRSHi8, 128 // t2LDRT, t2LDR_POST, t2LDR_PRE, t2LDRi8 129 BaseOp = &MI.getOperand(1); 130 Offset = (IndexMode == ARMII::IndexModePost) 131 ? 0 132 : (IndexMode == ARMII::IndexModePre || 133 IndexMode == ARMII::IndexModeUpd) 134 ? MI.getOperand(3).getImm() 135 : MI.getOperand(2).getImm(); 136 return true; 137 case ARMII::AddrModeT2_i12: 138 // t2LDRBi12, t2LDRHi12 139 // t2LDRSBi12, t2LDRSHi12 140 // t2LDRi12 141 BaseOp = &MI.getOperand(1); 142 Offset = MI.getOperand(2).getImm(); 143 return true; 144 case ARMII::AddrModeT2_i8s4: 145 // t2LDRD_POST, t2LDRD_PRE, t2LDRDi8 146 BaseOp = &MI.getOperand(2); 147 Offset = (IndexMode == ARMII::IndexModePost) 148 ? 0 149 : (IndexMode == ARMII::IndexModePre || 150 IndexMode == ARMII::IndexModeUpd) 151 ? MI.getOperand(4).getImm() 152 : MI.getOperand(3).getImm(); 153 return true; 154 case ARMII::AddrModeT1_1: 155 // tLDRBi, tLDRBr (watch out!), TLDRSB 156 case ARMII::AddrModeT1_2: 157 // tLDRHi, tLDRHr (watch out!), TLDRSH 158 case ARMII::AddrModeT1_4: 159 // tLDRi, tLDRr (watch out!) 160 BaseOp = &MI.getOperand(1); 161 Offset = MI.getOperand(2).isImm() ? MI.getOperand(2).getImm() : 0; 162 return MI.getOperand(2).isImm(); 163 } 164 return false; 165 } 166 167 ARMBankConflictHazardRecognizer::ARMBankConflictHazardRecognizer( 168 const ScheduleDAG *DAG, int64_t CPUBankMask, bool CPUAssumeITCMConflict) 169 : MF(DAG->MF), DL(DAG->MF.getDataLayout()), 170 DataMask(DataBankMask.getNumOccurrences() ? int64_t(DataBankMask) 171 : CPUBankMask), 172 AssumeITCMBankConflict(AssumeITCMConflict.getNumOccurrences() 173 ? AssumeITCMConflict 174 : CPUAssumeITCMConflict) { 175 MaxLookAhead = 1; 176 } 177 178 ScheduleHazardRecognizer::HazardType 179 ARMBankConflictHazardRecognizer::CheckOffsets(unsigned O0, unsigned O1) { 180 return (((O0 ^ O1) & DataMask) != 0) ? NoHazard : Hazard; 181 } 182 183 ScheduleHazardRecognizer::HazardType 184 ARMBankConflictHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { 185 MachineInstr &L0 = *SU->getInstr(); 186 if (!L0.mayLoad() || L0.mayStore() || L0.getNumMemOperands() != 1) 187 return NoHazard; 188 189 auto MO0 = *L0.memoperands().begin(); 190 auto BaseVal0 = MO0->getValue(); 191 auto BasePseudoVal0 = MO0->getPseudoValue(); 192 int64_t Offset0 = 0; 193 194 if (MO0->getSize() > 4) 195 return NoHazard; 196 197 bool SPvalid = false; 198 const MachineOperand *SP = nullptr; 199 int64_t SPOffset0 = 0; 200 201 for (auto L1 : Accesses) { 202 auto MO1 = *L1->memoperands().begin(); 203 auto BaseVal1 = MO1->getValue(); 204 auto BasePseudoVal1 = MO1->getPseudoValue(); 205 int64_t Offset1 = 0; 206 207 // Pointers to the same object 208 if (BaseVal0 && BaseVal1) { 209 const Value *Ptr0, *Ptr1; 210 Ptr0 = GetPointerBaseWithConstantOffset(BaseVal0, Offset0, DL, true); 211 Ptr1 = GetPointerBaseWithConstantOffset(BaseVal1, Offset1, DL, true); 212 if (Ptr0 == Ptr1 && Ptr0) 213 return CheckOffsets(Offset0, Offset1); 214 } 215 216 if (BasePseudoVal0 && BasePseudoVal1 && 217 BasePseudoVal0->kind() == BasePseudoVal1->kind() && 218 BasePseudoVal0->kind() == PseudoSourceValue::FixedStack) { 219 // Spills/fills 220 auto FS0 = cast<FixedStackPseudoSourceValue>(BasePseudoVal0); 221 auto FS1 = cast<FixedStackPseudoSourceValue>(BasePseudoVal1); 222 Offset0 = MF.getFrameInfo().getObjectOffset(FS0->getFrameIndex()); 223 Offset1 = MF.getFrameInfo().getObjectOffset(FS1->getFrameIndex()); 224 return CheckOffsets(Offset0, Offset1); 225 } 226 227 // Constant pools (likely in ITCM) 228 if (BasePseudoVal0 && BasePseudoVal1 && 229 BasePseudoVal0->kind() == BasePseudoVal1->kind() && 230 BasePseudoVal0->isConstantPool() && AssumeITCMBankConflict) 231 return Hazard; 232 233 // Is this a stack pointer-relative access? We could in general try to 234 // use "is this the same register and is it unchanged?", but the 235 // memory operand tracking is highly likely to have already found that. 236 // What we're after here is bank conflicts between different objects in 237 // the stack frame. 238 if (!SPvalid) { // set up SP 239 if (!getBaseOffset(L0, SP, SPOffset0) || SP->getReg().id() != ARM::SP) 240 SP = nullptr; 241 SPvalid = true; 242 } 243 if (SP) { 244 int64_t SPOffset1; 245 const MachineOperand *SP1; 246 if (getBaseOffset(*L1, SP1, SPOffset1) && SP1->getReg().id() == ARM::SP) 247 return CheckOffsets(SPOffset0, SPOffset1); 248 } 249 } 250 251 return NoHazard; 252 } 253 254 void ARMBankConflictHazardRecognizer::Reset() { Accesses.clear(); } 255 256 void ARMBankConflictHazardRecognizer::EmitInstruction(SUnit *SU) { 257 MachineInstr &MI = *SU->getInstr(); 258 if (!MI.mayLoad() || MI.mayStore() || MI.getNumMemOperands() != 1) 259 return; 260 261 auto MO = *MI.memoperands().begin(); 262 uint64_t Size1 = MO->getSize(); 263 if (Size1 > 4) 264 return; 265 Accesses.push_back(&MI); 266 } 267 268 void ARMBankConflictHazardRecognizer::AdvanceCycle() { Accesses.clear(); } 269 270 void ARMBankConflictHazardRecognizer::RecedeCycle() { Accesses.clear(); } 271