1 //===-- AArch64A53Fix835769.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // This pass changes code to work around Cortex-A53 erratum 835769. 9 // It works around it by inserting a nop instruction in code sequences that 10 // in some circumstances may trigger the erratum. 11 // It inserts a nop instruction between a sequence of the following 2 classes 12 // of instructions: 13 // instr 1: mem-instr (including loads, stores and prefetches). 14 // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers. 15 //===----------------------------------------------------------------------===// 16 17 #include "AArch64.h" 18 #include "AArch64Subtarget.h" 19 #include "llvm/ADT/Statistic.h" 20 #include "llvm/CodeGen/MachineFunction.h" 21 #include "llvm/CodeGen/MachineFunctionPass.h" 22 #include "llvm/CodeGen/MachineInstr.h" 23 #include "llvm/CodeGen/MachineInstrBuilder.h" 24 #include "llvm/CodeGen/MachineRegisterInfo.h" 25 #include "llvm/CodeGen/TargetInstrInfo.h" 26 #include "llvm/Support/Debug.h" 27 #include "llvm/Support/raw_ostream.h" 28 29 using namespace llvm; 30 31 #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769" 32 33 STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769"); 34 35 //===----------------------------------------------------------------------===// 36 // Helper functions 37 38 // Is the instruction a match for the instruction that comes first in the 39 // sequence of instructions that can trigger the erratum? 40 static bool isFirstInstructionInSequence(MachineInstr *MI) { 41 // Must return true if this instruction is a load, a store or a prefetch. 42 switch (MI->getOpcode()) { 43 case AArch64::PRFMl: 44 case AArch64::PRFMroW: 45 case AArch64::PRFMroX: 46 case AArch64::PRFMui: 47 case AArch64::PRFUMi: 48 return true; 49 default: 50 return MI->mayLoadOrStore(); 51 } 52 } 53 54 // Is the instruction a match for the instruction that comes second in the 55 // sequence that can trigger the erratum? 56 static bool isSecondInstructionInSequence(MachineInstr *MI) { 57 // Must return true for non-SIMD integer multiply-accumulates, writing 58 // to a 64-bit register. 59 switch (MI->getOpcode()) { 60 // Erratum cannot be triggered when the destination register is 32 bits, 61 // therefore only include the following. 62 case AArch64::MSUBXrrr: 63 case AArch64::MADDXrrr: 64 case AArch64::SMADDLrrr: 65 case AArch64::SMSUBLrrr: 66 case AArch64::UMADDLrrr: 67 case AArch64::UMSUBLrrr: 68 // Erratum can only be triggered by multiply-adds, not by regular 69 // non-accumulating multiplies, i.e. when Ra=XZR='11111' 70 return MI->getOperand(3).getReg() != AArch64::XZR; 71 default: 72 return false; 73 } 74 } 75 76 77 //===----------------------------------------------------------------------===// 78 79 namespace { 80 class AArch64A53Fix835769 : public MachineFunctionPass { 81 const TargetInstrInfo *TII; 82 83 public: 84 static char ID; 85 explicit AArch64A53Fix835769() : MachineFunctionPass(ID) { 86 initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry()); 87 } 88 89 bool runOnMachineFunction(MachineFunction &F) override; 90 91 MachineFunctionProperties getRequiredProperties() const override { 92 return MachineFunctionProperties().set( 93 MachineFunctionProperties::Property::NoVRegs); 94 } 95 96 StringRef getPassName() const override { 97 return "Workaround A53 erratum 835769 pass"; 98 } 99 100 void getAnalysisUsage(AnalysisUsage &AU) const override { 101 AU.setPreservesCFG(); 102 MachineFunctionPass::getAnalysisUsage(AU); 103 } 104 105 private: 106 bool runOnBasicBlock(MachineBasicBlock &MBB); 107 }; 108 char AArch64A53Fix835769::ID = 0; 109 110 } // end anonymous namespace 111 112 INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass", 113 "AArch64 fix for A53 erratum 835769", false, false) 114 115 //===----------------------------------------------------------------------===// 116 117 bool 118 AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) { 119 LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n"); 120 auto &STI = F.getSubtarget<AArch64Subtarget>(); 121 // Fix not requested, skip pass. 122 if (!STI.fixCortexA53_835769()) 123 return false; 124 125 bool Changed = false; 126 TII = STI.getInstrInfo(); 127 128 for (auto &MBB : F) { 129 Changed |= runOnBasicBlock(MBB); 130 } 131 return Changed; 132 } 133 134 // Return the block that was fallen through to get to MBB, if any, 135 // otherwise nullptr. 136 static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB, 137 const TargetInstrInfo *TII) { 138 // Get the previous machine basic block in the function. 139 MachineFunction::iterator MBBI(MBB); 140 141 // Can't go off top of function. 142 if (MBBI == MBB->getParent()->begin()) 143 return nullptr; 144 145 MachineBasicBlock *TBB = nullptr, *FBB = nullptr; 146 SmallVector<MachineOperand, 2> Cond; 147 148 MachineBasicBlock *PrevBB = &*std::prev(MBBI); 149 for (MachineBasicBlock *S : MBB->predecessors()) 150 if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB && 151 !FBB) 152 return S; 153 154 return nullptr; 155 } 156 157 // Iterate through fallen through blocks trying to find a previous non-pseudo if 158 // there is one, otherwise return nullptr. Only look for instructions in 159 // previous blocks, not the current block, since we only use this to look at 160 // previous blocks. 161 static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB, 162 const TargetInstrInfo *TII) { 163 MachineBasicBlock *FMBB = &MBB; 164 165 // If there is no non-pseudo in the current block, loop back around and try 166 // the previous block (if there is one). 167 while ((FMBB = getBBFallenThrough(FMBB, TII))) { 168 for (MachineInstr &I : llvm::reverse(*FMBB)) 169 if (!I.isPseudo()) 170 return &I; 171 } 172 173 // There was no previous non-pseudo in the fallen through blocks 174 return nullptr; 175 } 176 177 static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI, 178 const TargetInstrInfo *TII) { 179 // If we are the first instruction of the block, put the NOP at the end of 180 // the previous fallthrough block 181 if (MI == &MBB.front()) { 182 MachineInstr *I = getLastNonPseudo(MBB, TII); 183 assert(I && "Expected instruction"); 184 DebugLoc DL = I->getDebugLoc(); 185 BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0); 186 } 187 else { 188 DebugLoc DL = MI->getDebugLoc(); 189 BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0); 190 } 191 192 ++NumNopsAdded; 193 } 194 195 bool 196 AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) { 197 bool Changed = false; 198 LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB 199 << " - scanning instructions...\n"); 200 201 // First, scan the basic block, looking for a sequence of 2 instructions 202 // that match the conditions under which the erratum may trigger. 203 204 // List of terminating instructions in matching sequences 205 std::vector<MachineInstr*> Sequences; 206 unsigned Idx = 0; 207 MachineInstr *PrevInstr = nullptr; 208 209 // Try and find the last non-pseudo instruction in any fallen through blocks, 210 // if there isn't one, then we use nullptr to represent that. 211 PrevInstr = getLastNonPseudo(MBB, TII); 212 213 for (auto &MI : MBB) { 214 MachineInstr *CurrInstr = &MI; 215 LLVM_DEBUG(dbgs() << " Examining: " << MI); 216 if (PrevInstr) { 217 LLVM_DEBUG(dbgs() << " PrevInstr: " << *PrevInstr 218 << " CurrInstr: " << *CurrInstr 219 << " isFirstInstructionInSequence(PrevInstr): " 220 << isFirstInstructionInSequence(PrevInstr) << "\n" 221 << " isSecondInstructionInSequence(CurrInstr): " 222 << isSecondInstructionInSequence(CurrInstr) << "\n"); 223 if (isFirstInstructionInSequence(PrevInstr) && 224 isSecondInstructionInSequence(CurrInstr)) { 225 LLVM_DEBUG(dbgs() << " ** pattern found at Idx " << Idx << "!\n"); 226 (void) Idx; 227 Sequences.push_back(CurrInstr); 228 } 229 } 230 if (!CurrInstr->isPseudo()) 231 PrevInstr = CurrInstr; 232 ++Idx; 233 } 234 235 LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size() 236 << " occurrences of pattern found.\n"); 237 238 // Then update the basic block, inserting nops between the detected sequences. 239 for (auto &MI : Sequences) { 240 Changed = true; 241 insertNopBeforeInstruction(MBB, MI, TII); 242 } 243 244 return Changed; 245 } 246 247 // Factory function used by AArch64TargetMachine to add the pass to 248 // the passmanager. 249 FunctionPass *llvm::createAArch64A53Fix835769() { 250 return new AArch64A53Fix835769(); 251 } 252