1 //===-- MipsExpandPseudoInsts.cpp - Expand pseudo instructions ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains a pass that expands pseudo instructions into target 10 // instructions to allow proper scheduling, if-conversion, and other late 11 // optimizations. This pass should be run after register allocation but before 12 // the post-regalloc scheduling pass. 13 // 14 // This is currently only used for expanding atomic pseudos after register 15 // allocation. We do this to avoid the fast register allocator introducing 16 // spills between ll and sc. These stores cause some MIPS implementations to 17 // abort the atomic RMW sequence. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "Mips.h" 22 #include "MipsInstrInfo.h" 23 #include "MipsSubtarget.h" 24 #include "llvm/CodeGen/LivePhysRegs.h" 25 #include "llvm/CodeGen/MachineFunctionPass.h" 26 #include "llvm/CodeGen/MachineInstrBuilder.h" 27 28 using namespace llvm; 29 30 #define DEBUG_TYPE "mips-pseudo" 31 32 namespace { 33 class MipsExpandPseudo : public MachineFunctionPass { 34 public: 35 static char ID; 36 MipsExpandPseudo() : MachineFunctionPass(ID) {} 37 38 const MipsInstrInfo *TII; 39 const MipsSubtarget *STI; 40 41 bool runOnMachineFunction(MachineFunction &Fn) override; 42 43 MachineFunctionProperties getRequiredProperties() const override { 44 return MachineFunctionProperties().set( 45 MachineFunctionProperties::Property::NoVRegs); 46 } 47 48 StringRef getPassName() const override { 49 return "Mips pseudo instruction expansion pass"; 50 } 51 52 private: 53 bool expandAtomicCmpSwap(MachineBasicBlock &MBB, 54 MachineBasicBlock::iterator MBBI, 55 MachineBasicBlock::iterator &NextMBBI); 56 bool expandAtomicCmpSwapSubword(MachineBasicBlock &MBB, 57 MachineBasicBlock::iterator MBBI, 58 MachineBasicBlock::iterator &NextMBBI); 59 60 bool expandAtomicBinOp(MachineBasicBlock &BB, 61 MachineBasicBlock::iterator I, 62 MachineBasicBlock::iterator &NMBBI, unsigned Size); 63 bool expandAtomicBinOpSubword(MachineBasicBlock &BB, 64 MachineBasicBlock::iterator I, 65 MachineBasicBlock::iterator &NMBBI); 66 67 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 68 MachineBasicBlock::iterator &NMBB); 69 bool expandMBB(MachineBasicBlock &MBB); 70 }; 71 char MipsExpandPseudo::ID = 0; 72 } 73 74 bool MipsExpandPseudo::expandAtomicCmpSwapSubword( 75 MachineBasicBlock &BB, MachineBasicBlock::iterator I, 76 MachineBasicBlock::iterator &NMBBI) { 77 78 MachineFunction *MF = BB.getParent(); 79 80 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); 81 DebugLoc DL = I->getDebugLoc(); 82 unsigned LL, SC; 83 84 unsigned ZERO = Mips::ZERO; 85 unsigned BNE = Mips::BNE; 86 unsigned BEQ = Mips::BEQ; 87 unsigned SEOp = 88 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH; 89 90 if (STI->inMicroMipsMode()) { 91 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; 92 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; 93 BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM; 94 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; 95 } else { 96 LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) 97 : (ArePtrs64bit ? Mips::LL64 : Mips::LL); 98 SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) 99 : (ArePtrs64bit ? Mips::SC64 : Mips::SC); 100 } 101 102 unsigned Dest = I->getOperand(0).getReg(); 103 unsigned Ptr = I->getOperand(1).getReg(); 104 unsigned Mask = I->getOperand(2).getReg(); 105 unsigned ShiftCmpVal = I->getOperand(3).getReg(); 106 unsigned Mask2 = I->getOperand(4).getReg(); 107 unsigned ShiftNewVal = I->getOperand(5).getReg(); 108 unsigned ShiftAmnt = I->getOperand(6).getReg(); 109 unsigned Scratch = I->getOperand(7).getReg(); 110 unsigned Scratch2 = I->getOperand(8).getReg(); 111 112 // insert new blocks after the current block 113 const BasicBlock *LLVM_BB = BB.getBasicBlock(); 114 MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); 115 MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); 116 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); 117 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); 118 MachineFunction::iterator It = ++BB.getIterator(); 119 MF->insert(It, loop1MBB); 120 MF->insert(It, loop2MBB); 121 MF->insert(It, sinkMBB); 122 MF->insert(It, exitMBB); 123 124 // Transfer the remainder of BB and its successor edges to exitMBB. 125 exitMBB->splice(exitMBB->begin(), &BB, 126 std::next(MachineBasicBlock::iterator(I)), BB.end()); 127 exitMBB->transferSuccessorsAndUpdatePHIs(&BB); 128 129 // thisMBB: 130 // ... 131 // fallthrough --> loop1MBB 132 BB.addSuccessor(loop1MBB, BranchProbability::getOne()); 133 loop1MBB->addSuccessor(sinkMBB); 134 loop1MBB->addSuccessor(loop2MBB); 135 loop1MBB->normalizeSuccProbs(); 136 loop2MBB->addSuccessor(loop1MBB); 137 loop2MBB->addSuccessor(sinkMBB); 138 loop2MBB->normalizeSuccProbs(); 139 sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne()); 140 141 // loop1MBB: 142 // ll dest, 0(ptr) 143 // and Mask', dest, Mask 144 // bne Mask', ShiftCmpVal, exitMBB 145 BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0); 146 BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2) 147 .addReg(Scratch) 148 .addReg(Mask); 149 BuildMI(loop1MBB, DL, TII->get(BNE)) 150 .addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB); 151 152 // loop2MBB: 153 // and dest, dest, mask2 154 // or dest, dest, ShiftNewVal 155 // sc dest, dest, 0(ptr) 156 // beq dest, $0, loop1MBB 157 BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch) 158 .addReg(Scratch, RegState::Kill) 159 .addReg(Mask2); 160 BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch) 161 .addReg(Scratch, RegState::Kill) 162 .addReg(ShiftNewVal); 163 BuildMI(loop2MBB, DL, TII->get(SC), Scratch) 164 .addReg(Scratch, RegState::Kill) 165 .addReg(Ptr) 166 .addImm(0); 167 BuildMI(loop2MBB, DL, TII->get(BEQ)) 168 .addReg(Scratch, RegState::Kill) 169 .addReg(ZERO) 170 .addMBB(loop1MBB); 171 172 // sinkMBB: 173 // srl srlres, Mask', shiftamt 174 // sign_extend dest,srlres 175 BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest) 176 .addReg(Scratch2) 177 .addReg(ShiftAmnt); 178 if (STI->hasMips32r2()) { 179 BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest); 180 } else { 181 const unsigned ShiftImm = 182 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24; 183 BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest) 184 .addReg(Dest, RegState::Kill) 185 .addImm(ShiftImm); 186 BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest) 187 .addReg(Dest, RegState::Kill) 188 .addImm(ShiftImm); 189 } 190 191 LivePhysRegs LiveRegs; 192 computeAndAddLiveIns(LiveRegs, *loop1MBB); 193 computeAndAddLiveIns(LiveRegs, *loop2MBB); 194 computeAndAddLiveIns(LiveRegs, *sinkMBB); 195 computeAndAddLiveIns(LiveRegs, *exitMBB); 196 197 NMBBI = BB.end(); 198 I->eraseFromParent(); 199 return true; 200 } 201 202 bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB, 203 MachineBasicBlock::iterator I, 204 MachineBasicBlock::iterator &NMBBI) { 205 206 const unsigned Size = 207 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8; 208 MachineFunction *MF = BB.getParent(); 209 210 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); 211 DebugLoc DL = I->getDebugLoc(); 212 213 unsigned LL, SC, ZERO, BNE, BEQ, MOVE; 214 215 if (Size == 4) { 216 if (STI->inMicroMipsMode()) { 217 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; 218 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; 219 BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM; 220 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; 221 } else { 222 LL = STI->hasMips32r6() 223 ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) 224 : (ArePtrs64bit ? Mips::LL64 : Mips::LL); 225 SC = STI->hasMips32r6() 226 ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) 227 : (ArePtrs64bit ? Mips::SC64 : Mips::SC); 228 BNE = Mips::BNE; 229 BEQ = Mips::BEQ; 230 } 231 232 ZERO = Mips::ZERO; 233 MOVE = Mips::OR; 234 } else { 235 LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD; 236 SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD; 237 ZERO = Mips::ZERO_64; 238 BNE = Mips::BNE64; 239 BEQ = Mips::BEQ64; 240 MOVE = Mips::OR64; 241 } 242 243 unsigned Dest = I->getOperand(0).getReg(); 244 unsigned Ptr = I->getOperand(1).getReg(); 245 unsigned OldVal = I->getOperand(2).getReg(); 246 unsigned NewVal = I->getOperand(3).getReg(); 247 unsigned Scratch = I->getOperand(4).getReg(); 248 249 // insert new blocks after the current block 250 const BasicBlock *LLVM_BB = BB.getBasicBlock(); 251 MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); 252 MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); 253 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); 254 MachineFunction::iterator It = ++BB.getIterator(); 255 MF->insert(It, loop1MBB); 256 MF->insert(It, loop2MBB); 257 MF->insert(It, exitMBB); 258 259 // Transfer the remainder of BB and its successor edges to exitMBB. 260 exitMBB->splice(exitMBB->begin(), &BB, 261 std::next(MachineBasicBlock::iterator(I)), BB.end()); 262 exitMBB->transferSuccessorsAndUpdatePHIs(&BB); 263 264 // thisMBB: 265 // ... 266 // fallthrough --> loop1MBB 267 BB.addSuccessor(loop1MBB, BranchProbability::getOne()); 268 loop1MBB->addSuccessor(exitMBB); 269 loop1MBB->addSuccessor(loop2MBB); 270 loop1MBB->normalizeSuccProbs(); 271 loop2MBB->addSuccessor(loop1MBB); 272 loop2MBB->addSuccessor(exitMBB); 273 loop2MBB->normalizeSuccProbs(); 274 275 // loop1MBB: 276 // ll dest, 0(ptr) 277 // bne dest, oldval, exitMBB 278 BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0); 279 BuildMI(loop1MBB, DL, TII->get(BNE)) 280 .addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB); 281 282 // loop2MBB: 283 // move scratch, NewVal 284 // sc Scratch, Scratch, 0(ptr) 285 // beq Scratch, $0, loop1MBB 286 BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO); 287 BuildMI(loop2MBB, DL, TII->get(SC), Scratch) 288 .addReg(Scratch).addReg(Ptr).addImm(0); 289 BuildMI(loop2MBB, DL, TII->get(BEQ)) 290 .addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB); 291 292 LivePhysRegs LiveRegs; 293 computeAndAddLiveIns(LiveRegs, *loop1MBB); 294 computeAndAddLiveIns(LiveRegs, *loop2MBB); 295 computeAndAddLiveIns(LiveRegs, *exitMBB); 296 297 NMBBI = BB.end(); 298 I->eraseFromParent(); 299 return true; 300 } 301 302 bool MipsExpandPseudo::expandAtomicBinOpSubword( 303 MachineBasicBlock &BB, MachineBasicBlock::iterator I, 304 MachineBasicBlock::iterator &NMBBI) { 305 306 MachineFunction *MF = BB.getParent(); 307 308 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); 309 DebugLoc DL = I->getDebugLoc(); 310 311 unsigned LL, SC; 312 unsigned BEQ = Mips::BEQ; 313 unsigned SEOp = Mips::SEH; 314 315 if (STI->inMicroMipsMode()) { 316 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; 317 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; 318 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; 319 } else { 320 LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) 321 : (ArePtrs64bit ? Mips::LL64 : Mips::LL); 322 SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) 323 : (ArePtrs64bit ? Mips::SC64 : Mips::SC); 324 } 325 326 bool IsSwap = false; 327 bool IsNand = false; 328 329 unsigned Opcode = 0; 330 switch (I->getOpcode()) { 331 case Mips::ATOMIC_LOAD_NAND_I8_POSTRA: 332 SEOp = Mips::SEB; 333 LLVM_FALLTHROUGH; 334 case Mips::ATOMIC_LOAD_NAND_I16_POSTRA: 335 IsNand = true; 336 break; 337 case Mips::ATOMIC_SWAP_I8_POSTRA: 338 SEOp = Mips::SEB; 339 LLVM_FALLTHROUGH; 340 case Mips::ATOMIC_SWAP_I16_POSTRA: 341 IsSwap = true; 342 break; 343 case Mips::ATOMIC_LOAD_ADD_I8_POSTRA: 344 SEOp = Mips::SEB; 345 LLVM_FALLTHROUGH; 346 case Mips::ATOMIC_LOAD_ADD_I16_POSTRA: 347 Opcode = Mips::ADDu; 348 break; 349 case Mips::ATOMIC_LOAD_SUB_I8_POSTRA: 350 SEOp = Mips::SEB; 351 LLVM_FALLTHROUGH; 352 case Mips::ATOMIC_LOAD_SUB_I16_POSTRA: 353 Opcode = Mips::SUBu; 354 break; 355 case Mips::ATOMIC_LOAD_AND_I8_POSTRA: 356 SEOp = Mips::SEB; 357 LLVM_FALLTHROUGH; 358 case Mips::ATOMIC_LOAD_AND_I16_POSTRA: 359 Opcode = Mips::AND; 360 break; 361 case Mips::ATOMIC_LOAD_OR_I8_POSTRA: 362 SEOp = Mips::SEB; 363 LLVM_FALLTHROUGH; 364 case Mips::ATOMIC_LOAD_OR_I16_POSTRA: 365 Opcode = Mips::OR; 366 break; 367 case Mips::ATOMIC_LOAD_XOR_I8_POSTRA: 368 SEOp = Mips::SEB; 369 LLVM_FALLTHROUGH; 370 case Mips::ATOMIC_LOAD_XOR_I16_POSTRA: 371 Opcode = Mips::XOR; 372 break; 373 default: 374 llvm_unreachable("Unknown subword atomic pseudo for expansion!"); 375 } 376 377 unsigned Dest = I->getOperand(0).getReg(); 378 unsigned Ptr = I->getOperand(1).getReg(); 379 unsigned Incr = I->getOperand(2).getReg(); 380 unsigned Mask = I->getOperand(3).getReg(); 381 unsigned Mask2 = I->getOperand(4).getReg(); 382 unsigned ShiftAmnt = I->getOperand(5).getReg(); 383 unsigned OldVal = I->getOperand(6).getReg(); 384 unsigned BinOpRes = I->getOperand(7).getReg(); 385 unsigned StoreVal = I->getOperand(8).getReg(); 386 387 const BasicBlock *LLVM_BB = BB.getBasicBlock(); 388 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); 389 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); 390 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); 391 MachineFunction::iterator It = ++BB.getIterator(); 392 MF->insert(It, loopMBB); 393 MF->insert(It, sinkMBB); 394 MF->insert(It, exitMBB); 395 396 exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end()); 397 exitMBB->transferSuccessorsAndUpdatePHIs(&BB); 398 399 BB.addSuccessor(loopMBB, BranchProbability::getOne()); 400 loopMBB->addSuccessor(sinkMBB); 401 loopMBB->addSuccessor(loopMBB); 402 loopMBB->normalizeSuccProbs(); 403 404 BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); 405 if (IsNand) { 406 // and andres, oldval, incr2 407 // nor binopres, $0, andres 408 // and newval, binopres, mask 409 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) 410 .addReg(OldVal) 411 .addReg(Incr); 412 BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes) 413 .addReg(Mips::ZERO) 414 .addReg(BinOpRes); 415 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) 416 .addReg(BinOpRes) 417 .addReg(Mask); 418 } else if (!IsSwap) { 419 // <binop> binopres, oldval, incr2 420 // and newval, binopres, mask 421 BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes) 422 .addReg(OldVal) 423 .addReg(Incr); 424 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) 425 .addReg(BinOpRes) 426 .addReg(Mask); 427 } else { // atomic.swap 428 // and newval, incr2, mask 429 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) 430 .addReg(Incr) 431 .addReg(Mask); 432 } 433 434 // and StoreVal, OlddVal, Mask2 435 // or StoreVal, StoreVal, BinOpRes 436 // StoreVal<tied1> = sc StoreVal, 0(Ptr) 437 // beq StoreVal, zero, loopMBB 438 BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal) 439 .addReg(OldVal).addReg(Mask2); 440 BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal) 441 .addReg(StoreVal).addReg(BinOpRes); 442 BuildMI(loopMBB, DL, TII->get(SC), StoreVal) 443 .addReg(StoreVal).addReg(Ptr).addImm(0); 444 BuildMI(loopMBB, DL, TII->get(BEQ)) 445 .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB); 446 447 // sinkMBB: 448 // and maskedoldval1,oldval,mask 449 // srl srlres,maskedoldval1,shiftamt 450 // sign_extend dest,srlres 451 452 sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne()); 453 454 BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest) 455 .addReg(OldVal).addReg(Mask); 456 BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest) 457 .addReg(Dest).addReg(ShiftAmnt); 458 459 if (STI->hasMips32r2()) { 460 BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest); 461 } else { 462 const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24; 463 BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest) 464 .addReg(Dest, RegState::Kill) 465 .addImm(ShiftImm); 466 BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest) 467 .addReg(Dest, RegState::Kill) 468 .addImm(ShiftImm); 469 } 470 471 LivePhysRegs LiveRegs; 472 computeAndAddLiveIns(LiveRegs, *loopMBB); 473 computeAndAddLiveIns(LiveRegs, *sinkMBB); 474 computeAndAddLiveIns(LiveRegs, *exitMBB); 475 476 NMBBI = BB.end(); 477 I->eraseFromParent(); 478 479 return true; 480 } 481 482 bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB, 483 MachineBasicBlock::iterator I, 484 MachineBasicBlock::iterator &NMBBI, 485 unsigned Size) { 486 MachineFunction *MF = BB.getParent(); 487 488 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); 489 DebugLoc DL = I->getDebugLoc(); 490 491 unsigned LL, SC, ZERO, BEQ; 492 493 if (Size == 4) { 494 if (STI->inMicroMipsMode()) { 495 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; 496 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; 497 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; 498 } else { 499 LL = STI->hasMips32r6() 500 ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) 501 : (ArePtrs64bit ? Mips::LL64 : Mips::LL); 502 SC = STI->hasMips32r6() 503 ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) 504 : (ArePtrs64bit ? Mips::SC64 : Mips::SC); 505 BEQ = Mips::BEQ; 506 } 507 508 ZERO = Mips::ZERO; 509 } else { 510 LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD; 511 SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD; 512 ZERO = Mips::ZERO_64; 513 BEQ = Mips::BEQ64; 514 } 515 516 unsigned OldVal = I->getOperand(0).getReg(); 517 unsigned Ptr = I->getOperand(1).getReg(); 518 unsigned Incr = I->getOperand(2).getReg(); 519 unsigned Scratch = I->getOperand(3).getReg(); 520 521 unsigned Opcode = 0; 522 unsigned OR = 0; 523 unsigned AND = 0; 524 unsigned NOR = 0; 525 bool IsNand = false; 526 switch (I->getOpcode()) { 527 case Mips::ATOMIC_LOAD_ADD_I32_POSTRA: 528 Opcode = Mips::ADDu; 529 break; 530 case Mips::ATOMIC_LOAD_SUB_I32_POSTRA: 531 Opcode = Mips::SUBu; 532 break; 533 case Mips::ATOMIC_LOAD_AND_I32_POSTRA: 534 Opcode = Mips::AND; 535 break; 536 case Mips::ATOMIC_LOAD_OR_I32_POSTRA: 537 Opcode = Mips::OR; 538 break; 539 case Mips::ATOMIC_LOAD_XOR_I32_POSTRA: 540 Opcode = Mips::XOR; 541 break; 542 case Mips::ATOMIC_LOAD_NAND_I32_POSTRA: 543 IsNand = true; 544 AND = Mips::AND; 545 NOR = Mips::NOR; 546 break; 547 case Mips::ATOMIC_SWAP_I32_POSTRA: 548 OR = Mips::OR; 549 break; 550 case Mips::ATOMIC_LOAD_ADD_I64_POSTRA: 551 Opcode = Mips::DADDu; 552 break; 553 case Mips::ATOMIC_LOAD_SUB_I64_POSTRA: 554 Opcode = Mips::DSUBu; 555 break; 556 case Mips::ATOMIC_LOAD_AND_I64_POSTRA: 557 Opcode = Mips::AND64; 558 break; 559 case Mips::ATOMIC_LOAD_OR_I64_POSTRA: 560 Opcode = Mips::OR64; 561 break; 562 case Mips::ATOMIC_LOAD_XOR_I64_POSTRA: 563 Opcode = Mips::XOR64; 564 break; 565 case Mips::ATOMIC_LOAD_NAND_I64_POSTRA: 566 IsNand = true; 567 AND = Mips::AND64; 568 NOR = Mips::NOR64; 569 break; 570 case Mips::ATOMIC_SWAP_I64_POSTRA: 571 OR = Mips::OR64; 572 break; 573 default: 574 llvm_unreachable("Unknown pseudo atomic!"); 575 } 576 577 const BasicBlock *LLVM_BB = BB.getBasicBlock(); 578 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); 579 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); 580 MachineFunction::iterator It = ++BB.getIterator(); 581 MF->insert(It, loopMBB); 582 MF->insert(It, exitMBB); 583 584 exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end()); 585 exitMBB->transferSuccessorsAndUpdatePHIs(&BB); 586 587 BB.addSuccessor(loopMBB, BranchProbability::getOne()); 588 loopMBB->addSuccessor(exitMBB); 589 loopMBB->addSuccessor(loopMBB); 590 loopMBB->normalizeSuccProbs(); 591 592 BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); 593 assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!"); 594 assert((OldVal != Incr) && "Clobbered the wrong reg!"); 595 if (Opcode) { 596 BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr); 597 } else if (IsNand) { 598 assert(AND && NOR && 599 "Unknown nand instruction for atomic pseudo expansion"); 600 BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr); 601 BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch); 602 } else { 603 assert(OR && "Unknown instruction for atomic pseudo expansion!"); 604 BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO); 605 } 606 607 BuildMI(loopMBB, DL, TII->get(SC), Scratch).addReg(Scratch).addReg(Ptr).addImm(0); 608 BuildMI(loopMBB, DL, TII->get(BEQ)).addReg(Scratch).addReg(ZERO).addMBB(loopMBB); 609 610 NMBBI = BB.end(); 611 I->eraseFromParent(); 612 613 LivePhysRegs LiveRegs; 614 computeAndAddLiveIns(LiveRegs, *loopMBB); 615 computeAndAddLiveIns(LiveRegs, *exitMBB); 616 617 return true; 618 } 619 620 bool MipsExpandPseudo::expandMI(MachineBasicBlock &MBB, 621 MachineBasicBlock::iterator MBBI, 622 MachineBasicBlock::iterator &NMBB) { 623 624 bool Modified = false; 625 626 switch (MBBI->getOpcode()) { 627 case Mips::ATOMIC_CMP_SWAP_I32_POSTRA: 628 case Mips::ATOMIC_CMP_SWAP_I64_POSTRA: 629 return expandAtomicCmpSwap(MBB, MBBI, NMBB); 630 case Mips::ATOMIC_CMP_SWAP_I8_POSTRA: 631 case Mips::ATOMIC_CMP_SWAP_I16_POSTRA: 632 return expandAtomicCmpSwapSubword(MBB, MBBI, NMBB); 633 case Mips::ATOMIC_SWAP_I8_POSTRA: 634 case Mips::ATOMIC_SWAP_I16_POSTRA: 635 case Mips::ATOMIC_LOAD_NAND_I8_POSTRA: 636 case Mips::ATOMIC_LOAD_NAND_I16_POSTRA: 637 case Mips::ATOMIC_LOAD_ADD_I8_POSTRA: 638 case Mips::ATOMIC_LOAD_ADD_I16_POSTRA: 639 case Mips::ATOMIC_LOAD_SUB_I8_POSTRA: 640 case Mips::ATOMIC_LOAD_SUB_I16_POSTRA: 641 case Mips::ATOMIC_LOAD_AND_I8_POSTRA: 642 case Mips::ATOMIC_LOAD_AND_I16_POSTRA: 643 case Mips::ATOMIC_LOAD_OR_I8_POSTRA: 644 case Mips::ATOMIC_LOAD_OR_I16_POSTRA: 645 case Mips::ATOMIC_LOAD_XOR_I8_POSTRA: 646 case Mips::ATOMIC_LOAD_XOR_I16_POSTRA: 647 return expandAtomicBinOpSubword(MBB, MBBI, NMBB); 648 case Mips::ATOMIC_LOAD_ADD_I32_POSTRA: 649 case Mips::ATOMIC_LOAD_SUB_I32_POSTRA: 650 case Mips::ATOMIC_LOAD_AND_I32_POSTRA: 651 case Mips::ATOMIC_LOAD_OR_I32_POSTRA: 652 case Mips::ATOMIC_LOAD_XOR_I32_POSTRA: 653 case Mips::ATOMIC_LOAD_NAND_I32_POSTRA: 654 case Mips::ATOMIC_SWAP_I32_POSTRA: 655 return expandAtomicBinOp(MBB, MBBI, NMBB, 4); 656 case Mips::ATOMIC_LOAD_ADD_I64_POSTRA: 657 case Mips::ATOMIC_LOAD_SUB_I64_POSTRA: 658 case Mips::ATOMIC_LOAD_AND_I64_POSTRA: 659 case Mips::ATOMIC_LOAD_OR_I64_POSTRA: 660 case Mips::ATOMIC_LOAD_XOR_I64_POSTRA: 661 case Mips::ATOMIC_LOAD_NAND_I64_POSTRA: 662 case Mips::ATOMIC_SWAP_I64_POSTRA: 663 return expandAtomicBinOp(MBB, MBBI, NMBB, 8); 664 default: 665 return Modified; 666 } 667 } 668 669 bool MipsExpandPseudo::expandMBB(MachineBasicBlock &MBB) { 670 bool Modified = false; 671 672 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); 673 while (MBBI != E) { 674 MachineBasicBlock::iterator NMBBI = std::next(MBBI); 675 Modified |= expandMI(MBB, MBBI, NMBBI); 676 MBBI = NMBBI; 677 } 678 679 return Modified; 680 } 681 682 bool MipsExpandPseudo::runOnMachineFunction(MachineFunction &MF) { 683 STI = &static_cast<const MipsSubtarget &>(MF.getSubtarget()); 684 TII = STI->getInstrInfo(); 685 686 bool Modified = false; 687 for (MachineFunction::iterator MFI = MF.begin(), E = MF.end(); MFI != E; 688 ++MFI) 689 Modified |= expandMBB(*MFI); 690 691 if (Modified) 692 MF.RenumberBlocks(); 693 694 return Modified; 695 } 696 697 /// createMipsExpandPseudoPass - returns an instance of the pseudo instruction 698 /// expansion pass. 699 FunctionPass *llvm::createMipsExpandPseudoPass() { 700 return new MipsExpandPseudo(); 701 } 702