xref: /freebsd/contrib/llvm-project/llvm/lib/Target/Mips/MipsExpandPseudo.cpp (revision 6966ac055c3b7a39266fb982493330df7a097997)
1 //===-- MipsExpandPseudoInsts.cpp - Expand pseudo instructions ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a pass that expands pseudo instructions into target
10 // instructions to allow proper scheduling, if-conversion, and other late
11 // optimizations. This pass should be run after register allocation but before
12 // the post-regalloc scheduling pass.
13 //
14 // This is currently only used for expanding atomic pseudos after register
15 // allocation. We do this to avoid the fast register allocator introducing
16 // spills between ll and sc. These stores cause some MIPS implementations to
17 // abort the atomic RMW sequence.
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "Mips.h"
22 #include "MipsInstrInfo.h"
23 #include "MipsSubtarget.h"
24 #include "llvm/CodeGen/LivePhysRegs.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 
28 using namespace llvm;
29 
30 #define DEBUG_TYPE "mips-pseudo"
31 
32 namespace {
33   class MipsExpandPseudo : public MachineFunctionPass {
34   public:
35     static char ID;
36     MipsExpandPseudo() : MachineFunctionPass(ID) {}
37 
38     const MipsInstrInfo *TII;
39     const MipsSubtarget *STI;
40 
41     bool runOnMachineFunction(MachineFunction &Fn) override;
42 
43     MachineFunctionProperties getRequiredProperties() const override {
44       return MachineFunctionProperties().set(
45           MachineFunctionProperties::Property::NoVRegs);
46     }
47 
48     StringRef getPassName() const override {
49       return "Mips pseudo instruction expansion pass";
50     }
51 
52   private:
53     bool expandAtomicCmpSwap(MachineBasicBlock &MBB,
54                              MachineBasicBlock::iterator MBBI,
55                              MachineBasicBlock::iterator &NextMBBI);
56     bool expandAtomicCmpSwapSubword(MachineBasicBlock &MBB,
57                                     MachineBasicBlock::iterator MBBI,
58                                     MachineBasicBlock::iterator &NextMBBI);
59 
60     bool expandAtomicBinOp(MachineBasicBlock &BB,
61                            MachineBasicBlock::iterator I,
62                            MachineBasicBlock::iterator &NMBBI, unsigned Size);
63     bool expandAtomicBinOpSubword(MachineBasicBlock &BB,
64                                   MachineBasicBlock::iterator I,
65                                   MachineBasicBlock::iterator &NMBBI);
66 
67     bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
68                   MachineBasicBlock::iterator &NMBB);
69     bool expandMBB(MachineBasicBlock &MBB);
70    };
71   char MipsExpandPseudo::ID = 0;
72 }
73 
74 bool MipsExpandPseudo::expandAtomicCmpSwapSubword(
75     MachineBasicBlock &BB, MachineBasicBlock::iterator I,
76     MachineBasicBlock::iterator &NMBBI) {
77 
78   MachineFunction *MF = BB.getParent();
79 
80   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
81   DebugLoc DL = I->getDebugLoc();
82   unsigned LL, SC;
83 
84   unsigned ZERO = Mips::ZERO;
85   unsigned BNE = Mips::BNE;
86   unsigned BEQ = Mips::BEQ;
87   unsigned SEOp =
88       I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH;
89 
90   if (STI->inMicroMipsMode()) {
91       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
92       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
93       BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
94       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
95   } else {
96     LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
97                             : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
98     SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
99                             : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
100   }
101 
102   unsigned Dest = I->getOperand(0).getReg();
103   unsigned Ptr = I->getOperand(1).getReg();
104   unsigned Mask = I->getOperand(2).getReg();
105   unsigned ShiftCmpVal = I->getOperand(3).getReg();
106   unsigned Mask2 = I->getOperand(4).getReg();
107   unsigned ShiftNewVal = I->getOperand(5).getReg();
108   unsigned ShiftAmnt = I->getOperand(6).getReg();
109   unsigned Scratch = I->getOperand(7).getReg();
110   unsigned Scratch2 = I->getOperand(8).getReg();
111 
112   // insert new blocks after the current block
113   const BasicBlock *LLVM_BB = BB.getBasicBlock();
114   MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
115   MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
116   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
117   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
118   MachineFunction::iterator It = ++BB.getIterator();
119   MF->insert(It, loop1MBB);
120   MF->insert(It, loop2MBB);
121   MF->insert(It, sinkMBB);
122   MF->insert(It, exitMBB);
123 
124   // Transfer the remainder of BB and its successor edges to exitMBB.
125   exitMBB->splice(exitMBB->begin(), &BB,
126                   std::next(MachineBasicBlock::iterator(I)), BB.end());
127   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
128 
129   //  thisMBB:
130   //    ...
131   //    fallthrough --> loop1MBB
132   BB.addSuccessor(loop1MBB, BranchProbability::getOne());
133   loop1MBB->addSuccessor(sinkMBB);
134   loop1MBB->addSuccessor(loop2MBB);
135   loop1MBB->normalizeSuccProbs();
136   loop2MBB->addSuccessor(loop1MBB);
137   loop2MBB->addSuccessor(sinkMBB);
138   loop2MBB->normalizeSuccProbs();
139   sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
140 
141   // loop1MBB:
142   //   ll dest, 0(ptr)
143   //   and Mask', dest, Mask
144   //   bne Mask', ShiftCmpVal, exitMBB
145   BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0);
146   BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2)
147       .addReg(Scratch)
148       .addReg(Mask);
149   BuildMI(loop1MBB, DL, TII->get(BNE))
150     .addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB);
151 
152   // loop2MBB:
153   //   and dest, dest, mask2
154   //   or dest, dest, ShiftNewVal
155   //   sc dest, dest, 0(ptr)
156   //   beq dest, $0, loop1MBB
157   BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch)
158       .addReg(Scratch, RegState::Kill)
159       .addReg(Mask2);
160   BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch)
161       .addReg(Scratch, RegState::Kill)
162       .addReg(ShiftNewVal);
163   BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
164       .addReg(Scratch, RegState::Kill)
165       .addReg(Ptr)
166       .addImm(0);
167   BuildMI(loop2MBB, DL, TII->get(BEQ))
168       .addReg(Scratch, RegState::Kill)
169       .addReg(ZERO)
170       .addMBB(loop1MBB);
171 
172   //  sinkMBB:
173   //    srl     srlres, Mask', shiftamt
174   //    sign_extend dest,srlres
175   BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
176       .addReg(Scratch2)
177       .addReg(ShiftAmnt);
178   if (STI->hasMips32r2()) {
179     BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
180   } else {
181     const unsigned ShiftImm =
182         I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24;
183     BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
184         .addReg(Dest, RegState::Kill)
185         .addImm(ShiftImm);
186     BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
187         .addReg(Dest, RegState::Kill)
188         .addImm(ShiftImm);
189   }
190 
191   LivePhysRegs LiveRegs;
192   computeAndAddLiveIns(LiveRegs, *loop1MBB);
193   computeAndAddLiveIns(LiveRegs, *loop2MBB);
194   computeAndAddLiveIns(LiveRegs, *sinkMBB);
195   computeAndAddLiveIns(LiveRegs, *exitMBB);
196 
197   NMBBI = BB.end();
198   I->eraseFromParent();
199   return true;
200 }
201 
202 bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB,
203                                            MachineBasicBlock::iterator I,
204                                            MachineBasicBlock::iterator &NMBBI) {
205 
206   const unsigned Size =
207       I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8;
208   MachineFunction *MF = BB.getParent();
209 
210   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
211   DebugLoc DL = I->getDebugLoc();
212 
213   unsigned LL, SC, ZERO, BNE, BEQ, MOVE;
214 
215   if (Size == 4) {
216     if (STI->inMicroMipsMode()) {
217       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
218       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
219       BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
220       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
221     } else {
222       LL = STI->hasMips32r6()
223                ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
224                : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
225       SC = STI->hasMips32r6()
226                ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
227                : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
228       BNE = Mips::BNE;
229       BEQ = Mips::BEQ;
230     }
231 
232     ZERO = Mips::ZERO;
233     MOVE = Mips::OR;
234   } else {
235     LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
236     SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
237     ZERO = Mips::ZERO_64;
238     BNE = Mips::BNE64;
239     BEQ = Mips::BEQ64;
240     MOVE = Mips::OR64;
241   }
242 
243   unsigned Dest = I->getOperand(0).getReg();
244   unsigned Ptr = I->getOperand(1).getReg();
245   unsigned OldVal = I->getOperand(2).getReg();
246   unsigned NewVal = I->getOperand(3).getReg();
247   unsigned Scratch = I->getOperand(4).getReg();
248 
249   // insert new blocks after the current block
250   const BasicBlock *LLVM_BB = BB.getBasicBlock();
251   MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
252   MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
253   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
254   MachineFunction::iterator It = ++BB.getIterator();
255   MF->insert(It, loop1MBB);
256   MF->insert(It, loop2MBB);
257   MF->insert(It, exitMBB);
258 
259   // Transfer the remainder of BB and its successor edges to exitMBB.
260   exitMBB->splice(exitMBB->begin(), &BB,
261                   std::next(MachineBasicBlock::iterator(I)), BB.end());
262   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
263 
264   //  thisMBB:
265   //    ...
266   //    fallthrough --> loop1MBB
267   BB.addSuccessor(loop1MBB, BranchProbability::getOne());
268   loop1MBB->addSuccessor(exitMBB);
269   loop1MBB->addSuccessor(loop2MBB);
270   loop1MBB->normalizeSuccProbs();
271   loop2MBB->addSuccessor(loop1MBB);
272   loop2MBB->addSuccessor(exitMBB);
273   loop2MBB->normalizeSuccProbs();
274 
275   // loop1MBB:
276   //   ll dest, 0(ptr)
277   //   bne dest, oldval, exitMBB
278   BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0);
279   BuildMI(loop1MBB, DL, TII->get(BNE))
280     .addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB);
281 
282   // loop2MBB:
283   //   move scratch, NewVal
284   //   sc Scratch, Scratch, 0(ptr)
285   //   beq Scratch, $0, loop1MBB
286   BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO);
287   BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
288     .addReg(Scratch).addReg(Ptr).addImm(0);
289   BuildMI(loop2MBB, DL, TII->get(BEQ))
290     .addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB);
291 
292   LivePhysRegs LiveRegs;
293   computeAndAddLiveIns(LiveRegs, *loop1MBB);
294   computeAndAddLiveIns(LiveRegs, *loop2MBB);
295   computeAndAddLiveIns(LiveRegs, *exitMBB);
296 
297   NMBBI = BB.end();
298   I->eraseFromParent();
299   return true;
300 }
301 
302 bool MipsExpandPseudo::expandAtomicBinOpSubword(
303     MachineBasicBlock &BB, MachineBasicBlock::iterator I,
304     MachineBasicBlock::iterator &NMBBI) {
305 
306   MachineFunction *MF = BB.getParent();
307 
308   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
309   DebugLoc DL = I->getDebugLoc();
310 
311   unsigned LL, SC;
312   unsigned BEQ = Mips::BEQ;
313   unsigned SEOp = Mips::SEH;
314 
315   if (STI->inMicroMipsMode()) {
316       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
317       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
318       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
319   } else {
320     LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
321                             : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
322     SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
323                             : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
324   }
325 
326   bool IsSwap = false;
327   bool IsNand = false;
328 
329   unsigned Opcode = 0;
330   switch (I->getOpcode()) {
331   case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
332     SEOp = Mips::SEB;
333     LLVM_FALLTHROUGH;
334   case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
335     IsNand = true;
336     break;
337   case Mips::ATOMIC_SWAP_I8_POSTRA:
338     SEOp = Mips::SEB;
339     LLVM_FALLTHROUGH;
340   case Mips::ATOMIC_SWAP_I16_POSTRA:
341     IsSwap = true;
342     break;
343   case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
344     SEOp = Mips::SEB;
345     LLVM_FALLTHROUGH;
346   case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
347     Opcode = Mips::ADDu;
348     break;
349   case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
350     SEOp = Mips::SEB;
351     LLVM_FALLTHROUGH;
352   case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
353     Opcode = Mips::SUBu;
354     break;
355   case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
356     SEOp = Mips::SEB;
357     LLVM_FALLTHROUGH;
358   case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
359     Opcode = Mips::AND;
360     break;
361   case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
362     SEOp = Mips::SEB;
363     LLVM_FALLTHROUGH;
364   case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
365     Opcode = Mips::OR;
366     break;
367   case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
368     SEOp = Mips::SEB;
369     LLVM_FALLTHROUGH;
370   case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
371     Opcode = Mips::XOR;
372     break;
373   default:
374     llvm_unreachable("Unknown subword atomic pseudo for expansion!");
375   }
376 
377   unsigned Dest = I->getOperand(0).getReg();
378   unsigned Ptr = I->getOperand(1).getReg();
379   unsigned Incr = I->getOperand(2).getReg();
380   unsigned Mask = I->getOperand(3).getReg();
381   unsigned Mask2 = I->getOperand(4).getReg();
382   unsigned ShiftAmnt = I->getOperand(5).getReg();
383   unsigned OldVal = I->getOperand(6).getReg();
384   unsigned BinOpRes = I->getOperand(7).getReg();
385   unsigned StoreVal = I->getOperand(8).getReg();
386 
387   const BasicBlock *LLVM_BB = BB.getBasicBlock();
388   MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
389   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
390   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
391   MachineFunction::iterator It = ++BB.getIterator();
392   MF->insert(It, loopMBB);
393   MF->insert(It, sinkMBB);
394   MF->insert(It, exitMBB);
395 
396   exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
397   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
398 
399   BB.addSuccessor(loopMBB, BranchProbability::getOne());
400   loopMBB->addSuccessor(sinkMBB);
401   loopMBB->addSuccessor(loopMBB);
402   loopMBB->normalizeSuccProbs();
403 
404   BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
405   if (IsNand) {
406     //  and andres, oldval, incr2
407     //  nor binopres, $0, andres
408     //  and newval, binopres, mask
409     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
410         .addReg(OldVal)
411         .addReg(Incr);
412     BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes)
413         .addReg(Mips::ZERO)
414         .addReg(BinOpRes);
415     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
416         .addReg(BinOpRes)
417         .addReg(Mask);
418   } else if (!IsSwap) {
419     //  <binop> binopres, oldval, incr2
420     //  and newval, binopres, mask
421     BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes)
422         .addReg(OldVal)
423         .addReg(Incr);
424     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
425         .addReg(BinOpRes)
426         .addReg(Mask);
427   } else { // atomic.swap
428     //  and newval, incr2, mask
429     BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
430         .addReg(Incr)
431         .addReg(Mask);
432   }
433 
434   // and StoreVal, OlddVal, Mask2
435   // or StoreVal, StoreVal, BinOpRes
436   // StoreVal<tied1> = sc StoreVal, 0(Ptr)
437   // beq StoreVal, zero, loopMBB
438   BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal)
439     .addReg(OldVal).addReg(Mask2);
440   BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal)
441     .addReg(StoreVal).addReg(BinOpRes);
442   BuildMI(loopMBB, DL, TII->get(SC), StoreVal)
443     .addReg(StoreVal).addReg(Ptr).addImm(0);
444   BuildMI(loopMBB, DL, TII->get(BEQ))
445     .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB);
446 
447   //  sinkMBB:
448   //    and     maskedoldval1,oldval,mask
449   //    srl     srlres,maskedoldval1,shiftamt
450   //    sign_extend dest,srlres
451 
452   sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
453 
454   BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest)
455     .addReg(OldVal).addReg(Mask);
456   BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
457       .addReg(Dest).addReg(ShiftAmnt);
458 
459   if (STI->hasMips32r2()) {
460     BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
461   } else {
462     const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
463     BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
464         .addReg(Dest, RegState::Kill)
465         .addImm(ShiftImm);
466     BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
467         .addReg(Dest, RegState::Kill)
468         .addImm(ShiftImm);
469   }
470 
471   LivePhysRegs LiveRegs;
472   computeAndAddLiveIns(LiveRegs, *loopMBB);
473   computeAndAddLiveIns(LiveRegs, *sinkMBB);
474   computeAndAddLiveIns(LiveRegs, *exitMBB);
475 
476   NMBBI = BB.end();
477   I->eraseFromParent();
478 
479   return true;
480 }
481 
482 bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB,
483                                          MachineBasicBlock::iterator I,
484                                          MachineBasicBlock::iterator &NMBBI,
485                                          unsigned Size) {
486   MachineFunction *MF = BB.getParent();
487 
488   const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
489   DebugLoc DL = I->getDebugLoc();
490 
491   unsigned LL, SC, ZERO, BEQ;
492 
493   if (Size == 4) {
494     if (STI->inMicroMipsMode()) {
495       LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
496       SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
497       BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
498     } else {
499       LL = STI->hasMips32r6()
500                ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
501                : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
502       SC = STI->hasMips32r6()
503                ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
504                : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
505       BEQ = Mips::BEQ;
506     }
507 
508     ZERO = Mips::ZERO;
509   } else {
510     LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
511     SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
512     ZERO = Mips::ZERO_64;
513     BEQ = Mips::BEQ64;
514   }
515 
516   unsigned OldVal = I->getOperand(0).getReg();
517   unsigned Ptr = I->getOperand(1).getReg();
518   unsigned Incr = I->getOperand(2).getReg();
519   unsigned Scratch = I->getOperand(3).getReg();
520 
521   unsigned Opcode = 0;
522   unsigned OR = 0;
523   unsigned AND = 0;
524   unsigned NOR = 0;
525   bool IsNand = false;
526   switch (I->getOpcode()) {
527   case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
528     Opcode = Mips::ADDu;
529     break;
530   case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
531     Opcode = Mips::SUBu;
532     break;
533   case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
534     Opcode = Mips::AND;
535     break;
536   case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
537     Opcode = Mips::OR;
538     break;
539   case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
540     Opcode = Mips::XOR;
541     break;
542   case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
543     IsNand = true;
544     AND = Mips::AND;
545     NOR = Mips::NOR;
546     break;
547   case Mips::ATOMIC_SWAP_I32_POSTRA:
548     OR = Mips::OR;
549     break;
550   case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
551     Opcode = Mips::DADDu;
552     break;
553   case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
554     Opcode = Mips::DSUBu;
555     break;
556   case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
557     Opcode = Mips::AND64;
558     break;
559   case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
560     Opcode = Mips::OR64;
561     break;
562   case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
563     Opcode = Mips::XOR64;
564     break;
565   case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
566     IsNand = true;
567     AND = Mips::AND64;
568     NOR = Mips::NOR64;
569     break;
570   case Mips::ATOMIC_SWAP_I64_POSTRA:
571     OR = Mips::OR64;
572     break;
573   default:
574     llvm_unreachable("Unknown pseudo atomic!");
575   }
576 
577   const BasicBlock *LLVM_BB = BB.getBasicBlock();
578   MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
579   MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
580   MachineFunction::iterator It = ++BB.getIterator();
581   MF->insert(It, loopMBB);
582   MF->insert(It, exitMBB);
583 
584   exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
585   exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
586 
587   BB.addSuccessor(loopMBB, BranchProbability::getOne());
588   loopMBB->addSuccessor(exitMBB);
589   loopMBB->addSuccessor(loopMBB);
590   loopMBB->normalizeSuccProbs();
591 
592   BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
593   assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!");
594   assert((OldVal != Incr) && "Clobbered the wrong reg!");
595   if (Opcode) {
596     BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr);
597   } else if (IsNand) {
598     assert(AND && NOR &&
599            "Unknown nand instruction for atomic pseudo expansion");
600     BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr);
601     BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch);
602   } else {
603     assert(OR && "Unknown instruction for atomic pseudo expansion!");
604     BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO);
605   }
606 
607   BuildMI(loopMBB, DL, TII->get(SC), Scratch).addReg(Scratch).addReg(Ptr).addImm(0);
608   BuildMI(loopMBB, DL, TII->get(BEQ)).addReg(Scratch).addReg(ZERO).addMBB(loopMBB);
609 
610   NMBBI = BB.end();
611   I->eraseFromParent();
612 
613   LivePhysRegs LiveRegs;
614   computeAndAddLiveIns(LiveRegs, *loopMBB);
615   computeAndAddLiveIns(LiveRegs, *exitMBB);
616 
617   return true;
618 }
619 
620 bool MipsExpandPseudo::expandMI(MachineBasicBlock &MBB,
621                                 MachineBasicBlock::iterator MBBI,
622                                 MachineBasicBlock::iterator &NMBB) {
623 
624   bool Modified = false;
625 
626   switch (MBBI->getOpcode()) {
627   case Mips::ATOMIC_CMP_SWAP_I32_POSTRA:
628   case Mips::ATOMIC_CMP_SWAP_I64_POSTRA:
629     return expandAtomicCmpSwap(MBB, MBBI, NMBB);
630   case Mips::ATOMIC_CMP_SWAP_I8_POSTRA:
631   case Mips::ATOMIC_CMP_SWAP_I16_POSTRA:
632     return expandAtomicCmpSwapSubword(MBB, MBBI, NMBB);
633   case Mips::ATOMIC_SWAP_I8_POSTRA:
634   case Mips::ATOMIC_SWAP_I16_POSTRA:
635   case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
636   case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
637   case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
638   case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
639   case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
640   case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
641   case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
642   case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
643   case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
644   case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
645   case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
646   case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
647     return expandAtomicBinOpSubword(MBB, MBBI, NMBB);
648   case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
649   case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
650   case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
651   case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
652   case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
653   case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
654   case Mips::ATOMIC_SWAP_I32_POSTRA:
655     return expandAtomicBinOp(MBB, MBBI, NMBB, 4);
656   case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
657   case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
658   case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
659   case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
660   case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
661   case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
662   case Mips::ATOMIC_SWAP_I64_POSTRA:
663     return expandAtomicBinOp(MBB, MBBI, NMBB, 8);
664   default:
665     return Modified;
666   }
667 }
668 
669 bool MipsExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
670   bool Modified = false;
671 
672   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
673   while (MBBI != E) {
674     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
675     Modified |= expandMI(MBB, MBBI, NMBBI);
676     MBBI = NMBBI;
677   }
678 
679   return Modified;
680 }
681 
682 bool MipsExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
683   STI = &static_cast<const MipsSubtarget &>(MF.getSubtarget());
684   TII = STI->getInstrInfo();
685 
686   bool Modified = false;
687   for (MachineFunction::iterator MFI = MF.begin(), E = MF.end(); MFI != E;
688        ++MFI)
689     Modified |= expandMBB(*MFI);
690 
691   if (Modified)
692     MF.RenumberBlocks();
693 
694   return Modified;
695 }
696 
697 /// createMipsExpandPseudoPass - returns an instance of the pseudo instruction
698 /// expansion pass.
699 FunctionPass *llvm::createMipsExpandPseudoPass() {
700   return new MipsExpandPseudo();
701 }
702