45   AtomicRMWInst::BinOp Op;
77   Value *buildReduction(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
79   Value *buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
84   buildScanIteratively(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
88   void optimizeAtomic(Instruction &I, AtomicRMWInst::BinOp Op, unsigned ValIdx,
102   void visitAtomicRMWInst(AtomicRMWInst &I);
195 void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &I) {  in visitAtomicRMWInst()
205   AtomicRMWInst::BinOp Op = I.getOperation();  in visitAtomicRMWInst()
210   case AtomicRMWInst::Add:  in visitAtomicRMWInst()
211   case AtomicRMWInst::Sub:  in visitAtomicRMWInst()
212   case AtomicRMWInst::And:  in visitAtomicRMWInst()
213   case AtomicRMWInst::Or:  in visitAtomicRMWInst()
214   case AtomicRMWInst::Xor:  in visitAtomicRMWInst()
215   case AtomicRMWInst::Max:  in visitAtomicRMWInst()
216   case AtomicRMWInst::Min:  in visitAtomicRMWInst()
217   case AtomicRMWInst::UMax:  in visitAtomicRMWInst()
218   case AtomicRMWInst::UMin:  in visitAtomicRMWInst()
219   case AtomicRMWInst::FAdd:  in visitAtomicRMWInst()
220   case AtomicRMWInst::FSub:  in visitAtomicRMWInst()
221   case AtomicRMWInst::FMax:  in visitAtomicRMWInst()
222   case AtomicRMWInst::FMin:  in visitAtomicRMWInst()
227   if (AtomicRMWInst::isFPOperation(Op) &&  in visitAtomicRMWInst()
264   AtomicRMWInst::BinOp Op;  in visitIntrinsicInst()
273     Op = AtomicRMWInst::Add;  in visitIntrinsicInst()
279     Op = AtomicRMWInst::Sub;  in visitIntrinsicInst()
285     Op = AtomicRMWInst::And;  in visitIntrinsicInst()
291     Op = AtomicRMWInst::Or;  in visitIntrinsicInst()
297     Op = AtomicRMWInst::Xor;  in visitIntrinsicInst()
303     Op = AtomicRMWInst::Min;  in visitIntrinsicInst()
309     Op = AtomicRMWInst::UMin;  in visitIntrinsicInst()
315     Op = AtomicRMWInst::Max;  in visitIntrinsicInst()
321     Op = AtomicRMWInst::UMax;  in visitIntrinsicInst()
359 static Value *buildNonAtomicBinOp(IRBuilder<> &B, AtomicRMWInst::BinOp Op,  in buildNonAtomicBinOp()
366   case AtomicRMWInst::Add:  in buildNonAtomicBinOp()
368   case AtomicRMWInst::FAdd:  in buildNonAtomicBinOp()
370   case AtomicRMWInst::Sub:  in buildNonAtomicBinOp()
372   case AtomicRMWInst::FSub:  in buildNonAtomicBinOp()
374   case AtomicRMWInst::And:  in buildNonAtomicBinOp()
376   case AtomicRMWInst::Or:  in buildNonAtomicBinOp()
378   case AtomicRMWInst::Xor:  in buildNonAtomicBinOp()
381   case AtomicRMWInst::Max:  in buildNonAtomicBinOp()
384   case AtomicRMWInst::Min:  in buildNonAtomicBinOp()
387   case AtomicRMWInst::UMax:  in buildNonAtomicBinOp()
390   case AtomicRMWInst::UMin:  in buildNonAtomicBinOp()
393   case AtomicRMWInst::FMax:  in buildNonAtomicBinOp()
395   case AtomicRMWInst::FMin:  in buildNonAtomicBinOp()
405                                                  AtomicRMWInst::BinOp Op,  in buildReduction()
451                                             AtomicRMWInst::BinOp Op, Value *V,  in buildScan()
560     IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *const Identity, Value *V,  in buildScanIteratively()
624                                              AtomicRMWInst::BinOp Op) {  in getIdentityValueForAtomicOp()
630   case AtomicRMWInst::Add:  in getIdentityValueForAtomicOp()
631   case AtomicRMWInst::Sub:  in getIdentityValueForAtomicOp()
632   case AtomicRMWInst::Or:  in getIdentityValueForAtomicOp()
633   case AtomicRMWInst::Xor:  in getIdentityValueForAtomicOp()
634   case AtomicRMWInst::UMax:  in getIdentityValueForAtomicOp()
636   case AtomicRMWInst::And:  in getIdentityValueForAtomicOp()
637   case AtomicRMWInst::UMin:  in getIdentityValueForAtomicOp()
639   case AtomicRMWInst::Max:  in getIdentityValueForAtomicOp()
641   case AtomicRMWInst::Min:  in getIdentityValueForAtomicOp()
643   case AtomicRMWInst::FAdd:  in getIdentityValueForAtomicOp()
645   case AtomicRMWInst::FSub:  in getIdentityValueForAtomicOp()
647   case AtomicRMWInst::FMin:  in getIdentityValueForAtomicOp()
648   case AtomicRMWInst::FMax:  in getIdentityValueForAtomicOp()
663                                                AtomicRMWInst::BinOp Op,  in optimizeAtomic()
669   if (AtomicRMWInst::isFPOperation(Op)) {  in optimizeAtomic()
735   AtomicRMWInst::BinOp ScanOp = Op;  in optimizeAtomic()
736   if (Op == AtomicRMWInst::Sub) {  in optimizeAtomic()
737     ScanOp = AtomicRMWInst::Add;  in optimizeAtomic()
738   } else if (Op == AtomicRMWInst::FSub) {  in optimizeAtomic()
739     ScanOp = AtomicRMWInst::FAdd;  in optimizeAtomic()
790     case AtomicRMWInst::Add:  in optimizeAtomic()
791     case AtomicRMWInst::Sub: {  in optimizeAtomic()
799     case AtomicRMWInst::FAdd:  in optimizeAtomic()
800     case AtomicRMWInst::FSub: {  in optimizeAtomic()
807     case AtomicRMWInst::And:  in optimizeAtomic()
808     case AtomicRMWInst::Or:  in optimizeAtomic()
809     case AtomicRMWInst::Max:  in optimizeAtomic()
810     case AtomicRMWInst::Min:  in optimizeAtomic()
811     case AtomicRMWInst::UMax:  in optimizeAtomic()
812     case AtomicRMWInst::UMin:  in optimizeAtomic()
813     case AtomicRMWInst::FMin:  in optimizeAtomic()
814     case AtomicRMWInst::FMax:  in optimizeAtomic()
820     case AtomicRMWInst::Xor:  in optimizeAtomic()
933       case AtomicRMWInst::Add:  in optimizeAtomic()
934       case AtomicRMWInst::Sub:  in optimizeAtomic()
937       case AtomicRMWInst::And:  in optimizeAtomic()
938       case AtomicRMWInst::Or:  in optimizeAtomic()
939       case AtomicRMWInst::Max:  in optimizeAtomic()
940       case AtomicRMWInst::Min:  in optimizeAtomic()
941       case AtomicRMWInst::UMax:  in optimizeAtomic()
942       case AtomicRMWInst::UMin:  in optimizeAtomic()
943       case AtomicRMWInst::FMin:  in optimizeAtomic()
944       case AtomicRMWInst::FMax:  in optimizeAtomic()
947       case AtomicRMWInst::Xor:  in optimizeAtomic()
950       case AtomicRMWInst::FAdd:  in optimizeAtomic()
951       case AtomicRMWInst::FSub: {  in optimizeAtomic()