79   bool tryExpandAtomicRMW(AtomicRMWInst *AI);
80   AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI);
90       AtomicRMWInst *I, TargetLoweringBase::AtomicExpansionKind ExpansionKind);
91   AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
93   void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
105   bool isIdempotentRMW(AtomicRMWInst *RMWI);
106   bool simplifyIdempotentRMW(AtomicRMWInst *RMWI);
115   void expandAtomicRMWToLibcall(AtomicRMWInst *I);
119   llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
185 static unsigned getAtomicOpSize(AtomicRMWInst *RMWI) {  in getAtomicOpSize()
225     auto RMWI = dyn_cast<AtomicRMWInst>(I);  in run()
424 AtomicRMWInst *
425 AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {  in convertAtomicXchgToIntegerType()
438   auto *NewRMWI = Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, Addr, NewVal,  in convertAtomicXchgToIntegerType()
565   AtomicRMWInst *AI = Builder.CreateAtomicRMW(  in expandAtomicStore()
566       AtomicRMWInst::Xchg, SI->getPointerOperand(), SI->getValueOperand(),  in expandAtomicStore()
599 bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {  in tryExpandAtomicRMW()
648       AtomicRMWInst::BinOp Op = AI->getOperation();  in tryExpandAtomicRMW()
650       if (Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||  in tryExpandAtomicRMW()
651           Op == AtomicRMWInst::And) {  in tryExpandAtomicRMW()
835 static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,  in performMaskedAtomicOp()
843   case AtomicRMWInst::Xchg: {  in performMaskedAtomicOp()
848   case AtomicRMWInst::Or:  in performMaskedAtomicOp()
849   case AtomicRMWInst::Xor:  in performMaskedAtomicOp()
850   case AtomicRMWInst::And:  in performMaskedAtomicOp()
852   case AtomicRMWInst::Add:  in performMaskedAtomicOp()
853   case AtomicRMWInst::Sub:  in performMaskedAtomicOp()
854   case AtomicRMWInst::Nand: {  in performMaskedAtomicOp()
862   case AtomicRMWInst::Max:  in performMaskedAtomicOp()
863   case AtomicRMWInst::Min:  in performMaskedAtomicOp()
864   case AtomicRMWInst::UMax:  in performMaskedAtomicOp()
865   case AtomicRMWInst::UMin:  in performMaskedAtomicOp()
866   case AtomicRMWInst::FAdd:  in performMaskedAtomicOp()
867   case AtomicRMWInst::FSub:  in performMaskedAtomicOp()
868   case AtomicRMWInst::FMin:  in performMaskedAtomicOp()
869   case AtomicRMWInst::FMax:  in performMaskedAtomicOp()
870   case AtomicRMWInst::UIncWrap:  in performMaskedAtomicOp()
871   case AtomicRMWInst::UDecWrap: {  in performMaskedAtomicOp()
893     AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {  in expandPartwordAtomicRMW()
895   AtomicRMWInst::BinOp Op = AI->getOperation();  in expandPartwordAtomicRMW()
896   if (Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||  in expandPartwordAtomicRMW()
897       Op == AtomicRMWInst::And) {  in expandPartwordAtomicRMW()
911   if (Op == AtomicRMWInst::Xchg || Op == AtomicRMWInst::Add ||  in expandPartwordAtomicRMW()
912       Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Nand) {  in expandPartwordAtomicRMW()
972 AtomicRMWInst *AtomicExpandImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) {  in widenPartwordAtomicRMW()
974   AtomicRMWInst::BinOp Op = AI->getOperation();  in widenPartwordAtomicRMW()
976   assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||  in widenPartwordAtomicRMW()
977           Op == AtomicRMWInst::And) &&  in widenPartwordAtomicRMW()
990   if (Op == AtomicRMWInst::And)  in widenPartwordAtomicRMW()
996   AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(  in widenPartwordAtomicRMW()
1145 void AtomicExpandImpl::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {  in expandAtomicRMWToMaskedIntrinsic()
1156   AtomicRMWInst::BinOp RMWOp = AI->getOperation();  in expandAtomicRMWToMaskedIntrinsic()
1157   if (RMWOp == AtomicRMWInst::Max || RMWOp == AtomicRMWInst::Min)  in expandAtomicRMWToMaskedIntrinsic()
1534 bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {  in isIdempotentRMW()
1539   AtomicRMWInst::BinOp Op = RMWI->getOperation();  in isIdempotentRMW()
1541   case AtomicRMWInst::Add:  in isIdempotentRMW()
1542   case AtomicRMWInst::Sub:  in isIdempotentRMW()
1543   case AtomicRMWInst::Or:  in isIdempotentRMW()
1544   case AtomicRMWInst::Xor:  in isIdempotentRMW()
1546   case AtomicRMWInst::And:  in isIdempotentRMW()
1554 bool AtomicExpandImpl::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {  in simplifyIdempotentRMW()
1646 bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,  in expandAtomicRMWToCmpXchg()
1729 static ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) {  in GetRMWLibcall()
1760   case AtomicRMWInst::BAD_BINOP:  in GetRMWLibcall()
1762   case AtomicRMWInst::Xchg:  in GetRMWLibcall()
1764   case AtomicRMWInst::Add:  in GetRMWLibcall()
1766   case AtomicRMWInst::Sub:  in GetRMWLibcall()
1768   case AtomicRMWInst::And:  in GetRMWLibcall()
1770   case AtomicRMWInst::Or:  in GetRMWLibcall()
1772   case AtomicRMWInst::Xor:  in GetRMWLibcall()
1774   case AtomicRMWInst::Nand:  in GetRMWLibcall()
1776   case AtomicRMWInst::Max:  in GetRMWLibcall()
1777   case AtomicRMWInst::Min:  in GetRMWLibcall()
1778   case AtomicRMWInst::UMax:  in GetRMWLibcall()
1779   case AtomicRMWInst::UMin:  in GetRMWLibcall()
1780   case AtomicRMWInst::FMax:  in GetRMWLibcall()
1781   case AtomicRMWInst::FMin:  in GetRMWLibcall()
1782   case AtomicRMWInst::FAdd:  in GetRMWLibcall()
1783   case AtomicRMWInst::FSub:  in GetRMWLibcall()
1784   case AtomicRMWInst::UIncWrap:  in GetRMWLibcall()
1785   case AtomicRMWInst::UDecWrap:  in GetRMWLibcall()
1792 void AtomicExpandImpl::expandAtomicRMWToLibcall(AtomicRMWInst *I) {  in expandAtomicRMWToLibcall()