651   bool expandLoad(const SIMemOpInfo &MOI,
655   bool expandStore(const SIMemOpInfo &MOI,
659   bool expandAtomicFence(const SIMemOpInfo &MOI,
663   bool expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI,
2602 bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI,  in expandLoad()  argument
2608   if (MOI.isAtomic()) {  in expandLoad()
2609     const AtomicOrdering Order = MOI.getOrdering();  in expandLoad()
2613       Changed |= CC->enableLoadCacheBypass(MI, MOI.getScope(),  in expandLoad()
2614                                            MOI.getOrderingAddrSpace());  in expandLoad()
2618       Changed |= CC->insertWait(MI, MOI.getScope(), MOI.getOrderingAddrSpace(),  in expandLoad()
2620                                 MOI.getIsCrossAddressSpaceOrdering(),  in expandLoad()
2626           MI, MOI.getScope(), MOI.getInstrAddrSpace(), SIMemOp::LOAD,  in expandLoad()
2627           MOI.getIsCrossAddressSpaceOrdering(), Position::AFTER, Order);  in expandLoad()
2628       Changed |= CC->insertAcquire(MI, MOI.getScope(),  in expandLoad()
2629                                    MOI.getOrderingAddrSpace(),  in expandLoad()
2640       MI, MOI.getInstrAddrSpace(), SIMemOp::LOAD, MOI.isVolatile(),  in expandLoad()
2641       MOI.isNonTemporal(), MOI.isLastUse());  in expandLoad()
2646 bool SIMemoryLegalizer::expandStore(const SIMemOpInfo &MOI,  in expandStore()  argument
2652   if (MOI.isAtomic()) {  in expandStore()
2653     if (MOI.getOrdering() == AtomicOrdering::Monotonic ||  in expandStore()
2654         MOI.getOrdering() == AtomicOrdering::Release ||  in expandStore()
2655         MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {  in expandStore()
2656       Changed |= CC->enableStoreCacheBypass(MI, MOI.getScope(),  in expandStore()
2657                                             MOI.getOrderingAddrSpace());  in expandStore()
2660     if (MOI.getOrdering() == AtomicOrdering::Release ||  in expandStore()
2661         MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)  in expandStore()
2662       Changed |= CC->insertRelease(MI, MOI.getScope(),  in expandStore()
2663                                    MOI.getOrderingAddrSpace(),  in expandStore()
2664                                    MOI.getIsCrossAddressSpaceOrdering(),  in expandStore()
2674       MI, MOI.getInstrAddrSpace(), SIMemOp::STORE, MOI.isVolatile(),  in expandStore()
2675       MOI.isNonTemporal());  in expandStore()
2683 bool SIMemoryLegalizer::expandAtomicFence(const SIMemOpInfo &MOI,  in expandAtomicFence()  argument
2694       getFenceAddrSpaceMMRA(*MI, MOI.getOrderingAddrSpace());  in expandAtomicFence()
2696   if (MOI.isAtomic()) {  in expandAtomicFence()
2697     const AtomicOrdering Order = MOI.getOrdering();  in expandAtomicFence()
2700           MI, MOI.getScope(), OrderingAddrSpace, SIMemOp::LOAD | SIMemOp::STORE,  in expandAtomicFence()
2701           MOI.getIsCrossAddressSpaceOrdering(), Position::BEFORE, Order);  in expandAtomicFence()
2714       Changed |= CC->insertRelease(MI, MOI.getScope(), OrderingAddrSpace,  in expandAtomicFence()
2715                                    MOI.getIsCrossAddressSpaceOrdering(),  in expandAtomicFence()
2726       Changed |= CC->insertAcquire(MI, MOI.getScope(), OrderingAddrSpace,  in expandAtomicFence()
2735 bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI,  in expandAtomicCmpxchgOrRmw()  argument
2741   if (MOI.isAtomic()) {  in expandAtomicCmpxchgOrRmw()
2742     const AtomicOrdering Order = MOI.getOrdering();  in expandAtomicCmpxchgOrRmw()
2747       Changed |= CC->enableRMWCacheBypass(MI, MOI.getScope(),  in expandAtomicCmpxchgOrRmw()
2748                                           MOI.getInstrAddrSpace());  in expandAtomicCmpxchgOrRmw()
2754         MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent)  in expandAtomicCmpxchgOrRmw()
2755       Changed |= CC->insertRelease(MI, MOI.getScope(),  in expandAtomicCmpxchgOrRmw()
2756                                    MOI.getOrderingAddrSpace(),  in expandAtomicCmpxchgOrRmw()
2757                                    MOI.getIsCrossAddressSpaceOrdering(),  in expandAtomicCmpxchgOrRmw()
2763         MOI.getFailureOrdering() == AtomicOrdering::Acquire ||  in expandAtomicCmpxchgOrRmw()
2764         MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) {  in expandAtomicCmpxchgOrRmw()
2766           MI, MOI.getScope(), MOI.getInstrAddrSpace(),  in expandAtomicCmpxchgOrRmw()
2768           MOI.getIsCrossAddressSpaceOrdering(), Position::AFTER, Order);  in expandAtomicCmpxchgOrRmw()
2769       Changed |= CC->insertAcquire(MI, MOI.getScope(),  in expandAtomicCmpxchgOrRmw()
2770                                    MOI.getOrderingAddrSpace(),  in expandAtomicCmpxchgOrRmw()
2825       if (const auto &MOI = MOA.getLoadInfo(MI))  in run()  local
2826         Changed |= expandLoad(*MOI, MI);  in run()
2827       else if (const auto &MOI = MOA.getStoreInfo(MI)) {  in run()  local
2828         Changed |= expandStore(*MOI, MI);  in run()
2829       } else if (const auto &MOI = MOA.getAtomicFenceInfo(MI))  in run()  local
2830         Changed |= expandAtomicFence(*MOI, MI);  in run()
2831       else if (const auto &MOI = MOA.getAtomicCmpxchgOrRmwInfo(MI))  in run()  local
2832         Changed |= expandAtomicCmpxchgOrRmw(*MOI, MI);  in run()