1 //===- InstCombineAtomicRMW.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the visit functions for atomic rmw instructions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "InstCombineInternal.h" 14 #include "llvm/IR/Instructions.h" 15 16 using namespace llvm; 17 18 namespace { 19 /// Return true if and only if the given instruction does not modify the memory 20 /// location referenced. Note that an idemptent atomicrmw may still have 21 /// ordering effects on nearby instructions, or be volatile. 22 /// TODO: Common w/ the version in AtomicExpandPass, and change the term used. 23 /// Idemptotent is confusing in this context. 24 bool isIdempotentRMW(AtomicRMWInst& RMWI) { 25 if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand())) 26 switch(RMWI.getOperation()) { 27 case AtomicRMWInst::FAdd: // -0.0 28 return CF->isZero() && CF->isNegative(); 29 case AtomicRMWInst::FSub: // +0.0 30 return CF->isZero() && !CF->isNegative(); 31 default: 32 return false; 33 }; 34 35 auto C = dyn_cast<ConstantInt>(RMWI.getValOperand()); 36 if(!C) 37 return false; 38 39 switch(RMWI.getOperation()) { 40 case AtomicRMWInst::Add: 41 case AtomicRMWInst::Sub: 42 case AtomicRMWInst::Or: 43 case AtomicRMWInst::Xor: 44 return C->isZero(); 45 case AtomicRMWInst::And: 46 return C->isMinusOne(); 47 case AtomicRMWInst::Min: 48 return C->isMaxValue(true); 49 case AtomicRMWInst::Max: 50 return C->isMinValue(true); 51 case AtomicRMWInst::UMin: 52 return C->isMaxValue(false); 53 case AtomicRMWInst::UMax: 54 return C->isMinValue(false); 55 default: 56 return false; 57 } 58 } 59 60 /// Return true if the given instruction always produces a value in memory 61 /// equivalent to its value operand. 62 bool isSaturating(AtomicRMWInst& RMWI) { 63 if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand())) 64 switch (RMWI.getOperation()) { 65 case AtomicRMWInst::FMax: 66 // maxnum(x, +inf) -> +inf 67 return !CF->isNegative() && CF->isInfinity(); 68 case AtomicRMWInst::FMin: 69 // minnum(x, -inf) -> +inf 70 return CF->isNegative() && CF->isInfinity(); 71 case AtomicRMWInst::FAdd: 72 case AtomicRMWInst::FSub: 73 return CF->isNaN(); 74 default: 75 return false; 76 }; 77 78 auto C = dyn_cast<ConstantInt>(RMWI.getValOperand()); 79 if(!C) 80 return false; 81 82 switch(RMWI.getOperation()) { 83 default: 84 return false; 85 case AtomicRMWInst::Xchg: 86 return true; 87 case AtomicRMWInst::Or: 88 return C->isAllOnesValue(); 89 case AtomicRMWInst::And: 90 return C->isZero(); 91 case AtomicRMWInst::Min: 92 return C->isMinValue(true); 93 case AtomicRMWInst::Max: 94 return C->isMaxValue(true); 95 case AtomicRMWInst::UMin: 96 return C->isMinValue(false); 97 case AtomicRMWInst::UMax: 98 return C->isMaxValue(false); 99 }; 100 } 101 } // namespace 102 103 Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) { 104 105 // Volatile RMWs perform a load and a store, we cannot replace this by just a 106 // load or just a store. We chose not to canonicalize out of general paranoia 107 // about user expectations around volatile. 108 if (RMWI.isVolatile()) 109 return nullptr; 110 111 // Any atomicrmw op which produces a known result in memory can be 112 // replaced w/an atomicrmw xchg. 113 if (isSaturating(RMWI) && 114 RMWI.getOperation() != AtomicRMWInst::Xchg) { 115 RMWI.setOperation(AtomicRMWInst::Xchg); 116 return &RMWI; 117 } 118 119 AtomicOrdering Ordering = RMWI.getOrdering(); 120 assert(Ordering != AtomicOrdering::NotAtomic && 121 Ordering != AtomicOrdering::Unordered && 122 "AtomicRMWs don't make sense with Unordered or NotAtomic"); 123 124 // Any atomicrmw xchg with no uses can be converted to a atomic store if the 125 // ordering is compatible. 126 if (RMWI.getOperation() == AtomicRMWInst::Xchg && 127 RMWI.use_empty()) { 128 if (Ordering != AtomicOrdering::Release && 129 Ordering != AtomicOrdering::Monotonic) 130 return nullptr; 131 auto *SI = new StoreInst(RMWI.getValOperand(), 132 RMWI.getPointerOperand(), &RMWI); 133 SI->setAtomic(Ordering, RMWI.getSyncScopeID()); 134 SI->setAlignment(DL.getABITypeAlign(RMWI.getType())); 135 return eraseInstFromFunction(RMWI); 136 } 137 138 if (!isIdempotentRMW(RMWI)) 139 return nullptr; 140 141 // We chose to canonicalize all idempotent operations to an single 142 // operation code and constant. This makes it easier for the rest of the 143 // optimizer to match easily. The choices of or w/0 and fadd w/-0.0 are 144 // arbitrary. 145 if (RMWI.getType()->isIntegerTy() && 146 RMWI.getOperation() != AtomicRMWInst::Or) { 147 RMWI.setOperation(AtomicRMWInst::Or); 148 return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), 0)); 149 } else if (RMWI.getType()->isFloatingPointTy() && 150 RMWI.getOperation() != AtomicRMWInst::FAdd) { 151 RMWI.setOperation(AtomicRMWInst::FAdd); 152 return replaceOperand(RMWI, 1, ConstantFP::getNegativeZero(RMWI.getType())); 153 } 154 155 // Check if the required ordering is compatible with an atomic load. 156 if (Ordering != AtomicOrdering::Acquire && 157 Ordering != AtomicOrdering::Monotonic) 158 return nullptr; 159 160 LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand(), "", 161 false, DL.getABITypeAlign(RMWI.getType()), 162 Ordering, RMWI.getSyncScopeID()); 163 return Load; 164 } 165