1 //===- InstCombineAtomicRMW.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the visit functions for atomic rmw instructions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "InstCombineInternal.h" 14 #include "llvm/IR/Instructions.h" 15 #include "llvm/Transforms/InstCombine/InstCombiner.h" 16 17 using namespace llvm; 18 19 namespace { 20 /// Return true if and only if the given instruction does not modify the memory 21 /// location referenced. Note that an idemptent atomicrmw may still have 22 /// ordering effects on nearby instructions, or be volatile. 23 /// TODO: Common w/ the version in AtomicExpandPass, and change the term used. 24 /// Idemptotent is confusing in this context. 25 bool isIdempotentRMW(AtomicRMWInst& RMWI) { 26 if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand())) 27 switch(RMWI.getOperation()) { 28 case AtomicRMWInst::FAdd: // -0.0 29 return CF->isZero() && CF->isNegative(); 30 case AtomicRMWInst::FSub: // +0.0 31 return CF->isZero() && !CF->isNegative(); 32 default: 33 return false; 34 }; 35 36 auto C = dyn_cast<ConstantInt>(RMWI.getValOperand()); 37 if(!C) 38 return false; 39 40 switch(RMWI.getOperation()) { 41 case AtomicRMWInst::Add: 42 case AtomicRMWInst::Sub: 43 case AtomicRMWInst::Or: 44 case AtomicRMWInst::Xor: 45 return C->isZero(); 46 case AtomicRMWInst::And: 47 return C->isMinusOne(); 48 case AtomicRMWInst::Min: 49 return C->isMaxValue(true); 50 case AtomicRMWInst::Max: 51 return C->isMinValue(true); 52 case AtomicRMWInst::UMin: 53 return C->isMaxValue(false); 54 case AtomicRMWInst::UMax: 55 return C->isMinValue(false); 56 default: 57 return false; 58 } 59 } 60 61 /// Return true if the given instruction always produces a value in memory 62 /// equivalent to its value operand. 63 bool isSaturating(AtomicRMWInst& RMWI) { 64 if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand())) 65 switch(RMWI.getOperation()) { 66 case AtomicRMWInst::FAdd: 67 case AtomicRMWInst::FSub: 68 return CF->isNaN(); 69 default: 70 return false; 71 }; 72 73 auto C = dyn_cast<ConstantInt>(RMWI.getValOperand()); 74 if(!C) 75 return false; 76 77 switch(RMWI.getOperation()) { 78 default: 79 return false; 80 case AtomicRMWInst::Xchg: 81 return true; 82 case AtomicRMWInst::Or: 83 return C->isAllOnesValue(); 84 case AtomicRMWInst::And: 85 return C->isZero(); 86 case AtomicRMWInst::Min: 87 return C->isMinValue(true); 88 case AtomicRMWInst::Max: 89 return C->isMaxValue(true); 90 case AtomicRMWInst::UMin: 91 return C->isMinValue(false); 92 case AtomicRMWInst::UMax: 93 return C->isMaxValue(false); 94 }; 95 } 96 } // namespace 97 98 Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) { 99 100 // Volatile RMWs perform a load and a store, we cannot replace this by just a 101 // load or just a store. We chose not to canonicalize out of general paranoia 102 // about user expectations around volatile. 103 if (RMWI.isVolatile()) 104 return nullptr; 105 106 // Any atomicrmw op which produces a known result in memory can be 107 // replaced w/an atomicrmw xchg. 108 if (isSaturating(RMWI) && 109 RMWI.getOperation() != AtomicRMWInst::Xchg) { 110 RMWI.setOperation(AtomicRMWInst::Xchg); 111 return &RMWI; 112 } 113 114 AtomicOrdering Ordering = RMWI.getOrdering(); 115 assert(Ordering != AtomicOrdering::NotAtomic && 116 Ordering != AtomicOrdering::Unordered && 117 "AtomicRMWs don't make sense with Unordered or NotAtomic"); 118 119 // Any atomicrmw xchg with no uses can be converted to a atomic store if the 120 // ordering is compatible. 121 if (RMWI.getOperation() == AtomicRMWInst::Xchg && 122 RMWI.use_empty()) { 123 if (Ordering != AtomicOrdering::Release && 124 Ordering != AtomicOrdering::Monotonic) 125 return nullptr; 126 auto *SI = new StoreInst(RMWI.getValOperand(), 127 RMWI.getPointerOperand(), &RMWI); 128 SI->setAtomic(Ordering, RMWI.getSyncScopeID()); 129 SI->setAlignment(DL.getABITypeAlign(RMWI.getType())); 130 return eraseInstFromFunction(RMWI); 131 } 132 133 if (!isIdempotentRMW(RMWI)) 134 return nullptr; 135 136 // We chose to canonicalize all idempotent operations to an single 137 // operation code and constant. This makes it easier for the rest of the 138 // optimizer to match easily. The choices of or w/0 and fadd w/-0.0 are 139 // arbitrary. 140 if (RMWI.getType()->isIntegerTy() && 141 RMWI.getOperation() != AtomicRMWInst::Or) { 142 RMWI.setOperation(AtomicRMWInst::Or); 143 return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), 0)); 144 } else if (RMWI.getType()->isFloatingPointTy() && 145 RMWI.getOperation() != AtomicRMWInst::FAdd) { 146 RMWI.setOperation(AtomicRMWInst::FAdd); 147 return replaceOperand(RMWI, 1, ConstantFP::getNegativeZero(RMWI.getType())); 148 } 149 150 // Check if the required ordering is compatible with an atomic load. 151 if (Ordering != AtomicOrdering::Acquire && 152 Ordering != AtomicOrdering::Monotonic) 153 return nullptr; 154 155 LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand(), "", 156 false, DL.getABITypeAlign(RMWI.getType()), 157 Ordering, RMWI.getSyncScopeID()); 158 return Load; 159 } 160