1 //===- InstCombineAtomicRMW.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the visit functions for atomic rmw instructions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "InstCombineInternal.h" 14 #include "llvm/IR/Instructions.h" 15 16 using namespace llvm; 17 18 namespace { 19 /// Return true if and only if the given instruction does not modify the memory 20 /// location referenced. Note that an idemptent atomicrmw may still have 21 /// ordering effects on nearby instructions, or be volatile. 22 /// TODO: Common w/ the version in AtomicExpandPass, and change the term used. 23 /// Idemptotent is confusing in this context. 24 bool isIdempotentRMW(AtomicRMWInst& RMWI) { 25 if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand())) 26 switch(RMWI.getOperation()) { 27 case AtomicRMWInst::FAdd: // -0.0 28 return CF->isZero() && CF->isNegative(); 29 case AtomicRMWInst::FSub: // +0.0 30 return CF->isZero() && !CF->isNegative(); 31 default: 32 return false; 33 }; 34 35 auto C = dyn_cast<ConstantInt>(RMWI.getValOperand()); 36 if(!C) 37 return false; 38 39 switch(RMWI.getOperation()) { 40 case AtomicRMWInst::Add: 41 case AtomicRMWInst::Sub: 42 case AtomicRMWInst::Or: 43 case AtomicRMWInst::Xor: 44 return C->isZero(); 45 case AtomicRMWInst::And: 46 return C->isMinusOne(); 47 case AtomicRMWInst::Min: 48 return C->isMaxValue(true); 49 case AtomicRMWInst::Max: 50 return C->isMinValue(true); 51 case AtomicRMWInst::UMin: 52 return C->isMaxValue(false); 53 case AtomicRMWInst::UMax: 54 return C->isMinValue(false); 55 default: 56 return false; 57 } 58 } 59 60 /// Return true if the given instruction always produces a value in memory 61 /// equivalent to its value operand. 62 bool isSaturating(AtomicRMWInst& RMWI) { 63 if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand())) 64 switch (RMWI.getOperation()) { 65 case AtomicRMWInst::FMax: 66 // maxnum(x, +inf) -> +inf 67 return !CF->isNegative() && CF->isInfinity(); 68 case AtomicRMWInst::FMin: 69 // minnum(x, -inf) -> +inf 70 return CF->isNegative() && CF->isInfinity(); 71 case AtomicRMWInst::FAdd: 72 case AtomicRMWInst::FSub: 73 return CF->isNaN(); 74 default: 75 return false; 76 }; 77 78 auto C = dyn_cast<ConstantInt>(RMWI.getValOperand()); 79 if(!C) 80 return false; 81 82 switch(RMWI.getOperation()) { 83 default: 84 return false; 85 case AtomicRMWInst::Xchg: 86 return true; 87 case AtomicRMWInst::Or: 88 return C->isAllOnesValue(); 89 case AtomicRMWInst::And: 90 return C->isZero(); 91 case AtomicRMWInst::Min: 92 return C->isMinValue(true); 93 case AtomicRMWInst::Max: 94 return C->isMaxValue(true); 95 case AtomicRMWInst::UMin: 96 return C->isMinValue(false); 97 case AtomicRMWInst::UMax: 98 return C->isMaxValue(false); 99 }; 100 } 101 } // namespace 102 103 Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) { 104 105 // Volatile RMWs perform a load and a store, we cannot replace this by just a 106 // load or just a store. We chose not to canonicalize out of general paranoia 107 // about user expectations around volatile. 108 if (RMWI.isVolatile()) 109 return nullptr; 110 111 // Any atomicrmw op which produces a known result in memory can be 112 // replaced w/an atomicrmw xchg. 113 if (isSaturating(RMWI) && 114 RMWI.getOperation() != AtomicRMWInst::Xchg) { 115 RMWI.setOperation(AtomicRMWInst::Xchg); 116 return &RMWI; 117 } 118 119 assert(RMWI.getOrdering() != AtomicOrdering::NotAtomic && 120 RMWI.getOrdering() != AtomicOrdering::Unordered && 121 "AtomicRMWs don't make sense with Unordered or NotAtomic"); 122 123 if (!isIdempotentRMW(RMWI)) 124 return nullptr; 125 126 // We chose to canonicalize all idempotent operations to an single 127 // operation code and constant. This makes it easier for the rest of the 128 // optimizer to match easily. The choices of or w/0 and fadd w/-0.0 are 129 // arbitrary. 130 if (RMWI.getType()->isIntegerTy() && 131 RMWI.getOperation() != AtomicRMWInst::Or) { 132 RMWI.setOperation(AtomicRMWInst::Or); 133 return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), 0)); 134 } else if (RMWI.getType()->isFloatingPointTy() && 135 RMWI.getOperation() != AtomicRMWInst::FAdd) { 136 RMWI.setOperation(AtomicRMWInst::FAdd); 137 return replaceOperand(RMWI, 1, ConstantFP::getNegativeZero(RMWI.getType())); 138 } 139 140 return nullptr; 141 } 142