1 //===- InstCombineAtomicRMW.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the visit functions for atomic rmw instructions. 10 // 11 //===----------------------------------------------------------------------===// 12 #include "InstCombineInternal.h" 13 #include "llvm/IR/Instructions.h" 14 15 using namespace llvm; 16 17 namespace { 18 /// Return true if and only if the given instruction does not modify the memory 19 /// location referenced. Note that an idemptent atomicrmw may still have 20 /// ordering effects on nearby instructions, or be volatile. 21 /// TODO: Common w/ the version in AtomicExpandPass, and change the term used. 22 /// Idemptotent is confusing in this context. 23 bool isIdempotentRMW(AtomicRMWInst& RMWI) { 24 if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand())) 25 switch(RMWI.getOperation()) { 26 case AtomicRMWInst::FAdd: // -0.0 27 return CF->isZero() && CF->isNegative(); 28 case AtomicRMWInst::FSub: // +0.0 29 return CF->isZero() && !CF->isNegative(); 30 default: 31 return false; 32 }; 33 34 auto C = dyn_cast<ConstantInt>(RMWI.getValOperand()); 35 if(!C) 36 return false; 37 38 switch(RMWI.getOperation()) { 39 case AtomicRMWInst::Add: 40 case AtomicRMWInst::Sub: 41 case AtomicRMWInst::Or: 42 case AtomicRMWInst::Xor: 43 return C->isZero(); 44 case AtomicRMWInst::And: 45 return C->isMinusOne(); 46 case AtomicRMWInst::Min: 47 return C->isMaxValue(true); 48 case AtomicRMWInst::Max: 49 return C->isMinValue(true); 50 case AtomicRMWInst::UMin: 51 return C->isMaxValue(false); 52 case AtomicRMWInst::UMax: 53 return C->isMinValue(false); 54 default: 55 return false; 56 } 57 } 58 59 /// Return true if the given instruction always produces a value in memory 60 /// equivalent to its value operand. 61 bool isSaturating(AtomicRMWInst& RMWI) { 62 if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand())) 63 switch(RMWI.getOperation()) { 64 case AtomicRMWInst::FAdd: 65 case AtomicRMWInst::FSub: 66 return CF->isNaN(); 67 default: 68 return false; 69 }; 70 71 auto C = dyn_cast<ConstantInt>(RMWI.getValOperand()); 72 if(!C) 73 return false; 74 75 switch(RMWI.getOperation()) { 76 default: 77 return false; 78 case AtomicRMWInst::Xchg: 79 return true; 80 case AtomicRMWInst::Or: 81 return C->isAllOnesValue(); 82 case AtomicRMWInst::And: 83 return C->isZero(); 84 case AtomicRMWInst::Min: 85 return C->isMinValue(true); 86 case AtomicRMWInst::Max: 87 return C->isMaxValue(true); 88 case AtomicRMWInst::UMin: 89 return C->isMinValue(false); 90 case AtomicRMWInst::UMax: 91 return C->isMaxValue(false); 92 }; 93 } 94 } 95 96 Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) { 97 98 // Volatile RMWs perform a load and a store, we cannot replace this by just a 99 // load or just a store. We chose not to canonicalize out of general paranoia 100 // about user expectations around volatile. 101 if (RMWI.isVolatile()) 102 return nullptr; 103 104 // Any atomicrmw op which produces a known result in memory can be 105 // replaced w/an atomicrmw xchg. 106 if (isSaturating(RMWI) && 107 RMWI.getOperation() != AtomicRMWInst::Xchg) { 108 RMWI.setOperation(AtomicRMWInst::Xchg); 109 return &RMWI; 110 } 111 112 AtomicOrdering Ordering = RMWI.getOrdering(); 113 assert(Ordering != AtomicOrdering::NotAtomic && 114 Ordering != AtomicOrdering::Unordered && 115 "AtomicRMWs don't make sense with Unordered or NotAtomic"); 116 117 // Any atomicrmw xchg with no uses can be converted to a atomic store if the 118 // ordering is compatible. 119 if (RMWI.getOperation() == AtomicRMWInst::Xchg && 120 RMWI.use_empty()) { 121 if (Ordering != AtomicOrdering::Release && 122 Ordering != AtomicOrdering::Monotonic) 123 return nullptr; 124 auto *SI = new StoreInst(RMWI.getValOperand(), 125 RMWI.getPointerOperand(), &RMWI); 126 SI->setAtomic(Ordering, RMWI.getSyncScopeID()); 127 SI->setAlignment(DL.getABITypeAlignment(RMWI.getType())); 128 return eraseInstFromFunction(RMWI); 129 } 130 131 if (!isIdempotentRMW(RMWI)) 132 return nullptr; 133 134 // We chose to canonicalize all idempotent operations to an single 135 // operation code and constant. This makes it easier for the rest of the 136 // optimizer to match easily. The choices of or w/0 and fadd w/-0.0 are 137 // arbitrary. 138 if (RMWI.getType()->isIntegerTy() && 139 RMWI.getOperation() != AtomicRMWInst::Or) { 140 RMWI.setOperation(AtomicRMWInst::Or); 141 RMWI.setOperand(1, ConstantInt::get(RMWI.getType(), 0)); 142 return &RMWI; 143 } else if (RMWI.getType()->isFloatingPointTy() && 144 RMWI.getOperation() != AtomicRMWInst::FAdd) { 145 RMWI.setOperation(AtomicRMWInst::FAdd); 146 RMWI.setOperand(1, ConstantFP::getNegativeZero(RMWI.getType())); 147 return &RMWI; 148 } 149 150 // Check if the required ordering is compatible with an atomic load. 151 if (Ordering != AtomicOrdering::Acquire && 152 Ordering != AtomicOrdering::Monotonic) 153 return nullptr; 154 155 LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand()); 156 Load->setAtomic(Ordering, RMWI.getSyncScopeID()); 157 Load->setAlignment(DL.getABITypeAlignment(RMWI.getType())); 158 return Load; 159 } 160