1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass lowers atomic intrinsics to non-atomic form for use in a known 10 // non-preemptible environment. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/Utils/LowerAtomic.h" 15 #include "llvm/IR/Function.h" 16 #include "llvm/IR/IRBuilder.h" 17 18 using namespace llvm; 19 20 #define DEBUG_TYPE "loweratomic" 21 22 bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) { 23 IRBuilder<> Builder(CXI); 24 Value *Ptr = CXI->getPointerOperand(); 25 Value *Cmp = CXI->getCompareOperand(); 26 Value *Val = CXI->getNewValOperand(); 27 28 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); 29 Value *Equal = Builder.CreateICmpEQ(Orig, Cmp); 30 Value *Res = Builder.CreateSelect(Equal, Val, Orig); 31 Builder.CreateStore(Res, Ptr); 32 33 Res = Builder.CreateInsertValue(PoisonValue::get(CXI->getType()), Orig, 0); 34 Res = Builder.CreateInsertValue(Res, Equal, 1); 35 36 CXI->replaceAllUsesWith(Res); 37 CXI->eraseFromParent(); 38 return true; 39 } 40 41 Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op, 42 IRBuilderBase &Builder, Value *Loaded, 43 Value *Val) { 44 Value *NewVal; 45 switch (Op) { 46 case AtomicRMWInst::Xchg: 47 return Val; 48 case AtomicRMWInst::Add: 49 return Builder.CreateAdd(Loaded, Val, "new"); 50 case AtomicRMWInst::Sub: 51 return Builder.CreateSub(Loaded, Val, "new"); 52 case AtomicRMWInst::And: 53 return Builder.CreateAnd(Loaded, Val, "new"); 54 case AtomicRMWInst::Nand: 55 return Builder.CreateNot(Builder.CreateAnd(Loaded, Val), "new"); 56 case AtomicRMWInst::Or: 57 return Builder.CreateOr(Loaded, Val, "new"); 58 case AtomicRMWInst::Xor: 59 return Builder.CreateXor(Loaded, Val, "new"); 60 case AtomicRMWInst::Max: 61 NewVal = Builder.CreateICmpSGT(Loaded, Val); 62 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 63 case AtomicRMWInst::Min: 64 NewVal = Builder.CreateICmpSLE(Loaded, Val); 65 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 66 case AtomicRMWInst::UMax: 67 NewVal = Builder.CreateICmpUGT(Loaded, Val); 68 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 69 case AtomicRMWInst::UMin: 70 NewVal = Builder.CreateICmpULE(Loaded, Val); 71 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 72 case AtomicRMWInst::FAdd: 73 return Builder.CreateFAdd(Loaded, Val, "new"); 74 case AtomicRMWInst::FSub: 75 return Builder.CreateFSub(Loaded, Val, "new"); 76 case AtomicRMWInst::FMax: 77 return Builder.CreateMaxNum(Loaded, Val); 78 case AtomicRMWInst::FMin: 79 return Builder.CreateMinNum(Loaded, Val); 80 case AtomicRMWInst::UIncWrap: { 81 Constant *One = ConstantInt::get(Loaded->getType(), 1); 82 Value *Inc = Builder.CreateAdd(Loaded, One); 83 Value *Cmp = Builder.CreateICmpUGE(Loaded, Val); 84 Constant *Zero = ConstantInt::get(Loaded->getType(), 0); 85 return Builder.CreateSelect(Cmp, Zero, Inc, "new"); 86 } 87 case AtomicRMWInst::UDecWrap: { 88 Constant *Zero = ConstantInt::get(Loaded->getType(), 0); 89 Constant *One = ConstantInt::get(Loaded->getType(), 1); 90 91 Value *Dec = Builder.CreateSub(Loaded, One); 92 Value *CmpEq0 = Builder.CreateICmpEQ(Loaded, Zero); 93 Value *CmpOldGtVal = Builder.CreateICmpUGT(Loaded, Val); 94 Value *Or = Builder.CreateOr(CmpEq0, CmpOldGtVal); 95 return Builder.CreateSelect(Or, Val, Dec, "new"); 96 } 97 default: 98 llvm_unreachable("Unknown atomic op"); 99 } 100 } 101 102 bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) { 103 IRBuilder<> Builder(RMWI); 104 Builder.setIsFPConstrained( 105 RMWI->getFunction()->hasFnAttribute(Attribute::StrictFP)); 106 107 Value *Ptr = RMWI->getPointerOperand(); 108 Value *Val = RMWI->getValOperand(); 109 110 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); 111 Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val); 112 Builder.CreateStore(Res, Ptr); 113 RMWI->replaceAllUsesWith(Orig); 114 RMWI->eraseFromParent(); 115 return true; 116 } 117