1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass lowers atomic intrinsics to non-atomic form for use in a known 10 // non-preemptible environment. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/Utils/LowerAtomic.h" 15 #include "llvm/IR/Function.h" 16 #include "llvm/IR/IRBuilder.h" 17 #include "llvm/InitializePasses.h" 18 #include "llvm/Pass.h" 19 using namespace llvm; 20 21 #define DEBUG_TYPE "loweratomic" 22 23 bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) { 24 IRBuilder<> Builder(CXI); 25 Value *Ptr = CXI->getPointerOperand(); 26 Value *Cmp = CXI->getCompareOperand(); 27 Value *Val = CXI->getNewValOperand(); 28 29 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); 30 Value *Equal = Builder.CreateICmpEQ(Orig, Cmp); 31 Value *Res = Builder.CreateSelect(Equal, Val, Orig); 32 Builder.CreateStore(Res, Ptr); 33 34 Res = Builder.CreateInsertValue(PoisonValue::get(CXI->getType()), Orig, 0); 35 Res = Builder.CreateInsertValue(Res, Equal, 1); 36 37 CXI->replaceAllUsesWith(Res); 38 CXI->eraseFromParent(); 39 return true; 40 } 41 42 Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op, 43 IRBuilderBase &Builder, Value *Loaded, 44 Value *Val) { 45 Value *NewVal; 46 switch (Op) { 47 case AtomicRMWInst::Xchg: 48 return Val; 49 case AtomicRMWInst::Add: 50 return Builder.CreateAdd(Loaded, Val, "new"); 51 case AtomicRMWInst::Sub: 52 return Builder.CreateSub(Loaded, Val, "new"); 53 case AtomicRMWInst::And: 54 return Builder.CreateAnd(Loaded, Val, "new"); 55 case AtomicRMWInst::Nand: 56 return Builder.CreateNot(Builder.CreateAnd(Loaded, Val), "new"); 57 case AtomicRMWInst::Or: 58 return Builder.CreateOr(Loaded, Val, "new"); 59 case AtomicRMWInst::Xor: 60 return Builder.CreateXor(Loaded, Val, "new"); 61 case AtomicRMWInst::Max: 62 NewVal = Builder.CreateICmpSGT(Loaded, Val); 63 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 64 case AtomicRMWInst::Min: 65 NewVal = Builder.CreateICmpSLE(Loaded, Val); 66 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 67 case AtomicRMWInst::UMax: 68 NewVal = Builder.CreateICmpUGT(Loaded, Val); 69 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 70 case AtomicRMWInst::UMin: 71 NewVal = Builder.CreateICmpULE(Loaded, Val); 72 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 73 case AtomicRMWInst::FAdd: 74 return Builder.CreateFAdd(Loaded, Val, "new"); 75 case AtomicRMWInst::FSub: 76 return Builder.CreateFSub(Loaded, Val, "new"); 77 case AtomicRMWInst::FMax: 78 return Builder.CreateMaxNum(Loaded, Val); 79 case AtomicRMWInst::FMin: 80 return Builder.CreateMinNum(Loaded, Val); 81 case AtomicRMWInst::UIncWrap: { 82 Constant *One = ConstantInt::get(Loaded->getType(), 1); 83 Value *Inc = Builder.CreateAdd(Loaded, One); 84 Value *Cmp = Builder.CreateICmpUGE(Loaded, Val); 85 Constant *Zero = ConstantInt::get(Loaded->getType(), 0); 86 return Builder.CreateSelect(Cmp, Zero, Inc, "new"); 87 } 88 case AtomicRMWInst::UDecWrap: { 89 Constant *Zero = ConstantInt::get(Loaded->getType(), 0); 90 Constant *One = ConstantInt::get(Loaded->getType(), 1); 91 92 Value *Dec = Builder.CreateSub(Loaded, One); 93 Value *CmpEq0 = Builder.CreateICmpEQ(Loaded, Zero); 94 Value *CmpOldGtVal = Builder.CreateICmpUGT(Loaded, Val); 95 Value *Or = Builder.CreateOr(CmpEq0, CmpOldGtVal); 96 return Builder.CreateSelect(Or, Val, Dec, "new"); 97 } 98 default: 99 llvm_unreachable("Unknown atomic op"); 100 } 101 } 102 103 bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) { 104 IRBuilder<> Builder(RMWI); 105 Value *Ptr = RMWI->getPointerOperand(); 106 Value *Val = RMWI->getValOperand(); 107 108 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); 109 Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val); 110 Builder.CreateStore(Res, Ptr); 111 RMWI->replaceAllUsesWith(Orig); 112 RMWI->eraseFromParent(); 113 return true; 114 } 115