1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers atomic intrinsics to non-atomic form for use in a known
10 // non-preemptible environment.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Transforms/Utils/LowerAtomic.h"
15 #include "llvm/IR/Function.h"
16 #include "llvm/IR/IRBuilder.h"
17
18 using namespace llvm;
19
20 #define DEBUG_TYPE "loweratomic"
21
lowerAtomicCmpXchgInst(AtomicCmpXchgInst * CXI)22 bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
23 IRBuilder<> Builder(CXI);
24 Value *Ptr = CXI->getPointerOperand();
25 Value *Cmp = CXI->getCompareOperand();
26 Value *Val = CXI->getNewValOperand();
27
28 auto [Orig, Equal] =
29 buildCmpXchgValue(Builder, Ptr, Cmp, Val, CXI->getAlign());
30
31 Value *Res =
32 Builder.CreateInsertValue(PoisonValue::get(CXI->getType()), Orig, 0);
33 Res = Builder.CreateInsertValue(Res, Equal, 1);
34
35 CXI->replaceAllUsesWith(Res);
36 CXI->eraseFromParent();
37 return true;
38 }
39
buildCmpXchgValue(IRBuilderBase & Builder,Value * Ptr,Value * Cmp,Value * Val,Align Alignment)40 std::pair<Value *, Value *> llvm::buildCmpXchgValue(IRBuilderBase &Builder,
41 Value *Ptr, Value *Cmp,
42 Value *Val,
43 Align Alignment) {
44 LoadInst *Orig = Builder.CreateAlignedLoad(Val->getType(), Ptr, Alignment);
45 Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
46 Value *Res = Builder.CreateSelect(Equal, Val, Orig);
47 Builder.CreateAlignedStore(Res, Ptr, Alignment);
48
49 return {Orig, Equal};
50 }
51
buildAtomicRMWValue(AtomicRMWInst::BinOp Op,IRBuilderBase & Builder,Value * Loaded,Value * Val)52 Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,
53 IRBuilderBase &Builder, Value *Loaded,
54 Value *Val) {
55 Value *NewVal;
56 switch (Op) {
57 case AtomicRMWInst::Xchg:
58 return Val;
59 case AtomicRMWInst::Add:
60 return Builder.CreateAdd(Loaded, Val, "new");
61 case AtomicRMWInst::Sub:
62 return Builder.CreateSub(Loaded, Val, "new");
63 case AtomicRMWInst::And:
64 return Builder.CreateAnd(Loaded, Val, "new");
65 case AtomicRMWInst::Nand:
66 return Builder.CreateNot(Builder.CreateAnd(Loaded, Val), "new");
67 case AtomicRMWInst::Or:
68 return Builder.CreateOr(Loaded, Val, "new");
69 case AtomicRMWInst::Xor:
70 return Builder.CreateXor(Loaded, Val, "new");
71 case AtomicRMWInst::Max:
72 NewVal = Builder.CreateICmpSGT(Loaded, Val);
73 return Builder.CreateSelect(NewVal, Loaded, Val, "new");
74 case AtomicRMWInst::Min:
75 NewVal = Builder.CreateICmpSLE(Loaded, Val);
76 return Builder.CreateSelect(NewVal, Loaded, Val, "new");
77 case AtomicRMWInst::UMax:
78 NewVal = Builder.CreateICmpUGT(Loaded, Val);
79 return Builder.CreateSelect(NewVal, Loaded, Val, "new");
80 case AtomicRMWInst::UMin:
81 NewVal = Builder.CreateICmpULE(Loaded, Val);
82 return Builder.CreateSelect(NewVal, Loaded, Val, "new");
83 case AtomicRMWInst::FAdd:
84 return Builder.CreateFAdd(Loaded, Val, "new");
85 case AtomicRMWInst::FSub:
86 return Builder.CreateFSub(Loaded, Val, "new");
87 case AtomicRMWInst::FMax:
88 return Builder.CreateMaxNum(Loaded, Val);
89 case AtomicRMWInst::FMin:
90 return Builder.CreateMinNum(Loaded, Val);
91 case AtomicRMWInst::FMaximum:
92 return Builder.CreateMaximum(Loaded, Val);
93 case AtomicRMWInst::FMinimum:
94 return Builder.CreateMinimum(Loaded, Val);
95 case AtomicRMWInst::UIncWrap: {
96 Constant *One = ConstantInt::get(Loaded->getType(), 1);
97 Value *Inc = Builder.CreateAdd(Loaded, One);
98 Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
99 Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
100 return Builder.CreateSelect(Cmp, Zero, Inc, "new");
101 }
102 case AtomicRMWInst::UDecWrap: {
103 Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
104 Constant *One = ConstantInt::get(Loaded->getType(), 1);
105
106 Value *Dec = Builder.CreateSub(Loaded, One);
107 Value *CmpEq0 = Builder.CreateICmpEQ(Loaded, Zero);
108 Value *CmpOldGtVal = Builder.CreateICmpUGT(Loaded, Val);
109 Value *Or = Builder.CreateOr(CmpEq0, CmpOldGtVal);
110 return Builder.CreateSelect(Or, Val, Dec, "new");
111 }
112 case AtomicRMWInst::USubCond: {
113 Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
114 Value *Sub = Builder.CreateSub(Loaded, Val);
115 return Builder.CreateSelect(Cmp, Sub, Loaded, "new");
116 }
117 case AtomicRMWInst::USubSat:
118 return Builder.CreateIntrinsic(Intrinsic::usub_sat, Loaded->getType(),
119 {Loaded, Val}, nullptr, "new");
120 default:
121 llvm_unreachable("Unknown atomic op");
122 }
123 }
124
lowerAtomicRMWInst(AtomicRMWInst * RMWI)125 bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {
126 IRBuilder<> Builder(RMWI);
127 Builder.setIsFPConstrained(
128 RMWI->getFunction()->hasFnAttribute(Attribute::StrictFP));
129
130 Value *Ptr = RMWI->getPointerOperand();
131 Value *Val = RMWI->getValOperand();
132
133 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
134 Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val);
135 Builder.CreateStore(Res, Ptr);
136 RMWI->replaceAllUsesWith(Orig);
137 RMWI->eraseFromParent();
138 return true;
139 }
140