1 //===--- Atomic.cpp - Codegen of atomic operations ------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/Frontend/Atomic/Atomic.h"
10 #include "llvm/IR/DerivedTypes.h"
11 #include "llvm/IR/IRBuilder.h"
12 #include <utility>
13
14 using namespace llvm;
15
shouldCastToInt(Type * ValTy,bool CmpXchg)16 bool AtomicInfo::shouldCastToInt(Type *ValTy, bool CmpXchg) {
17 if (ValTy->isFloatingPointTy())
18 return ValTy->isX86_FP80Ty() || CmpXchg;
19 return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
20 }
21
EmitAtomicLoadOp(AtomicOrdering AO,bool IsVolatile,bool CmpXchg)22 Value *AtomicInfo::EmitAtomicLoadOp(AtomicOrdering AO, bool IsVolatile,
23 bool CmpXchg) {
24 Value *Ptr = getAtomicPointer();
25 Type *AtomicTy = Ty;
26 if (shouldCastToInt(Ty, CmpXchg))
27 AtomicTy = IntegerType::get(getLLVMContext(), AtomicSizeInBits);
28 LoadInst *Load =
29 Builder->CreateAlignedLoad(AtomicTy, Ptr, AtomicAlign, "atomic-load");
30 Load->setAtomic(AO);
31 if (IsVolatile)
32 Load->setVolatile(true);
33 decorateWithTBAA(Load);
34 return Load;
35 }
36
EmitAtomicLibcall(StringRef fnName,Type * ResultType,ArrayRef<Value * > Args)37 CallInst *AtomicInfo::EmitAtomicLibcall(StringRef fnName, Type *ResultType,
38 ArrayRef<Value *> Args) {
39 LLVMContext &ctx = Builder->getContext();
40 SmallVector<Type *, 6> ArgTys;
41 for (Value *Arg : Args)
42 ArgTys.push_back(Arg->getType());
43 FunctionType *FnType = FunctionType::get(ResultType, ArgTys, false);
44 Module *M = Builder->GetInsertBlock()->getModule();
45
46 // TODO: Use llvm::TargetLowering for Libcall ABI
47 AttrBuilder fnAttrBuilder(ctx);
48 fnAttrBuilder.addAttribute(Attribute::NoUnwind);
49 fnAttrBuilder.addAttribute(Attribute::WillReturn);
50 AttributeList fnAttrs =
51 AttributeList::get(ctx, AttributeList::FunctionIndex, fnAttrBuilder);
52 FunctionCallee LibcallFn = M->getOrInsertFunction(fnName, FnType, fnAttrs);
53 CallInst *Call = Builder->CreateCall(LibcallFn, Args);
54 return Call;
55 }
56
EmitAtomicCompareExchangeLibcall(Value * ExpectedVal,Value * DesiredVal,AtomicOrdering Success,AtomicOrdering Failure)57 std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchangeLibcall(
58 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,
59 AtomicOrdering Failure) {
60 LLVMContext &ctx = getLLVMContext();
61
62 // __atomic_compare_exchange's expected and desired are passed by pointers
63 // FIXME: types
64
65 // TODO: Get from llvm::TargetMachine / clang::TargetInfo
66 // if clang shares this codegen in future
67 constexpr uint64_t IntBits = 32;
68
69 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
70 // void *desired, int success, int failure);
71
72 Value *Args[6] = {
73 getAtomicSizeValue(),
74 getAtomicPointer(),
75 ExpectedVal,
76 DesiredVal,
77 Constant::getIntegerValue(IntegerType::get(ctx, IntBits),
78 APInt(IntBits, static_cast<uint64_t>(Success),
79 /*signed=*/true)),
80 Constant::getIntegerValue(IntegerType::get(ctx, IntBits),
81 APInt(IntBits, static_cast<uint64_t>(Failure),
82 /*signed=*/true)),
83 };
84 auto Result = EmitAtomicLibcall("__atomic_compare_exchange",
85 IntegerType::getInt1Ty(ctx), Args);
86 return std::make_pair(ExpectedVal, Result);
87 }
88
EmitAtomicCompareExchangeOp(Value * ExpectedVal,Value * DesiredVal,AtomicOrdering Success,AtomicOrdering Failure,bool IsVolatile,bool IsWeak)89 std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
90 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,
91 AtomicOrdering Failure, bool IsVolatile, bool IsWeak) {
92 // Do the atomic store.
93 Value *Addr = getAtomicAddressAsAtomicIntPointer();
94 auto *Inst = Builder->CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
95 getAtomicAlignment(), Success,
96 Failure, SyncScope::System);
97
98 // Other decoration.
99 Inst->setVolatile(IsVolatile);
100 Inst->setWeak(IsWeak);
101 auto *PreviousVal = Builder->CreateExtractValue(Inst, /*Idxs=*/0);
102 auto *SuccessFailureVal = Builder->CreateExtractValue(Inst, /*Idxs=*/1);
103 return std::make_pair(PreviousVal, SuccessFailureVal);
104 }
105
106 std::pair<LoadInst *, AllocaInst *>
EmitAtomicLoadLibcall(AtomicOrdering AO)107 AtomicInfo::EmitAtomicLoadLibcall(AtomicOrdering AO) {
108 LLVMContext &Ctx = getLLVMContext();
109 Type *SizedIntTy = Type::getIntNTy(Ctx, getAtomicSizeInBits());
110 Type *ResultTy;
111 SmallVector<Value *, 6> Args;
112 AttributeList Attr;
113 Module *M = Builder->GetInsertBlock()->getModule();
114 const DataLayout &DL = M->getDataLayout();
115 Args.push_back(
116 ConstantInt::get(DL.getIntPtrType(Ctx), this->getAtomicSizeInBits() / 8));
117
118 Value *PtrVal = getAtomicPointer();
119 PtrVal = Builder->CreateAddrSpaceCast(PtrVal, PointerType::getUnqual(Ctx));
120 Args.push_back(PtrVal);
121
122 auto CurrentIP = Builder->saveIP();
123 Builder->restoreIP(AllocaIP);
124 AllocaInst *AllocaResult =
125 CreateAlloca(Ty, getAtomicPointer()->getName() + "atomic.temp.load");
126 Builder->restoreIP(CurrentIP);
127 const Align AllocaAlignment = DL.getPrefTypeAlign(SizedIntTy);
128 AllocaResult->setAlignment(AllocaAlignment);
129 Args.push_back(AllocaResult);
130 Constant *OrderingVal =
131 ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(AO));
132 Args.push_back(OrderingVal);
133
134 ResultTy = Type::getVoidTy(Ctx);
135 SmallVector<Type *, 6> ArgTys;
136 for (Value *Arg : Args)
137 ArgTys.push_back(Arg->getType());
138 FunctionType *FnType = FunctionType::get(ResultTy, ArgTys, false);
139 FunctionCallee LibcallFn =
140 M->getOrInsertFunction("__atomic_load", FnType, Attr);
141 CallInst *Call = Builder->CreateCall(LibcallFn, Args);
142 Call->setAttributes(Attr);
143 return std::make_pair(
144 Builder->CreateAlignedLoad(Ty, AllocaResult, AllocaAlignment),
145 AllocaResult);
146 }
147
EmitAtomicStoreLibcall(AtomicOrdering AO,Value * Source)148 void AtomicInfo::EmitAtomicStoreLibcall(AtomicOrdering AO, Value *Source) {
149 LLVMContext &Ctx = getLLVMContext();
150 SmallVector<Value *, 6> Args;
151 AttributeList Attr;
152 Module *M = Builder->GetInsertBlock()->getModule();
153 const DataLayout &DL = M->getDataLayout();
154 Args.push_back(
155 ConstantInt::get(DL.getIntPtrType(Ctx), this->getAtomicSizeInBits() / 8));
156
157 Value *PtrVal = getAtomicPointer();
158 PtrVal = Builder->CreateAddrSpaceCast(PtrVal, PointerType::getUnqual(Ctx));
159 Args.push_back(PtrVal);
160
161 auto CurrentIP = Builder->saveIP();
162 Builder->restoreIP(AllocaIP);
163 Value *SourceAlloca = Builder->CreateAlloca(Source->getType());
164 Builder->restoreIP(CurrentIP);
165 Builder->CreateStore(Source, SourceAlloca);
166 SourceAlloca = Builder->CreatePointerBitCastOrAddrSpaceCast(
167 SourceAlloca, Builder->getPtrTy());
168 Args.push_back(SourceAlloca);
169
170 Constant *OrderingVal =
171 ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(AO));
172 Args.push_back(OrderingVal);
173
174 SmallVector<Type *, 6> ArgTys;
175 for (Value *Arg : Args)
176 ArgTys.push_back(Arg->getType());
177 FunctionType *FnType = FunctionType::get(Type::getVoidTy(Ctx), ArgTys, false);
178 FunctionCallee LibcallFn =
179 M->getOrInsertFunction("__atomic_store", FnType, Attr);
180 CallInst *Call = Builder->CreateCall(LibcallFn, Args);
181 Call->setAttributes(Attr);
182 }
183
EmitAtomicCompareExchange(Value * ExpectedVal,Value * DesiredVal,AtomicOrdering Success,AtomicOrdering Failure,bool IsVolatile,bool IsWeak)184 std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchange(
185 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,
186 AtomicOrdering Failure, bool IsVolatile, bool IsWeak) {
187 if (shouldUseLibcall())
188 return EmitAtomicCompareExchangeLibcall(ExpectedVal, DesiredVal, Success,
189 Failure);
190
191 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
192 Failure, IsVolatile, IsWeak);
193 return Res;
194 }
195