1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the code for emitting atomic operations.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "CGCall.h"
14 #include "CGRecordLayout.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/CodeGen/CGFunctionInfo.h"
20 #include "clang/Frontend/FrontendDiagnostic.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/Intrinsics.h"
24
25 using namespace clang;
26 using namespace CodeGen;
27
28 namespace {
29 class AtomicInfo {
30 CodeGenFunction &CGF;
31 QualType AtomicTy;
32 QualType ValueTy;
33 uint64_t AtomicSizeInBits;
34 uint64_t ValueSizeInBits;
35 CharUnits AtomicAlign;
36 CharUnits ValueAlign;
37 TypeEvaluationKind EvaluationKind;
38 bool UseLibcall;
39 LValue LVal;
40 CGBitFieldInfo BFI;
41 public:
AtomicInfo(CodeGenFunction & CGF,LValue & lvalue)42 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
43 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
44 EvaluationKind(TEK_Scalar), UseLibcall(true) {
45 assert(!lvalue.isGlobalReg());
46 ASTContext &C = CGF.getContext();
47 if (lvalue.isSimple()) {
48 AtomicTy = lvalue.getType();
49 if (auto *ATy = AtomicTy->getAs<AtomicType>())
50 ValueTy = ATy->getValueType();
51 else
52 ValueTy = AtomicTy;
53 EvaluationKind = CGF.getEvaluationKind(ValueTy);
54
55 uint64_t ValueAlignInBits;
56 uint64_t AtomicAlignInBits;
57 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
58 ValueSizeInBits = ValueTI.Width;
59 ValueAlignInBits = ValueTI.Align;
60
61 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
62 AtomicSizeInBits = AtomicTI.Width;
63 AtomicAlignInBits = AtomicTI.Align;
64
65 assert(ValueSizeInBits <= AtomicSizeInBits);
66 assert(ValueAlignInBits <= AtomicAlignInBits);
67
68 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
69 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
70 if (lvalue.getAlignment().isZero())
71 lvalue.setAlignment(AtomicAlign);
72
73 LVal = lvalue;
74 } else if (lvalue.isBitField()) {
75 ValueTy = lvalue.getType();
76 ValueSizeInBits = C.getTypeSize(ValueTy);
77 auto &OrigBFI = lvalue.getBitFieldInfo();
78 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
79 AtomicSizeInBits = C.toBits(
80 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
81 .alignTo(lvalue.getAlignment()));
82 llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF);
83 auto OffsetInChars =
84 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
85 lvalue.getAlignment();
86 llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(
87 CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity());
88 StoragePtr = CGF.Builder.CreateAddrSpaceCast(
89 StoragePtr, CGF.UnqualPtrTy, "atomic_bitfield_base");
90 BFI = OrigBFI;
91 BFI.Offset = Offset;
92 BFI.StorageSize = AtomicSizeInBits;
93 BFI.StorageOffset += OffsetInChars;
94 llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
95 LVal = LValue::MakeBitfield(
96 Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI,
97 lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo());
98 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
99 if (AtomicTy.isNull()) {
100 llvm::APInt Size(
101 /*numBits=*/32,
102 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
103 AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr,
104 ArraySizeModifier::Normal,
105 /*IndexTypeQuals=*/0);
106 }
107 AtomicAlign = ValueAlign = lvalue.getAlignment();
108 } else if (lvalue.isVectorElt()) {
109 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
110 ValueSizeInBits = C.getTypeSize(ValueTy);
111 AtomicTy = lvalue.getType();
112 AtomicSizeInBits = C.getTypeSize(AtomicTy);
113 AtomicAlign = ValueAlign = lvalue.getAlignment();
114 LVal = lvalue;
115 } else {
116 assert(lvalue.isExtVectorElt());
117 ValueTy = lvalue.getType();
118 ValueSizeInBits = C.getTypeSize(ValueTy);
119 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
120 lvalue.getType(), cast<llvm::FixedVectorType>(
121 lvalue.getExtVectorAddress().getElementType())
122 ->getNumElements());
123 AtomicSizeInBits = C.getTypeSize(AtomicTy);
124 AtomicAlign = ValueAlign = lvalue.getAlignment();
125 LVal = lvalue;
126 }
127 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
128 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
129 }
130
getAtomicType() const131 QualType getAtomicType() const { return AtomicTy; }
getValueType() const132 QualType getValueType() const { return ValueTy; }
getAtomicAlignment() const133 CharUnits getAtomicAlignment() const { return AtomicAlign; }
getAtomicSizeInBits() const134 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
getValueSizeInBits() const135 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
getEvaluationKind() const136 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
shouldUseLibcall() const137 bool shouldUseLibcall() const { return UseLibcall; }
getAtomicLValue() const138 const LValue &getAtomicLValue() const { return LVal; }
getAtomicPointer() const139 llvm::Value *getAtomicPointer() const {
140 if (LVal.isSimple())
141 return LVal.emitRawPointer(CGF);
142 else if (LVal.isBitField())
143 return LVal.getRawBitFieldPointer(CGF);
144 else if (LVal.isVectorElt())
145 return LVal.getRawVectorPointer(CGF);
146 assert(LVal.isExtVectorElt());
147 return LVal.getRawExtVectorPointer(CGF);
148 }
getAtomicAddress() const149 Address getAtomicAddress() const {
150 llvm::Type *ElTy;
151 if (LVal.isSimple())
152 ElTy = LVal.getAddress().getElementType();
153 else if (LVal.isBitField())
154 ElTy = LVal.getBitFieldAddress().getElementType();
155 else if (LVal.isVectorElt())
156 ElTy = LVal.getVectorAddress().getElementType();
157 else
158 ElTy = LVal.getExtVectorAddress().getElementType();
159 return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
160 }
161
getAtomicAddressAsAtomicIntPointer() const162 Address getAtomicAddressAsAtomicIntPointer() const {
163 return castToAtomicIntPointer(getAtomicAddress());
164 }
165
166 /// Is the atomic size larger than the underlying value type?
167 ///
168 /// Note that the absence of padding does not mean that atomic
169 /// objects are completely interchangeable with non-atomic
170 /// objects: we might have promoted the alignment of a type
171 /// without making it bigger.
hasPadding() const172 bool hasPadding() const {
173 return (ValueSizeInBits != AtomicSizeInBits);
174 }
175
176 bool emitMemSetZeroIfNecessary() const;
177
getAtomicSizeValue() const178 llvm::Value *getAtomicSizeValue() const {
179 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
180 return CGF.CGM.getSize(size);
181 }
182
183 /// Cast the given pointer to an integer pointer suitable for atomic
184 /// operations if the source.
185 Address castToAtomicIntPointer(Address Addr) const;
186
187 /// If Addr is compatible with the iN that will be used for an atomic
188 /// operation, bitcast it. Otherwise, create a temporary that is suitable
189 /// and copy the value across.
190 Address convertToAtomicIntPointer(Address Addr) const;
191
192 /// Turn an atomic-layout object into an r-value.
193 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
194 SourceLocation loc, bool AsValue) const;
195
196 llvm::Value *getScalarRValValueOrNull(RValue RVal) const;
197
198 /// Converts an rvalue to integer value if needed.
199 llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const;
200
201 RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot,
202 SourceLocation Loc, bool AsValue,
203 bool CmpXchg = false) const;
204
205 /// Copy an atomic r-value into atomic-layout memory.
206 void emitCopyIntoMemory(RValue rvalue) const;
207
208 /// Project an l-value down to the value field.
projectValue() const209 LValue projectValue() const {
210 assert(LVal.isSimple());
211 Address addr = getAtomicAddress();
212 if (hasPadding())
213 addr = CGF.Builder.CreateStructGEP(addr, 0);
214
215 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
216 LVal.getBaseInfo(), LVal.getTBAAInfo());
217 }
218
219 /// Emits atomic load.
220 /// \returns Loaded value.
221 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
222 bool AsValue, llvm::AtomicOrdering AO,
223 bool IsVolatile);
224
225 /// Emits atomic compare-and-exchange sequence.
226 /// \param Expected Expected value.
227 /// \param Desired Desired value.
228 /// \param Success Atomic ordering for success operation.
229 /// \param Failure Atomic ordering for failed operation.
230 /// \param IsWeak true if atomic operation is weak, false otherwise.
231 /// \returns Pair of values: previous value from storage (value type) and
232 /// boolean flag (i1 type) with true if success and false otherwise.
233 std::pair<RValue, llvm::Value *>
234 EmitAtomicCompareExchange(RValue Expected, RValue Desired,
235 llvm::AtomicOrdering Success =
236 llvm::AtomicOrdering::SequentiallyConsistent,
237 llvm::AtomicOrdering Failure =
238 llvm::AtomicOrdering::SequentiallyConsistent,
239 bool IsWeak = false);
240
241 /// Emits atomic update.
242 /// \param AO Atomic ordering.
243 /// \param UpdateOp Update operation for the current lvalue.
244 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
245 const llvm::function_ref<RValue(RValue)> &UpdateOp,
246 bool IsVolatile);
247 /// Emits atomic update.
248 /// \param AO Atomic ordering.
249 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
250 bool IsVolatile);
251
252 /// Materialize an atomic r-value in atomic-layout memory.
253 Address materializeRValue(RValue rvalue) const;
254
255 /// Creates temp alloca for intermediate operations on atomic value.
256 Address CreateTempAlloca() const;
257 private:
258 bool requiresMemSetZero(llvm::Type *type) const;
259
260
261 /// Emits atomic load as a libcall.
262 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
263 llvm::AtomicOrdering AO, bool IsVolatile);
264 /// Emits atomic load as LLVM instruction.
265 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,
266 bool CmpXchg = false);
267 /// Emits atomic compare-and-exchange op as a libcall.
268 llvm::Value *EmitAtomicCompareExchangeLibcall(
269 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
270 llvm::AtomicOrdering Success =
271 llvm::AtomicOrdering::SequentiallyConsistent,
272 llvm::AtomicOrdering Failure =
273 llvm::AtomicOrdering::SequentiallyConsistent);
274 /// Emits atomic compare-and-exchange op as LLVM instruction.
275 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
276 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
277 llvm::AtomicOrdering Success =
278 llvm::AtomicOrdering::SequentiallyConsistent,
279 llvm::AtomicOrdering Failure =
280 llvm::AtomicOrdering::SequentiallyConsistent,
281 bool IsWeak = false);
282 /// Emit atomic update as libcalls.
283 void
284 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
285 const llvm::function_ref<RValue(RValue)> &UpdateOp,
286 bool IsVolatile);
287 /// Emit atomic update as LLVM instructions.
288 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
289 const llvm::function_ref<RValue(RValue)> &UpdateOp,
290 bool IsVolatile);
291 /// Emit atomic update as libcalls.
292 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
293 bool IsVolatile);
294 /// Emit atomic update as LLVM instructions.
295 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
296 bool IsVolatile);
297 };
298 }
299
CreateTempAlloca() const300 Address AtomicInfo::CreateTempAlloca() const {
301 Address TempAlloca = CGF.CreateMemTemp(
302 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
303 : AtomicTy,
304 getAtomicAlignment(),
305 "atomic-temp");
306 // Cast to pointer to value type for bitfields.
307 if (LVal.isBitField())
308 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
309 TempAlloca, getAtomicAddress().getType(),
310 getAtomicAddress().getElementType());
311 return TempAlloca;
312 }
313
emitAtomicLibcall(CodeGenFunction & CGF,StringRef fnName,QualType resultType,CallArgList & args)314 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
315 StringRef fnName,
316 QualType resultType,
317 CallArgList &args) {
318 const CGFunctionInfo &fnInfo =
319 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
320 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
321 llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
322 fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
323 fnAttrB.addAttribute(llvm::Attribute::WillReturn);
324 llvm::AttributeList fnAttrs = llvm::AttributeList::get(
325 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
326
327 llvm::FunctionCallee fn =
328 CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
329 auto callee = CGCallee::forDirect(fn);
330 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
331 }
332
333 /// Does a store of the given IR type modify the full expected width?
isFullSizeType(CodeGenModule & CGM,llvm::Type * type,uint64_t expectedSize)334 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
335 uint64_t expectedSize) {
336 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
337 }
338
339 /// Does the atomic type require memsetting to zero before initialization?
340 ///
341 /// The IR type is provided as a way of making certain queries faster.
requiresMemSetZero(llvm::Type * type) const342 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
343 // If the atomic type has size padding, we definitely need a memset.
344 if (hasPadding()) return true;
345
346 // Otherwise, do some simple heuristics to try to avoid it:
347 switch (getEvaluationKind()) {
348 // For scalars and complexes, check whether the store size of the
349 // type uses the full size.
350 case TEK_Scalar:
351 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
352 case TEK_Complex:
353 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
354 AtomicSizeInBits / 2);
355
356 // Padding in structs has an undefined bit pattern. User beware.
357 case TEK_Aggregate:
358 return false;
359 }
360 llvm_unreachable("bad evaluation kind");
361 }
362
emitMemSetZeroIfNecessary() const363 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
364 assert(LVal.isSimple());
365 Address addr = LVal.getAddress();
366 if (!requiresMemSetZero(addr.getElementType()))
367 return false;
368
369 CGF.Builder.CreateMemSet(
370 addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0),
371 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
372 LVal.getAlignment().getAsAlign());
373 return true;
374 }
375
emitAtomicCmpXchg(CodeGenFunction & CGF,AtomicExpr * E,bool IsWeak,Address Dest,Address Ptr,Address Val1,Address Val2,uint64_t Size,llvm::AtomicOrdering SuccessOrder,llvm::AtomicOrdering FailureOrder,llvm::SyncScope::ID Scope)376 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
377 Address Dest, Address Ptr,
378 Address Val1, Address Val2,
379 uint64_t Size,
380 llvm::AtomicOrdering SuccessOrder,
381 llvm::AtomicOrdering FailureOrder,
382 llvm::SyncScope::ID Scope) {
383 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
384 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
385 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
386
387 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
388 Ptr, Expected, Desired, SuccessOrder, FailureOrder, Scope);
389 Pair->setVolatile(E->isVolatile());
390 Pair->setWeak(IsWeak);
391 CGF.getTargetHooks().setTargetAtomicMetadata(CGF, *Pair, E);
392
393 // Cmp holds the result of the compare-exchange operation: true on success,
394 // false on failure.
395 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
396 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
397
398 // This basic block is used to hold the store instruction if the operation
399 // failed.
400 llvm::BasicBlock *StoreExpectedBB =
401 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
402
403 // This basic block is the exit point of the operation, we should end up
404 // here regardless of whether or not the operation succeeded.
405 llvm::BasicBlock *ContinueBB =
406 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
407
408 // Update Expected if Expected isn't equal to Old, otherwise branch to the
409 // exit point.
410 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
411
412 CGF.Builder.SetInsertPoint(StoreExpectedBB);
413 // Update the memory at Expected with Old's value.
414 auto *I = CGF.Builder.CreateStore(Old, Val1);
415 CGF.addInstToCurrentSourceAtom(I, Old);
416
417 // Finally, branch to the exit point.
418 CGF.Builder.CreateBr(ContinueBB);
419
420 CGF.Builder.SetInsertPoint(ContinueBB);
421 // Update the memory at Dest with Cmp's value.
422 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
423 }
424
425 /// Given an ordering required on success, emit all possible cmpxchg
426 /// instructions to cope with the provided (but possibly only dynamically known)
427 /// FailureOrder.
emitAtomicCmpXchgFailureSet(CodeGenFunction & CGF,AtomicExpr * E,bool IsWeak,Address Dest,Address Ptr,Address Val1,Address Val2,llvm::Value * FailureOrderVal,uint64_t Size,llvm::AtomicOrdering SuccessOrder,llvm::SyncScope::ID Scope)428 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
429 bool IsWeak, Address Dest, Address Ptr,
430 Address Val1, Address Val2,
431 llvm::Value *FailureOrderVal,
432 uint64_t Size,
433 llvm::AtomicOrdering SuccessOrder,
434 llvm::SyncScope::ID Scope) {
435 llvm::AtomicOrdering FailureOrder;
436 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
437 auto FOS = FO->getSExtValue();
438 if (!llvm::isValidAtomicOrderingCABI(FOS))
439 FailureOrder = llvm::AtomicOrdering::Monotonic;
440 else
441 switch ((llvm::AtomicOrderingCABI)FOS) {
442 case llvm::AtomicOrderingCABI::relaxed:
443 // 31.7.2.18: "The failure argument shall not be memory_order_release
444 // nor memory_order_acq_rel". Fallback to monotonic.
445 case llvm::AtomicOrderingCABI::release:
446 case llvm::AtomicOrderingCABI::acq_rel:
447 FailureOrder = llvm::AtomicOrdering::Monotonic;
448 break;
449 case llvm::AtomicOrderingCABI::consume:
450 case llvm::AtomicOrderingCABI::acquire:
451 FailureOrder = llvm::AtomicOrdering::Acquire;
452 break;
453 case llvm::AtomicOrderingCABI::seq_cst:
454 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
455 break;
456 }
457 // Prior to c++17, "the failure argument shall be no stronger than the
458 // success argument". This condition has been lifted and the only
459 // precondition is 31.7.2.18. Effectively treat this as a DR and skip
460 // language version checks.
461 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
462 FailureOrder, Scope);
463 return;
464 }
465
466 // Create all the relevant BB's
467 auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
468 auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
469 auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
470 auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
471
472 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
473 // doesn't matter unless someone is crazy enough to use something that
474 // doesn't fold to a constant for the ordering.
475 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
476 // Implemented as acquire, since it's the closest in LLVM.
477 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
478 AcquireBB);
479 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
480 AcquireBB);
481 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
482 SeqCstBB);
483
484 // Emit all the different atomics
485 CGF.Builder.SetInsertPoint(MonotonicBB);
486 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
487 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
488 CGF.Builder.CreateBr(ContBB);
489
490 CGF.Builder.SetInsertPoint(AcquireBB);
491 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
492 llvm::AtomicOrdering::Acquire, Scope);
493 CGF.Builder.CreateBr(ContBB);
494
495 CGF.Builder.SetInsertPoint(SeqCstBB);
496 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
497 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
498 CGF.Builder.CreateBr(ContBB);
499
500 CGF.Builder.SetInsertPoint(ContBB);
501 }
502
503 /// Duplicate the atomic min/max operation in conventional IR for the builtin
504 /// variants that return the new rather than the original value.
EmitPostAtomicMinMax(CGBuilderTy & Builder,AtomicExpr::AtomicOp Op,bool IsSigned,llvm::Value * OldVal,llvm::Value * RHS)505 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
506 AtomicExpr::AtomicOp Op,
507 bool IsSigned,
508 llvm::Value *OldVal,
509 llvm::Value *RHS) {
510 llvm::CmpInst::Predicate Pred;
511 switch (Op) {
512 default:
513 llvm_unreachable("Unexpected min/max operation");
514 case AtomicExpr::AO__atomic_max_fetch:
515 case AtomicExpr::AO__scoped_atomic_max_fetch:
516 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
517 break;
518 case AtomicExpr::AO__atomic_min_fetch:
519 case AtomicExpr::AO__scoped_atomic_min_fetch:
520 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
521 break;
522 }
523 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
524 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
525 }
526
EmitAtomicOp(CodeGenFunction & CGF,AtomicExpr * E,Address Dest,Address Ptr,Address Val1,Address Val2,llvm::Value * IsWeak,llvm::Value * FailureOrder,uint64_t Size,llvm::AtomicOrdering Order,llvm::SyncScope::ID Scope)527 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
528 Address Ptr, Address Val1, Address Val2,
529 llvm::Value *IsWeak, llvm::Value *FailureOrder,
530 uint64_t Size, llvm::AtomicOrdering Order,
531 llvm::SyncScope::ID Scope) {
532 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
533 bool PostOpMinMax = false;
534 unsigned PostOp = 0;
535
536 switch (E->getOp()) {
537 case AtomicExpr::AO__c11_atomic_init:
538 case AtomicExpr::AO__opencl_atomic_init:
539 llvm_unreachable("Already handled!");
540
541 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
542 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
543 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
544 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
545 FailureOrder, Size, Order, Scope);
546 return;
547 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
548 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
549 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
550 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
551 FailureOrder, Size, Order, Scope);
552 return;
553 case AtomicExpr::AO__atomic_compare_exchange:
554 case AtomicExpr::AO__atomic_compare_exchange_n:
555 case AtomicExpr::AO__scoped_atomic_compare_exchange:
556 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
557 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
558 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
559 Val1, Val2, FailureOrder, Size, Order, Scope);
560 } else {
561 // Create all the relevant BB's
562 llvm::BasicBlock *StrongBB =
563 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
564 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
565 llvm::BasicBlock *ContBB =
566 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
567
568 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
569 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
570
571 CGF.Builder.SetInsertPoint(StrongBB);
572 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
573 FailureOrder, Size, Order, Scope);
574 CGF.Builder.CreateBr(ContBB);
575
576 CGF.Builder.SetInsertPoint(WeakBB);
577 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
578 FailureOrder, Size, Order, Scope);
579 CGF.Builder.CreateBr(ContBB);
580
581 CGF.Builder.SetInsertPoint(ContBB);
582 }
583 return;
584 }
585 case AtomicExpr::AO__c11_atomic_load:
586 case AtomicExpr::AO__opencl_atomic_load:
587 case AtomicExpr::AO__hip_atomic_load:
588 case AtomicExpr::AO__atomic_load_n:
589 case AtomicExpr::AO__atomic_load:
590 case AtomicExpr::AO__scoped_atomic_load_n:
591 case AtomicExpr::AO__scoped_atomic_load: {
592 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
593 Load->setAtomic(Order, Scope);
594 Load->setVolatile(E->isVolatile());
595 CGF.maybeAttachRangeForLoad(Load, E->getValueType(), E->getExprLoc());
596 auto *I = CGF.Builder.CreateStore(Load, Dest);
597 CGF.addInstToCurrentSourceAtom(I, Load);
598 return;
599 }
600
601 case AtomicExpr::AO__c11_atomic_store:
602 case AtomicExpr::AO__opencl_atomic_store:
603 case AtomicExpr::AO__hip_atomic_store:
604 case AtomicExpr::AO__atomic_store:
605 case AtomicExpr::AO__atomic_store_n:
606 case AtomicExpr::AO__scoped_atomic_store:
607 case AtomicExpr::AO__scoped_atomic_store_n: {
608 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
609 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
610 Store->setAtomic(Order, Scope);
611 Store->setVolatile(E->isVolatile());
612 CGF.addInstToCurrentSourceAtom(Store, LoadVal1);
613 return;
614 }
615
616 case AtomicExpr::AO__c11_atomic_exchange:
617 case AtomicExpr::AO__hip_atomic_exchange:
618 case AtomicExpr::AO__opencl_atomic_exchange:
619 case AtomicExpr::AO__atomic_exchange_n:
620 case AtomicExpr::AO__atomic_exchange:
621 case AtomicExpr::AO__scoped_atomic_exchange_n:
622 case AtomicExpr::AO__scoped_atomic_exchange:
623 Op = llvm::AtomicRMWInst::Xchg;
624 break;
625
626 case AtomicExpr::AO__atomic_add_fetch:
627 case AtomicExpr::AO__scoped_atomic_add_fetch:
628 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
629 : llvm::Instruction::Add;
630 [[fallthrough]];
631 case AtomicExpr::AO__c11_atomic_fetch_add:
632 case AtomicExpr::AO__hip_atomic_fetch_add:
633 case AtomicExpr::AO__opencl_atomic_fetch_add:
634 case AtomicExpr::AO__atomic_fetch_add:
635 case AtomicExpr::AO__scoped_atomic_fetch_add:
636 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
637 : llvm::AtomicRMWInst::Add;
638 break;
639
640 case AtomicExpr::AO__atomic_sub_fetch:
641 case AtomicExpr::AO__scoped_atomic_sub_fetch:
642 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
643 : llvm::Instruction::Sub;
644 [[fallthrough]];
645 case AtomicExpr::AO__c11_atomic_fetch_sub:
646 case AtomicExpr::AO__hip_atomic_fetch_sub:
647 case AtomicExpr::AO__opencl_atomic_fetch_sub:
648 case AtomicExpr::AO__atomic_fetch_sub:
649 case AtomicExpr::AO__scoped_atomic_fetch_sub:
650 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
651 : llvm::AtomicRMWInst::Sub;
652 break;
653
654 case AtomicExpr::AO__atomic_min_fetch:
655 case AtomicExpr::AO__scoped_atomic_min_fetch:
656 PostOpMinMax = true;
657 [[fallthrough]];
658 case AtomicExpr::AO__c11_atomic_fetch_min:
659 case AtomicExpr::AO__hip_atomic_fetch_min:
660 case AtomicExpr::AO__opencl_atomic_fetch_min:
661 case AtomicExpr::AO__atomic_fetch_min:
662 case AtomicExpr::AO__scoped_atomic_fetch_min:
663 Op = E->getValueType()->isFloatingType()
664 ? llvm::AtomicRMWInst::FMin
665 : (E->getValueType()->isSignedIntegerType()
666 ? llvm::AtomicRMWInst::Min
667 : llvm::AtomicRMWInst::UMin);
668 break;
669
670 case AtomicExpr::AO__atomic_max_fetch:
671 case AtomicExpr::AO__scoped_atomic_max_fetch:
672 PostOpMinMax = true;
673 [[fallthrough]];
674 case AtomicExpr::AO__c11_atomic_fetch_max:
675 case AtomicExpr::AO__hip_atomic_fetch_max:
676 case AtomicExpr::AO__opencl_atomic_fetch_max:
677 case AtomicExpr::AO__atomic_fetch_max:
678 case AtomicExpr::AO__scoped_atomic_fetch_max:
679 Op = E->getValueType()->isFloatingType()
680 ? llvm::AtomicRMWInst::FMax
681 : (E->getValueType()->isSignedIntegerType()
682 ? llvm::AtomicRMWInst::Max
683 : llvm::AtomicRMWInst::UMax);
684 break;
685
686 case AtomicExpr::AO__atomic_and_fetch:
687 case AtomicExpr::AO__scoped_atomic_and_fetch:
688 PostOp = llvm::Instruction::And;
689 [[fallthrough]];
690 case AtomicExpr::AO__c11_atomic_fetch_and:
691 case AtomicExpr::AO__hip_atomic_fetch_and:
692 case AtomicExpr::AO__opencl_atomic_fetch_and:
693 case AtomicExpr::AO__atomic_fetch_and:
694 case AtomicExpr::AO__scoped_atomic_fetch_and:
695 Op = llvm::AtomicRMWInst::And;
696 break;
697
698 case AtomicExpr::AO__atomic_or_fetch:
699 case AtomicExpr::AO__scoped_atomic_or_fetch:
700 PostOp = llvm::Instruction::Or;
701 [[fallthrough]];
702 case AtomicExpr::AO__c11_atomic_fetch_or:
703 case AtomicExpr::AO__hip_atomic_fetch_or:
704 case AtomicExpr::AO__opencl_atomic_fetch_or:
705 case AtomicExpr::AO__atomic_fetch_or:
706 case AtomicExpr::AO__scoped_atomic_fetch_or:
707 Op = llvm::AtomicRMWInst::Or;
708 break;
709
710 case AtomicExpr::AO__atomic_xor_fetch:
711 case AtomicExpr::AO__scoped_atomic_xor_fetch:
712 PostOp = llvm::Instruction::Xor;
713 [[fallthrough]];
714 case AtomicExpr::AO__c11_atomic_fetch_xor:
715 case AtomicExpr::AO__hip_atomic_fetch_xor:
716 case AtomicExpr::AO__opencl_atomic_fetch_xor:
717 case AtomicExpr::AO__atomic_fetch_xor:
718 case AtomicExpr::AO__scoped_atomic_fetch_xor:
719 Op = llvm::AtomicRMWInst::Xor;
720 break;
721
722 case AtomicExpr::AO__atomic_nand_fetch:
723 case AtomicExpr::AO__scoped_atomic_nand_fetch:
724 PostOp = llvm::Instruction::And; // the NOT is special cased below
725 [[fallthrough]];
726 case AtomicExpr::AO__c11_atomic_fetch_nand:
727 case AtomicExpr::AO__atomic_fetch_nand:
728 case AtomicExpr::AO__scoped_atomic_fetch_nand:
729 Op = llvm::AtomicRMWInst::Nand;
730 break;
731
732 case AtomicExpr::AO__atomic_test_and_set: {
733 llvm::AtomicRMWInst *RMWI =
734 CGF.emitAtomicRMWInst(llvm::AtomicRMWInst::Xchg, Ptr,
735 CGF.Builder.getInt8(1), Order, Scope, E);
736 RMWI->setVolatile(E->isVolatile());
737 llvm::Value *Result = CGF.Builder.CreateIsNotNull(RMWI, "tobool");
738 auto *I = CGF.Builder.CreateStore(Result, Dest);
739 CGF.addInstToCurrentSourceAtom(I, Result);
740 return;
741 }
742
743 case AtomicExpr::AO__atomic_clear: {
744 llvm::StoreInst *Store =
745 CGF.Builder.CreateStore(CGF.Builder.getInt8(0), Ptr);
746 Store->setAtomic(Order, Scope);
747 Store->setVolatile(E->isVolatile());
748 CGF.addInstToCurrentSourceAtom(Store, nullptr);
749 return;
750 }
751 }
752
753 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
754 llvm::AtomicRMWInst *RMWI =
755 CGF.emitAtomicRMWInst(Op, Ptr, LoadVal1, Order, Scope, E);
756 RMWI->setVolatile(E->isVolatile());
757
758 // For __atomic_*_fetch operations, perform the operation again to
759 // determine the value which was written.
760 llvm::Value *Result = RMWI;
761 if (PostOpMinMax)
762 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
763 E->getValueType()->isSignedIntegerType(),
764 RMWI, LoadVal1);
765 else if (PostOp)
766 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
767 LoadVal1);
768 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch ||
769 E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)
770 Result = CGF.Builder.CreateNot(Result);
771 auto *I = CGF.Builder.CreateStore(Result, Dest);
772 CGF.addInstToCurrentSourceAtom(I, Result);
773 }
774
775 // This function emits any expression (scalar, complex, or aggregate)
776 // into a temporary alloca.
777 static Address
EmitValToTemp(CodeGenFunction & CGF,Expr * E)778 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
779 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
780 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
781 /*Init*/ true);
782 return DeclPtr;
783 }
784
EmitAtomicOp(CodeGenFunction & CGF,AtomicExpr * Expr,Address Dest,Address Ptr,Address Val1,Address Val2,llvm::Value * IsWeak,llvm::Value * FailureOrder,uint64_t Size,llvm::AtomicOrdering Order,llvm::Value * Scope)785 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
786 Address Ptr, Address Val1, Address Val2,
787 llvm::Value *IsWeak, llvm::Value *FailureOrder,
788 uint64_t Size, llvm::AtomicOrdering Order,
789 llvm::Value *Scope) {
790 auto ScopeModel = Expr->getScopeModel();
791
792 // LLVM atomic instructions always have sync scope. If clang atomic
793 // expression has no scope operand, use default LLVM sync scope.
794 if (!ScopeModel) {
795 llvm::SyncScope::ID SS;
796 if (CGF.getLangOpts().OpenCL)
797 // OpenCL approach is: "The functions that do not have memory_scope
798 // argument have the same semantics as the corresponding functions with
799 // the memory_scope argument set to memory_scope_device." See ref.:
800 // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions
801 SS = CGF.getTargetHooks().getLLVMSyncScopeID(CGF.getLangOpts(),
802 SyncScope::OpenCLDevice,
803 Order, CGF.getLLVMContext());
804 else
805 SS = llvm::SyncScope::System;
806 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
807 Order, SS);
808 return;
809 }
810
811 // Handle constant scope.
812 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
813 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
814 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
815 Order, CGF.CGM.getLLVMContext());
816 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
817 Order, SCID);
818 return;
819 }
820
821 // Handle non-constant scope.
822 auto &Builder = CGF.Builder;
823 auto Scopes = ScopeModel->getRuntimeValues();
824 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
825 for (auto S : Scopes)
826 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
827
828 llvm::BasicBlock *ContBB =
829 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
830
831 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
832 // If unsupported sync scope is encountered at run time, assume a fallback
833 // sync scope value.
834 auto FallBack = ScopeModel->getFallBackValue();
835 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
836 for (auto S : Scopes) {
837 auto *B = BB[S];
838 if (S != FallBack)
839 SI->addCase(Builder.getInt32(S), B);
840
841 Builder.SetInsertPoint(B);
842 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
843 Order,
844 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
845 ScopeModel->map(S),
846 Order,
847 CGF.getLLVMContext()));
848 Builder.CreateBr(ContBB);
849 }
850
851 Builder.SetInsertPoint(ContBB);
852 }
853
EmitAtomicExpr(AtomicExpr * E)854 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
855 ApplyAtomGroup Grp(getDebugInfo());
856
857 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
858 QualType MemTy = AtomicTy;
859 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
860 MemTy = AT->getValueType();
861 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
862
863 Address Val1 = Address::invalid();
864 Address Val2 = Address::invalid();
865 Address Dest = Address::invalid();
866 Address Ptr = EmitPointerWithAlignment(E->getPtr());
867
868 if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
869 E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
870 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
871 EmitAtomicInit(E->getVal1(), lvalue);
872 return RValue::get(nullptr);
873 }
874
875 auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
876 uint64_t Size = TInfo.Width.getQuantity();
877 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
878
879 CharUnits MaxInlineWidth =
880 getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
881 DiagnosticsEngine &Diags = CGM.getDiags();
882 bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
883 bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
884 if (Misaligned) {
885 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
886 << (int)TInfo.Width.getQuantity()
887 << (int)Ptr.getAlignment().getQuantity();
888 }
889 if (Oversized) {
890 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
891 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
892 }
893
894 llvm::Value *Order = EmitScalarExpr(E->getOrder());
895 llvm::Value *Scope =
896 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
897 bool ShouldCastToIntPtrTy = true;
898
899 switch (E->getOp()) {
900 case AtomicExpr::AO__c11_atomic_init:
901 case AtomicExpr::AO__opencl_atomic_init:
902 llvm_unreachable("Already handled above with EmitAtomicInit!");
903
904 case AtomicExpr::AO__atomic_load_n:
905 case AtomicExpr::AO__scoped_atomic_load_n:
906 case AtomicExpr::AO__c11_atomic_load:
907 case AtomicExpr::AO__opencl_atomic_load:
908 case AtomicExpr::AO__hip_atomic_load:
909 case AtomicExpr::AO__atomic_test_and_set:
910 case AtomicExpr::AO__atomic_clear:
911 break;
912
913 case AtomicExpr::AO__atomic_load:
914 case AtomicExpr::AO__scoped_atomic_load:
915 Dest = EmitPointerWithAlignment(E->getVal1());
916 break;
917
918 case AtomicExpr::AO__atomic_store:
919 case AtomicExpr::AO__scoped_atomic_store:
920 Val1 = EmitPointerWithAlignment(E->getVal1());
921 break;
922
923 case AtomicExpr::AO__atomic_exchange:
924 case AtomicExpr::AO__scoped_atomic_exchange:
925 Val1 = EmitPointerWithAlignment(E->getVal1());
926 Dest = EmitPointerWithAlignment(E->getVal2());
927 break;
928
929 case AtomicExpr::AO__atomic_compare_exchange:
930 case AtomicExpr::AO__atomic_compare_exchange_n:
931 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
932 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
933 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
934 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
935 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
936 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
937 case AtomicExpr::AO__scoped_atomic_compare_exchange:
938 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
939 Val1 = EmitPointerWithAlignment(E->getVal1());
940 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
941 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
942 Val2 = EmitPointerWithAlignment(E->getVal2());
943 else
944 Val2 = EmitValToTemp(*this, E->getVal2());
945 OrderFail = EmitScalarExpr(E->getOrderFail());
946 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
947 E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
948 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||
949 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
950 IsWeak = EmitScalarExpr(E->getWeak());
951 break;
952
953 case AtomicExpr::AO__c11_atomic_fetch_add:
954 case AtomicExpr::AO__c11_atomic_fetch_sub:
955 case AtomicExpr::AO__hip_atomic_fetch_add:
956 case AtomicExpr::AO__hip_atomic_fetch_sub:
957 case AtomicExpr::AO__opencl_atomic_fetch_add:
958 case AtomicExpr::AO__opencl_atomic_fetch_sub:
959 if (MemTy->isPointerType()) {
960 // For pointer arithmetic, we're required to do a bit of math:
961 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
962 // ... but only for the C11 builtins. The GNU builtins expect the
963 // user to multiply by sizeof(T).
964 QualType Val1Ty = E->getVal1()->getType();
965 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
966 CharUnits PointeeIncAmt =
967 getContext().getTypeSizeInChars(MemTy->getPointeeType());
968 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
969 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
970 Val1 = Temp;
971 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
972 break;
973 }
974 [[fallthrough]];
975 case AtomicExpr::AO__atomic_fetch_add:
976 case AtomicExpr::AO__atomic_fetch_max:
977 case AtomicExpr::AO__atomic_fetch_min:
978 case AtomicExpr::AO__atomic_fetch_sub:
979 case AtomicExpr::AO__atomic_add_fetch:
980 case AtomicExpr::AO__atomic_max_fetch:
981 case AtomicExpr::AO__atomic_min_fetch:
982 case AtomicExpr::AO__atomic_sub_fetch:
983 case AtomicExpr::AO__c11_atomic_fetch_max:
984 case AtomicExpr::AO__c11_atomic_fetch_min:
985 case AtomicExpr::AO__opencl_atomic_fetch_max:
986 case AtomicExpr::AO__opencl_atomic_fetch_min:
987 case AtomicExpr::AO__hip_atomic_fetch_max:
988 case AtomicExpr::AO__hip_atomic_fetch_min:
989 case AtomicExpr::AO__scoped_atomic_fetch_add:
990 case AtomicExpr::AO__scoped_atomic_fetch_max:
991 case AtomicExpr::AO__scoped_atomic_fetch_min:
992 case AtomicExpr::AO__scoped_atomic_fetch_sub:
993 case AtomicExpr::AO__scoped_atomic_add_fetch:
994 case AtomicExpr::AO__scoped_atomic_max_fetch:
995 case AtomicExpr::AO__scoped_atomic_min_fetch:
996 case AtomicExpr::AO__scoped_atomic_sub_fetch:
997 ShouldCastToIntPtrTy = !MemTy->isFloatingType();
998 [[fallthrough]];
999
1000 case AtomicExpr::AO__atomic_fetch_and:
1001 case AtomicExpr::AO__atomic_fetch_nand:
1002 case AtomicExpr::AO__atomic_fetch_or:
1003 case AtomicExpr::AO__atomic_fetch_xor:
1004 case AtomicExpr::AO__atomic_and_fetch:
1005 case AtomicExpr::AO__atomic_nand_fetch:
1006 case AtomicExpr::AO__atomic_or_fetch:
1007 case AtomicExpr::AO__atomic_xor_fetch:
1008 case AtomicExpr::AO__atomic_store_n:
1009 case AtomicExpr::AO__atomic_exchange_n:
1010 case AtomicExpr::AO__c11_atomic_fetch_and:
1011 case AtomicExpr::AO__c11_atomic_fetch_nand:
1012 case AtomicExpr::AO__c11_atomic_fetch_or:
1013 case AtomicExpr::AO__c11_atomic_fetch_xor:
1014 case AtomicExpr::AO__c11_atomic_store:
1015 case AtomicExpr::AO__c11_atomic_exchange:
1016 case AtomicExpr::AO__hip_atomic_fetch_and:
1017 case AtomicExpr::AO__hip_atomic_fetch_or:
1018 case AtomicExpr::AO__hip_atomic_fetch_xor:
1019 case AtomicExpr::AO__hip_atomic_store:
1020 case AtomicExpr::AO__hip_atomic_exchange:
1021 case AtomicExpr::AO__opencl_atomic_fetch_and:
1022 case AtomicExpr::AO__opencl_atomic_fetch_or:
1023 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1024 case AtomicExpr::AO__opencl_atomic_store:
1025 case AtomicExpr::AO__opencl_atomic_exchange:
1026 case AtomicExpr::AO__scoped_atomic_fetch_and:
1027 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1028 case AtomicExpr::AO__scoped_atomic_fetch_or:
1029 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1030 case AtomicExpr::AO__scoped_atomic_and_fetch:
1031 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1032 case AtomicExpr::AO__scoped_atomic_or_fetch:
1033 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1034 case AtomicExpr::AO__scoped_atomic_store_n:
1035 case AtomicExpr::AO__scoped_atomic_exchange_n:
1036 Val1 = EmitValToTemp(*this, E->getVal1());
1037 break;
1038 }
1039
1040 QualType RValTy = E->getType().getUnqualifiedType();
1041
1042 // The inlined atomics only function on iN types, where N is a power of 2. We
1043 // need to make sure (via temporaries if necessary) that all incoming values
1044 // are compatible.
1045 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
1046 AtomicInfo Atomics(*this, AtomicVal);
1047
1048 if (ShouldCastToIntPtrTy) {
1049 Ptr = Atomics.castToAtomicIntPointer(Ptr);
1050 if (Val1.isValid())
1051 Val1 = Atomics.convertToAtomicIntPointer(Val1);
1052 if (Val2.isValid())
1053 Val2 = Atomics.convertToAtomicIntPointer(Val2);
1054 }
1055 if (Dest.isValid()) {
1056 if (ShouldCastToIntPtrTy)
1057 Dest = Atomics.castToAtomicIntPointer(Dest);
1058 } else if (E->isCmpXChg())
1059 Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
1060 else if (!RValTy->isVoidType()) {
1061 Dest = Atomics.CreateTempAlloca();
1062 if (ShouldCastToIntPtrTy)
1063 Dest = Atomics.castToAtomicIntPointer(Dest);
1064 }
1065
1066 bool PowerOf2Size = (Size & (Size - 1)) == 0;
1067 bool UseLibcall = !PowerOf2Size || (Size > 16);
1068
1069 // For atomics larger than 16 bytes, emit a libcall from the frontend. This
1070 // avoids the overhead of dealing with excessively-large value types in IR.
1071 // Non-power-of-2 values also lower to libcall here, as they are not currently
1072 // permitted in IR instructions (although that constraint could be relaxed in
1073 // the future). For other cases where a libcall is required on a given
1074 // platform, we let the backend handle it (this includes handling for all of
1075 // the size-optimized libcall variants, which are only valid up to 16 bytes.)
1076 //
1077 // See: https://llvm.org/docs/Atomics.html#libcalls-atomic
1078 if (UseLibcall) {
1079 CallArgList Args;
1080 // For non-optimized library calls, the size is the first parameter.
1081 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1082 getContext().getSizeType());
1083
1084 // The atomic address is the second parameter.
1085 // The OpenCL atomic library functions only accept pointer arguments to
1086 // generic address space.
1087 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1088 if (!E->isOpenCL())
1089 return V;
1090 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1091 if (AS == LangAS::opencl_generic)
1092 return V;
1093 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1094 auto *DestType = llvm::PointerType::get(getLLVMContext(), DestAS);
1095
1096 return getTargetHooks().performAddrSpaceCast(*this, V, AS, DestType,
1097 false);
1098 };
1099
1100 Args.add(RValue::get(CastToGenericAddrSpace(Ptr.emitRawPointer(*this),
1101 E->getPtr()->getType())),
1102 getContext().VoidPtrTy);
1103
1104 // The next 1-3 parameters are op-dependent.
1105 std::string LibCallName;
1106 QualType RetTy;
1107 bool HaveRetTy = false;
1108 switch (E->getOp()) {
1109 case AtomicExpr::AO__c11_atomic_init:
1110 case AtomicExpr::AO__opencl_atomic_init:
1111 llvm_unreachable("Already handled!");
1112
1113 // There is only one libcall for compare an exchange, because there is no
1114 // optimisation benefit possible from a libcall version of a weak compare
1115 // and exchange.
1116 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1117 // void *desired, int success, int failure)
1118 case AtomicExpr::AO__atomic_compare_exchange:
1119 case AtomicExpr::AO__atomic_compare_exchange_n:
1120 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1121 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1122 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1123 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1124 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1125 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1126 case AtomicExpr::AO__scoped_atomic_compare_exchange:
1127 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
1128 LibCallName = "__atomic_compare_exchange";
1129 RetTy = getContext().BoolTy;
1130 HaveRetTy = true;
1131 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
1132 E->getVal1()->getType())),
1133 getContext().VoidPtrTy);
1134 Args.add(RValue::get(CastToGenericAddrSpace(Val2.emitRawPointer(*this),
1135 E->getVal2()->getType())),
1136 getContext().VoidPtrTy);
1137 Args.add(RValue::get(Order), getContext().IntTy);
1138 Order = OrderFail;
1139 break;
1140 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1141 // int order)
1142 case AtomicExpr::AO__atomic_exchange:
1143 case AtomicExpr::AO__atomic_exchange_n:
1144 case AtomicExpr::AO__c11_atomic_exchange:
1145 case AtomicExpr::AO__hip_atomic_exchange:
1146 case AtomicExpr::AO__opencl_atomic_exchange:
1147 case AtomicExpr::AO__scoped_atomic_exchange:
1148 case AtomicExpr::AO__scoped_atomic_exchange_n:
1149 LibCallName = "__atomic_exchange";
1150 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
1151 E->getVal1()->getType())),
1152 getContext().VoidPtrTy);
1153 break;
1154 // void __atomic_store(size_t size, void *mem, void *val, int order)
1155 case AtomicExpr::AO__atomic_store:
1156 case AtomicExpr::AO__atomic_store_n:
1157 case AtomicExpr::AO__c11_atomic_store:
1158 case AtomicExpr::AO__hip_atomic_store:
1159 case AtomicExpr::AO__opencl_atomic_store:
1160 case AtomicExpr::AO__scoped_atomic_store:
1161 case AtomicExpr::AO__scoped_atomic_store_n:
1162 LibCallName = "__atomic_store";
1163 RetTy = getContext().VoidTy;
1164 HaveRetTy = true;
1165 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
1166 E->getVal1()->getType())),
1167 getContext().VoidPtrTy);
1168 break;
1169 // void __atomic_load(size_t size, void *mem, void *return, int order)
1170 case AtomicExpr::AO__atomic_load:
1171 case AtomicExpr::AO__atomic_load_n:
1172 case AtomicExpr::AO__c11_atomic_load:
1173 case AtomicExpr::AO__hip_atomic_load:
1174 case AtomicExpr::AO__opencl_atomic_load:
1175 case AtomicExpr::AO__scoped_atomic_load:
1176 case AtomicExpr::AO__scoped_atomic_load_n:
1177 LibCallName = "__atomic_load";
1178 break;
1179 case AtomicExpr::AO__atomic_add_fetch:
1180 case AtomicExpr::AO__scoped_atomic_add_fetch:
1181 case AtomicExpr::AO__atomic_fetch_add:
1182 case AtomicExpr::AO__c11_atomic_fetch_add:
1183 case AtomicExpr::AO__hip_atomic_fetch_add:
1184 case AtomicExpr::AO__opencl_atomic_fetch_add:
1185 case AtomicExpr::AO__scoped_atomic_fetch_add:
1186 case AtomicExpr::AO__atomic_and_fetch:
1187 case AtomicExpr::AO__scoped_atomic_and_fetch:
1188 case AtomicExpr::AO__atomic_fetch_and:
1189 case AtomicExpr::AO__c11_atomic_fetch_and:
1190 case AtomicExpr::AO__hip_atomic_fetch_and:
1191 case AtomicExpr::AO__opencl_atomic_fetch_and:
1192 case AtomicExpr::AO__scoped_atomic_fetch_and:
1193 case AtomicExpr::AO__atomic_or_fetch:
1194 case AtomicExpr::AO__scoped_atomic_or_fetch:
1195 case AtomicExpr::AO__atomic_fetch_or:
1196 case AtomicExpr::AO__c11_atomic_fetch_or:
1197 case AtomicExpr::AO__hip_atomic_fetch_or:
1198 case AtomicExpr::AO__opencl_atomic_fetch_or:
1199 case AtomicExpr::AO__scoped_atomic_fetch_or:
1200 case AtomicExpr::AO__atomic_sub_fetch:
1201 case AtomicExpr::AO__scoped_atomic_sub_fetch:
1202 case AtomicExpr::AO__atomic_fetch_sub:
1203 case AtomicExpr::AO__c11_atomic_fetch_sub:
1204 case AtomicExpr::AO__hip_atomic_fetch_sub:
1205 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1206 case AtomicExpr::AO__scoped_atomic_fetch_sub:
1207 case AtomicExpr::AO__atomic_xor_fetch:
1208 case AtomicExpr::AO__scoped_atomic_xor_fetch:
1209 case AtomicExpr::AO__atomic_fetch_xor:
1210 case AtomicExpr::AO__c11_atomic_fetch_xor:
1211 case AtomicExpr::AO__hip_atomic_fetch_xor:
1212 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1213 case AtomicExpr::AO__scoped_atomic_fetch_xor:
1214 case AtomicExpr::AO__atomic_nand_fetch:
1215 case AtomicExpr::AO__atomic_fetch_nand:
1216 case AtomicExpr::AO__c11_atomic_fetch_nand:
1217 case AtomicExpr::AO__scoped_atomic_fetch_nand:
1218 case AtomicExpr::AO__scoped_atomic_nand_fetch:
1219 case AtomicExpr::AO__atomic_min_fetch:
1220 case AtomicExpr::AO__atomic_fetch_min:
1221 case AtomicExpr::AO__c11_atomic_fetch_min:
1222 case AtomicExpr::AO__hip_atomic_fetch_min:
1223 case AtomicExpr::AO__opencl_atomic_fetch_min:
1224 case AtomicExpr::AO__scoped_atomic_fetch_min:
1225 case AtomicExpr::AO__scoped_atomic_min_fetch:
1226 case AtomicExpr::AO__atomic_max_fetch:
1227 case AtomicExpr::AO__atomic_fetch_max:
1228 case AtomicExpr::AO__c11_atomic_fetch_max:
1229 case AtomicExpr::AO__hip_atomic_fetch_max:
1230 case AtomicExpr::AO__opencl_atomic_fetch_max:
1231 case AtomicExpr::AO__scoped_atomic_fetch_max:
1232 case AtomicExpr::AO__scoped_atomic_max_fetch:
1233 case AtomicExpr::AO__atomic_test_and_set:
1234 case AtomicExpr::AO__atomic_clear:
1235 llvm_unreachable("Integral atomic operations always become atomicrmw!");
1236 }
1237
1238 if (E->isOpenCL()) {
1239 LibCallName =
1240 std::string("__opencl") + StringRef(LibCallName).drop_front(1).str();
1241 }
1242 // By default, assume we return a value of the atomic type.
1243 if (!HaveRetTy) {
1244 // Value is returned through parameter before the order.
1245 RetTy = getContext().VoidTy;
1246 Args.add(RValue::get(
1247 CastToGenericAddrSpace(Dest.emitRawPointer(*this), RetTy)),
1248 getContext().VoidPtrTy);
1249 }
1250 // Order is always the last parameter.
1251 Args.add(RValue::get(Order),
1252 getContext().IntTy);
1253 if (E->isOpenCL())
1254 Args.add(RValue::get(Scope), getContext().IntTy);
1255
1256 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1257 // The value is returned directly from the libcall.
1258 if (E->isCmpXChg())
1259 return Res;
1260
1261 if (RValTy->isVoidType())
1262 return RValue::get(nullptr);
1263
1264 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),
1265 RValTy, E->getExprLoc());
1266 }
1267
1268 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1269 E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1270 E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1271 E->getOp() == AtomicExpr::AO__atomic_store ||
1272 E->getOp() == AtomicExpr::AO__atomic_store_n ||
1273 E->getOp() == AtomicExpr::AO__scoped_atomic_store ||
1274 E->getOp() == AtomicExpr::AO__scoped_atomic_store_n ||
1275 E->getOp() == AtomicExpr::AO__atomic_clear;
1276 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1277 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1278 E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1279 E->getOp() == AtomicExpr::AO__atomic_load ||
1280 E->getOp() == AtomicExpr::AO__atomic_load_n ||
1281 E->getOp() == AtomicExpr::AO__scoped_atomic_load ||
1282 E->getOp() == AtomicExpr::AO__scoped_atomic_load_n;
1283
1284 if (isa<llvm::ConstantInt>(Order)) {
1285 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1286 // We should not ever get to a case where the ordering isn't a valid C ABI
1287 // value, but it's hard to enforce that in general.
1288 if (llvm::isValidAtomicOrderingCABI(ord))
1289 switch ((llvm::AtomicOrderingCABI)ord) {
1290 case llvm::AtomicOrderingCABI::relaxed:
1291 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1292 llvm::AtomicOrdering::Monotonic, Scope);
1293 break;
1294 case llvm::AtomicOrderingCABI::consume:
1295 case llvm::AtomicOrderingCABI::acquire:
1296 if (IsStore)
1297 break; // Avoid crashing on code with undefined behavior
1298 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1299 llvm::AtomicOrdering::Acquire, Scope);
1300 break;
1301 case llvm::AtomicOrderingCABI::release:
1302 if (IsLoad)
1303 break; // Avoid crashing on code with undefined behavior
1304 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1305 llvm::AtomicOrdering::Release, Scope);
1306 break;
1307 case llvm::AtomicOrderingCABI::acq_rel:
1308 if (IsLoad || IsStore)
1309 break; // Avoid crashing on code with undefined behavior
1310 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1311 llvm::AtomicOrdering::AcquireRelease, Scope);
1312 break;
1313 case llvm::AtomicOrderingCABI::seq_cst:
1314 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1315 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1316 break;
1317 }
1318 if (RValTy->isVoidType())
1319 return RValue::get(nullptr);
1320
1321 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),
1322 RValTy, E->getExprLoc());
1323 }
1324
1325 // Long case, when Order isn't obviously constant.
1326
1327 // Create all the relevant BB's
1328 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1329 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1330 *SeqCstBB = nullptr;
1331 MonotonicBB = createBasicBlock("monotonic", CurFn);
1332 if (!IsStore)
1333 AcquireBB = createBasicBlock("acquire", CurFn);
1334 if (!IsLoad)
1335 ReleaseBB = createBasicBlock("release", CurFn);
1336 if (!IsLoad && !IsStore)
1337 AcqRelBB = createBasicBlock("acqrel", CurFn);
1338 SeqCstBB = createBasicBlock("seqcst", CurFn);
1339 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1340
1341 // Create the switch for the split
1342 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1343 // doesn't matter unless someone is crazy enough to use something that
1344 // doesn't fold to a constant for the ordering.
1345 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1346 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1347
1348 // Emit all the different atomics
1349 Builder.SetInsertPoint(MonotonicBB);
1350 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1351 llvm::AtomicOrdering::Monotonic, Scope);
1352 Builder.CreateBr(ContBB);
1353 if (!IsStore) {
1354 Builder.SetInsertPoint(AcquireBB);
1355 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1356 llvm::AtomicOrdering::Acquire, Scope);
1357 Builder.CreateBr(ContBB);
1358 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1359 AcquireBB);
1360 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1361 AcquireBB);
1362 }
1363 if (!IsLoad) {
1364 Builder.SetInsertPoint(ReleaseBB);
1365 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1366 llvm::AtomicOrdering::Release, Scope);
1367 Builder.CreateBr(ContBB);
1368 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1369 ReleaseBB);
1370 }
1371 if (!IsLoad && !IsStore) {
1372 Builder.SetInsertPoint(AcqRelBB);
1373 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1374 llvm::AtomicOrdering::AcquireRelease, Scope);
1375 Builder.CreateBr(ContBB);
1376 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1377 AcqRelBB);
1378 }
1379 Builder.SetInsertPoint(SeqCstBB);
1380 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1381 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1382 Builder.CreateBr(ContBB);
1383 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1384 SeqCstBB);
1385
1386 // Cleanup and return
1387 Builder.SetInsertPoint(ContBB);
1388 if (RValTy->isVoidType())
1389 return RValue::get(nullptr);
1390
1391 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1392 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),
1393 RValTy, E->getExprLoc());
1394 }
1395
castToAtomicIntPointer(Address addr) const1396 Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
1397 llvm::IntegerType *ty =
1398 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1399 return addr.withElementType(ty);
1400 }
1401
convertToAtomicIntPointer(Address Addr) const1402 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1403 llvm::Type *Ty = Addr.getElementType();
1404 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1405 if (SourceSizeInBits != AtomicSizeInBits) {
1406 Address Tmp = CreateTempAlloca();
1407 CGF.Builder.CreateMemCpy(Tmp, Addr,
1408 std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1409 Addr = Tmp;
1410 }
1411
1412 return castToAtomicIntPointer(Addr);
1413 }
1414
convertAtomicTempToRValue(Address addr,AggValueSlot resultSlot,SourceLocation loc,bool asValue) const1415 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1416 AggValueSlot resultSlot,
1417 SourceLocation loc,
1418 bool asValue) const {
1419 if (LVal.isSimple()) {
1420 if (EvaluationKind == TEK_Aggregate)
1421 return resultSlot.asRValue();
1422
1423 // Drill into the padding structure if we have one.
1424 if (hasPadding())
1425 addr = CGF.Builder.CreateStructGEP(addr, 0);
1426
1427 // Otherwise, just convert the temporary to an r-value using the
1428 // normal conversion routine.
1429 return CGF.convertTempToRValue(addr, getValueType(), loc);
1430 }
1431 if (!asValue)
1432 // Get RValue from temp memory as atomic for non-simple lvalues
1433 return RValue::get(CGF.Builder.CreateLoad(addr));
1434 if (LVal.isBitField())
1435 return CGF.EmitLoadOfBitfieldLValue(
1436 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1437 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1438 if (LVal.isVectorElt())
1439 return CGF.EmitLoadOfLValue(
1440 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1441 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1442 assert(LVal.isExtVectorElt());
1443 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1444 addr, LVal.getExtVectorElts(), LVal.getType(),
1445 LVal.getBaseInfo(), TBAAAccessInfo()));
1446 }
1447
1448 /// Return true if \param ValTy is a type that should be casted to integer
1449 /// around the atomic memory operation. If \param CmpXchg is true, then the
1450 /// cast of a floating point type is made as that instruction can not have
1451 /// floating point operands. TODO: Allow compare-and-exchange and FP - see
1452 /// comment in AtomicExpandPass.cpp.
shouldCastToInt(llvm::Type * ValTy,bool CmpXchg)1453 static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) {
1454 if (ValTy->isFloatingPointTy())
1455 return ValTy->isX86_FP80Ty() || CmpXchg;
1456 return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
1457 }
1458
ConvertToValueOrAtomic(llvm::Value * Val,AggValueSlot ResultSlot,SourceLocation Loc,bool AsValue,bool CmpXchg) const1459 RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
1460 AggValueSlot ResultSlot,
1461 SourceLocation Loc, bool AsValue,
1462 bool CmpXchg) const {
1463 // Try not to in some easy cases.
1464 assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() ||
1465 Val->getType()->isIEEELikeFPTy()) &&
1466 "Expected integer, pointer or floating point value when converting "
1467 "result.");
1468 if (getEvaluationKind() == TEK_Scalar &&
1469 (((!LVal.isBitField() ||
1470 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1471 !hasPadding()) ||
1472 !AsValue)) {
1473 auto *ValTy = AsValue
1474 ? CGF.ConvertTypeForMem(ValueTy)
1475 : getAtomicAddress().getElementType();
1476 if (!shouldCastToInt(ValTy, CmpXchg)) {
1477 assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) &&
1478 "Different integer types.");
1479 return RValue::get(CGF.EmitFromMemory(Val, ValueTy));
1480 }
1481 if (llvm::CastInst::isBitCastable(Val->getType(), ValTy))
1482 return RValue::get(CGF.Builder.CreateBitCast(Val, ValTy));
1483 }
1484
1485 // Create a temporary. This needs to be big enough to hold the
1486 // atomic integer.
1487 Address Temp = Address::invalid();
1488 bool TempIsVolatile = false;
1489 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1490 assert(!ResultSlot.isIgnored());
1491 Temp = ResultSlot.getAddress();
1492 TempIsVolatile = ResultSlot.isVolatile();
1493 } else {
1494 Temp = CreateTempAlloca();
1495 }
1496
1497 // Slam the integer into the temporary.
1498 Address CastTemp = castToAtomicIntPointer(Temp);
1499 CGF.Builder.CreateStore(Val, CastTemp)->setVolatile(TempIsVolatile);
1500
1501 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1502 }
1503
EmitAtomicLoadLibcall(llvm::Value * AddForLoaded,llvm::AtomicOrdering AO,bool)1504 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1505 llvm::AtomicOrdering AO, bool) {
1506 // void __atomic_load(size_t size, void *mem, void *return, int order);
1507 CallArgList Args;
1508 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1509 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);
1510 Args.add(RValue::get(AddForLoaded), CGF.getContext().VoidPtrTy);
1511 Args.add(
1512 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1513 CGF.getContext().IntTy);
1514 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1515 }
1516
EmitAtomicLoadOp(llvm::AtomicOrdering AO,bool IsVolatile,bool CmpXchg)1517 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1518 bool IsVolatile, bool CmpXchg) {
1519 // Okay, we're doing this natively.
1520 Address Addr = getAtomicAddress();
1521 if (shouldCastToInt(Addr.getElementType(), CmpXchg))
1522 Addr = castToAtomicIntPointer(Addr);
1523 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1524 Load->setAtomic(AO);
1525
1526 // Other decoration.
1527 if (IsVolatile)
1528 Load->setVolatile(true);
1529 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1530 return Load;
1531 }
1532
1533 /// An LValue is a candidate for having its loads and stores be made atomic if
1534 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1535 /// performing such an operation can be performed without a libcall.
LValueIsSuitableForInlineAtomic(LValue LV)1536 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1537 if (!CGM.getLangOpts().MSVolatile) return false;
1538 AtomicInfo AI(*this, LV);
1539 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1540 // An atomic is inline if we don't need to use a libcall.
1541 bool AtomicIsInline = !AI.shouldUseLibcall();
1542 // MSVC doesn't seem to do this for types wider than a pointer.
1543 if (getContext().getTypeSize(LV.getType()) >
1544 getContext().getTypeSize(getContext().getIntPtrType()))
1545 return false;
1546 return IsVolatile && AtomicIsInline;
1547 }
1548
EmitAtomicLoad(LValue LV,SourceLocation SL,AggValueSlot Slot)1549 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1550 AggValueSlot Slot) {
1551 llvm::AtomicOrdering AO;
1552 bool IsVolatile = LV.isVolatileQualified();
1553 if (LV.getType()->isAtomicType()) {
1554 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1555 } else {
1556 AO = llvm::AtomicOrdering::Acquire;
1557 IsVolatile = true;
1558 }
1559 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1560 }
1561
EmitAtomicLoad(AggValueSlot ResultSlot,SourceLocation Loc,bool AsValue,llvm::AtomicOrdering AO,bool IsVolatile)1562 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1563 bool AsValue, llvm::AtomicOrdering AO,
1564 bool IsVolatile) {
1565 // Check whether we should use a library call.
1566 if (shouldUseLibcall()) {
1567 Address TempAddr = Address::invalid();
1568 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1569 assert(getEvaluationKind() == TEK_Aggregate);
1570 TempAddr = ResultSlot.getAddress();
1571 } else
1572 TempAddr = CreateTempAlloca();
1573
1574 EmitAtomicLoadLibcall(TempAddr.emitRawPointer(CGF), AO, IsVolatile);
1575
1576 // Okay, turn that back into the original value or whole atomic (for
1577 // non-simple lvalues) type.
1578 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1579 }
1580
1581 // Okay, we're doing this natively.
1582 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1583
1584 // If we're ignoring an aggregate return, don't do anything.
1585 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1586 return RValue::getAggregate(Address::invalid(), false);
1587
1588 // Okay, turn that back into the original value or atomic (for non-simple
1589 // lvalues) type.
1590 return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1591 }
1592
1593 /// Emit a load from an l-value of atomic type. Note that the r-value
1594 /// we produce is an r-value of the atomic *value* type.
EmitAtomicLoad(LValue src,SourceLocation loc,llvm::AtomicOrdering AO,bool IsVolatile,AggValueSlot resultSlot)1595 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1596 llvm::AtomicOrdering AO, bool IsVolatile,
1597 AggValueSlot resultSlot) {
1598 AtomicInfo Atomics(*this, src);
1599 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1600 IsVolatile);
1601 }
1602
1603 /// Copy an r-value into memory as part of storing to an atomic type.
1604 /// This needs to create a bit-pattern suitable for atomic operations.
emitCopyIntoMemory(RValue rvalue) const1605 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1606 assert(LVal.isSimple());
1607 // If we have an r-value, the rvalue should be of the atomic type,
1608 // which means that the caller is responsible for having zeroed
1609 // any padding. Just do an aggregate copy of that type.
1610 if (rvalue.isAggregate()) {
1611 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1612 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1613 getAtomicType());
1614 bool IsVolatile = rvalue.isVolatileQualified() ||
1615 LVal.isVolatileQualified();
1616 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1617 AggValueSlot::DoesNotOverlap, IsVolatile);
1618 return;
1619 }
1620
1621 // Okay, otherwise we're copying stuff.
1622
1623 // Zero out the buffer if necessary.
1624 emitMemSetZeroIfNecessary();
1625
1626 // Drill past the padding if present.
1627 LValue TempLVal = projectValue();
1628
1629 // Okay, store the rvalue in.
1630 if (rvalue.isScalar()) {
1631 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1632 } else {
1633 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1634 }
1635 }
1636
1637
1638 /// Materialize an r-value into memory for the purposes of storing it
1639 /// to an atomic type.
materializeRValue(RValue rvalue) const1640 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1641 // Aggregate r-values are already in memory, and EmitAtomicStore
1642 // requires them to be values of the atomic type.
1643 if (rvalue.isAggregate())
1644 return rvalue.getAggregateAddress();
1645
1646 // Otherwise, make a temporary and materialize into it.
1647 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1648 AtomicInfo Atomics(CGF, TempLV);
1649 Atomics.emitCopyIntoMemory(rvalue);
1650 return TempLV.getAddress();
1651 }
1652
getScalarRValValueOrNull(RValue RVal) const1653 llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const {
1654 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple()))
1655 return RVal.getScalarVal();
1656 return nullptr;
1657 }
1658
convertRValueToInt(RValue RVal,bool CmpXchg) const1659 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const {
1660 // If we've got a scalar value of the right size, try to avoid going
1661 // through memory. Floats get casted if needed by AtomicExpandPass.
1662 if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) {
1663 if (!shouldCastToInt(Value->getType(), CmpXchg))
1664 return CGF.EmitToMemory(Value, ValueTy);
1665 else {
1666 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1667 CGF.getLLVMContext(),
1668 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1669 if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1670 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1671 }
1672 }
1673 // Otherwise, we need to go through memory.
1674 // Put the r-value in memory.
1675 Address Addr = materializeRValue(RVal);
1676
1677 // Cast the temporary to the atomic int type and pull a value out.
1678 Addr = castToAtomicIntPointer(Addr);
1679 return CGF.Builder.CreateLoad(Addr);
1680 }
1681
EmitAtomicCompareExchangeOp(llvm::Value * ExpectedVal,llvm::Value * DesiredVal,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak)1682 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1683 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1684 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1685 // Do the atomic store.
1686 Address Addr = getAtomicAddressAsAtomicIntPointer();
1687 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
1688 Success, Failure);
1689 // Other decoration.
1690 Inst->setVolatile(LVal.isVolatileQualified());
1691 Inst->setWeak(IsWeak);
1692
1693 // Okay, turn that back into the original value type.
1694 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1695 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1696 return std::make_pair(PreviousVal, SuccessFailureVal);
1697 }
1698
1699 llvm::Value *
EmitAtomicCompareExchangeLibcall(llvm::Value * ExpectedAddr,llvm::Value * DesiredAddr,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure)1700 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1701 llvm::Value *DesiredAddr,
1702 llvm::AtomicOrdering Success,
1703 llvm::AtomicOrdering Failure) {
1704 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1705 // void *desired, int success, int failure);
1706 CallArgList Args;
1707 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1708 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);
1709 Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy);
1710 Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy);
1711 Args.add(RValue::get(
1712 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1713 CGF.getContext().IntTy);
1714 Args.add(RValue::get(
1715 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1716 CGF.getContext().IntTy);
1717 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1718 CGF.getContext().BoolTy, Args);
1719
1720 return SuccessFailureRVal.getScalarVal();
1721 }
1722
EmitAtomicCompareExchange(RValue Expected,RValue Desired,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak)1723 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1724 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1725 llvm::AtomicOrdering Failure, bool IsWeak) {
1726 // Check whether we should use a library call.
1727 if (shouldUseLibcall()) {
1728 // Produce a source address.
1729 Address ExpectedAddr = materializeRValue(Expected);
1730 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1731 llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF);
1732 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr,
1733 Success, Failure);
1734 return std::make_pair(
1735 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1736 SourceLocation(), /*AsValue=*/false),
1737 Res);
1738 }
1739
1740 // If we've got a scalar value of the right size, try to avoid going
1741 // through memory.
1742 auto *ExpectedVal = convertRValueToInt(Expected, /*CmpXchg=*/true);
1743 auto *DesiredVal = convertRValueToInt(Desired, /*CmpXchg=*/true);
1744 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1745 Failure, IsWeak);
1746 return std::make_pair(
1747 ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1748 SourceLocation(), /*AsValue=*/false,
1749 /*CmpXchg=*/true),
1750 Res.second);
1751 }
1752
1753 static void
EmitAtomicUpdateValue(CodeGenFunction & CGF,AtomicInfo & Atomics,RValue OldRVal,const llvm::function_ref<RValue (RValue)> & UpdateOp,Address DesiredAddr)1754 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1755 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1756 Address DesiredAddr) {
1757 RValue UpRVal;
1758 LValue AtomicLVal = Atomics.getAtomicLValue();
1759 LValue DesiredLVal;
1760 if (AtomicLVal.isSimple()) {
1761 UpRVal = OldRVal;
1762 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1763 } else {
1764 // Build new lvalue for temp address.
1765 Address Ptr = Atomics.materializeRValue(OldRVal);
1766 LValue UpdateLVal;
1767 if (AtomicLVal.isBitField()) {
1768 UpdateLVal =
1769 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1770 AtomicLVal.getType(),
1771 AtomicLVal.getBaseInfo(),
1772 AtomicLVal.getTBAAInfo());
1773 DesiredLVal =
1774 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1775 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1776 AtomicLVal.getTBAAInfo());
1777 } else if (AtomicLVal.isVectorElt()) {
1778 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1779 AtomicLVal.getType(),
1780 AtomicLVal.getBaseInfo(),
1781 AtomicLVal.getTBAAInfo());
1782 DesiredLVal = LValue::MakeVectorElt(
1783 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1784 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1785 } else {
1786 assert(AtomicLVal.isExtVectorElt());
1787 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1788 AtomicLVal.getType(),
1789 AtomicLVal.getBaseInfo(),
1790 AtomicLVal.getTBAAInfo());
1791 DesiredLVal = LValue::MakeExtVectorElt(
1792 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1793 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1794 }
1795 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1796 }
1797 // Store new value in the corresponding memory area.
1798 RValue NewRVal = UpdateOp(UpRVal);
1799 if (NewRVal.isScalar()) {
1800 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1801 } else {
1802 assert(NewRVal.isComplex());
1803 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1804 /*isInit=*/false);
1805 }
1806 }
1807
EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)1808 void AtomicInfo::EmitAtomicUpdateLibcall(
1809 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1810 bool IsVolatile) {
1811 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1812
1813 Address ExpectedAddr = CreateTempAlloca();
1814
1815 EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
1816 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1817 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1818 CGF.EmitBlock(ContBB);
1819 Address DesiredAddr = CreateTempAlloca();
1820 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1821 requiresMemSetZero(getAtomicAddress().getElementType())) {
1822 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1823 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1824 }
1825 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1826 AggValueSlot::ignored(),
1827 SourceLocation(), /*AsValue=*/false);
1828 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1829 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1830 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
1831 auto *Res =
1832 EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
1833 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1834 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1835 }
1836
EmitAtomicUpdateOp(llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)1837 void AtomicInfo::EmitAtomicUpdateOp(
1838 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1839 bool IsVolatile) {
1840 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1841
1842 // Do the atomic load.
1843 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true);
1844 // For non-simple lvalues perform compare-and-swap procedure.
1845 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1846 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1847 auto *CurBB = CGF.Builder.GetInsertBlock();
1848 CGF.EmitBlock(ContBB);
1849 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1850 /*NumReservedValues=*/2);
1851 PHI->addIncoming(OldVal, CurBB);
1852 Address NewAtomicAddr = CreateTempAlloca();
1853 Address NewAtomicIntAddr =
1854 shouldCastToInt(NewAtomicAddr.getElementType(), /*CmpXchg=*/true)
1855 ? castToAtomicIntPointer(NewAtomicAddr)
1856 : NewAtomicAddr;
1857
1858 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1859 requiresMemSetZero(getAtomicAddress().getElementType())) {
1860 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1861 }
1862 auto OldRVal = ConvertToValueOrAtomic(PHI, AggValueSlot::ignored(),
1863 SourceLocation(), /*AsValue=*/false,
1864 /*CmpXchg=*/true);
1865 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1866 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1867 // Try to write new value using cmpxchg operation.
1868 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1869 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1870 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1871 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1872 }
1873
EmitAtomicUpdateValue(CodeGenFunction & CGF,AtomicInfo & Atomics,RValue UpdateRVal,Address DesiredAddr)1874 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1875 RValue UpdateRVal, Address DesiredAddr) {
1876 LValue AtomicLVal = Atomics.getAtomicLValue();
1877 LValue DesiredLVal;
1878 // Build new lvalue for temp address.
1879 if (AtomicLVal.isBitField()) {
1880 DesiredLVal =
1881 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1882 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1883 AtomicLVal.getTBAAInfo());
1884 } else if (AtomicLVal.isVectorElt()) {
1885 DesiredLVal =
1886 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1887 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1888 AtomicLVal.getTBAAInfo());
1889 } else {
1890 assert(AtomicLVal.isExtVectorElt());
1891 DesiredLVal = LValue::MakeExtVectorElt(
1892 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1893 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1894 }
1895 // Store new value in the corresponding memory area.
1896 assert(UpdateRVal.isScalar());
1897 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1898 }
1899
EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,RValue UpdateRVal,bool IsVolatile)1900 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1901 RValue UpdateRVal, bool IsVolatile) {
1902 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1903
1904 Address ExpectedAddr = CreateTempAlloca();
1905
1906 EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
1907 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1908 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1909 CGF.EmitBlock(ContBB);
1910 Address DesiredAddr = CreateTempAlloca();
1911 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1912 requiresMemSetZero(getAtomicAddress().getElementType())) {
1913 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1914 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1915 }
1916 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1917 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1918 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
1919 auto *Res =
1920 EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
1921 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1922 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1923 }
1924
EmitAtomicUpdateOp(llvm::AtomicOrdering AO,RValue UpdateRVal,bool IsVolatile)1925 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1926 bool IsVolatile) {
1927 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1928
1929 // Do the atomic load.
1930 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true);
1931 // For non-simple lvalues perform compare-and-swap procedure.
1932 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1933 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1934 auto *CurBB = CGF.Builder.GetInsertBlock();
1935 CGF.EmitBlock(ContBB);
1936 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1937 /*NumReservedValues=*/2);
1938 PHI->addIncoming(OldVal, CurBB);
1939 Address NewAtomicAddr = CreateTempAlloca();
1940 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
1941 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1942 requiresMemSetZero(getAtomicAddress().getElementType())) {
1943 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1944 }
1945 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1946 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1947 // Try to write new value using cmpxchg operation.
1948 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1949 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1950 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1951 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1952 }
1953
EmitAtomicUpdate(llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)1954 void AtomicInfo::EmitAtomicUpdate(
1955 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1956 bool IsVolatile) {
1957 if (shouldUseLibcall()) {
1958 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1959 } else {
1960 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1961 }
1962 }
1963
EmitAtomicUpdate(llvm::AtomicOrdering AO,RValue UpdateRVal,bool IsVolatile)1964 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1965 bool IsVolatile) {
1966 if (shouldUseLibcall()) {
1967 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1968 } else {
1969 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1970 }
1971 }
1972
EmitAtomicStore(RValue rvalue,LValue lvalue,bool isInit)1973 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1974 bool isInit) {
1975 bool IsVolatile = lvalue.isVolatileQualified();
1976 llvm::AtomicOrdering AO;
1977 if (lvalue.getType()->isAtomicType()) {
1978 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1979 } else {
1980 AO = llvm::AtomicOrdering::Release;
1981 IsVolatile = true;
1982 }
1983 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1984 }
1985
1986 /// Emit a store to an l-value of atomic type.
1987 ///
1988 /// Note that the r-value is expected to be an r-value *of the atomic
1989 /// type*; this means that for aggregate r-values, it should include
1990 /// storage for any padding that was necessary.
EmitAtomicStore(RValue rvalue,LValue dest,llvm::AtomicOrdering AO,bool IsVolatile,bool isInit)1991 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1992 llvm::AtomicOrdering AO, bool IsVolatile,
1993 bool isInit) {
1994 // If this is an aggregate r-value, it should agree in type except
1995 // maybe for address-space qualification.
1996 assert(!rvalue.isAggregate() ||
1997 rvalue.getAggregateAddress().getElementType() ==
1998 dest.getAddress().getElementType());
1999
2000 AtomicInfo atomics(*this, dest);
2001 LValue LVal = atomics.getAtomicLValue();
2002
2003 // If this is an initialization, just put the value there normally.
2004 if (LVal.isSimple()) {
2005 if (isInit) {
2006 atomics.emitCopyIntoMemory(rvalue);
2007 return;
2008 }
2009
2010 // Check whether we should use a library call.
2011 if (atomics.shouldUseLibcall()) {
2012 // Produce a source address.
2013 Address srcAddr = atomics.materializeRValue(rvalue);
2014
2015 // void __atomic_store(size_t size, void *mem, void *val, int order)
2016 CallArgList args;
2017 args.add(RValue::get(atomics.getAtomicSizeValue()),
2018 getContext().getSizeType());
2019 args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy);
2020 args.add(RValue::get(srcAddr.emitRawPointer(*this)),
2021 getContext().VoidPtrTy);
2022 args.add(
2023 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2024 getContext().IntTy);
2025 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2026 return;
2027 }
2028
2029 // Okay, we're doing this natively.
2030 llvm::Value *ValToStore = atomics.convertRValueToInt(rvalue);
2031
2032 // Do the atomic store.
2033 Address Addr = atomics.getAtomicAddress();
2034 if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue))
2035 if (shouldCastToInt(Value->getType(), /*CmpXchg=*/false)) {
2036 Addr = atomics.castToAtomicIntPointer(Addr);
2037 ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(),
2038 /*isSigned=*/false);
2039 }
2040 llvm::StoreInst *store = Builder.CreateStore(ValToStore, Addr);
2041
2042 if (AO == llvm::AtomicOrdering::Acquire)
2043 AO = llvm::AtomicOrdering::Monotonic;
2044 else if (AO == llvm::AtomicOrdering::AcquireRelease)
2045 AO = llvm::AtomicOrdering::Release;
2046 // Initializations don't need to be atomic.
2047 if (!isInit)
2048 store->setAtomic(AO);
2049
2050 // Other decoration.
2051 if (IsVolatile)
2052 store->setVolatile(true);
2053 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2054 return;
2055 }
2056
2057 // Emit simple atomic update operation.
2058 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2059 }
2060
2061 /// Emit a compare-and-exchange op for atomic type.
2062 ///
EmitAtomicCompareExchange(LValue Obj,RValue Expected,RValue Desired,SourceLocation Loc,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak,AggValueSlot Slot)2063 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2064 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2065 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2066 AggValueSlot Slot) {
2067 // If this is an aggregate r-value, it should agree in type except
2068 // maybe for address-space qualification.
2069 assert(!Expected.isAggregate() ||
2070 Expected.getAggregateAddress().getElementType() ==
2071 Obj.getAddress().getElementType());
2072 assert(!Desired.isAggregate() ||
2073 Desired.getAggregateAddress().getElementType() ==
2074 Obj.getAddress().getElementType());
2075 AtomicInfo Atomics(*this, Obj);
2076
2077 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2078 IsWeak);
2079 }
2080
2081 llvm::AtomicRMWInst *
emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op,Address Addr,llvm::Value * Val,llvm::AtomicOrdering Order,llvm::SyncScope::ID SSID,const AtomicExpr * AE)2082 CodeGenFunction::emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr,
2083 llvm::Value *Val, llvm::AtomicOrdering Order,
2084 llvm::SyncScope::ID SSID,
2085 const AtomicExpr *AE) {
2086 llvm::AtomicRMWInst *RMW =
2087 Builder.CreateAtomicRMW(Op, Addr, Val, Order, SSID);
2088 getTargetHooks().setTargetAtomicMetadata(*this, *RMW, AE);
2089 return RMW;
2090 }
2091
EmitAtomicUpdate(LValue LVal,llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)2092 void CodeGenFunction::EmitAtomicUpdate(
2093 LValue LVal, llvm::AtomicOrdering AO,
2094 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2095 AtomicInfo Atomics(*this, LVal);
2096 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2097 }
2098
EmitAtomicInit(Expr * init,LValue dest)2099 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2100 AtomicInfo atomics(*this, dest);
2101
2102 switch (atomics.getEvaluationKind()) {
2103 case TEK_Scalar: {
2104 llvm::Value *value = EmitScalarExpr(init);
2105 atomics.emitCopyIntoMemory(RValue::get(value));
2106 return;
2107 }
2108
2109 case TEK_Complex: {
2110 ComplexPairTy value = EmitComplexExpr(init);
2111 atomics.emitCopyIntoMemory(RValue::getComplex(value));
2112 return;
2113 }
2114
2115 case TEK_Aggregate: {
2116 // Fix up the destination if the initializer isn't an expression
2117 // of atomic type.
2118 bool Zeroed = false;
2119 if (!init->getType()->isAtomicType()) {
2120 Zeroed = atomics.emitMemSetZeroIfNecessary();
2121 dest = atomics.projectValue();
2122 }
2123
2124 // Evaluate the expression directly into the destination.
2125 AggValueSlot slot = AggValueSlot::forLValue(
2126 dest, AggValueSlot::IsNotDestructed,
2127 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2128 AggValueSlot::DoesNotOverlap,
2129 Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2130
2131 EmitAggExpr(init, slot);
2132 return;
2133 }
2134 }
2135 llvm_unreachable("bad evaluation kind");
2136 }
2137