xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGAtomic.cpp (revision c1d255d3ffdbe447de3ab875bf4e7d7accc5bfc5)
1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the code for emitting atomic operations.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCall.h"
14 #include "CGRecordLayout.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/CodeGen/CGFunctionInfo.h"
20 #include "clang/Frontend/FrontendDiagnostic.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/Intrinsics.h"
24 #include "llvm/IR/Operator.h"
25 
26 using namespace clang;
27 using namespace CodeGen;
28 
29 namespace {
30   class AtomicInfo {
31     CodeGenFunction &CGF;
32     QualType AtomicTy;
33     QualType ValueTy;
34     uint64_t AtomicSizeInBits;
35     uint64_t ValueSizeInBits;
36     CharUnits AtomicAlign;
37     CharUnits ValueAlign;
38     TypeEvaluationKind EvaluationKind;
39     bool UseLibcall;
40     LValue LVal;
41     CGBitFieldInfo BFI;
42   public:
43     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44         : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45           EvaluationKind(TEK_Scalar), UseLibcall(true) {
46       assert(!lvalue.isGlobalReg());
47       ASTContext &C = CGF.getContext();
48       if (lvalue.isSimple()) {
49         AtomicTy = lvalue.getType();
50         if (auto *ATy = AtomicTy->getAs<AtomicType>())
51           ValueTy = ATy->getValueType();
52         else
53           ValueTy = AtomicTy;
54         EvaluationKind = CGF.getEvaluationKind(ValueTy);
55 
56         uint64_t ValueAlignInBits;
57         uint64_t AtomicAlignInBits;
58         TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59         ValueSizeInBits = ValueTI.Width;
60         ValueAlignInBits = ValueTI.Align;
61 
62         TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63         AtomicSizeInBits = AtomicTI.Width;
64         AtomicAlignInBits = AtomicTI.Align;
65 
66         assert(ValueSizeInBits <= AtomicSizeInBits);
67         assert(ValueAlignInBits <= AtomicAlignInBits);
68 
69         AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70         ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71         if (lvalue.getAlignment().isZero())
72           lvalue.setAlignment(AtomicAlign);
73 
74         LVal = lvalue;
75       } else if (lvalue.isBitField()) {
76         ValueTy = lvalue.getType();
77         ValueSizeInBits = C.getTypeSize(ValueTy);
78         auto &OrigBFI = lvalue.getBitFieldInfo();
79         auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80         AtomicSizeInBits = C.toBits(
81             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82                 .alignTo(lvalue.getAlignment()));
83         auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84         auto OffsetInChars =
85             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86             lvalue.getAlignment();
87         VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88             VoidPtrAddr, OffsetInChars.getQuantity());
89         auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90             VoidPtrAddr,
91             CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92             "atomic_bitfield_base");
93         BFI = OrigBFI;
94         BFI.Offset = Offset;
95         BFI.StorageSize = AtomicSizeInBits;
96         BFI.StorageOffset += OffsetInChars;
97         LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98                                     BFI, lvalue.getType(), lvalue.getBaseInfo(),
99                                     lvalue.getTBAAInfo());
100         AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
101         if (AtomicTy.isNull()) {
102           llvm::APInt Size(
103               /*numBits=*/32,
104               C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105           AtomicTy =
106               C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
107                                      /*IndexTypeQuals=*/0);
108         }
109         AtomicAlign = ValueAlign = lvalue.getAlignment();
110       } else if (lvalue.isVectorElt()) {
111         ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
112         ValueSizeInBits = C.getTypeSize(ValueTy);
113         AtomicTy = lvalue.getType();
114         AtomicSizeInBits = C.getTypeSize(AtomicTy);
115         AtomicAlign = ValueAlign = lvalue.getAlignment();
116         LVal = lvalue;
117       } else {
118         assert(lvalue.isExtVectorElt());
119         ValueTy = lvalue.getType();
120         ValueSizeInBits = C.getTypeSize(ValueTy);
121         AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122             lvalue.getType(), cast<llvm::FixedVectorType>(
123                                   lvalue.getExtVectorAddress().getElementType())
124                                   ->getNumElements());
125         AtomicSizeInBits = C.getTypeSize(AtomicTy);
126         AtomicAlign = ValueAlign = lvalue.getAlignment();
127         LVal = lvalue;
128       }
129       UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
130           AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
131     }
132 
133     QualType getAtomicType() const { return AtomicTy; }
134     QualType getValueType() const { return ValueTy; }
135     CharUnits getAtomicAlignment() const { return AtomicAlign; }
136     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137     uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139     bool shouldUseLibcall() const { return UseLibcall; }
140     const LValue &getAtomicLValue() const { return LVal; }
141     llvm::Value *getAtomicPointer() const {
142       if (LVal.isSimple())
143         return LVal.getPointer(CGF);
144       else if (LVal.isBitField())
145         return LVal.getBitFieldPointer();
146       else if (LVal.isVectorElt())
147         return LVal.getVectorPointer();
148       assert(LVal.isExtVectorElt());
149       return LVal.getExtVectorPointer();
150     }
151     Address getAtomicAddress() const {
152       return Address(getAtomicPointer(), getAtomicAlignment());
153     }
154 
155     Address getAtomicAddressAsAtomicIntPointer() const {
156       return emitCastToAtomicIntPointer(getAtomicAddress());
157     }
158 
159     /// Is the atomic size larger than the underlying value type?
160     ///
161     /// Note that the absence of padding does not mean that atomic
162     /// objects are completely interchangeable with non-atomic
163     /// objects: we might have promoted the alignment of a type
164     /// without making it bigger.
165     bool hasPadding() const {
166       return (ValueSizeInBits != AtomicSizeInBits);
167     }
168 
169     bool emitMemSetZeroIfNecessary() const;
170 
171     llvm::Value *getAtomicSizeValue() const {
172       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173       return CGF.CGM.getSize(size);
174     }
175 
176     /// Cast the given pointer to an integer pointer suitable for atomic
177     /// operations if the source.
178     Address emitCastToAtomicIntPointer(Address Addr) const;
179 
180     /// If Addr is compatible with the iN that will be used for an atomic
181     /// operation, bitcast it. Otherwise, create a temporary that is suitable
182     /// and copy the value across.
183     Address convertToAtomicIntPointer(Address Addr) const;
184 
185     /// Turn an atomic-layout object into an r-value.
186     RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187                                      SourceLocation loc, bool AsValue) const;
188 
189     /// Converts a rvalue to integer value.
190     llvm::Value *convertRValueToInt(RValue RVal) const;
191 
192     RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193                                      AggValueSlot ResultSlot,
194                                      SourceLocation Loc, bool AsValue) const;
195 
196     /// Copy an atomic r-value into atomic-layout memory.
197     void emitCopyIntoMemory(RValue rvalue) const;
198 
199     /// Project an l-value down to the value field.
200     LValue projectValue() const {
201       assert(LVal.isSimple());
202       Address addr = getAtomicAddress();
203       if (hasPadding())
204         addr = CGF.Builder.CreateStructGEP(addr, 0);
205 
206       return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207                               LVal.getBaseInfo(), LVal.getTBAAInfo());
208     }
209 
210     /// Emits atomic load.
211     /// \returns Loaded value.
212     RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213                           bool AsValue, llvm::AtomicOrdering AO,
214                           bool IsVolatile);
215 
216     /// Emits atomic compare-and-exchange sequence.
217     /// \param Expected Expected value.
218     /// \param Desired Desired value.
219     /// \param Success Atomic ordering for success operation.
220     /// \param Failure Atomic ordering for failed operation.
221     /// \param IsWeak true if atomic operation is weak, false otherwise.
222     /// \returns Pair of values: previous value from storage (value type) and
223     /// boolean flag (i1 type) with true if success and false otherwise.
224     std::pair<RValue, llvm::Value *>
225     EmitAtomicCompareExchange(RValue Expected, RValue Desired,
226                               llvm::AtomicOrdering Success =
227                                   llvm::AtomicOrdering::SequentiallyConsistent,
228                               llvm::AtomicOrdering Failure =
229                                   llvm::AtomicOrdering::SequentiallyConsistent,
230                               bool IsWeak = false);
231 
232     /// Emits atomic update.
233     /// \param AO Atomic ordering.
234     /// \param UpdateOp Update operation for the current lvalue.
235     void EmitAtomicUpdate(llvm::AtomicOrdering AO,
236                           const llvm::function_ref<RValue(RValue)> &UpdateOp,
237                           bool IsVolatile);
238     /// Emits atomic update.
239     /// \param AO Atomic ordering.
240     void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
241                           bool IsVolatile);
242 
243     /// Materialize an atomic r-value in atomic-layout memory.
244     Address materializeRValue(RValue rvalue) const;
245 
246     /// Creates temp alloca for intermediate operations on atomic value.
247     Address CreateTempAlloca() const;
248   private:
249     bool requiresMemSetZero(llvm::Type *type) const;
250 
251 
252     /// Emits atomic load as a libcall.
253     void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
254                                llvm::AtomicOrdering AO, bool IsVolatile);
255     /// Emits atomic load as LLVM instruction.
256     llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
257     /// Emits atomic compare-and-exchange op as a libcall.
258     llvm::Value *EmitAtomicCompareExchangeLibcall(
259         llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
260         llvm::AtomicOrdering Success =
261             llvm::AtomicOrdering::SequentiallyConsistent,
262         llvm::AtomicOrdering Failure =
263             llvm::AtomicOrdering::SequentiallyConsistent);
264     /// Emits atomic compare-and-exchange op as LLVM instruction.
265     std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
266         llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
267         llvm::AtomicOrdering Success =
268             llvm::AtomicOrdering::SequentiallyConsistent,
269         llvm::AtomicOrdering Failure =
270             llvm::AtomicOrdering::SequentiallyConsistent,
271         bool IsWeak = false);
272     /// Emit atomic update as libcalls.
273     void
274     EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
275                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
276                             bool IsVolatile);
277     /// Emit atomic update as LLVM instructions.
278     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
279                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
280                             bool IsVolatile);
281     /// Emit atomic update as libcalls.
282     void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283                                  bool IsVolatile);
284     /// Emit atomic update as LLVM instructions.
285     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
286                             bool IsVolatile);
287   };
288 }
289 
290 Address AtomicInfo::CreateTempAlloca() const {
291   Address TempAlloca = CGF.CreateMemTemp(
292       (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
293                                                                 : AtomicTy,
294       getAtomicAlignment(),
295       "atomic-temp");
296   // Cast to pointer to value type for bitfields.
297   if (LVal.isBitField())
298     return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
299         TempAlloca, getAtomicAddress().getType());
300   return TempAlloca;
301 }
302 
303 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
304                                 StringRef fnName,
305                                 QualType resultType,
306                                 CallArgList &args) {
307   const CGFunctionInfo &fnInfo =
308     CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
309   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
310   llvm::AttrBuilder fnAttrB;
311   fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
312   fnAttrB.addAttribute(llvm::Attribute::WillReturn);
313   llvm::AttributeList fnAttrs = llvm::AttributeList::get(
314       CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
315 
316   llvm::FunctionCallee fn =
317       CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
318   auto callee = CGCallee::forDirect(fn);
319   return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
320 }
321 
322 /// Does a store of the given IR type modify the full expected width?
323 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
324                            uint64_t expectedSize) {
325   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
326 }
327 
328 /// Does the atomic type require memsetting to zero before initialization?
329 ///
330 /// The IR type is provided as a way of making certain queries faster.
331 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
332   // If the atomic type has size padding, we definitely need a memset.
333   if (hasPadding()) return true;
334 
335   // Otherwise, do some simple heuristics to try to avoid it:
336   switch (getEvaluationKind()) {
337   // For scalars and complexes, check whether the store size of the
338   // type uses the full size.
339   case TEK_Scalar:
340     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
341   case TEK_Complex:
342     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
343                            AtomicSizeInBits / 2);
344 
345   // Padding in structs has an undefined bit pattern.  User beware.
346   case TEK_Aggregate:
347     return false;
348   }
349   llvm_unreachable("bad evaluation kind");
350 }
351 
352 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
353   assert(LVal.isSimple());
354   llvm::Value *addr = LVal.getPointer(CGF);
355   if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
356     return false;
357 
358   CGF.Builder.CreateMemSet(
359       addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
360       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
361       LVal.getAlignment().getAsAlign());
362   return true;
363 }
364 
365 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
366                               Address Dest, Address Ptr,
367                               Address Val1, Address Val2,
368                               uint64_t Size,
369                               llvm::AtomicOrdering SuccessOrder,
370                               llvm::AtomicOrdering FailureOrder,
371                               llvm::SyncScope::ID Scope) {
372   // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
373   llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
374   llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
375 
376   llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
377       Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
378       Scope);
379   Pair->setVolatile(E->isVolatile());
380   Pair->setWeak(IsWeak);
381 
382   // Cmp holds the result of the compare-exchange operation: true on success,
383   // false on failure.
384   llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
385   llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
386 
387   // This basic block is used to hold the store instruction if the operation
388   // failed.
389   llvm::BasicBlock *StoreExpectedBB =
390       CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
391 
392   // This basic block is the exit point of the operation, we should end up
393   // here regardless of whether or not the operation succeeded.
394   llvm::BasicBlock *ContinueBB =
395       CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
396 
397   // Update Expected if Expected isn't equal to Old, otherwise branch to the
398   // exit point.
399   CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
400 
401   CGF.Builder.SetInsertPoint(StoreExpectedBB);
402   // Update the memory at Expected with Old's value.
403   CGF.Builder.CreateStore(Old, Val1);
404   // Finally, branch to the exit point.
405   CGF.Builder.CreateBr(ContinueBB);
406 
407   CGF.Builder.SetInsertPoint(ContinueBB);
408   // Update the memory at Dest with Cmp's value.
409   CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
410 }
411 
412 /// Given an ordering required on success, emit all possible cmpxchg
413 /// instructions to cope with the provided (but possibly only dynamically known)
414 /// FailureOrder.
415 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
416                                         bool IsWeak, Address Dest, Address Ptr,
417                                         Address Val1, Address Val2,
418                                         llvm::Value *FailureOrderVal,
419                                         uint64_t Size,
420                                         llvm::AtomicOrdering SuccessOrder,
421                                         llvm::SyncScope::ID Scope) {
422   llvm::AtomicOrdering FailureOrder;
423   if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
424     auto FOS = FO->getSExtValue();
425     if (!llvm::isValidAtomicOrderingCABI(FOS))
426       FailureOrder = llvm::AtomicOrdering::Monotonic;
427     else
428       switch ((llvm::AtomicOrderingCABI)FOS) {
429       case llvm::AtomicOrderingCABI::relaxed:
430       case llvm::AtomicOrderingCABI::release:
431       case llvm::AtomicOrderingCABI::acq_rel:
432         FailureOrder = llvm::AtomicOrdering::Monotonic;
433         break;
434       case llvm::AtomicOrderingCABI::consume:
435       case llvm::AtomicOrderingCABI::acquire:
436         FailureOrder = llvm::AtomicOrdering::Acquire;
437         break;
438       case llvm::AtomicOrderingCABI::seq_cst:
439         FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
440         break;
441       }
442     if (isStrongerThan(FailureOrder, SuccessOrder)) {
443       // Don't assert on undefined behavior "failure argument shall be no
444       // stronger than the success argument".
445       FailureOrder =
446           llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
447     }
448     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
449                       FailureOrder, Scope);
450     return;
451   }
452 
453   // Create all the relevant BB's
454   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
455                    *SeqCstBB = nullptr;
456   MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
457   if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
458       SuccessOrder != llvm::AtomicOrdering::Release)
459     AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
460   if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
461     SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
462 
463   llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
464 
465   llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
466 
467   // Emit all the different atomics
468 
469   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
470   // doesn't matter unless someone is crazy enough to use something that
471   // doesn't fold to a constant for the ordering.
472   CGF.Builder.SetInsertPoint(MonotonicBB);
473   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
474                     Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
475   CGF.Builder.CreateBr(ContBB);
476 
477   if (AcquireBB) {
478     CGF.Builder.SetInsertPoint(AcquireBB);
479     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
480                       Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
481     CGF.Builder.CreateBr(ContBB);
482     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
483                 AcquireBB);
484     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
485                 AcquireBB);
486   }
487   if (SeqCstBB) {
488     CGF.Builder.SetInsertPoint(SeqCstBB);
489     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
490                       llvm::AtomicOrdering::SequentiallyConsistent, Scope);
491     CGF.Builder.CreateBr(ContBB);
492     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
493                 SeqCstBB);
494   }
495 
496   CGF.Builder.SetInsertPoint(ContBB);
497 }
498 
499 /// Duplicate the atomic min/max operation in conventional IR for the builtin
500 /// variants that return the new rather than the original value.
501 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
502                                          AtomicExpr::AtomicOp Op,
503                                          bool IsSigned,
504                                          llvm::Value *OldVal,
505                                          llvm::Value *RHS) {
506   llvm::CmpInst::Predicate Pred;
507   switch (Op) {
508   default:
509     llvm_unreachable("Unexpected min/max operation");
510   case AtomicExpr::AO__atomic_max_fetch:
511     Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
512     break;
513   case AtomicExpr::AO__atomic_min_fetch:
514     Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
515     break;
516   }
517   llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
518   return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
519 }
520 
521 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
522                          Address Ptr, Address Val1, Address Val2,
523                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
524                          uint64_t Size, llvm::AtomicOrdering Order,
525                          llvm::SyncScope::ID Scope) {
526   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
527   bool PostOpMinMax = false;
528   unsigned PostOp = 0;
529 
530   switch (E->getOp()) {
531   case AtomicExpr::AO__c11_atomic_init:
532   case AtomicExpr::AO__opencl_atomic_init:
533     llvm_unreachable("Already handled!");
534 
535   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
536   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
537     emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
538                                 FailureOrder, Size, Order, Scope);
539     return;
540   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
541   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
542     emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
543                                 FailureOrder, Size, Order, Scope);
544     return;
545   case AtomicExpr::AO__atomic_compare_exchange:
546   case AtomicExpr::AO__atomic_compare_exchange_n: {
547     if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
548       emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
549                                   Val1, Val2, FailureOrder, Size, Order, Scope);
550     } else {
551       // Create all the relevant BB's
552       llvm::BasicBlock *StrongBB =
553           CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
554       llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
555       llvm::BasicBlock *ContBB =
556           CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
557 
558       llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
559       SI->addCase(CGF.Builder.getInt1(false), StrongBB);
560 
561       CGF.Builder.SetInsertPoint(StrongBB);
562       emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
563                                   FailureOrder, Size, Order, Scope);
564       CGF.Builder.CreateBr(ContBB);
565 
566       CGF.Builder.SetInsertPoint(WeakBB);
567       emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
568                                   FailureOrder, Size, Order, Scope);
569       CGF.Builder.CreateBr(ContBB);
570 
571       CGF.Builder.SetInsertPoint(ContBB);
572     }
573     return;
574   }
575   case AtomicExpr::AO__c11_atomic_load:
576   case AtomicExpr::AO__opencl_atomic_load:
577   case AtomicExpr::AO__atomic_load_n:
578   case AtomicExpr::AO__atomic_load: {
579     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
580     Load->setAtomic(Order, Scope);
581     Load->setVolatile(E->isVolatile());
582     CGF.Builder.CreateStore(Load, Dest);
583     return;
584   }
585 
586   case AtomicExpr::AO__c11_atomic_store:
587   case AtomicExpr::AO__opencl_atomic_store:
588   case AtomicExpr::AO__atomic_store:
589   case AtomicExpr::AO__atomic_store_n: {
590     llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
591     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
592     Store->setAtomic(Order, Scope);
593     Store->setVolatile(E->isVolatile());
594     return;
595   }
596 
597   case AtomicExpr::AO__c11_atomic_exchange:
598   case AtomicExpr::AO__opencl_atomic_exchange:
599   case AtomicExpr::AO__atomic_exchange_n:
600   case AtomicExpr::AO__atomic_exchange:
601     Op = llvm::AtomicRMWInst::Xchg;
602     break;
603 
604   case AtomicExpr::AO__atomic_add_fetch:
605     PostOp = llvm::Instruction::Add;
606     LLVM_FALLTHROUGH;
607   case AtomicExpr::AO__c11_atomic_fetch_add:
608   case AtomicExpr::AO__opencl_atomic_fetch_add:
609   case AtomicExpr::AO__atomic_fetch_add:
610     Op = llvm::AtomicRMWInst::Add;
611     break;
612 
613   case AtomicExpr::AO__atomic_sub_fetch:
614     PostOp = llvm::Instruction::Sub;
615     LLVM_FALLTHROUGH;
616   case AtomicExpr::AO__c11_atomic_fetch_sub:
617   case AtomicExpr::AO__opencl_atomic_fetch_sub:
618   case AtomicExpr::AO__atomic_fetch_sub:
619     Op = llvm::AtomicRMWInst::Sub;
620     break;
621 
622   case AtomicExpr::AO__atomic_min_fetch:
623     PostOpMinMax = true;
624     LLVM_FALLTHROUGH;
625   case AtomicExpr::AO__c11_atomic_fetch_min:
626   case AtomicExpr::AO__opencl_atomic_fetch_min:
627   case AtomicExpr::AO__atomic_fetch_min:
628     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
629                                                   : llvm::AtomicRMWInst::UMin;
630     break;
631 
632   case AtomicExpr::AO__atomic_max_fetch:
633     PostOpMinMax = true;
634     LLVM_FALLTHROUGH;
635   case AtomicExpr::AO__c11_atomic_fetch_max:
636   case AtomicExpr::AO__opencl_atomic_fetch_max:
637   case AtomicExpr::AO__atomic_fetch_max:
638     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
639                                                   : llvm::AtomicRMWInst::UMax;
640     break;
641 
642   case AtomicExpr::AO__atomic_and_fetch:
643     PostOp = llvm::Instruction::And;
644     LLVM_FALLTHROUGH;
645   case AtomicExpr::AO__c11_atomic_fetch_and:
646   case AtomicExpr::AO__opencl_atomic_fetch_and:
647   case AtomicExpr::AO__atomic_fetch_and:
648     Op = llvm::AtomicRMWInst::And;
649     break;
650 
651   case AtomicExpr::AO__atomic_or_fetch:
652     PostOp = llvm::Instruction::Or;
653     LLVM_FALLTHROUGH;
654   case AtomicExpr::AO__c11_atomic_fetch_or:
655   case AtomicExpr::AO__opencl_atomic_fetch_or:
656   case AtomicExpr::AO__atomic_fetch_or:
657     Op = llvm::AtomicRMWInst::Or;
658     break;
659 
660   case AtomicExpr::AO__atomic_xor_fetch:
661     PostOp = llvm::Instruction::Xor;
662     LLVM_FALLTHROUGH;
663   case AtomicExpr::AO__c11_atomic_fetch_xor:
664   case AtomicExpr::AO__opencl_atomic_fetch_xor:
665   case AtomicExpr::AO__atomic_fetch_xor:
666     Op = llvm::AtomicRMWInst::Xor;
667     break;
668 
669   case AtomicExpr::AO__atomic_nand_fetch:
670     PostOp = llvm::Instruction::And; // the NOT is special cased below
671     LLVM_FALLTHROUGH;
672   case AtomicExpr::AO__atomic_fetch_nand:
673     Op = llvm::AtomicRMWInst::Nand;
674     break;
675   }
676 
677   llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
678   llvm::AtomicRMWInst *RMWI =
679       CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
680   RMWI->setVolatile(E->isVolatile());
681 
682   // For __atomic_*_fetch operations, perform the operation again to
683   // determine the value which was written.
684   llvm::Value *Result = RMWI;
685   if (PostOpMinMax)
686     Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
687                                   E->getValueType()->isSignedIntegerType(),
688                                   RMWI, LoadVal1);
689   else if (PostOp)
690     Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
691                                      LoadVal1);
692   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
693     Result = CGF.Builder.CreateNot(Result);
694   CGF.Builder.CreateStore(Result, Dest);
695 }
696 
697 // This function emits any expression (scalar, complex, or aggregate)
698 // into a temporary alloca.
699 static Address
700 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
701   Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
702   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
703                        /*Init*/ true);
704   return DeclPtr;
705 }
706 
707 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
708                          Address Ptr, Address Val1, Address Val2,
709                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
710                          uint64_t Size, llvm::AtomicOrdering Order,
711                          llvm::Value *Scope) {
712   auto ScopeModel = Expr->getScopeModel();
713 
714   // LLVM atomic instructions always have synch scope. If clang atomic
715   // expression has no scope operand, use default LLVM synch scope.
716   if (!ScopeModel) {
717     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
718                  Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
719     return;
720   }
721 
722   // Handle constant scope.
723   if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
724     auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
725         CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
726         Order, CGF.CGM.getLLVMContext());
727     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
728                  Order, SCID);
729     return;
730   }
731 
732   // Handle non-constant scope.
733   auto &Builder = CGF.Builder;
734   auto Scopes = ScopeModel->getRuntimeValues();
735   llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
736   for (auto S : Scopes)
737     BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
738 
739   llvm::BasicBlock *ContBB =
740       CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
741 
742   auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
743   // If unsupported synch scope is encountered at run time, assume a fallback
744   // synch scope value.
745   auto FallBack = ScopeModel->getFallBackValue();
746   llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
747   for (auto S : Scopes) {
748     auto *B = BB[S];
749     if (S != FallBack)
750       SI->addCase(Builder.getInt32(S), B);
751 
752     Builder.SetInsertPoint(B);
753     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
754                  Order,
755                  CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
756                                                          ScopeModel->map(S),
757                                                          Order,
758                                                          CGF.getLLVMContext()));
759     Builder.CreateBr(ContBB);
760   }
761 
762   Builder.SetInsertPoint(ContBB);
763 }
764 
765 static void
766 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
767                   bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
768                   SourceLocation Loc, CharUnits SizeInChars) {
769   if (UseOptimizedLibcall) {
770     // Load value and pass it to the function directly.
771     CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
772     int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
773     ValTy =
774         CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
775     llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
776                                                 SizeInBits)->getPointerTo();
777     Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
778     Val = CGF.EmitLoadOfScalar(Ptr, false,
779                                CGF.getContext().getPointerType(ValTy),
780                                Loc);
781     // Coerce the value into an appropriately sized integer type.
782     Args.add(RValue::get(Val), ValTy);
783   } else {
784     // Non-optimized functions always take a reference.
785     Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
786                          CGF.getContext().VoidPtrTy);
787   }
788 }
789 
790 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
791   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
792   QualType MemTy = AtomicTy;
793   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
794     MemTy = AT->getValueType();
795   llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
796 
797   Address Val1 = Address::invalid();
798   Address Val2 = Address::invalid();
799   Address Dest = Address::invalid();
800   Address Ptr = EmitPointerWithAlignment(E->getPtr());
801 
802   if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
803       E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
804     LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
805     EmitAtomicInit(E->getVal1(), lvalue);
806     return RValue::get(nullptr);
807   }
808 
809   auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
810   uint64_t Size = TInfo.Width.getQuantity();
811   unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
812 
813   bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
814   bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
815   bool UseLibcall = Misaligned | Oversized;
816   CharUnits MaxInlineWidth =
817       getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
818 
819   DiagnosticsEngine &Diags = CGM.getDiags();
820 
821   if (Misaligned) {
822     Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
823         << (int)TInfo.Width.getQuantity()
824         << (int)Ptr.getAlignment().getQuantity();
825   }
826 
827   if (Oversized) {
828     Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
829         << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
830   }
831 
832   llvm::Value *Order = EmitScalarExpr(E->getOrder());
833   llvm::Value *Scope =
834       E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
835 
836   switch (E->getOp()) {
837   case AtomicExpr::AO__c11_atomic_init:
838   case AtomicExpr::AO__opencl_atomic_init:
839     llvm_unreachable("Already handled above with EmitAtomicInit!");
840 
841   case AtomicExpr::AO__c11_atomic_load:
842   case AtomicExpr::AO__opencl_atomic_load:
843   case AtomicExpr::AO__atomic_load_n:
844     break;
845 
846   case AtomicExpr::AO__atomic_load:
847     Dest = EmitPointerWithAlignment(E->getVal1());
848     break;
849 
850   case AtomicExpr::AO__atomic_store:
851     Val1 = EmitPointerWithAlignment(E->getVal1());
852     break;
853 
854   case AtomicExpr::AO__atomic_exchange:
855     Val1 = EmitPointerWithAlignment(E->getVal1());
856     Dest = EmitPointerWithAlignment(E->getVal2());
857     break;
858 
859   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
860   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
861   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
862   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
863   case AtomicExpr::AO__atomic_compare_exchange_n:
864   case AtomicExpr::AO__atomic_compare_exchange:
865     Val1 = EmitPointerWithAlignment(E->getVal1());
866     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
867       Val2 = EmitPointerWithAlignment(E->getVal2());
868     else
869       Val2 = EmitValToTemp(*this, E->getVal2());
870     OrderFail = EmitScalarExpr(E->getOrderFail());
871     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
872         E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
873       IsWeak = EmitScalarExpr(E->getWeak());
874     break;
875 
876   case AtomicExpr::AO__c11_atomic_fetch_add:
877   case AtomicExpr::AO__c11_atomic_fetch_sub:
878   case AtomicExpr::AO__opencl_atomic_fetch_add:
879   case AtomicExpr::AO__opencl_atomic_fetch_sub:
880     if (MemTy->isPointerType()) {
881       // For pointer arithmetic, we're required to do a bit of math:
882       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
883       // ... but only for the C11 builtins. The GNU builtins expect the
884       // user to multiply by sizeof(T).
885       QualType Val1Ty = E->getVal1()->getType();
886       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
887       CharUnits PointeeIncAmt =
888           getContext().getTypeSizeInChars(MemTy->getPointeeType());
889       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
890       auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
891       Val1 = Temp;
892       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
893       break;
894     }
895       LLVM_FALLTHROUGH;
896   case AtomicExpr::AO__atomic_fetch_add:
897   case AtomicExpr::AO__atomic_fetch_sub:
898   case AtomicExpr::AO__atomic_add_fetch:
899   case AtomicExpr::AO__atomic_sub_fetch:
900   case AtomicExpr::AO__c11_atomic_store:
901   case AtomicExpr::AO__c11_atomic_exchange:
902   case AtomicExpr::AO__opencl_atomic_store:
903   case AtomicExpr::AO__opencl_atomic_exchange:
904   case AtomicExpr::AO__atomic_store_n:
905   case AtomicExpr::AO__atomic_exchange_n:
906   case AtomicExpr::AO__c11_atomic_fetch_and:
907   case AtomicExpr::AO__c11_atomic_fetch_or:
908   case AtomicExpr::AO__c11_atomic_fetch_xor:
909   case AtomicExpr::AO__c11_atomic_fetch_max:
910   case AtomicExpr::AO__c11_atomic_fetch_min:
911   case AtomicExpr::AO__opencl_atomic_fetch_and:
912   case AtomicExpr::AO__opencl_atomic_fetch_or:
913   case AtomicExpr::AO__opencl_atomic_fetch_xor:
914   case AtomicExpr::AO__opencl_atomic_fetch_min:
915   case AtomicExpr::AO__opencl_atomic_fetch_max:
916   case AtomicExpr::AO__atomic_fetch_and:
917   case AtomicExpr::AO__atomic_fetch_or:
918   case AtomicExpr::AO__atomic_fetch_xor:
919   case AtomicExpr::AO__atomic_fetch_nand:
920   case AtomicExpr::AO__atomic_and_fetch:
921   case AtomicExpr::AO__atomic_or_fetch:
922   case AtomicExpr::AO__atomic_xor_fetch:
923   case AtomicExpr::AO__atomic_nand_fetch:
924   case AtomicExpr::AO__atomic_max_fetch:
925   case AtomicExpr::AO__atomic_min_fetch:
926   case AtomicExpr::AO__atomic_fetch_max:
927   case AtomicExpr::AO__atomic_fetch_min:
928     Val1 = EmitValToTemp(*this, E->getVal1());
929     break;
930   }
931 
932   QualType RValTy = E->getType().getUnqualifiedType();
933 
934   // The inlined atomics only function on iN types, where N is a power of 2. We
935   // need to make sure (via temporaries if necessary) that all incoming values
936   // are compatible.
937   LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
938   AtomicInfo Atomics(*this, AtomicVal);
939 
940   Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
941   if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
942   if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
943   if (Dest.isValid())
944     Dest = Atomics.emitCastToAtomicIntPointer(Dest);
945   else if (E->isCmpXChg())
946     Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
947   else if (!RValTy->isVoidType())
948     Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
949 
950   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
951   if (UseLibcall) {
952     bool UseOptimizedLibcall = false;
953     switch (E->getOp()) {
954     case AtomicExpr::AO__c11_atomic_init:
955     case AtomicExpr::AO__opencl_atomic_init:
956       llvm_unreachable("Already handled above with EmitAtomicInit!");
957 
958     case AtomicExpr::AO__c11_atomic_fetch_add:
959     case AtomicExpr::AO__opencl_atomic_fetch_add:
960     case AtomicExpr::AO__atomic_fetch_add:
961     case AtomicExpr::AO__c11_atomic_fetch_and:
962     case AtomicExpr::AO__opencl_atomic_fetch_and:
963     case AtomicExpr::AO__atomic_fetch_and:
964     case AtomicExpr::AO__c11_atomic_fetch_or:
965     case AtomicExpr::AO__opencl_atomic_fetch_or:
966     case AtomicExpr::AO__atomic_fetch_or:
967     case AtomicExpr::AO__atomic_fetch_nand:
968     case AtomicExpr::AO__c11_atomic_fetch_sub:
969     case AtomicExpr::AO__opencl_atomic_fetch_sub:
970     case AtomicExpr::AO__atomic_fetch_sub:
971     case AtomicExpr::AO__c11_atomic_fetch_xor:
972     case AtomicExpr::AO__opencl_atomic_fetch_xor:
973     case AtomicExpr::AO__opencl_atomic_fetch_min:
974     case AtomicExpr::AO__opencl_atomic_fetch_max:
975     case AtomicExpr::AO__atomic_fetch_xor:
976     case AtomicExpr::AO__c11_atomic_fetch_max:
977     case AtomicExpr::AO__c11_atomic_fetch_min:
978     case AtomicExpr::AO__atomic_add_fetch:
979     case AtomicExpr::AO__atomic_and_fetch:
980     case AtomicExpr::AO__atomic_nand_fetch:
981     case AtomicExpr::AO__atomic_or_fetch:
982     case AtomicExpr::AO__atomic_sub_fetch:
983     case AtomicExpr::AO__atomic_xor_fetch:
984     case AtomicExpr::AO__atomic_fetch_max:
985     case AtomicExpr::AO__atomic_fetch_min:
986     case AtomicExpr::AO__atomic_max_fetch:
987     case AtomicExpr::AO__atomic_min_fetch:
988       // For these, only library calls for certain sizes exist.
989       UseOptimizedLibcall = true;
990       break;
991 
992     case AtomicExpr::AO__atomic_load:
993     case AtomicExpr::AO__atomic_store:
994     case AtomicExpr::AO__atomic_exchange:
995     case AtomicExpr::AO__atomic_compare_exchange:
996       // Use the generic version if we don't know that the operand will be
997       // suitably aligned for the optimized version.
998       if (Misaligned)
999         break;
1000       LLVM_FALLTHROUGH;
1001     case AtomicExpr::AO__c11_atomic_load:
1002     case AtomicExpr::AO__c11_atomic_store:
1003     case AtomicExpr::AO__c11_atomic_exchange:
1004     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1005     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1006     case AtomicExpr::AO__opencl_atomic_load:
1007     case AtomicExpr::AO__opencl_atomic_store:
1008     case AtomicExpr::AO__opencl_atomic_exchange:
1009     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1010     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1011     case AtomicExpr::AO__atomic_load_n:
1012     case AtomicExpr::AO__atomic_store_n:
1013     case AtomicExpr::AO__atomic_exchange_n:
1014     case AtomicExpr::AO__atomic_compare_exchange_n:
1015       // Only use optimized library calls for sizes for which they exist.
1016       // FIXME: Size == 16 optimized library functions exist too.
1017       if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
1018         UseOptimizedLibcall = true;
1019       break;
1020     }
1021 
1022     CallArgList Args;
1023     if (!UseOptimizedLibcall) {
1024       // For non-optimized library calls, the size is the first parameter
1025       Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1026                getContext().getSizeType());
1027     }
1028     // Atomic address is the first or second parameter
1029     // The OpenCL atomic library functions only accept pointer arguments to
1030     // generic address space.
1031     auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1032       if (!E->isOpenCL())
1033         return V;
1034       auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1035       if (AS == LangAS::opencl_generic)
1036         return V;
1037       auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1038       auto T = V->getType();
1039       auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
1040 
1041       return getTargetHooks().performAddrSpaceCast(
1042           *this, V, AS, LangAS::opencl_generic, DestType, false);
1043     };
1044 
1045     Args.add(RValue::get(CastToGenericAddrSpace(
1046                  EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
1047              getContext().VoidPtrTy);
1048 
1049     std::string LibCallName;
1050     QualType LoweredMemTy =
1051       MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
1052     QualType RetTy;
1053     bool HaveRetTy = false;
1054     llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1055     bool PostOpMinMax = false;
1056     switch (E->getOp()) {
1057     case AtomicExpr::AO__c11_atomic_init:
1058     case AtomicExpr::AO__opencl_atomic_init:
1059       llvm_unreachable("Already handled!");
1060 
1061     // There is only one libcall for compare an exchange, because there is no
1062     // optimisation benefit possible from a libcall version of a weak compare
1063     // and exchange.
1064     // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1065     //                                void *desired, int success, int failure)
1066     // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1067     //                                  int success, int failure)
1068     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1069     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1070     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1071     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1072     case AtomicExpr::AO__atomic_compare_exchange:
1073     case AtomicExpr::AO__atomic_compare_exchange_n:
1074       LibCallName = "__atomic_compare_exchange";
1075       RetTy = getContext().BoolTy;
1076       HaveRetTy = true;
1077       Args.add(
1078           RValue::get(CastToGenericAddrSpace(
1079               EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1080           getContext().VoidPtrTy);
1081       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1082                         MemTy, E->getExprLoc(), TInfo.Width);
1083       Args.add(RValue::get(Order), getContext().IntTy);
1084       Order = OrderFail;
1085       break;
1086     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1087     //                        int order)
1088     // T __atomic_exchange_N(T *mem, T val, int order)
1089     case AtomicExpr::AO__c11_atomic_exchange:
1090     case AtomicExpr::AO__opencl_atomic_exchange:
1091     case AtomicExpr::AO__atomic_exchange_n:
1092     case AtomicExpr::AO__atomic_exchange:
1093       LibCallName = "__atomic_exchange";
1094       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1095                         MemTy, E->getExprLoc(), TInfo.Width);
1096       break;
1097     // void __atomic_store(size_t size, void *mem, void *val, int order)
1098     // void __atomic_store_N(T *mem, T val, int order)
1099     case AtomicExpr::AO__c11_atomic_store:
1100     case AtomicExpr::AO__opencl_atomic_store:
1101     case AtomicExpr::AO__atomic_store:
1102     case AtomicExpr::AO__atomic_store_n:
1103       LibCallName = "__atomic_store";
1104       RetTy = getContext().VoidTy;
1105       HaveRetTy = true;
1106       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1107                         MemTy, E->getExprLoc(), TInfo.Width);
1108       break;
1109     // void __atomic_load(size_t size, void *mem, void *return, int order)
1110     // T __atomic_load_N(T *mem, int order)
1111     case AtomicExpr::AO__c11_atomic_load:
1112     case AtomicExpr::AO__opencl_atomic_load:
1113     case AtomicExpr::AO__atomic_load:
1114     case AtomicExpr::AO__atomic_load_n:
1115       LibCallName = "__atomic_load";
1116       break;
1117     // T __atomic_add_fetch_N(T *mem, T val, int order)
1118     // T __atomic_fetch_add_N(T *mem, T val, int order)
1119     case AtomicExpr::AO__atomic_add_fetch:
1120       PostOp = llvm::Instruction::Add;
1121       LLVM_FALLTHROUGH;
1122     case AtomicExpr::AO__c11_atomic_fetch_add:
1123     case AtomicExpr::AO__opencl_atomic_fetch_add:
1124     case AtomicExpr::AO__atomic_fetch_add:
1125       LibCallName = "__atomic_fetch_add";
1126       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1127                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1128       break;
1129     // T __atomic_and_fetch_N(T *mem, T val, int order)
1130     // T __atomic_fetch_and_N(T *mem, T val, int order)
1131     case AtomicExpr::AO__atomic_and_fetch:
1132       PostOp = llvm::Instruction::And;
1133       LLVM_FALLTHROUGH;
1134     case AtomicExpr::AO__c11_atomic_fetch_and:
1135     case AtomicExpr::AO__opencl_atomic_fetch_and:
1136     case AtomicExpr::AO__atomic_fetch_and:
1137       LibCallName = "__atomic_fetch_and";
1138       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1139                         MemTy, E->getExprLoc(), TInfo.Width);
1140       break;
1141     // T __atomic_or_fetch_N(T *mem, T val, int order)
1142     // T __atomic_fetch_or_N(T *mem, T val, int order)
1143     case AtomicExpr::AO__atomic_or_fetch:
1144       PostOp = llvm::Instruction::Or;
1145       LLVM_FALLTHROUGH;
1146     case AtomicExpr::AO__c11_atomic_fetch_or:
1147     case AtomicExpr::AO__opencl_atomic_fetch_or:
1148     case AtomicExpr::AO__atomic_fetch_or:
1149       LibCallName = "__atomic_fetch_or";
1150       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1151                         MemTy, E->getExprLoc(), TInfo.Width);
1152       break;
1153     // T __atomic_sub_fetch_N(T *mem, T val, int order)
1154     // T __atomic_fetch_sub_N(T *mem, T val, int order)
1155     case AtomicExpr::AO__atomic_sub_fetch:
1156       PostOp = llvm::Instruction::Sub;
1157       LLVM_FALLTHROUGH;
1158     case AtomicExpr::AO__c11_atomic_fetch_sub:
1159     case AtomicExpr::AO__opencl_atomic_fetch_sub:
1160     case AtomicExpr::AO__atomic_fetch_sub:
1161       LibCallName = "__atomic_fetch_sub";
1162       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1163                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1164       break;
1165     // T __atomic_xor_fetch_N(T *mem, T val, int order)
1166     // T __atomic_fetch_xor_N(T *mem, T val, int order)
1167     case AtomicExpr::AO__atomic_xor_fetch:
1168       PostOp = llvm::Instruction::Xor;
1169       LLVM_FALLTHROUGH;
1170     case AtomicExpr::AO__c11_atomic_fetch_xor:
1171     case AtomicExpr::AO__opencl_atomic_fetch_xor:
1172     case AtomicExpr::AO__atomic_fetch_xor:
1173       LibCallName = "__atomic_fetch_xor";
1174       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1175                         MemTy, E->getExprLoc(), TInfo.Width);
1176       break;
1177     case AtomicExpr::AO__atomic_min_fetch:
1178       PostOpMinMax = true;
1179       LLVM_FALLTHROUGH;
1180     case AtomicExpr::AO__c11_atomic_fetch_min:
1181     case AtomicExpr::AO__atomic_fetch_min:
1182     case AtomicExpr::AO__opencl_atomic_fetch_min:
1183       LibCallName = E->getValueType()->isSignedIntegerType()
1184                         ? "__atomic_fetch_min"
1185                         : "__atomic_fetch_umin";
1186       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1187                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1188       break;
1189     case AtomicExpr::AO__atomic_max_fetch:
1190       PostOpMinMax = true;
1191       LLVM_FALLTHROUGH;
1192     case AtomicExpr::AO__c11_atomic_fetch_max:
1193     case AtomicExpr::AO__atomic_fetch_max:
1194     case AtomicExpr::AO__opencl_atomic_fetch_max:
1195       LibCallName = E->getValueType()->isSignedIntegerType()
1196                         ? "__atomic_fetch_max"
1197                         : "__atomic_fetch_umax";
1198       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1199                         LoweredMemTy, E->getExprLoc(), TInfo.Width);
1200       break;
1201     // T __atomic_nand_fetch_N(T *mem, T val, int order)
1202     // T __atomic_fetch_nand_N(T *mem, T val, int order)
1203     case AtomicExpr::AO__atomic_nand_fetch:
1204       PostOp = llvm::Instruction::And; // the NOT is special cased below
1205       LLVM_FALLTHROUGH;
1206     case AtomicExpr::AO__atomic_fetch_nand:
1207       LibCallName = "__atomic_fetch_nand";
1208       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1209                         MemTy, E->getExprLoc(), TInfo.Width);
1210       break;
1211     }
1212 
1213     if (E->isOpenCL()) {
1214       LibCallName = std::string("__opencl") +
1215           StringRef(LibCallName).drop_front(1).str();
1216 
1217     }
1218     // Optimized functions have the size in their name.
1219     if (UseOptimizedLibcall)
1220       LibCallName += "_" + llvm::utostr(Size);
1221     // By default, assume we return a value of the atomic type.
1222     if (!HaveRetTy) {
1223       if (UseOptimizedLibcall) {
1224         // Value is returned directly.
1225         // The function returns an appropriately sized integer type.
1226         RetTy = getContext().getIntTypeForBitwidth(
1227             getContext().toBits(TInfo.Width), /*Signed=*/false);
1228       } else {
1229         // Value is returned through parameter before the order.
1230         RetTy = getContext().VoidTy;
1231         Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1232                  getContext().VoidPtrTy);
1233       }
1234     }
1235     // order is always the last parameter
1236     Args.add(RValue::get(Order),
1237              getContext().IntTy);
1238     if (E->isOpenCL())
1239       Args.add(RValue::get(Scope), getContext().IntTy);
1240 
1241     // PostOp is only needed for the atomic_*_fetch operations, and
1242     // thus is only needed for and implemented in the
1243     // UseOptimizedLibcall codepath.
1244     assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1245 
1246     RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1247     // The value is returned directly from the libcall.
1248     if (E->isCmpXChg())
1249       return Res;
1250 
1251     // The value is returned directly for optimized libcalls but the expr
1252     // provided an out-param.
1253     if (UseOptimizedLibcall && Res.getScalarVal()) {
1254       llvm::Value *ResVal = Res.getScalarVal();
1255       if (PostOpMinMax) {
1256         llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1257         ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1258                                       E->getValueType()->isSignedIntegerType(),
1259                                       ResVal, LoadVal1);
1260       } else if (PostOp) {
1261         llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1262         ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1263       }
1264       if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1265         ResVal = Builder.CreateNot(ResVal);
1266 
1267       Builder.CreateStore(
1268           ResVal,
1269           Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1270     }
1271 
1272     if (RValTy->isVoidType())
1273       return RValue::get(nullptr);
1274 
1275     return convertTempToRValue(
1276         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1277         RValTy, E->getExprLoc());
1278   }
1279 
1280   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1281                  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1282                  E->getOp() == AtomicExpr::AO__atomic_store ||
1283                  E->getOp() == AtomicExpr::AO__atomic_store_n;
1284   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1285                 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1286                 E->getOp() == AtomicExpr::AO__atomic_load ||
1287                 E->getOp() == AtomicExpr::AO__atomic_load_n;
1288 
1289   if (isa<llvm::ConstantInt>(Order)) {
1290     auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1291     // We should not ever get to a case where the ordering isn't a valid C ABI
1292     // value, but it's hard to enforce that in general.
1293     if (llvm::isValidAtomicOrderingCABI(ord))
1294       switch ((llvm::AtomicOrderingCABI)ord) {
1295       case llvm::AtomicOrderingCABI::relaxed:
1296         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1297                      llvm::AtomicOrdering::Monotonic, Scope);
1298         break;
1299       case llvm::AtomicOrderingCABI::consume:
1300       case llvm::AtomicOrderingCABI::acquire:
1301         if (IsStore)
1302           break; // Avoid crashing on code with undefined behavior
1303         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1304                      llvm::AtomicOrdering::Acquire, Scope);
1305         break;
1306       case llvm::AtomicOrderingCABI::release:
1307         if (IsLoad)
1308           break; // Avoid crashing on code with undefined behavior
1309         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1310                      llvm::AtomicOrdering::Release, Scope);
1311         break;
1312       case llvm::AtomicOrderingCABI::acq_rel:
1313         if (IsLoad || IsStore)
1314           break; // Avoid crashing on code with undefined behavior
1315         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1316                      llvm::AtomicOrdering::AcquireRelease, Scope);
1317         break;
1318       case llvm::AtomicOrderingCABI::seq_cst:
1319         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1320                      llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1321         break;
1322       }
1323     if (RValTy->isVoidType())
1324       return RValue::get(nullptr);
1325 
1326     return convertTempToRValue(
1327         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1328                                         Dest.getAddressSpace())),
1329         RValTy, E->getExprLoc());
1330   }
1331 
1332   // Long case, when Order isn't obviously constant.
1333 
1334   // Create all the relevant BB's
1335   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1336                    *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1337                    *SeqCstBB = nullptr;
1338   MonotonicBB = createBasicBlock("monotonic", CurFn);
1339   if (!IsStore)
1340     AcquireBB = createBasicBlock("acquire", CurFn);
1341   if (!IsLoad)
1342     ReleaseBB = createBasicBlock("release", CurFn);
1343   if (!IsLoad && !IsStore)
1344     AcqRelBB = createBasicBlock("acqrel", CurFn);
1345   SeqCstBB = createBasicBlock("seqcst", CurFn);
1346   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1347 
1348   // Create the switch for the split
1349   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1350   // doesn't matter unless someone is crazy enough to use something that
1351   // doesn't fold to a constant for the ordering.
1352   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1353   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1354 
1355   // Emit all the different atomics
1356   Builder.SetInsertPoint(MonotonicBB);
1357   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1358                llvm::AtomicOrdering::Monotonic, Scope);
1359   Builder.CreateBr(ContBB);
1360   if (!IsStore) {
1361     Builder.SetInsertPoint(AcquireBB);
1362     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1363                  llvm::AtomicOrdering::Acquire, Scope);
1364     Builder.CreateBr(ContBB);
1365     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1366                 AcquireBB);
1367     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1368                 AcquireBB);
1369   }
1370   if (!IsLoad) {
1371     Builder.SetInsertPoint(ReleaseBB);
1372     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1373                  llvm::AtomicOrdering::Release, Scope);
1374     Builder.CreateBr(ContBB);
1375     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1376                 ReleaseBB);
1377   }
1378   if (!IsLoad && !IsStore) {
1379     Builder.SetInsertPoint(AcqRelBB);
1380     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1381                  llvm::AtomicOrdering::AcquireRelease, Scope);
1382     Builder.CreateBr(ContBB);
1383     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1384                 AcqRelBB);
1385   }
1386   Builder.SetInsertPoint(SeqCstBB);
1387   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1388                llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1389   Builder.CreateBr(ContBB);
1390   SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1391               SeqCstBB);
1392 
1393   // Cleanup and return
1394   Builder.SetInsertPoint(ContBB);
1395   if (RValTy->isVoidType())
1396     return RValue::get(nullptr);
1397 
1398   assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1399   return convertTempToRValue(
1400       Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1401                                       Dest.getAddressSpace())),
1402       RValTy, E->getExprLoc());
1403 }
1404 
1405 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1406   unsigned addrspace =
1407     cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1408   llvm::IntegerType *ty =
1409     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1410   return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1411 }
1412 
1413 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1414   llvm::Type *Ty = Addr.getElementType();
1415   uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1416   if (SourceSizeInBits != AtomicSizeInBits) {
1417     Address Tmp = CreateTempAlloca();
1418     CGF.Builder.CreateMemCpy(Tmp, Addr,
1419                              std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1420     Addr = Tmp;
1421   }
1422 
1423   return emitCastToAtomicIntPointer(Addr);
1424 }
1425 
1426 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1427                                              AggValueSlot resultSlot,
1428                                              SourceLocation loc,
1429                                              bool asValue) const {
1430   if (LVal.isSimple()) {
1431     if (EvaluationKind == TEK_Aggregate)
1432       return resultSlot.asRValue();
1433 
1434     // Drill into the padding structure if we have one.
1435     if (hasPadding())
1436       addr = CGF.Builder.CreateStructGEP(addr, 0);
1437 
1438     // Otherwise, just convert the temporary to an r-value using the
1439     // normal conversion routine.
1440     return CGF.convertTempToRValue(addr, getValueType(), loc);
1441   }
1442   if (!asValue)
1443     // Get RValue from temp memory as atomic for non-simple lvalues
1444     return RValue::get(CGF.Builder.CreateLoad(addr));
1445   if (LVal.isBitField())
1446     return CGF.EmitLoadOfBitfieldLValue(
1447         LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1448                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1449   if (LVal.isVectorElt())
1450     return CGF.EmitLoadOfLValue(
1451         LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1452                               LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1453   assert(LVal.isExtVectorElt());
1454   return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1455       addr, LVal.getExtVectorElts(), LVal.getType(),
1456       LVal.getBaseInfo(), TBAAAccessInfo()));
1457 }
1458 
1459 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1460                                              AggValueSlot ResultSlot,
1461                                              SourceLocation Loc,
1462                                              bool AsValue) const {
1463   // Try not to in some easy cases.
1464   assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1465   if (getEvaluationKind() == TEK_Scalar &&
1466       (((!LVal.isBitField() ||
1467          LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1468         !hasPadding()) ||
1469        !AsValue)) {
1470     auto *ValTy = AsValue
1471                       ? CGF.ConvertTypeForMem(ValueTy)
1472                       : getAtomicAddress().getType()->getPointerElementType();
1473     if (ValTy->isIntegerTy()) {
1474       assert(IntVal->getType() == ValTy && "Different integer types.");
1475       return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1476     } else if (ValTy->isPointerTy())
1477       return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1478     else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1479       return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1480   }
1481 
1482   // Create a temporary.  This needs to be big enough to hold the
1483   // atomic integer.
1484   Address Temp = Address::invalid();
1485   bool TempIsVolatile = false;
1486   if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1487     assert(!ResultSlot.isIgnored());
1488     Temp = ResultSlot.getAddress();
1489     TempIsVolatile = ResultSlot.isVolatile();
1490   } else {
1491     Temp = CreateTempAlloca();
1492   }
1493 
1494   // Slam the integer into the temporary.
1495   Address CastTemp = emitCastToAtomicIntPointer(Temp);
1496   CGF.Builder.CreateStore(IntVal, CastTemp)
1497       ->setVolatile(TempIsVolatile);
1498 
1499   return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1500 }
1501 
1502 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1503                                        llvm::AtomicOrdering AO, bool) {
1504   // void __atomic_load(size_t size, void *mem, void *return, int order);
1505   CallArgList Args;
1506   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1507   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1508            CGF.getContext().VoidPtrTy);
1509   Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1510            CGF.getContext().VoidPtrTy);
1511   Args.add(
1512       RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1513       CGF.getContext().IntTy);
1514   emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1515 }
1516 
1517 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1518                                           bool IsVolatile) {
1519   // Okay, we're doing this natively.
1520   Address Addr = getAtomicAddressAsAtomicIntPointer();
1521   llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1522   Load->setAtomic(AO);
1523 
1524   // Other decoration.
1525   if (IsVolatile)
1526     Load->setVolatile(true);
1527   CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1528   return Load;
1529 }
1530 
1531 /// An LValue is a candidate for having its loads and stores be made atomic if
1532 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1533 /// performing such an operation can be performed without a libcall.
1534 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1535   if (!CGM.getCodeGenOpts().MSVolatile) return false;
1536   AtomicInfo AI(*this, LV);
1537   bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1538   // An atomic is inline if we don't need to use a libcall.
1539   bool AtomicIsInline = !AI.shouldUseLibcall();
1540   // MSVC doesn't seem to do this for types wider than a pointer.
1541   if (getContext().getTypeSize(LV.getType()) >
1542       getContext().getTypeSize(getContext().getIntPtrType()))
1543     return false;
1544   return IsVolatile && AtomicIsInline;
1545 }
1546 
1547 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1548                                        AggValueSlot Slot) {
1549   llvm::AtomicOrdering AO;
1550   bool IsVolatile = LV.isVolatileQualified();
1551   if (LV.getType()->isAtomicType()) {
1552     AO = llvm::AtomicOrdering::SequentiallyConsistent;
1553   } else {
1554     AO = llvm::AtomicOrdering::Acquire;
1555     IsVolatile = true;
1556   }
1557   return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1558 }
1559 
1560 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1561                                   bool AsValue, llvm::AtomicOrdering AO,
1562                                   bool IsVolatile) {
1563   // Check whether we should use a library call.
1564   if (shouldUseLibcall()) {
1565     Address TempAddr = Address::invalid();
1566     if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1567       assert(getEvaluationKind() == TEK_Aggregate);
1568       TempAddr = ResultSlot.getAddress();
1569     } else
1570       TempAddr = CreateTempAlloca();
1571 
1572     EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1573 
1574     // Okay, turn that back into the original value or whole atomic (for
1575     // non-simple lvalues) type.
1576     return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1577   }
1578 
1579   // Okay, we're doing this natively.
1580   auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1581 
1582   // If we're ignoring an aggregate return, don't do anything.
1583   if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1584     return RValue::getAggregate(Address::invalid(), false);
1585 
1586   // Okay, turn that back into the original value or atomic (for non-simple
1587   // lvalues) type.
1588   return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1589 }
1590 
1591 /// Emit a load from an l-value of atomic type.  Note that the r-value
1592 /// we produce is an r-value of the atomic *value* type.
1593 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1594                                        llvm::AtomicOrdering AO, bool IsVolatile,
1595                                        AggValueSlot resultSlot) {
1596   AtomicInfo Atomics(*this, src);
1597   return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1598                                 IsVolatile);
1599 }
1600 
1601 /// Copy an r-value into memory as part of storing to an atomic type.
1602 /// This needs to create a bit-pattern suitable for atomic operations.
1603 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1604   assert(LVal.isSimple());
1605   // If we have an r-value, the rvalue should be of the atomic type,
1606   // which means that the caller is responsible for having zeroed
1607   // any padding.  Just do an aggregate copy of that type.
1608   if (rvalue.isAggregate()) {
1609     LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1610     LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1611                                     getAtomicType());
1612     bool IsVolatile = rvalue.isVolatileQualified() ||
1613                       LVal.isVolatileQualified();
1614     CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1615                           AggValueSlot::DoesNotOverlap, IsVolatile);
1616     return;
1617   }
1618 
1619   // Okay, otherwise we're copying stuff.
1620 
1621   // Zero out the buffer if necessary.
1622   emitMemSetZeroIfNecessary();
1623 
1624   // Drill past the padding if present.
1625   LValue TempLVal = projectValue();
1626 
1627   // Okay, store the rvalue in.
1628   if (rvalue.isScalar()) {
1629     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1630   } else {
1631     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1632   }
1633 }
1634 
1635 
1636 /// Materialize an r-value into memory for the purposes of storing it
1637 /// to an atomic type.
1638 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1639   // Aggregate r-values are already in memory, and EmitAtomicStore
1640   // requires them to be values of the atomic type.
1641   if (rvalue.isAggregate())
1642     return rvalue.getAggregateAddress();
1643 
1644   // Otherwise, make a temporary and materialize into it.
1645   LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1646   AtomicInfo Atomics(CGF, TempLV);
1647   Atomics.emitCopyIntoMemory(rvalue);
1648   return TempLV.getAddress(CGF);
1649 }
1650 
1651 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1652   // If we've got a scalar value of the right size, try to avoid going
1653   // through memory.
1654   if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1655     llvm::Value *Value = RVal.getScalarVal();
1656     if (isa<llvm::IntegerType>(Value->getType()))
1657       return CGF.EmitToMemory(Value, ValueTy);
1658     else {
1659       llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1660           CGF.getLLVMContext(),
1661           LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1662       if (isa<llvm::PointerType>(Value->getType()))
1663         return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1664       else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1665         return CGF.Builder.CreateBitCast(Value, InputIntTy);
1666     }
1667   }
1668   // Otherwise, we need to go through memory.
1669   // Put the r-value in memory.
1670   Address Addr = materializeRValue(RVal);
1671 
1672   // Cast the temporary to the atomic int type and pull a value out.
1673   Addr = emitCastToAtomicIntPointer(Addr);
1674   return CGF.Builder.CreateLoad(Addr);
1675 }
1676 
1677 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1678     llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1679     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1680   // Do the atomic store.
1681   Address Addr = getAtomicAddressAsAtomicIntPointer();
1682   auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1683                                                ExpectedVal, DesiredVal,
1684                                                Success, Failure);
1685   // Other decoration.
1686   Inst->setVolatile(LVal.isVolatileQualified());
1687   Inst->setWeak(IsWeak);
1688 
1689   // Okay, turn that back into the original value type.
1690   auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1691   auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1692   return std::make_pair(PreviousVal, SuccessFailureVal);
1693 }
1694 
1695 llvm::Value *
1696 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1697                                              llvm::Value *DesiredAddr,
1698                                              llvm::AtomicOrdering Success,
1699                                              llvm::AtomicOrdering Failure) {
1700   // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1701   // void *desired, int success, int failure);
1702   CallArgList Args;
1703   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1704   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1705            CGF.getContext().VoidPtrTy);
1706   Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1707            CGF.getContext().VoidPtrTy);
1708   Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1709            CGF.getContext().VoidPtrTy);
1710   Args.add(RValue::get(
1711                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1712            CGF.getContext().IntTy);
1713   Args.add(RValue::get(
1714                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1715            CGF.getContext().IntTy);
1716   auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1717                                               CGF.getContext().BoolTy, Args);
1718 
1719   return SuccessFailureRVal.getScalarVal();
1720 }
1721 
1722 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1723     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1724     llvm::AtomicOrdering Failure, bool IsWeak) {
1725   if (isStrongerThan(Failure, Success))
1726     // Don't assert on undefined behavior "failure argument shall be no stronger
1727     // than the success argument".
1728     Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1729 
1730   // Check whether we should use a library call.
1731   if (shouldUseLibcall()) {
1732     // Produce a source address.
1733     Address ExpectedAddr = materializeRValue(Expected);
1734     Address DesiredAddr = materializeRValue(Desired);
1735     auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1736                                                  DesiredAddr.getPointer(),
1737                                                  Success, Failure);
1738     return std::make_pair(
1739         convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1740                                   SourceLocation(), /*AsValue=*/false),
1741         Res);
1742   }
1743 
1744   // If we've got a scalar value of the right size, try to avoid going
1745   // through memory.
1746   auto *ExpectedVal = convertRValueToInt(Expected);
1747   auto *DesiredVal = convertRValueToInt(Desired);
1748   auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1749                                          Failure, IsWeak);
1750   return std::make_pair(
1751       ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1752                                 SourceLocation(), /*AsValue=*/false),
1753       Res.second);
1754 }
1755 
1756 static void
1757 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1758                       const llvm::function_ref<RValue(RValue)> &UpdateOp,
1759                       Address DesiredAddr) {
1760   RValue UpRVal;
1761   LValue AtomicLVal = Atomics.getAtomicLValue();
1762   LValue DesiredLVal;
1763   if (AtomicLVal.isSimple()) {
1764     UpRVal = OldRVal;
1765     DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1766   } else {
1767     // Build new lvalue for temp address.
1768     Address Ptr = Atomics.materializeRValue(OldRVal);
1769     LValue UpdateLVal;
1770     if (AtomicLVal.isBitField()) {
1771       UpdateLVal =
1772           LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1773                                AtomicLVal.getType(),
1774                                AtomicLVal.getBaseInfo(),
1775                                AtomicLVal.getTBAAInfo());
1776       DesiredLVal =
1777           LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1778                                AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1779                                AtomicLVal.getTBAAInfo());
1780     } else if (AtomicLVal.isVectorElt()) {
1781       UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1782                                          AtomicLVal.getType(),
1783                                          AtomicLVal.getBaseInfo(),
1784                                          AtomicLVal.getTBAAInfo());
1785       DesiredLVal = LValue::MakeVectorElt(
1786           DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1787           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1788     } else {
1789       assert(AtomicLVal.isExtVectorElt());
1790       UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1791                                             AtomicLVal.getType(),
1792                                             AtomicLVal.getBaseInfo(),
1793                                             AtomicLVal.getTBAAInfo());
1794       DesiredLVal = LValue::MakeExtVectorElt(
1795           DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1796           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1797     }
1798     UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1799   }
1800   // Store new value in the corresponding memory area.
1801   RValue NewRVal = UpdateOp(UpRVal);
1802   if (NewRVal.isScalar()) {
1803     CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1804   } else {
1805     assert(NewRVal.isComplex());
1806     CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1807                            /*isInit=*/false);
1808   }
1809 }
1810 
1811 void AtomicInfo::EmitAtomicUpdateLibcall(
1812     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1813     bool IsVolatile) {
1814   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1815 
1816   Address ExpectedAddr = CreateTempAlloca();
1817 
1818   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1819   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1820   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1821   CGF.EmitBlock(ContBB);
1822   Address DesiredAddr = CreateTempAlloca();
1823   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1824       requiresMemSetZero(getAtomicAddress().getElementType())) {
1825     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1826     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1827   }
1828   auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1829                                            AggValueSlot::ignored(),
1830                                            SourceLocation(), /*AsValue=*/false);
1831   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1832   auto *Res =
1833       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1834                                        DesiredAddr.getPointer(),
1835                                        AO, Failure);
1836   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1837   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1838 }
1839 
1840 void AtomicInfo::EmitAtomicUpdateOp(
1841     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1842     bool IsVolatile) {
1843   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1844 
1845   // Do the atomic load.
1846   auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1847   // For non-simple lvalues perform compare-and-swap procedure.
1848   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1849   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1850   auto *CurBB = CGF.Builder.GetInsertBlock();
1851   CGF.EmitBlock(ContBB);
1852   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1853                                              /*NumReservedValues=*/2);
1854   PHI->addIncoming(OldVal, CurBB);
1855   Address NewAtomicAddr = CreateTempAlloca();
1856   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1857   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1858       requiresMemSetZero(getAtomicAddress().getElementType())) {
1859     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1860   }
1861   auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1862                                            SourceLocation(), /*AsValue=*/false);
1863   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1864   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1865   // Try to write new value using cmpxchg operation.
1866   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1867   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1868   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1869   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1870 }
1871 
1872 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1873                                   RValue UpdateRVal, Address DesiredAddr) {
1874   LValue AtomicLVal = Atomics.getAtomicLValue();
1875   LValue DesiredLVal;
1876   // Build new lvalue for temp address.
1877   if (AtomicLVal.isBitField()) {
1878     DesiredLVal =
1879         LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1880                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1881                              AtomicLVal.getTBAAInfo());
1882   } else if (AtomicLVal.isVectorElt()) {
1883     DesiredLVal =
1884         LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1885                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1886                               AtomicLVal.getTBAAInfo());
1887   } else {
1888     assert(AtomicLVal.isExtVectorElt());
1889     DesiredLVal = LValue::MakeExtVectorElt(
1890         DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1891         AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1892   }
1893   // Store new value in the corresponding memory area.
1894   assert(UpdateRVal.isScalar());
1895   CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1896 }
1897 
1898 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1899                                          RValue UpdateRVal, bool IsVolatile) {
1900   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1901 
1902   Address ExpectedAddr = CreateTempAlloca();
1903 
1904   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1905   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1906   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1907   CGF.EmitBlock(ContBB);
1908   Address DesiredAddr = CreateTempAlloca();
1909   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1910       requiresMemSetZero(getAtomicAddress().getElementType())) {
1911     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1912     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1913   }
1914   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1915   auto *Res =
1916       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1917                                        DesiredAddr.getPointer(),
1918                                        AO, Failure);
1919   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1920   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1921 }
1922 
1923 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1924                                     bool IsVolatile) {
1925   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1926 
1927   // Do the atomic load.
1928   auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1929   // For non-simple lvalues perform compare-and-swap procedure.
1930   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1931   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1932   auto *CurBB = CGF.Builder.GetInsertBlock();
1933   CGF.EmitBlock(ContBB);
1934   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1935                                              /*NumReservedValues=*/2);
1936   PHI->addIncoming(OldVal, CurBB);
1937   Address NewAtomicAddr = CreateTempAlloca();
1938   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1939   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1940       requiresMemSetZero(getAtomicAddress().getElementType())) {
1941     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1942   }
1943   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1944   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1945   // Try to write new value using cmpxchg operation.
1946   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1947   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1948   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1949   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1950 }
1951 
1952 void AtomicInfo::EmitAtomicUpdate(
1953     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1954     bool IsVolatile) {
1955   if (shouldUseLibcall()) {
1956     EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1957   } else {
1958     EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1959   }
1960 }
1961 
1962 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1963                                   bool IsVolatile) {
1964   if (shouldUseLibcall()) {
1965     EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1966   } else {
1967     EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1968   }
1969 }
1970 
1971 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1972                                       bool isInit) {
1973   bool IsVolatile = lvalue.isVolatileQualified();
1974   llvm::AtomicOrdering AO;
1975   if (lvalue.getType()->isAtomicType()) {
1976     AO = llvm::AtomicOrdering::SequentiallyConsistent;
1977   } else {
1978     AO = llvm::AtomicOrdering::Release;
1979     IsVolatile = true;
1980   }
1981   return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1982 }
1983 
1984 /// Emit a store to an l-value of atomic type.
1985 ///
1986 /// Note that the r-value is expected to be an r-value *of the atomic
1987 /// type*; this means that for aggregate r-values, it should include
1988 /// storage for any padding that was necessary.
1989 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1990                                       llvm::AtomicOrdering AO, bool IsVolatile,
1991                                       bool isInit) {
1992   // If this is an aggregate r-value, it should agree in type except
1993   // maybe for address-space qualification.
1994   assert(!rvalue.isAggregate() ||
1995          rvalue.getAggregateAddress().getElementType() ==
1996              dest.getAddress(*this).getElementType());
1997 
1998   AtomicInfo atomics(*this, dest);
1999   LValue LVal = atomics.getAtomicLValue();
2000 
2001   // If this is an initialization, just put the value there normally.
2002   if (LVal.isSimple()) {
2003     if (isInit) {
2004       atomics.emitCopyIntoMemory(rvalue);
2005       return;
2006     }
2007 
2008     // Check whether we should use a library call.
2009     if (atomics.shouldUseLibcall()) {
2010       // Produce a source address.
2011       Address srcAddr = atomics.materializeRValue(rvalue);
2012 
2013       // void __atomic_store(size_t size, void *mem, void *val, int order)
2014       CallArgList args;
2015       args.add(RValue::get(atomics.getAtomicSizeValue()),
2016                getContext().getSizeType());
2017       args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
2018                getContext().VoidPtrTy);
2019       args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
2020                getContext().VoidPtrTy);
2021       args.add(
2022           RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2023           getContext().IntTy);
2024       emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2025       return;
2026     }
2027 
2028     // Okay, we're doing this natively.
2029     llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2030 
2031     // Do the atomic store.
2032     Address addr =
2033         atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
2034     intValue = Builder.CreateIntCast(
2035         intValue, addr.getElementType(), /*isSigned=*/false);
2036     llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2037 
2038     if (AO == llvm::AtomicOrdering::Acquire)
2039       AO = llvm::AtomicOrdering::Monotonic;
2040     else if (AO == llvm::AtomicOrdering::AcquireRelease)
2041       AO = llvm::AtomicOrdering::Release;
2042     // Initializations don't need to be atomic.
2043     if (!isInit)
2044       store->setAtomic(AO);
2045 
2046     // Other decoration.
2047     if (IsVolatile)
2048       store->setVolatile(true);
2049     CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2050     return;
2051   }
2052 
2053   // Emit simple atomic update operation.
2054   atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2055 }
2056 
2057 /// Emit a compare-and-exchange op for atomic type.
2058 ///
2059 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2060     LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2061     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2062     AggValueSlot Slot) {
2063   // If this is an aggregate r-value, it should agree in type except
2064   // maybe for address-space qualification.
2065   assert(!Expected.isAggregate() ||
2066          Expected.getAggregateAddress().getElementType() ==
2067              Obj.getAddress(*this).getElementType());
2068   assert(!Desired.isAggregate() ||
2069          Desired.getAggregateAddress().getElementType() ==
2070              Obj.getAddress(*this).getElementType());
2071   AtomicInfo Atomics(*this, Obj);
2072 
2073   return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2074                                            IsWeak);
2075 }
2076 
2077 void CodeGenFunction::EmitAtomicUpdate(
2078     LValue LVal, llvm::AtomicOrdering AO,
2079     const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2080   AtomicInfo Atomics(*this, LVal);
2081   Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2082 }
2083 
2084 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2085   AtomicInfo atomics(*this, dest);
2086 
2087   switch (atomics.getEvaluationKind()) {
2088   case TEK_Scalar: {
2089     llvm::Value *value = EmitScalarExpr(init);
2090     atomics.emitCopyIntoMemory(RValue::get(value));
2091     return;
2092   }
2093 
2094   case TEK_Complex: {
2095     ComplexPairTy value = EmitComplexExpr(init);
2096     atomics.emitCopyIntoMemory(RValue::getComplex(value));
2097     return;
2098   }
2099 
2100   case TEK_Aggregate: {
2101     // Fix up the destination if the initializer isn't an expression
2102     // of atomic type.
2103     bool Zeroed = false;
2104     if (!init->getType()->isAtomicType()) {
2105       Zeroed = atomics.emitMemSetZeroIfNecessary();
2106       dest = atomics.projectValue();
2107     }
2108 
2109     // Evaluate the expression directly into the destination.
2110     AggValueSlot slot = AggValueSlot::forLValue(
2111         dest, *this, AggValueSlot::IsNotDestructed,
2112         AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2113         AggValueSlot::DoesNotOverlap,
2114         Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2115 
2116     EmitAggExpr(init, slot);
2117     return;
2118   }
2119   }
2120   llvm_unreachable("bad evaluation kind");
2121 }
2122