xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGAtomic.cpp (revision 2f513db72b034fd5ef7f080b11be5c711c15186a)
1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the code for emitting atomic operations.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCall.h"
14 #include "CGRecordLayout.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/CodeGen/CGFunctionInfo.h"
20 #include "clang/Frontend/FrontendDiagnostic.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/Intrinsics.h"
24 #include "llvm/IR/Operator.h"
25 
26 using namespace clang;
27 using namespace CodeGen;
28 
29 namespace {
30   class AtomicInfo {
31     CodeGenFunction &CGF;
32     QualType AtomicTy;
33     QualType ValueTy;
34     uint64_t AtomicSizeInBits;
35     uint64_t ValueSizeInBits;
36     CharUnits AtomicAlign;
37     CharUnits ValueAlign;
38     TypeEvaluationKind EvaluationKind;
39     bool UseLibcall;
40     LValue LVal;
41     CGBitFieldInfo BFI;
42   public:
43     AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44         : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45           EvaluationKind(TEK_Scalar), UseLibcall(true) {
46       assert(!lvalue.isGlobalReg());
47       ASTContext &C = CGF.getContext();
48       if (lvalue.isSimple()) {
49         AtomicTy = lvalue.getType();
50         if (auto *ATy = AtomicTy->getAs<AtomicType>())
51           ValueTy = ATy->getValueType();
52         else
53           ValueTy = AtomicTy;
54         EvaluationKind = CGF.getEvaluationKind(ValueTy);
55 
56         uint64_t ValueAlignInBits;
57         uint64_t AtomicAlignInBits;
58         TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59         ValueSizeInBits = ValueTI.Width;
60         ValueAlignInBits = ValueTI.Align;
61 
62         TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63         AtomicSizeInBits = AtomicTI.Width;
64         AtomicAlignInBits = AtomicTI.Align;
65 
66         assert(ValueSizeInBits <= AtomicSizeInBits);
67         assert(ValueAlignInBits <= AtomicAlignInBits);
68 
69         AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70         ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71         if (lvalue.getAlignment().isZero())
72           lvalue.setAlignment(AtomicAlign);
73 
74         LVal = lvalue;
75       } else if (lvalue.isBitField()) {
76         ValueTy = lvalue.getType();
77         ValueSizeInBits = C.getTypeSize(ValueTy);
78         auto &OrigBFI = lvalue.getBitFieldInfo();
79         auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80         AtomicSizeInBits = C.toBits(
81             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82                 .alignTo(lvalue.getAlignment()));
83         auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84         auto OffsetInChars =
85             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86             lvalue.getAlignment();
87         VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88             VoidPtrAddr, OffsetInChars.getQuantity());
89         auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90             VoidPtrAddr,
91             CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92             "atomic_bitfield_base");
93         BFI = OrigBFI;
94         BFI.Offset = Offset;
95         BFI.StorageSize = AtomicSizeInBits;
96         BFI.StorageOffset += OffsetInChars;
97         LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98                                     BFI, lvalue.getType(), lvalue.getBaseInfo(),
99                                     lvalue.getTBAAInfo());
100         AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
101         if (AtomicTy.isNull()) {
102           llvm::APInt Size(
103               /*numBits=*/32,
104               C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105           AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
106                                             /*IndexTypeQuals=*/0);
107         }
108         AtomicAlign = ValueAlign = lvalue.getAlignment();
109       } else if (lvalue.isVectorElt()) {
110         ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
111         ValueSizeInBits = C.getTypeSize(ValueTy);
112         AtomicTy = lvalue.getType();
113         AtomicSizeInBits = C.getTypeSize(AtomicTy);
114         AtomicAlign = ValueAlign = lvalue.getAlignment();
115         LVal = lvalue;
116       } else {
117         assert(lvalue.isExtVectorElt());
118         ValueTy = lvalue.getType();
119         ValueSizeInBits = C.getTypeSize(ValueTy);
120         AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121             lvalue.getType(), lvalue.getExtVectorAddress()
122                                   .getElementType()->getVectorNumElements());
123         AtomicSizeInBits = C.getTypeSize(AtomicTy);
124         AtomicAlign = ValueAlign = lvalue.getAlignment();
125         LVal = lvalue;
126       }
127       UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
128           AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
129     }
130 
131     QualType getAtomicType() const { return AtomicTy; }
132     QualType getValueType() const { return ValueTy; }
133     CharUnits getAtomicAlignment() const { return AtomicAlign; }
134     uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
135     uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
136     TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
137     bool shouldUseLibcall() const { return UseLibcall; }
138     const LValue &getAtomicLValue() const { return LVal; }
139     llvm::Value *getAtomicPointer() const {
140       if (LVal.isSimple())
141         return LVal.getPointer();
142       else if (LVal.isBitField())
143         return LVal.getBitFieldPointer();
144       else if (LVal.isVectorElt())
145         return LVal.getVectorPointer();
146       assert(LVal.isExtVectorElt());
147       return LVal.getExtVectorPointer();
148     }
149     Address getAtomicAddress() const {
150       return Address(getAtomicPointer(), getAtomicAlignment());
151     }
152 
153     Address getAtomicAddressAsAtomicIntPointer() const {
154       return emitCastToAtomicIntPointer(getAtomicAddress());
155     }
156 
157     /// Is the atomic size larger than the underlying value type?
158     ///
159     /// Note that the absence of padding does not mean that atomic
160     /// objects are completely interchangeable with non-atomic
161     /// objects: we might have promoted the alignment of a type
162     /// without making it bigger.
163     bool hasPadding() const {
164       return (ValueSizeInBits != AtomicSizeInBits);
165     }
166 
167     bool emitMemSetZeroIfNecessary() const;
168 
169     llvm::Value *getAtomicSizeValue() const {
170       CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
171       return CGF.CGM.getSize(size);
172     }
173 
174     /// Cast the given pointer to an integer pointer suitable for atomic
175     /// operations if the source.
176     Address emitCastToAtomicIntPointer(Address Addr) const;
177 
178     /// If Addr is compatible with the iN that will be used for an atomic
179     /// operation, bitcast it. Otherwise, create a temporary that is suitable
180     /// and copy the value across.
181     Address convertToAtomicIntPointer(Address Addr) const;
182 
183     /// Turn an atomic-layout object into an r-value.
184     RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
185                                      SourceLocation loc, bool AsValue) const;
186 
187     /// Converts a rvalue to integer value.
188     llvm::Value *convertRValueToInt(RValue RVal) const;
189 
190     RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
191                                      AggValueSlot ResultSlot,
192                                      SourceLocation Loc, bool AsValue) const;
193 
194     /// Copy an atomic r-value into atomic-layout memory.
195     void emitCopyIntoMemory(RValue rvalue) const;
196 
197     /// Project an l-value down to the value field.
198     LValue projectValue() const {
199       assert(LVal.isSimple());
200       Address addr = getAtomicAddress();
201       if (hasPadding())
202         addr = CGF.Builder.CreateStructGEP(addr, 0);
203 
204       return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
205                               LVal.getBaseInfo(), LVal.getTBAAInfo());
206     }
207 
208     /// Emits atomic load.
209     /// \returns Loaded value.
210     RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
211                           bool AsValue, llvm::AtomicOrdering AO,
212                           bool IsVolatile);
213 
214     /// Emits atomic compare-and-exchange sequence.
215     /// \param Expected Expected value.
216     /// \param Desired Desired value.
217     /// \param Success Atomic ordering for success operation.
218     /// \param Failure Atomic ordering for failed operation.
219     /// \param IsWeak true if atomic operation is weak, false otherwise.
220     /// \returns Pair of values: previous value from storage (value type) and
221     /// boolean flag (i1 type) with true if success and false otherwise.
222     std::pair<RValue, llvm::Value *>
223     EmitAtomicCompareExchange(RValue Expected, RValue Desired,
224                               llvm::AtomicOrdering Success =
225                                   llvm::AtomicOrdering::SequentiallyConsistent,
226                               llvm::AtomicOrdering Failure =
227                                   llvm::AtomicOrdering::SequentiallyConsistent,
228                               bool IsWeak = false);
229 
230     /// Emits atomic update.
231     /// \param AO Atomic ordering.
232     /// \param UpdateOp Update operation for the current lvalue.
233     void EmitAtomicUpdate(llvm::AtomicOrdering AO,
234                           const llvm::function_ref<RValue(RValue)> &UpdateOp,
235                           bool IsVolatile);
236     /// Emits atomic update.
237     /// \param AO Atomic ordering.
238     void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
239                           bool IsVolatile);
240 
241     /// Materialize an atomic r-value in atomic-layout memory.
242     Address materializeRValue(RValue rvalue) const;
243 
244     /// Creates temp alloca for intermediate operations on atomic value.
245     Address CreateTempAlloca() const;
246   private:
247     bool requiresMemSetZero(llvm::Type *type) const;
248 
249 
250     /// Emits atomic load as a libcall.
251     void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
252                                llvm::AtomicOrdering AO, bool IsVolatile);
253     /// Emits atomic load as LLVM instruction.
254     llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
255     /// Emits atomic compare-and-exchange op as a libcall.
256     llvm::Value *EmitAtomicCompareExchangeLibcall(
257         llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
258         llvm::AtomicOrdering Success =
259             llvm::AtomicOrdering::SequentiallyConsistent,
260         llvm::AtomicOrdering Failure =
261             llvm::AtomicOrdering::SequentiallyConsistent);
262     /// Emits atomic compare-and-exchange op as LLVM instruction.
263     std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
264         llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
265         llvm::AtomicOrdering Success =
266             llvm::AtomicOrdering::SequentiallyConsistent,
267         llvm::AtomicOrdering Failure =
268             llvm::AtomicOrdering::SequentiallyConsistent,
269         bool IsWeak = false);
270     /// Emit atomic update as libcalls.
271     void
272     EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
273                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
274                             bool IsVolatile);
275     /// Emit atomic update as LLVM instructions.
276     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
277                             const llvm::function_ref<RValue(RValue)> &UpdateOp,
278                             bool IsVolatile);
279     /// Emit atomic update as libcalls.
280     void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
281                                  bool IsVolatile);
282     /// Emit atomic update as LLVM instructions.
283     void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
284                             bool IsVolatile);
285   };
286 }
287 
288 Address AtomicInfo::CreateTempAlloca() const {
289   Address TempAlloca = CGF.CreateMemTemp(
290       (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
291                                                                 : AtomicTy,
292       getAtomicAlignment(),
293       "atomic-temp");
294   // Cast to pointer to value type for bitfields.
295   if (LVal.isBitField())
296     return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
297         TempAlloca, getAtomicAddress().getType());
298   return TempAlloca;
299 }
300 
301 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
302                                 StringRef fnName,
303                                 QualType resultType,
304                                 CallArgList &args) {
305   const CGFunctionInfo &fnInfo =
306     CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
307   llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
308   llvm::FunctionCallee fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
309   auto callee = CGCallee::forDirect(fn);
310   return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
311 }
312 
313 /// Does a store of the given IR type modify the full expected width?
314 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
315                            uint64_t expectedSize) {
316   return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
317 }
318 
319 /// Does the atomic type require memsetting to zero before initialization?
320 ///
321 /// The IR type is provided as a way of making certain queries faster.
322 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
323   // If the atomic type has size padding, we definitely need a memset.
324   if (hasPadding()) return true;
325 
326   // Otherwise, do some simple heuristics to try to avoid it:
327   switch (getEvaluationKind()) {
328   // For scalars and complexes, check whether the store size of the
329   // type uses the full size.
330   case TEK_Scalar:
331     return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
332   case TEK_Complex:
333     return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
334                            AtomicSizeInBits / 2);
335 
336   // Padding in structs has an undefined bit pattern.  User beware.
337   case TEK_Aggregate:
338     return false;
339   }
340   llvm_unreachable("bad evaluation kind");
341 }
342 
343 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
344   assert(LVal.isSimple());
345   llvm::Value *addr = LVal.getPointer();
346   if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
347     return false;
348 
349   CGF.Builder.CreateMemSet(
350       addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
351       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
352       LVal.getAlignment().getQuantity());
353   return true;
354 }
355 
356 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
357                               Address Dest, Address Ptr,
358                               Address Val1, Address Val2,
359                               uint64_t Size,
360                               llvm::AtomicOrdering SuccessOrder,
361                               llvm::AtomicOrdering FailureOrder,
362                               llvm::SyncScope::ID Scope) {
363   // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
364   llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
365   llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
366 
367   llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
368       Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
369       Scope);
370   Pair->setVolatile(E->isVolatile());
371   Pair->setWeak(IsWeak);
372 
373   // Cmp holds the result of the compare-exchange operation: true on success,
374   // false on failure.
375   llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
376   llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
377 
378   // This basic block is used to hold the store instruction if the operation
379   // failed.
380   llvm::BasicBlock *StoreExpectedBB =
381       CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
382 
383   // This basic block is the exit point of the operation, we should end up
384   // here regardless of whether or not the operation succeeded.
385   llvm::BasicBlock *ContinueBB =
386       CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
387 
388   // Update Expected if Expected isn't equal to Old, otherwise branch to the
389   // exit point.
390   CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
391 
392   CGF.Builder.SetInsertPoint(StoreExpectedBB);
393   // Update the memory at Expected with Old's value.
394   CGF.Builder.CreateStore(Old, Val1);
395   // Finally, branch to the exit point.
396   CGF.Builder.CreateBr(ContinueBB);
397 
398   CGF.Builder.SetInsertPoint(ContinueBB);
399   // Update the memory at Dest with Cmp's value.
400   CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
401 }
402 
403 /// Given an ordering required on success, emit all possible cmpxchg
404 /// instructions to cope with the provided (but possibly only dynamically known)
405 /// FailureOrder.
406 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
407                                         bool IsWeak, Address Dest, Address Ptr,
408                                         Address Val1, Address Val2,
409                                         llvm::Value *FailureOrderVal,
410                                         uint64_t Size,
411                                         llvm::AtomicOrdering SuccessOrder,
412                                         llvm::SyncScope::ID Scope) {
413   llvm::AtomicOrdering FailureOrder;
414   if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
415     auto FOS = FO->getSExtValue();
416     if (!llvm::isValidAtomicOrderingCABI(FOS))
417       FailureOrder = llvm::AtomicOrdering::Monotonic;
418     else
419       switch ((llvm::AtomicOrderingCABI)FOS) {
420       case llvm::AtomicOrderingCABI::relaxed:
421       case llvm::AtomicOrderingCABI::release:
422       case llvm::AtomicOrderingCABI::acq_rel:
423         FailureOrder = llvm::AtomicOrdering::Monotonic;
424         break;
425       case llvm::AtomicOrderingCABI::consume:
426       case llvm::AtomicOrderingCABI::acquire:
427         FailureOrder = llvm::AtomicOrdering::Acquire;
428         break;
429       case llvm::AtomicOrderingCABI::seq_cst:
430         FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
431         break;
432       }
433     if (isStrongerThan(FailureOrder, SuccessOrder)) {
434       // Don't assert on undefined behavior "failure argument shall be no
435       // stronger than the success argument".
436       FailureOrder =
437           llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
438     }
439     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
440                       FailureOrder, Scope);
441     return;
442   }
443 
444   // Create all the relevant BB's
445   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
446                    *SeqCstBB = nullptr;
447   MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
448   if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
449       SuccessOrder != llvm::AtomicOrdering::Release)
450     AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
451   if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
452     SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
453 
454   llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
455 
456   llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
457 
458   // Emit all the different atomics
459 
460   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
461   // doesn't matter unless someone is crazy enough to use something that
462   // doesn't fold to a constant for the ordering.
463   CGF.Builder.SetInsertPoint(MonotonicBB);
464   emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
465                     Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
466   CGF.Builder.CreateBr(ContBB);
467 
468   if (AcquireBB) {
469     CGF.Builder.SetInsertPoint(AcquireBB);
470     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
471                       Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
472     CGF.Builder.CreateBr(ContBB);
473     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
474                 AcquireBB);
475     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
476                 AcquireBB);
477   }
478   if (SeqCstBB) {
479     CGF.Builder.SetInsertPoint(SeqCstBB);
480     emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
481                       llvm::AtomicOrdering::SequentiallyConsistent, Scope);
482     CGF.Builder.CreateBr(ContBB);
483     SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
484                 SeqCstBB);
485   }
486 
487   CGF.Builder.SetInsertPoint(ContBB);
488 }
489 
490 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
491                          Address Ptr, Address Val1, Address Val2,
492                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
493                          uint64_t Size, llvm::AtomicOrdering Order,
494                          llvm::SyncScope::ID Scope) {
495   llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
496   llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
497 
498   switch (E->getOp()) {
499   case AtomicExpr::AO__c11_atomic_init:
500   case AtomicExpr::AO__opencl_atomic_init:
501     llvm_unreachable("Already handled!");
502 
503   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
504   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
505     emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
506                                 FailureOrder, Size, Order, Scope);
507     return;
508   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
509   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
510     emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
511                                 FailureOrder, Size, Order, Scope);
512     return;
513   case AtomicExpr::AO__atomic_compare_exchange:
514   case AtomicExpr::AO__atomic_compare_exchange_n: {
515     if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
516       emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
517                                   Val1, Val2, FailureOrder, Size, Order, Scope);
518     } else {
519       // Create all the relevant BB's
520       llvm::BasicBlock *StrongBB =
521           CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
522       llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
523       llvm::BasicBlock *ContBB =
524           CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
525 
526       llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
527       SI->addCase(CGF.Builder.getInt1(false), StrongBB);
528 
529       CGF.Builder.SetInsertPoint(StrongBB);
530       emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
531                                   FailureOrder, Size, Order, Scope);
532       CGF.Builder.CreateBr(ContBB);
533 
534       CGF.Builder.SetInsertPoint(WeakBB);
535       emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
536                                   FailureOrder, Size, Order, Scope);
537       CGF.Builder.CreateBr(ContBB);
538 
539       CGF.Builder.SetInsertPoint(ContBB);
540     }
541     return;
542   }
543   case AtomicExpr::AO__c11_atomic_load:
544   case AtomicExpr::AO__opencl_atomic_load:
545   case AtomicExpr::AO__atomic_load_n:
546   case AtomicExpr::AO__atomic_load: {
547     llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
548     Load->setAtomic(Order, Scope);
549     Load->setVolatile(E->isVolatile());
550     CGF.Builder.CreateStore(Load, Dest);
551     return;
552   }
553 
554   case AtomicExpr::AO__c11_atomic_store:
555   case AtomicExpr::AO__opencl_atomic_store:
556   case AtomicExpr::AO__atomic_store:
557   case AtomicExpr::AO__atomic_store_n: {
558     llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
559     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
560     Store->setAtomic(Order, Scope);
561     Store->setVolatile(E->isVolatile());
562     return;
563   }
564 
565   case AtomicExpr::AO__c11_atomic_exchange:
566   case AtomicExpr::AO__opencl_atomic_exchange:
567   case AtomicExpr::AO__atomic_exchange_n:
568   case AtomicExpr::AO__atomic_exchange:
569     Op = llvm::AtomicRMWInst::Xchg;
570     break;
571 
572   case AtomicExpr::AO__atomic_add_fetch:
573     PostOp = llvm::Instruction::Add;
574     LLVM_FALLTHROUGH;
575   case AtomicExpr::AO__c11_atomic_fetch_add:
576   case AtomicExpr::AO__opencl_atomic_fetch_add:
577   case AtomicExpr::AO__atomic_fetch_add:
578     Op = llvm::AtomicRMWInst::Add;
579     break;
580 
581   case AtomicExpr::AO__atomic_sub_fetch:
582     PostOp = llvm::Instruction::Sub;
583     LLVM_FALLTHROUGH;
584   case AtomicExpr::AO__c11_atomic_fetch_sub:
585   case AtomicExpr::AO__opencl_atomic_fetch_sub:
586   case AtomicExpr::AO__atomic_fetch_sub:
587     Op = llvm::AtomicRMWInst::Sub;
588     break;
589 
590   case AtomicExpr::AO__opencl_atomic_fetch_min:
591   case AtomicExpr::AO__atomic_fetch_min:
592     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
593                                                   : llvm::AtomicRMWInst::UMin;
594     break;
595 
596   case AtomicExpr::AO__opencl_atomic_fetch_max:
597   case AtomicExpr::AO__atomic_fetch_max:
598     Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
599                                                   : llvm::AtomicRMWInst::UMax;
600     break;
601 
602   case AtomicExpr::AO__atomic_and_fetch:
603     PostOp = llvm::Instruction::And;
604     LLVM_FALLTHROUGH;
605   case AtomicExpr::AO__c11_atomic_fetch_and:
606   case AtomicExpr::AO__opencl_atomic_fetch_and:
607   case AtomicExpr::AO__atomic_fetch_and:
608     Op = llvm::AtomicRMWInst::And;
609     break;
610 
611   case AtomicExpr::AO__atomic_or_fetch:
612     PostOp = llvm::Instruction::Or;
613     LLVM_FALLTHROUGH;
614   case AtomicExpr::AO__c11_atomic_fetch_or:
615   case AtomicExpr::AO__opencl_atomic_fetch_or:
616   case AtomicExpr::AO__atomic_fetch_or:
617     Op = llvm::AtomicRMWInst::Or;
618     break;
619 
620   case AtomicExpr::AO__atomic_xor_fetch:
621     PostOp = llvm::Instruction::Xor;
622     LLVM_FALLTHROUGH;
623   case AtomicExpr::AO__c11_atomic_fetch_xor:
624   case AtomicExpr::AO__opencl_atomic_fetch_xor:
625   case AtomicExpr::AO__atomic_fetch_xor:
626     Op = llvm::AtomicRMWInst::Xor;
627     break;
628 
629   case AtomicExpr::AO__atomic_nand_fetch:
630     PostOp = llvm::Instruction::And; // the NOT is special cased below
631     LLVM_FALLTHROUGH;
632   case AtomicExpr::AO__atomic_fetch_nand:
633     Op = llvm::AtomicRMWInst::Nand;
634     break;
635   }
636 
637   llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
638   llvm::AtomicRMWInst *RMWI =
639       CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
640   RMWI->setVolatile(E->isVolatile());
641 
642   // For __atomic_*_fetch operations, perform the operation again to
643   // determine the value which was written.
644   llvm::Value *Result = RMWI;
645   if (PostOp)
646     Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
647   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
648     Result = CGF.Builder.CreateNot(Result);
649   CGF.Builder.CreateStore(Result, Dest);
650 }
651 
652 // This function emits any expression (scalar, complex, or aggregate)
653 // into a temporary alloca.
654 static Address
655 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
656   Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
657   CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
658                        /*Init*/ true);
659   return DeclPtr;
660 }
661 
662 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
663                          Address Ptr, Address Val1, Address Val2,
664                          llvm::Value *IsWeak, llvm::Value *FailureOrder,
665                          uint64_t Size, llvm::AtomicOrdering Order,
666                          llvm::Value *Scope) {
667   auto ScopeModel = Expr->getScopeModel();
668 
669   // LLVM atomic instructions always have synch scope. If clang atomic
670   // expression has no scope operand, use default LLVM synch scope.
671   if (!ScopeModel) {
672     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
673                  Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
674     return;
675   }
676 
677   // Handle constant scope.
678   if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
679     auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
680         CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
681         Order, CGF.CGM.getLLVMContext());
682     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
683                  Order, SCID);
684     return;
685   }
686 
687   // Handle non-constant scope.
688   auto &Builder = CGF.Builder;
689   auto Scopes = ScopeModel->getRuntimeValues();
690   llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
691   for (auto S : Scopes)
692     BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
693 
694   llvm::BasicBlock *ContBB =
695       CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
696 
697   auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
698   // If unsupported synch scope is encountered at run time, assume a fallback
699   // synch scope value.
700   auto FallBack = ScopeModel->getFallBackValue();
701   llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
702   for (auto S : Scopes) {
703     auto *B = BB[S];
704     if (S != FallBack)
705       SI->addCase(Builder.getInt32(S), B);
706 
707     Builder.SetInsertPoint(B);
708     EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
709                  Order,
710                  CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
711                                                          ScopeModel->map(S),
712                                                          Order,
713                                                          CGF.getLLVMContext()));
714     Builder.CreateBr(ContBB);
715   }
716 
717   Builder.SetInsertPoint(ContBB);
718 }
719 
720 static void
721 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
722                   bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
723                   SourceLocation Loc, CharUnits SizeInChars) {
724   if (UseOptimizedLibcall) {
725     // Load value and pass it to the function directly.
726     CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
727     int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
728     ValTy =
729         CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
730     llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
731                                                 SizeInBits)->getPointerTo();
732     Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
733     Val = CGF.EmitLoadOfScalar(Ptr, false,
734                                CGF.getContext().getPointerType(ValTy),
735                                Loc);
736     // Coerce the value into an appropriately sized integer type.
737     Args.add(RValue::get(Val), ValTy);
738   } else {
739     // Non-optimized functions always take a reference.
740     Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
741                          CGF.getContext().VoidPtrTy);
742   }
743 }
744 
745 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
746   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
747   QualType MemTy = AtomicTy;
748   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
749     MemTy = AT->getValueType();
750   llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
751 
752   Address Val1 = Address::invalid();
753   Address Val2 = Address::invalid();
754   Address Dest = Address::invalid();
755   Address Ptr = EmitPointerWithAlignment(E->getPtr());
756 
757   if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
758       E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
759     LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
760     EmitAtomicInit(E->getVal1(), lvalue);
761     return RValue::get(nullptr);
762   }
763 
764   CharUnits sizeChars, alignChars;
765   std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
766   uint64_t Size = sizeChars.getQuantity();
767   unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
768 
769   bool Oversized = getContext().toBits(sizeChars) > MaxInlineWidthInBits;
770   bool Misaligned = (Ptr.getAlignment() % sizeChars) != 0;
771   bool UseLibcall = Misaligned | Oversized;
772 
773   if (UseLibcall) {
774     CGM.getDiags().Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
775         << !Oversized;
776   }
777 
778   llvm::Value *Order = EmitScalarExpr(E->getOrder());
779   llvm::Value *Scope =
780       E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
781 
782   switch (E->getOp()) {
783   case AtomicExpr::AO__c11_atomic_init:
784   case AtomicExpr::AO__opencl_atomic_init:
785     llvm_unreachable("Already handled above with EmitAtomicInit!");
786 
787   case AtomicExpr::AO__c11_atomic_load:
788   case AtomicExpr::AO__opencl_atomic_load:
789   case AtomicExpr::AO__atomic_load_n:
790     break;
791 
792   case AtomicExpr::AO__atomic_load:
793     Dest = EmitPointerWithAlignment(E->getVal1());
794     break;
795 
796   case AtomicExpr::AO__atomic_store:
797     Val1 = EmitPointerWithAlignment(E->getVal1());
798     break;
799 
800   case AtomicExpr::AO__atomic_exchange:
801     Val1 = EmitPointerWithAlignment(E->getVal1());
802     Dest = EmitPointerWithAlignment(E->getVal2());
803     break;
804 
805   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
806   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
807   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
808   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
809   case AtomicExpr::AO__atomic_compare_exchange_n:
810   case AtomicExpr::AO__atomic_compare_exchange:
811     Val1 = EmitPointerWithAlignment(E->getVal1());
812     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
813       Val2 = EmitPointerWithAlignment(E->getVal2());
814     else
815       Val2 = EmitValToTemp(*this, E->getVal2());
816     OrderFail = EmitScalarExpr(E->getOrderFail());
817     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
818         E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
819       IsWeak = EmitScalarExpr(E->getWeak());
820     break;
821 
822   case AtomicExpr::AO__c11_atomic_fetch_add:
823   case AtomicExpr::AO__c11_atomic_fetch_sub:
824   case AtomicExpr::AO__opencl_atomic_fetch_add:
825   case AtomicExpr::AO__opencl_atomic_fetch_sub:
826     if (MemTy->isPointerType()) {
827       // For pointer arithmetic, we're required to do a bit of math:
828       // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
829       // ... but only for the C11 builtins. The GNU builtins expect the
830       // user to multiply by sizeof(T).
831       QualType Val1Ty = E->getVal1()->getType();
832       llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
833       CharUnits PointeeIncAmt =
834           getContext().getTypeSizeInChars(MemTy->getPointeeType());
835       Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
836       auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
837       Val1 = Temp;
838       EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
839       break;
840     }
841       LLVM_FALLTHROUGH;
842   case AtomicExpr::AO__atomic_fetch_add:
843   case AtomicExpr::AO__atomic_fetch_sub:
844   case AtomicExpr::AO__atomic_add_fetch:
845   case AtomicExpr::AO__atomic_sub_fetch:
846   case AtomicExpr::AO__c11_atomic_store:
847   case AtomicExpr::AO__c11_atomic_exchange:
848   case AtomicExpr::AO__opencl_atomic_store:
849   case AtomicExpr::AO__opencl_atomic_exchange:
850   case AtomicExpr::AO__atomic_store_n:
851   case AtomicExpr::AO__atomic_exchange_n:
852   case AtomicExpr::AO__c11_atomic_fetch_and:
853   case AtomicExpr::AO__c11_atomic_fetch_or:
854   case AtomicExpr::AO__c11_atomic_fetch_xor:
855   case AtomicExpr::AO__opencl_atomic_fetch_and:
856   case AtomicExpr::AO__opencl_atomic_fetch_or:
857   case AtomicExpr::AO__opencl_atomic_fetch_xor:
858   case AtomicExpr::AO__opencl_atomic_fetch_min:
859   case AtomicExpr::AO__opencl_atomic_fetch_max:
860   case AtomicExpr::AO__atomic_fetch_and:
861   case AtomicExpr::AO__atomic_fetch_or:
862   case AtomicExpr::AO__atomic_fetch_xor:
863   case AtomicExpr::AO__atomic_fetch_nand:
864   case AtomicExpr::AO__atomic_and_fetch:
865   case AtomicExpr::AO__atomic_or_fetch:
866   case AtomicExpr::AO__atomic_xor_fetch:
867   case AtomicExpr::AO__atomic_nand_fetch:
868   case AtomicExpr::AO__atomic_fetch_min:
869   case AtomicExpr::AO__atomic_fetch_max:
870     Val1 = EmitValToTemp(*this, E->getVal1());
871     break;
872   }
873 
874   QualType RValTy = E->getType().getUnqualifiedType();
875 
876   // The inlined atomics only function on iN types, where N is a power of 2. We
877   // need to make sure (via temporaries if necessary) that all incoming values
878   // are compatible.
879   LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
880   AtomicInfo Atomics(*this, AtomicVal);
881 
882   Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
883   if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
884   if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
885   if (Dest.isValid())
886     Dest = Atomics.emitCastToAtomicIntPointer(Dest);
887   else if (E->isCmpXChg())
888     Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
889   else if (!RValTy->isVoidType())
890     Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
891 
892   // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
893   if (UseLibcall) {
894     bool UseOptimizedLibcall = false;
895     switch (E->getOp()) {
896     case AtomicExpr::AO__c11_atomic_init:
897     case AtomicExpr::AO__opencl_atomic_init:
898       llvm_unreachable("Already handled above with EmitAtomicInit!");
899 
900     case AtomicExpr::AO__c11_atomic_fetch_add:
901     case AtomicExpr::AO__opencl_atomic_fetch_add:
902     case AtomicExpr::AO__atomic_fetch_add:
903     case AtomicExpr::AO__c11_atomic_fetch_and:
904     case AtomicExpr::AO__opencl_atomic_fetch_and:
905     case AtomicExpr::AO__atomic_fetch_and:
906     case AtomicExpr::AO__c11_atomic_fetch_or:
907     case AtomicExpr::AO__opencl_atomic_fetch_or:
908     case AtomicExpr::AO__atomic_fetch_or:
909     case AtomicExpr::AO__atomic_fetch_nand:
910     case AtomicExpr::AO__c11_atomic_fetch_sub:
911     case AtomicExpr::AO__opencl_atomic_fetch_sub:
912     case AtomicExpr::AO__atomic_fetch_sub:
913     case AtomicExpr::AO__c11_atomic_fetch_xor:
914     case AtomicExpr::AO__opencl_atomic_fetch_xor:
915     case AtomicExpr::AO__opencl_atomic_fetch_min:
916     case AtomicExpr::AO__opencl_atomic_fetch_max:
917     case AtomicExpr::AO__atomic_fetch_xor:
918     case AtomicExpr::AO__atomic_add_fetch:
919     case AtomicExpr::AO__atomic_and_fetch:
920     case AtomicExpr::AO__atomic_nand_fetch:
921     case AtomicExpr::AO__atomic_or_fetch:
922     case AtomicExpr::AO__atomic_sub_fetch:
923     case AtomicExpr::AO__atomic_xor_fetch:
924     case AtomicExpr::AO__atomic_fetch_min:
925     case AtomicExpr::AO__atomic_fetch_max:
926       // For these, only library calls for certain sizes exist.
927       UseOptimizedLibcall = true;
928       break;
929 
930     case AtomicExpr::AO__atomic_load:
931     case AtomicExpr::AO__atomic_store:
932     case AtomicExpr::AO__atomic_exchange:
933     case AtomicExpr::AO__atomic_compare_exchange:
934       // Use the generic version if we don't know that the operand will be
935       // suitably aligned for the optimized version.
936       if (Misaligned)
937         break;
938       LLVM_FALLTHROUGH;
939     case AtomicExpr::AO__c11_atomic_load:
940     case AtomicExpr::AO__c11_atomic_store:
941     case AtomicExpr::AO__c11_atomic_exchange:
942     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
943     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
944     case AtomicExpr::AO__opencl_atomic_load:
945     case AtomicExpr::AO__opencl_atomic_store:
946     case AtomicExpr::AO__opencl_atomic_exchange:
947     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
948     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
949     case AtomicExpr::AO__atomic_load_n:
950     case AtomicExpr::AO__atomic_store_n:
951     case AtomicExpr::AO__atomic_exchange_n:
952     case AtomicExpr::AO__atomic_compare_exchange_n:
953       // Only use optimized library calls for sizes for which they exist.
954       // FIXME: Size == 16 optimized library functions exist too.
955       if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
956         UseOptimizedLibcall = true;
957       break;
958     }
959 
960     CallArgList Args;
961     if (!UseOptimizedLibcall) {
962       // For non-optimized library calls, the size is the first parameter
963       Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
964                getContext().getSizeType());
965     }
966     // Atomic address is the first or second parameter
967     // The OpenCL atomic library functions only accept pointer arguments to
968     // generic address space.
969     auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
970       if (!E->isOpenCL())
971         return V;
972       auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace();
973       if (AS == LangAS::opencl_generic)
974         return V;
975       auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
976       auto T = V->getType();
977       auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
978 
979       return getTargetHooks().performAddrSpaceCast(
980           *this, V, AS, LangAS::opencl_generic, DestType, false);
981     };
982 
983     Args.add(RValue::get(CastToGenericAddrSpace(
984                  EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
985              getContext().VoidPtrTy);
986 
987     std::string LibCallName;
988     QualType LoweredMemTy =
989       MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
990     QualType RetTy;
991     bool HaveRetTy = false;
992     llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
993     switch (E->getOp()) {
994     case AtomicExpr::AO__c11_atomic_init:
995     case AtomicExpr::AO__opencl_atomic_init:
996       llvm_unreachable("Already handled!");
997 
998     // There is only one libcall for compare an exchange, because there is no
999     // optimisation benefit possible from a libcall version of a weak compare
1000     // and exchange.
1001     // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1002     //                                void *desired, int success, int failure)
1003     // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1004     //                                  int success, int failure)
1005     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1006     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1007     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1008     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1009     case AtomicExpr::AO__atomic_compare_exchange:
1010     case AtomicExpr::AO__atomic_compare_exchange_n:
1011       LibCallName = "__atomic_compare_exchange";
1012       RetTy = getContext().BoolTy;
1013       HaveRetTy = true;
1014       Args.add(
1015           RValue::get(CastToGenericAddrSpace(
1016               EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1017           getContext().VoidPtrTy);
1018       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1019                         MemTy, E->getExprLoc(), sizeChars);
1020       Args.add(RValue::get(Order), getContext().IntTy);
1021       Order = OrderFail;
1022       break;
1023     // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1024     //                        int order)
1025     // T __atomic_exchange_N(T *mem, T val, int order)
1026     case AtomicExpr::AO__c11_atomic_exchange:
1027     case AtomicExpr::AO__opencl_atomic_exchange:
1028     case AtomicExpr::AO__atomic_exchange_n:
1029     case AtomicExpr::AO__atomic_exchange:
1030       LibCallName = "__atomic_exchange";
1031       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1032                         MemTy, E->getExprLoc(), sizeChars);
1033       break;
1034     // void __atomic_store(size_t size, void *mem, void *val, int order)
1035     // void __atomic_store_N(T *mem, T val, int order)
1036     case AtomicExpr::AO__c11_atomic_store:
1037     case AtomicExpr::AO__opencl_atomic_store:
1038     case AtomicExpr::AO__atomic_store:
1039     case AtomicExpr::AO__atomic_store_n:
1040       LibCallName = "__atomic_store";
1041       RetTy = getContext().VoidTy;
1042       HaveRetTy = true;
1043       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1044                         MemTy, E->getExprLoc(), sizeChars);
1045       break;
1046     // void __atomic_load(size_t size, void *mem, void *return, int order)
1047     // T __atomic_load_N(T *mem, int order)
1048     case AtomicExpr::AO__c11_atomic_load:
1049     case AtomicExpr::AO__opencl_atomic_load:
1050     case AtomicExpr::AO__atomic_load:
1051     case AtomicExpr::AO__atomic_load_n:
1052       LibCallName = "__atomic_load";
1053       break;
1054     // T __atomic_add_fetch_N(T *mem, T val, int order)
1055     // T __atomic_fetch_add_N(T *mem, T val, int order)
1056     case AtomicExpr::AO__atomic_add_fetch:
1057       PostOp = llvm::Instruction::Add;
1058       LLVM_FALLTHROUGH;
1059     case AtomicExpr::AO__c11_atomic_fetch_add:
1060     case AtomicExpr::AO__opencl_atomic_fetch_add:
1061     case AtomicExpr::AO__atomic_fetch_add:
1062       LibCallName = "__atomic_fetch_add";
1063       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1064                         LoweredMemTy, E->getExprLoc(), sizeChars);
1065       break;
1066     // T __atomic_and_fetch_N(T *mem, T val, int order)
1067     // T __atomic_fetch_and_N(T *mem, T val, int order)
1068     case AtomicExpr::AO__atomic_and_fetch:
1069       PostOp = llvm::Instruction::And;
1070       LLVM_FALLTHROUGH;
1071     case AtomicExpr::AO__c11_atomic_fetch_and:
1072     case AtomicExpr::AO__opencl_atomic_fetch_and:
1073     case AtomicExpr::AO__atomic_fetch_and:
1074       LibCallName = "__atomic_fetch_and";
1075       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1076                         MemTy, E->getExprLoc(), sizeChars);
1077       break;
1078     // T __atomic_or_fetch_N(T *mem, T val, int order)
1079     // T __atomic_fetch_or_N(T *mem, T val, int order)
1080     case AtomicExpr::AO__atomic_or_fetch:
1081       PostOp = llvm::Instruction::Or;
1082       LLVM_FALLTHROUGH;
1083     case AtomicExpr::AO__c11_atomic_fetch_or:
1084     case AtomicExpr::AO__opencl_atomic_fetch_or:
1085     case AtomicExpr::AO__atomic_fetch_or:
1086       LibCallName = "__atomic_fetch_or";
1087       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1088                         MemTy, E->getExprLoc(), sizeChars);
1089       break;
1090     // T __atomic_sub_fetch_N(T *mem, T val, int order)
1091     // T __atomic_fetch_sub_N(T *mem, T val, int order)
1092     case AtomicExpr::AO__atomic_sub_fetch:
1093       PostOp = llvm::Instruction::Sub;
1094       LLVM_FALLTHROUGH;
1095     case AtomicExpr::AO__c11_atomic_fetch_sub:
1096     case AtomicExpr::AO__opencl_atomic_fetch_sub:
1097     case AtomicExpr::AO__atomic_fetch_sub:
1098       LibCallName = "__atomic_fetch_sub";
1099       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1100                         LoweredMemTy, E->getExprLoc(), sizeChars);
1101       break;
1102     // T __atomic_xor_fetch_N(T *mem, T val, int order)
1103     // T __atomic_fetch_xor_N(T *mem, T val, int order)
1104     case AtomicExpr::AO__atomic_xor_fetch:
1105       PostOp = llvm::Instruction::Xor;
1106       LLVM_FALLTHROUGH;
1107     case AtomicExpr::AO__c11_atomic_fetch_xor:
1108     case AtomicExpr::AO__opencl_atomic_fetch_xor:
1109     case AtomicExpr::AO__atomic_fetch_xor:
1110       LibCallName = "__atomic_fetch_xor";
1111       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1112                         MemTy, E->getExprLoc(), sizeChars);
1113       break;
1114     case AtomicExpr::AO__atomic_fetch_min:
1115     case AtomicExpr::AO__opencl_atomic_fetch_min:
1116       LibCallName = E->getValueType()->isSignedIntegerType()
1117                         ? "__atomic_fetch_min"
1118                         : "__atomic_fetch_umin";
1119       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1120                         LoweredMemTy, E->getExprLoc(), sizeChars);
1121       break;
1122     case AtomicExpr::AO__atomic_fetch_max:
1123     case AtomicExpr::AO__opencl_atomic_fetch_max:
1124       LibCallName = E->getValueType()->isSignedIntegerType()
1125                         ? "__atomic_fetch_max"
1126                         : "__atomic_fetch_umax";
1127       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1128                         LoweredMemTy, E->getExprLoc(), sizeChars);
1129       break;
1130     // T __atomic_nand_fetch_N(T *mem, T val, int order)
1131     // T __atomic_fetch_nand_N(T *mem, T val, int order)
1132     case AtomicExpr::AO__atomic_nand_fetch:
1133       PostOp = llvm::Instruction::And; // the NOT is special cased below
1134       LLVM_FALLTHROUGH;
1135     case AtomicExpr::AO__atomic_fetch_nand:
1136       LibCallName = "__atomic_fetch_nand";
1137       AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1138                         MemTy, E->getExprLoc(), sizeChars);
1139       break;
1140     }
1141 
1142     if (E->isOpenCL()) {
1143       LibCallName = std::string("__opencl") +
1144           StringRef(LibCallName).drop_front(1).str();
1145 
1146     }
1147     // Optimized functions have the size in their name.
1148     if (UseOptimizedLibcall)
1149       LibCallName += "_" + llvm::utostr(Size);
1150     // By default, assume we return a value of the atomic type.
1151     if (!HaveRetTy) {
1152       if (UseOptimizedLibcall) {
1153         // Value is returned directly.
1154         // The function returns an appropriately sized integer type.
1155         RetTy = getContext().getIntTypeForBitwidth(
1156             getContext().toBits(sizeChars), /*Signed=*/false);
1157       } else {
1158         // Value is returned through parameter before the order.
1159         RetTy = getContext().VoidTy;
1160         Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1161                  getContext().VoidPtrTy);
1162       }
1163     }
1164     // order is always the last parameter
1165     Args.add(RValue::get(Order),
1166              getContext().IntTy);
1167     if (E->isOpenCL())
1168       Args.add(RValue::get(Scope), getContext().IntTy);
1169 
1170     // PostOp is only needed for the atomic_*_fetch operations, and
1171     // thus is only needed for and implemented in the
1172     // UseOptimizedLibcall codepath.
1173     assert(UseOptimizedLibcall || !PostOp);
1174 
1175     RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1176     // The value is returned directly from the libcall.
1177     if (E->isCmpXChg())
1178       return Res;
1179 
1180     // The value is returned directly for optimized libcalls but the expr
1181     // provided an out-param.
1182     if (UseOptimizedLibcall && Res.getScalarVal()) {
1183       llvm::Value *ResVal = Res.getScalarVal();
1184       if (PostOp) {
1185         llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1186         ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1187       }
1188       if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1189         ResVal = Builder.CreateNot(ResVal);
1190 
1191       Builder.CreateStore(
1192           ResVal,
1193           Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1194     }
1195 
1196     if (RValTy->isVoidType())
1197       return RValue::get(nullptr);
1198 
1199     return convertTempToRValue(
1200         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1201         RValTy, E->getExprLoc());
1202   }
1203 
1204   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1205                  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1206                  E->getOp() == AtomicExpr::AO__atomic_store ||
1207                  E->getOp() == AtomicExpr::AO__atomic_store_n;
1208   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1209                 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1210                 E->getOp() == AtomicExpr::AO__atomic_load ||
1211                 E->getOp() == AtomicExpr::AO__atomic_load_n;
1212 
1213   if (isa<llvm::ConstantInt>(Order)) {
1214     auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1215     // We should not ever get to a case where the ordering isn't a valid C ABI
1216     // value, but it's hard to enforce that in general.
1217     if (llvm::isValidAtomicOrderingCABI(ord))
1218       switch ((llvm::AtomicOrderingCABI)ord) {
1219       case llvm::AtomicOrderingCABI::relaxed:
1220         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1221                      llvm::AtomicOrdering::Monotonic, Scope);
1222         break;
1223       case llvm::AtomicOrderingCABI::consume:
1224       case llvm::AtomicOrderingCABI::acquire:
1225         if (IsStore)
1226           break; // Avoid crashing on code with undefined behavior
1227         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1228                      llvm::AtomicOrdering::Acquire, Scope);
1229         break;
1230       case llvm::AtomicOrderingCABI::release:
1231         if (IsLoad)
1232           break; // Avoid crashing on code with undefined behavior
1233         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1234                      llvm::AtomicOrdering::Release, Scope);
1235         break;
1236       case llvm::AtomicOrderingCABI::acq_rel:
1237         if (IsLoad || IsStore)
1238           break; // Avoid crashing on code with undefined behavior
1239         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1240                      llvm::AtomicOrdering::AcquireRelease, Scope);
1241         break;
1242       case llvm::AtomicOrderingCABI::seq_cst:
1243         EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1244                      llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1245         break;
1246       }
1247     if (RValTy->isVoidType())
1248       return RValue::get(nullptr);
1249 
1250     return convertTempToRValue(
1251         Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1252                                         Dest.getAddressSpace())),
1253         RValTy, E->getExprLoc());
1254   }
1255 
1256   // Long case, when Order isn't obviously constant.
1257 
1258   // Create all the relevant BB's
1259   llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1260                    *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1261                    *SeqCstBB = nullptr;
1262   MonotonicBB = createBasicBlock("monotonic", CurFn);
1263   if (!IsStore)
1264     AcquireBB = createBasicBlock("acquire", CurFn);
1265   if (!IsLoad)
1266     ReleaseBB = createBasicBlock("release", CurFn);
1267   if (!IsLoad && !IsStore)
1268     AcqRelBB = createBasicBlock("acqrel", CurFn);
1269   SeqCstBB = createBasicBlock("seqcst", CurFn);
1270   llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1271 
1272   // Create the switch for the split
1273   // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1274   // doesn't matter unless someone is crazy enough to use something that
1275   // doesn't fold to a constant for the ordering.
1276   Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1277   llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1278 
1279   // Emit all the different atomics
1280   Builder.SetInsertPoint(MonotonicBB);
1281   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1282                llvm::AtomicOrdering::Monotonic, Scope);
1283   Builder.CreateBr(ContBB);
1284   if (!IsStore) {
1285     Builder.SetInsertPoint(AcquireBB);
1286     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1287                  llvm::AtomicOrdering::Acquire, Scope);
1288     Builder.CreateBr(ContBB);
1289     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1290                 AcquireBB);
1291     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1292                 AcquireBB);
1293   }
1294   if (!IsLoad) {
1295     Builder.SetInsertPoint(ReleaseBB);
1296     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1297                  llvm::AtomicOrdering::Release, Scope);
1298     Builder.CreateBr(ContBB);
1299     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1300                 ReleaseBB);
1301   }
1302   if (!IsLoad && !IsStore) {
1303     Builder.SetInsertPoint(AcqRelBB);
1304     EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1305                  llvm::AtomicOrdering::AcquireRelease, Scope);
1306     Builder.CreateBr(ContBB);
1307     SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1308                 AcqRelBB);
1309   }
1310   Builder.SetInsertPoint(SeqCstBB);
1311   EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1312                llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1313   Builder.CreateBr(ContBB);
1314   SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1315               SeqCstBB);
1316 
1317   // Cleanup and return
1318   Builder.SetInsertPoint(ContBB);
1319   if (RValTy->isVoidType())
1320     return RValue::get(nullptr);
1321 
1322   assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1323   return convertTempToRValue(
1324       Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1325                                       Dest.getAddressSpace())),
1326       RValTy, E->getExprLoc());
1327 }
1328 
1329 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1330   unsigned addrspace =
1331     cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1332   llvm::IntegerType *ty =
1333     llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1334   return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1335 }
1336 
1337 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1338   llvm::Type *Ty = Addr.getElementType();
1339   uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1340   if (SourceSizeInBits != AtomicSizeInBits) {
1341     Address Tmp = CreateTempAlloca();
1342     CGF.Builder.CreateMemCpy(Tmp, Addr,
1343                              std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1344     Addr = Tmp;
1345   }
1346 
1347   return emitCastToAtomicIntPointer(Addr);
1348 }
1349 
1350 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1351                                              AggValueSlot resultSlot,
1352                                              SourceLocation loc,
1353                                              bool asValue) const {
1354   if (LVal.isSimple()) {
1355     if (EvaluationKind == TEK_Aggregate)
1356       return resultSlot.asRValue();
1357 
1358     // Drill into the padding structure if we have one.
1359     if (hasPadding())
1360       addr = CGF.Builder.CreateStructGEP(addr, 0);
1361 
1362     // Otherwise, just convert the temporary to an r-value using the
1363     // normal conversion routine.
1364     return CGF.convertTempToRValue(addr, getValueType(), loc);
1365   }
1366   if (!asValue)
1367     // Get RValue from temp memory as atomic for non-simple lvalues
1368     return RValue::get(CGF.Builder.CreateLoad(addr));
1369   if (LVal.isBitField())
1370     return CGF.EmitLoadOfBitfieldLValue(
1371         LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1372                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1373   if (LVal.isVectorElt())
1374     return CGF.EmitLoadOfLValue(
1375         LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1376                               LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1377   assert(LVal.isExtVectorElt());
1378   return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1379       addr, LVal.getExtVectorElts(), LVal.getType(),
1380       LVal.getBaseInfo(), TBAAAccessInfo()));
1381 }
1382 
1383 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1384                                              AggValueSlot ResultSlot,
1385                                              SourceLocation Loc,
1386                                              bool AsValue) const {
1387   // Try not to in some easy cases.
1388   assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1389   if (getEvaluationKind() == TEK_Scalar &&
1390       (((!LVal.isBitField() ||
1391          LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1392         !hasPadding()) ||
1393        !AsValue)) {
1394     auto *ValTy = AsValue
1395                       ? CGF.ConvertTypeForMem(ValueTy)
1396                       : getAtomicAddress().getType()->getPointerElementType();
1397     if (ValTy->isIntegerTy()) {
1398       assert(IntVal->getType() == ValTy && "Different integer types.");
1399       return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1400     } else if (ValTy->isPointerTy())
1401       return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1402     else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1403       return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1404   }
1405 
1406   // Create a temporary.  This needs to be big enough to hold the
1407   // atomic integer.
1408   Address Temp = Address::invalid();
1409   bool TempIsVolatile = false;
1410   if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1411     assert(!ResultSlot.isIgnored());
1412     Temp = ResultSlot.getAddress();
1413     TempIsVolatile = ResultSlot.isVolatile();
1414   } else {
1415     Temp = CreateTempAlloca();
1416   }
1417 
1418   // Slam the integer into the temporary.
1419   Address CastTemp = emitCastToAtomicIntPointer(Temp);
1420   CGF.Builder.CreateStore(IntVal, CastTemp)
1421       ->setVolatile(TempIsVolatile);
1422 
1423   return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1424 }
1425 
1426 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1427                                        llvm::AtomicOrdering AO, bool) {
1428   // void __atomic_load(size_t size, void *mem, void *return, int order);
1429   CallArgList Args;
1430   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1431   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1432            CGF.getContext().VoidPtrTy);
1433   Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1434            CGF.getContext().VoidPtrTy);
1435   Args.add(
1436       RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1437       CGF.getContext().IntTy);
1438   emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1439 }
1440 
1441 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1442                                           bool IsVolatile) {
1443   // Okay, we're doing this natively.
1444   Address Addr = getAtomicAddressAsAtomicIntPointer();
1445   llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1446   Load->setAtomic(AO);
1447 
1448   // Other decoration.
1449   if (IsVolatile)
1450     Load->setVolatile(true);
1451   CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1452   return Load;
1453 }
1454 
1455 /// An LValue is a candidate for having its loads and stores be made atomic if
1456 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1457 /// performing such an operation can be performed without a libcall.
1458 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1459   if (!CGM.getCodeGenOpts().MSVolatile) return false;
1460   AtomicInfo AI(*this, LV);
1461   bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1462   // An atomic is inline if we don't need to use a libcall.
1463   bool AtomicIsInline = !AI.shouldUseLibcall();
1464   // MSVC doesn't seem to do this for types wider than a pointer.
1465   if (getContext().getTypeSize(LV.getType()) >
1466       getContext().getTypeSize(getContext().getIntPtrType()))
1467     return false;
1468   return IsVolatile && AtomicIsInline;
1469 }
1470 
1471 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1472                                        AggValueSlot Slot) {
1473   llvm::AtomicOrdering AO;
1474   bool IsVolatile = LV.isVolatileQualified();
1475   if (LV.getType()->isAtomicType()) {
1476     AO = llvm::AtomicOrdering::SequentiallyConsistent;
1477   } else {
1478     AO = llvm::AtomicOrdering::Acquire;
1479     IsVolatile = true;
1480   }
1481   return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1482 }
1483 
1484 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1485                                   bool AsValue, llvm::AtomicOrdering AO,
1486                                   bool IsVolatile) {
1487   // Check whether we should use a library call.
1488   if (shouldUseLibcall()) {
1489     Address TempAddr = Address::invalid();
1490     if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1491       assert(getEvaluationKind() == TEK_Aggregate);
1492       TempAddr = ResultSlot.getAddress();
1493     } else
1494       TempAddr = CreateTempAlloca();
1495 
1496     EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1497 
1498     // Okay, turn that back into the original value or whole atomic (for
1499     // non-simple lvalues) type.
1500     return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1501   }
1502 
1503   // Okay, we're doing this natively.
1504   auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1505 
1506   // If we're ignoring an aggregate return, don't do anything.
1507   if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1508     return RValue::getAggregate(Address::invalid(), false);
1509 
1510   // Okay, turn that back into the original value or atomic (for non-simple
1511   // lvalues) type.
1512   return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1513 }
1514 
1515 /// Emit a load from an l-value of atomic type.  Note that the r-value
1516 /// we produce is an r-value of the atomic *value* type.
1517 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1518                                        llvm::AtomicOrdering AO, bool IsVolatile,
1519                                        AggValueSlot resultSlot) {
1520   AtomicInfo Atomics(*this, src);
1521   return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1522                                 IsVolatile);
1523 }
1524 
1525 /// Copy an r-value into memory as part of storing to an atomic type.
1526 /// This needs to create a bit-pattern suitable for atomic operations.
1527 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1528   assert(LVal.isSimple());
1529   // If we have an r-value, the rvalue should be of the atomic type,
1530   // which means that the caller is responsible for having zeroed
1531   // any padding.  Just do an aggregate copy of that type.
1532   if (rvalue.isAggregate()) {
1533     LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1534     LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1535                                     getAtomicType());
1536     bool IsVolatile = rvalue.isVolatileQualified() ||
1537                       LVal.isVolatileQualified();
1538     CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1539                           AggValueSlot::DoesNotOverlap, IsVolatile);
1540     return;
1541   }
1542 
1543   // Okay, otherwise we're copying stuff.
1544 
1545   // Zero out the buffer if necessary.
1546   emitMemSetZeroIfNecessary();
1547 
1548   // Drill past the padding if present.
1549   LValue TempLVal = projectValue();
1550 
1551   // Okay, store the rvalue in.
1552   if (rvalue.isScalar()) {
1553     CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1554   } else {
1555     CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1556   }
1557 }
1558 
1559 
1560 /// Materialize an r-value into memory for the purposes of storing it
1561 /// to an atomic type.
1562 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1563   // Aggregate r-values are already in memory, and EmitAtomicStore
1564   // requires them to be values of the atomic type.
1565   if (rvalue.isAggregate())
1566     return rvalue.getAggregateAddress();
1567 
1568   // Otherwise, make a temporary and materialize into it.
1569   LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1570   AtomicInfo Atomics(CGF, TempLV);
1571   Atomics.emitCopyIntoMemory(rvalue);
1572   return TempLV.getAddress();
1573 }
1574 
1575 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1576   // If we've got a scalar value of the right size, try to avoid going
1577   // through memory.
1578   if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1579     llvm::Value *Value = RVal.getScalarVal();
1580     if (isa<llvm::IntegerType>(Value->getType()))
1581       return CGF.EmitToMemory(Value, ValueTy);
1582     else {
1583       llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1584           CGF.getLLVMContext(),
1585           LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1586       if (isa<llvm::PointerType>(Value->getType()))
1587         return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1588       else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1589         return CGF.Builder.CreateBitCast(Value, InputIntTy);
1590     }
1591   }
1592   // Otherwise, we need to go through memory.
1593   // Put the r-value in memory.
1594   Address Addr = materializeRValue(RVal);
1595 
1596   // Cast the temporary to the atomic int type and pull a value out.
1597   Addr = emitCastToAtomicIntPointer(Addr);
1598   return CGF.Builder.CreateLoad(Addr);
1599 }
1600 
1601 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1602     llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1603     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1604   // Do the atomic store.
1605   Address Addr = getAtomicAddressAsAtomicIntPointer();
1606   auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1607                                                ExpectedVal, DesiredVal,
1608                                                Success, Failure);
1609   // Other decoration.
1610   Inst->setVolatile(LVal.isVolatileQualified());
1611   Inst->setWeak(IsWeak);
1612 
1613   // Okay, turn that back into the original value type.
1614   auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1615   auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1616   return std::make_pair(PreviousVal, SuccessFailureVal);
1617 }
1618 
1619 llvm::Value *
1620 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1621                                              llvm::Value *DesiredAddr,
1622                                              llvm::AtomicOrdering Success,
1623                                              llvm::AtomicOrdering Failure) {
1624   // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1625   // void *desired, int success, int failure);
1626   CallArgList Args;
1627   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1628   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1629            CGF.getContext().VoidPtrTy);
1630   Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1631            CGF.getContext().VoidPtrTy);
1632   Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1633            CGF.getContext().VoidPtrTy);
1634   Args.add(RValue::get(
1635                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1636            CGF.getContext().IntTy);
1637   Args.add(RValue::get(
1638                llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1639            CGF.getContext().IntTy);
1640   auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1641                                               CGF.getContext().BoolTy, Args);
1642 
1643   return SuccessFailureRVal.getScalarVal();
1644 }
1645 
1646 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1647     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1648     llvm::AtomicOrdering Failure, bool IsWeak) {
1649   if (isStrongerThan(Failure, Success))
1650     // Don't assert on undefined behavior "failure argument shall be no stronger
1651     // than the success argument".
1652     Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1653 
1654   // Check whether we should use a library call.
1655   if (shouldUseLibcall()) {
1656     // Produce a source address.
1657     Address ExpectedAddr = materializeRValue(Expected);
1658     Address DesiredAddr = materializeRValue(Desired);
1659     auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1660                                                  DesiredAddr.getPointer(),
1661                                                  Success, Failure);
1662     return std::make_pair(
1663         convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1664                                   SourceLocation(), /*AsValue=*/false),
1665         Res);
1666   }
1667 
1668   // If we've got a scalar value of the right size, try to avoid going
1669   // through memory.
1670   auto *ExpectedVal = convertRValueToInt(Expected);
1671   auto *DesiredVal = convertRValueToInt(Desired);
1672   auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1673                                          Failure, IsWeak);
1674   return std::make_pair(
1675       ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1676                                 SourceLocation(), /*AsValue=*/false),
1677       Res.second);
1678 }
1679 
1680 static void
1681 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1682                       const llvm::function_ref<RValue(RValue)> &UpdateOp,
1683                       Address DesiredAddr) {
1684   RValue UpRVal;
1685   LValue AtomicLVal = Atomics.getAtomicLValue();
1686   LValue DesiredLVal;
1687   if (AtomicLVal.isSimple()) {
1688     UpRVal = OldRVal;
1689     DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1690   } else {
1691     // Build new lvalue for temp address.
1692     Address Ptr = Atomics.materializeRValue(OldRVal);
1693     LValue UpdateLVal;
1694     if (AtomicLVal.isBitField()) {
1695       UpdateLVal =
1696           LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1697                                AtomicLVal.getType(),
1698                                AtomicLVal.getBaseInfo(),
1699                                AtomicLVal.getTBAAInfo());
1700       DesiredLVal =
1701           LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1702                                AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1703                                AtomicLVal.getTBAAInfo());
1704     } else if (AtomicLVal.isVectorElt()) {
1705       UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1706                                          AtomicLVal.getType(),
1707                                          AtomicLVal.getBaseInfo(),
1708                                          AtomicLVal.getTBAAInfo());
1709       DesiredLVal = LValue::MakeVectorElt(
1710           DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1711           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1712     } else {
1713       assert(AtomicLVal.isExtVectorElt());
1714       UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1715                                             AtomicLVal.getType(),
1716                                             AtomicLVal.getBaseInfo(),
1717                                             AtomicLVal.getTBAAInfo());
1718       DesiredLVal = LValue::MakeExtVectorElt(
1719           DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1720           AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1721     }
1722     UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1723   }
1724   // Store new value in the corresponding memory area.
1725   RValue NewRVal = UpdateOp(UpRVal);
1726   if (NewRVal.isScalar()) {
1727     CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1728   } else {
1729     assert(NewRVal.isComplex());
1730     CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1731                            /*isInit=*/false);
1732   }
1733 }
1734 
1735 void AtomicInfo::EmitAtomicUpdateLibcall(
1736     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1737     bool IsVolatile) {
1738   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1739 
1740   Address ExpectedAddr = CreateTempAlloca();
1741 
1742   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1743   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1744   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1745   CGF.EmitBlock(ContBB);
1746   Address DesiredAddr = CreateTempAlloca();
1747   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1748       requiresMemSetZero(getAtomicAddress().getElementType())) {
1749     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1750     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1751   }
1752   auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1753                                            AggValueSlot::ignored(),
1754                                            SourceLocation(), /*AsValue=*/false);
1755   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1756   auto *Res =
1757       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1758                                        DesiredAddr.getPointer(),
1759                                        AO, Failure);
1760   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1761   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1762 }
1763 
1764 void AtomicInfo::EmitAtomicUpdateOp(
1765     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1766     bool IsVolatile) {
1767   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1768 
1769   // Do the atomic load.
1770   auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1771   // For non-simple lvalues perform compare-and-swap procedure.
1772   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1773   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1774   auto *CurBB = CGF.Builder.GetInsertBlock();
1775   CGF.EmitBlock(ContBB);
1776   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1777                                              /*NumReservedValues=*/2);
1778   PHI->addIncoming(OldVal, CurBB);
1779   Address NewAtomicAddr = CreateTempAlloca();
1780   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1781   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1782       requiresMemSetZero(getAtomicAddress().getElementType())) {
1783     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1784   }
1785   auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1786                                            SourceLocation(), /*AsValue=*/false);
1787   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1788   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1789   // Try to write new value using cmpxchg operation.
1790   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1791   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1792   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1793   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1794 }
1795 
1796 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1797                                   RValue UpdateRVal, Address DesiredAddr) {
1798   LValue AtomicLVal = Atomics.getAtomicLValue();
1799   LValue DesiredLVal;
1800   // Build new lvalue for temp address.
1801   if (AtomicLVal.isBitField()) {
1802     DesiredLVal =
1803         LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1804                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1805                              AtomicLVal.getTBAAInfo());
1806   } else if (AtomicLVal.isVectorElt()) {
1807     DesiredLVal =
1808         LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1809                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1810                               AtomicLVal.getTBAAInfo());
1811   } else {
1812     assert(AtomicLVal.isExtVectorElt());
1813     DesiredLVal = LValue::MakeExtVectorElt(
1814         DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1815         AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1816   }
1817   // Store new value in the corresponding memory area.
1818   assert(UpdateRVal.isScalar());
1819   CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1820 }
1821 
1822 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1823                                          RValue UpdateRVal, bool IsVolatile) {
1824   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1825 
1826   Address ExpectedAddr = CreateTempAlloca();
1827 
1828   EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1829   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1830   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1831   CGF.EmitBlock(ContBB);
1832   Address DesiredAddr = CreateTempAlloca();
1833   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1834       requiresMemSetZero(getAtomicAddress().getElementType())) {
1835     auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1836     CGF.Builder.CreateStore(OldVal, DesiredAddr);
1837   }
1838   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1839   auto *Res =
1840       EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1841                                        DesiredAddr.getPointer(),
1842                                        AO, Failure);
1843   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1844   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1845 }
1846 
1847 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1848                                     bool IsVolatile) {
1849   auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1850 
1851   // Do the atomic load.
1852   auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1853   // For non-simple lvalues perform compare-and-swap procedure.
1854   auto *ContBB = CGF.createBasicBlock("atomic_cont");
1855   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1856   auto *CurBB = CGF.Builder.GetInsertBlock();
1857   CGF.EmitBlock(ContBB);
1858   llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1859                                              /*NumReservedValues=*/2);
1860   PHI->addIncoming(OldVal, CurBB);
1861   Address NewAtomicAddr = CreateTempAlloca();
1862   Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1863   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1864       requiresMemSetZero(getAtomicAddress().getElementType())) {
1865     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1866   }
1867   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1868   auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1869   // Try to write new value using cmpxchg operation.
1870   auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1871   PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1872   CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1873   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1874 }
1875 
1876 void AtomicInfo::EmitAtomicUpdate(
1877     llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1878     bool IsVolatile) {
1879   if (shouldUseLibcall()) {
1880     EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1881   } else {
1882     EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1883   }
1884 }
1885 
1886 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1887                                   bool IsVolatile) {
1888   if (shouldUseLibcall()) {
1889     EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1890   } else {
1891     EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1892   }
1893 }
1894 
1895 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1896                                       bool isInit) {
1897   bool IsVolatile = lvalue.isVolatileQualified();
1898   llvm::AtomicOrdering AO;
1899   if (lvalue.getType()->isAtomicType()) {
1900     AO = llvm::AtomicOrdering::SequentiallyConsistent;
1901   } else {
1902     AO = llvm::AtomicOrdering::Release;
1903     IsVolatile = true;
1904   }
1905   return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1906 }
1907 
1908 /// Emit a store to an l-value of atomic type.
1909 ///
1910 /// Note that the r-value is expected to be an r-value *of the atomic
1911 /// type*; this means that for aggregate r-values, it should include
1912 /// storage for any padding that was necessary.
1913 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1914                                       llvm::AtomicOrdering AO, bool IsVolatile,
1915                                       bool isInit) {
1916   // If this is an aggregate r-value, it should agree in type except
1917   // maybe for address-space qualification.
1918   assert(!rvalue.isAggregate() ||
1919          rvalue.getAggregateAddress().getElementType()
1920            == dest.getAddress().getElementType());
1921 
1922   AtomicInfo atomics(*this, dest);
1923   LValue LVal = atomics.getAtomicLValue();
1924 
1925   // If this is an initialization, just put the value there normally.
1926   if (LVal.isSimple()) {
1927     if (isInit) {
1928       atomics.emitCopyIntoMemory(rvalue);
1929       return;
1930     }
1931 
1932     // Check whether we should use a library call.
1933     if (atomics.shouldUseLibcall()) {
1934       // Produce a source address.
1935       Address srcAddr = atomics.materializeRValue(rvalue);
1936 
1937       // void __atomic_store(size_t size, void *mem, void *val, int order)
1938       CallArgList args;
1939       args.add(RValue::get(atomics.getAtomicSizeValue()),
1940                getContext().getSizeType());
1941       args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1942                getContext().VoidPtrTy);
1943       args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1944                getContext().VoidPtrTy);
1945       args.add(
1946           RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
1947           getContext().IntTy);
1948       emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1949       return;
1950     }
1951 
1952     // Okay, we're doing this natively.
1953     llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1954 
1955     // Do the atomic store.
1956     Address addr =
1957         atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1958     intValue = Builder.CreateIntCast(
1959         intValue, addr.getElementType(), /*isSigned=*/false);
1960     llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1961 
1962     // Initializations don't need to be atomic.
1963     if (!isInit)
1964       store->setAtomic(AO);
1965 
1966     // Other decoration.
1967     if (IsVolatile)
1968       store->setVolatile(true);
1969     CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
1970     return;
1971   }
1972 
1973   // Emit simple atomic update operation.
1974   atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1975 }
1976 
1977 /// Emit a compare-and-exchange op for atomic type.
1978 ///
1979 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1980     LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1981     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1982     AggValueSlot Slot) {
1983   // If this is an aggregate r-value, it should agree in type except
1984   // maybe for address-space qualification.
1985   assert(!Expected.isAggregate() ||
1986          Expected.getAggregateAddress().getElementType() ==
1987              Obj.getAddress().getElementType());
1988   assert(!Desired.isAggregate() ||
1989          Desired.getAggregateAddress().getElementType() ==
1990              Obj.getAddress().getElementType());
1991   AtomicInfo Atomics(*this, Obj);
1992 
1993   return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1994                                            IsWeak);
1995 }
1996 
1997 void CodeGenFunction::EmitAtomicUpdate(
1998     LValue LVal, llvm::AtomicOrdering AO,
1999     const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2000   AtomicInfo Atomics(*this, LVal);
2001   Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2002 }
2003 
2004 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2005   AtomicInfo atomics(*this, dest);
2006 
2007   switch (atomics.getEvaluationKind()) {
2008   case TEK_Scalar: {
2009     llvm::Value *value = EmitScalarExpr(init);
2010     atomics.emitCopyIntoMemory(RValue::get(value));
2011     return;
2012   }
2013 
2014   case TEK_Complex: {
2015     ComplexPairTy value = EmitComplexExpr(init);
2016     atomics.emitCopyIntoMemory(RValue::getComplex(value));
2017     return;
2018   }
2019 
2020   case TEK_Aggregate: {
2021     // Fix up the destination if the initializer isn't an expression
2022     // of atomic type.
2023     bool Zeroed = false;
2024     if (!init->getType()->isAtomicType()) {
2025       Zeroed = atomics.emitMemSetZeroIfNecessary();
2026       dest = atomics.projectValue();
2027     }
2028 
2029     // Evaluate the expression directly into the destination.
2030     AggValueSlot slot = AggValueSlot::forLValue(dest,
2031                                         AggValueSlot::IsNotDestructed,
2032                                         AggValueSlot::DoesNotNeedGCBarriers,
2033                                         AggValueSlot::IsNotAliased,
2034                                         AggValueSlot::DoesNotOverlap,
2035                                         Zeroed ? AggValueSlot::IsZeroed :
2036                                                  AggValueSlot::IsNotZeroed);
2037 
2038     EmitAggExpr(init, slot);
2039     return;
2040   }
2041   }
2042   llvm_unreachable("bad evaluation kind");
2043 }
2044