1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the code for emitting atomic operations. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CGCall.h" 14 #include "CGRecordLayout.h" 15 #include "CodeGenFunction.h" 16 #include "CodeGenModule.h" 17 #include "TargetInfo.h" 18 #include "clang/AST/ASTContext.h" 19 #include "clang/CodeGen/CGFunctionInfo.h" 20 #include "clang/Frontend/FrontendDiagnostic.h" 21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/IR/DataLayout.h" 23 #include "llvm/IR/Intrinsics.h" 24 #include "llvm/IR/Operator.h" 25 26 using namespace clang; 27 using namespace CodeGen; 28 29 namespace { 30 class AtomicInfo { 31 CodeGenFunction &CGF; 32 QualType AtomicTy; 33 QualType ValueTy; 34 uint64_t AtomicSizeInBits; 35 uint64_t ValueSizeInBits; 36 CharUnits AtomicAlign; 37 CharUnits ValueAlign; 38 TypeEvaluationKind EvaluationKind; 39 bool UseLibcall; 40 LValue LVal; 41 CGBitFieldInfo BFI; 42 public: 43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 45 EvaluationKind(TEK_Scalar), UseLibcall(true) { 46 assert(!lvalue.isGlobalReg()); 47 ASTContext &C = CGF.getContext(); 48 if (lvalue.isSimple()) { 49 AtomicTy = lvalue.getType(); 50 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 51 ValueTy = ATy->getValueType(); 52 else 53 ValueTy = AtomicTy; 54 EvaluationKind = CGF.getEvaluationKind(ValueTy); 55 56 uint64_t ValueAlignInBits; 57 uint64_t AtomicAlignInBits; 58 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 59 ValueSizeInBits = ValueTI.Width; 60 ValueAlignInBits = ValueTI.Align; 61 62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 63 AtomicSizeInBits = AtomicTI.Width; 64 AtomicAlignInBits = AtomicTI.Align; 65 66 assert(ValueSizeInBits <= AtomicSizeInBits); 67 assert(ValueAlignInBits <= AtomicAlignInBits); 68 69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 71 if (lvalue.getAlignment().isZero()) 72 lvalue.setAlignment(AtomicAlign); 73 74 LVal = lvalue; 75 } else if (lvalue.isBitField()) { 76 ValueTy = lvalue.getType(); 77 ValueSizeInBits = C.getTypeSize(ValueTy); 78 auto &OrigBFI = lvalue.getBitFieldInfo(); 79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 80 AtomicSizeInBits = C.toBits( 81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 82 .alignTo(lvalue.getAlignment())); 83 llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer(); 84 auto OffsetInChars = 85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 86 lvalue.getAlignment(); 87 llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64( 88 CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity()); 89 StoragePtr = CGF.Builder.CreateAddrSpaceCast( 90 StoragePtr, llvm::PointerType::getUnqual(CGF.getLLVMContext()), 91 "atomic_bitfield_base"); 92 BFI = OrigBFI; 93 BFI.Offset = Offset; 94 BFI.StorageSize = AtomicSizeInBits; 95 BFI.StorageOffset += OffsetInChars; 96 llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits); 97 LVal = LValue::MakeBitfield( 98 Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI, 99 lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo()); 100 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 101 if (AtomicTy.isNull()) { 102 llvm::APInt Size( 103 /*numBits=*/32, 104 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 105 AtomicTy = 106 C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal, 107 /*IndexTypeQuals=*/0); 108 } 109 AtomicAlign = ValueAlign = lvalue.getAlignment(); 110 } else if (lvalue.isVectorElt()) { 111 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType(); 112 ValueSizeInBits = C.getTypeSize(ValueTy); 113 AtomicTy = lvalue.getType(); 114 AtomicSizeInBits = C.getTypeSize(AtomicTy); 115 AtomicAlign = ValueAlign = lvalue.getAlignment(); 116 LVal = lvalue; 117 } else { 118 assert(lvalue.isExtVectorElt()); 119 ValueTy = lvalue.getType(); 120 ValueSizeInBits = C.getTypeSize(ValueTy); 121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 122 lvalue.getType(), cast<llvm::FixedVectorType>( 123 lvalue.getExtVectorAddress().getElementType()) 124 ->getNumElements()); 125 AtomicSizeInBits = C.getTypeSize(AtomicTy); 126 AtomicAlign = ValueAlign = lvalue.getAlignment(); 127 LVal = lvalue; 128 } 129 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 130 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 131 } 132 133 QualType getAtomicType() const { return AtomicTy; } 134 QualType getValueType() const { return ValueTy; } 135 CharUnits getAtomicAlignment() const { return AtomicAlign; } 136 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 137 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 138 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 139 bool shouldUseLibcall() const { return UseLibcall; } 140 const LValue &getAtomicLValue() const { return LVal; } 141 llvm::Value *getAtomicPointer() const { 142 if (LVal.isSimple()) 143 return LVal.getPointer(CGF); 144 else if (LVal.isBitField()) 145 return LVal.getBitFieldPointer(); 146 else if (LVal.isVectorElt()) 147 return LVal.getVectorPointer(); 148 assert(LVal.isExtVectorElt()); 149 return LVal.getExtVectorPointer(); 150 } 151 Address getAtomicAddress() const { 152 llvm::Type *ElTy; 153 if (LVal.isSimple()) 154 ElTy = LVal.getAddress(CGF).getElementType(); 155 else if (LVal.isBitField()) 156 ElTy = LVal.getBitFieldAddress().getElementType(); 157 else if (LVal.isVectorElt()) 158 ElTy = LVal.getVectorAddress().getElementType(); 159 else 160 ElTy = LVal.getExtVectorAddress().getElementType(); 161 return Address(getAtomicPointer(), ElTy, getAtomicAlignment()); 162 } 163 164 Address getAtomicAddressAsAtomicIntPointer() const { 165 return castToAtomicIntPointer(getAtomicAddress()); 166 } 167 168 /// Is the atomic size larger than the underlying value type? 169 /// 170 /// Note that the absence of padding does not mean that atomic 171 /// objects are completely interchangeable with non-atomic 172 /// objects: we might have promoted the alignment of a type 173 /// without making it bigger. 174 bool hasPadding() const { 175 return (ValueSizeInBits != AtomicSizeInBits); 176 } 177 178 bool emitMemSetZeroIfNecessary() const; 179 180 llvm::Value *getAtomicSizeValue() const { 181 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 182 return CGF.CGM.getSize(size); 183 } 184 185 /// Cast the given pointer to an integer pointer suitable for atomic 186 /// operations if the source. 187 Address castToAtomicIntPointer(Address Addr) const; 188 189 /// If Addr is compatible with the iN that will be used for an atomic 190 /// operation, bitcast it. Otherwise, create a temporary that is suitable 191 /// and copy the value across. 192 Address convertToAtomicIntPointer(Address Addr) const; 193 194 /// Turn an atomic-layout object into an r-value. 195 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 196 SourceLocation loc, bool AsValue) const; 197 198 /// Converts a rvalue to integer value. 199 llvm::Value *convertRValueToInt(RValue RVal) const; 200 201 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, 202 AggValueSlot ResultSlot, 203 SourceLocation Loc, bool AsValue) const; 204 205 /// Copy an atomic r-value into atomic-layout memory. 206 void emitCopyIntoMemory(RValue rvalue) const; 207 208 /// Project an l-value down to the value field. 209 LValue projectValue() const { 210 assert(LVal.isSimple()); 211 Address addr = getAtomicAddress(); 212 if (hasPadding()) 213 addr = CGF.Builder.CreateStructGEP(addr, 0); 214 215 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 216 LVal.getBaseInfo(), LVal.getTBAAInfo()); 217 } 218 219 /// Emits atomic load. 220 /// \returns Loaded value. 221 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 222 bool AsValue, llvm::AtomicOrdering AO, 223 bool IsVolatile); 224 225 /// Emits atomic compare-and-exchange sequence. 226 /// \param Expected Expected value. 227 /// \param Desired Desired value. 228 /// \param Success Atomic ordering for success operation. 229 /// \param Failure Atomic ordering for failed operation. 230 /// \param IsWeak true if atomic operation is weak, false otherwise. 231 /// \returns Pair of values: previous value from storage (value type) and 232 /// boolean flag (i1 type) with true if success and false otherwise. 233 std::pair<RValue, llvm::Value *> 234 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 235 llvm::AtomicOrdering Success = 236 llvm::AtomicOrdering::SequentiallyConsistent, 237 llvm::AtomicOrdering Failure = 238 llvm::AtomicOrdering::SequentiallyConsistent, 239 bool IsWeak = false); 240 241 /// Emits atomic update. 242 /// \param AO Atomic ordering. 243 /// \param UpdateOp Update operation for the current lvalue. 244 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 245 const llvm::function_ref<RValue(RValue)> &UpdateOp, 246 bool IsVolatile); 247 /// Emits atomic update. 248 /// \param AO Atomic ordering. 249 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 250 bool IsVolatile); 251 252 /// Materialize an atomic r-value in atomic-layout memory. 253 Address materializeRValue(RValue rvalue) const; 254 255 /// Creates temp alloca for intermediate operations on atomic value. 256 Address CreateTempAlloca() const; 257 private: 258 bool requiresMemSetZero(llvm::Type *type) const; 259 260 261 /// Emits atomic load as a libcall. 262 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 263 llvm::AtomicOrdering AO, bool IsVolatile); 264 /// Emits atomic load as LLVM instruction. 265 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); 266 /// Emits atomic compare-and-exchange op as a libcall. 267 llvm::Value *EmitAtomicCompareExchangeLibcall( 268 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 269 llvm::AtomicOrdering Success = 270 llvm::AtomicOrdering::SequentiallyConsistent, 271 llvm::AtomicOrdering Failure = 272 llvm::AtomicOrdering::SequentiallyConsistent); 273 /// Emits atomic compare-and-exchange op as LLVM instruction. 274 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 275 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 276 llvm::AtomicOrdering Success = 277 llvm::AtomicOrdering::SequentiallyConsistent, 278 llvm::AtomicOrdering Failure = 279 llvm::AtomicOrdering::SequentiallyConsistent, 280 bool IsWeak = false); 281 /// Emit atomic update as libcalls. 282 void 283 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 284 const llvm::function_ref<RValue(RValue)> &UpdateOp, 285 bool IsVolatile); 286 /// Emit atomic update as LLVM instructions. 287 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 288 const llvm::function_ref<RValue(RValue)> &UpdateOp, 289 bool IsVolatile); 290 /// Emit atomic update as libcalls. 291 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 292 bool IsVolatile); 293 /// Emit atomic update as LLVM instructions. 294 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 295 bool IsVolatile); 296 }; 297 } 298 299 Address AtomicInfo::CreateTempAlloca() const { 300 Address TempAlloca = CGF.CreateMemTemp( 301 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 302 : AtomicTy, 303 getAtomicAlignment(), 304 "atomic-temp"); 305 // Cast to pointer to value type for bitfields. 306 if (LVal.isBitField()) 307 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 308 TempAlloca, getAtomicAddress().getType(), 309 getAtomicAddress().getElementType()); 310 return TempAlloca; 311 } 312 313 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 314 StringRef fnName, 315 QualType resultType, 316 CallArgList &args) { 317 const CGFunctionInfo &fnInfo = 318 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 319 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 320 llvm::AttrBuilder fnAttrB(CGF.getLLVMContext()); 321 fnAttrB.addAttribute(llvm::Attribute::NoUnwind); 322 fnAttrB.addAttribute(llvm::Attribute::WillReturn); 323 llvm::AttributeList fnAttrs = llvm::AttributeList::get( 324 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB); 325 326 llvm::FunctionCallee fn = 327 CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs); 328 auto callee = CGCallee::forDirect(fn); 329 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); 330 } 331 332 /// Does a store of the given IR type modify the full expected width? 333 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 334 uint64_t expectedSize) { 335 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 336 } 337 338 /// Does the atomic type require memsetting to zero before initialization? 339 /// 340 /// The IR type is provided as a way of making certain queries faster. 341 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 342 // If the atomic type has size padding, we definitely need a memset. 343 if (hasPadding()) return true; 344 345 // Otherwise, do some simple heuristics to try to avoid it: 346 switch (getEvaluationKind()) { 347 // For scalars and complexes, check whether the store size of the 348 // type uses the full size. 349 case TEK_Scalar: 350 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 351 case TEK_Complex: 352 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 353 AtomicSizeInBits / 2); 354 355 // Padding in structs has an undefined bit pattern. User beware. 356 case TEK_Aggregate: 357 return false; 358 } 359 llvm_unreachable("bad evaluation kind"); 360 } 361 362 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 363 assert(LVal.isSimple()); 364 Address addr = LVal.getAddress(CGF); 365 if (!requiresMemSetZero(addr.getElementType())) 366 return false; 367 368 CGF.Builder.CreateMemSet( 369 addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0), 370 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 371 LVal.getAlignment().getAsAlign()); 372 return true; 373 } 374 375 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 376 Address Dest, Address Ptr, 377 Address Val1, Address Val2, 378 uint64_t Size, 379 llvm::AtomicOrdering SuccessOrder, 380 llvm::AtomicOrdering FailureOrder, 381 llvm::SyncScope::ID Scope) { 382 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 383 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 384 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 385 386 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 387 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder, 388 Scope); 389 Pair->setVolatile(E->isVolatile()); 390 Pair->setWeak(IsWeak); 391 392 // Cmp holds the result of the compare-exchange operation: true on success, 393 // false on failure. 394 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 395 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 396 397 // This basic block is used to hold the store instruction if the operation 398 // failed. 399 llvm::BasicBlock *StoreExpectedBB = 400 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 401 402 // This basic block is the exit point of the operation, we should end up 403 // here regardless of whether or not the operation succeeded. 404 llvm::BasicBlock *ContinueBB = 405 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 406 407 // Update Expected if Expected isn't equal to Old, otherwise branch to the 408 // exit point. 409 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 410 411 CGF.Builder.SetInsertPoint(StoreExpectedBB); 412 // Update the memory at Expected with Old's value. 413 CGF.Builder.CreateStore(Old, Val1); 414 // Finally, branch to the exit point. 415 CGF.Builder.CreateBr(ContinueBB); 416 417 CGF.Builder.SetInsertPoint(ContinueBB); 418 // Update the memory at Dest with Cmp's value. 419 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 420 } 421 422 /// Given an ordering required on success, emit all possible cmpxchg 423 /// instructions to cope with the provided (but possibly only dynamically known) 424 /// FailureOrder. 425 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 426 bool IsWeak, Address Dest, Address Ptr, 427 Address Val1, Address Val2, 428 llvm::Value *FailureOrderVal, 429 uint64_t Size, 430 llvm::AtomicOrdering SuccessOrder, 431 llvm::SyncScope::ID Scope) { 432 llvm::AtomicOrdering FailureOrder; 433 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 434 auto FOS = FO->getSExtValue(); 435 if (!llvm::isValidAtomicOrderingCABI(FOS)) 436 FailureOrder = llvm::AtomicOrdering::Monotonic; 437 else 438 switch ((llvm::AtomicOrderingCABI)FOS) { 439 case llvm::AtomicOrderingCABI::relaxed: 440 // 31.7.2.18: "The failure argument shall not be memory_order_release 441 // nor memory_order_acq_rel". Fallback to monotonic. 442 case llvm::AtomicOrderingCABI::release: 443 case llvm::AtomicOrderingCABI::acq_rel: 444 FailureOrder = llvm::AtomicOrdering::Monotonic; 445 break; 446 case llvm::AtomicOrderingCABI::consume: 447 case llvm::AtomicOrderingCABI::acquire: 448 FailureOrder = llvm::AtomicOrdering::Acquire; 449 break; 450 case llvm::AtomicOrderingCABI::seq_cst: 451 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 452 break; 453 } 454 // Prior to c++17, "the failure argument shall be no stronger than the 455 // success argument". This condition has been lifted and the only 456 // precondition is 31.7.2.18. Effectively treat this as a DR and skip 457 // language version checks. 458 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 459 FailureOrder, Scope); 460 return; 461 } 462 463 // Create all the relevant BB's 464 auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 465 auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 466 auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 467 auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 468 469 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 470 // doesn't matter unless someone is crazy enough to use something that 471 // doesn't fold to a constant for the ordering. 472 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 473 // Implemented as acquire, since it's the closest in LLVM. 474 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 475 AcquireBB); 476 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 477 AcquireBB); 478 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 479 SeqCstBB); 480 481 // Emit all the different atomics 482 CGF.Builder.SetInsertPoint(MonotonicBB); 483 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 484 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); 485 CGF.Builder.CreateBr(ContBB); 486 487 CGF.Builder.SetInsertPoint(AcquireBB); 488 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 489 llvm::AtomicOrdering::Acquire, Scope); 490 CGF.Builder.CreateBr(ContBB); 491 492 CGF.Builder.SetInsertPoint(SeqCstBB); 493 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 494 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 495 CGF.Builder.CreateBr(ContBB); 496 497 CGF.Builder.SetInsertPoint(ContBB); 498 } 499 500 /// Duplicate the atomic min/max operation in conventional IR for the builtin 501 /// variants that return the new rather than the original value. 502 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, 503 AtomicExpr::AtomicOp Op, 504 bool IsSigned, 505 llvm::Value *OldVal, 506 llvm::Value *RHS) { 507 llvm::CmpInst::Predicate Pred; 508 switch (Op) { 509 default: 510 llvm_unreachable("Unexpected min/max operation"); 511 case AtomicExpr::AO__atomic_max_fetch: 512 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT; 513 break; 514 case AtomicExpr::AO__atomic_min_fetch: 515 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT; 516 break; 517 } 518 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst"); 519 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval"); 520 } 521 522 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 523 Address Ptr, Address Val1, Address Val2, 524 llvm::Value *IsWeak, llvm::Value *FailureOrder, 525 uint64_t Size, llvm::AtomicOrdering Order, 526 llvm::SyncScope::ID Scope) { 527 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 528 bool PostOpMinMax = false; 529 unsigned PostOp = 0; 530 531 switch (E->getOp()) { 532 case AtomicExpr::AO__c11_atomic_init: 533 case AtomicExpr::AO__opencl_atomic_init: 534 llvm_unreachable("Already handled!"); 535 536 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 537 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 538 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 539 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 540 FailureOrder, Size, Order, Scope); 541 return; 542 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 543 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 544 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 545 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 546 FailureOrder, Size, Order, Scope); 547 return; 548 case AtomicExpr::AO__atomic_compare_exchange: 549 case AtomicExpr::AO__atomic_compare_exchange_n: { 550 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 551 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 552 Val1, Val2, FailureOrder, Size, Order, Scope); 553 } else { 554 // Create all the relevant BB's 555 llvm::BasicBlock *StrongBB = 556 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 557 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 558 llvm::BasicBlock *ContBB = 559 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 560 561 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 562 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 563 564 CGF.Builder.SetInsertPoint(StrongBB); 565 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 566 FailureOrder, Size, Order, Scope); 567 CGF.Builder.CreateBr(ContBB); 568 569 CGF.Builder.SetInsertPoint(WeakBB); 570 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 571 FailureOrder, Size, Order, Scope); 572 CGF.Builder.CreateBr(ContBB); 573 574 CGF.Builder.SetInsertPoint(ContBB); 575 } 576 return; 577 } 578 case AtomicExpr::AO__c11_atomic_load: 579 case AtomicExpr::AO__opencl_atomic_load: 580 case AtomicExpr::AO__hip_atomic_load: 581 case AtomicExpr::AO__atomic_load_n: 582 case AtomicExpr::AO__atomic_load: { 583 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 584 Load->setAtomic(Order, Scope); 585 Load->setVolatile(E->isVolatile()); 586 CGF.Builder.CreateStore(Load, Dest); 587 return; 588 } 589 590 case AtomicExpr::AO__c11_atomic_store: 591 case AtomicExpr::AO__opencl_atomic_store: 592 case AtomicExpr::AO__hip_atomic_store: 593 case AtomicExpr::AO__atomic_store: 594 case AtomicExpr::AO__atomic_store_n: { 595 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 596 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 597 Store->setAtomic(Order, Scope); 598 Store->setVolatile(E->isVolatile()); 599 return; 600 } 601 602 case AtomicExpr::AO__c11_atomic_exchange: 603 case AtomicExpr::AO__hip_atomic_exchange: 604 case AtomicExpr::AO__opencl_atomic_exchange: 605 case AtomicExpr::AO__atomic_exchange_n: 606 case AtomicExpr::AO__atomic_exchange: 607 Op = llvm::AtomicRMWInst::Xchg; 608 break; 609 610 case AtomicExpr::AO__atomic_add_fetch: 611 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd 612 : llvm::Instruction::Add; 613 [[fallthrough]]; 614 case AtomicExpr::AO__c11_atomic_fetch_add: 615 case AtomicExpr::AO__hip_atomic_fetch_add: 616 case AtomicExpr::AO__opencl_atomic_fetch_add: 617 case AtomicExpr::AO__atomic_fetch_add: 618 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd 619 : llvm::AtomicRMWInst::Add; 620 break; 621 622 case AtomicExpr::AO__atomic_sub_fetch: 623 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub 624 : llvm::Instruction::Sub; 625 [[fallthrough]]; 626 case AtomicExpr::AO__c11_atomic_fetch_sub: 627 case AtomicExpr::AO__hip_atomic_fetch_sub: 628 case AtomicExpr::AO__opencl_atomic_fetch_sub: 629 case AtomicExpr::AO__atomic_fetch_sub: 630 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub 631 : llvm::AtomicRMWInst::Sub; 632 break; 633 634 case AtomicExpr::AO__atomic_min_fetch: 635 PostOpMinMax = true; 636 [[fallthrough]]; 637 case AtomicExpr::AO__c11_atomic_fetch_min: 638 case AtomicExpr::AO__hip_atomic_fetch_min: 639 case AtomicExpr::AO__opencl_atomic_fetch_min: 640 case AtomicExpr::AO__atomic_fetch_min: 641 Op = E->getValueType()->isFloatingType() 642 ? llvm::AtomicRMWInst::FMin 643 : (E->getValueType()->isSignedIntegerType() 644 ? llvm::AtomicRMWInst::Min 645 : llvm::AtomicRMWInst::UMin); 646 break; 647 648 case AtomicExpr::AO__atomic_max_fetch: 649 PostOpMinMax = true; 650 [[fallthrough]]; 651 case AtomicExpr::AO__c11_atomic_fetch_max: 652 case AtomicExpr::AO__hip_atomic_fetch_max: 653 case AtomicExpr::AO__opencl_atomic_fetch_max: 654 case AtomicExpr::AO__atomic_fetch_max: 655 Op = E->getValueType()->isFloatingType() 656 ? llvm::AtomicRMWInst::FMax 657 : (E->getValueType()->isSignedIntegerType() 658 ? llvm::AtomicRMWInst::Max 659 : llvm::AtomicRMWInst::UMax); 660 break; 661 662 case AtomicExpr::AO__atomic_and_fetch: 663 PostOp = llvm::Instruction::And; 664 [[fallthrough]]; 665 case AtomicExpr::AO__c11_atomic_fetch_and: 666 case AtomicExpr::AO__hip_atomic_fetch_and: 667 case AtomicExpr::AO__opencl_atomic_fetch_and: 668 case AtomicExpr::AO__atomic_fetch_and: 669 Op = llvm::AtomicRMWInst::And; 670 break; 671 672 case AtomicExpr::AO__atomic_or_fetch: 673 PostOp = llvm::Instruction::Or; 674 [[fallthrough]]; 675 case AtomicExpr::AO__c11_atomic_fetch_or: 676 case AtomicExpr::AO__hip_atomic_fetch_or: 677 case AtomicExpr::AO__opencl_atomic_fetch_or: 678 case AtomicExpr::AO__atomic_fetch_or: 679 Op = llvm::AtomicRMWInst::Or; 680 break; 681 682 case AtomicExpr::AO__atomic_xor_fetch: 683 PostOp = llvm::Instruction::Xor; 684 [[fallthrough]]; 685 case AtomicExpr::AO__c11_atomic_fetch_xor: 686 case AtomicExpr::AO__hip_atomic_fetch_xor: 687 case AtomicExpr::AO__opencl_atomic_fetch_xor: 688 case AtomicExpr::AO__atomic_fetch_xor: 689 Op = llvm::AtomicRMWInst::Xor; 690 break; 691 692 case AtomicExpr::AO__atomic_nand_fetch: 693 PostOp = llvm::Instruction::And; // the NOT is special cased below 694 [[fallthrough]]; 695 case AtomicExpr::AO__c11_atomic_fetch_nand: 696 case AtomicExpr::AO__atomic_fetch_nand: 697 Op = llvm::AtomicRMWInst::Nand; 698 break; 699 } 700 701 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 702 llvm::AtomicRMWInst *RMWI = 703 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope); 704 RMWI->setVolatile(E->isVolatile()); 705 706 // For __atomic_*_fetch operations, perform the operation again to 707 // determine the value which was written. 708 llvm::Value *Result = RMWI; 709 if (PostOpMinMax) 710 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(), 711 E->getValueType()->isSignedIntegerType(), 712 RMWI, LoadVal1); 713 else if (PostOp) 714 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, 715 LoadVal1); 716 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 717 Result = CGF.Builder.CreateNot(Result); 718 CGF.Builder.CreateStore(Result, Dest); 719 } 720 721 // This function emits any expression (scalar, complex, or aggregate) 722 // into a temporary alloca. 723 static Address 724 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 725 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 726 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 727 /*Init*/ true); 728 return DeclPtr; 729 } 730 731 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, 732 Address Ptr, Address Val1, Address Val2, 733 llvm::Value *IsWeak, llvm::Value *FailureOrder, 734 uint64_t Size, llvm::AtomicOrdering Order, 735 llvm::Value *Scope) { 736 auto ScopeModel = Expr->getScopeModel(); 737 738 // LLVM atomic instructions always have synch scope. If clang atomic 739 // expression has no scope operand, use default LLVM synch scope. 740 if (!ScopeModel) { 741 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 742 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID("")); 743 return; 744 } 745 746 // Handle constant scope. 747 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { 748 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( 749 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()), 750 Order, CGF.CGM.getLLVMContext()); 751 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 752 Order, SCID); 753 return; 754 } 755 756 // Handle non-constant scope. 757 auto &Builder = CGF.Builder; 758 auto Scopes = ScopeModel->getRuntimeValues(); 759 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; 760 for (auto S : Scopes) 761 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); 762 763 llvm::BasicBlock *ContBB = 764 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn); 765 766 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); 767 // If unsupported synch scope is encountered at run time, assume a fallback 768 // synch scope value. 769 auto FallBack = ScopeModel->getFallBackValue(); 770 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); 771 for (auto S : Scopes) { 772 auto *B = BB[S]; 773 if (S != FallBack) 774 SI->addCase(Builder.getInt32(S), B); 775 776 Builder.SetInsertPoint(B); 777 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 778 Order, 779 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(), 780 ScopeModel->map(S), 781 Order, 782 CGF.getLLVMContext())); 783 Builder.CreateBr(ContBB); 784 } 785 786 Builder.SetInsertPoint(ContBB); 787 } 788 789 static void 790 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 791 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 792 SourceLocation Loc, CharUnits SizeInChars) { 793 if (UseOptimizedLibcall) { 794 // Load value and pass it to the function directly. 795 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); 796 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); 797 ValTy = 798 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); 799 llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits); 800 Address Ptr = Address(Val, ITy, Align); 801 Val = CGF.EmitLoadOfScalar(Ptr, false, 802 CGF.getContext().getPointerType(ValTy), 803 Loc); 804 // Coerce the value into an appropriately sized integer type. 805 Args.add(RValue::get(Val), ValTy); 806 } else { 807 // Non-optimized functions always take a reference. 808 Args.add(RValue::get(Val), CGF.getContext().VoidPtrTy); 809 } 810 } 811 812 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 813 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 814 QualType MemTy = AtomicTy; 815 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 816 MemTy = AT->getValueType(); 817 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 818 819 Address Val1 = Address::invalid(); 820 Address Val2 = Address::invalid(); 821 Address Dest = Address::invalid(); 822 Address Ptr = EmitPointerWithAlignment(E->getPtr()); 823 824 if (E->getOp() == AtomicExpr::AO__c11_atomic_init || 825 E->getOp() == AtomicExpr::AO__opencl_atomic_init) { 826 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 827 EmitAtomicInit(E->getVal1(), lvalue); 828 return RValue::get(nullptr); 829 } 830 831 auto TInfo = getContext().getTypeInfoInChars(AtomicTy); 832 uint64_t Size = TInfo.Width.getQuantity(); 833 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 834 835 bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits; 836 bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0; 837 bool UseLibcall = Misaligned | Oversized; 838 bool ShouldCastToIntPtrTy = true; 839 840 CharUnits MaxInlineWidth = 841 getContext().toCharUnitsFromBits(MaxInlineWidthInBits); 842 843 DiagnosticsEngine &Diags = CGM.getDiags(); 844 845 if (Misaligned) { 846 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned) 847 << (int)TInfo.Width.getQuantity() 848 << (int)Ptr.getAlignment().getQuantity(); 849 } 850 851 if (Oversized) { 852 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized) 853 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity(); 854 } 855 856 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 857 llvm::Value *Scope = 858 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; 859 860 switch (E->getOp()) { 861 case AtomicExpr::AO__c11_atomic_init: 862 case AtomicExpr::AO__opencl_atomic_init: 863 llvm_unreachable("Already handled above with EmitAtomicInit!"); 864 865 case AtomicExpr::AO__c11_atomic_load: 866 case AtomicExpr::AO__opencl_atomic_load: 867 case AtomicExpr::AO__hip_atomic_load: 868 case AtomicExpr::AO__atomic_load_n: 869 break; 870 871 case AtomicExpr::AO__atomic_load: 872 Dest = EmitPointerWithAlignment(E->getVal1()); 873 break; 874 875 case AtomicExpr::AO__atomic_store: 876 Val1 = EmitPointerWithAlignment(E->getVal1()); 877 break; 878 879 case AtomicExpr::AO__atomic_exchange: 880 Val1 = EmitPointerWithAlignment(E->getVal1()); 881 Dest = EmitPointerWithAlignment(E->getVal2()); 882 break; 883 884 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 885 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 886 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 887 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 888 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 889 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 890 case AtomicExpr::AO__atomic_compare_exchange_n: 891 case AtomicExpr::AO__atomic_compare_exchange: 892 Val1 = EmitPointerWithAlignment(E->getVal1()); 893 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 894 Val2 = EmitPointerWithAlignment(E->getVal2()); 895 else 896 Val2 = EmitValToTemp(*this, E->getVal2()); 897 OrderFail = EmitScalarExpr(E->getOrderFail()); 898 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || 899 E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 900 IsWeak = EmitScalarExpr(E->getWeak()); 901 break; 902 903 case AtomicExpr::AO__c11_atomic_fetch_add: 904 case AtomicExpr::AO__c11_atomic_fetch_sub: 905 case AtomicExpr::AO__hip_atomic_fetch_add: 906 case AtomicExpr::AO__hip_atomic_fetch_sub: 907 case AtomicExpr::AO__opencl_atomic_fetch_add: 908 case AtomicExpr::AO__opencl_atomic_fetch_sub: 909 if (MemTy->isPointerType()) { 910 // For pointer arithmetic, we're required to do a bit of math: 911 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 912 // ... but only for the C11 builtins. The GNU builtins expect the 913 // user to multiply by sizeof(T). 914 QualType Val1Ty = E->getVal1()->getType(); 915 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 916 CharUnits PointeeIncAmt = 917 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 918 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 919 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 920 Val1 = Temp; 921 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 922 break; 923 } 924 [[fallthrough]]; 925 case AtomicExpr::AO__atomic_fetch_add: 926 case AtomicExpr::AO__atomic_fetch_max: 927 case AtomicExpr::AO__atomic_fetch_min: 928 case AtomicExpr::AO__atomic_fetch_sub: 929 case AtomicExpr::AO__atomic_add_fetch: 930 case AtomicExpr::AO__atomic_max_fetch: 931 case AtomicExpr::AO__atomic_min_fetch: 932 case AtomicExpr::AO__atomic_sub_fetch: 933 case AtomicExpr::AO__c11_atomic_fetch_max: 934 case AtomicExpr::AO__c11_atomic_fetch_min: 935 case AtomicExpr::AO__opencl_atomic_fetch_max: 936 case AtomicExpr::AO__opencl_atomic_fetch_min: 937 case AtomicExpr::AO__hip_atomic_fetch_max: 938 case AtomicExpr::AO__hip_atomic_fetch_min: 939 ShouldCastToIntPtrTy = !MemTy->isFloatingType(); 940 [[fallthrough]]; 941 942 case AtomicExpr::AO__c11_atomic_store: 943 case AtomicExpr::AO__c11_atomic_exchange: 944 case AtomicExpr::AO__opencl_atomic_store: 945 case AtomicExpr::AO__hip_atomic_store: 946 case AtomicExpr::AO__opencl_atomic_exchange: 947 case AtomicExpr::AO__hip_atomic_exchange: 948 case AtomicExpr::AO__atomic_store_n: 949 case AtomicExpr::AO__atomic_exchange_n: 950 case AtomicExpr::AO__c11_atomic_fetch_and: 951 case AtomicExpr::AO__c11_atomic_fetch_or: 952 case AtomicExpr::AO__c11_atomic_fetch_xor: 953 case AtomicExpr::AO__c11_atomic_fetch_nand: 954 case AtomicExpr::AO__opencl_atomic_fetch_and: 955 case AtomicExpr::AO__opencl_atomic_fetch_or: 956 case AtomicExpr::AO__opencl_atomic_fetch_xor: 957 case AtomicExpr::AO__atomic_fetch_and: 958 case AtomicExpr::AO__hip_atomic_fetch_and: 959 case AtomicExpr::AO__atomic_fetch_or: 960 case AtomicExpr::AO__hip_atomic_fetch_or: 961 case AtomicExpr::AO__atomic_fetch_xor: 962 case AtomicExpr::AO__hip_atomic_fetch_xor: 963 case AtomicExpr::AO__atomic_fetch_nand: 964 case AtomicExpr::AO__atomic_and_fetch: 965 case AtomicExpr::AO__atomic_or_fetch: 966 case AtomicExpr::AO__atomic_xor_fetch: 967 case AtomicExpr::AO__atomic_nand_fetch: 968 Val1 = EmitValToTemp(*this, E->getVal1()); 969 break; 970 } 971 972 QualType RValTy = E->getType().getUnqualifiedType(); 973 974 // The inlined atomics only function on iN types, where N is a power of 2. We 975 // need to make sure (via temporaries if necessary) that all incoming values 976 // are compatible. 977 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 978 AtomicInfo Atomics(*this, AtomicVal); 979 980 if (ShouldCastToIntPtrTy) { 981 Ptr = Atomics.castToAtomicIntPointer(Ptr); 982 if (Val1.isValid()) 983 Val1 = Atomics.convertToAtomicIntPointer(Val1); 984 if (Val2.isValid()) 985 Val2 = Atomics.convertToAtomicIntPointer(Val2); 986 } 987 if (Dest.isValid()) { 988 if (ShouldCastToIntPtrTy) 989 Dest = Atomics.castToAtomicIntPointer(Dest); 990 } else if (E->isCmpXChg()) 991 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 992 else if (!RValTy->isVoidType()) { 993 Dest = Atomics.CreateTempAlloca(); 994 if (ShouldCastToIntPtrTy) 995 Dest = Atomics.castToAtomicIntPointer(Dest); 996 } 997 998 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 999 if (UseLibcall) { 1000 bool UseOptimizedLibcall = false; 1001 switch (E->getOp()) { 1002 case AtomicExpr::AO__c11_atomic_init: 1003 case AtomicExpr::AO__opencl_atomic_init: 1004 llvm_unreachable("Already handled above with EmitAtomicInit!"); 1005 1006 case AtomicExpr::AO__c11_atomic_fetch_add: 1007 case AtomicExpr::AO__opencl_atomic_fetch_add: 1008 case AtomicExpr::AO__atomic_fetch_add: 1009 case AtomicExpr::AO__hip_atomic_fetch_add: 1010 case AtomicExpr::AO__c11_atomic_fetch_and: 1011 case AtomicExpr::AO__opencl_atomic_fetch_and: 1012 case AtomicExpr::AO__hip_atomic_fetch_and: 1013 case AtomicExpr::AO__atomic_fetch_and: 1014 case AtomicExpr::AO__c11_atomic_fetch_or: 1015 case AtomicExpr::AO__opencl_atomic_fetch_or: 1016 case AtomicExpr::AO__hip_atomic_fetch_or: 1017 case AtomicExpr::AO__atomic_fetch_or: 1018 case AtomicExpr::AO__c11_atomic_fetch_nand: 1019 case AtomicExpr::AO__atomic_fetch_nand: 1020 case AtomicExpr::AO__c11_atomic_fetch_sub: 1021 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1022 case AtomicExpr::AO__atomic_fetch_sub: 1023 case AtomicExpr::AO__hip_atomic_fetch_sub: 1024 case AtomicExpr::AO__c11_atomic_fetch_xor: 1025 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1026 case AtomicExpr::AO__opencl_atomic_fetch_min: 1027 case AtomicExpr::AO__opencl_atomic_fetch_max: 1028 case AtomicExpr::AO__atomic_fetch_xor: 1029 case AtomicExpr::AO__hip_atomic_fetch_xor: 1030 case AtomicExpr::AO__c11_atomic_fetch_max: 1031 case AtomicExpr::AO__c11_atomic_fetch_min: 1032 case AtomicExpr::AO__atomic_add_fetch: 1033 case AtomicExpr::AO__atomic_and_fetch: 1034 case AtomicExpr::AO__atomic_nand_fetch: 1035 case AtomicExpr::AO__atomic_or_fetch: 1036 case AtomicExpr::AO__atomic_sub_fetch: 1037 case AtomicExpr::AO__atomic_xor_fetch: 1038 case AtomicExpr::AO__atomic_fetch_max: 1039 case AtomicExpr::AO__hip_atomic_fetch_max: 1040 case AtomicExpr::AO__atomic_fetch_min: 1041 case AtomicExpr::AO__hip_atomic_fetch_min: 1042 case AtomicExpr::AO__atomic_max_fetch: 1043 case AtomicExpr::AO__atomic_min_fetch: 1044 // For these, only library calls for certain sizes exist. 1045 UseOptimizedLibcall = true; 1046 break; 1047 1048 case AtomicExpr::AO__atomic_load: 1049 case AtomicExpr::AO__atomic_store: 1050 case AtomicExpr::AO__atomic_exchange: 1051 case AtomicExpr::AO__atomic_compare_exchange: 1052 // Use the generic version if we don't know that the operand will be 1053 // suitably aligned for the optimized version. 1054 if (Misaligned) 1055 break; 1056 [[fallthrough]]; 1057 case AtomicExpr::AO__c11_atomic_load: 1058 case AtomicExpr::AO__c11_atomic_store: 1059 case AtomicExpr::AO__c11_atomic_exchange: 1060 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 1061 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 1062 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 1063 case AtomicExpr::AO__opencl_atomic_load: 1064 case AtomicExpr::AO__hip_atomic_load: 1065 case AtomicExpr::AO__opencl_atomic_store: 1066 case AtomicExpr::AO__hip_atomic_store: 1067 case AtomicExpr::AO__opencl_atomic_exchange: 1068 case AtomicExpr::AO__hip_atomic_exchange: 1069 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 1070 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 1071 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 1072 case AtomicExpr::AO__atomic_load_n: 1073 case AtomicExpr::AO__atomic_store_n: 1074 case AtomicExpr::AO__atomic_exchange_n: 1075 case AtomicExpr::AO__atomic_compare_exchange_n: 1076 // Only use optimized library calls for sizes for which they exist. 1077 // FIXME: Size == 16 optimized library functions exist too. 1078 if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 1079 UseOptimizedLibcall = true; 1080 break; 1081 } 1082 1083 CallArgList Args; 1084 if (!UseOptimizedLibcall) { 1085 // For non-optimized library calls, the size is the first parameter 1086 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 1087 getContext().getSizeType()); 1088 } 1089 // Atomic address is the first or second parameter 1090 // The OpenCL atomic library functions only accept pointer arguments to 1091 // generic address space. 1092 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { 1093 if (!E->isOpenCL()) 1094 return V; 1095 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace(); 1096 if (AS == LangAS::opencl_generic) 1097 return V; 1098 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); 1099 auto *DestType = llvm::PointerType::get(getLLVMContext(), DestAS); 1100 1101 return getTargetHooks().performAddrSpaceCast( 1102 *this, V, AS, LangAS::opencl_generic, DestType, false); 1103 }; 1104 1105 Args.add(RValue::get(CastToGenericAddrSpace(Ptr.getPointer(), 1106 E->getPtr()->getType())), 1107 getContext().VoidPtrTy); 1108 1109 std::string LibCallName; 1110 QualType LoweredMemTy = 1111 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 1112 QualType RetTy; 1113 bool HaveRetTy = false; 1114 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 1115 bool PostOpMinMax = false; 1116 switch (E->getOp()) { 1117 case AtomicExpr::AO__c11_atomic_init: 1118 case AtomicExpr::AO__opencl_atomic_init: 1119 llvm_unreachable("Already handled!"); 1120 1121 // There is only one libcall for compare an exchange, because there is no 1122 // optimisation benefit possible from a libcall version of a weak compare 1123 // and exchange. 1124 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 1125 // void *desired, int success, int failure) 1126 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 1127 // int success, int failure) 1128 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 1129 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 1130 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 1131 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 1132 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 1133 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 1134 case AtomicExpr::AO__atomic_compare_exchange: 1135 case AtomicExpr::AO__atomic_compare_exchange_n: 1136 LibCallName = "__atomic_compare_exchange"; 1137 RetTy = getContext().BoolTy; 1138 HaveRetTy = true; 1139 Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(), 1140 E->getVal1()->getType())), 1141 getContext().VoidPtrTy); 1142 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), 1143 MemTy, E->getExprLoc(), TInfo.Width); 1144 Args.add(RValue::get(Order), getContext().IntTy); 1145 Order = OrderFail; 1146 break; 1147 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 1148 // int order) 1149 // T __atomic_exchange_N(T *mem, T val, int order) 1150 case AtomicExpr::AO__c11_atomic_exchange: 1151 case AtomicExpr::AO__opencl_atomic_exchange: 1152 case AtomicExpr::AO__atomic_exchange_n: 1153 case AtomicExpr::AO__atomic_exchange: 1154 case AtomicExpr::AO__hip_atomic_exchange: 1155 LibCallName = "__atomic_exchange"; 1156 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1157 MemTy, E->getExprLoc(), TInfo.Width); 1158 break; 1159 // void __atomic_store(size_t size, void *mem, void *val, int order) 1160 // void __atomic_store_N(T *mem, T val, int order) 1161 case AtomicExpr::AO__c11_atomic_store: 1162 case AtomicExpr::AO__opencl_atomic_store: 1163 case AtomicExpr::AO__hip_atomic_store: 1164 case AtomicExpr::AO__atomic_store: 1165 case AtomicExpr::AO__atomic_store_n: 1166 LibCallName = "__atomic_store"; 1167 RetTy = getContext().VoidTy; 1168 HaveRetTy = true; 1169 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1170 MemTy, E->getExprLoc(), TInfo.Width); 1171 break; 1172 // void __atomic_load(size_t size, void *mem, void *return, int order) 1173 // T __atomic_load_N(T *mem, int order) 1174 case AtomicExpr::AO__c11_atomic_load: 1175 case AtomicExpr::AO__opencl_atomic_load: 1176 case AtomicExpr::AO__hip_atomic_load: 1177 case AtomicExpr::AO__atomic_load: 1178 case AtomicExpr::AO__atomic_load_n: 1179 LibCallName = "__atomic_load"; 1180 break; 1181 // T __atomic_add_fetch_N(T *mem, T val, int order) 1182 // T __atomic_fetch_add_N(T *mem, T val, int order) 1183 case AtomicExpr::AO__atomic_add_fetch: 1184 PostOp = llvm::Instruction::Add; 1185 [[fallthrough]]; 1186 case AtomicExpr::AO__c11_atomic_fetch_add: 1187 case AtomicExpr::AO__opencl_atomic_fetch_add: 1188 case AtomicExpr::AO__atomic_fetch_add: 1189 case AtomicExpr::AO__hip_atomic_fetch_add: 1190 LibCallName = "__atomic_fetch_add"; 1191 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1192 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1193 break; 1194 // T __atomic_and_fetch_N(T *mem, T val, int order) 1195 // T __atomic_fetch_and_N(T *mem, T val, int order) 1196 case AtomicExpr::AO__atomic_and_fetch: 1197 PostOp = llvm::Instruction::And; 1198 [[fallthrough]]; 1199 case AtomicExpr::AO__c11_atomic_fetch_and: 1200 case AtomicExpr::AO__opencl_atomic_fetch_and: 1201 case AtomicExpr::AO__hip_atomic_fetch_and: 1202 case AtomicExpr::AO__atomic_fetch_and: 1203 LibCallName = "__atomic_fetch_and"; 1204 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1205 MemTy, E->getExprLoc(), TInfo.Width); 1206 break; 1207 // T __atomic_or_fetch_N(T *mem, T val, int order) 1208 // T __atomic_fetch_or_N(T *mem, T val, int order) 1209 case AtomicExpr::AO__atomic_or_fetch: 1210 PostOp = llvm::Instruction::Or; 1211 [[fallthrough]]; 1212 case AtomicExpr::AO__c11_atomic_fetch_or: 1213 case AtomicExpr::AO__opencl_atomic_fetch_or: 1214 case AtomicExpr::AO__hip_atomic_fetch_or: 1215 case AtomicExpr::AO__atomic_fetch_or: 1216 LibCallName = "__atomic_fetch_or"; 1217 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1218 MemTy, E->getExprLoc(), TInfo.Width); 1219 break; 1220 // T __atomic_sub_fetch_N(T *mem, T val, int order) 1221 // T __atomic_fetch_sub_N(T *mem, T val, int order) 1222 case AtomicExpr::AO__atomic_sub_fetch: 1223 PostOp = llvm::Instruction::Sub; 1224 [[fallthrough]]; 1225 case AtomicExpr::AO__c11_atomic_fetch_sub: 1226 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1227 case AtomicExpr::AO__hip_atomic_fetch_sub: 1228 case AtomicExpr::AO__atomic_fetch_sub: 1229 LibCallName = "__atomic_fetch_sub"; 1230 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1231 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1232 break; 1233 // T __atomic_xor_fetch_N(T *mem, T val, int order) 1234 // T __atomic_fetch_xor_N(T *mem, T val, int order) 1235 case AtomicExpr::AO__atomic_xor_fetch: 1236 PostOp = llvm::Instruction::Xor; 1237 [[fallthrough]]; 1238 case AtomicExpr::AO__c11_atomic_fetch_xor: 1239 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1240 case AtomicExpr::AO__hip_atomic_fetch_xor: 1241 case AtomicExpr::AO__atomic_fetch_xor: 1242 LibCallName = "__atomic_fetch_xor"; 1243 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1244 MemTy, E->getExprLoc(), TInfo.Width); 1245 break; 1246 case AtomicExpr::AO__atomic_min_fetch: 1247 PostOpMinMax = true; 1248 [[fallthrough]]; 1249 case AtomicExpr::AO__c11_atomic_fetch_min: 1250 case AtomicExpr::AO__atomic_fetch_min: 1251 case AtomicExpr::AO__hip_atomic_fetch_min: 1252 case AtomicExpr::AO__opencl_atomic_fetch_min: 1253 LibCallName = E->getValueType()->isSignedIntegerType() 1254 ? "__atomic_fetch_min" 1255 : "__atomic_fetch_umin"; 1256 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1257 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1258 break; 1259 case AtomicExpr::AO__atomic_max_fetch: 1260 PostOpMinMax = true; 1261 [[fallthrough]]; 1262 case AtomicExpr::AO__c11_atomic_fetch_max: 1263 case AtomicExpr::AO__atomic_fetch_max: 1264 case AtomicExpr::AO__hip_atomic_fetch_max: 1265 case AtomicExpr::AO__opencl_atomic_fetch_max: 1266 LibCallName = E->getValueType()->isSignedIntegerType() 1267 ? "__atomic_fetch_max" 1268 : "__atomic_fetch_umax"; 1269 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1270 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1271 break; 1272 // T __atomic_nand_fetch_N(T *mem, T val, int order) 1273 // T __atomic_fetch_nand_N(T *mem, T val, int order) 1274 case AtomicExpr::AO__atomic_nand_fetch: 1275 PostOp = llvm::Instruction::And; // the NOT is special cased below 1276 [[fallthrough]]; 1277 case AtomicExpr::AO__c11_atomic_fetch_nand: 1278 case AtomicExpr::AO__atomic_fetch_nand: 1279 LibCallName = "__atomic_fetch_nand"; 1280 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1281 MemTy, E->getExprLoc(), TInfo.Width); 1282 break; 1283 } 1284 1285 if (E->isOpenCL()) { 1286 LibCallName = std::string("__opencl") + 1287 StringRef(LibCallName).drop_front(1).str(); 1288 1289 } 1290 // Optimized functions have the size in their name. 1291 if (UseOptimizedLibcall) 1292 LibCallName += "_" + llvm::utostr(Size); 1293 // By default, assume we return a value of the atomic type. 1294 if (!HaveRetTy) { 1295 if (UseOptimizedLibcall) { 1296 // Value is returned directly. 1297 // The function returns an appropriately sized integer type. 1298 RetTy = getContext().getIntTypeForBitwidth( 1299 getContext().toBits(TInfo.Width), /*Signed=*/false); 1300 } else { 1301 // Value is returned through parameter before the order. 1302 RetTy = getContext().VoidTy; 1303 Args.add(RValue::get(Dest.getPointer()), getContext().VoidPtrTy); 1304 } 1305 } 1306 // order is always the last parameter 1307 Args.add(RValue::get(Order), 1308 getContext().IntTy); 1309 if (E->isOpenCL()) 1310 Args.add(RValue::get(Scope), getContext().IntTy); 1311 1312 // PostOp is only needed for the atomic_*_fetch operations, and 1313 // thus is only needed for and implemented in the 1314 // UseOptimizedLibcall codepath. 1315 assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax)); 1316 1317 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 1318 // The value is returned directly from the libcall. 1319 if (E->isCmpXChg()) 1320 return Res; 1321 1322 // The value is returned directly for optimized libcalls but the expr 1323 // provided an out-param. 1324 if (UseOptimizedLibcall && Res.getScalarVal()) { 1325 llvm::Value *ResVal = Res.getScalarVal(); 1326 if (PostOpMinMax) { 1327 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1328 ResVal = EmitPostAtomicMinMax(Builder, E->getOp(), 1329 E->getValueType()->isSignedIntegerType(), 1330 ResVal, LoadVal1); 1331 } else if (PostOp) { 1332 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1333 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); 1334 } 1335 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 1336 ResVal = Builder.CreateNot(ResVal); 1337 1338 Builder.CreateStore(ResVal, Dest.withElementType(ResVal->getType())); 1339 } 1340 1341 if (RValTy->isVoidType()) 1342 return RValue::get(nullptr); 1343 1344 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), 1345 RValTy, E->getExprLoc()); 1346 } 1347 1348 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1349 E->getOp() == AtomicExpr::AO__opencl_atomic_store || 1350 E->getOp() == AtomicExpr::AO__hip_atomic_store || 1351 E->getOp() == AtomicExpr::AO__atomic_store || 1352 E->getOp() == AtomicExpr::AO__atomic_store_n; 1353 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1354 E->getOp() == AtomicExpr::AO__opencl_atomic_load || 1355 E->getOp() == AtomicExpr::AO__hip_atomic_load || 1356 E->getOp() == AtomicExpr::AO__atomic_load || 1357 E->getOp() == AtomicExpr::AO__atomic_load_n; 1358 1359 if (isa<llvm::ConstantInt>(Order)) { 1360 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1361 // We should not ever get to a case where the ordering isn't a valid C ABI 1362 // value, but it's hard to enforce that in general. 1363 if (llvm::isValidAtomicOrderingCABI(ord)) 1364 switch ((llvm::AtomicOrderingCABI)ord) { 1365 case llvm::AtomicOrderingCABI::relaxed: 1366 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1367 llvm::AtomicOrdering::Monotonic, Scope); 1368 break; 1369 case llvm::AtomicOrderingCABI::consume: 1370 case llvm::AtomicOrderingCABI::acquire: 1371 if (IsStore) 1372 break; // Avoid crashing on code with undefined behavior 1373 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1374 llvm::AtomicOrdering::Acquire, Scope); 1375 break; 1376 case llvm::AtomicOrderingCABI::release: 1377 if (IsLoad) 1378 break; // Avoid crashing on code with undefined behavior 1379 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1380 llvm::AtomicOrdering::Release, Scope); 1381 break; 1382 case llvm::AtomicOrderingCABI::acq_rel: 1383 if (IsLoad || IsStore) 1384 break; // Avoid crashing on code with undefined behavior 1385 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1386 llvm::AtomicOrdering::AcquireRelease, Scope); 1387 break; 1388 case llvm::AtomicOrderingCABI::seq_cst: 1389 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1390 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1391 break; 1392 } 1393 if (RValTy->isVoidType()) 1394 return RValue::get(nullptr); 1395 1396 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), 1397 RValTy, E->getExprLoc()); 1398 } 1399 1400 // Long case, when Order isn't obviously constant. 1401 1402 // Create all the relevant BB's 1403 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1404 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1405 *SeqCstBB = nullptr; 1406 MonotonicBB = createBasicBlock("monotonic", CurFn); 1407 if (!IsStore) 1408 AcquireBB = createBasicBlock("acquire", CurFn); 1409 if (!IsLoad) 1410 ReleaseBB = createBasicBlock("release", CurFn); 1411 if (!IsLoad && !IsStore) 1412 AcqRelBB = createBasicBlock("acqrel", CurFn); 1413 SeqCstBB = createBasicBlock("seqcst", CurFn); 1414 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1415 1416 // Create the switch for the split 1417 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1418 // doesn't matter unless someone is crazy enough to use something that 1419 // doesn't fold to a constant for the ordering. 1420 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1421 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1422 1423 // Emit all the different atomics 1424 Builder.SetInsertPoint(MonotonicBB); 1425 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1426 llvm::AtomicOrdering::Monotonic, Scope); 1427 Builder.CreateBr(ContBB); 1428 if (!IsStore) { 1429 Builder.SetInsertPoint(AcquireBB); 1430 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1431 llvm::AtomicOrdering::Acquire, Scope); 1432 Builder.CreateBr(ContBB); 1433 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1434 AcquireBB); 1435 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1436 AcquireBB); 1437 } 1438 if (!IsLoad) { 1439 Builder.SetInsertPoint(ReleaseBB); 1440 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1441 llvm::AtomicOrdering::Release, Scope); 1442 Builder.CreateBr(ContBB); 1443 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1444 ReleaseBB); 1445 } 1446 if (!IsLoad && !IsStore) { 1447 Builder.SetInsertPoint(AcqRelBB); 1448 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1449 llvm::AtomicOrdering::AcquireRelease, Scope); 1450 Builder.CreateBr(ContBB); 1451 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1452 AcqRelBB); 1453 } 1454 Builder.SetInsertPoint(SeqCstBB); 1455 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1456 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1457 Builder.CreateBr(ContBB); 1458 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1459 SeqCstBB); 1460 1461 // Cleanup and return 1462 Builder.SetInsertPoint(ContBB); 1463 if (RValTy->isVoidType()) 1464 return RValue::get(nullptr); 1465 1466 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1467 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), 1468 RValTy, E->getExprLoc()); 1469 } 1470 1471 Address AtomicInfo::castToAtomicIntPointer(Address addr) const { 1472 llvm::IntegerType *ty = 1473 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1474 return addr.withElementType(ty); 1475 } 1476 1477 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1478 llvm::Type *Ty = Addr.getElementType(); 1479 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1480 if (SourceSizeInBits != AtomicSizeInBits) { 1481 Address Tmp = CreateTempAlloca(); 1482 CGF.Builder.CreateMemCpy(Tmp, Addr, 1483 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1484 Addr = Tmp; 1485 } 1486 1487 return castToAtomicIntPointer(Addr); 1488 } 1489 1490 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1491 AggValueSlot resultSlot, 1492 SourceLocation loc, 1493 bool asValue) const { 1494 if (LVal.isSimple()) { 1495 if (EvaluationKind == TEK_Aggregate) 1496 return resultSlot.asRValue(); 1497 1498 // Drill into the padding structure if we have one. 1499 if (hasPadding()) 1500 addr = CGF.Builder.CreateStructGEP(addr, 0); 1501 1502 // Otherwise, just convert the temporary to an r-value using the 1503 // normal conversion routine. 1504 return CGF.convertTempToRValue(addr, getValueType(), loc); 1505 } 1506 if (!asValue) 1507 // Get RValue from temp memory as atomic for non-simple lvalues 1508 return RValue::get(CGF.Builder.CreateLoad(addr)); 1509 if (LVal.isBitField()) 1510 return CGF.EmitLoadOfBitfieldLValue( 1511 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1512 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1513 if (LVal.isVectorElt()) 1514 return CGF.EmitLoadOfLValue( 1515 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1516 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1517 assert(LVal.isExtVectorElt()); 1518 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1519 addr, LVal.getExtVectorElts(), LVal.getType(), 1520 LVal.getBaseInfo(), TBAAAccessInfo())); 1521 } 1522 1523 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, 1524 AggValueSlot ResultSlot, 1525 SourceLocation Loc, 1526 bool AsValue) const { 1527 // Try not to in some easy cases. 1528 assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); 1529 if (getEvaluationKind() == TEK_Scalar && 1530 (((!LVal.isBitField() || 1531 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1532 !hasPadding()) || 1533 !AsValue)) { 1534 auto *ValTy = AsValue 1535 ? CGF.ConvertTypeForMem(ValueTy) 1536 : getAtomicAddress().getElementType(); 1537 if (ValTy->isIntegerTy()) { 1538 assert(IntVal->getType() == ValTy && "Different integer types."); 1539 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); 1540 } else if (ValTy->isPointerTy()) 1541 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); 1542 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) 1543 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); 1544 } 1545 1546 // Create a temporary. This needs to be big enough to hold the 1547 // atomic integer. 1548 Address Temp = Address::invalid(); 1549 bool TempIsVolatile = false; 1550 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1551 assert(!ResultSlot.isIgnored()); 1552 Temp = ResultSlot.getAddress(); 1553 TempIsVolatile = ResultSlot.isVolatile(); 1554 } else { 1555 Temp = CreateTempAlloca(); 1556 } 1557 1558 // Slam the integer into the temporary. 1559 Address CastTemp = castToAtomicIntPointer(Temp); 1560 CGF.Builder.CreateStore(IntVal, CastTemp) 1561 ->setVolatile(TempIsVolatile); 1562 1563 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1564 } 1565 1566 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1567 llvm::AtomicOrdering AO, bool) { 1568 // void __atomic_load(size_t size, void *mem, void *return, int order); 1569 CallArgList Args; 1570 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1571 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy); 1572 Args.add(RValue::get(AddForLoaded), CGF.getContext().VoidPtrTy); 1573 Args.add( 1574 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1575 CGF.getContext().IntTy); 1576 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1577 } 1578 1579 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1580 bool IsVolatile) { 1581 // Okay, we're doing this natively. 1582 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1583 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1584 Load->setAtomic(AO); 1585 1586 // Other decoration. 1587 if (IsVolatile) 1588 Load->setVolatile(true); 1589 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1590 return Load; 1591 } 1592 1593 /// An LValue is a candidate for having its loads and stores be made atomic if 1594 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1595 /// performing such an operation can be performed without a libcall. 1596 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1597 if (!CGM.getLangOpts().MSVolatile) return false; 1598 AtomicInfo AI(*this, LV); 1599 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1600 // An atomic is inline if we don't need to use a libcall. 1601 bool AtomicIsInline = !AI.shouldUseLibcall(); 1602 // MSVC doesn't seem to do this for types wider than a pointer. 1603 if (getContext().getTypeSize(LV.getType()) > 1604 getContext().getTypeSize(getContext().getIntPtrType())) 1605 return false; 1606 return IsVolatile && AtomicIsInline; 1607 } 1608 1609 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1610 AggValueSlot Slot) { 1611 llvm::AtomicOrdering AO; 1612 bool IsVolatile = LV.isVolatileQualified(); 1613 if (LV.getType()->isAtomicType()) { 1614 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1615 } else { 1616 AO = llvm::AtomicOrdering::Acquire; 1617 IsVolatile = true; 1618 } 1619 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1620 } 1621 1622 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1623 bool AsValue, llvm::AtomicOrdering AO, 1624 bool IsVolatile) { 1625 // Check whether we should use a library call. 1626 if (shouldUseLibcall()) { 1627 Address TempAddr = Address::invalid(); 1628 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1629 assert(getEvaluationKind() == TEK_Aggregate); 1630 TempAddr = ResultSlot.getAddress(); 1631 } else 1632 TempAddr = CreateTempAlloca(); 1633 1634 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); 1635 1636 // Okay, turn that back into the original value or whole atomic (for 1637 // non-simple lvalues) type. 1638 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1639 } 1640 1641 // Okay, we're doing this natively. 1642 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1643 1644 // If we're ignoring an aggregate return, don't do anything. 1645 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1646 return RValue::getAggregate(Address::invalid(), false); 1647 1648 // Okay, turn that back into the original value or atomic (for non-simple 1649 // lvalues) type. 1650 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1651 } 1652 1653 /// Emit a load from an l-value of atomic type. Note that the r-value 1654 /// we produce is an r-value of the atomic *value* type. 1655 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1656 llvm::AtomicOrdering AO, bool IsVolatile, 1657 AggValueSlot resultSlot) { 1658 AtomicInfo Atomics(*this, src); 1659 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1660 IsVolatile); 1661 } 1662 1663 /// Copy an r-value into memory as part of storing to an atomic type. 1664 /// This needs to create a bit-pattern suitable for atomic operations. 1665 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1666 assert(LVal.isSimple()); 1667 // If we have an r-value, the rvalue should be of the atomic type, 1668 // which means that the caller is responsible for having zeroed 1669 // any padding. Just do an aggregate copy of that type. 1670 if (rvalue.isAggregate()) { 1671 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); 1672 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), 1673 getAtomicType()); 1674 bool IsVolatile = rvalue.isVolatileQualified() || 1675 LVal.isVolatileQualified(); 1676 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), 1677 AggValueSlot::DoesNotOverlap, IsVolatile); 1678 return; 1679 } 1680 1681 // Okay, otherwise we're copying stuff. 1682 1683 // Zero out the buffer if necessary. 1684 emitMemSetZeroIfNecessary(); 1685 1686 // Drill past the padding if present. 1687 LValue TempLVal = projectValue(); 1688 1689 // Okay, store the rvalue in. 1690 if (rvalue.isScalar()) { 1691 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1692 } else { 1693 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1694 } 1695 } 1696 1697 1698 /// Materialize an r-value into memory for the purposes of storing it 1699 /// to an atomic type. 1700 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1701 // Aggregate r-values are already in memory, and EmitAtomicStore 1702 // requires them to be values of the atomic type. 1703 if (rvalue.isAggregate()) 1704 return rvalue.getAggregateAddress(); 1705 1706 // Otherwise, make a temporary and materialize into it. 1707 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1708 AtomicInfo Atomics(CGF, TempLV); 1709 Atomics.emitCopyIntoMemory(rvalue); 1710 return TempLV.getAddress(CGF); 1711 } 1712 1713 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { 1714 // If we've got a scalar value of the right size, try to avoid going 1715 // through memory. 1716 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { 1717 llvm::Value *Value = RVal.getScalarVal(); 1718 if (isa<llvm::IntegerType>(Value->getType())) 1719 return CGF.EmitToMemory(Value, ValueTy); 1720 else { 1721 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1722 CGF.getLLVMContext(), 1723 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1724 if (isa<llvm::PointerType>(Value->getType())) 1725 return CGF.Builder.CreatePtrToInt(Value, InputIntTy); 1726 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1727 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1728 } 1729 } 1730 // Otherwise, we need to go through memory. 1731 // Put the r-value in memory. 1732 Address Addr = materializeRValue(RVal); 1733 1734 // Cast the temporary to the atomic int type and pull a value out. 1735 Addr = castToAtomicIntPointer(Addr); 1736 return CGF.Builder.CreateLoad(Addr); 1737 } 1738 1739 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1740 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1741 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1742 // Do the atomic store. 1743 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1744 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), 1745 ExpectedVal, DesiredVal, 1746 Success, Failure); 1747 // Other decoration. 1748 Inst->setVolatile(LVal.isVolatileQualified()); 1749 Inst->setWeak(IsWeak); 1750 1751 // Okay, turn that back into the original value type. 1752 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1753 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1754 return std::make_pair(PreviousVal, SuccessFailureVal); 1755 } 1756 1757 llvm::Value * 1758 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1759 llvm::Value *DesiredAddr, 1760 llvm::AtomicOrdering Success, 1761 llvm::AtomicOrdering Failure) { 1762 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1763 // void *desired, int success, int failure); 1764 CallArgList Args; 1765 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1766 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy); 1767 Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy); 1768 Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy); 1769 Args.add(RValue::get( 1770 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1771 CGF.getContext().IntTy); 1772 Args.add(RValue::get( 1773 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1774 CGF.getContext().IntTy); 1775 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1776 CGF.getContext().BoolTy, Args); 1777 1778 return SuccessFailureRVal.getScalarVal(); 1779 } 1780 1781 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1782 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1783 llvm::AtomicOrdering Failure, bool IsWeak) { 1784 // Check whether we should use a library call. 1785 if (shouldUseLibcall()) { 1786 // Produce a source address. 1787 Address ExpectedAddr = materializeRValue(Expected); 1788 Address DesiredAddr = materializeRValue(Desired); 1789 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1790 DesiredAddr.getPointer(), 1791 Success, Failure); 1792 return std::make_pair( 1793 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1794 SourceLocation(), /*AsValue=*/false), 1795 Res); 1796 } 1797 1798 // If we've got a scalar value of the right size, try to avoid going 1799 // through memory. 1800 auto *ExpectedVal = convertRValueToInt(Expected); 1801 auto *DesiredVal = convertRValueToInt(Desired); 1802 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1803 Failure, IsWeak); 1804 return std::make_pair( 1805 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1806 SourceLocation(), /*AsValue=*/false), 1807 Res.second); 1808 } 1809 1810 static void 1811 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1812 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1813 Address DesiredAddr) { 1814 RValue UpRVal; 1815 LValue AtomicLVal = Atomics.getAtomicLValue(); 1816 LValue DesiredLVal; 1817 if (AtomicLVal.isSimple()) { 1818 UpRVal = OldRVal; 1819 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1820 } else { 1821 // Build new lvalue for temp address. 1822 Address Ptr = Atomics.materializeRValue(OldRVal); 1823 LValue UpdateLVal; 1824 if (AtomicLVal.isBitField()) { 1825 UpdateLVal = 1826 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1827 AtomicLVal.getType(), 1828 AtomicLVal.getBaseInfo(), 1829 AtomicLVal.getTBAAInfo()); 1830 DesiredLVal = 1831 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1832 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1833 AtomicLVal.getTBAAInfo()); 1834 } else if (AtomicLVal.isVectorElt()) { 1835 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1836 AtomicLVal.getType(), 1837 AtomicLVal.getBaseInfo(), 1838 AtomicLVal.getTBAAInfo()); 1839 DesiredLVal = LValue::MakeVectorElt( 1840 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1841 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1842 } else { 1843 assert(AtomicLVal.isExtVectorElt()); 1844 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1845 AtomicLVal.getType(), 1846 AtomicLVal.getBaseInfo(), 1847 AtomicLVal.getTBAAInfo()); 1848 DesiredLVal = LValue::MakeExtVectorElt( 1849 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1850 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1851 } 1852 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1853 } 1854 // Store new value in the corresponding memory area. 1855 RValue NewRVal = UpdateOp(UpRVal); 1856 if (NewRVal.isScalar()) { 1857 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1858 } else { 1859 assert(NewRVal.isComplex()); 1860 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1861 /*isInit=*/false); 1862 } 1863 } 1864 1865 void AtomicInfo::EmitAtomicUpdateLibcall( 1866 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1867 bool IsVolatile) { 1868 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1869 1870 Address ExpectedAddr = CreateTempAlloca(); 1871 1872 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1873 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1874 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1875 CGF.EmitBlock(ContBB); 1876 Address DesiredAddr = CreateTempAlloca(); 1877 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1878 requiresMemSetZero(getAtomicAddress().getElementType())) { 1879 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1880 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1881 } 1882 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1883 AggValueSlot::ignored(), 1884 SourceLocation(), /*AsValue=*/false); 1885 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1886 auto *Res = 1887 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1888 DesiredAddr.getPointer(), 1889 AO, Failure); 1890 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1891 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1892 } 1893 1894 void AtomicInfo::EmitAtomicUpdateOp( 1895 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1896 bool IsVolatile) { 1897 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1898 1899 // Do the atomic load. 1900 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); 1901 // For non-simple lvalues perform compare-and-swap procedure. 1902 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1903 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1904 auto *CurBB = CGF.Builder.GetInsertBlock(); 1905 CGF.EmitBlock(ContBB); 1906 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1907 /*NumReservedValues=*/2); 1908 PHI->addIncoming(OldVal, CurBB); 1909 Address NewAtomicAddr = CreateTempAlloca(); 1910 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr); 1911 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1912 requiresMemSetZero(getAtomicAddress().getElementType())) { 1913 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1914 } 1915 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), 1916 SourceLocation(), /*AsValue=*/false); 1917 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1918 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1919 // Try to write new value using cmpxchg operation. 1920 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1921 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1922 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1923 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1924 } 1925 1926 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1927 RValue UpdateRVal, Address DesiredAddr) { 1928 LValue AtomicLVal = Atomics.getAtomicLValue(); 1929 LValue DesiredLVal; 1930 // Build new lvalue for temp address. 1931 if (AtomicLVal.isBitField()) { 1932 DesiredLVal = 1933 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1934 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1935 AtomicLVal.getTBAAInfo()); 1936 } else if (AtomicLVal.isVectorElt()) { 1937 DesiredLVal = 1938 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1939 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1940 AtomicLVal.getTBAAInfo()); 1941 } else { 1942 assert(AtomicLVal.isExtVectorElt()); 1943 DesiredLVal = LValue::MakeExtVectorElt( 1944 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1945 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1946 } 1947 // Store new value in the corresponding memory area. 1948 assert(UpdateRVal.isScalar()); 1949 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1950 } 1951 1952 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1953 RValue UpdateRVal, bool IsVolatile) { 1954 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1955 1956 Address ExpectedAddr = CreateTempAlloca(); 1957 1958 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1959 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1960 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1961 CGF.EmitBlock(ContBB); 1962 Address DesiredAddr = CreateTempAlloca(); 1963 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1964 requiresMemSetZero(getAtomicAddress().getElementType())) { 1965 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1966 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1967 } 1968 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1969 auto *Res = 1970 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1971 DesiredAddr.getPointer(), 1972 AO, Failure); 1973 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1974 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1975 } 1976 1977 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1978 bool IsVolatile) { 1979 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1980 1981 // Do the atomic load. 1982 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); 1983 // For non-simple lvalues perform compare-and-swap procedure. 1984 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1985 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1986 auto *CurBB = CGF.Builder.GetInsertBlock(); 1987 CGF.EmitBlock(ContBB); 1988 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1989 /*NumReservedValues=*/2); 1990 PHI->addIncoming(OldVal, CurBB); 1991 Address NewAtomicAddr = CreateTempAlloca(); 1992 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr); 1993 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1994 requiresMemSetZero(getAtomicAddress().getElementType())) { 1995 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1996 } 1997 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 1998 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1999 // Try to write new value using cmpxchg operation. 2000 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 2001 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 2002 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 2003 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 2004 } 2005 2006 void AtomicInfo::EmitAtomicUpdate( 2007 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 2008 bool IsVolatile) { 2009 if (shouldUseLibcall()) { 2010 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 2011 } else { 2012 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 2013 } 2014 } 2015 2016 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 2017 bool IsVolatile) { 2018 if (shouldUseLibcall()) { 2019 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 2020 } else { 2021 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 2022 } 2023 } 2024 2025 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 2026 bool isInit) { 2027 bool IsVolatile = lvalue.isVolatileQualified(); 2028 llvm::AtomicOrdering AO; 2029 if (lvalue.getType()->isAtomicType()) { 2030 AO = llvm::AtomicOrdering::SequentiallyConsistent; 2031 } else { 2032 AO = llvm::AtomicOrdering::Release; 2033 IsVolatile = true; 2034 } 2035 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 2036 } 2037 2038 /// Emit a store to an l-value of atomic type. 2039 /// 2040 /// Note that the r-value is expected to be an r-value *of the atomic 2041 /// type*; this means that for aggregate r-values, it should include 2042 /// storage for any padding that was necessary. 2043 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 2044 llvm::AtomicOrdering AO, bool IsVolatile, 2045 bool isInit) { 2046 // If this is an aggregate r-value, it should agree in type except 2047 // maybe for address-space qualification. 2048 assert(!rvalue.isAggregate() || 2049 rvalue.getAggregateAddress().getElementType() == 2050 dest.getAddress(*this).getElementType()); 2051 2052 AtomicInfo atomics(*this, dest); 2053 LValue LVal = atomics.getAtomicLValue(); 2054 2055 // If this is an initialization, just put the value there normally. 2056 if (LVal.isSimple()) { 2057 if (isInit) { 2058 atomics.emitCopyIntoMemory(rvalue); 2059 return; 2060 } 2061 2062 // Check whether we should use a library call. 2063 if (atomics.shouldUseLibcall()) { 2064 // Produce a source address. 2065 Address srcAddr = atomics.materializeRValue(rvalue); 2066 2067 // void __atomic_store(size_t size, void *mem, void *val, int order) 2068 CallArgList args; 2069 args.add(RValue::get(atomics.getAtomicSizeValue()), 2070 getContext().getSizeType()); 2071 args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy); 2072 args.add(RValue::get(srcAddr.getPointer()), getContext().VoidPtrTy); 2073 args.add( 2074 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 2075 getContext().IntTy); 2076 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 2077 return; 2078 } 2079 2080 // Okay, we're doing this natively. 2081 llvm::Value *intValue = atomics.convertRValueToInt(rvalue); 2082 2083 // Do the atomic store. 2084 Address addr = atomics.castToAtomicIntPointer(atomics.getAtomicAddress()); 2085 intValue = Builder.CreateIntCast( 2086 intValue, addr.getElementType(), /*isSigned=*/false); 2087 llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 2088 2089 if (AO == llvm::AtomicOrdering::Acquire) 2090 AO = llvm::AtomicOrdering::Monotonic; 2091 else if (AO == llvm::AtomicOrdering::AcquireRelease) 2092 AO = llvm::AtomicOrdering::Release; 2093 // Initializations don't need to be atomic. 2094 if (!isInit) 2095 store->setAtomic(AO); 2096 2097 // Other decoration. 2098 if (IsVolatile) 2099 store->setVolatile(true); 2100 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 2101 return; 2102 } 2103 2104 // Emit simple atomic update operation. 2105 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 2106 } 2107 2108 /// Emit a compare-and-exchange op for atomic type. 2109 /// 2110 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 2111 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 2112 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 2113 AggValueSlot Slot) { 2114 // If this is an aggregate r-value, it should agree in type except 2115 // maybe for address-space qualification. 2116 assert(!Expected.isAggregate() || 2117 Expected.getAggregateAddress().getElementType() == 2118 Obj.getAddress(*this).getElementType()); 2119 assert(!Desired.isAggregate() || 2120 Desired.getAggregateAddress().getElementType() == 2121 Obj.getAddress(*this).getElementType()); 2122 AtomicInfo Atomics(*this, Obj); 2123 2124 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 2125 IsWeak); 2126 } 2127 2128 void CodeGenFunction::EmitAtomicUpdate( 2129 LValue LVal, llvm::AtomicOrdering AO, 2130 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 2131 AtomicInfo Atomics(*this, LVal); 2132 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 2133 } 2134 2135 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 2136 AtomicInfo atomics(*this, dest); 2137 2138 switch (atomics.getEvaluationKind()) { 2139 case TEK_Scalar: { 2140 llvm::Value *value = EmitScalarExpr(init); 2141 atomics.emitCopyIntoMemory(RValue::get(value)); 2142 return; 2143 } 2144 2145 case TEK_Complex: { 2146 ComplexPairTy value = EmitComplexExpr(init); 2147 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 2148 return; 2149 } 2150 2151 case TEK_Aggregate: { 2152 // Fix up the destination if the initializer isn't an expression 2153 // of atomic type. 2154 bool Zeroed = false; 2155 if (!init->getType()->isAtomicType()) { 2156 Zeroed = atomics.emitMemSetZeroIfNecessary(); 2157 dest = atomics.projectValue(); 2158 } 2159 2160 // Evaluate the expression directly into the destination. 2161 AggValueSlot slot = AggValueSlot::forLValue( 2162 dest, *this, AggValueSlot::IsNotDestructed, 2163 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, 2164 AggValueSlot::DoesNotOverlap, 2165 Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed); 2166 2167 EmitAggExpr(init, slot); 2168 return; 2169 } 2170 } 2171 llvm_unreachable("bad evaluation kind"); 2172 } 2173