1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the code for emitting atomic operations. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CGCall.h" 14 #include "CGRecordLayout.h" 15 #include "CodeGenFunction.h" 16 #include "CodeGenModule.h" 17 #include "TargetInfo.h" 18 #include "clang/AST/ASTContext.h" 19 #include "clang/CodeGen/CGFunctionInfo.h" 20 #include "clang/Frontend/FrontendDiagnostic.h" 21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/IR/DataLayout.h" 23 #include "llvm/IR/Intrinsics.h" 24 25 using namespace clang; 26 using namespace CodeGen; 27 28 namespace { 29 class AtomicInfo { 30 CodeGenFunction &CGF; 31 QualType AtomicTy; 32 QualType ValueTy; 33 uint64_t AtomicSizeInBits; 34 uint64_t ValueSizeInBits; 35 CharUnits AtomicAlign; 36 CharUnits ValueAlign; 37 TypeEvaluationKind EvaluationKind; 38 bool UseLibcall; 39 LValue LVal; 40 CGBitFieldInfo BFI; 41 public: 42 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 43 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 44 EvaluationKind(TEK_Scalar), UseLibcall(true) { 45 assert(!lvalue.isGlobalReg()); 46 ASTContext &C = CGF.getContext(); 47 if (lvalue.isSimple()) { 48 AtomicTy = lvalue.getType(); 49 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 50 ValueTy = ATy->getValueType(); 51 else 52 ValueTy = AtomicTy; 53 EvaluationKind = CGF.getEvaluationKind(ValueTy); 54 55 uint64_t ValueAlignInBits; 56 uint64_t AtomicAlignInBits; 57 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 58 ValueSizeInBits = ValueTI.Width; 59 ValueAlignInBits = ValueTI.Align; 60 61 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 62 AtomicSizeInBits = AtomicTI.Width; 63 AtomicAlignInBits = AtomicTI.Align; 64 65 assert(ValueSizeInBits <= AtomicSizeInBits); 66 assert(ValueAlignInBits <= AtomicAlignInBits); 67 68 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 69 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 70 if (lvalue.getAlignment().isZero()) 71 lvalue.setAlignment(AtomicAlign); 72 73 LVal = lvalue; 74 } else if (lvalue.isBitField()) { 75 ValueTy = lvalue.getType(); 76 ValueSizeInBits = C.getTypeSize(ValueTy); 77 auto &OrigBFI = lvalue.getBitFieldInfo(); 78 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 79 AtomicSizeInBits = C.toBits( 80 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 81 .alignTo(lvalue.getAlignment())); 82 llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF); 83 auto OffsetInChars = 84 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 85 lvalue.getAlignment(); 86 llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64( 87 CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity()); 88 StoragePtr = CGF.Builder.CreateAddrSpaceCast( 89 StoragePtr, CGF.UnqualPtrTy, "atomic_bitfield_base"); 90 BFI = OrigBFI; 91 BFI.Offset = Offset; 92 BFI.StorageSize = AtomicSizeInBits; 93 BFI.StorageOffset += OffsetInChars; 94 llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits); 95 LVal = LValue::MakeBitfield( 96 Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI, 97 lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo()); 98 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 99 if (AtomicTy.isNull()) { 100 llvm::APInt Size( 101 /*numBits=*/32, 102 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 103 AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr, 104 ArraySizeModifier::Normal, 105 /*IndexTypeQuals=*/0); 106 } 107 AtomicAlign = ValueAlign = lvalue.getAlignment(); 108 } else if (lvalue.isVectorElt()) { 109 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType(); 110 ValueSizeInBits = C.getTypeSize(ValueTy); 111 AtomicTy = lvalue.getType(); 112 AtomicSizeInBits = C.getTypeSize(AtomicTy); 113 AtomicAlign = ValueAlign = lvalue.getAlignment(); 114 LVal = lvalue; 115 } else { 116 assert(lvalue.isExtVectorElt()); 117 ValueTy = lvalue.getType(); 118 ValueSizeInBits = C.getTypeSize(ValueTy); 119 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 120 lvalue.getType(), cast<llvm::FixedVectorType>( 121 lvalue.getExtVectorAddress().getElementType()) 122 ->getNumElements()); 123 AtomicSizeInBits = C.getTypeSize(AtomicTy); 124 AtomicAlign = ValueAlign = lvalue.getAlignment(); 125 LVal = lvalue; 126 } 127 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 128 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 129 } 130 131 QualType getAtomicType() const { return AtomicTy; } 132 QualType getValueType() const { return ValueTy; } 133 CharUnits getAtomicAlignment() const { return AtomicAlign; } 134 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 135 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 136 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 137 bool shouldUseLibcall() const { return UseLibcall; } 138 const LValue &getAtomicLValue() const { return LVal; } 139 llvm::Value *getAtomicPointer() const { 140 if (LVal.isSimple()) 141 return LVal.emitRawPointer(CGF); 142 else if (LVal.isBitField()) 143 return LVal.getRawBitFieldPointer(CGF); 144 else if (LVal.isVectorElt()) 145 return LVal.getRawVectorPointer(CGF); 146 assert(LVal.isExtVectorElt()); 147 return LVal.getRawExtVectorPointer(CGF); 148 } 149 Address getAtomicAddress() const { 150 llvm::Type *ElTy; 151 if (LVal.isSimple()) 152 ElTy = LVal.getAddress().getElementType(); 153 else if (LVal.isBitField()) 154 ElTy = LVal.getBitFieldAddress().getElementType(); 155 else if (LVal.isVectorElt()) 156 ElTy = LVal.getVectorAddress().getElementType(); 157 else 158 ElTy = LVal.getExtVectorAddress().getElementType(); 159 return Address(getAtomicPointer(), ElTy, getAtomicAlignment()); 160 } 161 162 Address getAtomicAddressAsAtomicIntPointer() const { 163 return castToAtomicIntPointer(getAtomicAddress()); 164 } 165 166 /// Is the atomic size larger than the underlying value type? 167 /// 168 /// Note that the absence of padding does not mean that atomic 169 /// objects are completely interchangeable with non-atomic 170 /// objects: we might have promoted the alignment of a type 171 /// without making it bigger. 172 bool hasPadding() const { 173 return (ValueSizeInBits != AtomicSizeInBits); 174 } 175 176 bool emitMemSetZeroIfNecessary() const; 177 178 llvm::Value *getAtomicSizeValue() const { 179 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 180 return CGF.CGM.getSize(size); 181 } 182 183 /// Cast the given pointer to an integer pointer suitable for atomic 184 /// operations if the source. 185 Address castToAtomicIntPointer(Address Addr) const; 186 187 /// If Addr is compatible with the iN that will be used for an atomic 188 /// operation, bitcast it. Otherwise, create a temporary that is suitable 189 /// and copy the value across. 190 Address convertToAtomicIntPointer(Address Addr) const; 191 192 /// Turn an atomic-layout object into an r-value. 193 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 194 SourceLocation loc, bool AsValue) const; 195 196 llvm::Value *getScalarRValValueOrNull(RValue RVal) const; 197 198 /// Converts an rvalue to integer value if needed. 199 llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const; 200 201 RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot, 202 SourceLocation Loc, bool AsValue, 203 bool CmpXchg = false) const; 204 205 /// Copy an atomic r-value into atomic-layout memory. 206 void emitCopyIntoMemory(RValue rvalue) const; 207 208 /// Project an l-value down to the value field. 209 LValue projectValue() const { 210 assert(LVal.isSimple()); 211 Address addr = getAtomicAddress(); 212 if (hasPadding()) 213 addr = CGF.Builder.CreateStructGEP(addr, 0); 214 215 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 216 LVal.getBaseInfo(), LVal.getTBAAInfo()); 217 } 218 219 /// Emits atomic load. 220 /// \returns Loaded value. 221 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 222 bool AsValue, llvm::AtomicOrdering AO, 223 bool IsVolatile); 224 225 /// Emits atomic compare-and-exchange sequence. 226 /// \param Expected Expected value. 227 /// \param Desired Desired value. 228 /// \param Success Atomic ordering for success operation. 229 /// \param Failure Atomic ordering for failed operation. 230 /// \param IsWeak true if atomic operation is weak, false otherwise. 231 /// \returns Pair of values: previous value from storage (value type) and 232 /// boolean flag (i1 type) with true if success and false otherwise. 233 std::pair<RValue, llvm::Value *> 234 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 235 llvm::AtomicOrdering Success = 236 llvm::AtomicOrdering::SequentiallyConsistent, 237 llvm::AtomicOrdering Failure = 238 llvm::AtomicOrdering::SequentiallyConsistent, 239 bool IsWeak = false); 240 241 /// Emits atomic update. 242 /// \param AO Atomic ordering. 243 /// \param UpdateOp Update operation for the current lvalue. 244 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 245 const llvm::function_ref<RValue(RValue)> &UpdateOp, 246 bool IsVolatile); 247 /// Emits atomic update. 248 /// \param AO Atomic ordering. 249 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 250 bool IsVolatile); 251 252 /// Materialize an atomic r-value in atomic-layout memory. 253 Address materializeRValue(RValue rvalue) const; 254 255 /// Creates temp alloca for intermediate operations on atomic value. 256 Address CreateTempAlloca() const; 257 private: 258 bool requiresMemSetZero(llvm::Type *type) const; 259 260 261 /// Emits atomic load as a libcall. 262 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 263 llvm::AtomicOrdering AO, bool IsVolatile); 264 /// Emits atomic load as LLVM instruction. 265 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile, 266 bool CmpXchg = false); 267 /// Emits atomic compare-and-exchange op as a libcall. 268 llvm::Value *EmitAtomicCompareExchangeLibcall( 269 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 270 llvm::AtomicOrdering Success = 271 llvm::AtomicOrdering::SequentiallyConsistent, 272 llvm::AtomicOrdering Failure = 273 llvm::AtomicOrdering::SequentiallyConsistent); 274 /// Emits atomic compare-and-exchange op as LLVM instruction. 275 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 276 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 277 llvm::AtomicOrdering Success = 278 llvm::AtomicOrdering::SequentiallyConsistent, 279 llvm::AtomicOrdering Failure = 280 llvm::AtomicOrdering::SequentiallyConsistent, 281 bool IsWeak = false); 282 /// Emit atomic update as libcalls. 283 void 284 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 285 const llvm::function_ref<RValue(RValue)> &UpdateOp, 286 bool IsVolatile); 287 /// Emit atomic update as LLVM instructions. 288 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 289 const llvm::function_ref<RValue(RValue)> &UpdateOp, 290 bool IsVolatile); 291 /// Emit atomic update as libcalls. 292 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 293 bool IsVolatile); 294 /// Emit atomic update as LLVM instructions. 295 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 296 bool IsVolatile); 297 }; 298 } 299 300 Address AtomicInfo::CreateTempAlloca() const { 301 Address TempAlloca = CGF.CreateMemTemp( 302 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 303 : AtomicTy, 304 getAtomicAlignment(), 305 "atomic-temp"); 306 // Cast to pointer to value type for bitfields. 307 if (LVal.isBitField()) 308 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 309 TempAlloca, getAtomicAddress().getType(), 310 getAtomicAddress().getElementType()); 311 return TempAlloca; 312 } 313 314 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 315 StringRef fnName, 316 QualType resultType, 317 CallArgList &args) { 318 const CGFunctionInfo &fnInfo = 319 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 320 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 321 llvm::AttrBuilder fnAttrB(CGF.getLLVMContext()); 322 fnAttrB.addAttribute(llvm::Attribute::NoUnwind); 323 fnAttrB.addAttribute(llvm::Attribute::WillReturn); 324 llvm::AttributeList fnAttrs = llvm::AttributeList::get( 325 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB); 326 327 llvm::FunctionCallee fn = 328 CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs); 329 auto callee = CGCallee::forDirect(fn); 330 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); 331 } 332 333 /// Does a store of the given IR type modify the full expected width? 334 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 335 uint64_t expectedSize) { 336 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 337 } 338 339 /// Does the atomic type require memsetting to zero before initialization? 340 /// 341 /// The IR type is provided as a way of making certain queries faster. 342 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 343 // If the atomic type has size padding, we definitely need a memset. 344 if (hasPadding()) return true; 345 346 // Otherwise, do some simple heuristics to try to avoid it: 347 switch (getEvaluationKind()) { 348 // For scalars and complexes, check whether the store size of the 349 // type uses the full size. 350 case TEK_Scalar: 351 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 352 case TEK_Complex: 353 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 354 AtomicSizeInBits / 2); 355 356 // Padding in structs has an undefined bit pattern. User beware. 357 case TEK_Aggregate: 358 return false; 359 } 360 llvm_unreachable("bad evaluation kind"); 361 } 362 363 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 364 assert(LVal.isSimple()); 365 Address addr = LVal.getAddress(); 366 if (!requiresMemSetZero(addr.getElementType())) 367 return false; 368 369 CGF.Builder.CreateMemSet( 370 addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0), 371 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 372 LVal.getAlignment().getAsAlign()); 373 return true; 374 } 375 376 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 377 Address Dest, Address Ptr, 378 Address Val1, Address Val2, 379 uint64_t Size, 380 llvm::AtomicOrdering SuccessOrder, 381 llvm::AtomicOrdering FailureOrder, 382 llvm::SyncScope::ID Scope) { 383 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 384 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 385 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 386 387 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 388 Ptr, Expected, Desired, SuccessOrder, FailureOrder, Scope); 389 Pair->setVolatile(E->isVolatile()); 390 Pair->setWeak(IsWeak); 391 CGF.getTargetHooks().setTargetAtomicMetadata(CGF, *Pair, E); 392 393 // Cmp holds the result of the compare-exchange operation: true on success, 394 // false on failure. 395 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 396 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 397 398 // This basic block is used to hold the store instruction if the operation 399 // failed. 400 llvm::BasicBlock *StoreExpectedBB = 401 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 402 403 // This basic block is the exit point of the operation, we should end up 404 // here regardless of whether or not the operation succeeded. 405 llvm::BasicBlock *ContinueBB = 406 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 407 408 // Update Expected if Expected isn't equal to Old, otherwise branch to the 409 // exit point. 410 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 411 412 CGF.Builder.SetInsertPoint(StoreExpectedBB); 413 // Update the memory at Expected with Old's value. 414 auto *I = CGF.Builder.CreateStore(Old, Val1); 415 CGF.addInstToCurrentSourceAtom(I, Old); 416 417 // Finally, branch to the exit point. 418 CGF.Builder.CreateBr(ContinueBB); 419 420 CGF.Builder.SetInsertPoint(ContinueBB); 421 // Update the memory at Dest with Cmp's value. 422 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 423 } 424 425 /// Given an ordering required on success, emit all possible cmpxchg 426 /// instructions to cope with the provided (but possibly only dynamically known) 427 /// FailureOrder. 428 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 429 bool IsWeak, Address Dest, Address Ptr, 430 Address Val1, Address Val2, 431 llvm::Value *FailureOrderVal, 432 uint64_t Size, 433 llvm::AtomicOrdering SuccessOrder, 434 llvm::SyncScope::ID Scope) { 435 llvm::AtomicOrdering FailureOrder; 436 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 437 auto FOS = FO->getSExtValue(); 438 if (!llvm::isValidAtomicOrderingCABI(FOS)) 439 FailureOrder = llvm::AtomicOrdering::Monotonic; 440 else 441 switch ((llvm::AtomicOrderingCABI)FOS) { 442 case llvm::AtomicOrderingCABI::relaxed: 443 // 31.7.2.18: "The failure argument shall not be memory_order_release 444 // nor memory_order_acq_rel". Fallback to monotonic. 445 case llvm::AtomicOrderingCABI::release: 446 case llvm::AtomicOrderingCABI::acq_rel: 447 FailureOrder = llvm::AtomicOrdering::Monotonic; 448 break; 449 case llvm::AtomicOrderingCABI::consume: 450 case llvm::AtomicOrderingCABI::acquire: 451 FailureOrder = llvm::AtomicOrdering::Acquire; 452 break; 453 case llvm::AtomicOrderingCABI::seq_cst: 454 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 455 break; 456 } 457 // Prior to c++17, "the failure argument shall be no stronger than the 458 // success argument". This condition has been lifted and the only 459 // precondition is 31.7.2.18. Effectively treat this as a DR and skip 460 // language version checks. 461 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 462 FailureOrder, Scope); 463 return; 464 } 465 466 // Create all the relevant BB's 467 auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 468 auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 469 auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 470 auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 471 472 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 473 // doesn't matter unless someone is crazy enough to use something that 474 // doesn't fold to a constant for the ordering. 475 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 476 // Implemented as acquire, since it's the closest in LLVM. 477 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 478 AcquireBB); 479 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 480 AcquireBB); 481 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 482 SeqCstBB); 483 484 // Emit all the different atomics 485 CGF.Builder.SetInsertPoint(MonotonicBB); 486 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 487 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); 488 CGF.Builder.CreateBr(ContBB); 489 490 CGF.Builder.SetInsertPoint(AcquireBB); 491 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 492 llvm::AtomicOrdering::Acquire, Scope); 493 CGF.Builder.CreateBr(ContBB); 494 495 CGF.Builder.SetInsertPoint(SeqCstBB); 496 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 497 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 498 CGF.Builder.CreateBr(ContBB); 499 500 CGF.Builder.SetInsertPoint(ContBB); 501 } 502 503 /// Duplicate the atomic min/max operation in conventional IR for the builtin 504 /// variants that return the new rather than the original value. 505 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, 506 AtomicExpr::AtomicOp Op, 507 bool IsSigned, 508 llvm::Value *OldVal, 509 llvm::Value *RHS) { 510 llvm::CmpInst::Predicate Pred; 511 switch (Op) { 512 default: 513 llvm_unreachable("Unexpected min/max operation"); 514 case AtomicExpr::AO__atomic_max_fetch: 515 case AtomicExpr::AO__scoped_atomic_max_fetch: 516 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT; 517 break; 518 case AtomicExpr::AO__atomic_min_fetch: 519 case AtomicExpr::AO__scoped_atomic_min_fetch: 520 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT; 521 break; 522 } 523 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst"); 524 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval"); 525 } 526 527 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 528 Address Ptr, Address Val1, Address Val2, 529 llvm::Value *IsWeak, llvm::Value *FailureOrder, 530 uint64_t Size, llvm::AtomicOrdering Order, 531 llvm::SyncScope::ID Scope) { 532 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 533 bool PostOpMinMax = false; 534 unsigned PostOp = 0; 535 536 switch (E->getOp()) { 537 case AtomicExpr::AO__c11_atomic_init: 538 case AtomicExpr::AO__opencl_atomic_init: 539 llvm_unreachable("Already handled!"); 540 541 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 542 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 543 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 544 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 545 FailureOrder, Size, Order, Scope); 546 return; 547 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 548 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 549 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 550 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 551 FailureOrder, Size, Order, Scope); 552 return; 553 case AtomicExpr::AO__atomic_compare_exchange: 554 case AtomicExpr::AO__atomic_compare_exchange_n: 555 case AtomicExpr::AO__scoped_atomic_compare_exchange: 556 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: { 557 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 558 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 559 Val1, Val2, FailureOrder, Size, Order, Scope); 560 } else { 561 // Create all the relevant BB's 562 llvm::BasicBlock *StrongBB = 563 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 564 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 565 llvm::BasicBlock *ContBB = 566 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 567 568 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 569 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 570 571 CGF.Builder.SetInsertPoint(StrongBB); 572 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 573 FailureOrder, Size, Order, Scope); 574 CGF.Builder.CreateBr(ContBB); 575 576 CGF.Builder.SetInsertPoint(WeakBB); 577 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 578 FailureOrder, Size, Order, Scope); 579 CGF.Builder.CreateBr(ContBB); 580 581 CGF.Builder.SetInsertPoint(ContBB); 582 } 583 return; 584 } 585 case AtomicExpr::AO__c11_atomic_load: 586 case AtomicExpr::AO__opencl_atomic_load: 587 case AtomicExpr::AO__hip_atomic_load: 588 case AtomicExpr::AO__atomic_load_n: 589 case AtomicExpr::AO__atomic_load: 590 case AtomicExpr::AO__scoped_atomic_load_n: 591 case AtomicExpr::AO__scoped_atomic_load: { 592 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 593 Load->setAtomic(Order, Scope); 594 Load->setVolatile(E->isVolatile()); 595 CGF.maybeAttachRangeForLoad(Load, E->getValueType(), E->getExprLoc()); 596 auto *I = CGF.Builder.CreateStore(Load, Dest); 597 CGF.addInstToCurrentSourceAtom(I, Load); 598 return; 599 } 600 601 case AtomicExpr::AO__c11_atomic_store: 602 case AtomicExpr::AO__opencl_atomic_store: 603 case AtomicExpr::AO__hip_atomic_store: 604 case AtomicExpr::AO__atomic_store: 605 case AtomicExpr::AO__atomic_store_n: 606 case AtomicExpr::AO__scoped_atomic_store: 607 case AtomicExpr::AO__scoped_atomic_store_n: { 608 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 609 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 610 Store->setAtomic(Order, Scope); 611 Store->setVolatile(E->isVolatile()); 612 CGF.addInstToCurrentSourceAtom(Store, LoadVal1); 613 return; 614 } 615 616 case AtomicExpr::AO__c11_atomic_exchange: 617 case AtomicExpr::AO__hip_atomic_exchange: 618 case AtomicExpr::AO__opencl_atomic_exchange: 619 case AtomicExpr::AO__atomic_exchange_n: 620 case AtomicExpr::AO__atomic_exchange: 621 case AtomicExpr::AO__scoped_atomic_exchange_n: 622 case AtomicExpr::AO__scoped_atomic_exchange: 623 Op = llvm::AtomicRMWInst::Xchg; 624 break; 625 626 case AtomicExpr::AO__atomic_add_fetch: 627 case AtomicExpr::AO__scoped_atomic_add_fetch: 628 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd 629 : llvm::Instruction::Add; 630 [[fallthrough]]; 631 case AtomicExpr::AO__c11_atomic_fetch_add: 632 case AtomicExpr::AO__hip_atomic_fetch_add: 633 case AtomicExpr::AO__opencl_atomic_fetch_add: 634 case AtomicExpr::AO__atomic_fetch_add: 635 case AtomicExpr::AO__scoped_atomic_fetch_add: 636 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd 637 : llvm::AtomicRMWInst::Add; 638 break; 639 640 case AtomicExpr::AO__atomic_sub_fetch: 641 case AtomicExpr::AO__scoped_atomic_sub_fetch: 642 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub 643 : llvm::Instruction::Sub; 644 [[fallthrough]]; 645 case AtomicExpr::AO__c11_atomic_fetch_sub: 646 case AtomicExpr::AO__hip_atomic_fetch_sub: 647 case AtomicExpr::AO__opencl_atomic_fetch_sub: 648 case AtomicExpr::AO__atomic_fetch_sub: 649 case AtomicExpr::AO__scoped_atomic_fetch_sub: 650 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub 651 : llvm::AtomicRMWInst::Sub; 652 break; 653 654 case AtomicExpr::AO__atomic_min_fetch: 655 case AtomicExpr::AO__scoped_atomic_min_fetch: 656 PostOpMinMax = true; 657 [[fallthrough]]; 658 case AtomicExpr::AO__c11_atomic_fetch_min: 659 case AtomicExpr::AO__hip_atomic_fetch_min: 660 case AtomicExpr::AO__opencl_atomic_fetch_min: 661 case AtomicExpr::AO__atomic_fetch_min: 662 case AtomicExpr::AO__scoped_atomic_fetch_min: 663 Op = E->getValueType()->isFloatingType() 664 ? llvm::AtomicRMWInst::FMin 665 : (E->getValueType()->isSignedIntegerType() 666 ? llvm::AtomicRMWInst::Min 667 : llvm::AtomicRMWInst::UMin); 668 break; 669 670 case AtomicExpr::AO__atomic_max_fetch: 671 case AtomicExpr::AO__scoped_atomic_max_fetch: 672 PostOpMinMax = true; 673 [[fallthrough]]; 674 case AtomicExpr::AO__c11_atomic_fetch_max: 675 case AtomicExpr::AO__hip_atomic_fetch_max: 676 case AtomicExpr::AO__opencl_atomic_fetch_max: 677 case AtomicExpr::AO__atomic_fetch_max: 678 case AtomicExpr::AO__scoped_atomic_fetch_max: 679 Op = E->getValueType()->isFloatingType() 680 ? llvm::AtomicRMWInst::FMax 681 : (E->getValueType()->isSignedIntegerType() 682 ? llvm::AtomicRMWInst::Max 683 : llvm::AtomicRMWInst::UMax); 684 break; 685 686 case AtomicExpr::AO__atomic_and_fetch: 687 case AtomicExpr::AO__scoped_atomic_and_fetch: 688 PostOp = llvm::Instruction::And; 689 [[fallthrough]]; 690 case AtomicExpr::AO__c11_atomic_fetch_and: 691 case AtomicExpr::AO__hip_atomic_fetch_and: 692 case AtomicExpr::AO__opencl_atomic_fetch_and: 693 case AtomicExpr::AO__atomic_fetch_and: 694 case AtomicExpr::AO__scoped_atomic_fetch_and: 695 Op = llvm::AtomicRMWInst::And; 696 break; 697 698 case AtomicExpr::AO__atomic_or_fetch: 699 case AtomicExpr::AO__scoped_atomic_or_fetch: 700 PostOp = llvm::Instruction::Or; 701 [[fallthrough]]; 702 case AtomicExpr::AO__c11_atomic_fetch_or: 703 case AtomicExpr::AO__hip_atomic_fetch_or: 704 case AtomicExpr::AO__opencl_atomic_fetch_or: 705 case AtomicExpr::AO__atomic_fetch_or: 706 case AtomicExpr::AO__scoped_atomic_fetch_or: 707 Op = llvm::AtomicRMWInst::Or; 708 break; 709 710 case AtomicExpr::AO__atomic_xor_fetch: 711 case AtomicExpr::AO__scoped_atomic_xor_fetch: 712 PostOp = llvm::Instruction::Xor; 713 [[fallthrough]]; 714 case AtomicExpr::AO__c11_atomic_fetch_xor: 715 case AtomicExpr::AO__hip_atomic_fetch_xor: 716 case AtomicExpr::AO__opencl_atomic_fetch_xor: 717 case AtomicExpr::AO__atomic_fetch_xor: 718 case AtomicExpr::AO__scoped_atomic_fetch_xor: 719 Op = llvm::AtomicRMWInst::Xor; 720 break; 721 722 case AtomicExpr::AO__atomic_nand_fetch: 723 case AtomicExpr::AO__scoped_atomic_nand_fetch: 724 PostOp = llvm::Instruction::And; // the NOT is special cased below 725 [[fallthrough]]; 726 case AtomicExpr::AO__c11_atomic_fetch_nand: 727 case AtomicExpr::AO__atomic_fetch_nand: 728 case AtomicExpr::AO__scoped_atomic_fetch_nand: 729 Op = llvm::AtomicRMWInst::Nand; 730 break; 731 732 case AtomicExpr::AO__atomic_test_and_set: { 733 llvm::AtomicRMWInst *RMWI = 734 CGF.emitAtomicRMWInst(llvm::AtomicRMWInst::Xchg, Ptr, 735 CGF.Builder.getInt8(1), Order, Scope, E); 736 RMWI->setVolatile(E->isVolatile()); 737 llvm::Value *Result = CGF.Builder.CreateIsNotNull(RMWI, "tobool"); 738 auto *I = CGF.Builder.CreateStore(Result, Dest); 739 CGF.addInstToCurrentSourceAtom(I, Result); 740 return; 741 } 742 743 case AtomicExpr::AO__atomic_clear: { 744 llvm::StoreInst *Store = 745 CGF.Builder.CreateStore(CGF.Builder.getInt8(0), Ptr); 746 Store->setAtomic(Order, Scope); 747 Store->setVolatile(E->isVolatile()); 748 CGF.addInstToCurrentSourceAtom(Store, nullptr); 749 return; 750 } 751 } 752 753 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 754 llvm::AtomicRMWInst *RMWI = 755 CGF.emitAtomicRMWInst(Op, Ptr, LoadVal1, Order, Scope, E); 756 RMWI->setVolatile(E->isVolatile()); 757 758 // For __atomic_*_fetch operations, perform the operation again to 759 // determine the value which was written. 760 llvm::Value *Result = RMWI; 761 if (PostOpMinMax) 762 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(), 763 E->getValueType()->isSignedIntegerType(), 764 RMWI, LoadVal1); 765 else if (PostOp) 766 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, 767 LoadVal1); 768 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch || 769 E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch) 770 Result = CGF.Builder.CreateNot(Result); 771 auto *I = CGF.Builder.CreateStore(Result, Dest); 772 CGF.addInstToCurrentSourceAtom(I, Result); 773 } 774 775 // This function emits any expression (scalar, complex, or aggregate) 776 // into a temporary alloca. 777 static Address 778 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 779 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 780 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 781 /*Init*/ true); 782 return DeclPtr; 783 } 784 785 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, 786 Address Ptr, Address Val1, Address Val2, 787 llvm::Value *IsWeak, llvm::Value *FailureOrder, 788 uint64_t Size, llvm::AtomicOrdering Order, 789 llvm::Value *Scope) { 790 auto ScopeModel = Expr->getScopeModel(); 791 792 // LLVM atomic instructions always have sync scope. If clang atomic 793 // expression has no scope operand, use default LLVM sync scope. 794 if (!ScopeModel) { 795 llvm::SyncScope::ID SS; 796 if (CGF.getLangOpts().OpenCL) 797 // OpenCL approach is: "The functions that do not have memory_scope 798 // argument have the same semantics as the corresponding functions with 799 // the memory_scope argument set to memory_scope_device." See ref.: 800 // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions 801 SS = CGF.getTargetHooks().getLLVMSyncScopeID(CGF.getLangOpts(), 802 SyncScope::OpenCLDevice, 803 Order, CGF.getLLVMContext()); 804 else 805 SS = llvm::SyncScope::System; 806 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 807 Order, SS); 808 return; 809 } 810 811 // Handle constant scope. 812 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { 813 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( 814 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()), 815 Order, CGF.CGM.getLLVMContext()); 816 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 817 Order, SCID); 818 return; 819 } 820 821 // Handle non-constant scope. 822 auto &Builder = CGF.Builder; 823 auto Scopes = ScopeModel->getRuntimeValues(); 824 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; 825 for (auto S : Scopes) 826 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); 827 828 llvm::BasicBlock *ContBB = 829 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn); 830 831 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); 832 // If unsupported sync scope is encountered at run time, assume a fallback 833 // sync scope value. 834 auto FallBack = ScopeModel->getFallBackValue(); 835 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); 836 for (auto S : Scopes) { 837 auto *B = BB[S]; 838 if (S != FallBack) 839 SI->addCase(Builder.getInt32(S), B); 840 841 Builder.SetInsertPoint(B); 842 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 843 Order, 844 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(), 845 ScopeModel->map(S), 846 Order, 847 CGF.getLLVMContext())); 848 Builder.CreateBr(ContBB); 849 } 850 851 Builder.SetInsertPoint(ContBB); 852 } 853 854 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 855 ApplyAtomGroup Grp(getDebugInfo()); 856 857 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 858 QualType MemTy = AtomicTy; 859 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 860 MemTy = AT->getValueType(); 861 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 862 863 Address Val1 = Address::invalid(); 864 Address Val2 = Address::invalid(); 865 Address Dest = Address::invalid(); 866 Address Ptr = EmitPointerWithAlignment(E->getPtr()); 867 868 if (E->getOp() == AtomicExpr::AO__c11_atomic_init || 869 E->getOp() == AtomicExpr::AO__opencl_atomic_init) { 870 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 871 EmitAtomicInit(E->getVal1(), lvalue); 872 return RValue::get(nullptr); 873 } 874 875 auto TInfo = getContext().getTypeInfoInChars(AtomicTy); 876 uint64_t Size = TInfo.Width.getQuantity(); 877 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 878 879 CharUnits MaxInlineWidth = 880 getContext().toCharUnitsFromBits(MaxInlineWidthInBits); 881 DiagnosticsEngine &Diags = CGM.getDiags(); 882 bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0; 883 bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits; 884 if (Misaligned) { 885 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned) 886 << (int)TInfo.Width.getQuantity() 887 << (int)Ptr.getAlignment().getQuantity(); 888 } 889 if (Oversized) { 890 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized) 891 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity(); 892 } 893 894 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 895 llvm::Value *Scope = 896 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; 897 bool ShouldCastToIntPtrTy = true; 898 899 switch (E->getOp()) { 900 case AtomicExpr::AO__c11_atomic_init: 901 case AtomicExpr::AO__opencl_atomic_init: 902 llvm_unreachable("Already handled above with EmitAtomicInit!"); 903 904 case AtomicExpr::AO__atomic_load_n: 905 case AtomicExpr::AO__scoped_atomic_load_n: 906 case AtomicExpr::AO__c11_atomic_load: 907 case AtomicExpr::AO__opencl_atomic_load: 908 case AtomicExpr::AO__hip_atomic_load: 909 case AtomicExpr::AO__atomic_test_and_set: 910 case AtomicExpr::AO__atomic_clear: 911 break; 912 913 case AtomicExpr::AO__atomic_load: 914 case AtomicExpr::AO__scoped_atomic_load: 915 Dest = EmitPointerWithAlignment(E->getVal1()); 916 break; 917 918 case AtomicExpr::AO__atomic_store: 919 case AtomicExpr::AO__scoped_atomic_store: 920 Val1 = EmitPointerWithAlignment(E->getVal1()); 921 break; 922 923 case AtomicExpr::AO__atomic_exchange: 924 case AtomicExpr::AO__scoped_atomic_exchange: 925 Val1 = EmitPointerWithAlignment(E->getVal1()); 926 Dest = EmitPointerWithAlignment(E->getVal2()); 927 break; 928 929 case AtomicExpr::AO__atomic_compare_exchange: 930 case AtomicExpr::AO__atomic_compare_exchange_n: 931 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 932 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 933 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 934 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 935 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 936 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 937 case AtomicExpr::AO__scoped_atomic_compare_exchange: 938 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: 939 Val1 = EmitPointerWithAlignment(E->getVal1()); 940 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange || 941 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange) 942 Val2 = EmitPointerWithAlignment(E->getVal2()); 943 else 944 Val2 = EmitValToTemp(*this, E->getVal2()); 945 OrderFail = EmitScalarExpr(E->getOrderFail()); 946 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || 947 E->getOp() == AtomicExpr::AO__atomic_compare_exchange || 948 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n || 949 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange) 950 IsWeak = EmitScalarExpr(E->getWeak()); 951 break; 952 953 case AtomicExpr::AO__c11_atomic_fetch_add: 954 case AtomicExpr::AO__c11_atomic_fetch_sub: 955 case AtomicExpr::AO__hip_atomic_fetch_add: 956 case AtomicExpr::AO__hip_atomic_fetch_sub: 957 case AtomicExpr::AO__opencl_atomic_fetch_add: 958 case AtomicExpr::AO__opencl_atomic_fetch_sub: 959 if (MemTy->isPointerType()) { 960 // For pointer arithmetic, we're required to do a bit of math: 961 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 962 // ... but only for the C11 builtins. The GNU builtins expect the 963 // user to multiply by sizeof(T). 964 QualType Val1Ty = E->getVal1()->getType(); 965 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 966 CharUnits PointeeIncAmt = 967 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 968 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 969 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 970 Val1 = Temp; 971 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 972 break; 973 } 974 [[fallthrough]]; 975 case AtomicExpr::AO__atomic_fetch_add: 976 case AtomicExpr::AO__atomic_fetch_max: 977 case AtomicExpr::AO__atomic_fetch_min: 978 case AtomicExpr::AO__atomic_fetch_sub: 979 case AtomicExpr::AO__atomic_add_fetch: 980 case AtomicExpr::AO__atomic_max_fetch: 981 case AtomicExpr::AO__atomic_min_fetch: 982 case AtomicExpr::AO__atomic_sub_fetch: 983 case AtomicExpr::AO__c11_atomic_fetch_max: 984 case AtomicExpr::AO__c11_atomic_fetch_min: 985 case AtomicExpr::AO__opencl_atomic_fetch_max: 986 case AtomicExpr::AO__opencl_atomic_fetch_min: 987 case AtomicExpr::AO__hip_atomic_fetch_max: 988 case AtomicExpr::AO__hip_atomic_fetch_min: 989 case AtomicExpr::AO__scoped_atomic_fetch_add: 990 case AtomicExpr::AO__scoped_atomic_fetch_max: 991 case AtomicExpr::AO__scoped_atomic_fetch_min: 992 case AtomicExpr::AO__scoped_atomic_fetch_sub: 993 case AtomicExpr::AO__scoped_atomic_add_fetch: 994 case AtomicExpr::AO__scoped_atomic_max_fetch: 995 case AtomicExpr::AO__scoped_atomic_min_fetch: 996 case AtomicExpr::AO__scoped_atomic_sub_fetch: 997 ShouldCastToIntPtrTy = !MemTy->isFloatingType(); 998 [[fallthrough]]; 999 1000 case AtomicExpr::AO__atomic_fetch_and: 1001 case AtomicExpr::AO__atomic_fetch_nand: 1002 case AtomicExpr::AO__atomic_fetch_or: 1003 case AtomicExpr::AO__atomic_fetch_xor: 1004 case AtomicExpr::AO__atomic_and_fetch: 1005 case AtomicExpr::AO__atomic_nand_fetch: 1006 case AtomicExpr::AO__atomic_or_fetch: 1007 case AtomicExpr::AO__atomic_xor_fetch: 1008 case AtomicExpr::AO__atomic_store_n: 1009 case AtomicExpr::AO__atomic_exchange_n: 1010 case AtomicExpr::AO__c11_atomic_fetch_and: 1011 case AtomicExpr::AO__c11_atomic_fetch_nand: 1012 case AtomicExpr::AO__c11_atomic_fetch_or: 1013 case AtomicExpr::AO__c11_atomic_fetch_xor: 1014 case AtomicExpr::AO__c11_atomic_store: 1015 case AtomicExpr::AO__c11_atomic_exchange: 1016 case AtomicExpr::AO__hip_atomic_fetch_and: 1017 case AtomicExpr::AO__hip_atomic_fetch_or: 1018 case AtomicExpr::AO__hip_atomic_fetch_xor: 1019 case AtomicExpr::AO__hip_atomic_store: 1020 case AtomicExpr::AO__hip_atomic_exchange: 1021 case AtomicExpr::AO__opencl_atomic_fetch_and: 1022 case AtomicExpr::AO__opencl_atomic_fetch_or: 1023 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1024 case AtomicExpr::AO__opencl_atomic_store: 1025 case AtomicExpr::AO__opencl_atomic_exchange: 1026 case AtomicExpr::AO__scoped_atomic_fetch_and: 1027 case AtomicExpr::AO__scoped_atomic_fetch_nand: 1028 case AtomicExpr::AO__scoped_atomic_fetch_or: 1029 case AtomicExpr::AO__scoped_atomic_fetch_xor: 1030 case AtomicExpr::AO__scoped_atomic_and_fetch: 1031 case AtomicExpr::AO__scoped_atomic_nand_fetch: 1032 case AtomicExpr::AO__scoped_atomic_or_fetch: 1033 case AtomicExpr::AO__scoped_atomic_xor_fetch: 1034 case AtomicExpr::AO__scoped_atomic_store_n: 1035 case AtomicExpr::AO__scoped_atomic_exchange_n: 1036 Val1 = EmitValToTemp(*this, E->getVal1()); 1037 break; 1038 } 1039 1040 QualType RValTy = E->getType().getUnqualifiedType(); 1041 1042 // The inlined atomics only function on iN types, where N is a power of 2. We 1043 // need to make sure (via temporaries if necessary) that all incoming values 1044 // are compatible. 1045 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 1046 AtomicInfo Atomics(*this, AtomicVal); 1047 1048 if (ShouldCastToIntPtrTy) { 1049 Ptr = Atomics.castToAtomicIntPointer(Ptr); 1050 if (Val1.isValid()) 1051 Val1 = Atomics.convertToAtomicIntPointer(Val1); 1052 if (Val2.isValid()) 1053 Val2 = Atomics.convertToAtomicIntPointer(Val2); 1054 } 1055 if (Dest.isValid()) { 1056 if (ShouldCastToIntPtrTy) 1057 Dest = Atomics.castToAtomicIntPointer(Dest); 1058 } else if (E->isCmpXChg()) 1059 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 1060 else if (!RValTy->isVoidType()) { 1061 Dest = Atomics.CreateTempAlloca(); 1062 if (ShouldCastToIntPtrTy) 1063 Dest = Atomics.castToAtomicIntPointer(Dest); 1064 } 1065 1066 bool PowerOf2Size = (Size & (Size - 1)) == 0; 1067 bool UseLibcall = !PowerOf2Size || (Size > 16); 1068 1069 // For atomics larger than 16 bytes, emit a libcall from the frontend. This 1070 // avoids the overhead of dealing with excessively-large value types in IR. 1071 // Non-power-of-2 values also lower to libcall here, as they are not currently 1072 // permitted in IR instructions (although that constraint could be relaxed in 1073 // the future). For other cases where a libcall is required on a given 1074 // platform, we let the backend handle it (this includes handling for all of 1075 // the size-optimized libcall variants, which are only valid up to 16 bytes.) 1076 // 1077 // See: https://llvm.org/docs/Atomics.html#libcalls-atomic 1078 if (UseLibcall) { 1079 CallArgList Args; 1080 // For non-optimized library calls, the size is the first parameter. 1081 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 1082 getContext().getSizeType()); 1083 1084 // The atomic address is the second parameter. 1085 // The OpenCL atomic library functions only accept pointer arguments to 1086 // generic address space. 1087 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { 1088 if (!E->isOpenCL()) 1089 return V; 1090 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace(); 1091 if (AS == LangAS::opencl_generic) 1092 return V; 1093 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); 1094 auto *DestType = llvm::PointerType::get(getLLVMContext(), DestAS); 1095 1096 return getTargetHooks().performAddrSpaceCast(*this, V, AS, DestType, 1097 false); 1098 }; 1099 1100 Args.add(RValue::get(CastToGenericAddrSpace(Ptr.emitRawPointer(*this), 1101 E->getPtr()->getType())), 1102 getContext().VoidPtrTy); 1103 1104 // The next 1-3 parameters are op-dependent. 1105 std::string LibCallName; 1106 QualType RetTy; 1107 bool HaveRetTy = false; 1108 switch (E->getOp()) { 1109 case AtomicExpr::AO__c11_atomic_init: 1110 case AtomicExpr::AO__opencl_atomic_init: 1111 llvm_unreachable("Already handled!"); 1112 1113 // There is only one libcall for compare an exchange, because there is no 1114 // optimisation benefit possible from a libcall version of a weak compare 1115 // and exchange. 1116 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 1117 // void *desired, int success, int failure) 1118 case AtomicExpr::AO__atomic_compare_exchange: 1119 case AtomicExpr::AO__atomic_compare_exchange_n: 1120 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 1121 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 1122 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 1123 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 1124 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 1125 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 1126 case AtomicExpr::AO__scoped_atomic_compare_exchange: 1127 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: 1128 LibCallName = "__atomic_compare_exchange"; 1129 RetTy = getContext().BoolTy; 1130 HaveRetTy = true; 1131 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), 1132 E->getVal1()->getType())), 1133 getContext().VoidPtrTy); 1134 Args.add(RValue::get(CastToGenericAddrSpace(Val2.emitRawPointer(*this), 1135 E->getVal2()->getType())), 1136 getContext().VoidPtrTy); 1137 Args.add(RValue::get(Order), getContext().IntTy); 1138 Order = OrderFail; 1139 break; 1140 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 1141 // int order) 1142 case AtomicExpr::AO__atomic_exchange: 1143 case AtomicExpr::AO__atomic_exchange_n: 1144 case AtomicExpr::AO__c11_atomic_exchange: 1145 case AtomicExpr::AO__hip_atomic_exchange: 1146 case AtomicExpr::AO__opencl_atomic_exchange: 1147 case AtomicExpr::AO__scoped_atomic_exchange: 1148 case AtomicExpr::AO__scoped_atomic_exchange_n: 1149 LibCallName = "__atomic_exchange"; 1150 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), 1151 E->getVal1()->getType())), 1152 getContext().VoidPtrTy); 1153 break; 1154 // void __atomic_store(size_t size, void *mem, void *val, int order) 1155 case AtomicExpr::AO__atomic_store: 1156 case AtomicExpr::AO__atomic_store_n: 1157 case AtomicExpr::AO__c11_atomic_store: 1158 case AtomicExpr::AO__hip_atomic_store: 1159 case AtomicExpr::AO__opencl_atomic_store: 1160 case AtomicExpr::AO__scoped_atomic_store: 1161 case AtomicExpr::AO__scoped_atomic_store_n: 1162 LibCallName = "__atomic_store"; 1163 RetTy = getContext().VoidTy; 1164 HaveRetTy = true; 1165 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), 1166 E->getVal1()->getType())), 1167 getContext().VoidPtrTy); 1168 break; 1169 // void __atomic_load(size_t size, void *mem, void *return, int order) 1170 case AtomicExpr::AO__atomic_load: 1171 case AtomicExpr::AO__atomic_load_n: 1172 case AtomicExpr::AO__c11_atomic_load: 1173 case AtomicExpr::AO__hip_atomic_load: 1174 case AtomicExpr::AO__opencl_atomic_load: 1175 case AtomicExpr::AO__scoped_atomic_load: 1176 case AtomicExpr::AO__scoped_atomic_load_n: 1177 LibCallName = "__atomic_load"; 1178 break; 1179 case AtomicExpr::AO__atomic_add_fetch: 1180 case AtomicExpr::AO__scoped_atomic_add_fetch: 1181 case AtomicExpr::AO__atomic_fetch_add: 1182 case AtomicExpr::AO__c11_atomic_fetch_add: 1183 case AtomicExpr::AO__hip_atomic_fetch_add: 1184 case AtomicExpr::AO__opencl_atomic_fetch_add: 1185 case AtomicExpr::AO__scoped_atomic_fetch_add: 1186 case AtomicExpr::AO__atomic_and_fetch: 1187 case AtomicExpr::AO__scoped_atomic_and_fetch: 1188 case AtomicExpr::AO__atomic_fetch_and: 1189 case AtomicExpr::AO__c11_atomic_fetch_and: 1190 case AtomicExpr::AO__hip_atomic_fetch_and: 1191 case AtomicExpr::AO__opencl_atomic_fetch_and: 1192 case AtomicExpr::AO__scoped_atomic_fetch_and: 1193 case AtomicExpr::AO__atomic_or_fetch: 1194 case AtomicExpr::AO__scoped_atomic_or_fetch: 1195 case AtomicExpr::AO__atomic_fetch_or: 1196 case AtomicExpr::AO__c11_atomic_fetch_or: 1197 case AtomicExpr::AO__hip_atomic_fetch_or: 1198 case AtomicExpr::AO__opencl_atomic_fetch_or: 1199 case AtomicExpr::AO__scoped_atomic_fetch_or: 1200 case AtomicExpr::AO__atomic_sub_fetch: 1201 case AtomicExpr::AO__scoped_atomic_sub_fetch: 1202 case AtomicExpr::AO__atomic_fetch_sub: 1203 case AtomicExpr::AO__c11_atomic_fetch_sub: 1204 case AtomicExpr::AO__hip_atomic_fetch_sub: 1205 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1206 case AtomicExpr::AO__scoped_atomic_fetch_sub: 1207 case AtomicExpr::AO__atomic_xor_fetch: 1208 case AtomicExpr::AO__scoped_atomic_xor_fetch: 1209 case AtomicExpr::AO__atomic_fetch_xor: 1210 case AtomicExpr::AO__c11_atomic_fetch_xor: 1211 case AtomicExpr::AO__hip_atomic_fetch_xor: 1212 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1213 case AtomicExpr::AO__scoped_atomic_fetch_xor: 1214 case AtomicExpr::AO__atomic_nand_fetch: 1215 case AtomicExpr::AO__atomic_fetch_nand: 1216 case AtomicExpr::AO__c11_atomic_fetch_nand: 1217 case AtomicExpr::AO__scoped_atomic_fetch_nand: 1218 case AtomicExpr::AO__scoped_atomic_nand_fetch: 1219 case AtomicExpr::AO__atomic_min_fetch: 1220 case AtomicExpr::AO__atomic_fetch_min: 1221 case AtomicExpr::AO__c11_atomic_fetch_min: 1222 case AtomicExpr::AO__hip_atomic_fetch_min: 1223 case AtomicExpr::AO__opencl_atomic_fetch_min: 1224 case AtomicExpr::AO__scoped_atomic_fetch_min: 1225 case AtomicExpr::AO__scoped_atomic_min_fetch: 1226 case AtomicExpr::AO__atomic_max_fetch: 1227 case AtomicExpr::AO__atomic_fetch_max: 1228 case AtomicExpr::AO__c11_atomic_fetch_max: 1229 case AtomicExpr::AO__hip_atomic_fetch_max: 1230 case AtomicExpr::AO__opencl_atomic_fetch_max: 1231 case AtomicExpr::AO__scoped_atomic_fetch_max: 1232 case AtomicExpr::AO__scoped_atomic_max_fetch: 1233 case AtomicExpr::AO__atomic_test_and_set: 1234 case AtomicExpr::AO__atomic_clear: 1235 llvm_unreachable("Integral atomic operations always become atomicrmw!"); 1236 } 1237 1238 if (E->isOpenCL()) { 1239 LibCallName = 1240 std::string("__opencl") + StringRef(LibCallName).drop_front(1).str(); 1241 } 1242 // By default, assume we return a value of the atomic type. 1243 if (!HaveRetTy) { 1244 // Value is returned through parameter before the order. 1245 RetTy = getContext().VoidTy; 1246 Args.add(RValue::get( 1247 CastToGenericAddrSpace(Dest.emitRawPointer(*this), RetTy)), 1248 getContext().VoidPtrTy); 1249 } 1250 // Order is always the last parameter. 1251 Args.add(RValue::get(Order), 1252 getContext().IntTy); 1253 if (E->isOpenCL()) 1254 Args.add(RValue::get(Scope), getContext().IntTy); 1255 1256 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 1257 // The value is returned directly from the libcall. 1258 if (E->isCmpXChg()) 1259 return Res; 1260 1261 if (RValTy->isVoidType()) 1262 return RValue::get(nullptr); 1263 1264 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), 1265 RValTy, E->getExprLoc()); 1266 } 1267 1268 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1269 E->getOp() == AtomicExpr::AO__opencl_atomic_store || 1270 E->getOp() == AtomicExpr::AO__hip_atomic_store || 1271 E->getOp() == AtomicExpr::AO__atomic_store || 1272 E->getOp() == AtomicExpr::AO__atomic_store_n || 1273 E->getOp() == AtomicExpr::AO__scoped_atomic_store || 1274 E->getOp() == AtomicExpr::AO__scoped_atomic_store_n || 1275 E->getOp() == AtomicExpr::AO__atomic_clear; 1276 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1277 E->getOp() == AtomicExpr::AO__opencl_atomic_load || 1278 E->getOp() == AtomicExpr::AO__hip_atomic_load || 1279 E->getOp() == AtomicExpr::AO__atomic_load || 1280 E->getOp() == AtomicExpr::AO__atomic_load_n || 1281 E->getOp() == AtomicExpr::AO__scoped_atomic_load || 1282 E->getOp() == AtomicExpr::AO__scoped_atomic_load_n; 1283 1284 if (isa<llvm::ConstantInt>(Order)) { 1285 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1286 // We should not ever get to a case where the ordering isn't a valid C ABI 1287 // value, but it's hard to enforce that in general. 1288 if (llvm::isValidAtomicOrderingCABI(ord)) 1289 switch ((llvm::AtomicOrderingCABI)ord) { 1290 case llvm::AtomicOrderingCABI::relaxed: 1291 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1292 llvm::AtomicOrdering::Monotonic, Scope); 1293 break; 1294 case llvm::AtomicOrderingCABI::consume: 1295 case llvm::AtomicOrderingCABI::acquire: 1296 if (IsStore) 1297 break; // Avoid crashing on code with undefined behavior 1298 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1299 llvm::AtomicOrdering::Acquire, Scope); 1300 break; 1301 case llvm::AtomicOrderingCABI::release: 1302 if (IsLoad) 1303 break; // Avoid crashing on code with undefined behavior 1304 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1305 llvm::AtomicOrdering::Release, Scope); 1306 break; 1307 case llvm::AtomicOrderingCABI::acq_rel: 1308 if (IsLoad || IsStore) 1309 break; // Avoid crashing on code with undefined behavior 1310 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1311 llvm::AtomicOrdering::AcquireRelease, Scope); 1312 break; 1313 case llvm::AtomicOrderingCABI::seq_cst: 1314 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1315 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1316 break; 1317 } 1318 if (RValTy->isVoidType()) 1319 return RValue::get(nullptr); 1320 1321 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), 1322 RValTy, E->getExprLoc()); 1323 } 1324 1325 // Long case, when Order isn't obviously constant. 1326 1327 // Create all the relevant BB's 1328 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1329 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1330 *SeqCstBB = nullptr; 1331 MonotonicBB = createBasicBlock("monotonic", CurFn); 1332 if (!IsStore) 1333 AcquireBB = createBasicBlock("acquire", CurFn); 1334 if (!IsLoad) 1335 ReleaseBB = createBasicBlock("release", CurFn); 1336 if (!IsLoad && !IsStore) 1337 AcqRelBB = createBasicBlock("acqrel", CurFn); 1338 SeqCstBB = createBasicBlock("seqcst", CurFn); 1339 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1340 1341 // Create the switch for the split 1342 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1343 // doesn't matter unless someone is crazy enough to use something that 1344 // doesn't fold to a constant for the ordering. 1345 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1346 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1347 1348 // Emit all the different atomics 1349 Builder.SetInsertPoint(MonotonicBB); 1350 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1351 llvm::AtomicOrdering::Monotonic, Scope); 1352 Builder.CreateBr(ContBB); 1353 if (!IsStore) { 1354 Builder.SetInsertPoint(AcquireBB); 1355 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1356 llvm::AtomicOrdering::Acquire, Scope); 1357 Builder.CreateBr(ContBB); 1358 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1359 AcquireBB); 1360 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1361 AcquireBB); 1362 } 1363 if (!IsLoad) { 1364 Builder.SetInsertPoint(ReleaseBB); 1365 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1366 llvm::AtomicOrdering::Release, Scope); 1367 Builder.CreateBr(ContBB); 1368 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1369 ReleaseBB); 1370 } 1371 if (!IsLoad && !IsStore) { 1372 Builder.SetInsertPoint(AcqRelBB); 1373 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1374 llvm::AtomicOrdering::AcquireRelease, Scope); 1375 Builder.CreateBr(ContBB); 1376 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1377 AcqRelBB); 1378 } 1379 Builder.SetInsertPoint(SeqCstBB); 1380 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1381 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1382 Builder.CreateBr(ContBB); 1383 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1384 SeqCstBB); 1385 1386 // Cleanup and return 1387 Builder.SetInsertPoint(ContBB); 1388 if (RValTy->isVoidType()) 1389 return RValue::get(nullptr); 1390 1391 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1392 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), 1393 RValTy, E->getExprLoc()); 1394 } 1395 1396 Address AtomicInfo::castToAtomicIntPointer(Address addr) const { 1397 llvm::IntegerType *ty = 1398 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1399 return addr.withElementType(ty); 1400 } 1401 1402 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1403 llvm::Type *Ty = Addr.getElementType(); 1404 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1405 if (SourceSizeInBits != AtomicSizeInBits) { 1406 Address Tmp = CreateTempAlloca(); 1407 CGF.Builder.CreateMemCpy(Tmp, Addr, 1408 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1409 Addr = Tmp; 1410 } 1411 1412 return castToAtomicIntPointer(Addr); 1413 } 1414 1415 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1416 AggValueSlot resultSlot, 1417 SourceLocation loc, 1418 bool asValue) const { 1419 if (LVal.isSimple()) { 1420 if (EvaluationKind == TEK_Aggregate) 1421 return resultSlot.asRValue(); 1422 1423 // Drill into the padding structure if we have one. 1424 if (hasPadding()) 1425 addr = CGF.Builder.CreateStructGEP(addr, 0); 1426 1427 // Otherwise, just convert the temporary to an r-value using the 1428 // normal conversion routine. 1429 return CGF.convertTempToRValue(addr, getValueType(), loc); 1430 } 1431 if (!asValue) 1432 // Get RValue from temp memory as atomic for non-simple lvalues 1433 return RValue::get(CGF.Builder.CreateLoad(addr)); 1434 if (LVal.isBitField()) 1435 return CGF.EmitLoadOfBitfieldLValue( 1436 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1437 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1438 if (LVal.isVectorElt()) 1439 return CGF.EmitLoadOfLValue( 1440 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1441 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1442 assert(LVal.isExtVectorElt()); 1443 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1444 addr, LVal.getExtVectorElts(), LVal.getType(), 1445 LVal.getBaseInfo(), TBAAAccessInfo())); 1446 } 1447 1448 /// Return true if \param ValTy is a type that should be casted to integer 1449 /// around the atomic memory operation. If \param CmpXchg is true, then the 1450 /// cast of a floating point type is made as that instruction can not have 1451 /// floating point operands. TODO: Allow compare-and-exchange and FP - see 1452 /// comment in AtomicExpandPass.cpp. 1453 static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) { 1454 if (ValTy->isFloatingPointTy()) 1455 return ValTy->isX86_FP80Ty() || CmpXchg; 1456 return !ValTy->isIntegerTy() && !ValTy->isPointerTy(); 1457 } 1458 1459 RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val, 1460 AggValueSlot ResultSlot, 1461 SourceLocation Loc, bool AsValue, 1462 bool CmpXchg) const { 1463 // Try not to in some easy cases. 1464 assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() || 1465 Val->getType()->isIEEELikeFPTy()) && 1466 "Expected integer, pointer or floating point value when converting " 1467 "result."); 1468 if (getEvaluationKind() == TEK_Scalar && 1469 (((!LVal.isBitField() || 1470 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1471 !hasPadding()) || 1472 !AsValue)) { 1473 auto *ValTy = AsValue 1474 ? CGF.ConvertTypeForMem(ValueTy) 1475 : getAtomicAddress().getElementType(); 1476 if (!shouldCastToInt(ValTy, CmpXchg)) { 1477 assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) && 1478 "Different integer types."); 1479 return RValue::get(CGF.EmitFromMemory(Val, ValueTy)); 1480 } 1481 if (llvm::CastInst::isBitCastable(Val->getType(), ValTy)) 1482 return RValue::get(CGF.Builder.CreateBitCast(Val, ValTy)); 1483 } 1484 1485 // Create a temporary. This needs to be big enough to hold the 1486 // atomic integer. 1487 Address Temp = Address::invalid(); 1488 bool TempIsVolatile = false; 1489 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1490 assert(!ResultSlot.isIgnored()); 1491 Temp = ResultSlot.getAddress(); 1492 TempIsVolatile = ResultSlot.isVolatile(); 1493 } else { 1494 Temp = CreateTempAlloca(); 1495 } 1496 1497 // Slam the integer into the temporary. 1498 Address CastTemp = castToAtomicIntPointer(Temp); 1499 CGF.Builder.CreateStore(Val, CastTemp)->setVolatile(TempIsVolatile); 1500 1501 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1502 } 1503 1504 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1505 llvm::AtomicOrdering AO, bool) { 1506 // void __atomic_load(size_t size, void *mem, void *return, int order); 1507 CallArgList Args; 1508 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1509 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy); 1510 Args.add(RValue::get(AddForLoaded), CGF.getContext().VoidPtrTy); 1511 Args.add( 1512 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1513 CGF.getContext().IntTy); 1514 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1515 } 1516 1517 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1518 bool IsVolatile, bool CmpXchg) { 1519 // Okay, we're doing this natively. 1520 Address Addr = getAtomicAddress(); 1521 if (shouldCastToInt(Addr.getElementType(), CmpXchg)) 1522 Addr = castToAtomicIntPointer(Addr); 1523 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1524 Load->setAtomic(AO); 1525 1526 // Other decoration. 1527 if (IsVolatile) 1528 Load->setVolatile(true); 1529 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1530 return Load; 1531 } 1532 1533 /// An LValue is a candidate for having its loads and stores be made atomic if 1534 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1535 /// performing such an operation can be performed without a libcall. 1536 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1537 if (!CGM.getLangOpts().MSVolatile) return false; 1538 AtomicInfo AI(*this, LV); 1539 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1540 // An atomic is inline if we don't need to use a libcall. 1541 bool AtomicIsInline = !AI.shouldUseLibcall(); 1542 // MSVC doesn't seem to do this for types wider than a pointer. 1543 if (getContext().getTypeSize(LV.getType()) > 1544 getContext().getTypeSize(getContext().getIntPtrType())) 1545 return false; 1546 return IsVolatile && AtomicIsInline; 1547 } 1548 1549 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1550 AggValueSlot Slot) { 1551 llvm::AtomicOrdering AO; 1552 bool IsVolatile = LV.isVolatileQualified(); 1553 if (LV.getType()->isAtomicType()) { 1554 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1555 } else { 1556 AO = llvm::AtomicOrdering::Acquire; 1557 IsVolatile = true; 1558 } 1559 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1560 } 1561 1562 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1563 bool AsValue, llvm::AtomicOrdering AO, 1564 bool IsVolatile) { 1565 // Check whether we should use a library call. 1566 if (shouldUseLibcall()) { 1567 Address TempAddr = Address::invalid(); 1568 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1569 assert(getEvaluationKind() == TEK_Aggregate); 1570 TempAddr = ResultSlot.getAddress(); 1571 } else 1572 TempAddr = CreateTempAlloca(); 1573 1574 EmitAtomicLoadLibcall(TempAddr.emitRawPointer(CGF), AO, IsVolatile); 1575 1576 // Okay, turn that back into the original value or whole atomic (for 1577 // non-simple lvalues) type. 1578 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1579 } 1580 1581 // Okay, we're doing this natively. 1582 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1583 1584 // If we're ignoring an aggregate return, don't do anything. 1585 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1586 return RValue::getAggregate(Address::invalid(), false); 1587 1588 // Okay, turn that back into the original value or atomic (for non-simple 1589 // lvalues) type. 1590 return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1591 } 1592 1593 /// Emit a load from an l-value of atomic type. Note that the r-value 1594 /// we produce is an r-value of the atomic *value* type. 1595 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1596 llvm::AtomicOrdering AO, bool IsVolatile, 1597 AggValueSlot resultSlot) { 1598 AtomicInfo Atomics(*this, src); 1599 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1600 IsVolatile); 1601 } 1602 1603 /// Copy an r-value into memory as part of storing to an atomic type. 1604 /// This needs to create a bit-pattern suitable for atomic operations. 1605 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1606 assert(LVal.isSimple()); 1607 // If we have an r-value, the rvalue should be of the atomic type, 1608 // which means that the caller is responsible for having zeroed 1609 // any padding. Just do an aggregate copy of that type. 1610 if (rvalue.isAggregate()) { 1611 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); 1612 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), 1613 getAtomicType()); 1614 bool IsVolatile = rvalue.isVolatileQualified() || 1615 LVal.isVolatileQualified(); 1616 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), 1617 AggValueSlot::DoesNotOverlap, IsVolatile); 1618 return; 1619 } 1620 1621 // Okay, otherwise we're copying stuff. 1622 1623 // Zero out the buffer if necessary. 1624 emitMemSetZeroIfNecessary(); 1625 1626 // Drill past the padding if present. 1627 LValue TempLVal = projectValue(); 1628 1629 // Okay, store the rvalue in. 1630 if (rvalue.isScalar()) { 1631 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1632 } else { 1633 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1634 } 1635 } 1636 1637 1638 /// Materialize an r-value into memory for the purposes of storing it 1639 /// to an atomic type. 1640 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1641 // Aggregate r-values are already in memory, and EmitAtomicStore 1642 // requires them to be values of the atomic type. 1643 if (rvalue.isAggregate()) 1644 return rvalue.getAggregateAddress(); 1645 1646 // Otherwise, make a temporary and materialize into it. 1647 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1648 AtomicInfo Atomics(CGF, TempLV); 1649 Atomics.emitCopyIntoMemory(rvalue); 1650 return TempLV.getAddress(); 1651 } 1652 1653 llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const { 1654 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) 1655 return RVal.getScalarVal(); 1656 return nullptr; 1657 } 1658 1659 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const { 1660 // If we've got a scalar value of the right size, try to avoid going 1661 // through memory. Floats get casted if needed by AtomicExpandPass. 1662 if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) { 1663 if (!shouldCastToInt(Value->getType(), CmpXchg)) 1664 return CGF.EmitToMemory(Value, ValueTy); 1665 else { 1666 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1667 CGF.getLLVMContext(), 1668 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1669 if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1670 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1671 } 1672 } 1673 // Otherwise, we need to go through memory. 1674 // Put the r-value in memory. 1675 Address Addr = materializeRValue(RVal); 1676 1677 // Cast the temporary to the atomic int type and pull a value out. 1678 Addr = castToAtomicIntPointer(Addr); 1679 return CGF.Builder.CreateLoad(Addr); 1680 } 1681 1682 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1683 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1684 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1685 // Do the atomic store. 1686 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1687 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal, 1688 Success, Failure); 1689 // Other decoration. 1690 Inst->setVolatile(LVal.isVolatileQualified()); 1691 Inst->setWeak(IsWeak); 1692 1693 // Okay, turn that back into the original value type. 1694 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1695 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1696 return std::make_pair(PreviousVal, SuccessFailureVal); 1697 } 1698 1699 llvm::Value * 1700 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1701 llvm::Value *DesiredAddr, 1702 llvm::AtomicOrdering Success, 1703 llvm::AtomicOrdering Failure) { 1704 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1705 // void *desired, int success, int failure); 1706 CallArgList Args; 1707 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1708 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy); 1709 Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy); 1710 Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy); 1711 Args.add(RValue::get( 1712 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1713 CGF.getContext().IntTy); 1714 Args.add(RValue::get( 1715 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1716 CGF.getContext().IntTy); 1717 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1718 CGF.getContext().BoolTy, Args); 1719 1720 return SuccessFailureRVal.getScalarVal(); 1721 } 1722 1723 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1724 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1725 llvm::AtomicOrdering Failure, bool IsWeak) { 1726 // Check whether we should use a library call. 1727 if (shouldUseLibcall()) { 1728 // Produce a source address. 1729 Address ExpectedAddr = materializeRValue(Expected); 1730 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); 1731 llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF); 1732 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, 1733 Success, Failure); 1734 return std::make_pair( 1735 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1736 SourceLocation(), /*AsValue=*/false), 1737 Res); 1738 } 1739 1740 // If we've got a scalar value of the right size, try to avoid going 1741 // through memory. 1742 auto *ExpectedVal = convertRValueToInt(Expected, /*CmpXchg=*/true); 1743 auto *DesiredVal = convertRValueToInt(Desired, /*CmpXchg=*/true); 1744 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1745 Failure, IsWeak); 1746 return std::make_pair( 1747 ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1748 SourceLocation(), /*AsValue=*/false, 1749 /*CmpXchg=*/true), 1750 Res.second); 1751 } 1752 1753 static void 1754 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1755 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1756 Address DesiredAddr) { 1757 RValue UpRVal; 1758 LValue AtomicLVal = Atomics.getAtomicLValue(); 1759 LValue DesiredLVal; 1760 if (AtomicLVal.isSimple()) { 1761 UpRVal = OldRVal; 1762 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1763 } else { 1764 // Build new lvalue for temp address. 1765 Address Ptr = Atomics.materializeRValue(OldRVal); 1766 LValue UpdateLVal; 1767 if (AtomicLVal.isBitField()) { 1768 UpdateLVal = 1769 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1770 AtomicLVal.getType(), 1771 AtomicLVal.getBaseInfo(), 1772 AtomicLVal.getTBAAInfo()); 1773 DesiredLVal = 1774 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1775 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1776 AtomicLVal.getTBAAInfo()); 1777 } else if (AtomicLVal.isVectorElt()) { 1778 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1779 AtomicLVal.getType(), 1780 AtomicLVal.getBaseInfo(), 1781 AtomicLVal.getTBAAInfo()); 1782 DesiredLVal = LValue::MakeVectorElt( 1783 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1784 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1785 } else { 1786 assert(AtomicLVal.isExtVectorElt()); 1787 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1788 AtomicLVal.getType(), 1789 AtomicLVal.getBaseInfo(), 1790 AtomicLVal.getTBAAInfo()); 1791 DesiredLVal = LValue::MakeExtVectorElt( 1792 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1793 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1794 } 1795 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1796 } 1797 // Store new value in the corresponding memory area. 1798 RValue NewRVal = UpdateOp(UpRVal); 1799 if (NewRVal.isScalar()) { 1800 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1801 } else { 1802 assert(NewRVal.isComplex()); 1803 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1804 /*isInit=*/false); 1805 } 1806 } 1807 1808 void AtomicInfo::EmitAtomicUpdateLibcall( 1809 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1810 bool IsVolatile) { 1811 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1812 1813 Address ExpectedAddr = CreateTempAlloca(); 1814 1815 EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); 1816 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1817 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1818 CGF.EmitBlock(ContBB); 1819 Address DesiredAddr = CreateTempAlloca(); 1820 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1821 requiresMemSetZero(getAtomicAddress().getElementType())) { 1822 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1823 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1824 } 1825 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1826 AggValueSlot::ignored(), 1827 SourceLocation(), /*AsValue=*/false); 1828 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1829 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); 1830 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); 1831 auto *Res = 1832 EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); 1833 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1834 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1835 } 1836 1837 void AtomicInfo::EmitAtomicUpdateOp( 1838 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1839 bool IsVolatile) { 1840 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1841 1842 // Do the atomic load. 1843 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true); 1844 // For non-simple lvalues perform compare-and-swap procedure. 1845 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1846 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1847 auto *CurBB = CGF.Builder.GetInsertBlock(); 1848 CGF.EmitBlock(ContBB); 1849 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1850 /*NumReservedValues=*/2); 1851 PHI->addIncoming(OldVal, CurBB); 1852 Address NewAtomicAddr = CreateTempAlloca(); 1853 Address NewAtomicIntAddr = 1854 shouldCastToInt(NewAtomicAddr.getElementType(), /*CmpXchg=*/true) 1855 ? castToAtomicIntPointer(NewAtomicAddr) 1856 : NewAtomicAddr; 1857 1858 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1859 requiresMemSetZero(getAtomicAddress().getElementType())) { 1860 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1861 } 1862 auto OldRVal = ConvertToValueOrAtomic(PHI, AggValueSlot::ignored(), 1863 SourceLocation(), /*AsValue=*/false, 1864 /*CmpXchg=*/true); 1865 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1866 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1867 // Try to write new value using cmpxchg operation. 1868 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1869 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1870 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1871 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1872 } 1873 1874 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1875 RValue UpdateRVal, Address DesiredAddr) { 1876 LValue AtomicLVal = Atomics.getAtomicLValue(); 1877 LValue DesiredLVal; 1878 // Build new lvalue for temp address. 1879 if (AtomicLVal.isBitField()) { 1880 DesiredLVal = 1881 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1882 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1883 AtomicLVal.getTBAAInfo()); 1884 } else if (AtomicLVal.isVectorElt()) { 1885 DesiredLVal = 1886 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1887 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1888 AtomicLVal.getTBAAInfo()); 1889 } else { 1890 assert(AtomicLVal.isExtVectorElt()); 1891 DesiredLVal = LValue::MakeExtVectorElt( 1892 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1893 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1894 } 1895 // Store new value in the corresponding memory area. 1896 assert(UpdateRVal.isScalar()); 1897 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1898 } 1899 1900 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1901 RValue UpdateRVal, bool IsVolatile) { 1902 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1903 1904 Address ExpectedAddr = CreateTempAlloca(); 1905 1906 EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); 1907 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1908 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1909 CGF.EmitBlock(ContBB); 1910 Address DesiredAddr = CreateTempAlloca(); 1911 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1912 requiresMemSetZero(getAtomicAddress().getElementType())) { 1913 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1914 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1915 } 1916 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1917 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); 1918 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); 1919 auto *Res = 1920 EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); 1921 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1922 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1923 } 1924 1925 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1926 bool IsVolatile) { 1927 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1928 1929 // Do the atomic load. 1930 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true); 1931 // For non-simple lvalues perform compare-and-swap procedure. 1932 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1933 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1934 auto *CurBB = CGF.Builder.GetInsertBlock(); 1935 CGF.EmitBlock(ContBB); 1936 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1937 /*NumReservedValues=*/2); 1938 PHI->addIncoming(OldVal, CurBB); 1939 Address NewAtomicAddr = CreateTempAlloca(); 1940 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr); 1941 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1942 requiresMemSetZero(getAtomicAddress().getElementType())) { 1943 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1944 } 1945 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 1946 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1947 // Try to write new value using cmpxchg operation. 1948 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1949 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1950 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1951 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1952 } 1953 1954 void AtomicInfo::EmitAtomicUpdate( 1955 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1956 bool IsVolatile) { 1957 if (shouldUseLibcall()) { 1958 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 1959 } else { 1960 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 1961 } 1962 } 1963 1964 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 1965 bool IsVolatile) { 1966 if (shouldUseLibcall()) { 1967 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 1968 } else { 1969 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 1970 } 1971 } 1972 1973 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 1974 bool isInit) { 1975 bool IsVolatile = lvalue.isVolatileQualified(); 1976 llvm::AtomicOrdering AO; 1977 if (lvalue.getType()->isAtomicType()) { 1978 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1979 } else { 1980 AO = llvm::AtomicOrdering::Release; 1981 IsVolatile = true; 1982 } 1983 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 1984 } 1985 1986 /// Emit a store to an l-value of atomic type. 1987 /// 1988 /// Note that the r-value is expected to be an r-value *of the atomic 1989 /// type*; this means that for aggregate r-values, it should include 1990 /// storage for any padding that was necessary. 1991 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 1992 llvm::AtomicOrdering AO, bool IsVolatile, 1993 bool isInit) { 1994 // If this is an aggregate r-value, it should agree in type except 1995 // maybe for address-space qualification. 1996 assert(!rvalue.isAggregate() || 1997 rvalue.getAggregateAddress().getElementType() == 1998 dest.getAddress().getElementType()); 1999 2000 AtomicInfo atomics(*this, dest); 2001 LValue LVal = atomics.getAtomicLValue(); 2002 2003 // If this is an initialization, just put the value there normally. 2004 if (LVal.isSimple()) { 2005 if (isInit) { 2006 atomics.emitCopyIntoMemory(rvalue); 2007 return; 2008 } 2009 2010 // Check whether we should use a library call. 2011 if (atomics.shouldUseLibcall()) { 2012 // Produce a source address. 2013 Address srcAddr = atomics.materializeRValue(rvalue); 2014 2015 // void __atomic_store(size_t size, void *mem, void *val, int order) 2016 CallArgList args; 2017 args.add(RValue::get(atomics.getAtomicSizeValue()), 2018 getContext().getSizeType()); 2019 args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy); 2020 args.add(RValue::get(srcAddr.emitRawPointer(*this)), 2021 getContext().VoidPtrTy); 2022 args.add( 2023 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 2024 getContext().IntTy); 2025 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 2026 return; 2027 } 2028 2029 // Okay, we're doing this natively. 2030 llvm::Value *ValToStore = atomics.convertRValueToInt(rvalue); 2031 2032 // Do the atomic store. 2033 Address Addr = atomics.getAtomicAddress(); 2034 if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue)) 2035 if (shouldCastToInt(Value->getType(), /*CmpXchg=*/false)) { 2036 Addr = atomics.castToAtomicIntPointer(Addr); 2037 ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(), 2038 /*isSigned=*/false); 2039 } 2040 llvm::StoreInst *store = Builder.CreateStore(ValToStore, Addr); 2041 2042 if (AO == llvm::AtomicOrdering::Acquire) 2043 AO = llvm::AtomicOrdering::Monotonic; 2044 else if (AO == llvm::AtomicOrdering::AcquireRelease) 2045 AO = llvm::AtomicOrdering::Release; 2046 // Initializations don't need to be atomic. 2047 if (!isInit) 2048 store->setAtomic(AO); 2049 2050 // Other decoration. 2051 if (IsVolatile) 2052 store->setVolatile(true); 2053 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 2054 return; 2055 } 2056 2057 // Emit simple atomic update operation. 2058 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 2059 } 2060 2061 /// Emit a compare-and-exchange op for atomic type. 2062 /// 2063 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 2064 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 2065 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 2066 AggValueSlot Slot) { 2067 // If this is an aggregate r-value, it should agree in type except 2068 // maybe for address-space qualification. 2069 assert(!Expected.isAggregate() || 2070 Expected.getAggregateAddress().getElementType() == 2071 Obj.getAddress().getElementType()); 2072 assert(!Desired.isAggregate() || 2073 Desired.getAggregateAddress().getElementType() == 2074 Obj.getAddress().getElementType()); 2075 AtomicInfo Atomics(*this, Obj); 2076 2077 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 2078 IsWeak); 2079 } 2080 2081 llvm::AtomicRMWInst * 2082 CodeGenFunction::emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr, 2083 llvm::Value *Val, llvm::AtomicOrdering Order, 2084 llvm::SyncScope::ID SSID, 2085 const AtomicExpr *AE) { 2086 llvm::AtomicRMWInst *RMW = 2087 Builder.CreateAtomicRMW(Op, Addr, Val, Order, SSID); 2088 getTargetHooks().setTargetAtomicMetadata(*this, *RMW, AE); 2089 return RMW; 2090 } 2091 2092 void CodeGenFunction::EmitAtomicUpdate( 2093 LValue LVal, llvm::AtomicOrdering AO, 2094 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 2095 AtomicInfo Atomics(*this, LVal); 2096 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 2097 } 2098 2099 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 2100 AtomicInfo atomics(*this, dest); 2101 2102 switch (atomics.getEvaluationKind()) { 2103 case TEK_Scalar: { 2104 llvm::Value *value = EmitScalarExpr(init); 2105 atomics.emitCopyIntoMemory(RValue::get(value)); 2106 return; 2107 } 2108 2109 case TEK_Complex: { 2110 ComplexPairTy value = EmitComplexExpr(init); 2111 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 2112 return; 2113 } 2114 2115 case TEK_Aggregate: { 2116 // Fix up the destination if the initializer isn't an expression 2117 // of atomic type. 2118 bool Zeroed = false; 2119 if (!init->getType()->isAtomicType()) { 2120 Zeroed = atomics.emitMemSetZeroIfNecessary(); 2121 dest = atomics.projectValue(); 2122 } 2123 2124 // Evaluate the expression directly into the destination. 2125 AggValueSlot slot = AggValueSlot::forLValue( 2126 dest, AggValueSlot::IsNotDestructed, 2127 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, 2128 AggValueSlot::DoesNotOverlap, 2129 Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed); 2130 2131 EmitAggExpr(init, slot); 2132 return; 2133 } 2134 } 2135 llvm_unreachable("bad evaluation kind"); 2136 } 2137