1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the code for emitting atomic operations. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CGCall.h" 14 #include "CGRecordLayout.h" 15 #include "CodeGenFunction.h" 16 #include "CodeGenModule.h" 17 #include "TargetInfo.h" 18 #include "clang/AST/ASTContext.h" 19 #include "clang/CodeGen/CGFunctionInfo.h" 20 #include "clang/Frontend/FrontendDiagnostic.h" 21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/IR/DataLayout.h" 23 #include "llvm/IR/Intrinsics.h" 24 #include "llvm/IR/Operator.h" 25 26 using namespace clang; 27 using namespace CodeGen; 28 29 namespace { 30 class AtomicInfo { 31 CodeGenFunction &CGF; 32 QualType AtomicTy; 33 QualType ValueTy; 34 uint64_t AtomicSizeInBits; 35 uint64_t ValueSizeInBits; 36 CharUnits AtomicAlign; 37 CharUnits ValueAlign; 38 TypeEvaluationKind EvaluationKind; 39 bool UseLibcall; 40 LValue LVal; 41 CGBitFieldInfo BFI; 42 public: 43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 45 EvaluationKind(TEK_Scalar), UseLibcall(true) { 46 assert(!lvalue.isGlobalReg()); 47 ASTContext &C = CGF.getContext(); 48 if (lvalue.isSimple()) { 49 AtomicTy = lvalue.getType(); 50 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 51 ValueTy = ATy->getValueType(); 52 else 53 ValueTy = AtomicTy; 54 EvaluationKind = CGF.getEvaluationKind(ValueTy); 55 56 uint64_t ValueAlignInBits; 57 uint64_t AtomicAlignInBits; 58 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 59 ValueSizeInBits = ValueTI.Width; 60 ValueAlignInBits = ValueTI.Align; 61 62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 63 AtomicSizeInBits = AtomicTI.Width; 64 AtomicAlignInBits = AtomicTI.Align; 65 66 assert(ValueSizeInBits <= AtomicSizeInBits); 67 assert(ValueAlignInBits <= AtomicAlignInBits); 68 69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 71 if (lvalue.getAlignment().isZero()) 72 lvalue.setAlignment(AtomicAlign); 73 74 LVal = lvalue; 75 } else if (lvalue.isBitField()) { 76 ValueTy = lvalue.getType(); 77 ValueSizeInBits = C.getTypeSize(ValueTy); 78 auto &OrigBFI = lvalue.getBitFieldInfo(); 79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 80 AtomicSizeInBits = C.toBits( 81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 82 .alignTo(lvalue.getAlignment())); 83 llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF); 84 auto OffsetInChars = 85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 86 lvalue.getAlignment(); 87 llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64( 88 CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity()); 89 StoragePtr = CGF.Builder.CreateAddrSpaceCast( 90 StoragePtr, CGF.UnqualPtrTy, "atomic_bitfield_base"); 91 BFI = OrigBFI; 92 BFI.Offset = Offset; 93 BFI.StorageSize = AtomicSizeInBits; 94 BFI.StorageOffset += OffsetInChars; 95 llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits); 96 LVal = LValue::MakeBitfield( 97 Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI, 98 lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo()); 99 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 100 if (AtomicTy.isNull()) { 101 llvm::APInt Size( 102 /*numBits=*/32, 103 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 104 AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr, 105 ArraySizeModifier::Normal, 106 /*IndexTypeQuals=*/0); 107 } 108 AtomicAlign = ValueAlign = lvalue.getAlignment(); 109 } else if (lvalue.isVectorElt()) { 110 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType(); 111 ValueSizeInBits = C.getTypeSize(ValueTy); 112 AtomicTy = lvalue.getType(); 113 AtomicSizeInBits = C.getTypeSize(AtomicTy); 114 AtomicAlign = ValueAlign = lvalue.getAlignment(); 115 LVal = lvalue; 116 } else { 117 assert(lvalue.isExtVectorElt()); 118 ValueTy = lvalue.getType(); 119 ValueSizeInBits = C.getTypeSize(ValueTy); 120 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 121 lvalue.getType(), cast<llvm::FixedVectorType>( 122 lvalue.getExtVectorAddress().getElementType()) 123 ->getNumElements()); 124 AtomicSizeInBits = C.getTypeSize(AtomicTy); 125 AtomicAlign = ValueAlign = lvalue.getAlignment(); 126 LVal = lvalue; 127 } 128 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 129 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 130 } 131 132 QualType getAtomicType() const { return AtomicTy; } 133 QualType getValueType() const { return ValueTy; } 134 CharUnits getAtomicAlignment() const { return AtomicAlign; } 135 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 136 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 137 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 138 bool shouldUseLibcall() const { return UseLibcall; } 139 const LValue &getAtomicLValue() const { return LVal; } 140 llvm::Value *getAtomicPointer() const { 141 if (LVal.isSimple()) 142 return LVal.emitRawPointer(CGF); 143 else if (LVal.isBitField()) 144 return LVal.getRawBitFieldPointer(CGF); 145 else if (LVal.isVectorElt()) 146 return LVal.getRawVectorPointer(CGF); 147 assert(LVal.isExtVectorElt()); 148 return LVal.getRawExtVectorPointer(CGF); 149 } 150 Address getAtomicAddress() const { 151 llvm::Type *ElTy; 152 if (LVal.isSimple()) 153 ElTy = LVal.getAddress().getElementType(); 154 else if (LVal.isBitField()) 155 ElTy = LVal.getBitFieldAddress().getElementType(); 156 else if (LVal.isVectorElt()) 157 ElTy = LVal.getVectorAddress().getElementType(); 158 else 159 ElTy = LVal.getExtVectorAddress().getElementType(); 160 return Address(getAtomicPointer(), ElTy, getAtomicAlignment()); 161 } 162 163 Address getAtomicAddressAsAtomicIntPointer() const { 164 return castToAtomicIntPointer(getAtomicAddress()); 165 } 166 167 /// Is the atomic size larger than the underlying value type? 168 /// 169 /// Note that the absence of padding does not mean that atomic 170 /// objects are completely interchangeable with non-atomic 171 /// objects: we might have promoted the alignment of a type 172 /// without making it bigger. 173 bool hasPadding() const { 174 return (ValueSizeInBits != AtomicSizeInBits); 175 } 176 177 bool emitMemSetZeroIfNecessary() const; 178 179 llvm::Value *getAtomicSizeValue() const { 180 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 181 return CGF.CGM.getSize(size); 182 } 183 184 /// Cast the given pointer to an integer pointer suitable for atomic 185 /// operations if the source. 186 Address castToAtomicIntPointer(Address Addr) const; 187 188 /// If Addr is compatible with the iN that will be used for an atomic 189 /// operation, bitcast it. Otherwise, create a temporary that is suitable 190 /// and copy the value across. 191 Address convertToAtomicIntPointer(Address Addr) const; 192 193 /// Turn an atomic-layout object into an r-value. 194 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 195 SourceLocation loc, bool AsValue) const; 196 197 llvm::Value *getScalarRValValueOrNull(RValue RVal) const; 198 199 /// Converts an rvalue to integer value if needed. 200 llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const; 201 202 RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot, 203 SourceLocation Loc, bool AsValue, 204 bool CmpXchg = false) const; 205 206 /// Copy an atomic r-value into atomic-layout memory. 207 void emitCopyIntoMemory(RValue rvalue) const; 208 209 /// Project an l-value down to the value field. 210 LValue projectValue() const { 211 assert(LVal.isSimple()); 212 Address addr = getAtomicAddress(); 213 if (hasPadding()) 214 addr = CGF.Builder.CreateStructGEP(addr, 0); 215 216 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 217 LVal.getBaseInfo(), LVal.getTBAAInfo()); 218 } 219 220 /// Emits atomic load. 221 /// \returns Loaded value. 222 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 223 bool AsValue, llvm::AtomicOrdering AO, 224 bool IsVolatile); 225 226 /// Emits atomic compare-and-exchange sequence. 227 /// \param Expected Expected value. 228 /// \param Desired Desired value. 229 /// \param Success Atomic ordering for success operation. 230 /// \param Failure Atomic ordering for failed operation. 231 /// \param IsWeak true if atomic operation is weak, false otherwise. 232 /// \returns Pair of values: previous value from storage (value type) and 233 /// boolean flag (i1 type) with true if success and false otherwise. 234 std::pair<RValue, llvm::Value *> 235 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 236 llvm::AtomicOrdering Success = 237 llvm::AtomicOrdering::SequentiallyConsistent, 238 llvm::AtomicOrdering Failure = 239 llvm::AtomicOrdering::SequentiallyConsistent, 240 bool IsWeak = false); 241 242 /// Emits atomic update. 243 /// \param AO Atomic ordering. 244 /// \param UpdateOp Update operation for the current lvalue. 245 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 246 const llvm::function_ref<RValue(RValue)> &UpdateOp, 247 bool IsVolatile); 248 /// Emits atomic update. 249 /// \param AO Atomic ordering. 250 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 251 bool IsVolatile); 252 253 /// Materialize an atomic r-value in atomic-layout memory. 254 Address materializeRValue(RValue rvalue) const; 255 256 /// Creates temp alloca for intermediate operations on atomic value. 257 Address CreateTempAlloca() const; 258 private: 259 bool requiresMemSetZero(llvm::Type *type) const; 260 261 262 /// Emits atomic load as a libcall. 263 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 264 llvm::AtomicOrdering AO, bool IsVolatile); 265 /// Emits atomic load as LLVM instruction. 266 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile, 267 bool CmpXchg = false); 268 /// Emits atomic compare-and-exchange op as a libcall. 269 llvm::Value *EmitAtomicCompareExchangeLibcall( 270 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 271 llvm::AtomicOrdering Success = 272 llvm::AtomicOrdering::SequentiallyConsistent, 273 llvm::AtomicOrdering Failure = 274 llvm::AtomicOrdering::SequentiallyConsistent); 275 /// Emits atomic compare-and-exchange op as LLVM instruction. 276 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 277 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 278 llvm::AtomicOrdering Success = 279 llvm::AtomicOrdering::SequentiallyConsistent, 280 llvm::AtomicOrdering Failure = 281 llvm::AtomicOrdering::SequentiallyConsistent, 282 bool IsWeak = false); 283 /// Emit atomic update as libcalls. 284 void 285 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 286 const llvm::function_ref<RValue(RValue)> &UpdateOp, 287 bool IsVolatile); 288 /// Emit atomic update as LLVM instructions. 289 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 290 const llvm::function_ref<RValue(RValue)> &UpdateOp, 291 bool IsVolatile); 292 /// Emit atomic update as libcalls. 293 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 294 bool IsVolatile); 295 /// Emit atomic update as LLVM instructions. 296 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 297 bool IsVolatile); 298 }; 299 } 300 301 Address AtomicInfo::CreateTempAlloca() const { 302 Address TempAlloca = CGF.CreateMemTemp( 303 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 304 : AtomicTy, 305 getAtomicAlignment(), 306 "atomic-temp"); 307 // Cast to pointer to value type for bitfields. 308 if (LVal.isBitField()) 309 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 310 TempAlloca, getAtomicAddress().getType(), 311 getAtomicAddress().getElementType()); 312 return TempAlloca; 313 } 314 315 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 316 StringRef fnName, 317 QualType resultType, 318 CallArgList &args) { 319 const CGFunctionInfo &fnInfo = 320 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 321 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 322 llvm::AttrBuilder fnAttrB(CGF.getLLVMContext()); 323 fnAttrB.addAttribute(llvm::Attribute::NoUnwind); 324 fnAttrB.addAttribute(llvm::Attribute::WillReturn); 325 llvm::AttributeList fnAttrs = llvm::AttributeList::get( 326 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB); 327 328 llvm::FunctionCallee fn = 329 CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs); 330 auto callee = CGCallee::forDirect(fn); 331 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); 332 } 333 334 /// Does a store of the given IR type modify the full expected width? 335 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 336 uint64_t expectedSize) { 337 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 338 } 339 340 /// Does the atomic type require memsetting to zero before initialization? 341 /// 342 /// The IR type is provided as a way of making certain queries faster. 343 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 344 // If the atomic type has size padding, we definitely need a memset. 345 if (hasPadding()) return true; 346 347 // Otherwise, do some simple heuristics to try to avoid it: 348 switch (getEvaluationKind()) { 349 // For scalars and complexes, check whether the store size of the 350 // type uses the full size. 351 case TEK_Scalar: 352 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 353 case TEK_Complex: 354 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 355 AtomicSizeInBits / 2); 356 357 // Padding in structs has an undefined bit pattern. User beware. 358 case TEK_Aggregate: 359 return false; 360 } 361 llvm_unreachable("bad evaluation kind"); 362 } 363 364 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 365 assert(LVal.isSimple()); 366 Address addr = LVal.getAddress(); 367 if (!requiresMemSetZero(addr.getElementType())) 368 return false; 369 370 CGF.Builder.CreateMemSet( 371 addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0), 372 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 373 LVal.getAlignment().getAsAlign()); 374 return true; 375 } 376 377 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 378 Address Dest, Address Ptr, 379 Address Val1, Address Val2, 380 uint64_t Size, 381 llvm::AtomicOrdering SuccessOrder, 382 llvm::AtomicOrdering FailureOrder, 383 llvm::SyncScope::ID Scope) { 384 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 385 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 386 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 387 388 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 389 Ptr, Expected, Desired, SuccessOrder, FailureOrder, Scope); 390 Pair->setVolatile(E->isVolatile()); 391 Pair->setWeak(IsWeak); 392 393 // Cmp holds the result of the compare-exchange operation: true on success, 394 // false on failure. 395 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 396 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 397 398 // This basic block is used to hold the store instruction if the operation 399 // failed. 400 llvm::BasicBlock *StoreExpectedBB = 401 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 402 403 // This basic block is the exit point of the operation, we should end up 404 // here regardless of whether or not the operation succeeded. 405 llvm::BasicBlock *ContinueBB = 406 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 407 408 // Update Expected if Expected isn't equal to Old, otherwise branch to the 409 // exit point. 410 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 411 412 CGF.Builder.SetInsertPoint(StoreExpectedBB); 413 // Update the memory at Expected with Old's value. 414 CGF.Builder.CreateStore(Old, Val1); 415 // Finally, branch to the exit point. 416 CGF.Builder.CreateBr(ContinueBB); 417 418 CGF.Builder.SetInsertPoint(ContinueBB); 419 // Update the memory at Dest with Cmp's value. 420 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 421 } 422 423 /// Given an ordering required on success, emit all possible cmpxchg 424 /// instructions to cope with the provided (but possibly only dynamically known) 425 /// FailureOrder. 426 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 427 bool IsWeak, Address Dest, Address Ptr, 428 Address Val1, Address Val2, 429 llvm::Value *FailureOrderVal, 430 uint64_t Size, 431 llvm::AtomicOrdering SuccessOrder, 432 llvm::SyncScope::ID Scope) { 433 llvm::AtomicOrdering FailureOrder; 434 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 435 auto FOS = FO->getSExtValue(); 436 if (!llvm::isValidAtomicOrderingCABI(FOS)) 437 FailureOrder = llvm::AtomicOrdering::Monotonic; 438 else 439 switch ((llvm::AtomicOrderingCABI)FOS) { 440 case llvm::AtomicOrderingCABI::relaxed: 441 // 31.7.2.18: "The failure argument shall not be memory_order_release 442 // nor memory_order_acq_rel". Fallback to monotonic. 443 case llvm::AtomicOrderingCABI::release: 444 case llvm::AtomicOrderingCABI::acq_rel: 445 FailureOrder = llvm::AtomicOrdering::Monotonic; 446 break; 447 case llvm::AtomicOrderingCABI::consume: 448 case llvm::AtomicOrderingCABI::acquire: 449 FailureOrder = llvm::AtomicOrdering::Acquire; 450 break; 451 case llvm::AtomicOrderingCABI::seq_cst: 452 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 453 break; 454 } 455 // Prior to c++17, "the failure argument shall be no stronger than the 456 // success argument". This condition has been lifted and the only 457 // precondition is 31.7.2.18. Effectively treat this as a DR and skip 458 // language version checks. 459 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 460 FailureOrder, Scope); 461 return; 462 } 463 464 // Create all the relevant BB's 465 auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 466 auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 467 auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 468 auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 469 470 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 471 // doesn't matter unless someone is crazy enough to use something that 472 // doesn't fold to a constant for the ordering. 473 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 474 // Implemented as acquire, since it's the closest in LLVM. 475 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 476 AcquireBB); 477 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 478 AcquireBB); 479 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 480 SeqCstBB); 481 482 // Emit all the different atomics 483 CGF.Builder.SetInsertPoint(MonotonicBB); 484 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 485 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); 486 CGF.Builder.CreateBr(ContBB); 487 488 CGF.Builder.SetInsertPoint(AcquireBB); 489 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 490 llvm::AtomicOrdering::Acquire, Scope); 491 CGF.Builder.CreateBr(ContBB); 492 493 CGF.Builder.SetInsertPoint(SeqCstBB); 494 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 495 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 496 CGF.Builder.CreateBr(ContBB); 497 498 CGF.Builder.SetInsertPoint(ContBB); 499 } 500 501 /// Duplicate the atomic min/max operation in conventional IR for the builtin 502 /// variants that return the new rather than the original value. 503 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, 504 AtomicExpr::AtomicOp Op, 505 bool IsSigned, 506 llvm::Value *OldVal, 507 llvm::Value *RHS) { 508 llvm::CmpInst::Predicate Pred; 509 switch (Op) { 510 default: 511 llvm_unreachable("Unexpected min/max operation"); 512 case AtomicExpr::AO__atomic_max_fetch: 513 case AtomicExpr::AO__scoped_atomic_max_fetch: 514 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT; 515 break; 516 case AtomicExpr::AO__atomic_min_fetch: 517 case AtomicExpr::AO__scoped_atomic_min_fetch: 518 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT; 519 break; 520 } 521 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst"); 522 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval"); 523 } 524 525 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 526 Address Ptr, Address Val1, Address Val2, 527 llvm::Value *IsWeak, llvm::Value *FailureOrder, 528 uint64_t Size, llvm::AtomicOrdering Order, 529 llvm::SyncScope::ID Scope) { 530 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 531 bool PostOpMinMax = false; 532 unsigned PostOp = 0; 533 534 switch (E->getOp()) { 535 case AtomicExpr::AO__c11_atomic_init: 536 case AtomicExpr::AO__opencl_atomic_init: 537 llvm_unreachable("Already handled!"); 538 539 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 540 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 541 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 542 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 543 FailureOrder, Size, Order, Scope); 544 return; 545 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 546 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 547 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 548 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 549 FailureOrder, Size, Order, Scope); 550 return; 551 case AtomicExpr::AO__atomic_compare_exchange: 552 case AtomicExpr::AO__atomic_compare_exchange_n: 553 case AtomicExpr::AO__scoped_atomic_compare_exchange: 554 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: { 555 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 556 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 557 Val1, Val2, FailureOrder, Size, Order, Scope); 558 } else { 559 // Create all the relevant BB's 560 llvm::BasicBlock *StrongBB = 561 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 562 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 563 llvm::BasicBlock *ContBB = 564 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 565 566 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 567 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 568 569 CGF.Builder.SetInsertPoint(StrongBB); 570 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 571 FailureOrder, Size, Order, Scope); 572 CGF.Builder.CreateBr(ContBB); 573 574 CGF.Builder.SetInsertPoint(WeakBB); 575 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 576 FailureOrder, Size, Order, Scope); 577 CGF.Builder.CreateBr(ContBB); 578 579 CGF.Builder.SetInsertPoint(ContBB); 580 } 581 return; 582 } 583 case AtomicExpr::AO__c11_atomic_load: 584 case AtomicExpr::AO__opencl_atomic_load: 585 case AtomicExpr::AO__hip_atomic_load: 586 case AtomicExpr::AO__atomic_load_n: 587 case AtomicExpr::AO__atomic_load: 588 case AtomicExpr::AO__scoped_atomic_load_n: 589 case AtomicExpr::AO__scoped_atomic_load: { 590 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 591 Load->setAtomic(Order, Scope); 592 Load->setVolatile(E->isVolatile()); 593 CGF.Builder.CreateStore(Load, Dest); 594 return; 595 } 596 597 case AtomicExpr::AO__c11_atomic_store: 598 case AtomicExpr::AO__opencl_atomic_store: 599 case AtomicExpr::AO__hip_atomic_store: 600 case AtomicExpr::AO__atomic_store: 601 case AtomicExpr::AO__atomic_store_n: 602 case AtomicExpr::AO__scoped_atomic_store: 603 case AtomicExpr::AO__scoped_atomic_store_n: { 604 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 605 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 606 Store->setAtomic(Order, Scope); 607 Store->setVolatile(E->isVolatile()); 608 return; 609 } 610 611 case AtomicExpr::AO__c11_atomic_exchange: 612 case AtomicExpr::AO__hip_atomic_exchange: 613 case AtomicExpr::AO__opencl_atomic_exchange: 614 case AtomicExpr::AO__atomic_exchange_n: 615 case AtomicExpr::AO__atomic_exchange: 616 case AtomicExpr::AO__scoped_atomic_exchange_n: 617 case AtomicExpr::AO__scoped_atomic_exchange: 618 Op = llvm::AtomicRMWInst::Xchg; 619 break; 620 621 case AtomicExpr::AO__atomic_add_fetch: 622 case AtomicExpr::AO__scoped_atomic_add_fetch: 623 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd 624 : llvm::Instruction::Add; 625 [[fallthrough]]; 626 case AtomicExpr::AO__c11_atomic_fetch_add: 627 case AtomicExpr::AO__hip_atomic_fetch_add: 628 case AtomicExpr::AO__opencl_atomic_fetch_add: 629 case AtomicExpr::AO__atomic_fetch_add: 630 case AtomicExpr::AO__scoped_atomic_fetch_add: 631 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd 632 : llvm::AtomicRMWInst::Add; 633 break; 634 635 case AtomicExpr::AO__atomic_sub_fetch: 636 case AtomicExpr::AO__scoped_atomic_sub_fetch: 637 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub 638 : llvm::Instruction::Sub; 639 [[fallthrough]]; 640 case AtomicExpr::AO__c11_atomic_fetch_sub: 641 case AtomicExpr::AO__hip_atomic_fetch_sub: 642 case AtomicExpr::AO__opencl_atomic_fetch_sub: 643 case AtomicExpr::AO__atomic_fetch_sub: 644 case AtomicExpr::AO__scoped_atomic_fetch_sub: 645 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub 646 : llvm::AtomicRMWInst::Sub; 647 break; 648 649 case AtomicExpr::AO__atomic_min_fetch: 650 case AtomicExpr::AO__scoped_atomic_min_fetch: 651 PostOpMinMax = true; 652 [[fallthrough]]; 653 case AtomicExpr::AO__c11_atomic_fetch_min: 654 case AtomicExpr::AO__hip_atomic_fetch_min: 655 case AtomicExpr::AO__opencl_atomic_fetch_min: 656 case AtomicExpr::AO__atomic_fetch_min: 657 case AtomicExpr::AO__scoped_atomic_fetch_min: 658 Op = E->getValueType()->isFloatingType() 659 ? llvm::AtomicRMWInst::FMin 660 : (E->getValueType()->isSignedIntegerType() 661 ? llvm::AtomicRMWInst::Min 662 : llvm::AtomicRMWInst::UMin); 663 break; 664 665 case AtomicExpr::AO__atomic_max_fetch: 666 case AtomicExpr::AO__scoped_atomic_max_fetch: 667 PostOpMinMax = true; 668 [[fallthrough]]; 669 case AtomicExpr::AO__c11_atomic_fetch_max: 670 case AtomicExpr::AO__hip_atomic_fetch_max: 671 case AtomicExpr::AO__opencl_atomic_fetch_max: 672 case AtomicExpr::AO__atomic_fetch_max: 673 case AtomicExpr::AO__scoped_atomic_fetch_max: 674 Op = E->getValueType()->isFloatingType() 675 ? llvm::AtomicRMWInst::FMax 676 : (E->getValueType()->isSignedIntegerType() 677 ? llvm::AtomicRMWInst::Max 678 : llvm::AtomicRMWInst::UMax); 679 break; 680 681 case AtomicExpr::AO__atomic_and_fetch: 682 case AtomicExpr::AO__scoped_atomic_and_fetch: 683 PostOp = llvm::Instruction::And; 684 [[fallthrough]]; 685 case AtomicExpr::AO__c11_atomic_fetch_and: 686 case AtomicExpr::AO__hip_atomic_fetch_and: 687 case AtomicExpr::AO__opencl_atomic_fetch_and: 688 case AtomicExpr::AO__atomic_fetch_and: 689 case AtomicExpr::AO__scoped_atomic_fetch_and: 690 Op = llvm::AtomicRMWInst::And; 691 break; 692 693 case AtomicExpr::AO__atomic_or_fetch: 694 case AtomicExpr::AO__scoped_atomic_or_fetch: 695 PostOp = llvm::Instruction::Or; 696 [[fallthrough]]; 697 case AtomicExpr::AO__c11_atomic_fetch_or: 698 case AtomicExpr::AO__hip_atomic_fetch_or: 699 case AtomicExpr::AO__opencl_atomic_fetch_or: 700 case AtomicExpr::AO__atomic_fetch_or: 701 case AtomicExpr::AO__scoped_atomic_fetch_or: 702 Op = llvm::AtomicRMWInst::Or; 703 break; 704 705 case AtomicExpr::AO__atomic_xor_fetch: 706 case AtomicExpr::AO__scoped_atomic_xor_fetch: 707 PostOp = llvm::Instruction::Xor; 708 [[fallthrough]]; 709 case AtomicExpr::AO__c11_atomic_fetch_xor: 710 case AtomicExpr::AO__hip_atomic_fetch_xor: 711 case AtomicExpr::AO__opencl_atomic_fetch_xor: 712 case AtomicExpr::AO__atomic_fetch_xor: 713 case AtomicExpr::AO__scoped_atomic_fetch_xor: 714 Op = llvm::AtomicRMWInst::Xor; 715 break; 716 717 case AtomicExpr::AO__atomic_nand_fetch: 718 case AtomicExpr::AO__scoped_atomic_nand_fetch: 719 PostOp = llvm::Instruction::And; // the NOT is special cased below 720 [[fallthrough]]; 721 case AtomicExpr::AO__c11_atomic_fetch_nand: 722 case AtomicExpr::AO__atomic_fetch_nand: 723 case AtomicExpr::AO__scoped_atomic_fetch_nand: 724 Op = llvm::AtomicRMWInst::Nand; 725 break; 726 } 727 728 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 729 llvm::AtomicRMWInst *RMWI = 730 CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order, Scope); 731 RMWI->setVolatile(E->isVolatile()); 732 733 // For __atomic_*_fetch operations, perform the operation again to 734 // determine the value which was written. 735 llvm::Value *Result = RMWI; 736 if (PostOpMinMax) 737 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(), 738 E->getValueType()->isSignedIntegerType(), 739 RMWI, LoadVal1); 740 else if (PostOp) 741 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, 742 LoadVal1); 743 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch || 744 E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch) 745 Result = CGF.Builder.CreateNot(Result); 746 CGF.Builder.CreateStore(Result, Dest); 747 } 748 749 // This function emits any expression (scalar, complex, or aggregate) 750 // into a temporary alloca. 751 static Address 752 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 753 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 754 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 755 /*Init*/ true); 756 return DeclPtr; 757 } 758 759 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, 760 Address Ptr, Address Val1, Address Val2, 761 llvm::Value *IsWeak, llvm::Value *FailureOrder, 762 uint64_t Size, llvm::AtomicOrdering Order, 763 llvm::Value *Scope) { 764 auto ScopeModel = Expr->getScopeModel(); 765 766 // LLVM atomic instructions always have synch scope. If clang atomic 767 // expression has no scope operand, use default LLVM synch scope. 768 if (!ScopeModel) { 769 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 770 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID("")); 771 return; 772 } 773 774 // Handle constant scope. 775 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { 776 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( 777 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()), 778 Order, CGF.CGM.getLLVMContext()); 779 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 780 Order, SCID); 781 return; 782 } 783 784 // Handle non-constant scope. 785 auto &Builder = CGF.Builder; 786 auto Scopes = ScopeModel->getRuntimeValues(); 787 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; 788 for (auto S : Scopes) 789 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); 790 791 llvm::BasicBlock *ContBB = 792 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn); 793 794 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); 795 // If unsupported synch scope is encountered at run time, assume a fallback 796 // synch scope value. 797 auto FallBack = ScopeModel->getFallBackValue(); 798 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); 799 for (auto S : Scopes) { 800 auto *B = BB[S]; 801 if (S != FallBack) 802 SI->addCase(Builder.getInt32(S), B); 803 804 Builder.SetInsertPoint(B); 805 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 806 Order, 807 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(), 808 ScopeModel->map(S), 809 Order, 810 CGF.getLLVMContext())); 811 Builder.CreateBr(ContBB); 812 } 813 814 Builder.SetInsertPoint(ContBB); 815 } 816 817 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 818 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 819 QualType MemTy = AtomicTy; 820 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 821 MemTy = AT->getValueType(); 822 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 823 824 Address Val1 = Address::invalid(); 825 Address Val2 = Address::invalid(); 826 Address Dest = Address::invalid(); 827 Address Ptr = EmitPointerWithAlignment(E->getPtr()); 828 829 if (E->getOp() == AtomicExpr::AO__c11_atomic_init || 830 E->getOp() == AtomicExpr::AO__opencl_atomic_init) { 831 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 832 EmitAtomicInit(E->getVal1(), lvalue); 833 return RValue::get(nullptr); 834 } 835 836 auto TInfo = getContext().getTypeInfoInChars(AtomicTy); 837 uint64_t Size = TInfo.Width.getQuantity(); 838 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 839 840 CharUnits MaxInlineWidth = 841 getContext().toCharUnitsFromBits(MaxInlineWidthInBits); 842 DiagnosticsEngine &Diags = CGM.getDiags(); 843 bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0; 844 bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits; 845 if (Misaligned) { 846 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned) 847 << (int)TInfo.Width.getQuantity() 848 << (int)Ptr.getAlignment().getQuantity(); 849 } 850 if (Oversized) { 851 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized) 852 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity(); 853 } 854 855 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 856 llvm::Value *Scope = 857 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; 858 bool ShouldCastToIntPtrTy = true; 859 860 switch (E->getOp()) { 861 case AtomicExpr::AO__c11_atomic_init: 862 case AtomicExpr::AO__opencl_atomic_init: 863 llvm_unreachable("Already handled above with EmitAtomicInit!"); 864 865 case AtomicExpr::AO__atomic_load_n: 866 case AtomicExpr::AO__scoped_atomic_load_n: 867 case AtomicExpr::AO__c11_atomic_load: 868 case AtomicExpr::AO__opencl_atomic_load: 869 case AtomicExpr::AO__hip_atomic_load: 870 break; 871 872 case AtomicExpr::AO__atomic_load: 873 case AtomicExpr::AO__scoped_atomic_load: 874 Dest = EmitPointerWithAlignment(E->getVal1()); 875 break; 876 877 case AtomicExpr::AO__atomic_store: 878 case AtomicExpr::AO__scoped_atomic_store: 879 Val1 = EmitPointerWithAlignment(E->getVal1()); 880 break; 881 882 case AtomicExpr::AO__atomic_exchange: 883 case AtomicExpr::AO__scoped_atomic_exchange: 884 Val1 = EmitPointerWithAlignment(E->getVal1()); 885 Dest = EmitPointerWithAlignment(E->getVal2()); 886 break; 887 888 case AtomicExpr::AO__atomic_compare_exchange: 889 case AtomicExpr::AO__atomic_compare_exchange_n: 890 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 891 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 892 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 893 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 894 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 895 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 896 case AtomicExpr::AO__scoped_atomic_compare_exchange: 897 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: 898 Val1 = EmitPointerWithAlignment(E->getVal1()); 899 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange || 900 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange) 901 Val2 = EmitPointerWithAlignment(E->getVal2()); 902 else 903 Val2 = EmitValToTemp(*this, E->getVal2()); 904 OrderFail = EmitScalarExpr(E->getOrderFail()); 905 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || 906 E->getOp() == AtomicExpr::AO__atomic_compare_exchange || 907 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n || 908 E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange) 909 IsWeak = EmitScalarExpr(E->getWeak()); 910 break; 911 912 case AtomicExpr::AO__c11_atomic_fetch_add: 913 case AtomicExpr::AO__c11_atomic_fetch_sub: 914 case AtomicExpr::AO__hip_atomic_fetch_add: 915 case AtomicExpr::AO__hip_atomic_fetch_sub: 916 case AtomicExpr::AO__opencl_atomic_fetch_add: 917 case AtomicExpr::AO__opencl_atomic_fetch_sub: 918 if (MemTy->isPointerType()) { 919 // For pointer arithmetic, we're required to do a bit of math: 920 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 921 // ... but only for the C11 builtins. The GNU builtins expect the 922 // user to multiply by sizeof(T). 923 QualType Val1Ty = E->getVal1()->getType(); 924 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 925 CharUnits PointeeIncAmt = 926 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 927 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 928 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 929 Val1 = Temp; 930 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 931 break; 932 } 933 [[fallthrough]]; 934 case AtomicExpr::AO__atomic_fetch_add: 935 case AtomicExpr::AO__atomic_fetch_max: 936 case AtomicExpr::AO__atomic_fetch_min: 937 case AtomicExpr::AO__atomic_fetch_sub: 938 case AtomicExpr::AO__atomic_add_fetch: 939 case AtomicExpr::AO__atomic_max_fetch: 940 case AtomicExpr::AO__atomic_min_fetch: 941 case AtomicExpr::AO__atomic_sub_fetch: 942 case AtomicExpr::AO__c11_atomic_fetch_max: 943 case AtomicExpr::AO__c11_atomic_fetch_min: 944 case AtomicExpr::AO__opencl_atomic_fetch_max: 945 case AtomicExpr::AO__opencl_atomic_fetch_min: 946 case AtomicExpr::AO__hip_atomic_fetch_max: 947 case AtomicExpr::AO__hip_atomic_fetch_min: 948 case AtomicExpr::AO__scoped_atomic_fetch_add: 949 case AtomicExpr::AO__scoped_atomic_fetch_max: 950 case AtomicExpr::AO__scoped_atomic_fetch_min: 951 case AtomicExpr::AO__scoped_atomic_fetch_sub: 952 case AtomicExpr::AO__scoped_atomic_add_fetch: 953 case AtomicExpr::AO__scoped_atomic_max_fetch: 954 case AtomicExpr::AO__scoped_atomic_min_fetch: 955 case AtomicExpr::AO__scoped_atomic_sub_fetch: 956 ShouldCastToIntPtrTy = !MemTy->isFloatingType(); 957 [[fallthrough]]; 958 959 case AtomicExpr::AO__atomic_fetch_and: 960 case AtomicExpr::AO__atomic_fetch_nand: 961 case AtomicExpr::AO__atomic_fetch_or: 962 case AtomicExpr::AO__atomic_fetch_xor: 963 case AtomicExpr::AO__atomic_and_fetch: 964 case AtomicExpr::AO__atomic_nand_fetch: 965 case AtomicExpr::AO__atomic_or_fetch: 966 case AtomicExpr::AO__atomic_xor_fetch: 967 case AtomicExpr::AO__atomic_store_n: 968 case AtomicExpr::AO__atomic_exchange_n: 969 case AtomicExpr::AO__c11_atomic_fetch_and: 970 case AtomicExpr::AO__c11_atomic_fetch_nand: 971 case AtomicExpr::AO__c11_atomic_fetch_or: 972 case AtomicExpr::AO__c11_atomic_fetch_xor: 973 case AtomicExpr::AO__c11_atomic_store: 974 case AtomicExpr::AO__c11_atomic_exchange: 975 case AtomicExpr::AO__hip_atomic_fetch_and: 976 case AtomicExpr::AO__hip_atomic_fetch_or: 977 case AtomicExpr::AO__hip_atomic_fetch_xor: 978 case AtomicExpr::AO__hip_atomic_store: 979 case AtomicExpr::AO__hip_atomic_exchange: 980 case AtomicExpr::AO__opencl_atomic_fetch_and: 981 case AtomicExpr::AO__opencl_atomic_fetch_or: 982 case AtomicExpr::AO__opencl_atomic_fetch_xor: 983 case AtomicExpr::AO__opencl_atomic_store: 984 case AtomicExpr::AO__opencl_atomic_exchange: 985 case AtomicExpr::AO__scoped_atomic_fetch_and: 986 case AtomicExpr::AO__scoped_atomic_fetch_nand: 987 case AtomicExpr::AO__scoped_atomic_fetch_or: 988 case AtomicExpr::AO__scoped_atomic_fetch_xor: 989 case AtomicExpr::AO__scoped_atomic_and_fetch: 990 case AtomicExpr::AO__scoped_atomic_nand_fetch: 991 case AtomicExpr::AO__scoped_atomic_or_fetch: 992 case AtomicExpr::AO__scoped_atomic_xor_fetch: 993 case AtomicExpr::AO__scoped_atomic_store_n: 994 case AtomicExpr::AO__scoped_atomic_exchange_n: 995 Val1 = EmitValToTemp(*this, E->getVal1()); 996 break; 997 } 998 999 QualType RValTy = E->getType().getUnqualifiedType(); 1000 1001 // The inlined atomics only function on iN types, where N is a power of 2. We 1002 // need to make sure (via temporaries if necessary) that all incoming values 1003 // are compatible. 1004 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 1005 AtomicInfo Atomics(*this, AtomicVal); 1006 1007 if (ShouldCastToIntPtrTy) { 1008 Ptr = Atomics.castToAtomicIntPointer(Ptr); 1009 if (Val1.isValid()) 1010 Val1 = Atomics.convertToAtomicIntPointer(Val1); 1011 if (Val2.isValid()) 1012 Val2 = Atomics.convertToAtomicIntPointer(Val2); 1013 } 1014 if (Dest.isValid()) { 1015 if (ShouldCastToIntPtrTy) 1016 Dest = Atomics.castToAtomicIntPointer(Dest); 1017 } else if (E->isCmpXChg()) 1018 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 1019 else if (!RValTy->isVoidType()) { 1020 Dest = Atomics.CreateTempAlloca(); 1021 if (ShouldCastToIntPtrTy) 1022 Dest = Atomics.castToAtomicIntPointer(Dest); 1023 } 1024 1025 bool PowerOf2Size = (Size & (Size - 1)) == 0; 1026 bool UseLibcall = !PowerOf2Size || (Size > 16); 1027 1028 // For atomics larger than 16 bytes, emit a libcall from the frontend. This 1029 // avoids the overhead of dealing with excessively-large value types in IR. 1030 // Non-power-of-2 values also lower to libcall here, as they are not currently 1031 // permitted in IR instructions (although that constraint could be relaxed in 1032 // the future). For other cases where a libcall is required on a given 1033 // platform, we let the backend handle it (this includes handling for all of 1034 // the size-optimized libcall variants, which are only valid up to 16 bytes.) 1035 // 1036 // See: https://llvm.org/docs/Atomics.html#libcalls-atomic 1037 if (UseLibcall) { 1038 CallArgList Args; 1039 // For non-optimized library calls, the size is the first parameter. 1040 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 1041 getContext().getSizeType()); 1042 1043 // The atomic address is the second parameter. 1044 // The OpenCL atomic library functions only accept pointer arguments to 1045 // generic address space. 1046 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { 1047 if (!E->isOpenCL()) 1048 return V; 1049 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace(); 1050 if (AS == LangAS::opencl_generic) 1051 return V; 1052 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); 1053 auto *DestType = llvm::PointerType::get(getLLVMContext(), DestAS); 1054 1055 return getTargetHooks().performAddrSpaceCast( 1056 *this, V, AS, LangAS::opencl_generic, DestType, false); 1057 }; 1058 1059 Args.add(RValue::get(CastToGenericAddrSpace(Ptr.emitRawPointer(*this), 1060 E->getPtr()->getType())), 1061 getContext().VoidPtrTy); 1062 1063 // The next 1-3 parameters are op-dependent. 1064 std::string LibCallName; 1065 QualType RetTy; 1066 bool HaveRetTy = false; 1067 switch (E->getOp()) { 1068 case AtomicExpr::AO__c11_atomic_init: 1069 case AtomicExpr::AO__opencl_atomic_init: 1070 llvm_unreachable("Already handled!"); 1071 1072 // There is only one libcall for compare an exchange, because there is no 1073 // optimisation benefit possible from a libcall version of a weak compare 1074 // and exchange. 1075 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 1076 // void *desired, int success, int failure) 1077 case AtomicExpr::AO__atomic_compare_exchange: 1078 case AtomicExpr::AO__atomic_compare_exchange_n: 1079 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 1080 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 1081 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 1082 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 1083 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 1084 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 1085 case AtomicExpr::AO__scoped_atomic_compare_exchange: 1086 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: 1087 LibCallName = "__atomic_compare_exchange"; 1088 RetTy = getContext().BoolTy; 1089 HaveRetTy = true; 1090 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), 1091 E->getVal1()->getType())), 1092 getContext().VoidPtrTy); 1093 Args.add(RValue::get(CastToGenericAddrSpace(Val2.emitRawPointer(*this), 1094 E->getVal2()->getType())), 1095 getContext().VoidPtrTy); 1096 Args.add(RValue::get(Order), getContext().IntTy); 1097 Order = OrderFail; 1098 break; 1099 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 1100 // int order) 1101 case AtomicExpr::AO__atomic_exchange: 1102 case AtomicExpr::AO__atomic_exchange_n: 1103 case AtomicExpr::AO__c11_atomic_exchange: 1104 case AtomicExpr::AO__hip_atomic_exchange: 1105 case AtomicExpr::AO__opencl_atomic_exchange: 1106 case AtomicExpr::AO__scoped_atomic_exchange: 1107 case AtomicExpr::AO__scoped_atomic_exchange_n: 1108 LibCallName = "__atomic_exchange"; 1109 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), 1110 E->getVal1()->getType())), 1111 getContext().VoidPtrTy); 1112 break; 1113 // void __atomic_store(size_t size, void *mem, void *val, int order) 1114 case AtomicExpr::AO__atomic_store: 1115 case AtomicExpr::AO__atomic_store_n: 1116 case AtomicExpr::AO__c11_atomic_store: 1117 case AtomicExpr::AO__hip_atomic_store: 1118 case AtomicExpr::AO__opencl_atomic_store: 1119 case AtomicExpr::AO__scoped_atomic_store: 1120 case AtomicExpr::AO__scoped_atomic_store_n: 1121 LibCallName = "__atomic_store"; 1122 RetTy = getContext().VoidTy; 1123 HaveRetTy = true; 1124 Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this), 1125 E->getVal1()->getType())), 1126 getContext().VoidPtrTy); 1127 break; 1128 // void __atomic_load(size_t size, void *mem, void *return, int order) 1129 case AtomicExpr::AO__atomic_load: 1130 case AtomicExpr::AO__atomic_load_n: 1131 case AtomicExpr::AO__c11_atomic_load: 1132 case AtomicExpr::AO__hip_atomic_load: 1133 case AtomicExpr::AO__opencl_atomic_load: 1134 case AtomicExpr::AO__scoped_atomic_load: 1135 case AtomicExpr::AO__scoped_atomic_load_n: 1136 LibCallName = "__atomic_load"; 1137 break; 1138 case AtomicExpr::AO__atomic_add_fetch: 1139 case AtomicExpr::AO__scoped_atomic_add_fetch: 1140 case AtomicExpr::AO__atomic_fetch_add: 1141 case AtomicExpr::AO__c11_atomic_fetch_add: 1142 case AtomicExpr::AO__hip_atomic_fetch_add: 1143 case AtomicExpr::AO__opencl_atomic_fetch_add: 1144 case AtomicExpr::AO__scoped_atomic_fetch_add: 1145 case AtomicExpr::AO__atomic_and_fetch: 1146 case AtomicExpr::AO__scoped_atomic_and_fetch: 1147 case AtomicExpr::AO__atomic_fetch_and: 1148 case AtomicExpr::AO__c11_atomic_fetch_and: 1149 case AtomicExpr::AO__hip_atomic_fetch_and: 1150 case AtomicExpr::AO__opencl_atomic_fetch_and: 1151 case AtomicExpr::AO__scoped_atomic_fetch_and: 1152 case AtomicExpr::AO__atomic_or_fetch: 1153 case AtomicExpr::AO__scoped_atomic_or_fetch: 1154 case AtomicExpr::AO__atomic_fetch_or: 1155 case AtomicExpr::AO__c11_atomic_fetch_or: 1156 case AtomicExpr::AO__hip_atomic_fetch_or: 1157 case AtomicExpr::AO__opencl_atomic_fetch_or: 1158 case AtomicExpr::AO__scoped_atomic_fetch_or: 1159 case AtomicExpr::AO__atomic_sub_fetch: 1160 case AtomicExpr::AO__scoped_atomic_sub_fetch: 1161 case AtomicExpr::AO__atomic_fetch_sub: 1162 case AtomicExpr::AO__c11_atomic_fetch_sub: 1163 case AtomicExpr::AO__hip_atomic_fetch_sub: 1164 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1165 case AtomicExpr::AO__scoped_atomic_fetch_sub: 1166 case AtomicExpr::AO__atomic_xor_fetch: 1167 case AtomicExpr::AO__scoped_atomic_xor_fetch: 1168 case AtomicExpr::AO__atomic_fetch_xor: 1169 case AtomicExpr::AO__c11_atomic_fetch_xor: 1170 case AtomicExpr::AO__hip_atomic_fetch_xor: 1171 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1172 case AtomicExpr::AO__scoped_atomic_fetch_xor: 1173 case AtomicExpr::AO__atomic_nand_fetch: 1174 case AtomicExpr::AO__atomic_fetch_nand: 1175 case AtomicExpr::AO__c11_atomic_fetch_nand: 1176 case AtomicExpr::AO__scoped_atomic_fetch_nand: 1177 case AtomicExpr::AO__scoped_atomic_nand_fetch: 1178 case AtomicExpr::AO__atomic_min_fetch: 1179 case AtomicExpr::AO__atomic_fetch_min: 1180 case AtomicExpr::AO__c11_atomic_fetch_min: 1181 case AtomicExpr::AO__hip_atomic_fetch_min: 1182 case AtomicExpr::AO__opencl_atomic_fetch_min: 1183 case AtomicExpr::AO__scoped_atomic_fetch_min: 1184 case AtomicExpr::AO__scoped_atomic_min_fetch: 1185 case AtomicExpr::AO__atomic_max_fetch: 1186 case AtomicExpr::AO__atomic_fetch_max: 1187 case AtomicExpr::AO__c11_atomic_fetch_max: 1188 case AtomicExpr::AO__hip_atomic_fetch_max: 1189 case AtomicExpr::AO__opencl_atomic_fetch_max: 1190 case AtomicExpr::AO__scoped_atomic_fetch_max: 1191 case AtomicExpr::AO__scoped_atomic_max_fetch: 1192 llvm_unreachable("Integral atomic operations always become atomicrmw!"); 1193 } 1194 1195 if (E->isOpenCL()) { 1196 LibCallName = 1197 std::string("__opencl") + StringRef(LibCallName).drop_front(1).str(); 1198 } 1199 // By default, assume we return a value of the atomic type. 1200 if (!HaveRetTy) { 1201 // Value is returned through parameter before the order. 1202 RetTy = getContext().VoidTy; 1203 Args.add(RValue::get( 1204 CastToGenericAddrSpace(Dest.emitRawPointer(*this), RetTy)), 1205 getContext().VoidPtrTy); 1206 } 1207 // Order is always the last parameter. 1208 Args.add(RValue::get(Order), 1209 getContext().IntTy); 1210 if (E->isOpenCL()) 1211 Args.add(RValue::get(Scope), getContext().IntTy); 1212 1213 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 1214 // The value is returned directly from the libcall. 1215 if (E->isCmpXChg()) 1216 return Res; 1217 1218 if (RValTy->isVoidType()) 1219 return RValue::get(nullptr); 1220 1221 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), 1222 RValTy, E->getExprLoc()); 1223 } 1224 1225 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1226 E->getOp() == AtomicExpr::AO__opencl_atomic_store || 1227 E->getOp() == AtomicExpr::AO__hip_atomic_store || 1228 E->getOp() == AtomicExpr::AO__atomic_store || 1229 E->getOp() == AtomicExpr::AO__atomic_store_n || 1230 E->getOp() == AtomicExpr::AO__scoped_atomic_store || 1231 E->getOp() == AtomicExpr::AO__scoped_atomic_store_n; 1232 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1233 E->getOp() == AtomicExpr::AO__opencl_atomic_load || 1234 E->getOp() == AtomicExpr::AO__hip_atomic_load || 1235 E->getOp() == AtomicExpr::AO__atomic_load || 1236 E->getOp() == AtomicExpr::AO__atomic_load_n || 1237 E->getOp() == AtomicExpr::AO__scoped_atomic_load || 1238 E->getOp() == AtomicExpr::AO__scoped_atomic_load_n; 1239 1240 if (isa<llvm::ConstantInt>(Order)) { 1241 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1242 // We should not ever get to a case where the ordering isn't a valid C ABI 1243 // value, but it's hard to enforce that in general. 1244 if (llvm::isValidAtomicOrderingCABI(ord)) 1245 switch ((llvm::AtomicOrderingCABI)ord) { 1246 case llvm::AtomicOrderingCABI::relaxed: 1247 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1248 llvm::AtomicOrdering::Monotonic, Scope); 1249 break; 1250 case llvm::AtomicOrderingCABI::consume: 1251 case llvm::AtomicOrderingCABI::acquire: 1252 if (IsStore) 1253 break; // Avoid crashing on code with undefined behavior 1254 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1255 llvm::AtomicOrdering::Acquire, Scope); 1256 break; 1257 case llvm::AtomicOrderingCABI::release: 1258 if (IsLoad) 1259 break; // Avoid crashing on code with undefined behavior 1260 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1261 llvm::AtomicOrdering::Release, Scope); 1262 break; 1263 case llvm::AtomicOrderingCABI::acq_rel: 1264 if (IsLoad || IsStore) 1265 break; // Avoid crashing on code with undefined behavior 1266 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1267 llvm::AtomicOrdering::AcquireRelease, Scope); 1268 break; 1269 case llvm::AtomicOrderingCABI::seq_cst: 1270 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1271 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1272 break; 1273 } 1274 if (RValTy->isVoidType()) 1275 return RValue::get(nullptr); 1276 1277 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), 1278 RValTy, E->getExprLoc()); 1279 } 1280 1281 // Long case, when Order isn't obviously constant. 1282 1283 // Create all the relevant BB's 1284 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1285 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1286 *SeqCstBB = nullptr; 1287 MonotonicBB = createBasicBlock("monotonic", CurFn); 1288 if (!IsStore) 1289 AcquireBB = createBasicBlock("acquire", CurFn); 1290 if (!IsLoad) 1291 ReleaseBB = createBasicBlock("release", CurFn); 1292 if (!IsLoad && !IsStore) 1293 AcqRelBB = createBasicBlock("acqrel", CurFn); 1294 SeqCstBB = createBasicBlock("seqcst", CurFn); 1295 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1296 1297 // Create the switch for the split 1298 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1299 // doesn't matter unless someone is crazy enough to use something that 1300 // doesn't fold to a constant for the ordering. 1301 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1302 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1303 1304 // Emit all the different atomics 1305 Builder.SetInsertPoint(MonotonicBB); 1306 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1307 llvm::AtomicOrdering::Monotonic, Scope); 1308 Builder.CreateBr(ContBB); 1309 if (!IsStore) { 1310 Builder.SetInsertPoint(AcquireBB); 1311 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1312 llvm::AtomicOrdering::Acquire, Scope); 1313 Builder.CreateBr(ContBB); 1314 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1315 AcquireBB); 1316 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1317 AcquireBB); 1318 } 1319 if (!IsLoad) { 1320 Builder.SetInsertPoint(ReleaseBB); 1321 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1322 llvm::AtomicOrdering::Release, Scope); 1323 Builder.CreateBr(ContBB); 1324 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1325 ReleaseBB); 1326 } 1327 if (!IsLoad && !IsStore) { 1328 Builder.SetInsertPoint(AcqRelBB); 1329 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1330 llvm::AtomicOrdering::AcquireRelease, Scope); 1331 Builder.CreateBr(ContBB); 1332 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1333 AcqRelBB); 1334 } 1335 Builder.SetInsertPoint(SeqCstBB); 1336 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1337 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1338 Builder.CreateBr(ContBB); 1339 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1340 SeqCstBB); 1341 1342 // Cleanup and return 1343 Builder.SetInsertPoint(ContBB); 1344 if (RValTy->isVoidType()) 1345 return RValue::get(nullptr); 1346 1347 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1348 return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)), 1349 RValTy, E->getExprLoc()); 1350 } 1351 1352 Address AtomicInfo::castToAtomicIntPointer(Address addr) const { 1353 llvm::IntegerType *ty = 1354 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1355 return addr.withElementType(ty); 1356 } 1357 1358 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1359 llvm::Type *Ty = Addr.getElementType(); 1360 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1361 if (SourceSizeInBits != AtomicSizeInBits) { 1362 Address Tmp = CreateTempAlloca(); 1363 CGF.Builder.CreateMemCpy(Tmp, Addr, 1364 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1365 Addr = Tmp; 1366 } 1367 1368 return castToAtomicIntPointer(Addr); 1369 } 1370 1371 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1372 AggValueSlot resultSlot, 1373 SourceLocation loc, 1374 bool asValue) const { 1375 if (LVal.isSimple()) { 1376 if (EvaluationKind == TEK_Aggregate) 1377 return resultSlot.asRValue(); 1378 1379 // Drill into the padding structure if we have one. 1380 if (hasPadding()) 1381 addr = CGF.Builder.CreateStructGEP(addr, 0); 1382 1383 // Otherwise, just convert the temporary to an r-value using the 1384 // normal conversion routine. 1385 return CGF.convertTempToRValue(addr, getValueType(), loc); 1386 } 1387 if (!asValue) 1388 // Get RValue from temp memory as atomic for non-simple lvalues 1389 return RValue::get(CGF.Builder.CreateLoad(addr)); 1390 if (LVal.isBitField()) 1391 return CGF.EmitLoadOfBitfieldLValue( 1392 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1393 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1394 if (LVal.isVectorElt()) 1395 return CGF.EmitLoadOfLValue( 1396 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1397 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1398 assert(LVal.isExtVectorElt()); 1399 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1400 addr, LVal.getExtVectorElts(), LVal.getType(), 1401 LVal.getBaseInfo(), TBAAAccessInfo())); 1402 } 1403 1404 /// Return true if \param ValTy is a type that should be casted to integer 1405 /// around the atomic memory operation. If \param CmpXchg is true, then the 1406 /// cast of a floating point type is made as that instruction can not have 1407 /// floating point operands. TODO: Allow compare-and-exchange and FP - see 1408 /// comment in AtomicExpandPass.cpp. 1409 static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) { 1410 if (ValTy->isFloatingPointTy()) 1411 return ValTy->isX86_FP80Ty() || CmpXchg; 1412 return !ValTy->isIntegerTy() && !ValTy->isPointerTy(); 1413 } 1414 1415 RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val, 1416 AggValueSlot ResultSlot, 1417 SourceLocation Loc, bool AsValue, 1418 bool CmpXchg) const { 1419 // Try not to in some easy cases. 1420 assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() || 1421 Val->getType()->isIEEELikeFPTy()) && 1422 "Expected integer, pointer or floating point value when converting " 1423 "result."); 1424 if (getEvaluationKind() == TEK_Scalar && 1425 (((!LVal.isBitField() || 1426 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1427 !hasPadding()) || 1428 !AsValue)) { 1429 auto *ValTy = AsValue 1430 ? CGF.ConvertTypeForMem(ValueTy) 1431 : getAtomicAddress().getElementType(); 1432 if (!shouldCastToInt(ValTy, CmpXchg)) { 1433 assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) && 1434 "Different integer types."); 1435 return RValue::get(CGF.EmitFromMemory(Val, ValueTy)); 1436 } 1437 if (llvm::CastInst::isBitCastable(Val->getType(), ValTy)) 1438 return RValue::get(CGF.Builder.CreateBitCast(Val, ValTy)); 1439 } 1440 1441 // Create a temporary. This needs to be big enough to hold the 1442 // atomic integer. 1443 Address Temp = Address::invalid(); 1444 bool TempIsVolatile = false; 1445 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1446 assert(!ResultSlot.isIgnored()); 1447 Temp = ResultSlot.getAddress(); 1448 TempIsVolatile = ResultSlot.isVolatile(); 1449 } else { 1450 Temp = CreateTempAlloca(); 1451 } 1452 1453 // Slam the integer into the temporary. 1454 Address CastTemp = castToAtomicIntPointer(Temp); 1455 CGF.Builder.CreateStore(Val, CastTemp)->setVolatile(TempIsVolatile); 1456 1457 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1458 } 1459 1460 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1461 llvm::AtomicOrdering AO, bool) { 1462 // void __atomic_load(size_t size, void *mem, void *return, int order); 1463 CallArgList Args; 1464 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1465 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy); 1466 Args.add(RValue::get(AddForLoaded), CGF.getContext().VoidPtrTy); 1467 Args.add( 1468 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1469 CGF.getContext().IntTy); 1470 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1471 } 1472 1473 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1474 bool IsVolatile, bool CmpXchg) { 1475 // Okay, we're doing this natively. 1476 Address Addr = getAtomicAddress(); 1477 if (shouldCastToInt(Addr.getElementType(), CmpXchg)) 1478 Addr = castToAtomicIntPointer(Addr); 1479 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1480 Load->setAtomic(AO); 1481 1482 // Other decoration. 1483 if (IsVolatile) 1484 Load->setVolatile(true); 1485 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1486 return Load; 1487 } 1488 1489 /// An LValue is a candidate for having its loads and stores be made atomic if 1490 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1491 /// performing such an operation can be performed without a libcall. 1492 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1493 if (!CGM.getLangOpts().MSVolatile) return false; 1494 AtomicInfo AI(*this, LV); 1495 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1496 // An atomic is inline if we don't need to use a libcall. 1497 bool AtomicIsInline = !AI.shouldUseLibcall(); 1498 // MSVC doesn't seem to do this for types wider than a pointer. 1499 if (getContext().getTypeSize(LV.getType()) > 1500 getContext().getTypeSize(getContext().getIntPtrType())) 1501 return false; 1502 return IsVolatile && AtomicIsInline; 1503 } 1504 1505 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1506 AggValueSlot Slot) { 1507 llvm::AtomicOrdering AO; 1508 bool IsVolatile = LV.isVolatileQualified(); 1509 if (LV.getType()->isAtomicType()) { 1510 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1511 } else { 1512 AO = llvm::AtomicOrdering::Acquire; 1513 IsVolatile = true; 1514 } 1515 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1516 } 1517 1518 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1519 bool AsValue, llvm::AtomicOrdering AO, 1520 bool IsVolatile) { 1521 // Check whether we should use a library call. 1522 if (shouldUseLibcall()) { 1523 Address TempAddr = Address::invalid(); 1524 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1525 assert(getEvaluationKind() == TEK_Aggregate); 1526 TempAddr = ResultSlot.getAddress(); 1527 } else 1528 TempAddr = CreateTempAlloca(); 1529 1530 EmitAtomicLoadLibcall(TempAddr.emitRawPointer(CGF), AO, IsVolatile); 1531 1532 // Okay, turn that back into the original value or whole atomic (for 1533 // non-simple lvalues) type. 1534 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1535 } 1536 1537 // Okay, we're doing this natively. 1538 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1539 1540 // If we're ignoring an aggregate return, don't do anything. 1541 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1542 return RValue::getAggregate(Address::invalid(), false); 1543 1544 // Okay, turn that back into the original value or atomic (for non-simple 1545 // lvalues) type. 1546 return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1547 } 1548 1549 /// Emit a load from an l-value of atomic type. Note that the r-value 1550 /// we produce is an r-value of the atomic *value* type. 1551 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1552 llvm::AtomicOrdering AO, bool IsVolatile, 1553 AggValueSlot resultSlot) { 1554 AtomicInfo Atomics(*this, src); 1555 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1556 IsVolatile); 1557 } 1558 1559 /// Copy an r-value into memory as part of storing to an atomic type. 1560 /// This needs to create a bit-pattern suitable for atomic operations. 1561 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1562 assert(LVal.isSimple()); 1563 // If we have an r-value, the rvalue should be of the atomic type, 1564 // which means that the caller is responsible for having zeroed 1565 // any padding. Just do an aggregate copy of that type. 1566 if (rvalue.isAggregate()) { 1567 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); 1568 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), 1569 getAtomicType()); 1570 bool IsVolatile = rvalue.isVolatileQualified() || 1571 LVal.isVolatileQualified(); 1572 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), 1573 AggValueSlot::DoesNotOverlap, IsVolatile); 1574 return; 1575 } 1576 1577 // Okay, otherwise we're copying stuff. 1578 1579 // Zero out the buffer if necessary. 1580 emitMemSetZeroIfNecessary(); 1581 1582 // Drill past the padding if present. 1583 LValue TempLVal = projectValue(); 1584 1585 // Okay, store the rvalue in. 1586 if (rvalue.isScalar()) { 1587 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1588 } else { 1589 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1590 } 1591 } 1592 1593 1594 /// Materialize an r-value into memory for the purposes of storing it 1595 /// to an atomic type. 1596 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1597 // Aggregate r-values are already in memory, and EmitAtomicStore 1598 // requires them to be values of the atomic type. 1599 if (rvalue.isAggregate()) 1600 return rvalue.getAggregateAddress(); 1601 1602 // Otherwise, make a temporary and materialize into it. 1603 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1604 AtomicInfo Atomics(CGF, TempLV); 1605 Atomics.emitCopyIntoMemory(rvalue); 1606 return TempLV.getAddress(); 1607 } 1608 1609 llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const { 1610 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) 1611 return RVal.getScalarVal(); 1612 return nullptr; 1613 } 1614 1615 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const { 1616 // If we've got a scalar value of the right size, try to avoid going 1617 // through memory. Floats get casted if needed by AtomicExpandPass. 1618 if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) { 1619 if (!shouldCastToInt(Value->getType(), CmpXchg)) 1620 return CGF.EmitToMemory(Value, ValueTy); 1621 else { 1622 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1623 CGF.getLLVMContext(), 1624 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1625 if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1626 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1627 } 1628 } 1629 // Otherwise, we need to go through memory. 1630 // Put the r-value in memory. 1631 Address Addr = materializeRValue(RVal); 1632 1633 // Cast the temporary to the atomic int type and pull a value out. 1634 Addr = castToAtomicIntPointer(Addr); 1635 return CGF.Builder.CreateLoad(Addr); 1636 } 1637 1638 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1639 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1640 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1641 // Do the atomic store. 1642 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1643 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal, 1644 Success, Failure); 1645 // Other decoration. 1646 Inst->setVolatile(LVal.isVolatileQualified()); 1647 Inst->setWeak(IsWeak); 1648 1649 // Okay, turn that back into the original value type. 1650 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1651 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1652 return std::make_pair(PreviousVal, SuccessFailureVal); 1653 } 1654 1655 llvm::Value * 1656 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1657 llvm::Value *DesiredAddr, 1658 llvm::AtomicOrdering Success, 1659 llvm::AtomicOrdering Failure) { 1660 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1661 // void *desired, int success, int failure); 1662 CallArgList Args; 1663 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1664 Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy); 1665 Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy); 1666 Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy); 1667 Args.add(RValue::get( 1668 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1669 CGF.getContext().IntTy); 1670 Args.add(RValue::get( 1671 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1672 CGF.getContext().IntTy); 1673 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1674 CGF.getContext().BoolTy, Args); 1675 1676 return SuccessFailureRVal.getScalarVal(); 1677 } 1678 1679 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1680 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1681 llvm::AtomicOrdering Failure, bool IsWeak) { 1682 // Check whether we should use a library call. 1683 if (shouldUseLibcall()) { 1684 // Produce a source address. 1685 Address ExpectedAddr = materializeRValue(Expected); 1686 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); 1687 llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF); 1688 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, 1689 Success, Failure); 1690 return std::make_pair( 1691 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1692 SourceLocation(), /*AsValue=*/false), 1693 Res); 1694 } 1695 1696 // If we've got a scalar value of the right size, try to avoid going 1697 // through memory. 1698 auto *ExpectedVal = convertRValueToInt(Expected, /*CmpXchg=*/true); 1699 auto *DesiredVal = convertRValueToInt(Desired, /*CmpXchg=*/true); 1700 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1701 Failure, IsWeak); 1702 return std::make_pair( 1703 ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1704 SourceLocation(), /*AsValue=*/false, 1705 /*CmpXchg=*/true), 1706 Res.second); 1707 } 1708 1709 static void 1710 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1711 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1712 Address DesiredAddr) { 1713 RValue UpRVal; 1714 LValue AtomicLVal = Atomics.getAtomicLValue(); 1715 LValue DesiredLVal; 1716 if (AtomicLVal.isSimple()) { 1717 UpRVal = OldRVal; 1718 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1719 } else { 1720 // Build new lvalue for temp address. 1721 Address Ptr = Atomics.materializeRValue(OldRVal); 1722 LValue UpdateLVal; 1723 if (AtomicLVal.isBitField()) { 1724 UpdateLVal = 1725 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1726 AtomicLVal.getType(), 1727 AtomicLVal.getBaseInfo(), 1728 AtomicLVal.getTBAAInfo()); 1729 DesiredLVal = 1730 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1731 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1732 AtomicLVal.getTBAAInfo()); 1733 } else if (AtomicLVal.isVectorElt()) { 1734 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1735 AtomicLVal.getType(), 1736 AtomicLVal.getBaseInfo(), 1737 AtomicLVal.getTBAAInfo()); 1738 DesiredLVal = LValue::MakeVectorElt( 1739 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1740 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1741 } else { 1742 assert(AtomicLVal.isExtVectorElt()); 1743 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1744 AtomicLVal.getType(), 1745 AtomicLVal.getBaseInfo(), 1746 AtomicLVal.getTBAAInfo()); 1747 DesiredLVal = LValue::MakeExtVectorElt( 1748 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1749 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1750 } 1751 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1752 } 1753 // Store new value in the corresponding memory area. 1754 RValue NewRVal = UpdateOp(UpRVal); 1755 if (NewRVal.isScalar()) { 1756 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1757 } else { 1758 assert(NewRVal.isComplex()); 1759 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1760 /*isInit=*/false); 1761 } 1762 } 1763 1764 void AtomicInfo::EmitAtomicUpdateLibcall( 1765 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1766 bool IsVolatile) { 1767 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1768 1769 Address ExpectedAddr = CreateTempAlloca(); 1770 1771 EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); 1772 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1773 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1774 CGF.EmitBlock(ContBB); 1775 Address DesiredAddr = CreateTempAlloca(); 1776 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1777 requiresMemSetZero(getAtomicAddress().getElementType())) { 1778 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1779 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1780 } 1781 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1782 AggValueSlot::ignored(), 1783 SourceLocation(), /*AsValue=*/false); 1784 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1785 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); 1786 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); 1787 auto *Res = 1788 EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); 1789 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1790 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1791 } 1792 1793 void AtomicInfo::EmitAtomicUpdateOp( 1794 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1795 bool IsVolatile) { 1796 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1797 1798 // Do the atomic load. 1799 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true); 1800 // For non-simple lvalues perform compare-and-swap procedure. 1801 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1802 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1803 auto *CurBB = CGF.Builder.GetInsertBlock(); 1804 CGF.EmitBlock(ContBB); 1805 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1806 /*NumReservedValues=*/2); 1807 PHI->addIncoming(OldVal, CurBB); 1808 Address NewAtomicAddr = CreateTempAlloca(); 1809 Address NewAtomicIntAddr = 1810 shouldCastToInt(NewAtomicAddr.getElementType(), /*CmpXchg=*/true) 1811 ? castToAtomicIntPointer(NewAtomicAddr) 1812 : NewAtomicAddr; 1813 1814 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1815 requiresMemSetZero(getAtomicAddress().getElementType())) { 1816 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1817 } 1818 auto OldRVal = ConvertToValueOrAtomic(PHI, AggValueSlot::ignored(), 1819 SourceLocation(), /*AsValue=*/false, 1820 /*CmpXchg=*/true); 1821 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1822 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1823 // Try to write new value using cmpxchg operation. 1824 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1825 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1826 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1827 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1828 } 1829 1830 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1831 RValue UpdateRVal, Address DesiredAddr) { 1832 LValue AtomicLVal = Atomics.getAtomicLValue(); 1833 LValue DesiredLVal; 1834 // Build new lvalue for temp address. 1835 if (AtomicLVal.isBitField()) { 1836 DesiredLVal = 1837 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1838 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1839 AtomicLVal.getTBAAInfo()); 1840 } else if (AtomicLVal.isVectorElt()) { 1841 DesiredLVal = 1842 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1843 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1844 AtomicLVal.getTBAAInfo()); 1845 } else { 1846 assert(AtomicLVal.isExtVectorElt()); 1847 DesiredLVal = LValue::MakeExtVectorElt( 1848 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1849 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1850 } 1851 // Store new value in the corresponding memory area. 1852 assert(UpdateRVal.isScalar()); 1853 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1854 } 1855 1856 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1857 RValue UpdateRVal, bool IsVolatile) { 1858 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1859 1860 Address ExpectedAddr = CreateTempAlloca(); 1861 1862 EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile); 1863 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1864 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1865 CGF.EmitBlock(ContBB); 1866 Address DesiredAddr = CreateTempAlloca(); 1867 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1868 requiresMemSetZero(getAtomicAddress().getElementType())) { 1869 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1870 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1871 } 1872 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1873 llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF); 1874 llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF); 1875 auto *Res = 1876 EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure); 1877 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1878 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1879 } 1880 1881 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1882 bool IsVolatile) { 1883 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1884 1885 // Do the atomic load. 1886 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true); 1887 // For non-simple lvalues perform compare-and-swap procedure. 1888 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1889 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1890 auto *CurBB = CGF.Builder.GetInsertBlock(); 1891 CGF.EmitBlock(ContBB); 1892 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1893 /*NumReservedValues=*/2); 1894 PHI->addIncoming(OldVal, CurBB); 1895 Address NewAtomicAddr = CreateTempAlloca(); 1896 Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr); 1897 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1898 requiresMemSetZero(getAtomicAddress().getElementType())) { 1899 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1900 } 1901 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 1902 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1903 // Try to write new value using cmpxchg operation. 1904 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1905 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1906 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1907 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1908 } 1909 1910 void AtomicInfo::EmitAtomicUpdate( 1911 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1912 bool IsVolatile) { 1913 if (shouldUseLibcall()) { 1914 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 1915 } else { 1916 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 1917 } 1918 } 1919 1920 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 1921 bool IsVolatile) { 1922 if (shouldUseLibcall()) { 1923 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 1924 } else { 1925 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 1926 } 1927 } 1928 1929 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 1930 bool isInit) { 1931 bool IsVolatile = lvalue.isVolatileQualified(); 1932 llvm::AtomicOrdering AO; 1933 if (lvalue.getType()->isAtomicType()) { 1934 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1935 } else { 1936 AO = llvm::AtomicOrdering::Release; 1937 IsVolatile = true; 1938 } 1939 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 1940 } 1941 1942 /// Emit a store to an l-value of atomic type. 1943 /// 1944 /// Note that the r-value is expected to be an r-value *of the atomic 1945 /// type*; this means that for aggregate r-values, it should include 1946 /// storage for any padding that was necessary. 1947 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 1948 llvm::AtomicOrdering AO, bool IsVolatile, 1949 bool isInit) { 1950 // If this is an aggregate r-value, it should agree in type except 1951 // maybe for address-space qualification. 1952 assert(!rvalue.isAggregate() || 1953 rvalue.getAggregateAddress().getElementType() == 1954 dest.getAddress().getElementType()); 1955 1956 AtomicInfo atomics(*this, dest); 1957 LValue LVal = atomics.getAtomicLValue(); 1958 1959 // If this is an initialization, just put the value there normally. 1960 if (LVal.isSimple()) { 1961 if (isInit) { 1962 atomics.emitCopyIntoMemory(rvalue); 1963 return; 1964 } 1965 1966 // Check whether we should use a library call. 1967 if (atomics.shouldUseLibcall()) { 1968 // Produce a source address. 1969 Address srcAddr = atomics.materializeRValue(rvalue); 1970 1971 // void __atomic_store(size_t size, void *mem, void *val, int order) 1972 CallArgList args; 1973 args.add(RValue::get(atomics.getAtomicSizeValue()), 1974 getContext().getSizeType()); 1975 args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy); 1976 args.add(RValue::get(srcAddr.emitRawPointer(*this)), 1977 getContext().VoidPtrTy); 1978 args.add( 1979 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 1980 getContext().IntTy); 1981 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 1982 return; 1983 } 1984 1985 // Okay, we're doing this natively. 1986 llvm::Value *ValToStore = atomics.convertRValueToInt(rvalue); 1987 1988 // Do the atomic store. 1989 Address Addr = atomics.getAtomicAddress(); 1990 if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue)) 1991 if (shouldCastToInt(Value->getType(), /*CmpXchg=*/false)) { 1992 Addr = atomics.castToAtomicIntPointer(Addr); 1993 ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(), 1994 /*isSigned=*/false); 1995 } 1996 llvm::StoreInst *store = Builder.CreateStore(ValToStore, Addr); 1997 1998 if (AO == llvm::AtomicOrdering::Acquire) 1999 AO = llvm::AtomicOrdering::Monotonic; 2000 else if (AO == llvm::AtomicOrdering::AcquireRelease) 2001 AO = llvm::AtomicOrdering::Release; 2002 // Initializations don't need to be atomic. 2003 if (!isInit) 2004 store->setAtomic(AO); 2005 2006 // Other decoration. 2007 if (IsVolatile) 2008 store->setVolatile(true); 2009 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 2010 return; 2011 } 2012 2013 // Emit simple atomic update operation. 2014 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 2015 } 2016 2017 /// Emit a compare-and-exchange op for atomic type. 2018 /// 2019 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 2020 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 2021 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 2022 AggValueSlot Slot) { 2023 // If this is an aggregate r-value, it should agree in type except 2024 // maybe for address-space qualification. 2025 assert(!Expected.isAggregate() || 2026 Expected.getAggregateAddress().getElementType() == 2027 Obj.getAddress().getElementType()); 2028 assert(!Desired.isAggregate() || 2029 Desired.getAggregateAddress().getElementType() == 2030 Obj.getAddress().getElementType()); 2031 AtomicInfo Atomics(*this, Obj); 2032 2033 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 2034 IsWeak); 2035 } 2036 2037 void CodeGenFunction::EmitAtomicUpdate( 2038 LValue LVal, llvm::AtomicOrdering AO, 2039 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 2040 AtomicInfo Atomics(*this, LVal); 2041 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 2042 } 2043 2044 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 2045 AtomicInfo atomics(*this, dest); 2046 2047 switch (atomics.getEvaluationKind()) { 2048 case TEK_Scalar: { 2049 llvm::Value *value = EmitScalarExpr(init); 2050 atomics.emitCopyIntoMemory(RValue::get(value)); 2051 return; 2052 } 2053 2054 case TEK_Complex: { 2055 ComplexPairTy value = EmitComplexExpr(init); 2056 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 2057 return; 2058 } 2059 2060 case TEK_Aggregate: { 2061 // Fix up the destination if the initializer isn't an expression 2062 // of atomic type. 2063 bool Zeroed = false; 2064 if (!init->getType()->isAtomicType()) { 2065 Zeroed = atomics.emitMemSetZeroIfNecessary(); 2066 dest = atomics.projectValue(); 2067 } 2068 2069 // Evaluate the expression directly into the destination. 2070 AggValueSlot slot = AggValueSlot::forLValue( 2071 dest, AggValueSlot::IsNotDestructed, 2072 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, 2073 AggValueSlot::DoesNotOverlap, 2074 Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed); 2075 2076 EmitAggExpr(init, slot); 2077 return; 2078 } 2079 } 2080 llvm_unreachable("bad evaluation kind"); 2081 } 2082