1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the code for emitting atomic operations. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CGCall.h" 14 #include "CGRecordLayout.h" 15 #include "CodeGenFunction.h" 16 #include "CodeGenModule.h" 17 #include "TargetInfo.h" 18 #include "clang/AST/ASTContext.h" 19 #include "clang/CodeGen/CGFunctionInfo.h" 20 #include "clang/Frontend/FrontendDiagnostic.h" 21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/IR/DataLayout.h" 23 #include "llvm/IR/Intrinsics.h" 24 #include "llvm/IR/Operator.h" 25 26 using namespace clang; 27 using namespace CodeGen; 28 29 namespace { 30 class AtomicInfo { 31 CodeGenFunction &CGF; 32 QualType AtomicTy; 33 QualType ValueTy; 34 uint64_t AtomicSizeInBits; 35 uint64_t ValueSizeInBits; 36 CharUnits AtomicAlign; 37 CharUnits ValueAlign; 38 TypeEvaluationKind EvaluationKind; 39 bool UseLibcall; 40 LValue LVal; 41 CGBitFieldInfo BFI; 42 public: 43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 45 EvaluationKind(TEK_Scalar), UseLibcall(true) { 46 assert(!lvalue.isGlobalReg()); 47 ASTContext &C = CGF.getContext(); 48 if (lvalue.isSimple()) { 49 AtomicTy = lvalue.getType(); 50 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 51 ValueTy = ATy->getValueType(); 52 else 53 ValueTy = AtomicTy; 54 EvaluationKind = CGF.getEvaluationKind(ValueTy); 55 56 uint64_t ValueAlignInBits; 57 uint64_t AtomicAlignInBits; 58 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 59 ValueSizeInBits = ValueTI.Width; 60 ValueAlignInBits = ValueTI.Align; 61 62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 63 AtomicSizeInBits = AtomicTI.Width; 64 AtomicAlignInBits = AtomicTI.Align; 65 66 assert(ValueSizeInBits <= AtomicSizeInBits); 67 assert(ValueAlignInBits <= AtomicAlignInBits); 68 69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 71 if (lvalue.getAlignment().isZero()) 72 lvalue.setAlignment(AtomicAlign); 73 74 LVal = lvalue; 75 } else if (lvalue.isBitField()) { 76 ValueTy = lvalue.getType(); 77 ValueSizeInBits = C.getTypeSize(ValueTy); 78 auto &OrigBFI = lvalue.getBitFieldInfo(); 79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 80 AtomicSizeInBits = C.toBits( 81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 82 .alignTo(lvalue.getAlignment())); 83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); 84 auto OffsetInChars = 85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 86 lvalue.getAlignment(); 87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( 88 CGF.Int8Ty, VoidPtrAddr, OffsetInChars.getQuantity()); 89 llvm::Type *IntTy = CGF.Builder.getIntNTy(AtomicSizeInBits); 90 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 91 VoidPtrAddr, IntTy->getPointerTo(), "atomic_bitfield_base"); 92 BFI = OrigBFI; 93 BFI.Offset = Offset; 94 BFI.StorageSize = AtomicSizeInBits; 95 BFI.StorageOffset += OffsetInChars; 96 LVal = LValue::MakeBitfield(Address(Addr, IntTy, lvalue.getAlignment()), 97 BFI, lvalue.getType(), lvalue.getBaseInfo(), 98 lvalue.getTBAAInfo()); 99 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 100 if (AtomicTy.isNull()) { 101 llvm::APInt Size( 102 /*numBits=*/32, 103 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 104 AtomicTy = 105 C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal, 106 /*IndexTypeQuals=*/0); 107 } 108 AtomicAlign = ValueAlign = lvalue.getAlignment(); 109 } else if (lvalue.isVectorElt()) { 110 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType(); 111 ValueSizeInBits = C.getTypeSize(ValueTy); 112 AtomicTy = lvalue.getType(); 113 AtomicSizeInBits = C.getTypeSize(AtomicTy); 114 AtomicAlign = ValueAlign = lvalue.getAlignment(); 115 LVal = lvalue; 116 } else { 117 assert(lvalue.isExtVectorElt()); 118 ValueTy = lvalue.getType(); 119 ValueSizeInBits = C.getTypeSize(ValueTy); 120 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 121 lvalue.getType(), cast<llvm::FixedVectorType>( 122 lvalue.getExtVectorAddress().getElementType()) 123 ->getNumElements()); 124 AtomicSizeInBits = C.getTypeSize(AtomicTy); 125 AtomicAlign = ValueAlign = lvalue.getAlignment(); 126 LVal = lvalue; 127 } 128 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 129 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 130 } 131 132 QualType getAtomicType() const { return AtomicTy; } 133 QualType getValueType() const { return ValueTy; } 134 CharUnits getAtomicAlignment() const { return AtomicAlign; } 135 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 136 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 137 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 138 bool shouldUseLibcall() const { return UseLibcall; } 139 const LValue &getAtomicLValue() const { return LVal; } 140 llvm::Value *getAtomicPointer() const { 141 if (LVal.isSimple()) 142 return LVal.getPointer(CGF); 143 else if (LVal.isBitField()) 144 return LVal.getBitFieldPointer(); 145 else if (LVal.isVectorElt()) 146 return LVal.getVectorPointer(); 147 assert(LVal.isExtVectorElt()); 148 return LVal.getExtVectorPointer(); 149 } 150 Address getAtomicAddress() const { 151 llvm::Type *ElTy; 152 if (LVal.isSimple()) 153 ElTy = LVal.getAddress(CGF).getElementType(); 154 else if (LVal.isBitField()) 155 ElTy = LVal.getBitFieldAddress().getElementType(); 156 else if (LVal.isVectorElt()) 157 ElTy = LVal.getVectorAddress().getElementType(); 158 else 159 ElTy = LVal.getExtVectorAddress().getElementType(); 160 return Address(getAtomicPointer(), ElTy, getAtomicAlignment()); 161 } 162 163 Address getAtomicAddressAsAtomicIntPointer() const { 164 return emitCastToAtomicIntPointer(getAtomicAddress()); 165 } 166 167 /// Is the atomic size larger than the underlying value type? 168 /// 169 /// Note that the absence of padding does not mean that atomic 170 /// objects are completely interchangeable with non-atomic 171 /// objects: we might have promoted the alignment of a type 172 /// without making it bigger. 173 bool hasPadding() const { 174 return (ValueSizeInBits != AtomicSizeInBits); 175 } 176 177 bool emitMemSetZeroIfNecessary() const; 178 179 llvm::Value *getAtomicSizeValue() const { 180 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 181 return CGF.CGM.getSize(size); 182 } 183 184 /// Cast the given pointer to an integer pointer suitable for atomic 185 /// operations if the source. 186 Address emitCastToAtomicIntPointer(Address Addr) const; 187 188 /// If Addr is compatible with the iN that will be used for an atomic 189 /// operation, bitcast it. Otherwise, create a temporary that is suitable 190 /// and copy the value across. 191 Address convertToAtomicIntPointer(Address Addr) const; 192 193 /// Turn an atomic-layout object into an r-value. 194 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 195 SourceLocation loc, bool AsValue) const; 196 197 /// Converts a rvalue to integer value. 198 llvm::Value *convertRValueToInt(RValue RVal) const; 199 200 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, 201 AggValueSlot ResultSlot, 202 SourceLocation Loc, bool AsValue) const; 203 204 /// Copy an atomic r-value into atomic-layout memory. 205 void emitCopyIntoMemory(RValue rvalue) const; 206 207 /// Project an l-value down to the value field. 208 LValue projectValue() const { 209 assert(LVal.isSimple()); 210 Address addr = getAtomicAddress(); 211 if (hasPadding()) 212 addr = CGF.Builder.CreateStructGEP(addr, 0); 213 214 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 215 LVal.getBaseInfo(), LVal.getTBAAInfo()); 216 } 217 218 /// Emits atomic load. 219 /// \returns Loaded value. 220 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 221 bool AsValue, llvm::AtomicOrdering AO, 222 bool IsVolatile); 223 224 /// Emits atomic compare-and-exchange sequence. 225 /// \param Expected Expected value. 226 /// \param Desired Desired value. 227 /// \param Success Atomic ordering for success operation. 228 /// \param Failure Atomic ordering for failed operation. 229 /// \param IsWeak true if atomic operation is weak, false otherwise. 230 /// \returns Pair of values: previous value from storage (value type) and 231 /// boolean flag (i1 type) with true if success and false otherwise. 232 std::pair<RValue, llvm::Value *> 233 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 234 llvm::AtomicOrdering Success = 235 llvm::AtomicOrdering::SequentiallyConsistent, 236 llvm::AtomicOrdering Failure = 237 llvm::AtomicOrdering::SequentiallyConsistent, 238 bool IsWeak = false); 239 240 /// Emits atomic update. 241 /// \param AO Atomic ordering. 242 /// \param UpdateOp Update operation for the current lvalue. 243 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 244 const llvm::function_ref<RValue(RValue)> &UpdateOp, 245 bool IsVolatile); 246 /// Emits atomic update. 247 /// \param AO Atomic ordering. 248 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 249 bool IsVolatile); 250 251 /// Materialize an atomic r-value in atomic-layout memory. 252 Address materializeRValue(RValue rvalue) const; 253 254 /// Creates temp alloca for intermediate operations on atomic value. 255 Address CreateTempAlloca() const; 256 private: 257 bool requiresMemSetZero(llvm::Type *type) const; 258 259 260 /// Emits atomic load as a libcall. 261 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 262 llvm::AtomicOrdering AO, bool IsVolatile); 263 /// Emits atomic load as LLVM instruction. 264 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); 265 /// Emits atomic compare-and-exchange op as a libcall. 266 llvm::Value *EmitAtomicCompareExchangeLibcall( 267 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 268 llvm::AtomicOrdering Success = 269 llvm::AtomicOrdering::SequentiallyConsistent, 270 llvm::AtomicOrdering Failure = 271 llvm::AtomicOrdering::SequentiallyConsistent); 272 /// Emits atomic compare-and-exchange op as LLVM instruction. 273 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 274 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 275 llvm::AtomicOrdering Success = 276 llvm::AtomicOrdering::SequentiallyConsistent, 277 llvm::AtomicOrdering Failure = 278 llvm::AtomicOrdering::SequentiallyConsistent, 279 bool IsWeak = false); 280 /// Emit atomic update as libcalls. 281 void 282 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 283 const llvm::function_ref<RValue(RValue)> &UpdateOp, 284 bool IsVolatile); 285 /// Emit atomic update as LLVM instructions. 286 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 287 const llvm::function_ref<RValue(RValue)> &UpdateOp, 288 bool IsVolatile); 289 /// Emit atomic update as libcalls. 290 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 291 bool IsVolatile); 292 /// Emit atomic update as LLVM instructions. 293 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 294 bool IsVolatile); 295 }; 296 } 297 298 Address AtomicInfo::CreateTempAlloca() const { 299 Address TempAlloca = CGF.CreateMemTemp( 300 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 301 : AtomicTy, 302 getAtomicAlignment(), 303 "atomic-temp"); 304 // Cast to pointer to value type for bitfields. 305 if (LVal.isBitField()) 306 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 307 TempAlloca, getAtomicAddress().getType(), 308 getAtomicAddress().getElementType()); 309 return TempAlloca; 310 } 311 312 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 313 StringRef fnName, 314 QualType resultType, 315 CallArgList &args) { 316 const CGFunctionInfo &fnInfo = 317 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 318 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 319 llvm::AttrBuilder fnAttrB(CGF.getLLVMContext()); 320 fnAttrB.addAttribute(llvm::Attribute::NoUnwind); 321 fnAttrB.addAttribute(llvm::Attribute::WillReturn); 322 llvm::AttributeList fnAttrs = llvm::AttributeList::get( 323 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB); 324 325 llvm::FunctionCallee fn = 326 CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs); 327 auto callee = CGCallee::forDirect(fn); 328 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); 329 } 330 331 /// Does a store of the given IR type modify the full expected width? 332 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 333 uint64_t expectedSize) { 334 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 335 } 336 337 /// Does the atomic type require memsetting to zero before initialization? 338 /// 339 /// The IR type is provided as a way of making certain queries faster. 340 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 341 // If the atomic type has size padding, we definitely need a memset. 342 if (hasPadding()) return true; 343 344 // Otherwise, do some simple heuristics to try to avoid it: 345 switch (getEvaluationKind()) { 346 // For scalars and complexes, check whether the store size of the 347 // type uses the full size. 348 case TEK_Scalar: 349 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 350 case TEK_Complex: 351 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 352 AtomicSizeInBits / 2); 353 354 // Padding in structs has an undefined bit pattern. User beware. 355 case TEK_Aggregate: 356 return false; 357 } 358 llvm_unreachable("bad evaluation kind"); 359 } 360 361 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 362 assert(LVal.isSimple()); 363 Address addr = LVal.getAddress(CGF); 364 if (!requiresMemSetZero(addr.getElementType())) 365 return false; 366 367 CGF.Builder.CreateMemSet( 368 addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0), 369 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 370 LVal.getAlignment().getAsAlign()); 371 return true; 372 } 373 374 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 375 Address Dest, Address Ptr, 376 Address Val1, Address Val2, 377 uint64_t Size, 378 llvm::AtomicOrdering SuccessOrder, 379 llvm::AtomicOrdering FailureOrder, 380 llvm::SyncScope::ID Scope) { 381 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 382 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 383 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 384 385 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 386 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder, 387 Scope); 388 Pair->setVolatile(E->isVolatile()); 389 Pair->setWeak(IsWeak); 390 391 // Cmp holds the result of the compare-exchange operation: true on success, 392 // false on failure. 393 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 394 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 395 396 // This basic block is used to hold the store instruction if the operation 397 // failed. 398 llvm::BasicBlock *StoreExpectedBB = 399 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 400 401 // This basic block is the exit point of the operation, we should end up 402 // here regardless of whether or not the operation succeeded. 403 llvm::BasicBlock *ContinueBB = 404 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 405 406 // Update Expected if Expected isn't equal to Old, otherwise branch to the 407 // exit point. 408 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 409 410 CGF.Builder.SetInsertPoint(StoreExpectedBB); 411 // Update the memory at Expected with Old's value. 412 CGF.Builder.CreateStore(Old, Val1); 413 // Finally, branch to the exit point. 414 CGF.Builder.CreateBr(ContinueBB); 415 416 CGF.Builder.SetInsertPoint(ContinueBB); 417 // Update the memory at Dest with Cmp's value. 418 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 419 } 420 421 /// Given an ordering required on success, emit all possible cmpxchg 422 /// instructions to cope with the provided (but possibly only dynamically known) 423 /// FailureOrder. 424 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 425 bool IsWeak, Address Dest, Address Ptr, 426 Address Val1, Address Val2, 427 llvm::Value *FailureOrderVal, 428 uint64_t Size, 429 llvm::AtomicOrdering SuccessOrder, 430 llvm::SyncScope::ID Scope) { 431 llvm::AtomicOrdering FailureOrder; 432 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 433 auto FOS = FO->getSExtValue(); 434 if (!llvm::isValidAtomicOrderingCABI(FOS)) 435 FailureOrder = llvm::AtomicOrdering::Monotonic; 436 else 437 switch ((llvm::AtomicOrderingCABI)FOS) { 438 case llvm::AtomicOrderingCABI::relaxed: 439 // 31.7.2.18: "The failure argument shall not be memory_order_release 440 // nor memory_order_acq_rel". Fallback to monotonic. 441 case llvm::AtomicOrderingCABI::release: 442 case llvm::AtomicOrderingCABI::acq_rel: 443 FailureOrder = llvm::AtomicOrdering::Monotonic; 444 break; 445 case llvm::AtomicOrderingCABI::consume: 446 case llvm::AtomicOrderingCABI::acquire: 447 FailureOrder = llvm::AtomicOrdering::Acquire; 448 break; 449 case llvm::AtomicOrderingCABI::seq_cst: 450 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 451 break; 452 } 453 // Prior to c++17, "the failure argument shall be no stronger than the 454 // success argument". This condition has been lifted and the only 455 // precondition is 31.7.2.18. Effectively treat this as a DR and skip 456 // language version checks. 457 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 458 FailureOrder, Scope); 459 return; 460 } 461 462 // Create all the relevant BB's 463 auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 464 auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 465 auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 466 auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 467 468 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 469 // doesn't matter unless someone is crazy enough to use something that 470 // doesn't fold to a constant for the ordering. 471 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 472 // Implemented as acquire, since it's the closest in LLVM. 473 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 474 AcquireBB); 475 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 476 AcquireBB); 477 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 478 SeqCstBB); 479 480 // Emit all the different atomics 481 CGF.Builder.SetInsertPoint(MonotonicBB); 482 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 483 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); 484 CGF.Builder.CreateBr(ContBB); 485 486 CGF.Builder.SetInsertPoint(AcquireBB); 487 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 488 llvm::AtomicOrdering::Acquire, Scope); 489 CGF.Builder.CreateBr(ContBB); 490 491 CGF.Builder.SetInsertPoint(SeqCstBB); 492 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 493 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 494 CGF.Builder.CreateBr(ContBB); 495 496 CGF.Builder.SetInsertPoint(ContBB); 497 } 498 499 /// Duplicate the atomic min/max operation in conventional IR for the builtin 500 /// variants that return the new rather than the original value. 501 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, 502 AtomicExpr::AtomicOp Op, 503 bool IsSigned, 504 llvm::Value *OldVal, 505 llvm::Value *RHS) { 506 llvm::CmpInst::Predicate Pred; 507 switch (Op) { 508 default: 509 llvm_unreachable("Unexpected min/max operation"); 510 case AtomicExpr::AO__atomic_max_fetch: 511 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT; 512 break; 513 case AtomicExpr::AO__atomic_min_fetch: 514 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT; 515 break; 516 } 517 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst"); 518 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval"); 519 } 520 521 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 522 Address Ptr, Address Val1, Address Val2, 523 llvm::Value *IsWeak, llvm::Value *FailureOrder, 524 uint64_t Size, llvm::AtomicOrdering Order, 525 llvm::SyncScope::ID Scope) { 526 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 527 bool PostOpMinMax = false; 528 unsigned PostOp = 0; 529 530 switch (E->getOp()) { 531 case AtomicExpr::AO__c11_atomic_init: 532 case AtomicExpr::AO__opencl_atomic_init: 533 llvm_unreachable("Already handled!"); 534 535 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 536 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 537 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 538 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 539 FailureOrder, Size, Order, Scope); 540 return; 541 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 542 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 543 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 544 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 545 FailureOrder, Size, Order, Scope); 546 return; 547 case AtomicExpr::AO__atomic_compare_exchange: 548 case AtomicExpr::AO__atomic_compare_exchange_n: { 549 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 550 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 551 Val1, Val2, FailureOrder, Size, Order, Scope); 552 } else { 553 // Create all the relevant BB's 554 llvm::BasicBlock *StrongBB = 555 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 556 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 557 llvm::BasicBlock *ContBB = 558 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 559 560 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 561 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 562 563 CGF.Builder.SetInsertPoint(StrongBB); 564 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 565 FailureOrder, Size, Order, Scope); 566 CGF.Builder.CreateBr(ContBB); 567 568 CGF.Builder.SetInsertPoint(WeakBB); 569 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 570 FailureOrder, Size, Order, Scope); 571 CGF.Builder.CreateBr(ContBB); 572 573 CGF.Builder.SetInsertPoint(ContBB); 574 } 575 return; 576 } 577 case AtomicExpr::AO__c11_atomic_load: 578 case AtomicExpr::AO__opencl_atomic_load: 579 case AtomicExpr::AO__hip_atomic_load: 580 case AtomicExpr::AO__atomic_load_n: 581 case AtomicExpr::AO__atomic_load: { 582 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 583 Load->setAtomic(Order, Scope); 584 Load->setVolatile(E->isVolatile()); 585 CGF.Builder.CreateStore(Load, Dest); 586 return; 587 } 588 589 case AtomicExpr::AO__c11_atomic_store: 590 case AtomicExpr::AO__opencl_atomic_store: 591 case AtomicExpr::AO__hip_atomic_store: 592 case AtomicExpr::AO__atomic_store: 593 case AtomicExpr::AO__atomic_store_n: { 594 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 595 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 596 Store->setAtomic(Order, Scope); 597 Store->setVolatile(E->isVolatile()); 598 return; 599 } 600 601 case AtomicExpr::AO__c11_atomic_exchange: 602 case AtomicExpr::AO__hip_atomic_exchange: 603 case AtomicExpr::AO__opencl_atomic_exchange: 604 case AtomicExpr::AO__atomic_exchange_n: 605 case AtomicExpr::AO__atomic_exchange: 606 Op = llvm::AtomicRMWInst::Xchg; 607 break; 608 609 case AtomicExpr::AO__atomic_add_fetch: 610 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd 611 : llvm::Instruction::Add; 612 [[fallthrough]]; 613 case AtomicExpr::AO__c11_atomic_fetch_add: 614 case AtomicExpr::AO__hip_atomic_fetch_add: 615 case AtomicExpr::AO__opencl_atomic_fetch_add: 616 case AtomicExpr::AO__atomic_fetch_add: 617 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd 618 : llvm::AtomicRMWInst::Add; 619 break; 620 621 case AtomicExpr::AO__atomic_sub_fetch: 622 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub 623 : llvm::Instruction::Sub; 624 [[fallthrough]]; 625 case AtomicExpr::AO__c11_atomic_fetch_sub: 626 case AtomicExpr::AO__opencl_atomic_fetch_sub: 627 case AtomicExpr::AO__atomic_fetch_sub: 628 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub 629 : llvm::AtomicRMWInst::Sub; 630 break; 631 632 case AtomicExpr::AO__atomic_min_fetch: 633 PostOpMinMax = true; 634 [[fallthrough]]; 635 case AtomicExpr::AO__c11_atomic_fetch_min: 636 case AtomicExpr::AO__hip_atomic_fetch_min: 637 case AtomicExpr::AO__opencl_atomic_fetch_min: 638 case AtomicExpr::AO__atomic_fetch_min: 639 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min 640 : llvm::AtomicRMWInst::UMin; 641 break; 642 643 case AtomicExpr::AO__atomic_max_fetch: 644 PostOpMinMax = true; 645 [[fallthrough]]; 646 case AtomicExpr::AO__c11_atomic_fetch_max: 647 case AtomicExpr::AO__hip_atomic_fetch_max: 648 case AtomicExpr::AO__opencl_atomic_fetch_max: 649 case AtomicExpr::AO__atomic_fetch_max: 650 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max 651 : llvm::AtomicRMWInst::UMax; 652 break; 653 654 case AtomicExpr::AO__atomic_and_fetch: 655 PostOp = llvm::Instruction::And; 656 [[fallthrough]]; 657 case AtomicExpr::AO__c11_atomic_fetch_and: 658 case AtomicExpr::AO__hip_atomic_fetch_and: 659 case AtomicExpr::AO__opencl_atomic_fetch_and: 660 case AtomicExpr::AO__atomic_fetch_and: 661 Op = llvm::AtomicRMWInst::And; 662 break; 663 664 case AtomicExpr::AO__atomic_or_fetch: 665 PostOp = llvm::Instruction::Or; 666 [[fallthrough]]; 667 case AtomicExpr::AO__c11_atomic_fetch_or: 668 case AtomicExpr::AO__hip_atomic_fetch_or: 669 case AtomicExpr::AO__opencl_atomic_fetch_or: 670 case AtomicExpr::AO__atomic_fetch_or: 671 Op = llvm::AtomicRMWInst::Or; 672 break; 673 674 case AtomicExpr::AO__atomic_xor_fetch: 675 PostOp = llvm::Instruction::Xor; 676 [[fallthrough]]; 677 case AtomicExpr::AO__c11_atomic_fetch_xor: 678 case AtomicExpr::AO__hip_atomic_fetch_xor: 679 case AtomicExpr::AO__opencl_atomic_fetch_xor: 680 case AtomicExpr::AO__atomic_fetch_xor: 681 Op = llvm::AtomicRMWInst::Xor; 682 break; 683 684 case AtomicExpr::AO__atomic_nand_fetch: 685 PostOp = llvm::Instruction::And; // the NOT is special cased below 686 [[fallthrough]]; 687 case AtomicExpr::AO__c11_atomic_fetch_nand: 688 case AtomicExpr::AO__atomic_fetch_nand: 689 Op = llvm::AtomicRMWInst::Nand; 690 break; 691 } 692 693 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 694 llvm::AtomicRMWInst *RMWI = 695 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope); 696 RMWI->setVolatile(E->isVolatile()); 697 698 // For __atomic_*_fetch operations, perform the operation again to 699 // determine the value which was written. 700 llvm::Value *Result = RMWI; 701 if (PostOpMinMax) 702 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(), 703 E->getValueType()->isSignedIntegerType(), 704 RMWI, LoadVal1); 705 else if (PostOp) 706 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, 707 LoadVal1); 708 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 709 Result = CGF.Builder.CreateNot(Result); 710 CGF.Builder.CreateStore(Result, Dest); 711 } 712 713 // This function emits any expression (scalar, complex, or aggregate) 714 // into a temporary alloca. 715 static Address 716 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 717 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 718 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 719 /*Init*/ true); 720 return DeclPtr; 721 } 722 723 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, 724 Address Ptr, Address Val1, Address Val2, 725 llvm::Value *IsWeak, llvm::Value *FailureOrder, 726 uint64_t Size, llvm::AtomicOrdering Order, 727 llvm::Value *Scope) { 728 auto ScopeModel = Expr->getScopeModel(); 729 730 // LLVM atomic instructions always have synch scope. If clang atomic 731 // expression has no scope operand, use default LLVM synch scope. 732 if (!ScopeModel) { 733 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 734 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID("")); 735 return; 736 } 737 738 // Handle constant scope. 739 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { 740 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( 741 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()), 742 Order, CGF.CGM.getLLVMContext()); 743 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 744 Order, SCID); 745 return; 746 } 747 748 // Handle non-constant scope. 749 auto &Builder = CGF.Builder; 750 auto Scopes = ScopeModel->getRuntimeValues(); 751 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; 752 for (auto S : Scopes) 753 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); 754 755 llvm::BasicBlock *ContBB = 756 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn); 757 758 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); 759 // If unsupported synch scope is encountered at run time, assume a fallback 760 // synch scope value. 761 auto FallBack = ScopeModel->getFallBackValue(); 762 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); 763 for (auto S : Scopes) { 764 auto *B = BB[S]; 765 if (S != FallBack) 766 SI->addCase(Builder.getInt32(S), B); 767 768 Builder.SetInsertPoint(B); 769 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 770 Order, 771 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(), 772 ScopeModel->map(S), 773 Order, 774 CGF.getLLVMContext())); 775 Builder.CreateBr(ContBB); 776 } 777 778 Builder.SetInsertPoint(ContBB); 779 } 780 781 static void 782 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 783 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 784 SourceLocation Loc, CharUnits SizeInChars) { 785 if (UseOptimizedLibcall) { 786 // Load value and pass it to the function directly. 787 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); 788 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); 789 ValTy = 790 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); 791 llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits); 792 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, ITy->getPointerTo()), 793 ITy, Align); 794 Val = CGF.EmitLoadOfScalar(Ptr, false, 795 CGF.getContext().getPointerType(ValTy), 796 Loc); 797 // Coerce the value into an appropriately sized integer type. 798 Args.add(RValue::get(Val), ValTy); 799 } else { 800 // Non-optimized functions always take a reference. 801 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), 802 CGF.getContext().VoidPtrTy); 803 } 804 } 805 806 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 807 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 808 QualType MemTy = AtomicTy; 809 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 810 MemTy = AT->getValueType(); 811 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 812 813 Address Val1 = Address::invalid(); 814 Address Val2 = Address::invalid(); 815 Address Dest = Address::invalid(); 816 Address Ptr = EmitPointerWithAlignment(E->getPtr()); 817 818 if (E->getOp() == AtomicExpr::AO__c11_atomic_init || 819 E->getOp() == AtomicExpr::AO__opencl_atomic_init) { 820 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 821 EmitAtomicInit(E->getVal1(), lvalue); 822 return RValue::get(nullptr); 823 } 824 825 auto TInfo = getContext().getTypeInfoInChars(AtomicTy); 826 uint64_t Size = TInfo.Width.getQuantity(); 827 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 828 829 bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits; 830 bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0; 831 bool UseLibcall = Misaligned | Oversized; 832 bool ShouldCastToIntPtrTy = true; 833 834 CharUnits MaxInlineWidth = 835 getContext().toCharUnitsFromBits(MaxInlineWidthInBits); 836 837 DiagnosticsEngine &Diags = CGM.getDiags(); 838 839 if (Misaligned) { 840 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned) 841 << (int)TInfo.Width.getQuantity() 842 << (int)Ptr.getAlignment().getQuantity(); 843 } 844 845 if (Oversized) { 846 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized) 847 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity(); 848 } 849 850 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 851 llvm::Value *Scope = 852 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; 853 854 switch (E->getOp()) { 855 case AtomicExpr::AO__c11_atomic_init: 856 case AtomicExpr::AO__opencl_atomic_init: 857 llvm_unreachable("Already handled above with EmitAtomicInit!"); 858 859 case AtomicExpr::AO__c11_atomic_load: 860 case AtomicExpr::AO__opencl_atomic_load: 861 case AtomicExpr::AO__hip_atomic_load: 862 case AtomicExpr::AO__atomic_load_n: 863 break; 864 865 case AtomicExpr::AO__atomic_load: 866 Dest = EmitPointerWithAlignment(E->getVal1()); 867 break; 868 869 case AtomicExpr::AO__atomic_store: 870 Val1 = EmitPointerWithAlignment(E->getVal1()); 871 break; 872 873 case AtomicExpr::AO__atomic_exchange: 874 Val1 = EmitPointerWithAlignment(E->getVal1()); 875 Dest = EmitPointerWithAlignment(E->getVal2()); 876 break; 877 878 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 879 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 880 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 881 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 882 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 883 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 884 case AtomicExpr::AO__atomic_compare_exchange_n: 885 case AtomicExpr::AO__atomic_compare_exchange: 886 Val1 = EmitPointerWithAlignment(E->getVal1()); 887 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 888 Val2 = EmitPointerWithAlignment(E->getVal2()); 889 else 890 Val2 = EmitValToTemp(*this, E->getVal2()); 891 OrderFail = EmitScalarExpr(E->getOrderFail()); 892 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || 893 E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 894 IsWeak = EmitScalarExpr(E->getWeak()); 895 break; 896 897 case AtomicExpr::AO__c11_atomic_fetch_add: 898 case AtomicExpr::AO__c11_atomic_fetch_sub: 899 case AtomicExpr::AO__hip_atomic_fetch_add: 900 case AtomicExpr::AO__opencl_atomic_fetch_add: 901 case AtomicExpr::AO__opencl_atomic_fetch_sub: 902 if (MemTy->isPointerType()) { 903 // For pointer arithmetic, we're required to do a bit of math: 904 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 905 // ... but only for the C11 builtins. The GNU builtins expect the 906 // user to multiply by sizeof(T). 907 QualType Val1Ty = E->getVal1()->getType(); 908 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 909 CharUnits PointeeIncAmt = 910 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 911 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 912 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 913 Val1 = Temp; 914 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 915 break; 916 } 917 [[fallthrough]]; 918 case AtomicExpr::AO__atomic_fetch_add: 919 case AtomicExpr::AO__atomic_fetch_sub: 920 case AtomicExpr::AO__atomic_add_fetch: 921 case AtomicExpr::AO__atomic_sub_fetch: 922 ShouldCastToIntPtrTy = !MemTy->isFloatingType(); 923 [[fallthrough]]; 924 925 case AtomicExpr::AO__c11_atomic_store: 926 case AtomicExpr::AO__c11_atomic_exchange: 927 case AtomicExpr::AO__opencl_atomic_store: 928 case AtomicExpr::AO__hip_atomic_store: 929 case AtomicExpr::AO__opencl_atomic_exchange: 930 case AtomicExpr::AO__hip_atomic_exchange: 931 case AtomicExpr::AO__atomic_store_n: 932 case AtomicExpr::AO__atomic_exchange_n: 933 case AtomicExpr::AO__c11_atomic_fetch_and: 934 case AtomicExpr::AO__c11_atomic_fetch_or: 935 case AtomicExpr::AO__c11_atomic_fetch_xor: 936 case AtomicExpr::AO__c11_atomic_fetch_nand: 937 case AtomicExpr::AO__c11_atomic_fetch_max: 938 case AtomicExpr::AO__c11_atomic_fetch_min: 939 case AtomicExpr::AO__opencl_atomic_fetch_and: 940 case AtomicExpr::AO__opencl_atomic_fetch_or: 941 case AtomicExpr::AO__opencl_atomic_fetch_xor: 942 case AtomicExpr::AO__opencl_atomic_fetch_min: 943 case AtomicExpr::AO__opencl_atomic_fetch_max: 944 case AtomicExpr::AO__atomic_fetch_and: 945 case AtomicExpr::AO__hip_atomic_fetch_and: 946 case AtomicExpr::AO__atomic_fetch_or: 947 case AtomicExpr::AO__hip_atomic_fetch_or: 948 case AtomicExpr::AO__atomic_fetch_xor: 949 case AtomicExpr::AO__hip_atomic_fetch_xor: 950 case AtomicExpr::AO__atomic_fetch_nand: 951 case AtomicExpr::AO__atomic_and_fetch: 952 case AtomicExpr::AO__atomic_or_fetch: 953 case AtomicExpr::AO__atomic_xor_fetch: 954 case AtomicExpr::AO__atomic_nand_fetch: 955 case AtomicExpr::AO__atomic_max_fetch: 956 case AtomicExpr::AO__atomic_min_fetch: 957 case AtomicExpr::AO__atomic_fetch_max: 958 case AtomicExpr::AO__hip_atomic_fetch_max: 959 case AtomicExpr::AO__atomic_fetch_min: 960 case AtomicExpr::AO__hip_atomic_fetch_min: 961 Val1 = EmitValToTemp(*this, E->getVal1()); 962 break; 963 } 964 965 QualType RValTy = E->getType().getUnqualifiedType(); 966 967 // The inlined atomics only function on iN types, where N is a power of 2. We 968 // need to make sure (via temporaries if necessary) that all incoming values 969 // are compatible. 970 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 971 AtomicInfo Atomics(*this, AtomicVal); 972 973 if (ShouldCastToIntPtrTy) { 974 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); 975 if (Val1.isValid()) 976 Val1 = Atomics.convertToAtomicIntPointer(Val1); 977 if (Val2.isValid()) 978 Val2 = Atomics.convertToAtomicIntPointer(Val2); 979 } 980 if (Dest.isValid()) { 981 if (ShouldCastToIntPtrTy) 982 Dest = Atomics.emitCastToAtomicIntPointer(Dest); 983 } else if (E->isCmpXChg()) 984 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 985 else if (!RValTy->isVoidType()) { 986 Dest = Atomics.CreateTempAlloca(); 987 if (ShouldCastToIntPtrTy) 988 Dest = Atomics.emitCastToAtomicIntPointer(Dest); 989 } 990 991 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 992 if (UseLibcall) { 993 bool UseOptimizedLibcall = false; 994 switch (E->getOp()) { 995 case AtomicExpr::AO__c11_atomic_init: 996 case AtomicExpr::AO__opencl_atomic_init: 997 llvm_unreachable("Already handled above with EmitAtomicInit!"); 998 999 case AtomicExpr::AO__c11_atomic_fetch_add: 1000 case AtomicExpr::AO__opencl_atomic_fetch_add: 1001 case AtomicExpr::AO__atomic_fetch_add: 1002 case AtomicExpr::AO__hip_atomic_fetch_add: 1003 case AtomicExpr::AO__c11_atomic_fetch_and: 1004 case AtomicExpr::AO__opencl_atomic_fetch_and: 1005 case AtomicExpr::AO__hip_atomic_fetch_and: 1006 case AtomicExpr::AO__atomic_fetch_and: 1007 case AtomicExpr::AO__c11_atomic_fetch_or: 1008 case AtomicExpr::AO__opencl_atomic_fetch_or: 1009 case AtomicExpr::AO__hip_atomic_fetch_or: 1010 case AtomicExpr::AO__atomic_fetch_or: 1011 case AtomicExpr::AO__c11_atomic_fetch_nand: 1012 case AtomicExpr::AO__atomic_fetch_nand: 1013 case AtomicExpr::AO__c11_atomic_fetch_sub: 1014 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1015 case AtomicExpr::AO__atomic_fetch_sub: 1016 case AtomicExpr::AO__c11_atomic_fetch_xor: 1017 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1018 case AtomicExpr::AO__opencl_atomic_fetch_min: 1019 case AtomicExpr::AO__opencl_atomic_fetch_max: 1020 case AtomicExpr::AO__atomic_fetch_xor: 1021 case AtomicExpr::AO__hip_atomic_fetch_xor: 1022 case AtomicExpr::AO__c11_atomic_fetch_max: 1023 case AtomicExpr::AO__c11_atomic_fetch_min: 1024 case AtomicExpr::AO__atomic_add_fetch: 1025 case AtomicExpr::AO__atomic_and_fetch: 1026 case AtomicExpr::AO__atomic_nand_fetch: 1027 case AtomicExpr::AO__atomic_or_fetch: 1028 case AtomicExpr::AO__atomic_sub_fetch: 1029 case AtomicExpr::AO__atomic_xor_fetch: 1030 case AtomicExpr::AO__atomic_fetch_max: 1031 case AtomicExpr::AO__hip_atomic_fetch_max: 1032 case AtomicExpr::AO__atomic_fetch_min: 1033 case AtomicExpr::AO__hip_atomic_fetch_min: 1034 case AtomicExpr::AO__atomic_max_fetch: 1035 case AtomicExpr::AO__atomic_min_fetch: 1036 // For these, only library calls for certain sizes exist. 1037 UseOptimizedLibcall = true; 1038 break; 1039 1040 case AtomicExpr::AO__atomic_load: 1041 case AtomicExpr::AO__atomic_store: 1042 case AtomicExpr::AO__atomic_exchange: 1043 case AtomicExpr::AO__atomic_compare_exchange: 1044 // Use the generic version if we don't know that the operand will be 1045 // suitably aligned for the optimized version. 1046 if (Misaligned) 1047 break; 1048 [[fallthrough]]; 1049 case AtomicExpr::AO__c11_atomic_load: 1050 case AtomicExpr::AO__c11_atomic_store: 1051 case AtomicExpr::AO__c11_atomic_exchange: 1052 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 1053 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 1054 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 1055 case AtomicExpr::AO__opencl_atomic_load: 1056 case AtomicExpr::AO__hip_atomic_load: 1057 case AtomicExpr::AO__opencl_atomic_store: 1058 case AtomicExpr::AO__hip_atomic_store: 1059 case AtomicExpr::AO__opencl_atomic_exchange: 1060 case AtomicExpr::AO__hip_atomic_exchange: 1061 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 1062 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 1063 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 1064 case AtomicExpr::AO__atomic_load_n: 1065 case AtomicExpr::AO__atomic_store_n: 1066 case AtomicExpr::AO__atomic_exchange_n: 1067 case AtomicExpr::AO__atomic_compare_exchange_n: 1068 // Only use optimized library calls for sizes for which they exist. 1069 // FIXME: Size == 16 optimized library functions exist too. 1070 if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 1071 UseOptimizedLibcall = true; 1072 break; 1073 } 1074 1075 CallArgList Args; 1076 if (!UseOptimizedLibcall) { 1077 // For non-optimized library calls, the size is the first parameter 1078 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 1079 getContext().getSizeType()); 1080 } 1081 // Atomic address is the first or second parameter 1082 // The OpenCL atomic library functions only accept pointer arguments to 1083 // generic address space. 1084 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { 1085 if (!E->isOpenCL()) 1086 return V; 1087 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace(); 1088 if (AS == LangAS::opencl_generic) 1089 return V; 1090 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); 1091 auto T = llvm::cast<llvm::PointerType>(V->getType()); 1092 auto *DestType = llvm::PointerType::getWithSamePointeeType(T, DestAS); 1093 1094 return getTargetHooks().performAddrSpaceCast( 1095 *this, V, AS, LangAS::opencl_generic, DestType, false); 1096 }; 1097 1098 Args.add(RValue::get(CastToGenericAddrSpace( 1099 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())), 1100 getContext().VoidPtrTy); 1101 1102 std::string LibCallName; 1103 QualType LoweredMemTy = 1104 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 1105 QualType RetTy; 1106 bool HaveRetTy = false; 1107 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 1108 bool PostOpMinMax = false; 1109 switch (E->getOp()) { 1110 case AtomicExpr::AO__c11_atomic_init: 1111 case AtomicExpr::AO__opencl_atomic_init: 1112 llvm_unreachable("Already handled!"); 1113 1114 // There is only one libcall for compare an exchange, because there is no 1115 // optimisation benefit possible from a libcall version of a weak compare 1116 // and exchange. 1117 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 1118 // void *desired, int success, int failure) 1119 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 1120 // int success, int failure) 1121 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 1122 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 1123 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 1124 case AtomicExpr::AO__hip_atomic_compare_exchange_weak: 1125 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 1126 case AtomicExpr::AO__hip_atomic_compare_exchange_strong: 1127 case AtomicExpr::AO__atomic_compare_exchange: 1128 case AtomicExpr::AO__atomic_compare_exchange_n: 1129 LibCallName = "__atomic_compare_exchange"; 1130 RetTy = getContext().BoolTy; 1131 HaveRetTy = true; 1132 Args.add( 1133 RValue::get(CastToGenericAddrSpace( 1134 EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())), 1135 getContext().VoidPtrTy); 1136 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), 1137 MemTy, E->getExprLoc(), TInfo.Width); 1138 Args.add(RValue::get(Order), getContext().IntTy); 1139 Order = OrderFail; 1140 break; 1141 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 1142 // int order) 1143 // T __atomic_exchange_N(T *mem, T val, int order) 1144 case AtomicExpr::AO__c11_atomic_exchange: 1145 case AtomicExpr::AO__opencl_atomic_exchange: 1146 case AtomicExpr::AO__atomic_exchange_n: 1147 case AtomicExpr::AO__atomic_exchange: 1148 case AtomicExpr::AO__hip_atomic_exchange: 1149 LibCallName = "__atomic_exchange"; 1150 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1151 MemTy, E->getExprLoc(), TInfo.Width); 1152 break; 1153 // void __atomic_store(size_t size, void *mem, void *val, int order) 1154 // void __atomic_store_N(T *mem, T val, int order) 1155 case AtomicExpr::AO__c11_atomic_store: 1156 case AtomicExpr::AO__opencl_atomic_store: 1157 case AtomicExpr::AO__hip_atomic_store: 1158 case AtomicExpr::AO__atomic_store: 1159 case AtomicExpr::AO__atomic_store_n: 1160 LibCallName = "__atomic_store"; 1161 RetTy = getContext().VoidTy; 1162 HaveRetTy = true; 1163 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1164 MemTy, E->getExprLoc(), TInfo.Width); 1165 break; 1166 // void __atomic_load(size_t size, void *mem, void *return, int order) 1167 // T __atomic_load_N(T *mem, int order) 1168 case AtomicExpr::AO__c11_atomic_load: 1169 case AtomicExpr::AO__opencl_atomic_load: 1170 case AtomicExpr::AO__hip_atomic_load: 1171 case AtomicExpr::AO__atomic_load: 1172 case AtomicExpr::AO__atomic_load_n: 1173 LibCallName = "__atomic_load"; 1174 break; 1175 // T __atomic_add_fetch_N(T *mem, T val, int order) 1176 // T __atomic_fetch_add_N(T *mem, T val, int order) 1177 case AtomicExpr::AO__atomic_add_fetch: 1178 PostOp = llvm::Instruction::Add; 1179 [[fallthrough]]; 1180 case AtomicExpr::AO__c11_atomic_fetch_add: 1181 case AtomicExpr::AO__opencl_atomic_fetch_add: 1182 case AtomicExpr::AO__atomic_fetch_add: 1183 case AtomicExpr::AO__hip_atomic_fetch_add: 1184 LibCallName = "__atomic_fetch_add"; 1185 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1186 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1187 break; 1188 // T __atomic_and_fetch_N(T *mem, T val, int order) 1189 // T __atomic_fetch_and_N(T *mem, T val, int order) 1190 case AtomicExpr::AO__atomic_and_fetch: 1191 PostOp = llvm::Instruction::And; 1192 [[fallthrough]]; 1193 case AtomicExpr::AO__c11_atomic_fetch_and: 1194 case AtomicExpr::AO__opencl_atomic_fetch_and: 1195 case AtomicExpr::AO__hip_atomic_fetch_and: 1196 case AtomicExpr::AO__atomic_fetch_and: 1197 LibCallName = "__atomic_fetch_and"; 1198 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1199 MemTy, E->getExprLoc(), TInfo.Width); 1200 break; 1201 // T __atomic_or_fetch_N(T *mem, T val, int order) 1202 // T __atomic_fetch_or_N(T *mem, T val, int order) 1203 case AtomicExpr::AO__atomic_or_fetch: 1204 PostOp = llvm::Instruction::Or; 1205 [[fallthrough]]; 1206 case AtomicExpr::AO__c11_atomic_fetch_or: 1207 case AtomicExpr::AO__opencl_atomic_fetch_or: 1208 case AtomicExpr::AO__hip_atomic_fetch_or: 1209 case AtomicExpr::AO__atomic_fetch_or: 1210 LibCallName = "__atomic_fetch_or"; 1211 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1212 MemTy, E->getExprLoc(), TInfo.Width); 1213 break; 1214 // T __atomic_sub_fetch_N(T *mem, T val, int order) 1215 // T __atomic_fetch_sub_N(T *mem, T val, int order) 1216 case AtomicExpr::AO__atomic_sub_fetch: 1217 PostOp = llvm::Instruction::Sub; 1218 [[fallthrough]]; 1219 case AtomicExpr::AO__c11_atomic_fetch_sub: 1220 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1221 case AtomicExpr::AO__atomic_fetch_sub: 1222 LibCallName = "__atomic_fetch_sub"; 1223 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1224 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1225 break; 1226 // T __atomic_xor_fetch_N(T *mem, T val, int order) 1227 // T __atomic_fetch_xor_N(T *mem, T val, int order) 1228 case AtomicExpr::AO__atomic_xor_fetch: 1229 PostOp = llvm::Instruction::Xor; 1230 [[fallthrough]]; 1231 case AtomicExpr::AO__c11_atomic_fetch_xor: 1232 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1233 case AtomicExpr::AO__hip_atomic_fetch_xor: 1234 case AtomicExpr::AO__atomic_fetch_xor: 1235 LibCallName = "__atomic_fetch_xor"; 1236 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1237 MemTy, E->getExprLoc(), TInfo.Width); 1238 break; 1239 case AtomicExpr::AO__atomic_min_fetch: 1240 PostOpMinMax = true; 1241 [[fallthrough]]; 1242 case AtomicExpr::AO__c11_atomic_fetch_min: 1243 case AtomicExpr::AO__atomic_fetch_min: 1244 case AtomicExpr::AO__hip_atomic_fetch_min: 1245 case AtomicExpr::AO__opencl_atomic_fetch_min: 1246 LibCallName = E->getValueType()->isSignedIntegerType() 1247 ? "__atomic_fetch_min" 1248 : "__atomic_fetch_umin"; 1249 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1250 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1251 break; 1252 case AtomicExpr::AO__atomic_max_fetch: 1253 PostOpMinMax = true; 1254 [[fallthrough]]; 1255 case AtomicExpr::AO__c11_atomic_fetch_max: 1256 case AtomicExpr::AO__atomic_fetch_max: 1257 case AtomicExpr::AO__hip_atomic_fetch_max: 1258 case AtomicExpr::AO__opencl_atomic_fetch_max: 1259 LibCallName = E->getValueType()->isSignedIntegerType() 1260 ? "__atomic_fetch_max" 1261 : "__atomic_fetch_umax"; 1262 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1263 LoweredMemTy, E->getExprLoc(), TInfo.Width); 1264 break; 1265 // T __atomic_nand_fetch_N(T *mem, T val, int order) 1266 // T __atomic_fetch_nand_N(T *mem, T val, int order) 1267 case AtomicExpr::AO__atomic_nand_fetch: 1268 PostOp = llvm::Instruction::And; // the NOT is special cased below 1269 [[fallthrough]]; 1270 case AtomicExpr::AO__c11_atomic_fetch_nand: 1271 case AtomicExpr::AO__atomic_fetch_nand: 1272 LibCallName = "__atomic_fetch_nand"; 1273 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1274 MemTy, E->getExprLoc(), TInfo.Width); 1275 break; 1276 } 1277 1278 if (E->isOpenCL()) { 1279 LibCallName = std::string("__opencl") + 1280 StringRef(LibCallName).drop_front(1).str(); 1281 1282 } 1283 // Optimized functions have the size in their name. 1284 if (UseOptimizedLibcall) 1285 LibCallName += "_" + llvm::utostr(Size); 1286 // By default, assume we return a value of the atomic type. 1287 if (!HaveRetTy) { 1288 if (UseOptimizedLibcall) { 1289 // Value is returned directly. 1290 // The function returns an appropriately sized integer type. 1291 RetTy = getContext().getIntTypeForBitwidth( 1292 getContext().toBits(TInfo.Width), /*Signed=*/false); 1293 } else { 1294 // Value is returned through parameter before the order. 1295 RetTy = getContext().VoidTy; 1296 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), 1297 getContext().VoidPtrTy); 1298 } 1299 } 1300 // order is always the last parameter 1301 Args.add(RValue::get(Order), 1302 getContext().IntTy); 1303 if (E->isOpenCL()) 1304 Args.add(RValue::get(Scope), getContext().IntTy); 1305 1306 // PostOp is only needed for the atomic_*_fetch operations, and 1307 // thus is only needed for and implemented in the 1308 // UseOptimizedLibcall codepath. 1309 assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax)); 1310 1311 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 1312 // The value is returned directly from the libcall. 1313 if (E->isCmpXChg()) 1314 return Res; 1315 1316 // The value is returned directly for optimized libcalls but the expr 1317 // provided an out-param. 1318 if (UseOptimizedLibcall && Res.getScalarVal()) { 1319 llvm::Value *ResVal = Res.getScalarVal(); 1320 if (PostOpMinMax) { 1321 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1322 ResVal = EmitPostAtomicMinMax(Builder, E->getOp(), 1323 E->getValueType()->isSignedIntegerType(), 1324 ResVal, LoadVal1); 1325 } else if (PostOp) { 1326 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1327 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); 1328 } 1329 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 1330 ResVal = Builder.CreateNot(ResVal); 1331 1332 Builder.CreateStore( 1333 ResVal, Builder.CreateElementBitCast(Dest, ResVal->getType())); 1334 } 1335 1336 if (RValTy->isVoidType()) 1337 return RValue::get(nullptr); 1338 1339 return convertTempToRValue( 1340 Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)), 1341 RValTy, E->getExprLoc()); 1342 } 1343 1344 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1345 E->getOp() == AtomicExpr::AO__opencl_atomic_store || 1346 E->getOp() == AtomicExpr::AO__hip_atomic_store || 1347 E->getOp() == AtomicExpr::AO__atomic_store || 1348 E->getOp() == AtomicExpr::AO__atomic_store_n; 1349 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1350 E->getOp() == AtomicExpr::AO__opencl_atomic_load || 1351 E->getOp() == AtomicExpr::AO__hip_atomic_load || 1352 E->getOp() == AtomicExpr::AO__atomic_load || 1353 E->getOp() == AtomicExpr::AO__atomic_load_n; 1354 1355 if (isa<llvm::ConstantInt>(Order)) { 1356 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1357 // We should not ever get to a case where the ordering isn't a valid C ABI 1358 // value, but it's hard to enforce that in general. 1359 if (llvm::isValidAtomicOrderingCABI(ord)) 1360 switch ((llvm::AtomicOrderingCABI)ord) { 1361 case llvm::AtomicOrderingCABI::relaxed: 1362 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1363 llvm::AtomicOrdering::Monotonic, Scope); 1364 break; 1365 case llvm::AtomicOrderingCABI::consume: 1366 case llvm::AtomicOrderingCABI::acquire: 1367 if (IsStore) 1368 break; // Avoid crashing on code with undefined behavior 1369 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1370 llvm::AtomicOrdering::Acquire, Scope); 1371 break; 1372 case llvm::AtomicOrderingCABI::release: 1373 if (IsLoad) 1374 break; // Avoid crashing on code with undefined behavior 1375 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1376 llvm::AtomicOrdering::Release, Scope); 1377 break; 1378 case llvm::AtomicOrderingCABI::acq_rel: 1379 if (IsLoad || IsStore) 1380 break; // Avoid crashing on code with undefined behavior 1381 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1382 llvm::AtomicOrdering::AcquireRelease, Scope); 1383 break; 1384 case llvm::AtomicOrderingCABI::seq_cst: 1385 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1386 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1387 break; 1388 } 1389 if (RValTy->isVoidType()) 1390 return RValue::get(nullptr); 1391 1392 return convertTempToRValue( 1393 Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)), 1394 RValTy, E->getExprLoc()); 1395 } 1396 1397 // Long case, when Order isn't obviously constant. 1398 1399 // Create all the relevant BB's 1400 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1401 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1402 *SeqCstBB = nullptr; 1403 MonotonicBB = createBasicBlock("monotonic", CurFn); 1404 if (!IsStore) 1405 AcquireBB = createBasicBlock("acquire", CurFn); 1406 if (!IsLoad) 1407 ReleaseBB = createBasicBlock("release", CurFn); 1408 if (!IsLoad && !IsStore) 1409 AcqRelBB = createBasicBlock("acqrel", CurFn); 1410 SeqCstBB = createBasicBlock("seqcst", CurFn); 1411 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1412 1413 // Create the switch for the split 1414 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1415 // doesn't matter unless someone is crazy enough to use something that 1416 // doesn't fold to a constant for the ordering. 1417 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1418 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1419 1420 // Emit all the different atomics 1421 Builder.SetInsertPoint(MonotonicBB); 1422 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1423 llvm::AtomicOrdering::Monotonic, Scope); 1424 Builder.CreateBr(ContBB); 1425 if (!IsStore) { 1426 Builder.SetInsertPoint(AcquireBB); 1427 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1428 llvm::AtomicOrdering::Acquire, Scope); 1429 Builder.CreateBr(ContBB); 1430 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1431 AcquireBB); 1432 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1433 AcquireBB); 1434 } 1435 if (!IsLoad) { 1436 Builder.SetInsertPoint(ReleaseBB); 1437 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1438 llvm::AtomicOrdering::Release, Scope); 1439 Builder.CreateBr(ContBB); 1440 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1441 ReleaseBB); 1442 } 1443 if (!IsLoad && !IsStore) { 1444 Builder.SetInsertPoint(AcqRelBB); 1445 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1446 llvm::AtomicOrdering::AcquireRelease, Scope); 1447 Builder.CreateBr(ContBB); 1448 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1449 AcqRelBB); 1450 } 1451 Builder.SetInsertPoint(SeqCstBB); 1452 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1453 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1454 Builder.CreateBr(ContBB); 1455 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1456 SeqCstBB); 1457 1458 // Cleanup and return 1459 Builder.SetInsertPoint(ContBB); 1460 if (RValTy->isVoidType()) 1461 return RValue::get(nullptr); 1462 1463 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1464 return convertTempToRValue( 1465 Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)), 1466 RValTy, E->getExprLoc()); 1467 } 1468 1469 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { 1470 llvm::IntegerType *ty = 1471 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1472 return CGF.Builder.CreateElementBitCast(addr, ty); 1473 } 1474 1475 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1476 llvm::Type *Ty = Addr.getElementType(); 1477 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1478 if (SourceSizeInBits != AtomicSizeInBits) { 1479 Address Tmp = CreateTempAlloca(); 1480 CGF.Builder.CreateMemCpy(Tmp, Addr, 1481 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1482 Addr = Tmp; 1483 } 1484 1485 return emitCastToAtomicIntPointer(Addr); 1486 } 1487 1488 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1489 AggValueSlot resultSlot, 1490 SourceLocation loc, 1491 bool asValue) const { 1492 if (LVal.isSimple()) { 1493 if (EvaluationKind == TEK_Aggregate) 1494 return resultSlot.asRValue(); 1495 1496 // Drill into the padding structure if we have one. 1497 if (hasPadding()) 1498 addr = CGF.Builder.CreateStructGEP(addr, 0); 1499 1500 // Otherwise, just convert the temporary to an r-value using the 1501 // normal conversion routine. 1502 return CGF.convertTempToRValue(addr, getValueType(), loc); 1503 } 1504 if (!asValue) 1505 // Get RValue from temp memory as atomic for non-simple lvalues 1506 return RValue::get(CGF.Builder.CreateLoad(addr)); 1507 if (LVal.isBitField()) 1508 return CGF.EmitLoadOfBitfieldLValue( 1509 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1510 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1511 if (LVal.isVectorElt()) 1512 return CGF.EmitLoadOfLValue( 1513 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1514 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1515 assert(LVal.isExtVectorElt()); 1516 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1517 addr, LVal.getExtVectorElts(), LVal.getType(), 1518 LVal.getBaseInfo(), TBAAAccessInfo())); 1519 } 1520 1521 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, 1522 AggValueSlot ResultSlot, 1523 SourceLocation Loc, 1524 bool AsValue) const { 1525 // Try not to in some easy cases. 1526 assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); 1527 if (getEvaluationKind() == TEK_Scalar && 1528 (((!LVal.isBitField() || 1529 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1530 !hasPadding()) || 1531 !AsValue)) { 1532 auto *ValTy = AsValue 1533 ? CGF.ConvertTypeForMem(ValueTy) 1534 : getAtomicAddress().getElementType(); 1535 if (ValTy->isIntegerTy()) { 1536 assert(IntVal->getType() == ValTy && "Different integer types."); 1537 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); 1538 } else if (ValTy->isPointerTy()) 1539 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); 1540 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) 1541 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); 1542 } 1543 1544 // Create a temporary. This needs to be big enough to hold the 1545 // atomic integer. 1546 Address Temp = Address::invalid(); 1547 bool TempIsVolatile = false; 1548 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1549 assert(!ResultSlot.isIgnored()); 1550 Temp = ResultSlot.getAddress(); 1551 TempIsVolatile = ResultSlot.isVolatile(); 1552 } else { 1553 Temp = CreateTempAlloca(); 1554 } 1555 1556 // Slam the integer into the temporary. 1557 Address CastTemp = emitCastToAtomicIntPointer(Temp); 1558 CGF.Builder.CreateStore(IntVal, CastTemp) 1559 ->setVolatile(TempIsVolatile); 1560 1561 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1562 } 1563 1564 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1565 llvm::AtomicOrdering AO, bool) { 1566 // void __atomic_load(size_t size, void *mem, void *return, int order); 1567 CallArgList Args; 1568 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1569 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1570 CGF.getContext().VoidPtrTy); 1571 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), 1572 CGF.getContext().VoidPtrTy); 1573 Args.add( 1574 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1575 CGF.getContext().IntTy); 1576 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1577 } 1578 1579 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1580 bool IsVolatile) { 1581 // Okay, we're doing this natively. 1582 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1583 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1584 Load->setAtomic(AO); 1585 1586 // Other decoration. 1587 if (IsVolatile) 1588 Load->setVolatile(true); 1589 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1590 return Load; 1591 } 1592 1593 /// An LValue is a candidate for having its loads and stores be made atomic if 1594 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1595 /// performing such an operation can be performed without a libcall. 1596 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1597 if (!CGM.getLangOpts().MSVolatile) return false; 1598 AtomicInfo AI(*this, LV); 1599 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1600 // An atomic is inline if we don't need to use a libcall. 1601 bool AtomicIsInline = !AI.shouldUseLibcall(); 1602 // MSVC doesn't seem to do this for types wider than a pointer. 1603 if (getContext().getTypeSize(LV.getType()) > 1604 getContext().getTypeSize(getContext().getIntPtrType())) 1605 return false; 1606 return IsVolatile && AtomicIsInline; 1607 } 1608 1609 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1610 AggValueSlot Slot) { 1611 llvm::AtomicOrdering AO; 1612 bool IsVolatile = LV.isVolatileQualified(); 1613 if (LV.getType()->isAtomicType()) { 1614 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1615 } else { 1616 AO = llvm::AtomicOrdering::Acquire; 1617 IsVolatile = true; 1618 } 1619 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1620 } 1621 1622 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1623 bool AsValue, llvm::AtomicOrdering AO, 1624 bool IsVolatile) { 1625 // Check whether we should use a library call. 1626 if (shouldUseLibcall()) { 1627 Address TempAddr = Address::invalid(); 1628 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1629 assert(getEvaluationKind() == TEK_Aggregate); 1630 TempAddr = ResultSlot.getAddress(); 1631 } else 1632 TempAddr = CreateTempAlloca(); 1633 1634 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); 1635 1636 // Okay, turn that back into the original value or whole atomic (for 1637 // non-simple lvalues) type. 1638 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1639 } 1640 1641 // Okay, we're doing this natively. 1642 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1643 1644 // If we're ignoring an aggregate return, don't do anything. 1645 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1646 return RValue::getAggregate(Address::invalid(), false); 1647 1648 // Okay, turn that back into the original value or atomic (for non-simple 1649 // lvalues) type. 1650 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1651 } 1652 1653 /// Emit a load from an l-value of atomic type. Note that the r-value 1654 /// we produce is an r-value of the atomic *value* type. 1655 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1656 llvm::AtomicOrdering AO, bool IsVolatile, 1657 AggValueSlot resultSlot) { 1658 AtomicInfo Atomics(*this, src); 1659 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1660 IsVolatile); 1661 } 1662 1663 /// Copy an r-value into memory as part of storing to an atomic type. 1664 /// This needs to create a bit-pattern suitable for atomic operations. 1665 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1666 assert(LVal.isSimple()); 1667 // If we have an r-value, the rvalue should be of the atomic type, 1668 // which means that the caller is responsible for having zeroed 1669 // any padding. Just do an aggregate copy of that type. 1670 if (rvalue.isAggregate()) { 1671 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); 1672 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), 1673 getAtomicType()); 1674 bool IsVolatile = rvalue.isVolatileQualified() || 1675 LVal.isVolatileQualified(); 1676 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), 1677 AggValueSlot::DoesNotOverlap, IsVolatile); 1678 return; 1679 } 1680 1681 // Okay, otherwise we're copying stuff. 1682 1683 // Zero out the buffer if necessary. 1684 emitMemSetZeroIfNecessary(); 1685 1686 // Drill past the padding if present. 1687 LValue TempLVal = projectValue(); 1688 1689 // Okay, store the rvalue in. 1690 if (rvalue.isScalar()) { 1691 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1692 } else { 1693 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1694 } 1695 } 1696 1697 1698 /// Materialize an r-value into memory for the purposes of storing it 1699 /// to an atomic type. 1700 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1701 // Aggregate r-values are already in memory, and EmitAtomicStore 1702 // requires them to be values of the atomic type. 1703 if (rvalue.isAggregate()) 1704 return rvalue.getAggregateAddress(); 1705 1706 // Otherwise, make a temporary and materialize into it. 1707 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1708 AtomicInfo Atomics(CGF, TempLV); 1709 Atomics.emitCopyIntoMemory(rvalue); 1710 return TempLV.getAddress(CGF); 1711 } 1712 1713 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { 1714 // If we've got a scalar value of the right size, try to avoid going 1715 // through memory. 1716 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { 1717 llvm::Value *Value = RVal.getScalarVal(); 1718 if (isa<llvm::IntegerType>(Value->getType())) 1719 return CGF.EmitToMemory(Value, ValueTy); 1720 else { 1721 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1722 CGF.getLLVMContext(), 1723 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1724 if (isa<llvm::PointerType>(Value->getType())) 1725 return CGF.Builder.CreatePtrToInt(Value, InputIntTy); 1726 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1727 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1728 } 1729 } 1730 // Otherwise, we need to go through memory. 1731 // Put the r-value in memory. 1732 Address Addr = materializeRValue(RVal); 1733 1734 // Cast the temporary to the atomic int type and pull a value out. 1735 Addr = emitCastToAtomicIntPointer(Addr); 1736 return CGF.Builder.CreateLoad(Addr); 1737 } 1738 1739 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1740 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1741 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1742 // Do the atomic store. 1743 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1744 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), 1745 ExpectedVal, DesiredVal, 1746 Success, Failure); 1747 // Other decoration. 1748 Inst->setVolatile(LVal.isVolatileQualified()); 1749 Inst->setWeak(IsWeak); 1750 1751 // Okay, turn that back into the original value type. 1752 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1753 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1754 return std::make_pair(PreviousVal, SuccessFailureVal); 1755 } 1756 1757 llvm::Value * 1758 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1759 llvm::Value *DesiredAddr, 1760 llvm::AtomicOrdering Success, 1761 llvm::AtomicOrdering Failure) { 1762 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1763 // void *desired, int success, int failure); 1764 CallArgList Args; 1765 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1766 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1767 CGF.getContext().VoidPtrTy); 1768 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), 1769 CGF.getContext().VoidPtrTy); 1770 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), 1771 CGF.getContext().VoidPtrTy); 1772 Args.add(RValue::get( 1773 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1774 CGF.getContext().IntTy); 1775 Args.add(RValue::get( 1776 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1777 CGF.getContext().IntTy); 1778 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1779 CGF.getContext().BoolTy, Args); 1780 1781 return SuccessFailureRVal.getScalarVal(); 1782 } 1783 1784 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1785 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1786 llvm::AtomicOrdering Failure, bool IsWeak) { 1787 // Check whether we should use a library call. 1788 if (shouldUseLibcall()) { 1789 // Produce a source address. 1790 Address ExpectedAddr = materializeRValue(Expected); 1791 Address DesiredAddr = materializeRValue(Desired); 1792 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1793 DesiredAddr.getPointer(), 1794 Success, Failure); 1795 return std::make_pair( 1796 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1797 SourceLocation(), /*AsValue=*/false), 1798 Res); 1799 } 1800 1801 // If we've got a scalar value of the right size, try to avoid going 1802 // through memory. 1803 auto *ExpectedVal = convertRValueToInt(Expected); 1804 auto *DesiredVal = convertRValueToInt(Desired); 1805 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1806 Failure, IsWeak); 1807 return std::make_pair( 1808 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1809 SourceLocation(), /*AsValue=*/false), 1810 Res.second); 1811 } 1812 1813 static void 1814 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1815 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1816 Address DesiredAddr) { 1817 RValue UpRVal; 1818 LValue AtomicLVal = Atomics.getAtomicLValue(); 1819 LValue DesiredLVal; 1820 if (AtomicLVal.isSimple()) { 1821 UpRVal = OldRVal; 1822 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1823 } else { 1824 // Build new lvalue for temp address. 1825 Address Ptr = Atomics.materializeRValue(OldRVal); 1826 LValue UpdateLVal; 1827 if (AtomicLVal.isBitField()) { 1828 UpdateLVal = 1829 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1830 AtomicLVal.getType(), 1831 AtomicLVal.getBaseInfo(), 1832 AtomicLVal.getTBAAInfo()); 1833 DesiredLVal = 1834 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1835 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1836 AtomicLVal.getTBAAInfo()); 1837 } else if (AtomicLVal.isVectorElt()) { 1838 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1839 AtomicLVal.getType(), 1840 AtomicLVal.getBaseInfo(), 1841 AtomicLVal.getTBAAInfo()); 1842 DesiredLVal = LValue::MakeVectorElt( 1843 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1844 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1845 } else { 1846 assert(AtomicLVal.isExtVectorElt()); 1847 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1848 AtomicLVal.getType(), 1849 AtomicLVal.getBaseInfo(), 1850 AtomicLVal.getTBAAInfo()); 1851 DesiredLVal = LValue::MakeExtVectorElt( 1852 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1853 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1854 } 1855 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1856 } 1857 // Store new value in the corresponding memory area. 1858 RValue NewRVal = UpdateOp(UpRVal); 1859 if (NewRVal.isScalar()) { 1860 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1861 } else { 1862 assert(NewRVal.isComplex()); 1863 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1864 /*isInit=*/false); 1865 } 1866 } 1867 1868 void AtomicInfo::EmitAtomicUpdateLibcall( 1869 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1870 bool IsVolatile) { 1871 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1872 1873 Address ExpectedAddr = CreateTempAlloca(); 1874 1875 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1876 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1877 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1878 CGF.EmitBlock(ContBB); 1879 Address DesiredAddr = CreateTempAlloca(); 1880 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1881 requiresMemSetZero(getAtomicAddress().getElementType())) { 1882 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1883 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1884 } 1885 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1886 AggValueSlot::ignored(), 1887 SourceLocation(), /*AsValue=*/false); 1888 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1889 auto *Res = 1890 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1891 DesiredAddr.getPointer(), 1892 AO, Failure); 1893 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1894 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1895 } 1896 1897 void AtomicInfo::EmitAtomicUpdateOp( 1898 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1899 bool IsVolatile) { 1900 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1901 1902 // Do the atomic load. 1903 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); 1904 // For non-simple lvalues perform compare-and-swap procedure. 1905 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1906 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1907 auto *CurBB = CGF.Builder.GetInsertBlock(); 1908 CGF.EmitBlock(ContBB); 1909 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1910 /*NumReservedValues=*/2); 1911 PHI->addIncoming(OldVal, CurBB); 1912 Address NewAtomicAddr = CreateTempAlloca(); 1913 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1914 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1915 requiresMemSetZero(getAtomicAddress().getElementType())) { 1916 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1917 } 1918 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), 1919 SourceLocation(), /*AsValue=*/false); 1920 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1921 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1922 // Try to write new value using cmpxchg operation. 1923 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1924 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1925 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1926 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1927 } 1928 1929 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1930 RValue UpdateRVal, Address DesiredAddr) { 1931 LValue AtomicLVal = Atomics.getAtomicLValue(); 1932 LValue DesiredLVal; 1933 // Build new lvalue for temp address. 1934 if (AtomicLVal.isBitField()) { 1935 DesiredLVal = 1936 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1937 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1938 AtomicLVal.getTBAAInfo()); 1939 } else if (AtomicLVal.isVectorElt()) { 1940 DesiredLVal = 1941 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1942 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1943 AtomicLVal.getTBAAInfo()); 1944 } else { 1945 assert(AtomicLVal.isExtVectorElt()); 1946 DesiredLVal = LValue::MakeExtVectorElt( 1947 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1948 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1949 } 1950 // Store new value in the corresponding memory area. 1951 assert(UpdateRVal.isScalar()); 1952 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1953 } 1954 1955 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1956 RValue UpdateRVal, bool IsVolatile) { 1957 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1958 1959 Address ExpectedAddr = CreateTempAlloca(); 1960 1961 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1962 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1963 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1964 CGF.EmitBlock(ContBB); 1965 Address DesiredAddr = CreateTempAlloca(); 1966 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1967 requiresMemSetZero(getAtomicAddress().getElementType())) { 1968 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1969 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1970 } 1971 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1972 auto *Res = 1973 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1974 DesiredAddr.getPointer(), 1975 AO, Failure); 1976 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1977 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1978 } 1979 1980 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1981 bool IsVolatile) { 1982 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1983 1984 // Do the atomic load. 1985 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); 1986 // For non-simple lvalues perform compare-and-swap procedure. 1987 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1988 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1989 auto *CurBB = CGF.Builder.GetInsertBlock(); 1990 CGF.EmitBlock(ContBB); 1991 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1992 /*NumReservedValues=*/2); 1993 PHI->addIncoming(OldVal, CurBB); 1994 Address NewAtomicAddr = CreateTempAlloca(); 1995 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1996 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1997 requiresMemSetZero(getAtomicAddress().getElementType())) { 1998 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1999 } 2000 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 2001 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 2002 // Try to write new value using cmpxchg operation. 2003 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 2004 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 2005 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 2006 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 2007 } 2008 2009 void AtomicInfo::EmitAtomicUpdate( 2010 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 2011 bool IsVolatile) { 2012 if (shouldUseLibcall()) { 2013 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 2014 } else { 2015 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 2016 } 2017 } 2018 2019 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 2020 bool IsVolatile) { 2021 if (shouldUseLibcall()) { 2022 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 2023 } else { 2024 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 2025 } 2026 } 2027 2028 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 2029 bool isInit) { 2030 bool IsVolatile = lvalue.isVolatileQualified(); 2031 llvm::AtomicOrdering AO; 2032 if (lvalue.getType()->isAtomicType()) { 2033 AO = llvm::AtomicOrdering::SequentiallyConsistent; 2034 } else { 2035 AO = llvm::AtomicOrdering::Release; 2036 IsVolatile = true; 2037 } 2038 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 2039 } 2040 2041 /// Emit a store to an l-value of atomic type. 2042 /// 2043 /// Note that the r-value is expected to be an r-value *of the atomic 2044 /// type*; this means that for aggregate r-values, it should include 2045 /// storage for any padding that was necessary. 2046 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 2047 llvm::AtomicOrdering AO, bool IsVolatile, 2048 bool isInit) { 2049 // If this is an aggregate r-value, it should agree in type except 2050 // maybe for address-space qualification. 2051 assert(!rvalue.isAggregate() || 2052 rvalue.getAggregateAddress().getElementType() == 2053 dest.getAddress(*this).getElementType()); 2054 2055 AtomicInfo atomics(*this, dest); 2056 LValue LVal = atomics.getAtomicLValue(); 2057 2058 // If this is an initialization, just put the value there normally. 2059 if (LVal.isSimple()) { 2060 if (isInit) { 2061 atomics.emitCopyIntoMemory(rvalue); 2062 return; 2063 } 2064 2065 // Check whether we should use a library call. 2066 if (atomics.shouldUseLibcall()) { 2067 // Produce a source address. 2068 Address srcAddr = atomics.materializeRValue(rvalue); 2069 2070 // void __atomic_store(size_t size, void *mem, void *val, int order) 2071 CallArgList args; 2072 args.add(RValue::get(atomics.getAtomicSizeValue()), 2073 getContext().getSizeType()); 2074 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), 2075 getContext().VoidPtrTy); 2076 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), 2077 getContext().VoidPtrTy); 2078 args.add( 2079 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 2080 getContext().IntTy); 2081 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 2082 return; 2083 } 2084 2085 // Okay, we're doing this natively. 2086 llvm::Value *intValue = atomics.convertRValueToInt(rvalue); 2087 2088 // Do the atomic store. 2089 Address addr = 2090 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); 2091 intValue = Builder.CreateIntCast( 2092 intValue, addr.getElementType(), /*isSigned=*/false); 2093 llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 2094 2095 if (AO == llvm::AtomicOrdering::Acquire) 2096 AO = llvm::AtomicOrdering::Monotonic; 2097 else if (AO == llvm::AtomicOrdering::AcquireRelease) 2098 AO = llvm::AtomicOrdering::Release; 2099 // Initializations don't need to be atomic. 2100 if (!isInit) 2101 store->setAtomic(AO); 2102 2103 // Other decoration. 2104 if (IsVolatile) 2105 store->setVolatile(true); 2106 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 2107 return; 2108 } 2109 2110 // Emit simple atomic update operation. 2111 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 2112 } 2113 2114 /// Emit a compare-and-exchange op for atomic type. 2115 /// 2116 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 2117 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 2118 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 2119 AggValueSlot Slot) { 2120 // If this is an aggregate r-value, it should agree in type except 2121 // maybe for address-space qualification. 2122 assert(!Expected.isAggregate() || 2123 Expected.getAggregateAddress().getElementType() == 2124 Obj.getAddress(*this).getElementType()); 2125 assert(!Desired.isAggregate() || 2126 Desired.getAggregateAddress().getElementType() == 2127 Obj.getAddress(*this).getElementType()); 2128 AtomicInfo Atomics(*this, Obj); 2129 2130 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 2131 IsWeak); 2132 } 2133 2134 void CodeGenFunction::EmitAtomicUpdate( 2135 LValue LVal, llvm::AtomicOrdering AO, 2136 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 2137 AtomicInfo Atomics(*this, LVal); 2138 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 2139 } 2140 2141 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 2142 AtomicInfo atomics(*this, dest); 2143 2144 switch (atomics.getEvaluationKind()) { 2145 case TEK_Scalar: { 2146 llvm::Value *value = EmitScalarExpr(init); 2147 atomics.emitCopyIntoMemory(RValue::get(value)); 2148 return; 2149 } 2150 2151 case TEK_Complex: { 2152 ComplexPairTy value = EmitComplexExpr(init); 2153 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 2154 return; 2155 } 2156 2157 case TEK_Aggregate: { 2158 // Fix up the destination if the initializer isn't an expression 2159 // of atomic type. 2160 bool Zeroed = false; 2161 if (!init->getType()->isAtomicType()) { 2162 Zeroed = atomics.emitMemSetZeroIfNecessary(); 2163 dest = atomics.projectValue(); 2164 } 2165 2166 // Evaluate the expression directly into the destination. 2167 AggValueSlot slot = AggValueSlot::forLValue( 2168 dest, *this, AggValueSlot::IsNotDestructed, 2169 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, 2170 AggValueSlot::DoesNotOverlap, 2171 Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed); 2172 2173 EmitAggExpr(init, slot); 2174 return; 2175 } 2176 } 2177 llvm_unreachable("bad evaluation kind"); 2178 } 2179