1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the code for emitting atomic operations. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CGCall.h" 14 #include "CGRecordLayout.h" 15 #include "CodeGenFunction.h" 16 #include "CodeGenModule.h" 17 #include "TargetInfo.h" 18 #include "clang/AST/ASTContext.h" 19 #include "clang/CodeGen/CGFunctionInfo.h" 20 #include "clang/Frontend/FrontendDiagnostic.h" 21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/IR/DataLayout.h" 23 #include "llvm/IR/Intrinsics.h" 24 #include "llvm/IR/Operator.h" 25 26 using namespace clang; 27 using namespace CodeGen; 28 29 namespace { 30 class AtomicInfo { 31 CodeGenFunction &CGF; 32 QualType AtomicTy; 33 QualType ValueTy; 34 uint64_t AtomicSizeInBits; 35 uint64_t ValueSizeInBits; 36 CharUnits AtomicAlign; 37 CharUnits ValueAlign; 38 TypeEvaluationKind EvaluationKind; 39 bool UseLibcall; 40 LValue LVal; 41 CGBitFieldInfo BFI; 42 public: 43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 45 EvaluationKind(TEK_Scalar), UseLibcall(true) { 46 assert(!lvalue.isGlobalReg()); 47 ASTContext &C = CGF.getContext(); 48 if (lvalue.isSimple()) { 49 AtomicTy = lvalue.getType(); 50 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 51 ValueTy = ATy->getValueType(); 52 else 53 ValueTy = AtomicTy; 54 EvaluationKind = CGF.getEvaluationKind(ValueTy); 55 56 uint64_t ValueAlignInBits; 57 uint64_t AtomicAlignInBits; 58 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 59 ValueSizeInBits = ValueTI.Width; 60 ValueAlignInBits = ValueTI.Align; 61 62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 63 AtomicSizeInBits = AtomicTI.Width; 64 AtomicAlignInBits = AtomicTI.Align; 65 66 assert(ValueSizeInBits <= AtomicSizeInBits); 67 assert(ValueAlignInBits <= AtomicAlignInBits); 68 69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 71 if (lvalue.getAlignment().isZero()) 72 lvalue.setAlignment(AtomicAlign); 73 74 LVal = lvalue; 75 } else if (lvalue.isBitField()) { 76 ValueTy = lvalue.getType(); 77 ValueSizeInBits = C.getTypeSize(ValueTy); 78 auto &OrigBFI = lvalue.getBitFieldInfo(); 79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 80 AtomicSizeInBits = C.toBits( 81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 82 .alignTo(lvalue.getAlignment())); 83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); 84 auto OffsetInChars = 85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 86 lvalue.getAlignment(); 87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( 88 VoidPtrAddr, OffsetInChars.getQuantity()); 89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 90 VoidPtrAddr, 91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), 92 "atomic_bitfield_base"); 93 BFI = OrigBFI; 94 BFI.Offset = Offset; 95 BFI.StorageSize = AtomicSizeInBits; 96 BFI.StorageOffset += OffsetInChars; 97 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()), 98 BFI, lvalue.getType(), lvalue.getBaseInfo(), 99 lvalue.getTBAAInfo()); 100 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 101 if (AtomicTy.isNull()) { 102 llvm::APInt Size( 103 /*numBits=*/32, 104 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 105 AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal, 106 /*IndexTypeQuals=*/0); 107 } 108 AtomicAlign = ValueAlign = lvalue.getAlignment(); 109 } else if (lvalue.isVectorElt()) { 110 ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType(); 111 ValueSizeInBits = C.getTypeSize(ValueTy); 112 AtomicTy = lvalue.getType(); 113 AtomicSizeInBits = C.getTypeSize(AtomicTy); 114 AtomicAlign = ValueAlign = lvalue.getAlignment(); 115 LVal = lvalue; 116 } else { 117 assert(lvalue.isExtVectorElt()); 118 ValueTy = lvalue.getType(); 119 ValueSizeInBits = C.getTypeSize(ValueTy); 120 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 121 lvalue.getType(), lvalue.getExtVectorAddress() 122 .getElementType()->getVectorNumElements()); 123 AtomicSizeInBits = C.getTypeSize(AtomicTy); 124 AtomicAlign = ValueAlign = lvalue.getAlignment(); 125 LVal = lvalue; 126 } 127 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 128 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 129 } 130 131 QualType getAtomicType() const { return AtomicTy; } 132 QualType getValueType() const { return ValueTy; } 133 CharUnits getAtomicAlignment() const { return AtomicAlign; } 134 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 135 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 136 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 137 bool shouldUseLibcall() const { return UseLibcall; } 138 const LValue &getAtomicLValue() const { return LVal; } 139 llvm::Value *getAtomicPointer() const { 140 if (LVal.isSimple()) 141 return LVal.getPointer(); 142 else if (LVal.isBitField()) 143 return LVal.getBitFieldPointer(); 144 else if (LVal.isVectorElt()) 145 return LVal.getVectorPointer(); 146 assert(LVal.isExtVectorElt()); 147 return LVal.getExtVectorPointer(); 148 } 149 Address getAtomicAddress() const { 150 return Address(getAtomicPointer(), getAtomicAlignment()); 151 } 152 153 Address getAtomicAddressAsAtomicIntPointer() const { 154 return emitCastToAtomicIntPointer(getAtomicAddress()); 155 } 156 157 /// Is the atomic size larger than the underlying value type? 158 /// 159 /// Note that the absence of padding does not mean that atomic 160 /// objects are completely interchangeable with non-atomic 161 /// objects: we might have promoted the alignment of a type 162 /// without making it bigger. 163 bool hasPadding() const { 164 return (ValueSizeInBits != AtomicSizeInBits); 165 } 166 167 bool emitMemSetZeroIfNecessary() const; 168 169 llvm::Value *getAtomicSizeValue() const { 170 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 171 return CGF.CGM.getSize(size); 172 } 173 174 /// Cast the given pointer to an integer pointer suitable for atomic 175 /// operations if the source. 176 Address emitCastToAtomicIntPointer(Address Addr) const; 177 178 /// If Addr is compatible with the iN that will be used for an atomic 179 /// operation, bitcast it. Otherwise, create a temporary that is suitable 180 /// and copy the value across. 181 Address convertToAtomicIntPointer(Address Addr) const; 182 183 /// Turn an atomic-layout object into an r-value. 184 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 185 SourceLocation loc, bool AsValue) const; 186 187 /// Converts a rvalue to integer value. 188 llvm::Value *convertRValueToInt(RValue RVal) const; 189 190 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, 191 AggValueSlot ResultSlot, 192 SourceLocation Loc, bool AsValue) const; 193 194 /// Copy an atomic r-value into atomic-layout memory. 195 void emitCopyIntoMemory(RValue rvalue) const; 196 197 /// Project an l-value down to the value field. 198 LValue projectValue() const { 199 assert(LVal.isSimple()); 200 Address addr = getAtomicAddress(); 201 if (hasPadding()) 202 addr = CGF.Builder.CreateStructGEP(addr, 0); 203 204 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 205 LVal.getBaseInfo(), LVal.getTBAAInfo()); 206 } 207 208 /// Emits atomic load. 209 /// \returns Loaded value. 210 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 211 bool AsValue, llvm::AtomicOrdering AO, 212 bool IsVolatile); 213 214 /// Emits atomic compare-and-exchange sequence. 215 /// \param Expected Expected value. 216 /// \param Desired Desired value. 217 /// \param Success Atomic ordering for success operation. 218 /// \param Failure Atomic ordering for failed operation. 219 /// \param IsWeak true if atomic operation is weak, false otherwise. 220 /// \returns Pair of values: previous value from storage (value type) and 221 /// boolean flag (i1 type) with true if success and false otherwise. 222 std::pair<RValue, llvm::Value *> 223 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 224 llvm::AtomicOrdering Success = 225 llvm::AtomicOrdering::SequentiallyConsistent, 226 llvm::AtomicOrdering Failure = 227 llvm::AtomicOrdering::SequentiallyConsistent, 228 bool IsWeak = false); 229 230 /// Emits atomic update. 231 /// \param AO Atomic ordering. 232 /// \param UpdateOp Update operation for the current lvalue. 233 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 234 const llvm::function_ref<RValue(RValue)> &UpdateOp, 235 bool IsVolatile); 236 /// Emits atomic update. 237 /// \param AO Atomic ordering. 238 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 239 bool IsVolatile); 240 241 /// Materialize an atomic r-value in atomic-layout memory. 242 Address materializeRValue(RValue rvalue) const; 243 244 /// Creates temp alloca for intermediate operations on atomic value. 245 Address CreateTempAlloca() const; 246 private: 247 bool requiresMemSetZero(llvm::Type *type) const; 248 249 250 /// Emits atomic load as a libcall. 251 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 252 llvm::AtomicOrdering AO, bool IsVolatile); 253 /// Emits atomic load as LLVM instruction. 254 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); 255 /// Emits atomic compare-and-exchange op as a libcall. 256 llvm::Value *EmitAtomicCompareExchangeLibcall( 257 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 258 llvm::AtomicOrdering Success = 259 llvm::AtomicOrdering::SequentiallyConsistent, 260 llvm::AtomicOrdering Failure = 261 llvm::AtomicOrdering::SequentiallyConsistent); 262 /// Emits atomic compare-and-exchange op as LLVM instruction. 263 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 264 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 265 llvm::AtomicOrdering Success = 266 llvm::AtomicOrdering::SequentiallyConsistent, 267 llvm::AtomicOrdering Failure = 268 llvm::AtomicOrdering::SequentiallyConsistent, 269 bool IsWeak = false); 270 /// Emit atomic update as libcalls. 271 void 272 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 273 const llvm::function_ref<RValue(RValue)> &UpdateOp, 274 bool IsVolatile); 275 /// Emit atomic update as LLVM instructions. 276 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 277 const llvm::function_ref<RValue(RValue)> &UpdateOp, 278 bool IsVolatile); 279 /// Emit atomic update as libcalls. 280 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 281 bool IsVolatile); 282 /// Emit atomic update as LLVM instructions. 283 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 284 bool IsVolatile); 285 }; 286 } 287 288 Address AtomicInfo::CreateTempAlloca() const { 289 Address TempAlloca = CGF.CreateMemTemp( 290 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 291 : AtomicTy, 292 getAtomicAlignment(), 293 "atomic-temp"); 294 // Cast to pointer to value type for bitfields. 295 if (LVal.isBitField()) 296 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 297 TempAlloca, getAtomicAddress().getType()); 298 return TempAlloca; 299 } 300 301 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 302 StringRef fnName, 303 QualType resultType, 304 CallArgList &args) { 305 const CGFunctionInfo &fnInfo = 306 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 307 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 308 llvm::FunctionCallee fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); 309 auto callee = CGCallee::forDirect(fn); 310 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); 311 } 312 313 /// Does a store of the given IR type modify the full expected width? 314 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 315 uint64_t expectedSize) { 316 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 317 } 318 319 /// Does the atomic type require memsetting to zero before initialization? 320 /// 321 /// The IR type is provided as a way of making certain queries faster. 322 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 323 // If the atomic type has size padding, we definitely need a memset. 324 if (hasPadding()) return true; 325 326 // Otherwise, do some simple heuristics to try to avoid it: 327 switch (getEvaluationKind()) { 328 // For scalars and complexes, check whether the store size of the 329 // type uses the full size. 330 case TEK_Scalar: 331 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 332 case TEK_Complex: 333 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 334 AtomicSizeInBits / 2); 335 336 // Padding in structs has an undefined bit pattern. User beware. 337 case TEK_Aggregate: 338 return false; 339 } 340 llvm_unreachable("bad evaluation kind"); 341 } 342 343 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 344 assert(LVal.isSimple()); 345 llvm::Value *addr = LVal.getPointer(); 346 if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 347 return false; 348 349 CGF.Builder.CreateMemSet( 350 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 351 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 352 LVal.getAlignment().getQuantity()); 353 return true; 354 } 355 356 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 357 Address Dest, Address Ptr, 358 Address Val1, Address Val2, 359 uint64_t Size, 360 llvm::AtomicOrdering SuccessOrder, 361 llvm::AtomicOrdering FailureOrder, 362 llvm::SyncScope::ID Scope) { 363 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 364 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 365 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 366 367 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 368 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder, 369 Scope); 370 Pair->setVolatile(E->isVolatile()); 371 Pair->setWeak(IsWeak); 372 373 // Cmp holds the result of the compare-exchange operation: true on success, 374 // false on failure. 375 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 376 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 377 378 // This basic block is used to hold the store instruction if the operation 379 // failed. 380 llvm::BasicBlock *StoreExpectedBB = 381 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 382 383 // This basic block is the exit point of the operation, we should end up 384 // here regardless of whether or not the operation succeeded. 385 llvm::BasicBlock *ContinueBB = 386 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 387 388 // Update Expected if Expected isn't equal to Old, otherwise branch to the 389 // exit point. 390 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 391 392 CGF.Builder.SetInsertPoint(StoreExpectedBB); 393 // Update the memory at Expected with Old's value. 394 CGF.Builder.CreateStore(Old, Val1); 395 // Finally, branch to the exit point. 396 CGF.Builder.CreateBr(ContinueBB); 397 398 CGF.Builder.SetInsertPoint(ContinueBB); 399 // Update the memory at Dest with Cmp's value. 400 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 401 } 402 403 /// Given an ordering required on success, emit all possible cmpxchg 404 /// instructions to cope with the provided (but possibly only dynamically known) 405 /// FailureOrder. 406 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 407 bool IsWeak, Address Dest, Address Ptr, 408 Address Val1, Address Val2, 409 llvm::Value *FailureOrderVal, 410 uint64_t Size, 411 llvm::AtomicOrdering SuccessOrder, 412 llvm::SyncScope::ID Scope) { 413 llvm::AtomicOrdering FailureOrder; 414 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 415 auto FOS = FO->getSExtValue(); 416 if (!llvm::isValidAtomicOrderingCABI(FOS)) 417 FailureOrder = llvm::AtomicOrdering::Monotonic; 418 else 419 switch ((llvm::AtomicOrderingCABI)FOS) { 420 case llvm::AtomicOrderingCABI::relaxed: 421 case llvm::AtomicOrderingCABI::release: 422 case llvm::AtomicOrderingCABI::acq_rel: 423 FailureOrder = llvm::AtomicOrdering::Monotonic; 424 break; 425 case llvm::AtomicOrderingCABI::consume: 426 case llvm::AtomicOrderingCABI::acquire: 427 FailureOrder = llvm::AtomicOrdering::Acquire; 428 break; 429 case llvm::AtomicOrderingCABI::seq_cst: 430 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 431 break; 432 } 433 if (isStrongerThan(FailureOrder, SuccessOrder)) { 434 // Don't assert on undefined behavior "failure argument shall be no 435 // stronger than the success argument". 436 FailureOrder = 437 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder); 438 } 439 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 440 FailureOrder, Scope); 441 return; 442 } 443 444 // Create all the relevant BB's 445 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 446 *SeqCstBB = nullptr; 447 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 448 if (SuccessOrder != llvm::AtomicOrdering::Monotonic && 449 SuccessOrder != llvm::AtomicOrdering::Release) 450 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 451 if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent) 452 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 453 454 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 455 456 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 457 458 // Emit all the different atomics 459 460 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 461 // doesn't matter unless someone is crazy enough to use something that 462 // doesn't fold to a constant for the ordering. 463 CGF.Builder.SetInsertPoint(MonotonicBB); 464 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 465 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); 466 CGF.Builder.CreateBr(ContBB); 467 468 if (AcquireBB) { 469 CGF.Builder.SetInsertPoint(AcquireBB); 470 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 471 Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope); 472 CGF.Builder.CreateBr(ContBB); 473 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 474 AcquireBB); 475 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 476 AcquireBB); 477 } 478 if (SeqCstBB) { 479 CGF.Builder.SetInsertPoint(SeqCstBB); 480 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 481 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 482 CGF.Builder.CreateBr(ContBB); 483 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 484 SeqCstBB); 485 } 486 487 CGF.Builder.SetInsertPoint(ContBB); 488 } 489 490 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 491 Address Ptr, Address Val1, Address Val2, 492 llvm::Value *IsWeak, llvm::Value *FailureOrder, 493 uint64_t Size, llvm::AtomicOrdering Order, 494 llvm::SyncScope::ID Scope) { 495 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 496 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 497 498 switch (E->getOp()) { 499 case AtomicExpr::AO__c11_atomic_init: 500 case AtomicExpr::AO__opencl_atomic_init: 501 llvm_unreachable("Already handled!"); 502 503 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 504 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 505 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 506 FailureOrder, Size, Order, Scope); 507 return; 508 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 509 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 510 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 511 FailureOrder, Size, Order, Scope); 512 return; 513 case AtomicExpr::AO__atomic_compare_exchange: 514 case AtomicExpr::AO__atomic_compare_exchange_n: { 515 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 516 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 517 Val1, Val2, FailureOrder, Size, Order, Scope); 518 } else { 519 // Create all the relevant BB's 520 llvm::BasicBlock *StrongBB = 521 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 522 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 523 llvm::BasicBlock *ContBB = 524 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 525 526 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 527 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 528 529 CGF.Builder.SetInsertPoint(StrongBB); 530 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 531 FailureOrder, Size, Order, Scope); 532 CGF.Builder.CreateBr(ContBB); 533 534 CGF.Builder.SetInsertPoint(WeakBB); 535 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 536 FailureOrder, Size, Order, Scope); 537 CGF.Builder.CreateBr(ContBB); 538 539 CGF.Builder.SetInsertPoint(ContBB); 540 } 541 return; 542 } 543 case AtomicExpr::AO__c11_atomic_load: 544 case AtomicExpr::AO__opencl_atomic_load: 545 case AtomicExpr::AO__atomic_load_n: 546 case AtomicExpr::AO__atomic_load: { 547 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 548 Load->setAtomic(Order, Scope); 549 Load->setVolatile(E->isVolatile()); 550 CGF.Builder.CreateStore(Load, Dest); 551 return; 552 } 553 554 case AtomicExpr::AO__c11_atomic_store: 555 case AtomicExpr::AO__opencl_atomic_store: 556 case AtomicExpr::AO__atomic_store: 557 case AtomicExpr::AO__atomic_store_n: { 558 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 559 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 560 Store->setAtomic(Order, Scope); 561 Store->setVolatile(E->isVolatile()); 562 return; 563 } 564 565 case AtomicExpr::AO__c11_atomic_exchange: 566 case AtomicExpr::AO__opencl_atomic_exchange: 567 case AtomicExpr::AO__atomic_exchange_n: 568 case AtomicExpr::AO__atomic_exchange: 569 Op = llvm::AtomicRMWInst::Xchg; 570 break; 571 572 case AtomicExpr::AO__atomic_add_fetch: 573 PostOp = llvm::Instruction::Add; 574 LLVM_FALLTHROUGH; 575 case AtomicExpr::AO__c11_atomic_fetch_add: 576 case AtomicExpr::AO__opencl_atomic_fetch_add: 577 case AtomicExpr::AO__atomic_fetch_add: 578 Op = llvm::AtomicRMWInst::Add; 579 break; 580 581 case AtomicExpr::AO__atomic_sub_fetch: 582 PostOp = llvm::Instruction::Sub; 583 LLVM_FALLTHROUGH; 584 case AtomicExpr::AO__c11_atomic_fetch_sub: 585 case AtomicExpr::AO__opencl_atomic_fetch_sub: 586 case AtomicExpr::AO__atomic_fetch_sub: 587 Op = llvm::AtomicRMWInst::Sub; 588 break; 589 590 case AtomicExpr::AO__opencl_atomic_fetch_min: 591 case AtomicExpr::AO__atomic_fetch_min: 592 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min 593 : llvm::AtomicRMWInst::UMin; 594 break; 595 596 case AtomicExpr::AO__opencl_atomic_fetch_max: 597 case AtomicExpr::AO__atomic_fetch_max: 598 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max 599 : llvm::AtomicRMWInst::UMax; 600 break; 601 602 case AtomicExpr::AO__atomic_and_fetch: 603 PostOp = llvm::Instruction::And; 604 LLVM_FALLTHROUGH; 605 case AtomicExpr::AO__c11_atomic_fetch_and: 606 case AtomicExpr::AO__opencl_atomic_fetch_and: 607 case AtomicExpr::AO__atomic_fetch_and: 608 Op = llvm::AtomicRMWInst::And; 609 break; 610 611 case AtomicExpr::AO__atomic_or_fetch: 612 PostOp = llvm::Instruction::Or; 613 LLVM_FALLTHROUGH; 614 case AtomicExpr::AO__c11_atomic_fetch_or: 615 case AtomicExpr::AO__opencl_atomic_fetch_or: 616 case AtomicExpr::AO__atomic_fetch_or: 617 Op = llvm::AtomicRMWInst::Or; 618 break; 619 620 case AtomicExpr::AO__atomic_xor_fetch: 621 PostOp = llvm::Instruction::Xor; 622 LLVM_FALLTHROUGH; 623 case AtomicExpr::AO__c11_atomic_fetch_xor: 624 case AtomicExpr::AO__opencl_atomic_fetch_xor: 625 case AtomicExpr::AO__atomic_fetch_xor: 626 Op = llvm::AtomicRMWInst::Xor; 627 break; 628 629 case AtomicExpr::AO__atomic_nand_fetch: 630 PostOp = llvm::Instruction::And; // the NOT is special cased below 631 LLVM_FALLTHROUGH; 632 case AtomicExpr::AO__atomic_fetch_nand: 633 Op = llvm::AtomicRMWInst::Nand; 634 break; 635 } 636 637 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 638 llvm::AtomicRMWInst *RMWI = 639 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope); 640 RMWI->setVolatile(E->isVolatile()); 641 642 // For __atomic_*_fetch operations, perform the operation again to 643 // determine the value which was written. 644 llvm::Value *Result = RMWI; 645 if (PostOp) 646 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); 647 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 648 Result = CGF.Builder.CreateNot(Result); 649 CGF.Builder.CreateStore(Result, Dest); 650 } 651 652 // This function emits any expression (scalar, complex, or aggregate) 653 // into a temporary alloca. 654 static Address 655 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 656 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 657 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 658 /*Init*/ true); 659 return DeclPtr; 660 } 661 662 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, 663 Address Ptr, Address Val1, Address Val2, 664 llvm::Value *IsWeak, llvm::Value *FailureOrder, 665 uint64_t Size, llvm::AtomicOrdering Order, 666 llvm::Value *Scope) { 667 auto ScopeModel = Expr->getScopeModel(); 668 669 // LLVM atomic instructions always have synch scope. If clang atomic 670 // expression has no scope operand, use default LLVM synch scope. 671 if (!ScopeModel) { 672 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 673 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID("")); 674 return; 675 } 676 677 // Handle constant scope. 678 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { 679 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( 680 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()), 681 Order, CGF.CGM.getLLVMContext()); 682 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 683 Order, SCID); 684 return; 685 } 686 687 // Handle non-constant scope. 688 auto &Builder = CGF.Builder; 689 auto Scopes = ScopeModel->getRuntimeValues(); 690 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; 691 for (auto S : Scopes) 692 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); 693 694 llvm::BasicBlock *ContBB = 695 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn); 696 697 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); 698 // If unsupported synch scope is encountered at run time, assume a fallback 699 // synch scope value. 700 auto FallBack = ScopeModel->getFallBackValue(); 701 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); 702 for (auto S : Scopes) { 703 auto *B = BB[S]; 704 if (S != FallBack) 705 SI->addCase(Builder.getInt32(S), B); 706 707 Builder.SetInsertPoint(B); 708 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 709 Order, 710 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(), 711 ScopeModel->map(S), 712 Order, 713 CGF.getLLVMContext())); 714 Builder.CreateBr(ContBB); 715 } 716 717 Builder.SetInsertPoint(ContBB); 718 } 719 720 static void 721 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 722 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 723 SourceLocation Loc, CharUnits SizeInChars) { 724 if (UseOptimizedLibcall) { 725 // Load value and pass it to the function directly. 726 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); 727 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); 728 ValTy = 729 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); 730 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(), 731 SizeInBits)->getPointerTo(); 732 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align); 733 Val = CGF.EmitLoadOfScalar(Ptr, false, 734 CGF.getContext().getPointerType(ValTy), 735 Loc); 736 // Coerce the value into an appropriately sized integer type. 737 Args.add(RValue::get(Val), ValTy); 738 } else { 739 // Non-optimized functions always take a reference. 740 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), 741 CGF.getContext().VoidPtrTy); 742 } 743 } 744 745 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 746 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 747 QualType MemTy = AtomicTy; 748 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 749 MemTy = AT->getValueType(); 750 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 751 752 Address Val1 = Address::invalid(); 753 Address Val2 = Address::invalid(); 754 Address Dest = Address::invalid(); 755 Address Ptr = EmitPointerWithAlignment(E->getPtr()); 756 757 if (E->getOp() == AtomicExpr::AO__c11_atomic_init || 758 E->getOp() == AtomicExpr::AO__opencl_atomic_init) { 759 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 760 EmitAtomicInit(E->getVal1(), lvalue); 761 return RValue::get(nullptr); 762 } 763 764 CharUnits sizeChars, alignChars; 765 std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy); 766 uint64_t Size = sizeChars.getQuantity(); 767 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 768 769 bool Oversized = getContext().toBits(sizeChars) > MaxInlineWidthInBits; 770 bool Misaligned = (Ptr.getAlignment() % sizeChars) != 0; 771 bool UseLibcall = Misaligned | Oversized; 772 773 if (UseLibcall) { 774 CGM.getDiags().Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned) 775 << !Oversized; 776 } 777 778 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 779 llvm::Value *Scope = 780 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; 781 782 switch (E->getOp()) { 783 case AtomicExpr::AO__c11_atomic_init: 784 case AtomicExpr::AO__opencl_atomic_init: 785 llvm_unreachable("Already handled above with EmitAtomicInit!"); 786 787 case AtomicExpr::AO__c11_atomic_load: 788 case AtomicExpr::AO__opencl_atomic_load: 789 case AtomicExpr::AO__atomic_load_n: 790 break; 791 792 case AtomicExpr::AO__atomic_load: 793 Dest = EmitPointerWithAlignment(E->getVal1()); 794 break; 795 796 case AtomicExpr::AO__atomic_store: 797 Val1 = EmitPointerWithAlignment(E->getVal1()); 798 break; 799 800 case AtomicExpr::AO__atomic_exchange: 801 Val1 = EmitPointerWithAlignment(E->getVal1()); 802 Dest = EmitPointerWithAlignment(E->getVal2()); 803 break; 804 805 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 806 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 807 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 808 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 809 case AtomicExpr::AO__atomic_compare_exchange_n: 810 case AtomicExpr::AO__atomic_compare_exchange: 811 Val1 = EmitPointerWithAlignment(E->getVal1()); 812 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 813 Val2 = EmitPointerWithAlignment(E->getVal2()); 814 else 815 Val2 = EmitValToTemp(*this, E->getVal2()); 816 OrderFail = EmitScalarExpr(E->getOrderFail()); 817 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || 818 E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 819 IsWeak = EmitScalarExpr(E->getWeak()); 820 break; 821 822 case AtomicExpr::AO__c11_atomic_fetch_add: 823 case AtomicExpr::AO__c11_atomic_fetch_sub: 824 case AtomicExpr::AO__opencl_atomic_fetch_add: 825 case AtomicExpr::AO__opencl_atomic_fetch_sub: 826 if (MemTy->isPointerType()) { 827 // For pointer arithmetic, we're required to do a bit of math: 828 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 829 // ... but only for the C11 builtins. The GNU builtins expect the 830 // user to multiply by sizeof(T). 831 QualType Val1Ty = E->getVal1()->getType(); 832 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 833 CharUnits PointeeIncAmt = 834 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 835 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 836 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 837 Val1 = Temp; 838 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 839 break; 840 } 841 LLVM_FALLTHROUGH; 842 case AtomicExpr::AO__atomic_fetch_add: 843 case AtomicExpr::AO__atomic_fetch_sub: 844 case AtomicExpr::AO__atomic_add_fetch: 845 case AtomicExpr::AO__atomic_sub_fetch: 846 case AtomicExpr::AO__c11_atomic_store: 847 case AtomicExpr::AO__c11_atomic_exchange: 848 case AtomicExpr::AO__opencl_atomic_store: 849 case AtomicExpr::AO__opencl_atomic_exchange: 850 case AtomicExpr::AO__atomic_store_n: 851 case AtomicExpr::AO__atomic_exchange_n: 852 case AtomicExpr::AO__c11_atomic_fetch_and: 853 case AtomicExpr::AO__c11_atomic_fetch_or: 854 case AtomicExpr::AO__c11_atomic_fetch_xor: 855 case AtomicExpr::AO__opencl_atomic_fetch_and: 856 case AtomicExpr::AO__opencl_atomic_fetch_or: 857 case AtomicExpr::AO__opencl_atomic_fetch_xor: 858 case AtomicExpr::AO__opencl_atomic_fetch_min: 859 case AtomicExpr::AO__opencl_atomic_fetch_max: 860 case AtomicExpr::AO__atomic_fetch_and: 861 case AtomicExpr::AO__atomic_fetch_or: 862 case AtomicExpr::AO__atomic_fetch_xor: 863 case AtomicExpr::AO__atomic_fetch_nand: 864 case AtomicExpr::AO__atomic_and_fetch: 865 case AtomicExpr::AO__atomic_or_fetch: 866 case AtomicExpr::AO__atomic_xor_fetch: 867 case AtomicExpr::AO__atomic_nand_fetch: 868 case AtomicExpr::AO__atomic_fetch_min: 869 case AtomicExpr::AO__atomic_fetch_max: 870 Val1 = EmitValToTemp(*this, E->getVal1()); 871 break; 872 } 873 874 QualType RValTy = E->getType().getUnqualifiedType(); 875 876 // The inlined atomics only function on iN types, where N is a power of 2. We 877 // need to make sure (via temporaries if necessary) that all incoming values 878 // are compatible. 879 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 880 AtomicInfo Atomics(*this, AtomicVal); 881 882 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); 883 if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1); 884 if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2); 885 if (Dest.isValid()) 886 Dest = Atomics.emitCastToAtomicIntPointer(Dest); 887 else if (E->isCmpXChg()) 888 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 889 else if (!RValTy->isVoidType()) 890 Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca()); 891 892 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 893 if (UseLibcall) { 894 bool UseOptimizedLibcall = false; 895 switch (E->getOp()) { 896 case AtomicExpr::AO__c11_atomic_init: 897 case AtomicExpr::AO__opencl_atomic_init: 898 llvm_unreachable("Already handled above with EmitAtomicInit!"); 899 900 case AtomicExpr::AO__c11_atomic_fetch_add: 901 case AtomicExpr::AO__opencl_atomic_fetch_add: 902 case AtomicExpr::AO__atomic_fetch_add: 903 case AtomicExpr::AO__c11_atomic_fetch_and: 904 case AtomicExpr::AO__opencl_atomic_fetch_and: 905 case AtomicExpr::AO__atomic_fetch_and: 906 case AtomicExpr::AO__c11_atomic_fetch_or: 907 case AtomicExpr::AO__opencl_atomic_fetch_or: 908 case AtomicExpr::AO__atomic_fetch_or: 909 case AtomicExpr::AO__atomic_fetch_nand: 910 case AtomicExpr::AO__c11_atomic_fetch_sub: 911 case AtomicExpr::AO__opencl_atomic_fetch_sub: 912 case AtomicExpr::AO__atomic_fetch_sub: 913 case AtomicExpr::AO__c11_atomic_fetch_xor: 914 case AtomicExpr::AO__opencl_atomic_fetch_xor: 915 case AtomicExpr::AO__opencl_atomic_fetch_min: 916 case AtomicExpr::AO__opencl_atomic_fetch_max: 917 case AtomicExpr::AO__atomic_fetch_xor: 918 case AtomicExpr::AO__atomic_add_fetch: 919 case AtomicExpr::AO__atomic_and_fetch: 920 case AtomicExpr::AO__atomic_nand_fetch: 921 case AtomicExpr::AO__atomic_or_fetch: 922 case AtomicExpr::AO__atomic_sub_fetch: 923 case AtomicExpr::AO__atomic_xor_fetch: 924 case AtomicExpr::AO__atomic_fetch_min: 925 case AtomicExpr::AO__atomic_fetch_max: 926 // For these, only library calls for certain sizes exist. 927 UseOptimizedLibcall = true; 928 break; 929 930 case AtomicExpr::AO__atomic_load: 931 case AtomicExpr::AO__atomic_store: 932 case AtomicExpr::AO__atomic_exchange: 933 case AtomicExpr::AO__atomic_compare_exchange: 934 // Use the generic version if we don't know that the operand will be 935 // suitably aligned for the optimized version. 936 if (Misaligned) 937 break; 938 LLVM_FALLTHROUGH; 939 case AtomicExpr::AO__c11_atomic_load: 940 case AtomicExpr::AO__c11_atomic_store: 941 case AtomicExpr::AO__c11_atomic_exchange: 942 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 943 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 944 case AtomicExpr::AO__opencl_atomic_load: 945 case AtomicExpr::AO__opencl_atomic_store: 946 case AtomicExpr::AO__opencl_atomic_exchange: 947 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 948 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 949 case AtomicExpr::AO__atomic_load_n: 950 case AtomicExpr::AO__atomic_store_n: 951 case AtomicExpr::AO__atomic_exchange_n: 952 case AtomicExpr::AO__atomic_compare_exchange_n: 953 // Only use optimized library calls for sizes for which they exist. 954 // FIXME: Size == 16 optimized library functions exist too. 955 if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 956 UseOptimizedLibcall = true; 957 break; 958 } 959 960 CallArgList Args; 961 if (!UseOptimizedLibcall) { 962 // For non-optimized library calls, the size is the first parameter 963 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 964 getContext().getSizeType()); 965 } 966 // Atomic address is the first or second parameter 967 // The OpenCL atomic library functions only accept pointer arguments to 968 // generic address space. 969 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { 970 if (!E->isOpenCL()) 971 return V; 972 auto AS = PT->getAs<PointerType>()->getPointeeType().getAddressSpace(); 973 if (AS == LangAS::opencl_generic) 974 return V; 975 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); 976 auto T = V->getType(); 977 auto *DestType = T->getPointerElementType()->getPointerTo(DestAS); 978 979 return getTargetHooks().performAddrSpaceCast( 980 *this, V, AS, LangAS::opencl_generic, DestType, false); 981 }; 982 983 Args.add(RValue::get(CastToGenericAddrSpace( 984 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())), 985 getContext().VoidPtrTy); 986 987 std::string LibCallName; 988 QualType LoweredMemTy = 989 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 990 QualType RetTy; 991 bool HaveRetTy = false; 992 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 993 switch (E->getOp()) { 994 case AtomicExpr::AO__c11_atomic_init: 995 case AtomicExpr::AO__opencl_atomic_init: 996 llvm_unreachable("Already handled!"); 997 998 // There is only one libcall for compare an exchange, because there is no 999 // optimisation benefit possible from a libcall version of a weak compare 1000 // and exchange. 1001 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 1002 // void *desired, int success, int failure) 1003 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 1004 // int success, int failure) 1005 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 1006 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 1007 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 1008 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 1009 case AtomicExpr::AO__atomic_compare_exchange: 1010 case AtomicExpr::AO__atomic_compare_exchange_n: 1011 LibCallName = "__atomic_compare_exchange"; 1012 RetTy = getContext().BoolTy; 1013 HaveRetTy = true; 1014 Args.add( 1015 RValue::get(CastToGenericAddrSpace( 1016 EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())), 1017 getContext().VoidPtrTy); 1018 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), 1019 MemTy, E->getExprLoc(), sizeChars); 1020 Args.add(RValue::get(Order), getContext().IntTy); 1021 Order = OrderFail; 1022 break; 1023 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 1024 // int order) 1025 // T __atomic_exchange_N(T *mem, T val, int order) 1026 case AtomicExpr::AO__c11_atomic_exchange: 1027 case AtomicExpr::AO__opencl_atomic_exchange: 1028 case AtomicExpr::AO__atomic_exchange_n: 1029 case AtomicExpr::AO__atomic_exchange: 1030 LibCallName = "__atomic_exchange"; 1031 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1032 MemTy, E->getExprLoc(), sizeChars); 1033 break; 1034 // void __atomic_store(size_t size, void *mem, void *val, int order) 1035 // void __atomic_store_N(T *mem, T val, int order) 1036 case AtomicExpr::AO__c11_atomic_store: 1037 case AtomicExpr::AO__opencl_atomic_store: 1038 case AtomicExpr::AO__atomic_store: 1039 case AtomicExpr::AO__atomic_store_n: 1040 LibCallName = "__atomic_store"; 1041 RetTy = getContext().VoidTy; 1042 HaveRetTy = true; 1043 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1044 MemTy, E->getExprLoc(), sizeChars); 1045 break; 1046 // void __atomic_load(size_t size, void *mem, void *return, int order) 1047 // T __atomic_load_N(T *mem, int order) 1048 case AtomicExpr::AO__c11_atomic_load: 1049 case AtomicExpr::AO__opencl_atomic_load: 1050 case AtomicExpr::AO__atomic_load: 1051 case AtomicExpr::AO__atomic_load_n: 1052 LibCallName = "__atomic_load"; 1053 break; 1054 // T __atomic_add_fetch_N(T *mem, T val, int order) 1055 // T __atomic_fetch_add_N(T *mem, T val, int order) 1056 case AtomicExpr::AO__atomic_add_fetch: 1057 PostOp = llvm::Instruction::Add; 1058 LLVM_FALLTHROUGH; 1059 case AtomicExpr::AO__c11_atomic_fetch_add: 1060 case AtomicExpr::AO__opencl_atomic_fetch_add: 1061 case AtomicExpr::AO__atomic_fetch_add: 1062 LibCallName = "__atomic_fetch_add"; 1063 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1064 LoweredMemTy, E->getExprLoc(), sizeChars); 1065 break; 1066 // T __atomic_and_fetch_N(T *mem, T val, int order) 1067 // T __atomic_fetch_and_N(T *mem, T val, int order) 1068 case AtomicExpr::AO__atomic_and_fetch: 1069 PostOp = llvm::Instruction::And; 1070 LLVM_FALLTHROUGH; 1071 case AtomicExpr::AO__c11_atomic_fetch_and: 1072 case AtomicExpr::AO__opencl_atomic_fetch_and: 1073 case AtomicExpr::AO__atomic_fetch_and: 1074 LibCallName = "__atomic_fetch_and"; 1075 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1076 MemTy, E->getExprLoc(), sizeChars); 1077 break; 1078 // T __atomic_or_fetch_N(T *mem, T val, int order) 1079 // T __atomic_fetch_or_N(T *mem, T val, int order) 1080 case AtomicExpr::AO__atomic_or_fetch: 1081 PostOp = llvm::Instruction::Or; 1082 LLVM_FALLTHROUGH; 1083 case AtomicExpr::AO__c11_atomic_fetch_or: 1084 case AtomicExpr::AO__opencl_atomic_fetch_or: 1085 case AtomicExpr::AO__atomic_fetch_or: 1086 LibCallName = "__atomic_fetch_or"; 1087 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1088 MemTy, E->getExprLoc(), sizeChars); 1089 break; 1090 // T __atomic_sub_fetch_N(T *mem, T val, int order) 1091 // T __atomic_fetch_sub_N(T *mem, T val, int order) 1092 case AtomicExpr::AO__atomic_sub_fetch: 1093 PostOp = llvm::Instruction::Sub; 1094 LLVM_FALLTHROUGH; 1095 case AtomicExpr::AO__c11_atomic_fetch_sub: 1096 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1097 case AtomicExpr::AO__atomic_fetch_sub: 1098 LibCallName = "__atomic_fetch_sub"; 1099 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1100 LoweredMemTy, E->getExprLoc(), sizeChars); 1101 break; 1102 // T __atomic_xor_fetch_N(T *mem, T val, int order) 1103 // T __atomic_fetch_xor_N(T *mem, T val, int order) 1104 case AtomicExpr::AO__atomic_xor_fetch: 1105 PostOp = llvm::Instruction::Xor; 1106 LLVM_FALLTHROUGH; 1107 case AtomicExpr::AO__c11_atomic_fetch_xor: 1108 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1109 case AtomicExpr::AO__atomic_fetch_xor: 1110 LibCallName = "__atomic_fetch_xor"; 1111 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1112 MemTy, E->getExprLoc(), sizeChars); 1113 break; 1114 case AtomicExpr::AO__atomic_fetch_min: 1115 case AtomicExpr::AO__opencl_atomic_fetch_min: 1116 LibCallName = E->getValueType()->isSignedIntegerType() 1117 ? "__atomic_fetch_min" 1118 : "__atomic_fetch_umin"; 1119 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1120 LoweredMemTy, E->getExprLoc(), sizeChars); 1121 break; 1122 case AtomicExpr::AO__atomic_fetch_max: 1123 case AtomicExpr::AO__opencl_atomic_fetch_max: 1124 LibCallName = E->getValueType()->isSignedIntegerType() 1125 ? "__atomic_fetch_max" 1126 : "__atomic_fetch_umax"; 1127 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1128 LoweredMemTy, E->getExprLoc(), sizeChars); 1129 break; 1130 // T __atomic_nand_fetch_N(T *mem, T val, int order) 1131 // T __atomic_fetch_nand_N(T *mem, T val, int order) 1132 case AtomicExpr::AO__atomic_nand_fetch: 1133 PostOp = llvm::Instruction::And; // the NOT is special cased below 1134 LLVM_FALLTHROUGH; 1135 case AtomicExpr::AO__atomic_fetch_nand: 1136 LibCallName = "__atomic_fetch_nand"; 1137 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1138 MemTy, E->getExprLoc(), sizeChars); 1139 break; 1140 } 1141 1142 if (E->isOpenCL()) { 1143 LibCallName = std::string("__opencl") + 1144 StringRef(LibCallName).drop_front(1).str(); 1145 1146 } 1147 // Optimized functions have the size in their name. 1148 if (UseOptimizedLibcall) 1149 LibCallName += "_" + llvm::utostr(Size); 1150 // By default, assume we return a value of the atomic type. 1151 if (!HaveRetTy) { 1152 if (UseOptimizedLibcall) { 1153 // Value is returned directly. 1154 // The function returns an appropriately sized integer type. 1155 RetTy = getContext().getIntTypeForBitwidth( 1156 getContext().toBits(sizeChars), /*Signed=*/false); 1157 } else { 1158 // Value is returned through parameter before the order. 1159 RetTy = getContext().VoidTy; 1160 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), 1161 getContext().VoidPtrTy); 1162 } 1163 } 1164 // order is always the last parameter 1165 Args.add(RValue::get(Order), 1166 getContext().IntTy); 1167 if (E->isOpenCL()) 1168 Args.add(RValue::get(Scope), getContext().IntTy); 1169 1170 // PostOp is only needed for the atomic_*_fetch operations, and 1171 // thus is only needed for and implemented in the 1172 // UseOptimizedLibcall codepath. 1173 assert(UseOptimizedLibcall || !PostOp); 1174 1175 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 1176 // The value is returned directly from the libcall. 1177 if (E->isCmpXChg()) 1178 return Res; 1179 1180 // The value is returned directly for optimized libcalls but the expr 1181 // provided an out-param. 1182 if (UseOptimizedLibcall && Res.getScalarVal()) { 1183 llvm::Value *ResVal = Res.getScalarVal(); 1184 if (PostOp) { 1185 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1186 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); 1187 } 1188 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 1189 ResVal = Builder.CreateNot(ResVal); 1190 1191 Builder.CreateStore( 1192 ResVal, 1193 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo())); 1194 } 1195 1196 if (RValTy->isVoidType()) 1197 return RValue::get(nullptr); 1198 1199 return convertTempToRValue( 1200 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1201 RValTy, E->getExprLoc()); 1202 } 1203 1204 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1205 E->getOp() == AtomicExpr::AO__opencl_atomic_store || 1206 E->getOp() == AtomicExpr::AO__atomic_store || 1207 E->getOp() == AtomicExpr::AO__atomic_store_n; 1208 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1209 E->getOp() == AtomicExpr::AO__opencl_atomic_load || 1210 E->getOp() == AtomicExpr::AO__atomic_load || 1211 E->getOp() == AtomicExpr::AO__atomic_load_n; 1212 1213 if (isa<llvm::ConstantInt>(Order)) { 1214 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1215 // We should not ever get to a case where the ordering isn't a valid C ABI 1216 // value, but it's hard to enforce that in general. 1217 if (llvm::isValidAtomicOrderingCABI(ord)) 1218 switch ((llvm::AtomicOrderingCABI)ord) { 1219 case llvm::AtomicOrderingCABI::relaxed: 1220 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1221 llvm::AtomicOrdering::Monotonic, Scope); 1222 break; 1223 case llvm::AtomicOrderingCABI::consume: 1224 case llvm::AtomicOrderingCABI::acquire: 1225 if (IsStore) 1226 break; // Avoid crashing on code with undefined behavior 1227 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1228 llvm::AtomicOrdering::Acquire, Scope); 1229 break; 1230 case llvm::AtomicOrderingCABI::release: 1231 if (IsLoad) 1232 break; // Avoid crashing on code with undefined behavior 1233 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1234 llvm::AtomicOrdering::Release, Scope); 1235 break; 1236 case llvm::AtomicOrderingCABI::acq_rel: 1237 if (IsLoad || IsStore) 1238 break; // Avoid crashing on code with undefined behavior 1239 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1240 llvm::AtomicOrdering::AcquireRelease, Scope); 1241 break; 1242 case llvm::AtomicOrderingCABI::seq_cst: 1243 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1244 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1245 break; 1246 } 1247 if (RValTy->isVoidType()) 1248 return RValue::get(nullptr); 1249 1250 return convertTempToRValue( 1251 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1252 Dest.getAddressSpace())), 1253 RValTy, E->getExprLoc()); 1254 } 1255 1256 // Long case, when Order isn't obviously constant. 1257 1258 // Create all the relevant BB's 1259 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1260 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1261 *SeqCstBB = nullptr; 1262 MonotonicBB = createBasicBlock("monotonic", CurFn); 1263 if (!IsStore) 1264 AcquireBB = createBasicBlock("acquire", CurFn); 1265 if (!IsLoad) 1266 ReleaseBB = createBasicBlock("release", CurFn); 1267 if (!IsLoad && !IsStore) 1268 AcqRelBB = createBasicBlock("acqrel", CurFn); 1269 SeqCstBB = createBasicBlock("seqcst", CurFn); 1270 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1271 1272 // Create the switch for the split 1273 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1274 // doesn't matter unless someone is crazy enough to use something that 1275 // doesn't fold to a constant for the ordering. 1276 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1277 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1278 1279 // Emit all the different atomics 1280 Builder.SetInsertPoint(MonotonicBB); 1281 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1282 llvm::AtomicOrdering::Monotonic, Scope); 1283 Builder.CreateBr(ContBB); 1284 if (!IsStore) { 1285 Builder.SetInsertPoint(AcquireBB); 1286 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1287 llvm::AtomicOrdering::Acquire, Scope); 1288 Builder.CreateBr(ContBB); 1289 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1290 AcquireBB); 1291 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1292 AcquireBB); 1293 } 1294 if (!IsLoad) { 1295 Builder.SetInsertPoint(ReleaseBB); 1296 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1297 llvm::AtomicOrdering::Release, Scope); 1298 Builder.CreateBr(ContBB); 1299 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1300 ReleaseBB); 1301 } 1302 if (!IsLoad && !IsStore) { 1303 Builder.SetInsertPoint(AcqRelBB); 1304 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1305 llvm::AtomicOrdering::AcquireRelease, Scope); 1306 Builder.CreateBr(ContBB); 1307 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1308 AcqRelBB); 1309 } 1310 Builder.SetInsertPoint(SeqCstBB); 1311 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1312 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1313 Builder.CreateBr(ContBB); 1314 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1315 SeqCstBB); 1316 1317 // Cleanup and return 1318 Builder.SetInsertPoint(ContBB); 1319 if (RValTy->isVoidType()) 1320 return RValue::get(nullptr); 1321 1322 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1323 return convertTempToRValue( 1324 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1325 Dest.getAddressSpace())), 1326 RValTy, E->getExprLoc()); 1327 } 1328 1329 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { 1330 unsigned addrspace = 1331 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace(); 1332 llvm::IntegerType *ty = 1333 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1334 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 1335 } 1336 1337 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1338 llvm::Type *Ty = Addr.getElementType(); 1339 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1340 if (SourceSizeInBits != AtomicSizeInBits) { 1341 Address Tmp = CreateTempAlloca(); 1342 CGF.Builder.CreateMemCpy(Tmp, Addr, 1343 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1344 Addr = Tmp; 1345 } 1346 1347 return emitCastToAtomicIntPointer(Addr); 1348 } 1349 1350 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1351 AggValueSlot resultSlot, 1352 SourceLocation loc, 1353 bool asValue) const { 1354 if (LVal.isSimple()) { 1355 if (EvaluationKind == TEK_Aggregate) 1356 return resultSlot.asRValue(); 1357 1358 // Drill into the padding structure if we have one. 1359 if (hasPadding()) 1360 addr = CGF.Builder.CreateStructGEP(addr, 0); 1361 1362 // Otherwise, just convert the temporary to an r-value using the 1363 // normal conversion routine. 1364 return CGF.convertTempToRValue(addr, getValueType(), loc); 1365 } 1366 if (!asValue) 1367 // Get RValue from temp memory as atomic for non-simple lvalues 1368 return RValue::get(CGF.Builder.CreateLoad(addr)); 1369 if (LVal.isBitField()) 1370 return CGF.EmitLoadOfBitfieldLValue( 1371 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1372 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1373 if (LVal.isVectorElt()) 1374 return CGF.EmitLoadOfLValue( 1375 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1376 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1377 assert(LVal.isExtVectorElt()); 1378 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1379 addr, LVal.getExtVectorElts(), LVal.getType(), 1380 LVal.getBaseInfo(), TBAAAccessInfo())); 1381 } 1382 1383 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, 1384 AggValueSlot ResultSlot, 1385 SourceLocation Loc, 1386 bool AsValue) const { 1387 // Try not to in some easy cases. 1388 assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); 1389 if (getEvaluationKind() == TEK_Scalar && 1390 (((!LVal.isBitField() || 1391 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1392 !hasPadding()) || 1393 !AsValue)) { 1394 auto *ValTy = AsValue 1395 ? CGF.ConvertTypeForMem(ValueTy) 1396 : getAtomicAddress().getType()->getPointerElementType(); 1397 if (ValTy->isIntegerTy()) { 1398 assert(IntVal->getType() == ValTy && "Different integer types."); 1399 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); 1400 } else if (ValTy->isPointerTy()) 1401 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); 1402 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) 1403 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); 1404 } 1405 1406 // Create a temporary. This needs to be big enough to hold the 1407 // atomic integer. 1408 Address Temp = Address::invalid(); 1409 bool TempIsVolatile = false; 1410 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1411 assert(!ResultSlot.isIgnored()); 1412 Temp = ResultSlot.getAddress(); 1413 TempIsVolatile = ResultSlot.isVolatile(); 1414 } else { 1415 Temp = CreateTempAlloca(); 1416 } 1417 1418 // Slam the integer into the temporary. 1419 Address CastTemp = emitCastToAtomicIntPointer(Temp); 1420 CGF.Builder.CreateStore(IntVal, CastTemp) 1421 ->setVolatile(TempIsVolatile); 1422 1423 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1424 } 1425 1426 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1427 llvm::AtomicOrdering AO, bool) { 1428 // void __atomic_load(size_t size, void *mem, void *return, int order); 1429 CallArgList Args; 1430 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1431 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1432 CGF.getContext().VoidPtrTy); 1433 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), 1434 CGF.getContext().VoidPtrTy); 1435 Args.add( 1436 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1437 CGF.getContext().IntTy); 1438 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1439 } 1440 1441 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1442 bool IsVolatile) { 1443 // Okay, we're doing this natively. 1444 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1445 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1446 Load->setAtomic(AO); 1447 1448 // Other decoration. 1449 if (IsVolatile) 1450 Load->setVolatile(true); 1451 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1452 return Load; 1453 } 1454 1455 /// An LValue is a candidate for having its loads and stores be made atomic if 1456 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1457 /// performing such an operation can be performed without a libcall. 1458 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1459 if (!CGM.getCodeGenOpts().MSVolatile) return false; 1460 AtomicInfo AI(*this, LV); 1461 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1462 // An atomic is inline if we don't need to use a libcall. 1463 bool AtomicIsInline = !AI.shouldUseLibcall(); 1464 // MSVC doesn't seem to do this for types wider than a pointer. 1465 if (getContext().getTypeSize(LV.getType()) > 1466 getContext().getTypeSize(getContext().getIntPtrType())) 1467 return false; 1468 return IsVolatile && AtomicIsInline; 1469 } 1470 1471 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1472 AggValueSlot Slot) { 1473 llvm::AtomicOrdering AO; 1474 bool IsVolatile = LV.isVolatileQualified(); 1475 if (LV.getType()->isAtomicType()) { 1476 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1477 } else { 1478 AO = llvm::AtomicOrdering::Acquire; 1479 IsVolatile = true; 1480 } 1481 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1482 } 1483 1484 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1485 bool AsValue, llvm::AtomicOrdering AO, 1486 bool IsVolatile) { 1487 // Check whether we should use a library call. 1488 if (shouldUseLibcall()) { 1489 Address TempAddr = Address::invalid(); 1490 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1491 assert(getEvaluationKind() == TEK_Aggregate); 1492 TempAddr = ResultSlot.getAddress(); 1493 } else 1494 TempAddr = CreateTempAlloca(); 1495 1496 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); 1497 1498 // Okay, turn that back into the original value or whole atomic (for 1499 // non-simple lvalues) type. 1500 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1501 } 1502 1503 // Okay, we're doing this natively. 1504 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1505 1506 // If we're ignoring an aggregate return, don't do anything. 1507 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1508 return RValue::getAggregate(Address::invalid(), false); 1509 1510 // Okay, turn that back into the original value or atomic (for non-simple 1511 // lvalues) type. 1512 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1513 } 1514 1515 /// Emit a load from an l-value of atomic type. Note that the r-value 1516 /// we produce is an r-value of the atomic *value* type. 1517 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1518 llvm::AtomicOrdering AO, bool IsVolatile, 1519 AggValueSlot resultSlot) { 1520 AtomicInfo Atomics(*this, src); 1521 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1522 IsVolatile); 1523 } 1524 1525 /// Copy an r-value into memory as part of storing to an atomic type. 1526 /// This needs to create a bit-pattern suitable for atomic operations. 1527 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1528 assert(LVal.isSimple()); 1529 // If we have an r-value, the rvalue should be of the atomic type, 1530 // which means that the caller is responsible for having zeroed 1531 // any padding. Just do an aggregate copy of that type. 1532 if (rvalue.isAggregate()) { 1533 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); 1534 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), 1535 getAtomicType()); 1536 bool IsVolatile = rvalue.isVolatileQualified() || 1537 LVal.isVolatileQualified(); 1538 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), 1539 AggValueSlot::DoesNotOverlap, IsVolatile); 1540 return; 1541 } 1542 1543 // Okay, otherwise we're copying stuff. 1544 1545 // Zero out the buffer if necessary. 1546 emitMemSetZeroIfNecessary(); 1547 1548 // Drill past the padding if present. 1549 LValue TempLVal = projectValue(); 1550 1551 // Okay, store the rvalue in. 1552 if (rvalue.isScalar()) { 1553 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1554 } else { 1555 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1556 } 1557 } 1558 1559 1560 /// Materialize an r-value into memory for the purposes of storing it 1561 /// to an atomic type. 1562 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1563 // Aggregate r-values are already in memory, and EmitAtomicStore 1564 // requires them to be values of the atomic type. 1565 if (rvalue.isAggregate()) 1566 return rvalue.getAggregateAddress(); 1567 1568 // Otherwise, make a temporary and materialize into it. 1569 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1570 AtomicInfo Atomics(CGF, TempLV); 1571 Atomics.emitCopyIntoMemory(rvalue); 1572 return TempLV.getAddress(); 1573 } 1574 1575 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { 1576 // If we've got a scalar value of the right size, try to avoid going 1577 // through memory. 1578 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { 1579 llvm::Value *Value = RVal.getScalarVal(); 1580 if (isa<llvm::IntegerType>(Value->getType())) 1581 return CGF.EmitToMemory(Value, ValueTy); 1582 else { 1583 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1584 CGF.getLLVMContext(), 1585 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1586 if (isa<llvm::PointerType>(Value->getType())) 1587 return CGF.Builder.CreatePtrToInt(Value, InputIntTy); 1588 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1589 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1590 } 1591 } 1592 // Otherwise, we need to go through memory. 1593 // Put the r-value in memory. 1594 Address Addr = materializeRValue(RVal); 1595 1596 // Cast the temporary to the atomic int type and pull a value out. 1597 Addr = emitCastToAtomicIntPointer(Addr); 1598 return CGF.Builder.CreateLoad(Addr); 1599 } 1600 1601 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1602 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1603 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1604 // Do the atomic store. 1605 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1606 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), 1607 ExpectedVal, DesiredVal, 1608 Success, Failure); 1609 // Other decoration. 1610 Inst->setVolatile(LVal.isVolatileQualified()); 1611 Inst->setWeak(IsWeak); 1612 1613 // Okay, turn that back into the original value type. 1614 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1615 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1616 return std::make_pair(PreviousVal, SuccessFailureVal); 1617 } 1618 1619 llvm::Value * 1620 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1621 llvm::Value *DesiredAddr, 1622 llvm::AtomicOrdering Success, 1623 llvm::AtomicOrdering Failure) { 1624 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1625 // void *desired, int success, int failure); 1626 CallArgList Args; 1627 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1628 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1629 CGF.getContext().VoidPtrTy); 1630 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), 1631 CGF.getContext().VoidPtrTy); 1632 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), 1633 CGF.getContext().VoidPtrTy); 1634 Args.add(RValue::get( 1635 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1636 CGF.getContext().IntTy); 1637 Args.add(RValue::get( 1638 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1639 CGF.getContext().IntTy); 1640 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1641 CGF.getContext().BoolTy, Args); 1642 1643 return SuccessFailureRVal.getScalarVal(); 1644 } 1645 1646 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1647 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1648 llvm::AtomicOrdering Failure, bool IsWeak) { 1649 if (isStrongerThan(Failure, Success)) 1650 // Don't assert on undefined behavior "failure argument shall be no stronger 1651 // than the success argument". 1652 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); 1653 1654 // Check whether we should use a library call. 1655 if (shouldUseLibcall()) { 1656 // Produce a source address. 1657 Address ExpectedAddr = materializeRValue(Expected); 1658 Address DesiredAddr = materializeRValue(Desired); 1659 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1660 DesiredAddr.getPointer(), 1661 Success, Failure); 1662 return std::make_pair( 1663 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1664 SourceLocation(), /*AsValue=*/false), 1665 Res); 1666 } 1667 1668 // If we've got a scalar value of the right size, try to avoid going 1669 // through memory. 1670 auto *ExpectedVal = convertRValueToInt(Expected); 1671 auto *DesiredVal = convertRValueToInt(Desired); 1672 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1673 Failure, IsWeak); 1674 return std::make_pair( 1675 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1676 SourceLocation(), /*AsValue=*/false), 1677 Res.second); 1678 } 1679 1680 static void 1681 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1682 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1683 Address DesiredAddr) { 1684 RValue UpRVal; 1685 LValue AtomicLVal = Atomics.getAtomicLValue(); 1686 LValue DesiredLVal; 1687 if (AtomicLVal.isSimple()) { 1688 UpRVal = OldRVal; 1689 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1690 } else { 1691 // Build new lvalue for temp address. 1692 Address Ptr = Atomics.materializeRValue(OldRVal); 1693 LValue UpdateLVal; 1694 if (AtomicLVal.isBitField()) { 1695 UpdateLVal = 1696 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1697 AtomicLVal.getType(), 1698 AtomicLVal.getBaseInfo(), 1699 AtomicLVal.getTBAAInfo()); 1700 DesiredLVal = 1701 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1702 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1703 AtomicLVal.getTBAAInfo()); 1704 } else if (AtomicLVal.isVectorElt()) { 1705 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1706 AtomicLVal.getType(), 1707 AtomicLVal.getBaseInfo(), 1708 AtomicLVal.getTBAAInfo()); 1709 DesiredLVal = LValue::MakeVectorElt( 1710 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1711 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1712 } else { 1713 assert(AtomicLVal.isExtVectorElt()); 1714 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1715 AtomicLVal.getType(), 1716 AtomicLVal.getBaseInfo(), 1717 AtomicLVal.getTBAAInfo()); 1718 DesiredLVal = LValue::MakeExtVectorElt( 1719 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1720 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1721 } 1722 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1723 } 1724 // Store new value in the corresponding memory area. 1725 RValue NewRVal = UpdateOp(UpRVal); 1726 if (NewRVal.isScalar()) { 1727 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1728 } else { 1729 assert(NewRVal.isComplex()); 1730 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1731 /*isInit=*/false); 1732 } 1733 } 1734 1735 void AtomicInfo::EmitAtomicUpdateLibcall( 1736 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1737 bool IsVolatile) { 1738 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1739 1740 Address ExpectedAddr = CreateTempAlloca(); 1741 1742 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1743 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1744 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1745 CGF.EmitBlock(ContBB); 1746 Address DesiredAddr = CreateTempAlloca(); 1747 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1748 requiresMemSetZero(getAtomicAddress().getElementType())) { 1749 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1750 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1751 } 1752 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1753 AggValueSlot::ignored(), 1754 SourceLocation(), /*AsValue=*/false); 1755 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1756 auto *Res = 1757 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1758 DesiredAddr.getPointer(), 1759 AO, Failure); 1760 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1761 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1762 } 1763 1764 void AtomicInfo::EmitAtomicUpdateOp( 1765 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1766 bool IsVolatile) { 1767 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1768 1769 // Do the atomic load. 1770 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1771 // For non-simple lvalues perform compare-and-swap procedure. 1772 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1773 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1774 auto *CurBB = CGF.Builder.GetInsertBlock(); 1775 CGF.EmitBlock(ContBB); 1776 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1777 /*NumReservedValues=*/2); 1778 PHI->addIncoming(OldVal, CurBB); 1779 Address NewAtomicAddr = CreateTempAlloca(); 1780 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1781 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1782 requiresMemSetZero(getAtomicAddress().getElementType())) { 1783 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1784 } 1785 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), 1786 SourceLocation(), /*AsValue=*/false); 1787 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1788 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1789 // Try to write new value using cmpxchg operation. 1790 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1791 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1792 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1793 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1794 } 1795 1796 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1797 RValue UpdateRVal, Address DesiredAddr) { 1798 LValue AtomicLVal = Atomics.getAtomicLValue(); 1799 LValue DesiredLVal; 1800 // Build new lvalue for temp address. 1801 if (AtomicLVal.isBitField()) { 1802 DesiredLVal = 1803 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1804 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1805 AtomicLVal.getTBAAInfo()); 1806 } else if (AtomicLVal.isVectorElt()) { 1807 DesiredLVal = 1808 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1809 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1810 AtomicLVal.getTBAAInfo()); 1811 } else { 1812 assert(AtomicLVal.isExtVectorElt()); 1813 DesiredLVal = LValue::MakeExtVectorElt( 1814 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1815 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1816 } 1817 // Store new value in the corresponding memory area. 1818 assert(UpdateRVal.isScalar()); 1819 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1820 } 1821 1822 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1823 RValue UpdateRVal, bool IsVolatile) { 1824 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1825 1826 Address ExpectedAddr = CreateTempAlloca(); 1827 1828 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1829 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1830 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1831 CGF.EmitBlock(ContBB); 1832 Address DesiredAddr = CreateTempAlloca(); 1833 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1834 requiresMemSetZero(getAtomicAddress().getElementType())) { 1835 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1836 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1837 } 1838 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1839 auto *Res = 1840 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1841 DesiredAddr.getPointer(), 1842 AO, Failure); 1843 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1844 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1845 } 1846 1847 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1848 bool IsVolatile) { 1849 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1850 1851 // Do the atomic load. 1852 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1853 // For non-simple lvalues perform compare-and-swap procedure. 1854 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1855 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1856 auto *CurBB = CGF.Builder.GetInsertBlock(); 1857 CGF.EmitBlock(ContBB); 1858 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1859 /*NumReservedValues=*/2); 1860 PHI->addIncoming(OldVal, CurBB); 1861 Address NewAtomicAddr = CreateTempAlloca(); 1862 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1863 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1864 requiresMemSetZero(getAtomicAddress().getElementType())) { 1865 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1866 } 1867 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 1868 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1869 // Try to write new value using cmpxchg operation. 1870 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1871 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1872 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1873 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1874 } 1875 1876 void AtomicInfo::EmitAtomicUpdate( 1877 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1878 bool IsVolatile) { 1879 if (shouldUseLibcall()) { 1880 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 1881 } else { 1882 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 1883 } 1884 } 1885 1886 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 1887 bool IsVolatile) { 1888 if (shouldUseLibcall()) { 1889 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 1890 } else { 1891 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 1892 } 1893 } 1894 1895 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 1896 bool isInit) { 1897 bool IsVolatile = lvalue.isVolatileQualified(); 1898 llvm::AtomicOrdering AO; 1899 if (lvalue.getType()->isAtomicType()) { 1900 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1901 } else { 1902 AO = llvm::AtomicOrdering::Release; 1903 IsVolatile = true; 1904 } 1905 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 1906 } 1907 1908 /// Emit a store to an l-value of atomic type. 1909 /// 1910 /// Note that the r-value is expected to be an r-value *of the atomic 1911 /// type*; this means that for aggregate r-values, it should include 1912 /// storage for any padding that was necessary. 1913 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 1914 llvm::AtomicOrdering AO, bool IsVolatile, 1915 bool isInit) { 1916 // If this is an aggregate r-value, it should agree in type except 1917 // maybe for address-space qualification. 1918 assert(!rvalue.isAggregate() || 1919 rvalue.getAggregateAddress().getElementType() 1920 == dest.getAddress().getElementType()); 1921 1922 AtomicInfo atomics(*this, dest); 1923 LValue LVal = atomics.getAtomicLValue(); 1924 1925 // If this is an initialization, just put the value there normally. 1926 if (LVal.isSimple()) { 1927 if (isInit) { 1928 atomics.emitCopyIntoMemory(rvalue); 1929 return; 1930 } 1931 1932 // Check whether we should use a library call. 1933 if (atomics.shouldUseLibcall()) { 1934 // Produce a source address. 1935 Address srcAddr = atomics.materializeRValue(rvalue); 1936 1937 // void __atomic_store(size_t size, void *mem, void *val, int order) 1938 CallArgList args; 1939 args.add(RValue::get(atomics.getAtomicSizeValue()), 1940 getContext().getSizeType()); 1941 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), 1942 getContext().VoidPtrTy); 1943 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), 1944 getContext().VoidPtrTy); 1945 args.add( 1946 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 1947 getContext().IntTy); 1948 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 1949 return; 1950 } 1951 1952 // Okay, we're doing this natively. 1953 llvm::Value *intValue = atomics.convertRValueToInt(rvalue); 1954 1955 // Do the atomic store. 1956 Address addr = 1957 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); 1958 intValue = Builder.CreateIntCast( 1959 intValue, addr.getElementType(), /*isSigned=*/false); 1960 llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 1961 1962 // Initializations don't need to be atomic. 1963 if (!isInit) 1964 store->setAtomic(AO); 1965 1966 // Other decoration. 1967 if (IsVolatile) 1968 store->setVolatile(true); 1969 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 1970 return; 1971 } 1972 1973 // Emit simple atomic update operation. 1974 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 1975 } 1976 1977 /// Emit a compare-and-exchange op for atomic type. 1978 /// 1979 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 1980 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 1981 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 1982 AggValueSlot Slot) { 1983 // If this is an aggregate r-value, it should agree in type except 1984 // maybe for address-space qualification. 1985 assert(!Expected.isAggregate() || 1986 Expected.getAggregateAddress().getElementType() == 1987 Obj.getAddress().getElementType()); 1988 assert(!Desired.isAggregate() || 1989 Desired.getAggregateAddress().getElementType() == 1990 Obj.getAddress().getElementType()); 1991 AtomicInfo Atomics(*this, Obj); 1992 1993 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 1994 IsWeak); 1995 } 1996 1997 void CodeGenFunction::EmitAtomicUpdate( 1998 LValue LVal, llvm::AtomicOrdering AO, 1999 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 2000 AtomicInfo Atomics(*this, LVal); 2001 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 2002 } 2003 2004 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 2005 AtomicInfo atomics(*this, dest); 2006 2007 switch (atomics.getEvaluationKind()) { 2008 case TEK_Scalar: { 2009 llvm::Value *value = EmitScalarExpr(init); 2010 atomics.emitCopyIntoMemory(RValue::get(value)); 2011 return; 2012 } 2013 2014 case TEK_Complex: { 2015 ComplexPairTy value = EmitComplexExpr(init); 2016 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 2017 return; 2018 } 2019 2020 case TEK_Aggregate: { 2021 // Fix up the destination if the initializer isn't an expression 2022 // of atomic type. 2023 bool Zeroed = false; 2024 if (!init->getType()->isAtomicType()) { 2025 Zeroed = atomics.emitMemSetZeroIfNecessary(); 2026 dest = atomics.projectValue(); 2027 } 2028 2029 // Evaluate the expression directly into the destination. 2030 AggValueSlot slot = AggValueSlot::forLValue(dest, 2031 AggValueSlot::IsNotDestructed, 2032 AggValueSlot::DoesNotNeedGCBarriers, 2033 AggValueSlot::IsNotAliased, 2034 AggValueSlot::DoesNotOverlap, 2035 Zeroed ? AggValueSlot::IsZeroed : 2036 AggValueSlot::IsNotZeroed); 2037 2038 EmitAggExpr(init, slot); 2039 return; 2040 } 2041 } 2042 llvm_unreachable("bad evaluation kind"); 2043 } 2044