Lines Matching refs:AI
79 bool tryExpandAtomicRMW(AtomicRMWInst *AI);
91 AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
93 void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
119 llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
565 AtomicRMWInst *AI = Builder.CreateAtomicRMW( in expandAtomicStore() local
571 tryExpandAtomicRMW(AI); in expandAtomicStore()
599 bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) { in tryExpandAtomicRMW() argument
600 LLVMContext &Ctx = AI->getModule()->getContext(); in tryExpandAtomicRMW()
601 TargetLowering::AtomicExpansionKind Kind = TLI->shouldExpandAtomicRMWInIR(AI); in tryExpandAtomicRMW()
607 unsigned ValueSize = getAtomicOpSize(AI); in tryExpandAtomicRMW()
609 expandPartwordAtomicRMW(AI, in tryExpandAtomicRMW()
613 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded, in tryExpandAtomicRMW()
614 AI->getValOperand()); in tryExpandAtomicRMW()
616 expandAtomicOpToLLSC(AI, AI->getType(), AI->getPointerOperand(), in tryExpandAtomicRMW()
617 AI->getAlign(), AI->getOrdering(), PerformOp); in tryExpandAtomicRMW()
623 unsigned ValueSize = getAtomicOpSize(AI); in tryExpandAtomicRMW()
625 expandPartwordAtomicRMW(AI, in tryExpandAtomicRMW()
630 auto MemScope = SSNs[AI->getSyncScopeID()].empty() in tryExpandAtomicRMW()
632 : SSNs[AI->getSyncScopeID()]; in tryExpandAtomicRMW()
633 OptimizationRemarkEmitter ORE(AI->getFunction()); in tryExpandAtomicRMW()
635 return OptimizationRemark(DEBUG_TYPE, "Passed", AI) in tryExpandAtomicRMW()
637 << AI->getOperationName(AI->getOperation()) << " operation at " in tryExpandAtomicRMW()
640 expandAtomicRMWToCmpXchg(AI, createCmpXchgInstFun); in tryExpandAtomicRMW()
646 unsigned ValueSize = getAtomicOpSize(AI); in tryExpandAtomicRMW()
648 AtomicRMWInst::BinOp Op = AI->getOperation(); in tryExpandAtomicRMW()
652 tryExpandAtomicRMW(widenPartwordAtomicRMW(AI)); in tryExpandAtomicRMW()
656 expandAtomicRMWToMaskedIntrinsic(AI); in tryExpandAtomicRMW()
660 TLI->emitBitTestAtomicRMWIntrinsic(AI); in tryExpandAtomicRMW()
664 TLI->emitCmpArithAtomicRMWIntrinsic(AI); in tryExpandAtomicRMW()
668 return lowerAtomicRMWInst(AI); in tryExpandAtomicRMW()
670 TLI->emitExpandAtomicRMW(AI); in tryExpandAtomicRMW()
893 AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) { in expandPartwordAtomicRMW() argument
895 AtomicRMWInst::BinOp Op = AI->getOperation(); in expandPartwordAtomicRMW()
898 tryExpandAtomicRMW(widenPartwordAtomicRMW(AI)); in expandPartwordAtomicRMW()
901 AtomicOrdering MemOpOrder = AI->getOrdering(); in expandPartwordAtomicRMW()
902 SyncScope::ID SSID = AI->getSyncScopeID(); in expandPartwordAtomicRMW()
904 ReplacementIRBuilder Builder(AI, *DL); in expandPartwordAtomicRMW()
907 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(), in expandPartwordAtomicRMW()
908 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); in expandPartwordAtomicRMW()
913 Value *ValOp = Builder.CreateBitCast(AI->getValOperand(), PMV.IntValueType); in expandPartwordAtomicRMW()
921 AI->getValOperand(), PMV); in expandPartwordAtomicRMW()
937 AI->replaceAllUsesWith(FinalOldResult); in expandPartwordAtomicRMW()
938 AI->eraseFromParent(); in expandPartwordAtomicRMW()
972 AtomicRMWInst *AtomicExpandImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) { in widenPartwordAtomicRMW() argument
973 ReplacementIRBuilder Builder(AI, *DL); in widenPartwordAtomicRMW()
974 AtomicRMWInst::BinOp Op = AI->getOperation(); in widenPartwordAtomicRMW()
981 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(), in widenPartwordAtomicRMW()
982 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); in widenPartwordAtomicRMW()
985 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), in widenPartwordAtomicRMW()
998 AI->getOrdering(), AI->getSyncScopeID()); in widenPartwordAtomicRMW()
1000 copyMetadataForAtomic(*NewAI, *AI); in widenPartwordAtomicRMW()
1003 AI->replaceAllUsesWith(FinalOldResult); in widenPartwordAtomicRMW()
1004 AI->eraseFromParent(); in widenPartwordAtomicRMW()
1145 void AtomicExpandImpl::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) { in expandAtomicRMWToMaskedIntrinsic() argument
1146 ReplacementIRBuilder Builder(AI, *DL); in expandAtomicRMWToMaskedIntrinsic()
1149 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(), in expandAtomicRMWToMaskedIntrinsic()
1150 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8); in expandAtomicRMWToMaskedIntrinsic()
1156 AtomicRMWInst::BinOp RMWOp = AI->getOperation(); in expandAtomicRMWToMaskedIntrinsic()
1161 Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType), in expandAtomicRMWToMaskedIntrinsic()
1164 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt, in expandAtomicRMWToMaskedIntrinsic()
1165 AI->getOrdering()); in expandAtomicRMWToMaskedIntrinsic()
1167 AI->replaceAllUsesWith(FinalOldResult); in expandAtomicRMWToMaskedIntrinsic()
1168 AI->eraseFromParent(); in expandAtomicRMWToMaskedIntrinsic()
1646 bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, in expandAtomicRMWToCmpXchg() argument
1648 ReplacementIRBuilder Builder(AI, AI->getDataLayout()); in expandAtomicRMWToCmpXchg()
1650 AI->getFunction()->hasFnAttribute(Attribute::StrictFP)); in expandAtomicRMWToCmpXchg()
1655 Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(), in expandAtomicRMWToCmpXchg()
1656 AI->getOrdering(), AI->getSyncScopeID(), in expandAtomicRMWToCmpXchg()
1658 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded, in expandAtomicRMWToCmpXchg()
1659 AI->getValOperand()); in expandAtomicRMWToCmpXchg()
1663 AI->replaceAllUsesWith(Loaded); in expandAtomicRMWToCmpXchg()
1664 AI->eraseFromParent(); in expandAtomicRMWToCmpXchg()