| /freebsd/contrib/llvm-project/llvm/lib/Transforms/InstCombine/ |
| H A D | InstCombineAtomicRMW.cpp | 24 bool isIdempotentRMW(AtomicRMWInst& RMWI) { in isIdempotentRMW()
27 case AtomicRMWInst::FAdd: // -0.0 in isIdempotentRMW()
29 case AtomicRMWInst::FSub: // +0.0 in isIdempotentRMW()
40 case AtomicRMWInst::Add: in isIdempotentRMW()
41 case AtomicRMWInst::Sub: in isIdempotentRMW()
42 case AtomicRMWInst::Or: in isIdempotentRMW()
43 case AtomicRMWInst::Xor: in isIdempotentRMW()
45 case AtomicRMWInst::And: in isIdempotentRMW()
47 case AtomicRMWInst::Min: in isIdempotentRMW()
49 case AtomicRMWInst::Max: in isIdempotentRMW()
[all …]
|
| /freebsd/contrib/llvm-project/llvm/lib/Target/AMDGPU/ |
| H A D | AMDGPUAtomicOptimizer.cpp | 45 AtomicRMWInst::BinOp Op;
77 Value *buildReduction(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
79 Value *buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
84 buildScanIteratively(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
88 void optimizeAtomic(Instruction &I, AtomicRMWInst::BinOp Op, unsigned ValIdx,
102 void visitAtomicRMWInst(AtomicRMWInst &I);
195 void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &I) { in visitAtomicRMWInst()
205 AtomicRMWInst::BinOp Op = I.getOperation(); in visitAtomicRMWInst()
210 case AtomicRMWInst::Add: in visitAtomicRMWInst()
211 case AtomicRMWInst::Sub: in visitAtomicRMWInst()
[all …]
|
| H A D | AMDGPULowerBufferFatPointers.cpp | 750 PtrParts visitAtomicRMWInst(AtomicRMWInst &AI);
1082 !isa<AtomicRMWInst>(I) && Order != AtomicOrdering::NotAtomic; in handleMemoryInst()
1100 else if (auto *RMW = dyn_cast<AtomicRMWInst>(I)) { in handleMemoryInst()
1102 case AtomicRMWInst::Xchg: in handleMemoryInst()
1105 case AtomicRMWInst::Add: in handleMemoryInst()
1108 case AtomicRMWInst::Sub: in handleMemoryInst()
1111 case AtomicRMWInst::And: in handleMemoryInst()
1114 case AtomicRMWInst::Or: in handleMemoryInst()
1117 case AtomicRMWInst::Xor: in handleMemoryInst()
1120 case AtomicRMWInst::Max: in handleMemoryInst()
[all …]
|
| /freebsd/contrib/llvm-project/llvm/lib/Transforms/Utils/ |
| H A D | LowerAtomic.cpp | 41 Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op, in buildAtomicRMWValue()
46 case AtomicRMWInst::Xchg: in buildAtomicRMWValue()
48 case AtomicRMWInst::Add: in buildAtomicRMWValue()
50 case AtomicRMWInst::Sub: in buildAtomicRMWValue()
52 case AtomicRMWInst::And: in buildAtomicRMWValue()
54 case AtomicRMWInst::Nand: in buildAtomicRMWValue()
56 case AtomicRMWInst::Or: in buildAtomicRMWValue()
58 case AtomicRMWInst::Xor: in buildAtomicRMWValue()
60 case AtomicRMWInst::Max: in buildAtomicRMWValue()
63 case AtomicRMWInst::Min: in buildAtomicRMWValue()
[all …]
|
| H A D | FunctionComparator.cpp | 759 if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(L)) { in cmpOperations()
761 cast<AtomicRMWInst>(R)->getOperation())) in cmpOperations()
764 cast<AtomicRMWInst>(R)->isVolatile())) in cmpOperations()
767 cast<AtomicRMWInst>(R)->getOrdering())) in cmpOperations()
770 cast<AtomicRMWInst>(R)->getSyncScopeID()); in cmpOperations()
|
| /freebsd/contrib/llvm-project/llvm/lib/Target/LoongArch/ |
| H A D | LoongArchExpandAtomicPseudoInsts.cpp | 51 MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
56 AtomicRMWInst::BinOp, bool IsMasked, int Width,
92 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32, in expandMI()
95 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 32, in expandMI()
98 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI); in expandMI()
100 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI); in expandMI()
102 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32, in expandMI()
105 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64, in expandMI()
108 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32, in expandMI()
111 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst in expandMI()
[all...] |
| H A D | LoongArchISelLowering.h | 189 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
191 Value *emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI,
|
| /freebsd/contrib/llvm-project/llvm/lib/CodeGen/ |
| H A D | AtomicExpandPass.cpp | 79 bool tryExpandAtomicRMW(AtomicRMWInst *AI);
80 AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI);
90 AtomicRMWInst *I, TargetLoweringBase::AtomicExpansionKind ExpansionKind);
91 AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
93 void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
105 bool isIdempotentRMW(AtomicRMWInst *RMWI);
106 bool simplifyIdempotentRMW(AtomicRMWInst *RMWI);
115 void expandAtomicRMWToLibcall(AtomicRMWInst *I);
119 llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
185 static unsigned getAtomicOpSize(AtomicRMWInst *RMWI) { in getAtomicOpSize()
[all …]
|
| /freebsd/contrib/llvm-project/llvm/lib/Target/RISCV/ |
| H A D | RISCVExpandAtomicPseudoInsts.cpp | 52 MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
57 AtomicRMWInst::BinOp, bool IsMasked, int Width,
115 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32, in expandMI()
118 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64, in expandMI()
121 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32, in expandMI()
124 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI); in expandMI()
126 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI); in expandMI()
128 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32, in expandMI()
131 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32, in expandMI()
134 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst in expandMI()
[all...] |
| /freebsd/contrib/llvm-project/llvm/lib/Target/NVPTX/ |
| H A D | NVPTXAtomicLower.cpp | 46 SmallVector<AtomicRMWInst *> LocalMemoryAtomics; in runOnFunction()
48 if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(&I)) in runOnFunction()
53 for (AtomicRMWInst *RMWI : LocalMemoryAtomics) in runOnFunction()
|
| /freebsd/contrib/llvm-project/llvm/include/llvm/Transforms/Utils/ |
| H A D | LowerAtomic.h | 29 bool lowerAtomicRMWInst(AtomicRMWInst *RMWI);
33 Value *buildAtomicRMWValue(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder,
|
| /freebsd/contrib/llvm-project/llvm/include/llvm/CodeGen/ |
| H A D | AtomicExpandUtils.h | 18 class AtomicRMWInst; variable
60 bool expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg);
|
| /freebsd/contrib/llvm-project/llvm/lib/Transforms/Instrumentation/ |
| H A D | ThreadSanitizer.cpp | 164 FunctionCallee TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1]
282 for (unsigned Op = AtomicRMWInst::FIRST_BINOP; in initialize()
283 Op <= AtomicRMWInst::LAST_BINOP; ++Op) { in initialize()
286 if (Op == AtomicRMWInst::Xchg) in initialize()
288 else if (Op == AtomicRMWInst::Add) in initialize()
290 else if (Op == AtomicRMWInst::Sub) in initialize()
292 else if (Op == AtomicRMWInst::And) in initialize()
294 else if (Op == AtomicRMWInst::Or) in initialize()
296 else if (Op == AtomicRMWInst::Xor) in initialize()
298 else if (Op == AtomicRMWInst::Nand) in initialize()
[all …]
|
| H A D | BoundsChecking.cpp | 172 } else if (AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) { in addBoundsChecking()
|
| /freebsd/contrib/llvm-project/llvm/lib/Transforms/Scalar/ |
| H A D | LowerAtomicPass.cpp | 47 else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(&Inst)) in runOnBasicBlock()
|
| /freebsd/contrib/llvm-project/llvm/lib/IR/ |
| H A D | Core.cpp | 2120 if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(P)) in LLVMGetAlignment()
2140 else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(P)) in LLVMSetAlignment()
3897 static AtomicRMWInst::BinOp mapFromLLVMRMWBinOp(LLVMAtomicRMWBinOp BinOp) { in mapFromLLVMRMWBinOp()
3899 case LLVMAtomicRMWBinOpXchg: return AtomicRMWInst::Xchg; in mapFromLLVMRMWBinOp()
3900 case LLVMAtomicRMWBinOpAdd: return AtomicRMWInst::Add; in mapFromLLVMRMWBinOp()
3901 case LLVMAtomicRMWBinOpSub: return AtomicRMWInst::Sub; in mapFromLLVMRMWBinOp()
3902 case LLVMAtomicRMWBinOpAnd: return AtomicRMWInst::And; in mapFromLLVMRMWBinOp()
3903 case LLVMAtomicRMWBinOpNand: return AtomicRMWInst::Nand; in mapFromLLVMRMWBinOp()
3904 case LLVMAtomicRMWBinOpOr: return AtomicRMWInst::Or; in mapFromLLVMRMWBinOp()
3905 case LLVMAtomicRMWBinOpXor: return AtomicRMWInst::Xor; in mapFromLLVMRMWBinOp()
[all …]
|
| H A D | Instructions.cpp | 1368 void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val, in Init()
1389 AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, in AtomicRMWInst() function in AtomicRMWInst
1393 OperandTraits<AtomicRMWInst>::op_begin(this), in AtomicRMWInst()
1394 OperandTraits<AtomicRMWInst>::operands(this), InsertBefore) { in AtomicRMWInst()
1398 StringRef AtomicRMWInst::getOperationName(BinOp Op) { in getOperationName()
1400 case AtomicRMWInst::Xchg: in getOperationName()
1402 case AtomicRMWInst::Add: in getOperationName()
1404 case AtomicRMWInst::Sub: in getOperationName()
1406 case AtomicRMWInst::And: in getOperationName()
1408 case AtomicRMWInst::Nand: in getOperationName()
[all …]
|
| H A D | Instruction.cpp | 840 if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I1)) in hasSameSpecialState()
841 return RMWI->getOperation() == cast<AtomicRMWInst>(I2)->getOperation() && in hasSameSpecialState()
842 RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() && in hasSameSpecialState()
843 RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() && in hasSameSpecialState()
844 RMWI->getSyncScopeID() == cast<AtomicRMWInst>(I2)->getSyncScopeID(); in hasSameSpecialState()
1013 return cast<AtomicRMWInst>(this)->isVolatile(); in isVolatile()
|
| /freebsd/contrib/llvm-project/clang/lib/CodeGen/ |
| H A D | CGAtomic.cpp | 530 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; in EmitAtomicOp()
618 Op = llvm::AtomicRMWInst::Xchg; in EmitAtomicOp()
631 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd in EmitAtomicOp()
632 : llvm::AtomicRMWInst::Add; in EmitAtomicOp()
645 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub in EmitAtomicOp()
646 : llvm::AtomicRMWInst::Sub; in EmitAtomicOp()
659 ? llvm::AtomicRMWInst::FMin in EmitAtomicOp()
661 ? llvm::AtomicRMWInst::Min in EmitAtomicOp()
662 : llvm::AtomicRMWInst::UMin); in EmitAtomicOp()
675 ? llvm::AtomicRMWInst::FMax in EmitAtomicOp()
[all …]
|
| /freebsd/contrib/llvm-project/llvm/lib/Frontend/OpenMP/ |
| H A D | OMPIRBuilder.cpp | 7795 Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, in createAtomicUpdate()
7809 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) && in createAtomicUpdate()
7810 (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) && in createAtomicUpdate()
7822 AtomicRMWInst::BinOp RMWOp) { in emitRMWOpAsInstruction()
7824 case AtomicRMWInst::Add: in emitRMWOpAsInstruction()
7826 case AtomicRMWInst::Sub: in emitRMWOpAsInstruction()
7828 case AtomicRMWInst::And: in emitRMWOpAsInstruction()
7830 case AtomicRMWInst::Nand: in emitRMWOpAsInstruction()
7832 case AtomicRMWInst::Or: in emitRMWOpAsInstruction()
7834 case AtomicRMWInst::Xor: in emitRMWOpAsInstruction()
[all …]
|
| /freebsd/contrib/llvm-project/llvm/lib/Target/DirectX/DXILWriter/ |
| H A D | DXILBitcodeWriter.cpp | 165 static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op);
542 unsigned DXILBitcodeWriter::getEncodedRMWOperation(AtomicRMWInst::BinOp Op) { in getEncodedRMWOperation()
546 case AtomicRMWInst::Xchg: in getEncodedRMWOperation()
548 case AtomicRMWInst::Add: in getEncodedRMWOperation()
550 case AtomicRMWInst::Sub: in getEncodedRMWOperation()
552 case AtomicRMWInst::And: in getEncodedRMWOperation()
554 case AtomicRMWInst::Nand: in getEncodedRMWOperation()
556 case AtomicRMWInst::Or: in getEncodedRMWOperation()
558 case AtomicRMWInst::Xor: in getEncodedRMWOperation()
560 case AtomicRMWInst::Max: in getEncodedRMWOperation()
[all …]
|
| /freebsd/contrib/llvm-project/llvm/include/llvm/Analysis/ |
| H A D | MemoryLocation.h | 35 class AtomicRMWInst; variable
259 static MemoryLocation get(const AtomicRMWInst *RMWI);
|
| /freebsd/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/ |
| H A D | IRTranslator.cpp | 280 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) in getMemOpAlign()
3242 const AtomicRMWInst &I = cast<AtomicRMWInst>(U); in translateAtomicRMW()
3253 case AtomicRMWInst::Xchg: in translateAtomicRMW()
3256 case AtomicRMWInst::Add: in translateAtomicRMW()
3259 case AtomicRMWInst::Sub: in translateAtomicRMW()
3262 case AtomicRMWInst::And: in translateAtomicRMW()
3265 case AtomicRMWInst::Nand: in translateAtomicRMW()
3268 case AtomicRMWInst::Or: in translateAtomicRMW()
3271 case AtomicRMWInst::Xor: in translateAtomicRMW()
3274 case AtomicRMWInst::Max: in translateAtomicRMW()
[all …]
|
| /freebsd/contrib/llvm-project/llvm/lib/Analysis/ |
| H A D | MemoryLocation.cpp | 68 MemoryLocation MemoryLocation::get(const AtomicRMWInst *RMWI) { in get()
89 return get(cast<AtomicRMWInst>(Inst)); in getOrNone()
|
| /freebsd/contrib/llvm-project/llvm/lib/Target/X86/ |
| H A D | X86ISelLowering.h | 1749 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
1751 shouldExpandLogicAtomicRMWInIR(AtomicRMWInst *AI) const;
1752 void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
1753 void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
1756 lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;
|