| /freebsd/contrib/llvm-project/llvm/lib/Transforms/Vectorize/ |
| H A D | VPlanValue.h | 44 class VPValue {
66 VPValue(const unsigned char SC, Value *UV = nullptr, VPDef *Def = nullptr);
88 VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV, nullptr) {} in VPValue() function
90 VPValue(VPDef *Def, Value *UV = nullptr) : VPValue(VPVRecipeSC, UV, Def) {} in VPValue() function
92 VPValue(Value *UV, VPDef *Def) : VPValue(VPValueSC, UV, Def) {} in VPValue() function
93 VPValue(const VPValue &) = delete;
94 VPValue &operator=(const VPValue &) = delete;
96 virtual ~VPValue();
149 void replaceAllUsesWith(VPValue *New);
155 VPValue *New,
[all …]
|
| H A D | VPlan.h | 275 DenseMap<VPValue *, PerPartValuesTy> PerPartOutput;
278 DenseMap<VPValue *, ScalarsPerPartValuesTy> PerPartScalars;
284 Value *get(VPValue *Def, unsigned Part, bool IsScalar = false);
287 Value *get(VPValue *Def, const VPIteration &Instance);
289 bool hasVectorValue(VPValue *Def, unsigned Part) { in hasVectorValue()
295 bool hasScalarValue(VPValue *Def, VPIteration Instance) { in hasScalarValue()
307 void set(VPValue *Def, Value *V, unsigned Part, bool IsScalar = false) {
322 void reset(VPValue *Def, Value *V, unsigned Part) { in reset()
330 void set(VPValue *Def, Value *V, const VPIteration &Instance) { in set()
344 void reset(VPValue *Def, Value *V, const VPIteration &Instance) { in reset()
[all …]
|
| H A D | VPlanSLP.cpp | 49 void VPlanSlp::addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New) { in addCombined()
50 if (all_of(Operands, [](VPValue *V) { in addCombined()
54 for (VPValue *V : Operands) { in addCombined()
68 bool VPlanSlp::areVectorizable(ArrayRef<VPValue *> Operands) const { in areVectorizable()
70 if (!all_of(Operands, [](VPValue *Op) { in areVectorizable()
86 if (!all_of(Operands, [Opcode, Width](VPValue *Op) { in areVectorizable()
96 if (any_of(Operands, [this](VPValue *Op) { in areVectorizable()
104 [](VPValue *Op) { return Op->hasMoreThanOneUniqueUser(); })) { in areVectorizable()
133 if (!all_of(Operands, [](VPValue *Op) { in areVectorizable()
143 if (!all_of(Operands, [](VPValue *Op) { in areVectorizable()
[all …]
|
| H A D | VPRecipeBuilder.h | 50 DenseMap<std::pair<BasicBlock *, BasicBlock *>, VPValue *>;
51 using BlockMaskCacheTy = DenseMap<BasicBlock *, VPValue *>;
73 ArrayRef<VPValue *> Operands,
79 ArrayRef<VPValue *> Operands,
85 tryToOptimizeInductionTruncate(TruncInst *I, ArrayRef<VPValue *> Operands,
91 VPBlendRecipe *tryToBlend(PHINode *Phi, ArrayRef<VPValue *> Operands);
96 VPWidenCallRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands,
102 VPWidenRecipe *tryToWiden(Instruction *I, ArrayRef<VPValue *> Operands,
116 ArrayRef<VPValue *> Operands,
135 VPValue *getBlockInMask(BasicBlock *BB) const;
[all …]
|
| H A D | LoopVectorizationPlanner.h | 57 ArrayRef<VPValue *> Operands, DebugLoc DL,
63 std::initializer_list<VPValue *> Operands,
65 return createInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name);
142 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
152 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
158 std::initializer_list<VPValue *> Operands,
164 VPValue *createNot(VPValue *Operand, DebugLoc DL = {},
169 VPValue *createAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
174 VPValue *createOr(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
182 VPValue *createLogicalAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
[all …]
|
| H A D | VPlanTransforms.cpp | 48 VPValue *VPV = Ingredient.getVPSingleValue(); in VPInstructionsToVPRecipes()
58 VPValue *Start = Plan->getOrAddLiveIn(II->getStartValue()); in VPInstructionsToVPRecipes()
59 VPValue *Step = in VPInstructionsToVPRecipes()
118 for (VPValue *Op : Recipe.operands()) in sinkScalarOperands()
173 for (VPValue *Op : SinkCandidate->operands()) in sinkScalarOperands()
184 VPValue *getPredicatedMask(VPRegionBlock *R) { in getPredicatedMask()
237 VPValue *Mask1 = getPredicatedMask(Region1); in mergeReplicateRegionsIntoSuccessors()
238 VPValue *Mask2 = getPredicatedMask(Region2); in mergeReplicateRegionsIntoSuccessors()
273 VPValue *PredInst1 = in mergeReplicateRegionsIntoSuccessors()
275 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue(); in mergeReplicateRegionsIntoSuccessors()
[all …]
|
| H A D | VPlan.cpp | 61 raw_ostream &llvm::operator<<(raw_ostream &OS, const VPValue &V) { in operator <<()
83 VPValue::VPValue(const unsigned char SC, Value *UV, VPDef *Def) in VPValue() function in VPValue
89 VPValue::~VPValue() { in ~VPValue()
96 void VPValue::print(raw_ostream &OS, VPSlotTracker &SlotTracker) const { in print()
103 void VPValue::dump() const { in dump()
120 VPRecipeBase *VPValue::getDefiningRecipe() { in getDefiningRecipe()
124 const VPRecipeBase *VPValue::getDefiningRecipe() const { in getDefiningRecipe()
227 Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) { in get()
254 Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) { in get()
401 void VPTransformState::packScalarIntoVectorValue(VPValue *Def, in packScalarIntoVectorValue()
[all …]
|
| H A D | VPlanPatternMatch.h | 37 inline class_match<VPValue> m_VPValue() { return class_match<VPValue>(); } in m_VPValue()
61 bool match(VPValue *VPV) { in match()
107 inline bind_ty<VPValue> m_VPValue(VPValue *&V) { return V; } in m_VPValue()
137 bool match(const VPValue *V) {
168 bool match(const VPValue *V) {
308 bool match(const VPValue *V) {
326 bool match(const VPValue *V) {
|
| H A D | VPlanHCFGBuilder.cpp | 54 DenseMap<Value *, VPValue *> IRDef2VPValue;
70 VPValue *getOrCreateVPOperand(Value *IRVal);
126 VPValue *VPVal = IRDef2VPValue[Phi]; in fixPhiNodes()
257 VPValue *PlainCFGBuilder::getOrCreateVPOperand(Value *IRVal) { in getOrCreateVPOperand()
275 VPValue *NewVPVal = Plan.getOrAddLiveIn(IRVal); in getOrCreateVPOperand()
298 VPValue *Cond = getOrCreateVPOperand(Br->getCondition()); in createVPInstructionsForVPBB()
306 VPValue *NewVPV; in createVPInstructionsForVPBB()
317 SmallVector<VPValue *, 4> VPOperands; in createVPInstructionsForVPBB()
|
| H A D | VPlanAnalysis.h | 18 class VPValue; variable
40 DenseMap<const VPValue *, Type *> CachedTypes;
61 Type *inferScalarType(const VPValue *V);
|
| H A D | VPlanAnalysis.cpp | 23 VPValue *Inc = R->getIncomingValue(I); in inferScalarTypeForRecipe()
37 VPValue *OtherV = R->getOperand(Op); in inferScalarTypeForRecipe()
52 VPValue *OtherV = R->getOperand(2); in inferScalarTypeForRecipe()
147 VPValue *OtherV = R->getOperand(2); in inferScalarTypeForRecipe()
233 Type *VPTypeAnalysis::inferScalarType(const VPValue *V) { in inferScalarType()
308 for (VPValue *Op : Cur->operands()) { in collectEphemeralRecipesForVPlan()
|
| H A D | VPlanRecipes.cpp | 196 VPValue *ExitValue = getOperand(0); in fixPhi()
331 VPValue *A, VPValue *B, DebugLoc DL, in VPInstruction()
333 : VPRecipeWithIRFlags(VPDef::VPInstructionSC, ArrayRef<VPValue *>({A, B}), in VPInstruction()
341 std::initializer_list<VPValue *> Operands, in VPInstruction()
558 VPValue *LoopExitingDef = getOperand(1); in generatePerPart()
747 bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const { in onlyFirstLaneUsed()
771 bool VPInstruction::onlyFirstPartUsed(const VPValue *Op) const { in onlyFirstPartUsed()
947 interleaveComma(arg_operands(), O, [&O, &SlotTracker](VPValue *Op) { in print()
1078 for (VPValue *VPOp : operands()) in execute()
1160 VPValue *Op = getOperand(0); in execute()
[all …]
|
| H A D | LoopVectorize.cpp | 2385 VPValue *Operand = I.value(); in scalarizeInstruction()
2810 VPValue *StepVPV = Plan.getSCEVExpansion(II.getStep()); in fixupIVUsers()
3103 VPValue *Inc = VPPhi->getIncomingValue(i); in fixNonInductionPHIs()
4500 VPValue *ToCheck = in willGenerateVectors()
6774 VPValue *VPBuilder::createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B, in createICmp()
7743 iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
7745 std::function<VPValue *(Value *)> Fn = [this](Value *Op) { in mapToVPValues()
7755 VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) { in createEdgeMask()
7764 VPValue *SrcMask = getBlockInMask(Src); in createEdgeMask()
7779 VPValue *EdgeMask = getVPValueOrAddLiveIn(BI->getCondition(), Plan); in createEdgeMask()
[all …]
|
| H A D | VPlanVerifier.cpp | 129 for (const VPValue *V : R.definedValues()) { in verifyVPBasicBlock()
|