Home
last modified time | relevance | path

Searched refs:VPValue (Results 1 – 21 of 21) sorted by relevance

/freebsd/contrib/llvm-project/llvm/lib/Transforms/Vectorize/
H A DVPlanValue.h48 class LLVM_ABI_FOR_TEST VPValue {
67 VPValue(const unsigned char SC, Value *UV = nullptr, VPDef *Def = nullptr);
70 VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV, nullptr) {} in VPValue() function
72 VPValue(VPDef *Def, Value *UV = nullptr) : VPValue(VPVRecipeSC, UV, Def) {} in VPValue() function
74 VPValue(Value *UV, VPDef *Def) : VPValue(VPValueSC, UV, Def) {} in VPValue() function
95 VPValue(const VPValue &) = delete;
96 VPValue &operator=(const VPValue &) = delete;
98 virtual ~VPValue();
151 void replaceAllUsesWith(VPValue *New);
157 VPValue *New,
[all …]
H A DVPlanPredicator.cpp32 VPValue *>;
33 using BlockMaskCacheTy = DenseMap<VPBasicBlock *, VPValue *>;
43 VPValue *createEdgeMask(VPBasicBlock *Src, VPBasicBlock *Dst);
46 VPValue *getBlockInMask(VPBasicBlock *VPBB) const { in getBlockInMask()
52 void setBlockInMask(VPBasicBlock *VPBB, VPValue *Mask) { in setBlockInMask()
61 VPValue *setEdgeMask(const VPBasicBlock *Src, const VPBasicBlock *Dst, in setEdgeMask()
62 VPValue *Mask) { in setEdgeMask()
70 VPValue *getEdgeMask(const VPBasicBlock *Src, const VPBasicBlock *Dst) const { in getEdgeMask()
79 VPValue *createBlockInMask(VPBasicBlock *VPBB);
88 VPValue *VPPredicator::createEdgeMask(VPBasicBlock *Src, VPBasicBlock *Dst) { in createEdgeMask()
[all …]
H A DVPlan.h402 VPRecipeBase(const unsigned char SC, ArrayRef<VPValue *> Operands,
500 static inline bool classof(const VPValue *V) { \
518 class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
520 VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
522 : VPRecipeBase(SC, Operands, DL), VPValue(this) {} in VPRecipeBase()
524 VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
526 : VPRecipeBase(SC, Operands, DL), VPValue(this, UV) {} in VPRecipeBase()
863 VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
867 VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands, in VPRecipeWithIRFlags()
871 VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
[all …]
H A DVPlanSLP.cpp105 void VPlanSlp::addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New) { in addCombined()
106 if (all_of(Operands, [](VPValue *V) { in addCombined()
110 for (VPValue *V : Operands) { in addCombined()
124 bool VPlanSlp::areVectorizable(ArrayRef<VPValue *> Operands) const { in areVectorizable()
126 if (!all_of(Operands, [](VPValue *Op) { in areVectorizable()
142 if (!all_of(Operands, [Opcode, Width](VPValue *Op) { in areVectorizable()
152 if (any_of(Operands, [this](VPValue *Op) { in areVectorizable()
160 [](VPValue *Op) { return Op->hasMoreThanOneUniqueUser(); })) { in areVectorizable()
189 if (!all_of(Operands, [](VPValue *Op) { in areVectorizable()
199 if (!all_of(Operands, [](VPValue *Op) { in areVectorizable()
[all …]
H A DLoopVectorizationPlanner.h66 ArrayRef<VPValue *> Operands, DebugLoc DL,
151 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
161 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
165 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
174 std::initializer_list<VPValue *> Operands,
183 std::initializer_list<VPValue *> Operands,
191 VPInstruction *createNot(VPValue *Operand,
197 VPInstruction *createAnd(VPValue *LHS, VPValue *RHS,
203 VPInstruction *createOr(VPValue *LHS, VPValue *RHS,
212 VPInstruction *createLogicalAnd(VPValue *LHS, VPValue *RHS,
[all …]
H A DVPRecipeBuilder.h73 DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache;
100 ArrayRef<VPValue *> Operands,
106 ArrayRef<VPValue *> Operands,
112 tryToOptimizeInductionTruncate(TruncInst *I, ArrayRef<VPValue *> Operands,
118 VPSingleDefRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands,
124 VPWidenRecipe *tryToWiden(Instruction *I, ArrayRef<VPValue *> Operands);
131 ArrayRef<VPValue *> Operands);
152 DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache, in VPRecipeBuilder()
175 ArrayRef<VPValue *> Operands,
187 VPValue *getBlockInMask(VPBasicBlock *VPBB) const { in getBlockInMask()
[all …]
H A DVPlanSLP.h28 class VPValue; variable
78 DenseMap<SmallVector<VPValue *, 4>, VPInstruction *> BundleToCombined;
93 typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
105 bool areVectorizable(ArrayRef<VPValue *> Operands) const;
108 void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
120 std::pair<OpMode, VPValue *> getBest(OpMode Mode, VPValue *Last,
121 SmallPtrSetImpl<VPValue *> &Candidates,
126 void dumpBundle(ArrayRef<VPValue *> Values);
136 LLVM_ABI_FOR_TEST VPInstruction *buildGraph(ArrayRef<VPValue *> Operands);
H A DVPlanTransforms.cpp59 VPValue *VPV = Ingredient.getVPSingleValue(); in tryToConvertVPInstructionsToVPRecipes()
72 VPValue *Start = Plan->getOrAddLiveIn(II->getStartValue()); in tryToConvertVPInstructionsToVPRecipes()
73 VPValue *Step = in tryToConvertVPInstructionsToVPRecipes()
138 for (VPValue *Op : Recipe.operands()) in sinkScalarOperands()
202 for (VPValue *Op : SinkCandidate->operands()) in sinkScalarOperands()
213 VPValue *getPredicatedMask(VPRegionBlock *R) { in getPredicatedMask()
266 VPValue *Mask1 = getPredicatedMask(Region1); in mergeReplicateRegionsIntoSuccessors()
267 VPValue *Mask2 = getPredicatedMask(Region2); in mergeReplicateRegionsIntoSuccessors()
302 VPValue *PredInst1 = in mergeReplicateRegionsIntoSuccessors()
304 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue(); in mergeReplicateRegionsIntoSuccessors()
[all …]
H A DVPlanHelpers.h224 DenseMap<const VPValue *, Value *> VPV2Vector;
226 DenseMap<const VPValue *, SmallVector<Value *, 4>> VPV2Scalars;
231 Value *get(const VPValue *Def, bool IsScalar = false);
234 Value *get(const VPValue *Def, const VPLane &Lane);
236 bool hasVectorValue(const VPValue *Def) { in hasVectorValue()
240 bool hasScalarValue(const VPValue *Def, VPLane Lane) { in hasScalarValue()
250 void set(const VPValue *Def, Value *V, bool IsScalar = false) {
261 void reset(const VPValue *Def, Value *V) { in reset()
267 void set(const VPValue *Def, Value *V, const VPLane &Lane) { in set()
277 void reset(const VPValue *Def, Value *V, const VPLane &Lane) { in reset()
[all …]
H A DVPlanUtils.cpp17 bool vputils::onlyFirstLaneUsed(const VPValue *Def) { in onlyFirstLaneUsed()
22 bool vputils::onlyFirstPartUsed(const VPValue *Def) { in onlyFirstPartUsed()
27 VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, in getOrCreateVPValueForSCEVExpr()
31 VPValue *Expanded = nullptr; in getOrCreateVPValueForSCEVExpr()
51 bool vputils::isHeaderMask(const VPValue *V, VPlan &Plan) { in isHeaderMask()
55 auto IsWideCanonicalIV = [](VPValue *A) { in isHeaderMask()
61 VPValue *A, *B; in isHeaderMask()
75 const SCEV *vputils::getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE) { in getSCEVExprForVPValue()
86 bool vputils::isUniformAcrossVFsAndUFs(VPValue *V) { in isUniformAcrossVFsAndUFs()
H A DVPlanUnroll.cpp51 DenseMap<VPValue *, SmallVector<VPValue *>> VPV2Parts;
71 VPValue *getConstantVPV(unsigned Part) { in getConstantVPV()
82 VPValue *getValueForPart(VPValue *V, unsigned Part) { in getValueForPart()
110 bool contains(VPValue *VPV) const { return VPV2Parts.contains(VPV); } in contains()
160 VPValue *ScalarStep = IV->getStepValue(); in unrollWidenInductionByUF()
180 VPValue *Prev = IV; in unrollWidenInductionByUF()
237 VPValue *StartV; in unrollHeaderPHIByUF()
292 VPValue *Op; in unrollRecipeByUF()
350 VPValue *Op1; in unrollBlock()
365 VPValue *Op0; in unrollBlock()
[all …]
H A DVPlanConstruction.cpp51 DenseMap<Value *, VPValue *> IRDef2VPValue;
63 VPValue *getOrCreateVPOperand(Value *IRVal);
93 VPValue *VPVal = IRDef2VPValue[Phi]; in fixHeaderPhis()
147 VPValue *PlainCFGBuilder::getOrCreateVPOperand(Value *IRVal) { in getOrCreateVPOperand()
165 VPValue *NewVPVal = Plan->getOrAddLiveIn(IRVal); in getOrCreateVPOperand()
189 VPValue *Cond = getOrCreateVPOperand(Br->getCondition()); in createVPInstructionsForVPBB()
198 SmallVector<VPValue *> Ops = {getOrCreateVPOperand(SI->getCondition())}; in createVPInstructionsForVPBB()
219 DenseMap<const VPBasicBlock *, VPValue *> VPPredToIncomingValue; in createVPInstructionsForVPBB()
231 SmallVector<VPValue *, 4> VPOperands; in createVPInstructionsForVPBB()
569 VPValue *Cmp; in prepareForVectorization()
[all …]
H A DVPlanUtils.h23 bool onlyFirstLaneUsed(const VPValue *Def);
26 bool onlyFirstPartUsed(const VPValue *Def);
33 VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
38 const SCEV *getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE);
42 inline bool isSingleScalar(const VPValue *VPV) { in isSingleScalar()
89 bool isHeaderMask(const VPValue *V, VPlan &Plan);
95 bool isUniformAcrossVFsAndUFs(VPValue *V);
H A DVPlan.cpp88 VPValue::VPValue(const unsigned char SC, Value *UV, VPDef *Def) in VPValue() function in VPValue
94 VPValue::~VPValue() { in ~VPValue()
101 void VPValue::print(raw_ostream &OS, VPSlotTracker &SlotTracker) const { in print()
108 void VPValue::dump() const { in dump()
125 VPRecipeBase *VPValue::getDefiningRecipe() { in getDefiningRecipe()
129 const VPRecipeBase *VPValue::getDefiningRecipe() const { in getDefiningRecipe()
252 Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) { in get()
284 Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) { in get()
407 Value *VPTransformState::packScalarIntoVectorizedValue(const VPValue *Def, in packScalarIntoVectorizedValue()
932 VPValue DummyValue; in ~VPlan()
[all …]
H A DVPlanAnalysis.h20 class VPValue; variable
44 DenseMap<const VPValue *, Type *> CachedTypes;
67 Type *inferScalarType(const VPValue *V);
H A DVPlanPatternMatch.h42 inline class_match<VPValue> m_VPValue() { return class_match<VPValue>(); } in m_VPValue()
60 const VPValue *Val;
62 specificval_ty(const VPValue *V) : Val(V) {} in specificval_ty()
64 bool match(VPValue *VPV) const { return VPV == Val; } in match()
67 inline specificval_ty m_Specific(const VPValue *VPV) { return VPV; } in m_Specific()
72 VPValue *const &Val;
74 deferredval_ty(VPValue *const &V) : Val(V) {} in deferredval_ty()
76 bool match(VPValue *const V) const { return V == Val; } in match()
85 inline deferredval_ty m_Deferred(VPValue *const &V) { return V; } in m_Deferred()
97 bool match(VPValue *VPV) const { in match()
[all …]
H A DVPlanAnalysis.cpp47 VPValue *Inc = R->getIncomingValue(I); in inferScalarTypeForRecipe()
61 VPValue *OtherV = R->getOperand(Op); in inferScalarTypeForRecipe()
80 VPValue *OtherV = R->getOperand(2); in inferScalarTypeForRecipe()
194 VPValue *OtherV = R->getOperand(2); in inferScalarTypeForRecipe()
257 Type *VPTypeAnalysis::inferScalarType(const VPValue *V) { in inferScalarType()
336 for (VPValue *Op : Cur->operands()) { in collectEphemeralRecipesForVPlan()
431 SmallSetVector<VPValue *, 8> LoopInvariants; in calculateRegisterUsageForPlan()
448 for (VPValue *U : R.operands()) { in calculateRegisterUsageForPlan()
576 for (VPValue *DefV : R->definedValues()) { in calculateRegisterUsageForPlan()
H A DVPlanTransforms.h228 DenseMap<VPValue *, VPValue *> &EndValues);
248 static DenseMap<VPBasicBlock *, VPValue *>
H A DVPlanRecipes.cpp302 VPValue *Op = getOperand(0); in computeCost()
415 VPValue *
434 VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, in VPInstruction()
862 for (VPValue *Op : drop_begin(operands())) in generate()
1061 bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const { in onlyFirstLaneUsed()
1098 bool VPInstruction::onlyFirstPartUsed(const VPValue *Op) const { in onlyFirstPartUsed()
1315 VPValue *Exiting = getOperand(0); in extractLastLaneOfFirstOperand()
1333 VPValue *ExitValue = Op; in execute()
1469 interleaveComma(args(), O, [&O, &SlotTracker](VPValue *Op) { in print()
1565 bool VPWidenIntrinsicRecipe::onlyFirstLaneUsed(const VPValue *Op) const { in onlyFirstLaneUsed()
[all …]
H A DLoopVectorize.cpp2711 VPValue *Inc = VPPhi->getIncomingValue(Idx); in fixNonInductionPHIs()
4209 VPValue *ToCheck = in willGenerateVectors()
7137 VPValue *StartVPV = RdxResult->getOperand(1); in getStartValueFromReductionResult()
7150 VPValue *Incoming = EpiResumePhiR->getOperand(0); in fixReductionScalarResumeWhenVectorizingEpilog()
7269 VPValue *Exp = BestVPlan.getOrAddLiveIn(ExpandedSCEVs[ExpSCEV->getSCEV()]); in executePlan()
7630 VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands, in tryToWidenMemory()
7651 VPValue *Mask = nullptr; in tryToWidenMemory()
7663 VPValue *Ptr = isa<LoadInst>(I) ? Operands[0] : Operands[1]; in tryToWidenMemory()
7702 VPValue *Start, const InductionDescriptor &IndDesc, in createWidenInductionRecipes()
7709 VPValue *Step = in createWidenInductionRecipes()
[all …]
H A DVPlanVerifier.cpp146 SmallVector<const VPValue *> Ops(R.operands()); in verifyEVLRecipe()
233 for (const VPValue *V : R.definedValues()) { in verifyVPBasicBlock()
246 VPValue *IncomingVPV = Phi->getIncomingValue(Idx); in verifyVPBasicBlock()