Home
last modified time | relevance | path

Searched refs:VF (Results 1 – 25 of 98) sorted by relevance

1234

/freebsd/contrib/llvm-project/llvm/lib/Transforms/Utils/
H A DInjectTLIMappings.cpp38 /// the CallInst \p CI with a vectorization factor of \p VF lanes. For each
41 static void addVariantDeclaration(CallInst &CI, const ElementCount &VF, in addVariantDeclaration() argument
52 assert(Info->Shape.VF == VF && "Mangled name does not match VF"); in addVariantDeclaration()
94 auto AddVariantDecl = [&](const ElementCount &VF, bool Predicate) { in addMappingsFromTLI()
95 const VecDesc *VD = TLI.getVectorMappingInfo(ScalarName, VF, Predicate); in addMappingsFromTLI()
104 addVariantDeclaration(CI, VF, VD); in addMappingsFromTLI()
113 for (ElementCount VF = ElementCount::getFixed(2); in addMappingsFromTLI()
114 ElementCount::isKnownLE(VF, WidestFixedV in addMappingsFromTLI() local
93 __anon6d2b60130102(const ElementCount &VF, bool Predicate) addMappingsFromTLI() argument
118 for (ElementCount VF = ElementCount::getScalable(2); addMappingsFromTLI() local
[all...]
/freebsd/contrib/llvm-project/llvm/lib/Transforms/Vectorize/
H A DLoopVectorize.cpp481 AC(AC), ORE(ORE), VF(VecWidth), UF(UnrollFactor), in InnerLoopVectorizer()
639 ElementCount VF; member in llvm::InnerLoopVectorizer
917 Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, in createStepForVF() argument
920 return B.CreateElementCount(Ty, VF.multiplyCoefficientBy(Step)); in createStepForVF()
924 Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF) { in getRuntimeVF() argument
925 return B.CreateElementCount(Ty, VF); in getRuntimeVF()
964 VectorizationFactor VF, unsigned IC) { in reportVectorization() argument
973 << ore::NV("VectorizationFactor", VF.Width) in reportVectorization()
1055 unsigned selectInterleaveCount(ElementCount VF, InstructionCost LoopCost);
1064 void setCostBasedWideningDecision(ElementCount VF);
[all …]
H A DVPlanRecipes.cpp199 : VPLane::getLastLaneForVF(State.VF); in fixPhi()
285 InstructionCost VPRecipeBase::cost(ElementCount VF, VPCostContext &Ctx) { in cost() argument
287 if (Ctx.skipCostComputation(UI, VF.isVector())) in cost()
290 InstructionCost RecipeCost = computeCost(VF, Ctx); in cost()
296 dbgs() << "Cost of " << RecipeCost << " for VF " << VF << ": "; in cost()
302 InstructionCost VPRecipeBase::computeCost(ElementCount VF, in computeCost() argument
313 return UI ? Ctx.getLegacyCost(UI, VF) : 0; in computeCost()
421 if (State.VF.isScalar()) in generatePerPart()
426 auto *PredTy = VectorType::get(Int1Ty, State.VF); in generatePerPart()
458 createStepForVF(Builder, ScalarTC->getType(), State.VF, State.UF); in generatePerPart()
[all …]
H A DVPlan.cpp71 const ElementCount &VF) const { in getAsRuntimeExpr()
75 return Builder.CreateSub(getRuntimeVF(Builder, Builder.getInt32Ty(), VF), in getAsRuntimeExpr()
76 Builder.getInt32(VF.getKnownMinValue() - Lane)); in getAsRuntimeExpr()
219 VPTransformState::VPTransformState(ElementCount VF, unsigned UF, LoopInfo *LI, in VPTransformState() argument
223 : VF(VF), UF(UF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan), in VPTransformState()
233 .PerPartScalars[Def][Instance.Part][Instance.Lane.mapToCacheIndex(VF)]; in get()
248 Value *Lane = Instance.Lane.getAsRuntimeExpr(Builder, VF); in get()
256 assert((VF.isScalar() || Def->isLiveIn() || hasVectorValue(Def, Part) || in get()
271 if (VF.isScalar()) in get()
284 Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast"); in get()
[all …]
H A DVPlan.h80 Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF);
83 Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF,
125 ElementCount VF; member
128 iterator(ElementCount VF) : VF(VF) {} in iterator()
130 bool operator==(const iterator &Other) const { return VF == Other.VF; }
132 ElementCount operator*() const { return VF; }
135 VF *= 2;
182 static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset) { in getLaneFromEnd() argument
183 assert(Offset > 0 && Offset <= VF.getKnownMinValue() && in getLaneFromEnd()
185 unsigned LaneOffset = VF.getKnownMinValue() - Offset; in getLaneFromEnd()
[all …]
H A DSLPVectorizer.cpp251 static FixedVectorType *getWidenedType(Type *ScalarTy, unsigned VF) { in getWidenedType() argument
253 VF * getNumElements(ScalarTy)); in getWidenedType()
460 static SmallBitVector buildUseMask(int VF, ArrayRef<int> Mask, in buildUseMask() argument
462 SmallBitVector UseMask(VF, true); in buildUseMask()
469 if (MaskArg == UseMask::FirstArg && Value < VF) in buildUseMask()
471 else if (MaskArg == UseMask::SecondArg && Value >= VF) in buildUseMask()
472 UseMask.reset(Value - VF); in buildUseMask()
898 Mappings.front().Shape.VF != BaseMappings.front().Shape.VF || in getSameOpcode()
4276 const int VF = GetVF(I); in findReusedOrderedScalars() local
4277 if (VF == 0) in findReusedOrderedScalars()
[all …]
H A DLoopVectorizationPlanner.h345 InstructionCost cost(VPlan &Plan, ElementCount VF) const;
366 VPlan &getBestPlanFor(ElementCount VF) const;
385 executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan,
396 bool hasPlanWithVF(ElementCount VF) const { in hasPlanWithVF() argument
398 [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); }); in hasPlanWithVF()
462 bool isCandidateForEpilogueVectorization(const ElementCount VF) const;
H A DLoopVectorizationLegality.cpp568 bool LoopVectorizationLegality::isUniform(Value *V, ElementCount VF) const { in isUniform()
571 if (VF.isScalable()) in isUniform()
573 if (VF.isScalar()) in isUniform()
585 unsigned FixedVF = VF.getKnownMinValue(); in isUniform()
602 ElementCount VF) const { in isUniformMemOp()
610 return isUniform(Ptr, VF) && !blockNeedsPredication(I.getParent()); in isUniformMemOp()
776 for (ElementCount VF = ElementCount::getFixed(2); in isTLIScalarize() local
777 ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2) in isTLIScalarize()
778 Scalarize &= !TLI.isFunctionVectorizable(ScalarName, VF); in isTLIScalarize()
779 for (ElementCount VF = ElementCount::getScalable(1); in isTLIScalarize() local
[all …]
/freebsd/contrib/llvm-project/llvm/lib/Target/SystemZ/
H A DSystemZTargetTransformInfo.cpp532 unsigned VF = VTy->getNumElements(); in getArithmeticInstrCost() local
546 return VF * DivMulSeqCost + in getArithmeticInstrCost()
549 if ((SignedDivRem || UnsignedDivRem) && VF > 4) in getArithmeticInstrCost()
573 (VF * ScalarCost) + in getArithmeticInstrCost()
577 if (VF == 2) in getArithmeticInstrCost()
592 InstructionCost Cost = (VF * LIBCALL_COST) + in getArithmeticInstrCost()
595 if (VF == 2 && ScalarBits == 32) in getArithmeticInstrCost()
681 unsigned VF = cast<FixedVectorType>(SrcTy)->getNumElements(); in getVectorTruncCost() local
691 if (VF == 8 && SrcTy->getScalarSizeInBits() == 64 && in getVectorTruncCost()
725 static Type *getCmpOpsType(const Instruction *I, unsigned VF = 1) { in getCmpOpsType() argument
[all …]
/freebsd/contrib/llvm-project/llvm/lib/Analysis/
H A DVFABIDemangling.cpp
H A DTargetLibraryInfo.cpp1239 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX) \ argument
1240 {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX},
1246 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX) \ argument
1247 {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX},
1253 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX) \ argument
1254 {SCAL, VEC, VF, MASK, VABI_PREFIX},
1261 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX) \ argument
1262 {SCAL, VEC, VF, MASK, VABI_PREFIX},
1269 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX) \ argument
1270 {SCAL, VEC, VF, MASK, VABI_PREFIX},
[all …]
H A DVectorUtils.cpp519 for (int Idx = 0, VF = FirstMask.size(); Idx < VF; ++Idx) { in processShuffleMasks() local
523 FirstMask[Idx] = SecondMask[Idx] + VF; in processShuffleMasks()
528 for (int Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) { in processShuffleMasks() local
891 llvm::createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, in createBitMaskForGaps() argument
901 for (unsigned i = 0; i < VF; i++) in createBitMaskForGaps()
911 llvm::createReplicatedMask(unsigned ReplicationFactor, unsigned VF) { in createReplicatedMask() argument
913 for (unsigned i = 0; i < VF; i++) in createReplicatedMask()
920 llvm::SmallVector<int, 16> llvm::createInterleaveMask(unsigned VF, in createInterleaveMask() argument
923 for (unsigned i = 0; i < VF; i++) in createInterleaveMask()
925 Mask.push_back(j * VF + i); in createInterleaveMask()
[all …]
H A DTargetTransformInfo.cpp778 unsigned TargetTransformInfo::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, in getStoreMinimumVF() argument
780 return TTIImpl->getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy); in getStoreMinimumVF()
832 unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const { in getMaxInterleaveFactor()
833 return TTIImpl->getMaxInterleaveFactor(VF); in getMaxInterleaveFactor()
1051 Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, in getReplicationShuffleCost() argument
1054 EltTy, ReplicationFactor, VF, DemandedDstElts, CostKind); in getReplicationShuffleCost()
1271 const RecurrenceDescriptor &RdxDesc, ElementCount VF) const { in isLegalToVectorizeReduction()
1272 return TTIImpl->isLegalToVectorizeReduction(RdxDesc, VF); in isLegalToVectorizeReduction()
1279 unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF, in getLoadVectorFactor() argument
1283 return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy); in getLoadVectorFactor()
[all …]
/freebsd/contrib/llvm-project/llvm/lib/Target/AArch64/
H A DAArch64TargetTransformInfo.h151 unsigned getMaxNumElements(ElementCount VF) const { in getMaxNumElements() argument
152 if (!VF.isScalable()) in getMaxNumElements()
153 return VF.getFixedValue(); in getMaxNumElements()
155 return VF.getKnownMinValue() * ST->getVScaleForTuning(); in getMaxNumElements()
158 unsigned getMaxInterleaveFactor(ElementCount VF);
387 ElementCount VF) const;
424 unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, in getStoreMinimumVF() argument
427 if (ScalarMemTy->isIntegerTy(8) && isPowerOf2_32(VF) && VF >= 4) in getStoreMinimumVF()
430 return BaseT::getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy); in getStoreMinimumVF()
/freebsd/contrib/llvm-project/llvm/include/llvm/Analysis/
H A DTargetLibraryInfo.h202 bool isFunctionVectorizable(StringRef F, const ElementCount &VF) const { in isFunctionVectorizable() argument
203 return !(getVectorizedFunction(F, VF, false).empty() && in isFunctionVectorizable()
204 getVectorizedFunction(F, VF, true).empty()); in isFunctionVectorizable()
213 StringRef getVectorizedFunction(StringRef F, const ElementCount &VF,
219 const VecDesc *getVectorMappingInfo(StringRef F, const ElementCount &VF,
389 bool isFunctionVectorizable(StringRef F, const ElementCount &VF) const { in isFunctionVectorizable() argument
390 return Impl->isFunctionVectorizable(F, VF); in isFunctionVectorizable()
395 StringRef getVectorizedFunction(StringRef F, const ElementCount &VF,
397 return Impl->getVectorizedFunction(F, VF, Masked);
399 const VecDesc *getVectorMappingInfo(StringRef F, const ElementCount &VF, in getVectorMappingInfo() argument
[all …]
H A DVectorUtils.h83 std::optional<ElementCount> VF = std::nullopt) {
89 if (!VF || Info.Shape.VF == *VF)
141 inline Type *ToVectorTy(Type *Scalar, unsigned VF) { in ToVectorTy() argument
142 return ToVectorTy(Scalar, ElementCount::getFixed(VF)); in ToVectorTy()
350 Constant *createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF,
366 unsigned VF);
379 llvm::SmallVector<int, 16> createInterleaveMask(unsigned VF, unsigned NumVecs);
394 unsigned VF);
H A DTargetTransformInfo.h1189 unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
1260 unsigned getMaxInterleaveFactor(ElementCount VF) const;
1413 int VF,
1661 ElementCount VF) const;
1668 unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
1674 unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
1997 virtual unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
2034 virtual unsigned getMaxInterleaveFactor(ElementCount VF) = 0;
2072 getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
2163 ElementCount VF) const = 0;
[all …]
H A DTargetTransformInfoImpl.h502 unsigned getStoreMinimumVF(unsigned VF, Type *, Type *) const { return VF; } in getStoreMinimumVF() argument
546 unsigned getMaxInterleaveFactor(ElementCount VF) const { return 1; } in getMaxInterleaveFactor() argument
678 unsigned getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF, in getReplicationShuffleCost() argument
907 ElementCount VF) const { in isLegalToVectorizeReduction() argument
913 unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize, in getLoadVectorFactor() argument
916 return VF; in getLoadVectorFactor()
919 unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize, in getStoreVectorFactor() argument
922 return VF; in getStoreVectorFactor()
1378 int ReplicationFactor, VF; in getInstructionCost() local
1379 if (Shuffle->isReplicationMask(ReplicationFactor, VF)) { in getInstructionCost()
[all …]
/freebsd/contrib/llvm-project/llvm/lib/Target/X86/
H A DX86InterleavedAccess.cpp439 int VF = VT.getVectorNumElements(); in createShuffleStride() local
442 for (int i = 0, LaneSize = VF / LaneCount; i != LaneSize; ++i) in createShuffleStride()
452 int VF = VT.getVectorNumElements() / std::max(VectorSize / 128, 1); in setGroupSize() local
454 int GroupSize = std::ceil((VF - FirstGroupElement) / 3.0); in setGroupSize()
456 FirstGroupElement = ((GroupSize)*3 + FirstGroupElement) % VF; in setGroupSize()
611 int VF = VT.getVectorNumElements(); in group2Shuffle() local
615 IndexGroup[(Index * 3) % (VF / Lane)] = Index; in group2Shuffle()
619 for (int i = 0; i < VF / Lane; i++) { in group2Shuffle()
/freebsd/contrib/llvm-project/llvm/lib/Target/RISCV/
H A DRISCVTargetTransformInfo.h305 ElementCount VF) const { in isLegalToVectorizeReduction() argument
306 if (!VF.isScalable()) in isLegalToVectorizeReduction()
334 unsigned getMaxInterleaveFactor(ElementCount VF) { in getMaxInterleaveFactor() argument
336 if (VF.isScalable()) in getMaxInterleaveFactor()
340 return VF.isScalar() ? 1 : ST->getMaxInterleaveFactor(); in getMaxInterleaveFactor()
/freebsd/tools/tools/vop_table/
H A Dvop_table.pl167 @VF = <V>;
176 foreach $v (@VF) {
/freebsd/contrib/llvm-project/llvm/lib/Target/AMDGPU/
H A DR600TargetTransformInfo.cpp85 unsigned R600TTIImpl::getMaxInterleaveFactor(ElementCount VF) { in getMaxInterleaveFactor() argument
88 if (VF.isScalar()) in getMaxInterleaveFactor()
H A DAMDGPUTargetTransformInfo.h124 unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
127 unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
150 unsigned getMaxInterleaveFactor(ElementCount VF);
/freebsd/contrib/llvm-project/llvm/lib/Target/PowerPC/MCTargetDesc/
H A DPPCMCTargetDesc.h263 static const MCPhysReg VFRegs[32] = PPC_REGS0_31(PPC::VF); \
268 static const MCPhysReg VSFRegs[64] = PPC_REGS_LO_HI(PPC::F, PPC::VF); \
269 static const MCPhysReg VSSRegs[64] = PPC_REGS_LO_HI(PPC::F, PPC::VF); \
/freebsd/contrib/llvm-project/llvm/include/llvm/CodeGen/
H A DBasicTTIImpl.h220 unsigned VF = VT->getNumElements(); variable
230 PointerType::get(VT->getElementType(), 0), VF),
236 VF * thisT()->getMemoryOpCost(Opcode, VT->getElementType(), Alignment,
254 FixedVectorType::get(Type::getInt1Ty(DataTy->getContext()), VF),
256 VF * (thisT()->getCFInstrCost(Instruction::Br, CostKind) +
357 unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, in getStoreMinimumVF() argument
359 auto &&IsSupportedByTarget = [this, ScalarMemTy, ScalarValTy](unsigned VF) { in getStoreMinimumVF()
360 auto *SrcTy = FixedVectorType::get(ScalarMemTy, VF / 2); in getStoreMinimumVF()
367 getTLI()->getValueType(DL, FixedVectorType::get(ScalarValTy, VF / 2)); in getStoreMinimumVF()
372 while (VF > 2 && IsSupportedByTarget(VF)) in getStoreMinimumVF()
[all …]

1234