Home
last modified time | relevance | path

Searched refs:VF (Results 1 – 25 of 118) sorted by relevance

12345

/freebsd/contrib/llvm-project/llvm/lib/Transforms/Vectorize/
H A DVPlanHelpers.h47 Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF);
50 Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF,
96 ElementCount VF; member
99 iterator(ElementCount VF) : VF(VF) {} in iterator()
101 bool operator==(const iterator &Other) const { return VF == Other.VF; }
103 ElementCount operator*() const { return VF; }
106 VF *= 2;
152 static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset) { in getLaneFromEnd() argument
153 assert(Offset > 0 && Offset <= VF.getKnownMinValue() && in getLaneFromEnd()
155 unsigned LaneOffset = VF.getKnownMinValue() - Offset; in getLaneFromEnd()
[all …]
H A DLoopVectorize.cpp491 AC(AC), ORE(ORE), VF(VecWidth), in InnerLoopVectorizer()
536 Value *createIterationCountCheck(ElementCount VF, unsigned UF) const;
584 ElementCount VF; member in llvm::InnerLoopVectorizer
816 Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, in createStepForVF() argument
819 return B.CreateElementCount(Ty, VF.multiplyCoefficientBy(Step)); in createStepForVF()
823 Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF) { in getRuntimeVF() argument
824 return B.CreateElementCount(Ty, VF); in getRuntimeVF()
856 VectorizationFactor VF, unsigned IC) { in reportVectorization() argument
865 << ore::NV("VectorizationFactor", VF.Width) in reportVectorization()
952 bool useMaxBandwidth(ElementCount VF);
[all …]
H A DVPlanRecipes.cpp251 InstructionCost VPRecipeBase::cost(ElementCount VF, VPCostContext &Ctx) { in cost() argument
265 if (UI && Ctx.skipCostComputation(UI, VF.isVector())) { in cost()
268 RecipeCost = computeCost(VF, Ctx); in cost()
275 dbgs() << "Cost of " << RecipeCost << " for VF " << VF << ": "; in cost()
281 InstructionCost VPRecipeBase::computeCost(ElementCount VF, in computeCost() argument
299 VPPartialReductionRecipe::computeCost(ElementCount VF, in computeCost() argument
359 return Reduction->computeCost(VF, Ctx); in computeCost()
363 PhiType, VF, ExtAType, ExtBType, in computeCost()
576 assert(State.VF.isVector() && "Only extract elements from vectors"); in generate()
615 if (State.VF.isScalar()) in generate()
[all …]
H A DVPlan.cpp76 const ElementCount &VF) const { in getAsRuntimeExpr()
80 return Builder.CreateSub(getRuntimeVF(Builder, Builder.getInt32Ty(), VF), in getAsRuntimeExpr()
81 Builder.getInt32(VF.getKnownMinValue() - Lane)); in getAsRuntimeExpr()
244 ElementCount VF, LoopInfo *LI, in VPTransformState() argument
248 : TTI(TTI), VF(VF), CFG(DT), LI(LI), AC(AC), Builder(Builder), Plan(Plan), in VPTransformState()
257 return Data.VPV2Scalars[Def][Lane.mapToCacheIndex(VF)]; in get()
278 Value *LaneV = Lane.getAsRuntimeExpr(Builder, VF); in get()
286 assert((VF.isScalar() || Def->isLiveIn() || hasVectorValue(Def) || in get()
305 if (VF.isScalar()) in get()
318 Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast"); in get()
[all …]
H A DEVLIndVarSimplify.cpp86 uint64_t VF = V.udiv(*Fixed).getLimitedValue(); in getVFFromIndVar() local
87 if (VF && llvm::isUInt<32>(VF) && in getVFFromIndVar()
90 return VF; in getVFFromIndVar()
163 uint32_t VF = getVFFromIndVar(StepV, *L.getHeader()->getParent()); in run() local
164 if (!VF) { in run()
177 LLVM_DEBUG(dbgs() << "Using VF=" << VF << " for loop " << L.getName() in run()
188 m_Value(RemTC), m_SpecificInt(VF), in run()
H A DVPlan.h343 virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx) = 0;
422 InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
491 virtual InstructionCost computeCost(ElementCount VF,
1082 InstructionCost computeCost(ElementCount VF,
1185 InstructionCost computeCost(ElementCount VF, in computeCost() argument
1289 InstructionCost computeCost(ElementCount VF,
1383 InstructionCost computeCost(ElementCount VF,
1436 InstructionCost computeCost(ElementCount VF,
1509 InstructionCost computeCost(ElementCount VF,
1571 InstructionCost computeCost(ElementCount VF,
[all …]
H A DVPlanAnalysis.cpp500 auto GetRegUsage = [&TTICapture](Type *Ty, ElementCount VF) -> unsigned { in calculateRegisterUsageForPlan() argument
502 (VF.isScalable() && in calculateRegisterUsageForPlan()
505 return TTICapture.getRegUsageForType(VectorType::get(Ty, VF)); in calculateRegisterUsageForPlan()
570 ElementCount VF = VFs[J].divideCoefficientBy(ScaleFactor); in calculateRegisterUsageForPlan() local
571 LLVM_DEBUG(if (VF != VFs[J]) { in calculateRegisterUsageForPlan()
572 dbgs() << "LV(REG): Scaled down VF from " << VFs[J] << " to " << VF in calculateRegisterUsageForPlan()
579 RegUsage[ClassID] += GetRegUsage(ScalarTy, VF); in calculateRegisterUsageForPlan()
615 ElementCount VF = IsScalar ? ElementCount::getFixed(1) : VFs[Idx]; in calculateRegisterUsageForPlan() local
617 VF.isVector(), TypeInfo.inferScalarType(In)); in calculateRegisterUsageForPlan()
618 Invariant[ClassID] += GetRegUsage(TypeInfo.inferScalarType(In), VF); in calculateRegisterUsageForPlan()
H A DLoopVectorizationPlanner.h309 VPValue *VF, DebugLoc DL) { in createScalarIVSteps() argument
311 IV, Step, VF, InductionOpcode, in createScalarIVSteps()
455 InstructionCost cost(VPlan &Plan, ElementCount VF) const;
460 InstructionCost precomputeCosts(VPlan &Plan, ElementCount VF,
484 VPlan &getPlanFor(ElementCount VF) const;
500 DenseMap<const SCEV *, Value *> executePlan(ElementCount VF, unsigned UF,
512 bool hasPlanWithVF(ElementCount VF) const { in hasPlanWithVF() argument
514 [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); }); in hasPlanWithVF()
597 bool isCandidateForEpilogueVectorization(const ElementCount VF) const;
H A DSLPVectorizer.cpp268 static FixedVectorType *getWidenedType(Type *ScalarTy, unsigned VF) { in getWidenedType() argument
270 VF * getNumElements(ScalarTy)); in getWidenedType()
663 static SmallBitVector buildUseMask(int VF, ArrayRef<int> Mask, in buildUseMask() argument
665 SmallBitVector UseMask(VF, true); in buildUseMask()
672 if (MaskArg == UseMask::FirstArg && Value < VF) in buildUseMask()
674 else if (MaskArg == UseMask::SecondArg && Value >= VF) in buildUseMask()
675 UseMask.reset(Value - VF); in buildUseMask()
1482 Mappings.front().Shape.VF != BaseMappings.front().Shape.VF || in getSameOpcode()
1648 unsigned VF) { in replicateMask() argument
1651 SmallVector<Constant *> NewVal(Val.size() * VF); in replicateMask()
[all …]
H A DLoopIdiomVectorize.cpp134 LoopIdiomVectorize(LoopIdiomVectorizeStyle S, unsigned VF, DominatorTree *DT, in LoopIdiomVectorize() argument
137 : VectorizeStyle(S), ByteCompareVF(VF), DT(DT), LI(LI), TTI(TTI), DL(DL) { in LoopIdiomVectorize()
173 unsigned VF, Type *CharTy, BasicBlock *ExitSucc,
178 void transformFindFirstByte(PHINode *IndPhi, unsigned VF, Type *CharTy,
556 auto *VF = ConstantInt::get(I32Type, ByteCompareVF); in createPredicatedFindMismatch() local
559 {I64Type}, {AVL, VF, Builder.getTrue()}); in createPredicatedFindMismatch()
1086 unsigned VF = 128 / CharTy->getIntegerBitWidth(); in recognizeFindFirstByte() local
1088 ScalableVectorType::get(CharTy, VF), FixedVectorType::get(CharTy, VF), in recognizeFindFirstByte()
1089 ScalableVectorType::get(Type::getInt1Ty(Ctx), VF)}; in recognizeFindFirstByte()
1169 transformFindFirstByte(IndPhi, VF, CharTy, ExitSucc, ExitFail, SearchStart, in recognizeFindFirstByte()
[all …]
H A DVPlanTransforms.h118 static void replicateByVF(VPlan &Plan, ElementCount VF);
239 static void narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
254 addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF,
H A DLoopVectorizationLegality.cpp566 bool LoopVectorizationLegality::isUniform(Value *V, ElementCount VF) const { in isUniform()
569 if (VF.isScalable()) in isUniform()
571 if (VF.isScalar()) in isUniform()
583 unsigned FixedVF = VF.getKnownMinValue(); in isUniform()
600 ElementCount VF) const { in isUniformMemOp()
608 return isUniform(Ptr, VF) && !blockNeedsPredication(I.getParent()); in isUniformMemOp()
771 for (ElementCount VF = ElementCount::getFixed(2); in isTLIScalarize() local
772 ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2) in isTLIScalarize()
773 Scalarize &= !TLI.isFunctionVectorizable(ScalarName, VF); in isTLIScalarize()
774 for (ElementCount VF = ElementCount::getScalable(1); in isTLIScalarize() local
[all …]
/freebsd/contrib/llvm-project/llvm/lib/Transforms/Utils/
H A DInjectTLIMappings.cpp40 static void addVariantDeclaration(CallInst &CI, const ElementCount &VF, in addVariantDeclaration() argument
51 assert(Info->Shape.VF == VF && "Mangled name does not match VF"); in addVariantDeclaration()
94 auto AddVariantDecl = [&](const ElementCount &VF, bool Predicate) { in addMappingsFromTLI() argument
95 const VecDesc *VD = TLI.getVectorMappingInfo(ScalarName, VF, Predicate); in addMappingsFromTLI()
104 addVariantDeclaration(CI, VF, VD); in addMappingsFromTLI()
113 for (ElementCount VF = ElementCount::getFixed(2); in addMappingsFromTLI() local
114 ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2) in addMappingsFromTLI()
115 AddVariantDecl(VF, Predicated); in addMappingsFromTLI()
117 for (ElementCount VF = ElementCount::getScalable(2); in addMappingsFromTLI() local
118 ElementCount::isKnownLE(VF, WidestScalableVF); VF *= 2) in addMappingsFromTLI()
[all …]
/freebsd/contrib/llvm-project/llvm/lib/Analysis/
H A DVFABIDemangling.cpp
H A DTargetLibraryInfo.cpp1299 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX, CC) \ argument
1300 {SCAL, VEC, VF, MASK, VABI_PREFIX, CC},
1319 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX) \ argument
1320 {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX, /* CC = */ std::nullopt},
1326 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, VABI_PREFIX) \ argument
1327 {SCAL, VEC, VF, /* MASK = */ false, VABI_PREFIX, /* CC = */ std::nullopt},
1333 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX) \ argument
1334 {SCAL, VEC, VF, MASK, VABI_PREFIX, /* CC = */ std::nullopt},
1341 #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF, MASK, VABI_PREFIX) \ argument
1342 {SCAL, VEC, VF, MASK, VABI_PREFIX, /* CC = */ std::nullopt},
[all …]
/freebsd/contrib/llvm-project/llvm/lib/Target/SystemZ/
H A DSystemZTargetTransformInfo.cpp648 unsigned VF = VTy->getNumElements(); in getArithmeticInstrCost() local
662 return VF * DivMulSeqCost + in getArithmeticInstrCost()
668 else if (VF > 4) in getArithmeticInstrCost()
693 (VF * ScalarCost) + in getArithmeticInstrCost()
697 if (VF == 2) in getArithmeticInstrCost()
713 (VF * LIBCALL_COST) + in getArithmeticInstrCost()
716 if (VF == 2 && ScalarBits == 32) in getArithmeticInstrCost()
804 unsigned VF = cast<FixedVectorType>(SrcTy)->getNumElements(); in getVectorTruncCost() local
814 if (VF == 8 && SrcTy->getScalarSizeInBits() == 64 && in getVectorTruncCost()
848 static Type *getCmpOpsType(const Instruction *I, unsigned VF = 1) { in getCmpOpsType() argument
[all …]
/freebsd/contrib/llvm-project/llvm/include/llvm/Analysis/
H A DTargetLibraryInfo.h206 bool isFunctionVectorizable(StringRef F, const ElementCount &VF) const { in isFunctionVectorizable() argument
207 return !(getVectorizedFunction(F, VF, false).empty() && in isFunctionVectorizable()
208 getVectorizedFunction(F, VF, true).empty()); in isFunctionVectorizable()
217 LLVM_ABI StringRef getVectorizedFunction(StringRef F, const ElementCount &VF,
224 getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const;
397 bool isFunctionVectorizable(StringRef F, const ElementCount &VF) const { in isFunctionVectorizable() argument
398 return Impl->isFunctionVectorizable(F, VF); in isFunctionVectorizable()
403 StringRef getVectorizedFunction(StringRef F, const ElementCount &VF,
405 return Impl->getVectorizedFunction(F, VF, Masked);
407 const VecDesc *getVectorMappingInfo(StringRef F, const ElementCount &VF, in getVectorMappingInfo() argument
[all …]
H A DVectorUtils.h85 std::optional<ElementCount> VF = std::nullopt) {
91 if (!VF || Info.Shape.VF == *VF)
403 createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF,
419 createReplicatedMask(unsigned ReplicationFactor, unsigned VF);
432 LLVM_ABI llvm::SmallVector<int, 16> createInterleaveMask(unsigned VF,
448 createStrideMask(unsigned Start, unsigned Stride, unsigned VF);
/freebsd/contrib/llvm-project/llvm/lib/Target/AArch64/
H A DAArch64TargetTransformInfo.h166 unsigned getMaxNumElements(ElementCount VF) const { in getMaxNumElements() argument
167 if (!VF.isScalable()) in getMaxNumElements()
168 return VF.getFixedValue(); in getMaxNumElements()
170 return VF.getKnownMinValue() * ST->getVScaleForTuning(); in getMaxNumElements()
173 unsigned getMaxInterleaveFactor(ElementCount VF) const override;
386 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
434 ElementCount VF) const override;
479 unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, in getStoreMinimumVF() argument
482 if (ScalarMemTy->isIntegerTy(8) && isPowerOf2_32(VF) && VF >= 4) in getStoreMinimumVF()
485 return BaseT::getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy); in getStoreMinimumVF()
/freebsd/contrib/llvm-project/llvm/lib/IR/
H A DVFABIDemangler.cpp97 unsigned VF = 0; in tryParseVLEN() local
98 if (ParseString.consumeInteger(10, VF)) in tryParseVLEN()
102 if (VF == 0) in tryParseVLEN()
105 ParsedVF = {VF, false}; in tryParseVLEN()
560 ElementCount VF = Info.Shape.VF; in createFunctionType() local
565 VectorType::get(Type::getInt1Ty(ScalarFTy->getContext()), VF); in createFunctionType()
572 OperandTy = VectorType::get(OperandTy, VF); in createFunctionType()
578 RetTy = toVectorizedTy(RetTy, VF); in createFunctionType()
H A DVectorTypeUtils.cpp50 ElementCount VF = cast<VectorType>(ElemTys.front())->getElementCount(); in isVectorizedStructTy() local
52 return Ty->isVectorTy() && cast<VectorType>(Ty)->getElementCount() == VF; in isVectorizedStructTy()
/freebsd/contrib/llvm-project/llvm/include/llvm/IR/
H A DVFABIDemangler.h85 ElementCount VF; // Vectorization factor. member
89 return std::tie(VF, Parameters) == std::tie(Other.VF, Other.Parameters);
H A DVectorTypeUtils.h26 inline Type *toVectorTy(Type *Scalar, unsigned VF) { in toVectorTy() argument
27 return toVectorTy(Scalar, ElementCount::getFixed(VF)); in toVectorTy()
/freebsd/contrib/llvm-project/llvm/lib/Target/X86/
H A DX86InterleavedAccess.cpp436 int VF = VT.getVectorNumElements(); in createShuffleStride() local
439 for (int i = 0, LaneSize = VF / LaneCount; i != LaneSize; ++i) in createShuffleStride()
449 int VF = VT.getVectorNumElements() / std::max(VectorSize / 128, 1); in setGroupSize() local
451 int GroupSize = std::ceil((VF - FirstGroupElement) / 3.0); in setGroupSize()
453 FirstGroupElement = ((GroupSize)*3 + FirstGroupElement) % VF; in setGroupSize()
608 int VF = VT.getVectorNumElements(); in group2Shuffle() local
612 IndexGroup[(Index * 3) % (VF / Lane)] = Index; in group2Shuffle()
616 for (int i = 0; i < VF / Lane; i++) { in group2Shuffle()
/freebsd/contrib/llvm-project/llvm/lib/Target/RISCV/
H A DRISCVTargetTransformInfo.h105 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
357 ElementCount VF) const override { in isLegalToVectorizeReduction() argument
358 if (!VF.isScalable()) in isLegalToVectorizeReduction()
390 unsigned getMaxInterleaveFactor(ElementCount VF) const override { in getMaxInterleaveFactor() argument
392 if (VF.isScalable()) in getMaxInterleaveFactor()
396 return VF.isScalar() ? 1 : ST->getMaxInterleaveFactor(); in getMaxInterleaveFactor()

12345