| /freebsd/contrib/llvm-project/llvm/lib/Transforms/Vectorize/ |
| H A D | VPlanSLP.cpp | 61 auto *VPInst = dyn_cast<VPInstruction>(&VPI); in visitBlock()
73 Old2New[IG] = new InterleaveGroup<VPInstruction>( in visitBlock()
98 VPInstruction *VPlanSlp::markFailed() { in markFailed()
105 void VPlanSlp::addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New) { in addCombined()
107 return cast<VPInstruction>(V)->getUnderlyingInstr(); in addCombined()
111 Type *T = cast<VPInstruction>(V)->getUnderlyingInstr()->getType(); in addCombined()
127 return Op && isa<VPInstruction>(Op) && in areVectorizable()
128 cast<VPInstruction>(Op)->getUnderlyingInstr(); in areVectorizable()
139 cast<VPInstruction>(Operands[0])->getUnderlyingInstr(); in areVectorizable()
143 const Instruction *I = cast<VPInstruction>(Op)->getUnderlyingInstr(); in areVectorizable()
[all …]
|
| H A D | VPlanSLP.h | 29 class VPInstruction; variable
32 DenseMap<VPInstruction *, InterleaveGroup<VPInstruction> *>
38 InterleaveGroup<VPInstruction> *>;
57 SmallPtrSet<InterleaveGroup<VPInstruction> *, 4> DelSet( in ~VPInterleavedAccessInfo()
66 InterleaveGroup<VPInstruction> *
67 getInterleaveGroup(VPInstruction *Instr) const { in getInterleaveGroup()
78 DenseMap<SmallVector<VPValue *, 4>, VPInstruction *> BundleToCombined;
93 typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
108 void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
111 VPInstruction *markFailed();
[all …]
|
| H A D | LoopVectorizationPlanner.h | 65 VPInstruction *createInstruction(unsigned Opcode,
68 return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name));
151 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
157 VPInstruction *NewVPInst = createInstruction(Opcode, Operands, DL, Name);
161 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
165 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
170 new VPInstruction(Opcode, Operands, Flags, DL, Name));
173 VPInstruction *createNaryOp(unsigned Opcode,
182 VPInstruction *createOverflowingOp(unsigned Opcode,
188 new VPInstruction(Opcode, Operands, WrapFlags, DL, Name));
[all …]
|
| H A D | VPlanAnalysis.cpp | 55 Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) { in inferScalarTypeForRecipe()
76 case VPInstruction::ReductionStartVector: in inferScalarTypeForRecipe()
88 case VPInstruction::ActiveLaneMask: in inferScalarTypeForRecipe()
93 case VPInstruction::ComputeAnyOfResult: in inferScalarTypeForRecipe()
95 case VPInstruction::ComputeFindIVResult: in inferScalarTypeForRecipe()
96 case VPInstruction::ComputeReductionResult: { in inferScalarTypeForRecipe()
99 case VPInstruction::ExplicitVectorLength: in inferScalarTypeForRecipe()
105 case VPInstruction::FirstOrderRecurrenceSplice: in inferScalarTypeForRecipe()
106 case VPInstruction::Not: in inferScalarTypeForRecipe()
107 case VPInstruction::CalculateTripCountMinusVF: in inferScalarTypeForRecipe()
[all …]
|
| H A D | VPlanUnroll.cpp | 162 VPInstruction *VectorStep = Builder.createNaryOp( in unrollWidenInductionByUF()
163 VPInstruction::WideIVStep, {&Plan.getVF(), ScalarStep}, IVTy, Flags, in unrollWidenInductionByUF()
188 VPInstruction *Add = Builder.createNaryOp(AddOpc, in unrollWidenInductionByUF()
232 if (auto *VPI = dyn_cast<VPInstruction>(RdxPhi->getStartValue())) { in unrollHeaderPHIByUF()
233 assert(VPI->getOpcode() == VPInstruction::ReductionStartVector && in unrollHeaderPHIByUF()
263 if (auto *VPI = dyn_cast<VPInstruction>(&R)) { in unrollRecipeByUF()
293 if (match(&R, m_VPInstruction<VPInstruction::FirstOrderRecurrenceSplice>( in unrollRecipeByUF()
317 match(Copy, m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>( in unrollRecipeByUF()
351 if (match(&R, m_VPInstruction<VPInstruction::AnyOf>(m_VPValue(Op1))) || in unrollBlock()
352 match(&R, m_VPInstruction<VPInstruction::FirstActiveLane>( in unrollBlock()
[all …]
|
| H A D | VPlanRecipes.cpp | 55 return cast<VPInstruction>(this)->opcodeMayReadOrWriteFromMemory(); in mayWriteToMemory()
107 return cast<VPInstruction>(this)->opcodeMayReadOrWriteFromMemory(); in mayReadFromMemory()
288 (isa<VPInstruction>(this) && in isPhi()
289 cast<VPInstruction>(this)->getOpcode() == Instruction::PHI) || in isPhi()
294 auto *VPI = dyn_cast<VPInstruction>(this); in isScalarCast()
434 VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, in VPInstruction() function in VPInstruction
447 unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) { in getNumOperandsForOpcode()
455 case VPInstruction::StepVector: in getNumOperandsForOpcode()
461 case VPInstruction::AnyOf: in getNumOperandsForOpcode()
462 case VPInstruction::BranchOnCond: in getNumOperandsForOpcode()
[all …]
|
| H A D | VPlanPatternMatch.h | 196 inline bind_ty<VPInstruction> m_VPInstruction(VPInstruction *&V) { return V; } in m_VPInstruction()
225 assert(Opcode == VPInstruction::BuildVector && in match()
227 auto *VPI = dyn_cast<VPInstruction>(R); in match()
228 return VPI && VPI->getOpcode() == VPInstruction::BuildVector; in match()
283 UnaryRecipe_match<Op0_t, Opcode, VPInstruction>;
286 using ZeroOpVPInstruction_match = ZeroOpRecipe_match<Opcode, VPInstruction>;
291 VPWidenCastRecipe, VPInstruction>;
301 VPInstruction>;
311 VPInstruction>;
317 VPReplicateRecipe, VPWidenCastRecipe, VPInstruction>;
[all …]
|
| H A D | VPlanVerifier.cpp | 43 bool verifyEVLRecipe(const VPInstruction &EVL) const;
82 !isa<VPInstruction>(*RecipeI) && in verifyPhiRecipes()
83 cast<VPInstruction>(RecipeI)->getOpcode() == Instruction::PHI) { in verifyPhiRecipes()
138 bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const { in verifyEVLRecipe()
139 if (EVL.getOpcode() != VPInstruction::ExplicitVectorLength) { in verifyEVLRecipe()
173 .Case<VPInstruction>([&](const VPInstruction *I) { in verifyEVLRecipe()
292 if (const auto *EVL = dyn_cast<VPInstruction>(&R)) { in verifyVPBasicBlock()
293 if (EVL->getOpcode() == VPInstruction::ExplicitVectorLength && in verifyVPBasicBlock()
472 auto *LastInst = dyn_cast<VPInstruction>(std::prev(Exiting->end())); in verify()
473 if (!LastInst || (LastInst->getOpcode() != VPInstruction::BranchOnCount && in verify()
[all …]
|
| H A D | VPlanConstruction.cpp | 190 VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst); in createVPInstructionsForVPBB()
237 NewR = cast<VPInstruction>( in createVPInstructionsForVPBB()
389 assert(cast<VPInstruction>(Term)->getOpcode() == in canonicalHeaderAndLatch()
390 VPInstruction::BranchOnCond && in canonicalHeaderAndLatch()
392 auto *Not = new VPInstruction(VPInstruction::Not, {Term->getOperand(0)}); in canonicalHeaderAndLatch()
454 Builder.createNaryOp(VPInstruction::BranchOnCount, in addCanonicalIVRecipes()
579 Builder.createNaryOp(VPInstruction::BranchOnCond, {Cmp}, LatchDL); in prepareForVectorization()
622 .createNaryOp(VPInstruction::BranchOnCond, {CondVPV}, in attachCheckBlock()
724 auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator()); in handleMaxMinNumReductions()
725 assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount && in handleMaxMinNumReductions()
[all …]
|
| H A D | VPlanTransforms.cpp | 78 assert(isa<VPInstruction>(&Ingredient) && in tryToConvertVPInstructionsToVPRecipes()
779 m_VPInstruction<VPInstruction::FirstActiveLane>( in optimizeEarlyExitInductionUser()
796 DebugLoc DL = cast<VPInstruction>(Op)->getDebugLoc(); in optimizeEarlyExitInductionUser()
798 B.createNaryOp(VPInstruction::FirstActiveLane, Mask, DL); in optimizeEarlyExitInductionUser()
833 if (!match(Op, m_VPInstruction<VPInstruction::ExtractLastElement>( in optimizeLatchExitInductionUser()
957 case VPInstruction::LogicalAnd: in tryToFoldLiveIns()
960 case VPInstruction::Not: in tryToFoldLiveIns()
975 case VPInstruction::PtrAdd: in tryToFoldLiveIns()
999 .Case<VPInstruction, VPWidenRecipe, VPWidenCastRecipe, in simplifyRecipe()
1126 if (match(Def, m_VPInstruction<VPInstruction::WideIVStep>( in simplifyRecipe()
[all …]
|
| H A D | VPlanPredicator.cpp | 39 void createSwitchEdgeMasks(VPInstruction *SI);
102 auto *Term = cast<VPInstruction>(Src->getTerminator()); in createEdgeMask()
108 assert(Term->getOpcode() == VPInstruction::BranchOnCond && in createEdgeMask()
178 void VPPredicator::createSwitchEdgeMasks(VPInstruction *SI) { in createSwitchEdgeMasks()
|
| H A D | VPlanAnalysis.h | 22 class VPInstruction; variable
52 Type *inferScalarTypeForRecipe(const VPInstruction *R);
|
| H A D | VPlanUtils.h | 50 case VPInstruction::Broadcast: in isSingleScalar()
51 case VPInstruction::PtrAdd: in isSingleScalar()
79 if (auto *VPI = dyn_cast<VPInstruction>(VPV)) in isSingleScalar()
|
| H A D | VPlanUtils.cpp | 95 m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>( in isUniformAcrossVFsAndUFs()
117 .Case<VPInstruction>([](const auto *VPI) { in isUniformAcrossVFsAndUFs()
|
| H A D | LoopVectorize.cpp | 4079 .Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe, in emitInvalidCostRemarks()
4283 auto *VPI = dyn_cast<VPInstruction>(&R); in selectVectorizationFactor()
4287 case VPInstruction::ActiveLaneMask: in selectVectorizationFactor()
4288 case VPInstruction::ExplicitVectorLength: in selectVectorizationFactor()
6932 auto *VPI = dyn_cast<VPInstruction>(U); in planContainsAdditionalSimplifications()
6934 VPInstruction::FirstOrderRecurrenceSplice; in planContainsAdditionalSimplifications()
7133 static Value *getStartValueFromReductionResult(VPInstruction *RdxResult) { in getStartValueFromReductionResult()
7135 assert(RdxResult->getOpcode() == VPInstruction::ComputeFindIVResult && in getStartValueFromReductionResult()
7153 auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming); in fixReductionScalarResumeWhenVectorizingEpilog()
7155 (EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult && in fixReductionScalarResumeWhenVectorizingEpilog()
[all …]
|
| H A D | VPlan.h | 947 class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
1055 VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL = {},
1060 VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
1066 VPInstruction *clone() override { in VP_CLASSOF_IMPL()
1068 auto *New = new VPInstruction(Opcode, Operands, *this, getDebugLoc(), Name); in VP_CLASSOF_IMPL()
1108 case VPInstruction::BranchOnCond: in hasResult()
1109 case VPInstruction::BranchOnCount: in hasResult()
1142 class VPInstructionWithType : public VPInstruction {
1150 : VPInstruction(Opcode, Operands, Flags, DL, Name), ResultTy(ResultTy) {} in VPInstruction() function
1157 auto *VPI = dyn_cast<VPInstruction>(R); in classof()
[all …]
|
| H A D | VPlan.cpp | 267 auto *BuildVector = cast<VPInstruction>(Def); in get()
653 bool IsSwitch = isa<VPInstruction>(R) && in hasConditionalTerminator()
654 cast<VPInstruction>(R)->getOpcode() == Instruction::Switch; in hasConditionalTerminator()
1063 bool NeedsScalar = isa<VPInstruction>(PhiR) || in execute()
1507 auto *VPI = dyn_cast_or_null<VPInstruction>(V->getDefiningRecipe()); in assignName()
|