| /freebsd/contrib/llvm-project/llvm/lib/Transforms/Vectorize/ |
| H A D | VPlanPredicator.cpp | 31 DenseMap<std::pair<const VPBasicBlock *, const VPBasicBlock *>,
33 using BlockMaskCacheTy = DenseMap<VPBasicBlock *, VPValue *>;
43 VPValue *createEdgeMask(VPBasicBlock *Src, VPBasicBlock *Dst);
46 VPValue *getBlockInMask(VPBasicBlock *VPBB) const { in getBlockInMask()
52 void setBlockInMask(VPBasicBlock *VPBB, VPValue *Mask) { in setBlockInMask()
61 VPValue *setEdgeMask(const VPBasicBlock *Src, const VPBasicBlock *Dst, in setEdgeMask()
70 VPValue *getEdgeMask(const VPBasicBlock *Src, const VPBasicBlock *Dst) const { in getEdgeMask()
75 void createHeaderMask(VPBasicBlock *HeaderVPBB, bool FoldTail);
79 VPValue *createBlockInMask(VPBasicBlock *VPBB);
82 void convertPhisToBlends(VPBasicBlock *VPBB);
[all …]
|
| H A D | VPlanConstruction.cpp | 49 DenseMap<BasicBlock *, VPBasicBlock *> BB2VPBB;
57 void setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB);
59 VPBasicBlock *getOrCreateVPBB(BasicBlock *BB);
64 void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB);
77 void PlainCFGBuilder::setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB) { in setVPBBPredsFromBB()
111 VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) { in getOrCreateVPBB()
120 VPBasicBlock *VPBB = Plan->createVPBasicBlock(Name); in getOrCreateVPBB()
173 void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, in createVPInstructionsForVPBB()
219 DenseMap<const VPBasicBlock *, VPValue *> VPPredToIncomingValue; in createVPInstructionsForVPBB()
275 VPBasicBlock *VPBB = getOrCreateVPBB(BB); in buildPlainCFG()
[all …]
|
| H A D | LoopVectorizationPlanner.h | 55 VPBasicBlock *BB = nullptr;
56 VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator();
73 VPBuilder(VPBasicBlock *InsertBB) { setInsertPoint(InsertBB); } in VPBuilder()
75 VPBuilder(VPBasicBlock *TheBB, VPBasicBlock::iterator IP) { in VPBuilder()
83 InsertPt = VPBasicBlock::iterator(); in clearInsertionPoint()
86 VPBasicBlock *getInsertBlock() const { return BB; } in getInsertBlock()
87 VPBasicBlock::iterator getInsertPoint() const { return InsertPt; } in getInsertPoint()
98 VPBasicBlock *Block = nullptr;
99 VPBasicBlock::iterator Point;
106 VPInsertPoint(VPBasicBlock *InsertBlock, VPBasicBlock::iterator InsertPoint) in VPInsertPoint()
[all …]
|
| H A D | VPlan.cpp | 68 const VPBasicBlock *Parent = R.getParent(); in operator <<()
160 const VPBasicBlock *VPBlockBase::getEntryBasicBlock() const { in getEntryBasicBlock()
164 return cast<VPBasicBlock>(Block); in getEntryBasicBlock()
167 VPBasicBlock *VPBlockBase::getEntryBasicBlock() { in getEntryBasicBlock()
171 return cast<VPBasicBlock>(Block); in getEntryBasicBlock()
180 const VPBasicBlock *VPBlockBase::getExitingBasicBlock() const { in getExitingBasicBlock()
184 return cast<VPBasicBlock>(Block); in getExitingBasicBlock()
187 VPBasicBlock *VPBlockBase::getExitingBasicBlock() { in getExitingBasicBlock()
191 return cast<VPBasicBlock>(Block); in getExitingBasicBlock()
212 auto *VPBB = dyn_cast<VPBasicBlock>(VPB); in isHeader()
[all …]
|
| H A D | VPlanUnroll.cpp | 64 VPBasicBlock::iterator InsertPtForPhi);
69 VPBasicBlock::iterator InsertPtForPhi);
136 zip(VPBlockUtils::blocksOnly<VPBasicBlock>(PartI), in unrollReplicateRegionByUF()
137 VPBlockUtils::blocksOnly<VPBasicBlock>(Part0))) { in unrollReplicateRegionByUF()
151 VPWidenIntOrFpInductionRecipe *IV, VPBasicBlock::iterator InsertPtForPhi) { in unrollWidenInductionByUF()
152 VPBasicBlock *PH = cast<VPBasicBlock>( in unrollWidenInductionByUF()
203 VPBasicBlock::iterator InsertPtForPhi) { in unrollHeaderPHIByUF()
286 VPBasicBlock &VPBB = *R.getParent(); in unrollRecipeByUF()
342 auto *VPBB = cast<VPBasicBlock>(VPB); in unrollBlock()
407 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) { in unrollByUF()
[all …]
|
| H A D | VPlanVerifier.cpp | 38 bool verifyPhiRecipes(const VPBasicBlock *VPBB);
45 bool verifyVPBasicBlock(const VPBasicBlock *VPBB);
72 bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) { in verifyPhiRecipes()
213 bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { in verifyVPBasicBlock()
250 const VPBasicBlock *IncomingVPBB = Phi->getIncomingBlock(Idx); in verifyVPBasicBlock()
325 auto *VPBB = dyn_cast<VPBasicBlock>(VPB); in verifyBlock()
448 const VPBasicBlock *Entry = dyn_cast<VPBasicBlock>(TopRegion->getEntry()); in verify()
460 const VPBasicBlock *Exiting = dyn_cast<VPBasicBlock>(TopRegion->getExiting()); in verify()
|
| H A D | VPlanTransforms.cpp | 49 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) { in tryToConvertVPInstructionsToVPRecipes()
129 SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList; in sinkScalarOperands()
131 VPBasicBlock *EntryVPBB = VPR->getEntryBasicBlock(); in sinkScalarOperands()
134 VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(EntryVPBB->getSuccessors()[0]); in sinkScalarOperands()
148 VPBasicBlock *SinkTo; in sinkScalarOperands()
214 auto *EntryBB = dyn_cast<VPBasicBlock>(R->getEntry()); in getPredicatedMask()
223 static VPBasicBlock *getPredicatedThenBlock(VPRegionBlock *R) { in getPredicatedThenBlock()
224 auto *EntryBB = cast<VPBasicBlock>(R->getEntry()); in getPredicatedThenBlock()
228 auto *Succ0 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[0]); in getPredicatedThenBlock()
229 auto *Succ1 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[1]); in getPredicatedThenBlock()
[all …]
|
| H A D | VPlanTransforms.h | 199 static void handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
200 VPBasicBlock *EarlyExitVPBB,
201 VPlan &Plan, VPBasicBlock *HeaderVPBB,
202 VPBasicBlock *LatchVPBB,
248 static DenseMap<VPBasicBlock *, VPValue *>
|
| H A D | VPlanSLP.h | 24 class VPBasicBlock; variable
84 const VPBasicBlock &BB;
130 VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB) : IAI(IAI), BB(BB) {} in VPlanSlp()
|
| H A D | VPlan.h | 59 class VPBasicBlock; variable
189 const VPBasicBlock *getEntryBasicBlock() const;
190 VPBasicBlock *getEntryBasicBlock();
195 const VPBasicBlock *getExitingBasicBlock() const;
196 VPBasicBlock *getExitingBasicBlock();
389 : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
392 friend VPBasicBlock; variable
396 VPBasicBlock *Parent = nullptr;
412 VPBasicBlock *getParent() { return Parent; } in getParent()
413 const VPBasicBlock *getParent() const { return Parent; } in getParent()
[all …]
|
| H A D | VPlanHelpers.h | 39 class VPBasicBlock; variable
299 VPBasicBlock *PrevVPBB = nullptr;
311 SmallDenseMap<const VPBasicBlock *, BasicBlock *> VPBB2IRBB;
398 void assignNames(const VPBasicBlock *VPBB);
439 void dumpBasicBlock(const VPBasicBlock *BasicBlock);
|
| H A D | VPRecipeBuilder.h | 73 DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache;
152 DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache, in VPRecipeBuilder() argument
187 VPValue *getBlockInMask(VPBasicBlock *VPBB) const { in getBlockInMask()
|
| H A D | VPlanUtils.cpp | 131 VPBasicBlock *vputils::getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT) { in getFirstLoopHeader()
136 return I == DepthFirst.end() ? nullptr : cast<VPBasicBlock>(*I); in getFirstLoopHeader()
|
| H A D | VPlanAnalysis.cpp | 319 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>( in collectEphemeralRecipesForVPlan()
441 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) { in calculateRegisterUsageForPlan()
|
| H A D | VPlanSLP.cpp | 57 if (VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(Block)) { in visitBlock()
171 VPBasicBlock *Parent = cast<VPInstruction>(Operands[0])->getParent(); in areVectorizable()
|
| H A D | LoopVectorize.cpp | 2314 for (VPRecipeBase &R : cast<VPBasicBlock>(ScalarPH)->phis()) { in introduceCheckBlockInVPlan()
2413 static void replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB) { in replaceVPBBWithIRVPBB()
2671 VPBasicBlock *HeaderVPBB = in fixVectorizedLoop()
2702 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) { in fixNonInductionPHIs()
2712 const VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx); in fixNonInductionPHIs()
4025 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) { in emitInvalidCostRemarks()
4135 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>( in willGenerateVectors()
4280 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>( in selectVectorizationFactor()
6917 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) { in planContainsAdditionalSimplifications()
7236 VPBasicBlock *VectorPH = cast<VPBasicBlock>(BestVPlan.getVectorPreheader()); in executePlan()
[all …]
|
| H A D | VPlanUtils.h | 99 VPBasicBlock *getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT);
|
| H A D | VPlanRecipes.cpp | 215 void VPRecipeBase::insertBefore(VPBasicBlock &BB, in insertBefore()
245 void VPRecipeBase::moveBefore(VPBasicBlock &BB, in moveBefore()
540 static BranchInst *createCondBranch(Value *Cond, VPBasicBlock *VPBB, in createCondBranch()
546 VPBasicBlock *SecondVPSucc = cast<VPBasicBlock>(VPBB->getSuccessors()[1]); in createCondBranch()
|