xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/ExpandVectorPredication.cpp (revision 924226fba12cc9a228c73b956e1b7fa24c60b055)
1 //===----- CodeGen/ExpandVectorPredication.cpp - Expand VP intrinsics -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements IR expansion for vector predication intrinsics, allowing
10 // targets to enable vector predication until just before codegen.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/ExpandVectorPredication.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/Analysis/TargetTransformInfo.h"
17 #include "llvm/Analysis/ValueTracking.h"
18 #include "llvm/CodeGen/Passes.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/IRBuilder.h"
22 #include "llvm/IR/InstIterator.h"
23 #include "llvm/IR/Instructions.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/Intrinsics.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/InitializePasses.h"
28 #include "llvm/Pass.h"
29 #include "llvm/Support/CommandLine.h"
30 #include "llvm/Support/Compiler.h"
31 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/MathExtras.h"
33 
34 using namespace llvm;
35 
36 using VPLegalization = TargetTransformInfo::VPLegalization;
37 using VPTransform = TargetTransformInfo::VPLegalization::VPTransform;
38 
39 // Keep this in sync with TargetTransformInfo::VPLegalization.
40 #define VPINTERNAL_VPLEGAL_CASES                                               \
41   VPINTERNAL_CASE(Legal)                                                       \
42   VPINTERNAL_CASE(Discard)                                                     \
43   VPINTERNAL_CASE(Convert)
44 
45 #define VPINTERNAL_CASE(X) "|" #X
46 
47 // Override options.
48 static cl::opt<std::string> EVLTransformOverride(
49     "expandvp-override-evl-transform", cl::init(""), cl::Hidden,
50     cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
51              ". If non-empty, ignore "
52              "TargetTransformInfo and "
53              "always use this transformation for the %evl parameter (Used in "
54              "testing)."));
55 
56 static cl::opt<std::string> MaskTransformOverride(
57     "expandvp-override-mask-transform", cl::init(""), cl::Hidden,
58     cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
59              ". If non-empty, Ignore "
60              "TargetTransformInfo and "
61              "always use this transformation for the %mask parameter (Used in "
62              "testing)."));
63 
64 #undef VPINTERNAL_CASE
65 #define VPINTERNAL_CASE(X) .Case(#X, VPLegalization::X)
66 
67 static VPTransform parseOverrideOption(const std::string &TextOpt) {
68   return StringSwitch<VPTransform>(TextOpt) VPINTERNAL_VPLEGAL_CASES;
69 }
70 
71 #undef VPINTERNAL_VPLEGAL_CASES
72 
73 // Whether any override options are set.
74 static bool anyExpandVPOverridesSet() {
75   return !EVLTransformOverride.empty() || !MaskTransformOverride.empty();
76 }
77 
78 #define DEBUG_TYPE "expandvp"
79 
80 STATISTIC(NumFoldedVL, "Number of folded vector length params");
81 STATISTIC(NumLoweredVPOps, "Number of folded vector predication operations");
82 
83 ///// Helpers {
84 
85 /// \returns Whether the vector mask \p MaskVal has all lane bits set.
86 static bool isAllTrueMask(Value *MaskVal) {
87   auto *ConstVec = dyn_cast<ConstantVector>(MaskVal);
88   return ConstVec && ConstVec->isAllOnesValue();
89 }
90 
91 /// \returns A non-excepting divisor constant for this type.
92 static Constant *getSafeDivisor(Type *DivTy) {
93   assert(DivTy->isIntOrIntVectorTy() && "Unsupported divisor type");
94   return ConstantInt::get(DivTy, 1u, false);
95 }
96 
97 /// Transfer operation properties from \p OldVPI to \p NewVal.
98 static void transferDecorations(Value &NewVal, VPIntrinsic &VPI) {
99   auto *NewInst = dyn_cast<Instruction>(&NewVal);
100   if (!NewInst || !isa<FPMathOperator>(NewVal))
101     return;
102 
103   auto *OldFMOp = dyn_cast<FPMathOperator>(&VPI);
104   if (!OldFMOp)
105     return;
106 
107   NewInst->setFastMathFlags(OldFMOp->getFastMathFlags());
108 }
109 
110 /// Transfer all properties from \p OldOp to \p NewOp and replace all uses.
111 /// OldVP gets erased.
112 static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp) {
113   transferDecorations(NewOp, OldOp);
114   OldOp.replaceAllUsesWith(&NewOp);
115   OldOp.eraseFromParent();
116 }
117 
118 //// } Helpers
119 
120 namespace {
121 
122 // Expansion pass state at function scope.
123 struct CachingVPExpander {
124   Function &F;
125   const TargetTransformInfo &TTI;
126 
127   /// \returns A (fixed length) vector with ascending integer indices
128   /// (<0, 1, ..., NumElems-1>).
129   /// \p Builder
130   ///    Used for instruction creation.
131   /// \p LaneTy
132   ///    Integer element type of the result vector.
133   /// \p NumElems
134   ///    Number of vector elements.
135   Value *createStepVector(IRBuilder<> &Builder, Type *LaneTy,
136                           unsigned NumElems);
137 
138   /// \returns A bitmask that is true where the lane position is less-than \p
139   /// EVLParam
140   ///
141   /// \p Builder
142   ///    Used for instruction creation.
143   /// \p VLParam
144   ///    The explicit vector length parameter to test against the lane
145   ///    positions.
146   /// \p ElemCount
147   ///    Static (potentially scalable) number of vector elements.
148   Value *convertEVLToMask(IRBuilder<> &Builder, Value *EVLParam,
149                           ElementCount ElemCount);
150 
151   Value *foldEVLIntoMask(VPIntrinsic &VPI);
152 
153   /// "Remove" the %evl parameter of \p PI by setting it to the static vector
154   /// length of the operation.
155   void discardEVLParameter(VPIntrinsic &PI);
156 
157   /// \brief Lower this VP binary operator to a unpredicated binary operator.
158   Value *expandPredicationInBinaryOperator(IRBuilder<> &Builder,
159                                            VPIntrinsic &PI);
160 
161   /// \brief Lower this VP reduction to a call to an unpredicated reduction
162   /// intrinsic.
163   Value *expandPredicationInReduction(IRBuilder<> &Builder,
164                                       VPReductionIntrinsic &PI);
165 
166   /// \brief Query TTI and expand the vector predication in \p P accordingly.
167   Value *expandPredication(VPIntrinsic &PI);
168 
169   /// \brief  Determine how and whether the VPIntrinsic \p VPI shall be
170   /// expanded. This overrides TTI with the cl::opts listed at the top of this
171   /// file.
172   VPLegalization getVPLegalizationStrategy(const VPIntrinsic &VPI) const;
173   bool UsingTTIOverrides;
174 
175 public:
176   CachingVPExpander(Function &F, const TargetTransformInfo &TTI)
177       : F(F), TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {}
178 
179   bool expandVectorPredication();
180 };
181 
182 //// CachingVPExpander {
183 
184 Value *CachingVPExpander::createStepVector(IRBuilder<> &Builder, Type *LaneTy,
185                                            unsigned NumElems) {
186   // TODO add caching
187   SmallVector<Constant *, 16> ConstElems;
188 
189   for (unsigned Idx = 0; Idx < NumElems; ++Idx)
190     ConstElems.push_back(ConstantInt::get(LaneTy, Idx, false));
191 
192   return ConstantVector::get(ConstElems);
193 }
194 
195 Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder,
196                                            Value *EVLParam,
197                                            ElementCount ElemCount) {
198   // TODO add caching
199   // Scalable vector %evl conversion.
200   if (ElemCount.isScalable()) {
201     auto *M = Builder.GetInsertBlock()->getModule();
202     Type *BoolVecTy = VectorType::get(Builder.getInt1Ty(), ElemCount);
203     Function *ActiveMaskFunc = Intrinsic::getDeclaration(
204         M, Intrinsic::get_active_lane_mask, {BoolVecTy, EVLParam->getType()});
205     // `get_active_lane_mask` performs an implicit less-than comparison.
206     Value *ConstZero = Builder.getInt32(0);
207     return Builder.CreateCall(ActiveMaskFunc, {ConstZero, EVLParam});
208   }
209 
210   // Fixed vector %evl conversion.
211   Type *LaneTy = EVLParam->getType();
212   unsigned NumElems = ElemCount.getFixedValue();
213   Value *VLSplat = Builder.CreateVectorSplat(NumElems, EVLParam);
214   Value *IdxVec = createStepVector(Builder, LaneTy, NumElems);
215   return Builder.CreateICmp(CmpInst::ICMP_ULT, IdxVec, VLSplat);
216 }
217 
218 Value *
219 CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder,
220                                                      VPIntrinsic &VPI) {
221   assert((isSafeToSpeculativelyExecute(&VPI) ||
222           VPI.canIgnoreVectorLengthParam()) &&
223          "Implicitly dropping %evl in non-speculatable operator!");
224 
225   auto OC = static_cast<Instruction::BinaryOps>(*VPI.getFunctionalOpcode());
226   assert(Instruction::isBinaryOp(OC));
227 
228   Value *Op0 = VPI.getOperand(0);
229   Value *Op1 = VPI.getOperand(1);
230   Value *Mask = VPI.getMaskParam();
231 
232   // Blend in safe operands.
233   if (Mask && !isAllTrueMask(Mask)) {
234     switch (OC) {
235     default:
236       // Can safely ignore the predicate.
237       break;
238 
239     // Division operators need a safe divisor on masked-off lanes (1).
240     case Instruction::UDiv:
241     case Instruction::SDiv:
242     case Instruction::URem:
243     case Instruction::SRem:
244       // 2nd operand must not be zero.
245       Value *SafeDivisor = getSafeDivisor(VPI.getType());
246       Op1 = Builder.CreateSelect(Mask, Op1, SafeDivisor);
247     }
248   }
249 
250   Value *NewBinOp = Builder.CreateBinOp(OC, Op0, Op1, VPI.getName());
251 
252   replaceOperation(*NewBinOp, VPI);
253   return NewBinOp;
254 }
255 
256 static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
257                                          Type *EltTy) {
258   bool Negative = false;
259   unsigned EltBits = EltTy->getScalarSizeInBits();
260   switch (VPI.getIntrinsicID()) {
261   default:
262     llvm_unreachable("Expecting a VP reduction intrinsic");
263   case Intrinsic::vp_reduce_add:
264   case Intrinsic::vp_reduce_or:
265   case Intrinsic::vp_reduce_xor:
266   case Intrinsic::vp_reduce_umax:
267     return Constant::getNullValue(EltTy);
268   case Intrinsic::vp_reduce_mul:
269     return ConstantInt::get(EltTy, 1, /*IsSigned*/ false);
270   case Intrinsic::vp_reduce_and:
271   case Intrinsic::vp_reduce_umin:
272     return ConstantInt::getAllOnesValue(EltTy);
273   case Intrinsic::vp_reduce_smin:
274     return ConstantInt::get(EltTy->getContext(),
275                             APInt::getSignedMaxValue(EltBits));
276   case Intrinsic::vp_reduce_smax:
277     return ConstantInt::get(EltTy->getContext(),
278                             APInt::getSignedMinValue(EltBits));
279   case Intrinsic::vp_reduce_fmax:
280     Negative = true;
281     LLVM_FALLTHROUGH;
282   case Intrinsic::vp_reduce_fmin: {
283     FastMathFlags Flags = VPI.getFastMathFlags();
284     const fltSemantics &Semantics = EltTy->getFltSemantics();
285     return !Flags.noNaNs() ? ConstantFP::getQNaN(EltTy, Negative)
286            : !Flags.noInfs()
287                ? ConstantFP::getInfinity(EltTy, Negative)
288                : ConstantFP::get(EltTy,
289                                  APFloat::getLargest(Semantics, Negative));
290   }
291   case Intrinsic::vp_reduce_fadd:
292     return ConstantFP::getNegativeZero(EltTy);
293   case Intrinsic::vp_reduce_fmul:
294     return ConstantFP::get(EltTy, 1.0);
295   }
296 }
297 
298 Value *
299 CachingVPExpander::expandPredicationInReduction(IRBuilder<> &Builder,
300                                                 VPReductionIntrinsic &VPI) {
301   assert((isSafeToSpeculativelyExecute(&VPI) ||
302           VPI.canIgnoreVectorLengthParam()) &&
303          "Implicitly dropping %evl in non-speculatable operator!");
304 
305   Value *Mask = VPI.getMaskParam();
306   Value *RedOp = VPI.getOperand(VPI.getVectorParamPos());
307 
308   // Insert neutral element in masked-out positions
309   if (Mask && !isAllTrueMask(Mask)) {
310     auto *NeutralElt = getNeutralReductionElement(VPI, VPI.getType());
311     auto *NeutralVector = Builder.CreateVectorSplat(
312         cast<VectorType>(RedOp->getType())->getElementCount(), NeutralElt);
313     RedOp = Builder.CreateSelect(Mask, RedOp, NeutralVector);
314   }
315 
316   Value *Reduction;
317   Value *Start = VPI.getOperand(VPI.getStartParamPos());
318 
319   switch (VPI.getIntrinsicID()) {
320   default:
321     llvm_unreachable("Impossible reduction kind");
322   case Intrinsic::vp_reduce_add:
323     Reduction = Builder.CreateAddReduce(RedOp);
324     Reduction = Builder.CreateAdd(Reduction, Start);
325     break;
326   case Intrinsic::vp_reduce_mul:
327     Reduction = Builder.CreateMulReduce(RedOp);
328     Reduction = Builder.CreateMul(Reduction, Start);
329     break;
330   case Intrinsic::vp_reduce_and:
331     Reduction = Builder.CreateAndReduce(RedOp);
332     Reduction = Builder.CreateAnd(Reduction, Start);
333     break;
334   case Intrinsic::vp_reduce_or:
335     Reduction = Builder.CreateOrReduce(RedOp);
336     Reduction = Builder.CreateOr(Reduction, Start);
337     break;
338   case Intrinsic::vp_reduce_xor:
339     Reduction = Builder.CreateXorReduce(RedOp);
340     Reduction = Builder.CreateXor(Reduction, Start);
341     break;
342   case Intrinsic::vp_reduce_smax:
343     Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ true);
344     Reduction =
345         Builder.CreateBinaryIntrinsic(Intrinsic::smax, Reduction, Start);
346     break;
347   case Intrinsic::vp_reduce_smin:
348     Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ true);
349     Reduction =
350         Builder.CreateBinaryIntrinsic(Intrinsic::smin, Reduction, Start);
351     break;
352   case Intrinsic::vp_reduce_umax:
353     Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ false);
354     Reduction =
355         Builder.CreateBinaryIntrinsic(Intrinsic::umax, Reduction, Start);
356     break;
357   case Intrinsic::vp_reduce_umin:
358     Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ false);
359     Reduction =
360         Builder.CreateBinaryIntrinsic(Intrinsic::umin, Reduction, Start);
361     break;
362   case Intrinsic::vp_reduce_fmax:
363     Reduction = Builder.CreateFPMaxReduce(RedOp);
364     transferDecorations(*Reduction, VPI);
365     Reduction =
366         Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, Reduction, Start);
367     break;
368   case Intrinsic::vp_reduce_fmin:
369     Reduction = Builder.CreateFPMinReduce(RedOp);
370     transferDecorations(*Reduction, VPI);
371     Reduction =
372         Builder.CreateBinaryIntrinsic(Intrinsic::minnum, Reduction, Start);
373     break;
374   case Intrinsic::vp_reduce_fadd:
375     Reduction = Builder.CreateFAddReduce(Start, RedOp);
376     break;
377   case Intrinsic::vp_reduce_fmul:
378     Reduction = Builder.CreateFMulReduce(Start, RedOp);
379     break;
380   }
381 
382   replaceOperation(*Reduction, VPI);
383   return Reduction;
384 }
385 
386 void CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
387   LLVM_DEBUG(dbgs() << "Discard EVL parameter in " << VPI << "\n");
388 
389   if (VPI.canIgnoreVectorLengthParam())
390     return;
391 
392   Value *EVLParam = VPI.getVectorLengthParam();
393   if (!EVLParam)
394     return;
395 
396   ElementCount StaticElemCount = VPI.getStaticVectorLength();
397   Value *MaxEVL = nullptr;
398   Type *Int32Ty = Type::getInt32Ty(VPI.getContext());
399   if (StaticElemCount.isScalable()) {
400     // TODO add caching
401     auto *M = VPI.getModule();
402     Function *VScaleFunc =
403         Intrinsic::getDeclaration(M, Intrinsic::vscale, Int32Ty);
404     IRBuilder<> Builder(VPI.getParent(), VPI.getIterator());
405     Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue());
406     Value *VScale = Builder.CreateCall(VScaleFunc, {}, "vscale");
407     MaxEVL = Builder.CreateMul(VScale, FactorConst, "scalable_size",
408                                /*NUW*/ true, /*NSW*/ false);
409   } else {
410     MaxEVL = ConstantInt::get(Int32Ty, StaticElemCount.getFixedValue(), false);
411   }
412   VPI.setVectorLengthParam(MaxEVL);
413 }
414 
415 Value *CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) {
416   LLVM_DEBUG(dbgs() << "Folding vlen for " << VPI << '\n');
417 
418   IRBuilder<> Builder(&VPI);
419 
420   // Ineffective %evl parameter and so nothing to do here.
421   if (VPI.canIgnoreVectorLengthParam())
422     return &VPI;
423 
424   // Only VP intrinsics can have an %evl parameter.
425   Value *OldMaskParam = VPI.getMaskParam();
426   Value *OldEVLParam = VPI.getVectorLengthParam();
427   assert(OldMaskParam && "no mask param to fold the vl param into");
428   assert(OldEVLParam && "no EVL param to fold away");
429 
430   LLVM_DEBUG(dbgs() << "OLD evl: " << *OldEVLParam << '\n');
431   LLVM_DEBUG(dbgs() << "OLD mask: " << *OldMaskParam << '\n');
432 
433   // Convert the %evl predication into vector mask predication.
434   ElementCount ElemCount = VPI.getStaticVectorLength();
435   Value *VLMask = convertEVLToMask(Builder, OldEVLParam, ElemCount);
436   Value *NewMaskParam = Builder.CreateAnd(VLMask, OldMaskParam);
437   VPI.setMaskParam(NewMaskParam);
438 
439   // Drop the %evl parameter.
440   discardEVLParameter(VPI);
441   assert(VPI.canIgnoreVectorLengthParam() &&
442          "transformation did not render the evl param ineffective!");
443 
444   // Reassess the modified instruction.
445   return &VPI;
446 }
447 
448 Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
449   LLVM_DEBUG(dbgs() << "Lowering to unpredicated op: " << VPI << '\n');
450 
451   IRBuilder<> Builder(&VPI);
452 
453   // Try lowering to a LLVM instruction first.
454   auto OC = VPI.getFunctionalOpcode();
455 
456   if (OC && Instruction::isBinaryOp(*OC))
457     return expandPredicationInBinaryOperator(Builder, VPI);
458 
459   if (auto *VPRI = dyn_cast<VPReductionIntrinsic>(&VPI))
460     return expandPredicationInReduction(Builder, *VPRI);
461 
462   return &VPI;
463 }
464 
465 //// } CachingVPExpander
466 
467 struct TransformJob {
468   VPIntrinsic *PI;
469   TargetTransformInfo::VPLegalization Strategy;
470   TransformJob(VPIntrinsic *PI, TargetTransformInfo::VPLegalization InitStrat)
471       : PI(PI), Strategy(InitStrat) {}
472 
473   bool isDone() const { return Strategy.shouldDoNothing(); }
474 };
475 
476 void sanitizeStrategy(Instruction &I, VPLegalization &LegalizeStrat) {
477   // Speculatable instructions do not strictly need predication.
478   if (isSafeToSpeculativelyExecute(&I)) {
479     // Converting a speculatable VP intrinsic means dropping %mask and %evl.
480     // No need to expand %evl into the %mask only to ignore that code.
481     if (LegalizeStrat.OpStrategy == VPLegalization::Convert)
482       LegalizeStrat.EVLParamStrategy = VPLegalization::Discard;
483     return;
484   }
485 
486   // We have to preserve the predicating effect of %evl for this
487   // non-speculatable VP intrinsic.
488   // 1) Never discard %evl.
489   // 2) If this VP intrinsic will be expanded to non-VP code, make sure that
490   //    %evl gets folded into %mask.
491   if ((LegalizeStrat.EVLParamStrategy == VPLegalization::Discard) ||
492       (LegalizeStrat.OpStrategy == VPLegalization::Convert)) {
493     LegalizeStrat.EVLParamStrategy = VPLegalization::Convert;
494   }
495 }
496 
497 VPLegalization
498 CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
499   auto VPStrat = TTI.getVPLegalizationStrategy(VPI);
500   if (LLVM_LIKELY(!UsingTTIOverrides)) {
501     // No overrides - we are in production.
502     return VPStrat;
503   }
504 
505   // Overrides set - we are in testing, the following does not need to be
506   // efficient.
507   VPStrat.EVLParamStrategy = parseOverrideOption(EVLTransformOverride);
508   VPStrat.OpStrategy = parseOverrideOption(MaskTransformOverride);
509   return VPStrat;
510 }
511 
512 /// \brief Expand llvm.vp.* intrinsics as requested by \p TTI.
513 bool CachingVPExpander::expandVectorPredication() {
514   SmallVector<TransformJob, 16> Worklist;
515 
516   // Collect all VPIntrinsics that need expansion and determine their expansion
517   // strategy.
518   for (auto &I : instructions(F)) {
519     auto *VPI = dyn_cast<VPIntrinsic>(&I);
520     if (!VPI)
521       continue;
522     auto VPStrat = getVPLegalizationStrategy(*VPI);
523     sanitizeStrategy(I, VPStrat);
524     if (!VPStrat.shouldDoNothing())
525       Worklist.emplace_back(VPI, VPStrat);
526   }
527   if (Worklist.empty())
528     return false;
529 
530   // Transform all VPIntrinsics on the worklist.
531   LLVM_DEBUG(dbgs() << "\n:::: Transforming " << Worklist.size()
532                     << " instructions ::::\n");
533   for (TransformJob Job : Worklist) {
534     // Transform the EVL parameter.
535     switch (Job.Strategy.EVLParamStrategy) {
536     case VPLegalization::Legal:
537       break;
538     case VPLegalization::Discard:
539       discardEVLParameter(*Job.PI);
540       break;
541     case VPLegalization::Convert:
542       if (foldEVLIntoMask(*Job.PI))
543         ++NumFoldedVL;
544       break;
545     }
546     Job.Strategy.EVLParamStrategy = VPLegalization::Legal;
547 
548     // Replace with a non-predicated operation.
549     switch (Job.Strategy.OpStrategy) {
550     case VPLegalization::Legal:
551       break;
552     case VPLegalization::Discard:
553       llvm_unreachable("Invalid strategy for operators.");
554     case VPLegalization::Convert:
555       expandPredication(*Job.PI);
556       ++NumLoweredVPOps;
557       break;
558     }
559     Job.Strategy.OpStrategy = VPLegalization::Legal;
560 
561     assert(Job.isDone() && "incomplete transformation");
562   }
563 
564   return true;
565 }
566 class ExpandVectorPredication : public FunctionPass {
567 public:
568   static char ID;
569   ExpandVectorPredication() : FunctionPass(ID) {
570     initializeExpandVectorPredicationPass(*PassRegistry::getPassRegistry());
571   }
572 
573   bool runOnFunction(Function &F) override {
574     const auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
575     CachingVPExpander VPExpander(F, *TTI);
576     return VPExpander.expandVectorPredication();
577   }
578 
579   void getAnalysisUsage(AnalysisUsage &AU) const override {
580     AU.addRequired<TargetTransformInfoWrapperPass>();
581     AU.setPreservesCFG();
582   }
583 };
584 } // namespace
585 
586 char ExpandVectorPredication::ID;
587 INITIALIZE_PASS_BEGIN(ExpandVectorPredication, "expandvp",
588                       "Expand vector predication intrinsics", false, false)
589 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
590 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
591 INITIALIZE_PASS_END(ExpandVectorPredication, "expandvp",
592                     "Expand vector predication intrinsics", false, false)
593 
594 FunctionPass *llvm::createExpandVectorPredicationPass() {
595   return new ExpandVectorPredication();
596 }
597 
598 PreservedAnalyses
599 ExpandVectorPredicationPass::run(Function &F, FunctionAnalysisManager &AM) {
600   const auto &TTI = AM.getResult<TargetIRAnalysis>(F);
601   CachingVPExpander VPExpander(F, TTI);
602   if (!VPExpander.expandVectorPredication())
603     return PreservedAnalyses::all();
604   PreservedAnalyses PA;
605   PA.preserveSet<CFGAnalyses>();
606   return PA;
607 }
608