151   Value *createStepVector(IRBuilder<> &Builder, Type *LaneTy,
164   Value *convertEVLToMask(IRBuilder<> &Builder, Value *EVLParam,
174   Value *expandPredicationInBinaryOperator(IRBuilder<> &Builder,
178   Value *expandPredicationToIntCall(IRBuilder<> &Builder, VPIntrinsic &PI,
182   Value *expandPredicationToFPCall(IRBuilder<> &Builder, VPIntrinsic &PI,
186   Value *expandPredicationInReduction(IRBuilder<> &Builder,
190   Value *expandPredicationToCastIntrinsic(IRBuilder<> &Builder,
194   Value *expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,
198   Value *expandPredicationInComparison(IRBuilder<> &Builder,
218 Value *CachingVPExpander::createStepVector(IRBuilder<> &Builder, Type *LaneTy,  in createStepVector()  argument
229 Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder,  in convertEVLToMask()  argument
235     auto *M = Builder.GetInsertBlock()->getModule();  in convertEVLToMask()
236     Type *BoolVecTy = VectorType::get(Builder.getInt1Ty(), ElemCount);  in convertEVLToMask()
240     Value *ConstZero = Builder.getInt32(0);  in convertEVLToMask()
241     return Builder.CreateCall(ActiveMaskFunc, {ConstZero, EVLParam});  in convertEVLToMask()
247   Value *VLSplat = Builder.CreateVectorSplat(NumElems, EVLParam);  in convertEVLToMask()
248   Value *IdxVec = createStepVector(Builder, LaneTy, NumElems);  in convertEVLToMask()
249   return Builder.CreateICmp(CmpInst::ICMP_ULT, IdxVec, VLSplat);  in convertEVLToMask()
253 CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder,  in expandPredicationInBinaryOperator()  argument
279       Op1 = Builder.CreateSelect(Mask, Op1, SafeDivisor);  in expandPredicationInBinaryOperator()
283   Value *NewBinOp = Builder.CreateBinOp(OC, Op0, Op1, VPI.getName());  in expandPredicationInBinaryOperator()
290     IRBuilder<> &Builder, VPIntrinsic &VPI, unsigned UnpredicatedIntrinsicID) {  in expandPredicationToIntCall()  argument
301     Value *NewOp = Builder.CreateCall(Fn, {Op0, Op1}, VPI.getName());  in expandPredicationToIntCall()
310     Value *NewOp = Builder.CreateCall(Fn, {Op}, VPI.getName());  in expandPredicationToIntCall()
319     IRBuilder<> &Builder, VPIntrinsic &VPI, unsigned UnpredicatedIntrinsicID) {  in expandPredicationToFPCall()  argument
329     Value *NewOp = Builder.CreateCall(Fn, {Op0}, VPI.getName());  in expandPredicationToFPCall()
339     Value *NewOp = Builder.CreateCall(Fn, {Op0, Op1}, VPI.getName());  in expandPredicationToFPCall()
355           Builder.CreateConstrainedFPCall(Fn, {Op0, Op1, Op2}, VPI.getName());  in expandPredicationToFPCall()
357       NewOp = Builder.CreateCall(Fn, {Op0, Op1, Op2}, VPI.getName());  in expandPredicationToFPCall()
415 CachingVPExpander::expandPredicationInReduction(IRBuilder<> &Builder,  in expandPredicationInReduction()  argument
426     auto *NeutralVector = Builder.CreateVectorSplat(  in expandPredicationInReduction()
428     RedOp = Builder.CreateSelect(Mask, RedOp, NeutralVector);  in expandPredicationInReduction()
438     Reduction = Builder.CreateAddReduce(RedOp);  in expandPredicationInReduction()
439     Reduction = Builder.CreateAdd(Reduction, Start);  in expandPredicationInReduction()
442     Reduction = Builder.CreateMulReduce(RedOp);  in expandPredicationInReduction()
443     Reduction = Builder.CreateMul(Reduction, Start);  in expandPredicationInReduction()
446     Reduction = Builder.CreateAndReduce(RedOp);  in expandPredicationInReduction()
447     Reduction = Builder.CreateAnd(Reduction, Start);  in expandPredicationInReduction()
450     Reduction = Builder.CreateOrReduce(RedOp);  in expandPredicationInReduction()
451     Reduction = Builder.CreateOr(Reduction, Start);  in expandPredicationInReduction()
454     Reduction = Builder.CreateXorReduce(RedOp);  in expandPredicationInReduction()
455     Reduction = Builder.CreateXor(Reduction, Start);  in expandPredicationInReduction()
458     Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ true);  in expandPredicationInReduction()
460         Builder.CreateBinaryIntrinsic(Intrinsic::smax, Reduction, Start);  in expandPredicationInReduction()
463     Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ true);  in expandPredicationInReduction()
465         Builder.CreateBinaryIntrinsic(Intrinsic::smin, Reduction, Start);  in expandPredicationInReduction()
468     Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ false);  in expandPredicationInReduction()
470         Builder.CreateBinaryIntrinsic(Intrinsic::umax, Reduction, Start);  in expandPredicationInReduction()
473     Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ false);  in expandPredicationInReduction()
475         Builder.CreateBinaryIntrinsic(Intrinsic::umin, Reduction, Start);  in expandPredicationInReduction()
478     Reduction = Builder.CreateFPMaxReduce(RedOp);  in expandPredicationInReduction()
481         Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, Reduction, Start);  in expandPredicationInReduction()
484     Reduction = Builder.CreateFPMinReduce(RedOp);  in expandPredicationInReduction()
487         Builder.CreateBinaryIntrinsic(Intrinsic::minnum, Reduction, Start);  in expandPredicationInReduction()
490     Reduction = Builder.CreateFPMaximumReduce(RedOp);  in expandPredicationInReduction()
493         Builder.CreateBinaryIntrinsic(Intrinsic::maximum, Reduction, Start);  in expandPredicationInReduction()
496     Reduction = Builder.CreateFPMinimumReduce(RedOp);  in expandPredicationInReduction()
499         Builder.CreateBinaryIntrinsic(Intrinsic::minimum, Reduction, Start);  in expandPredicationInReduction()
502     Reduction = Builder.CreateFAddReduce(Start, RedOp);  in expandPredicationInReduction()
505     Reduction = Builder.CreateFMulReduce(Start, RedOp);  in expandPredicationInReduction()
513 Value *CachingVPExpander::expandPredicationToCastIntrinsic(IRBuilder<> &Builder,  in expandPredicationToCastIntrinsic()  argument
521         Builder.CreateSExt(VPI.getOperand(0), VPI.getType(), VPI.getName());  in expandPredicationToCastIntrinsic()
525         Builder.CreateZExt(VPI.getOperand(0), VPI.getType(), VPI.getName());  in expandPredicationToCastIntrinsic()
529         Builder.CreateTrunc(VPI.getOperand(0), VPI.getType(), VPI.getName());  in expandPredicationToCastIntrinsic()
533         Builder.CreateIntToPtr(VPI.getOperand(0), VPI.getType(), VPI.getName());  in expandPredicationToCastIntrinsic()
537         Builder.CreatePtrToInt(VPI.getOperand(0), VPI.getType(), VPI.getName());  in expandPredicationToCastIntrinsic()
541         Builder.CreateFPToSI(VPI.getOperand(0), VPI.getType(), VPI.getName());  in expandPredicationToCastIntrinsic()
546         Builder.CreateFPToUI(VPI.getOperand(0), VPI.getType(), VPI.getName());  in expandPredicationToCastIntrinsic()
550         Builder.CreateSIToFP(VPI.getOperand(0), VPI.getType(), VPI.getName());  in expandPredicationToCastIntrinsic()
554         Builder.CreateUIToFP(VPI.getOperand(0), VPI.getType(), VPI.getName());  in expandPredicationToCastIntrinsic()
558         Builder.CreateFPTrunc(VPI.getOperand(0), VPI.getType(), VPI.getName());  in expandPredicationToCastIntrinsic()
562         Builder.CreateFPExt(VPI.getOperand(0), VPI.getType(), VPI.getName());  in expandPredicationToCastIntrinsic()
570 CachingVPExpander::expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder,  in expandPredicationInMemoryIntrinsic()  argument
590           Builder.CreateStore(DataParam, PtrParam, /*IsVolatile*/ false);  in expandPredicationInMemoryIntrinsic()
595       NewMemoryInst = Builder.CreateMaskedStore(  in expandPredicationInMemoryIntrinsic()
602           Builder.CreateLoad(VPI.getType(), PtrParam, /*IsVolatile*/ false);  in expandPredicationInMemoryIntrinsic()
607       NewMemoryInst = Builder.CreateMaskedLoad(  in expandPredicationInMemoryIntrinsic()
614     NewMemoryInst = Builder.CreateMaskedScatter(  in expandPredicationInMemoryIntrinsic()
621     NewMemoryInst = Builder.CreateMaskedGather(  in expandPredicationInMemoryIntrinsic()
634 Value *CachingVPExpander::expandPredicationInComparison(IRBuilder<> &Builder,  in expandPredicationInComparison()  argument
646   auto *NewCmp = Builder.CreateCmp(Pred, Op0, Op1);  in expandPredicationInComparison()
670     IRBuilder<> Builder(VPI.getParent(), VPI.getIterator());  in discardEVLParameter()  local
671     Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue());  in discardEVLParameter()
672     Value *VScale = Builder.CreateCall(VScaleFunc, {}, "vscale");  in discardEVLParameter()
673     MaxEVL = Builder.CreateMul(VScale, FactorConst, "scalable_size",  in discardEVLParameter()
684   IRBuilder<> Builder(&VPI);  in foldEVLIntoMask()  local
701   Value *VLMask = convertEVLToMask(Builder, OldEVLParam, ElemCount);  in foldEVLIntoMask()
702   Value *NewMaskParam = Builder.CreateAnd(VLMask, OldMaskParam);  in foldEVLIntoMask()
717   IRBuilder<> Builder(&VPI);  in expandPredication()  local
723     return expandPredicationInBinaryOperator(Builder, VPI);  in expandPredication()
726     return expandPredicationInReduction(Builder, *VPRI);  in expandPredication()
729     return expandPredicationInComparison(Builder, *VPCmp);  in expandPredication()
732     return expandPredicationToCastIntrinsic(Builder, VPI);  in expandPredication()
739     Value *NewNegOp = Builder.CreateFNeg(VPI.getOperand(0), VPI.getName());  in expandPredication()
750     return expandPredicationToIntCall(Builder, VPI,  in expandPredication()
760     return expandPredicationToFPCall(Builder, VPI,  in expandPredication()
766     return expandPredicationInMemoryIntrinsic(Builder, VPI);  in expandPredication()
770     if (Value *Call = expandPredicationToFPCall(Builder, VPI, *CID))  in expandPredication()