1 //===----- CodeGen/ExpandVectorPredication.cpp - Expand VP intrinsics -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass implements IR expansion for vector predication intrinsics, allowing 10 // targets to enable vector predication until just before codegen. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/ExpandVectorPredication.h" 15 #include "llvm/ADT/Statistic.h" 16 #include "llvm/Analysis/TargetTransformInfo.h" 17 #include "llvm/Analysis/ValueTracking.h" 18 #include "llvm/CodeGen/Passes.h" 19 #include "llvm/IR/Constants.h" 20 #include "llvm/IR/Function.h" 21 #include "llvm/IR/IRBuilder.h" 22 #include "llvm/IR/InstIterator.h" 23 #include "llvm/IR/Instructions.h" 24 #include "llvm/IR/IntrinsicInst.h" 25 #include "llvm/IR/Intrinsics.h" 26 #include "llvm/IR/Module.h" 27 #include "llvm/InitializePasses.h" 28 #include "llvm/Pass.h" 29 #include "llvm/Support/CommandLine.h" 30 #include "llvm/Support/Compiler.h" 31 #include "llvm/Support/Debug.h" 32 #include "llvm/Support/MathExtras.h" 33 34 using namespace llvm; 35 36 using VPLegalization = TargetTransformInfo::VPLegalization; 37 using VPTransform = TargetTransformInfo::VPLegalization::VPTransform; 38 39 // Keep this in sync with TargetTransformInfo::VPLegalization. 40 #define VPINTERNAL_VPLEGAL_CASES \ 41 VPINTERNAL_CASE(Legal) \ 42 VPINTERNAL_CASE(Discard) \ 43 VPINTERNAL_CASE(Convert) 44 45 #define VPINTERNAL_CASE(X) "|" #X 46 47 // Override options. 48 static cl::opt<std::string> EVLTransformOverride( 49 "expandvp-override-evl-transform", cl::init(""), cl::Hidden, 50 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES 51 ". If non-empty, ignore " 52 "TargetTransformInfo and " 53 "always use this transformation for the %evl parameter (Used in " 54 "testing).")); 55 56 static cl::opt<std::string> MaskTransformOverride( 57 "expandvp-override-mask-transform", cl::init(""), cl::Hidden, 58 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES 59 ". If non-empty, Ignore " 60 "TargetTransformInfo and " 61 "always use this transformation for the %mask parameter (Used in " 62 "testing).")); 63 64 #undef VPINTERNAL_CASE 65 #define VPINTERNAL_CASE(X) .Case(#X, VPLegalization::X) 66 67 static VPTransform parseOverrideOption(const std::string &TextOpt) { 68 return StringSwitch<VPTransform>(TextOpt) VPINTERNAL_VPLEGAL_CASES; 69 } 70 71 #undef VPINTERNAL_VPLEGAL_CASES 72 73 // Whether any override options are set. 74 static bool anyExpandVPOverridesSet() { 75 return !EVLTransformOverride.empty() || !MaskTransformOverride.empty(); 76 } 77 78 #define DEBUG_TYPE "expandvp" 79 80 STATISTIC(NumFoldedVL, "Number of folded vector length params"); 81 STATISTIC(NumLoweredVPOps, "Number of folded vector predication operations"); 82 83 ///// Helpers { 84 85 /// \returns Whether the vector mask \p MaskVal has all lane bits set. 86 static bool isAllTrueMask(Value *MaskVal) { 87 auto *ConstVec = dyn_cast<ConstantVector>(MaskVal); 88 return ConstVec && ConstVec->isAllOnesValue(); 89 } 90 91 /// \returns A non-excepting divisor constant for this type. 92 static Constant *getSafeDivisor(Type *DivTy) { 93 assert(DivTy->isIntOrIntVectorTy() && "Unsupported divisor type"); 94 return ConstantInt::get(DivTy, 1u, false); 95 } 96 97 /// Transfer operation properties from \p OldVPI to \p NewVal. 98 static void transferDecorations(Value &NewVal, VPIntrinsic &VPI) { 99 auto *NewInst = dyn_cast<Instruction>(&NewVal); 100 if (!NewInst || !isa<FPMathOperator>(NewVal)) 101 return; 102 103 auto *OldFMOp = dyn_cast<FPMathOperator>(&VPI); 104 if (!OldFMOp) 105 return; 106 107 NewInst->setFastMathFlags(OldFMOp->getFastMathFlags()); 108 } 109 110 /// Transfer all properties from \p OldOp to \p NewOp and replace all uses. 111 /// OldVP gets erased. 112 static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp) { 113 transferDecorations(NewOp, OldOp); 114 OldOp.replaceAllUsesWith(&NewOp); 115 OldOp.eraseFromParent(); 116 } 117 118 //// } Helpers 119 120 namespace { 121 122 // Expansion pass state at function scope. 123 struct CachingVPExpander { 124 Function &F; 125 const TargetTransformInfo &TTI; 126 127 /// \returns A (fixed length) vector with ascending integer indices 128 /// (<0, 1, ..., NumElems-1>). 129 /// \p Builder 130 /// Used for instruction creation. 131 /// \p LaneTy 132 /// Integer element type of the result vector. 133 /// \p NumElems 134 /// Number of vector elements. 135 Value *createStepVector(IRBuilder<> &Builder, Type *LaneTy, 136 unsigned NumElems); 137 138 /// \returns A bitmask that is true where the lane position is less-than \p 139 /// EVLParam 140 /// 141 /// \p Builder 142 /// Used for instruction creation. 143 /// \p VLParam 144 /// The explicit vector length parameter to test against the lane 145 /// positions. 146 /// \p ElemCount 147 /// Static (potentially scalable) number of vector elements. 148 Value *convertEVLToMask(IRBuilder<> &Builder, Value *EVLParam, 149 ElementCount ElemCount); 150 151 Value *foldEVLIntoMask(VPIntrinsic &VPI); 152 153 /// "Remove" the %evl parameter of \p PI by setting it to the static vector 154 /// length of the operation. 155 void discardEVLParameter(VPIntrinsic &PI); 156 157 /// \brief Lower this VP binary operator to a unpredicated binary operator. 158 Value *expandPredicationInBinaryOperator(IRBuilder<> &Builder, 159 VPIntrinsic &PI); 160 161 /// \brief Lower this VP reduction to a call to an unpredicated reduction 162 /// intrinsic. 163 Value *expandPredicationInReduction(IRBuilder<> &Builder, 164 VPReductionIntrinsic &PI); 165 166 /// \brief Query TTI and expand the vector predication in \p P accordingly. 167 Value *expandPredication(VPIntrinsic &PI); 168 169 /// \brief Determine how and whether the VPIntrinsic \p VPI shall be 170 /// expanded. This overrides TTI with the cl::opts listed at the top of this 171 /// file. 172 VPLegalization getVPLegalizationStrategy(const VPIntrinsic &VPI) const; 173 bool UsingTTIOverrides; 174 175 public: 176 CachingVPExpander(Function &F, const TargetTransformInfo &TTI) 177 : F(F), TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {} 178 179 bool expandVectorPredication(); 180 }; 181 182 //// CachingVPExpander { 183 184 Value *CachingVPExpander::createStepVector(IRBuilder<> &Builder, Type *LaneTy, 185 unsigned NumElems) { 186 // TODO add caching 187 SmallVector<Constant *, 16> ConstElems; 188 189 for (unsigned Idx = 0; Idx < NumElems; ++Idx) 190 ConstElems.push_back(ConstantInt::get(LaneTy, Idx, false)); 191 192 return ConstantVector::get(ConstElems); 193 } 194 195 Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder, 196 Value *EVLParam, 197 ElementCount ElemCount) { 198 // TODO add caching 199 // Scalable vector %evl conversion. 200 if (ElemCount.isScalable()) { 201 auto *M = Builder.GetInsertBlock()->getModule(); 202 Type *BoolVecTy = VectorType::get(Builder.getInt1Ty(), ElemCount); 203 Function *ActiveMaskFunc = Intrinsic::getDeclaration( 204 M, Intrinsic::get_active_lane_mask, {BoolVecTy, EVLParam->getType()}); 205 // `get_active_lane_mask` performs an implicit less-than comparison. 206 Value *ConstZero = Builder.getInt32(0); 207 return Builder.CreateCall(ActiveMaskFunc, {ConstZero, EVLParam}); 208 } 209 210 // Fixed vector %evl conversion. 211 Type *LaneTy = EVLParam->getType(); 212 unsigned NumElems = ElemCount.getFixedValue(); 213 Value *VLSplat = Builder.CreateVectorSplat(NumElems, EVLParam); 214 Value *IdxVec = createStepVector(Builder, LaneTy, NumElems); 215 return Builder.CreateICmp(CmpInst::ICMP_ULT, IdxVec, VLSplat); 216 } 217 218 Value * 219 CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder, 220 VPIntrinsic &VPI) { 221 assert((isSafeToSpeculativelyExecute(&VPI) || 222 VPI.canIgnoreVectorLengthParam()) && 223 "Implicitly dropping %evl in non-speculatable operator!"); 224 225 auto OC = static_cast<Instruction::BinaryOps>(*VPI.getFunctionalOpcode()); 226 assert(Instruction::isBinaryOp(OC)); 227 228 Value *Op0 = VPI.getOperand(0); 229 Value *Op1 = VPI.getOperand(1); 230 Value *Mask = VPI.getMaskParam(); 231 232 // Blend in safe operands. 233 if (Mask && !isAllTrueMask(Mask)) { 234 switch (OC) { 235 default: 236 // Can safely ignore the predicate. 237 break; 238 239 // Division operators need a safe divisor on masked-off lanes (1). 240 case Instruction::UDiv: 241 case Instruction::SDiv: 242 case Instruction::URem: 243 case Instruction::SRem: 244 // 2nd operand must not be zero. 245 Value *SafeDivisor = getSafeDivisor(VPI.getType()); 246 Op1 = Builder.CreateSelect(Mask, Op1, SafeDivisor); 247 } 248 } 249 250 Value *NewBinOp = Builder.CreateBinOp(OC, Op0, Op1, VPI.getName()); 251 252 replaceOperation(*NewBinOp, VPI); 253 return NewBinOp; 254 } 255 256 static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI, 257 Type *EltTy) { 258 bool Negative = false; 259 unsigned EltBits = EltTy->getScalarSizeInBits(); 260 switch (VPI.getIntrinsicID()) { 261 default: 262 llvm_unreachable("Expecting a VP reduction intrinsic"); 263 case Intrinsic::vp_reduce_add: 264 case Intrinsic::vp_reduce_or: 265 case Intrinsic::vp_reduce_xor: 266 case Intrinsic::vp_reduce_umax: 267 return Constant::getNullValue(EltTy); 268 case Intrinsic::vp_reduce_mul: 269 return ConstantInt::get(EltTy, 1, /*IsSigned*/ false); 270 case Intrinsic::vp_reduce_and: 271 case Intrinsic::vp_reduce_umin: 272 return ConstantInt::getAllOnesValue(EltTy); 273 case Intrinsic::vp_reduce_smin: 274 return ConstantInt::get(EltTy->getContext(), 275 APInt::getSignedMaxValue(EltBits)); 276 case Intrinsic::vp_reduce_smax: 277 return ConstantInt::get(EltTy->getContext(), 278 APInt::getSignedMinValue(EltBits)); 279 case Intrinsic::vp_reduce_fmax: 280 Negative = true; 281 LLVM_FALLTHROUGH; 282 case Intrinsic::vp_reduce_fmin: { 283 FastMathFlags Flags = VPI.getFastMathFlags(); 284 const fltSemantics &Semantics = EltTy->getFltSemantics(); 285 return !Flags.noNaNs() ? ConstantFP::getQNaN(EltTy, Negative) 286 : !Flags.noInfs() 287 ? ConstantFP::getInfinity(EltTy, Negative) 288 : ConstantFP::get(EltTy, 289 APFloat::getLargest(Semantics, Negative)); 290 } 291 case Intrinsic::vp_reduce_fadd: 292 return ConstantFP::getNegativeZero(EltTy); 293 case Intrinsic::vp_reduce_fmul: 294 return ConstantFP::get(EltTy, 1.0); 295 } 296 } 297 298 Value * 299 CachingVPExpander::expandPredicationInReduction(IRBuilder<> &Builder, 300 VPReductionIntrinsic &VPI) { 301 assert((isSafeToSpeculativelyExecute(&VPI) || 302 VPI.canIgnoreVectorLengthParam()) && 303 "Implicitly dropping %evl in non-speculatable operator!"); 304 305 Value *Mask = VPI.getMaskParam(); 306 Value *RedOp = VPI.getOperand(VPI.getVectorParamPos()); 307 308 // Insert neutral element in masked-out positions 309 if (Mask && !isAllTrueMask(Mask)) { 310 auto *NeutralElt = getNeutralReductionElement(VPI, VPI.getType()); 311 auto *NeutralVector = Builder.CreateVectorSplat( 312 cast<VectorType>(RedOp->getType())->getElementCount(), NeutralElt); 313 RedOp = Builder.CreateSelect(Mask, RedOp, NeutralVector); 314 } 315 316 Value *Reduction; 317 Value *Start = VPI.getOperand(VPI.getStartParamPos()); 318 319 switch (VPI.getIntrinsicID()) { 320 default: 321 llvm_unreachable("Impossible reduction kind"); 322 case Intrinsic::vp_reduce_add: 323 Reduction = Builder.CreateAddReduce(RedOp); 324 Reduction = Builder.CreateAdd(Reduction, Start); 325 break; 326 case Intrinsic::vp_reduce_mul: 327 Reduction = Builder.CreateMulReduce(RedOp); 328 Reduction = Builder.CreateMul(Reduction, Start); 329 break; 330 case Intrinsic::vp_reduce_and: 331 Reduction = Builder.CreateAndReduce(RedOp); 332 Reduction = Builder.CreateAnd(Reduction, Start); 333 break; 334 case Intrinsic::vp_reduce_or: 335 Reduction = Builder.CreateOrReduce(RedOp); 336 Reduction = Builder.CreateOr(Reduction, Start); 337 break; 338 case Intrinsic::vp_reduce_xor: 339 Reduction = Builder.CreateXorReduce(RedOp); 340 Reduction = Builder.CreateXor(Reduction, Start); 341 break; 342 case Intrinsic::vp_reduce_smax: 343 Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ true); 344 Reduction = 345 Builder.CreateBinaryIntrinsic(Intrinsic::smax, Reduction, Start); 346 break; 347 case Intrinsic::vp_reduce_smin: 348 Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ true); 349 Reduction = 350 Builder.CreateBinaryIntrinsic(Intrinsic::smin, Reduction, Start); 351 break; 352 case Intrinsic::vp_reduce_umax: 353 Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ false); 354 Reduction = 355 Builder.CreateBinaryIntrinsic(Intrinsic::umax, Reduction, Start); 356 break; 357 case Intrinsic::vp_reduce_umin: 358 Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ false); 359 Reduction = 360 Builder.CreateBinaryIntrinsic(Intrinsic::umin, Reduction, Start); 361 break; 362 case Intrinsic::vp_reduce_fmax: 363 Reduction = Builder.CreateFPMaxReduce(RedOp); 364 transferDecorations(*Reduction, VPI); 365 Reduction = 366 Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, Reduction, Start); 367 break; 368 case Intrinsic::vp_reduce_fmin: 369 Reduction = Builder.CreateFPMinReduce(RedOp); 370 transferDecorations(*Reduction, VPI); 371 Reduction = 372 Builder.CreateBinaryIntrinsic(Intrinsic::minnum, Reduction, Start); 373 break; 374 case Intrinsic::vp_reduce_fadd: 375 Reduction = Builder.CreateFAddReduce(Start, RedOp); 376 break; 377 case Intrinsic::vp_reduce_fmul: 378 Reduction = Builder.CreateFMulReduce(Start, RedOp); 379 break; 380 } 381 382 replaceOperation(*Reduction, VPI); 383 return Reduction; 384 } 385 386 void CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) { 387 LLVM_DEBUG(dbgs() << "Discard EVL parameter in " << VPI << "\n"); 388 389 if (VPI.canIgnoreVectorLengthParam()) 390 return; 391 392 Value *EVLParam = VPI.getVectorLengthParam(); 393 if (!EVLParam) 394 return; 395 396 ElementCount StaticElemCount = VPI.getStaticVectorLength(); 397 Value *MaxEVL = nullptr; 398 Type *Int32Ty = Type::getInt32Ty(VPI.getContext()); 399 if (StaticElemCount.isScalable()) { 400 // TODO add caching 401 auto *M = VPI.getModule(); 402 Function *VScaleFunc = 403 Intrinsic::getDeclaration(M, Intrinsic::vscale, Int32Ty); 404 IRBuilder<> Builder(VPI.getParent(), VPI.getIterator()); 405 Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue()); 406 Value *VScale = Builder.CreateCall(VScaleFunc, {}, "vscale"); 407 MaxEVL = Builder.CreateMul(VScale, FactorConst, "scalable_size", 408 /*NUW*/ true, /*NSW*/ false); 409 } else { 410 MaxEVL = ConstantInt::get(Int32Ty, StaticElemCount.getFixedValue(), false); 411 } 412 VPI.setVectorLengthParam(MaxEVL); 413 } 414 415 Value *CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) { 416 LLVM_DEBUG(dbgs() << "Folding vlen for " << VPI << '\n'); 417 418 IRBuilder<> Builder(&VPI); 419 420 // Ineffective %evl parameter and so nothing to do here. 421 if (VPI.canIgnoreVectorLengthParam()) 422 return &VPI; 423 424 // Only VP intrinsics can have an %evl parameter. 425 Value *OldMaskParam = VPI.getMaskParam(); 426 Value *OldEVLParam = VPI.getVectorLengthParam(); 427 assert(OldMaskParam && "no mask param to fold the vl param into"); 428 assert(OldEVLParam && "no EVL param to fold away"); 429 430 LLVM_DEBUG(dbgs() << "OLD evl: " << *OldEVLParam << '\n'); 431 LLVM_DEBUG(dbgs() << "OLD mask: " << *OldMaskParam << '\n'); 432 433 // Convert the %evl predication into vector mask predication. 434 ElementCount ElemCount = VPI.getStaticVectorLength(); 435 Value *VLMask = convertEVLToMask(Builder, OldEVLParam, ElemCount); 436 Value *NewMaskParam = Builder.CreateAnd(VLMask, OldMaskParam); 437 VPI.setMaskParam(NewMaskParam); 438 439 // Drop the %evl parameter. 440 discardEVLParameter(VPI); 441 assert(VPI.canIgnoreVectorLengthParam() && 442 "transformation did not render the evl param ineffective!"); 443 444 // Reassess the modified instruction. 445 return &VPI; 446 } 447 448 Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) { 449 LLVM_DEBUG(dbgs() << "Lowering to unpredicated op: " << VPI << '\n'); 450 451 IRBuilder<> Builder(&VPI); 452 453 // Try lowering to a LLVM instruction first. 454 auto OC = VPI.getFunctionalOpcode(); 455 456 if (OC && Instruction::isBinaryOp(*OC)) 457 return expandPredicationInBinaryOperator(Builder, VPI); 458 459 if (auto *VPRI = dyn_cast<VPReductionIntrinsic>(&VPI)) 460 return expandPredicationInReduction(Builder, *VPRI); 461 462 return &VPI; 463 } 464 465 //// } CachingVPExpander 466 467 struct TransformJob { 468 VPIntrinsic *PI; 469 TargetTransformInfo::VPLegalization Strategy; 470 TransformJob(VPIntrinsic *PI, TargetTransformInfo::VPLegalization InitStrat) 471 : PI(PI), Strategy(InitStrat) {} 472 473 bool isDone() const { return Strategy.shouldDoNothing(); } 474 }; 475 476 void sanitizeStrategy(Instruction &I, VPLegalization &LegalizeStrat) { 477 // Speculatable instructions do not strictly need predication. 478 if (isSafeToSpeculativelyExecute(&I)) { 479 // Converting a speculatable VP intrinsic means dropping %mask and %evl. 480 // No need to expand %evl into the %mask only to ignore that code. 481 if (LegalizeStrat.OpStrategy == VPLegalization::Convert) 482 LegalizeStrat.EVLParamStrategy = VPLegalization::Discard; 483 return; 484 } 485 486 // We have to preserve the predicating effect of %evl for this 487 // non-speculatable VP intrinsic. 488 // 1) Never discard %evl. 489 // 2) If this VP intrinsic will be expanded to non-VP code, make sure that 490 // %evl gets folded into %mask. 491 if ((LegalizeStrat.EVLParamStrategy == VPLegalization::Discard) || 492 (LegalizeStrat.OpStrategy == VPLegalization::Convert)) { 493 LegalizeStrat.EVLParamStrategy = VPLegalization::Convert; 494 } 495 } 496 497 VPLegalization 498 CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const { 499 auto VPStrat = TTI.getVPLegalizationStrategy(VPI); 500 if (LLVM_LIKELY(!UsingTTIOverrides)) { 501 // No overrides - we are in production. 502 return VPStrat; 503 } 504 505 // Overrides set - we are in testing, the following does not need to be 506 // efficient. 507 VPStrat.EVLParamStrategy = parseOverrideOption(EVLTransformOverride); 508 VPStrat.OpStrategy = parseOverrideOption(MaskTransformOverride); 509 return VPStrat; 510 } 511 512 /// \brief Expand llvm.vp.* intrinsics as requested by \p TTI. 513 bool CachingVPExpander::expandVectorPredication() { 514 SmallVector<TransformJob, 16> Worklist; 515 516 // Collect all VPIntrinsics that need expansion and determine their expansion 517 // strategy. 518 for (auto &I : instructions(F)) { 519 auto *VPI = dyn_cast<VPIntrinsic>(&I); 520 if (!VPI) 521 continue; 522 auto VPStrat = getVPLegalizationStrategy(*VPI); 523 sanitizeStrategy(I, VPStrat); 524 if (!VPStrat.shouldDoNothing()) 525 Worklist.emplace_back(VPI, VPStrat); 526 } 527 if (Worklist.empty()) 528 return false; 529 530 // Transform all VPIntrinsics on the worklist. 531 LLVM_DEBUG(dbgs() << "\n:::: Transforming " << Worklist.size() 532 << " instructions ::::\n"); 533 for (TransformJob Job : Worklist) { 534 // Transform the EVL parameter. 535 switch (Job.Strategy.EVLParamStrategy) { 536 case VPLegalization::Legal: 537 break; 538 case VPLegalization::Discard: 539 discardEVLParameter(*Job.PI); 540 break; 541 case VPLegalization::Convert: 542 if (foldEVLIntoMask(*Job.PI)) 543 ++NumFoldedVL; 544 break; 545 } 546 Job.Strategy.EVLParamStrategy = VPLegalization::Legal; 547 548 // Replace with a non-predicated operation. 549 switch (Job.Strategy.OpStrategy) { 550 case VPLegalization::Legal: 551 break; 552 case VPLegalization::Discard: 553 llvm_unreachable("Invalid strategy for operators."); 554 case VPLegalization::Convert: 555 expandPredication(*Job.PI); 556 ++NumLoweredVPOps; 557 break; 558 } 559 Job.Strategy.OpStrategy = VPLegalization::Legal; 560 561 assert(Job.isDone() && "incomplete transformation"); 562 } 563 564 return true; 565 } 566 class ExpandVectorPredication : public FunctionPass { 567 public: 568 static char ID; 569 ExpandVectorPredication() : FunctionPass(ID) { 570 initializeExpandVectorPredicationPass(*PassRegistry::getPassRegistry()); 571 } 572 573 bool runOnFunction(Function &F) override { 574 const auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 575 CachingVPExpander VPExpander(F, *TTI); 576 return VPExpander.expandVectorPredication(); 577 } 578 579 void getAnalysisUsage(AnalysisUsage &AU) const override { 580 AU.addRequired<TargetTransformInfoWrapperPass>(); 581 AU.setPreservesCFG(); 582 } 583 }; 584 } // namespace 585 586 char ExpandVectorPredication::ID; 587 INITIALIZE_PASS_BEGIN(ExpandVectorPredication, "expandvp", 588 "Expand vector predication intrinsics", false, false) 589 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) 590 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 591 INITIALIZE_PASS_END(ExpandVectorPredication, "expandvp", 592 "Expand vector predication intrinsics", false, false) 593 594 FunctionPass *llvm::createExpandVectorPredicationPass() { 595 return new ExpandVectorPredication(); 596 } 597 598 PreservedAnalyses 599 ExpandVectorPredicationPass::run(Function &F, FunctionAnalysisManager &AM) { 600 const auto &TTI = AM.getResult<TargetIRAnalysis>(F); 601 CachingVPExpander VPExpander(F, TTI); 602 if (!VPExpander.expandVectorPredication()) 603 return PreservedAnalyses::all(); 604 PreservedAnalyses PA; 605 PA.preserveSet<CFGAnalyses>(); 606 return PA; 607 } 608