1 //===- MVETailPredication.cpp - MVE Tail Predication ------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Armv8.1m introduced MVE, M-Profile Vector Extension, and low-overhead 11 /// branches to help accelerate DSP applications. These two extensions, 12 /// combined with a new form of predication called tail-predication, can be used 13 /// to provide implicit vector predication within a low-overhead loop. 14 /// This is implicit because the predicate of active/inactive lanes is 15 /// calculated by hardware, and thus does not need to be explicitly passed 16 /// to vector instructions. The instructions responsible for this are the 17 /// DLSTP and WLSTP instructions, which setup a tail-predicated loop and the 18 /// the total number of data elements processed by the loop. The loop-end 19 /// LETP instruction is responsible for decrementing and setting the remaining 20 /// elements to be processed and generating the mask of active lanes. 21 /// 22 /// The HardwareLoops pass inserts intrinsics identifying loops that the 23 /// backend will attempt to convert into a low-overhead loop. The vectorizer is 24 /// responsible for generating a vectorized loop in which the lanes are 25 /// predicated upon the iteration counter. This pass looks at these predicated 26 /// vector loops, that are targets for low-overhead loops, and prepares it for 27 /// code generation. Once the vectorizer has produced a masked loop, there's a 28 /// couple of final forms: 29 /// - A tail-predicated loop, with implicit predication. 30 /// - A loop containing multiple VCPT instructions, predicating multiple VPT 31 /// blocks of instructions operating on different vector types. 32 /// 33 /// This pass: 34 /// 1) Checks if the predicates of the masked load/store instructions are 35 /// generated by intrinsic @llvm.get.active.lanes(). This intrinsic consumes 36 /// the Backedge Taken Count (BTC) of the scalar loop as its second argument, 37 /// which we extract to set up the number of elements processed by the loop. 38 /// 2) Intrinsic @llvm.get.active.lanes() is then replaced by the MVE target 39 /// specific VCTP intrinsic to represent the effect of tail predication. 40 /// This will be picked up by the ARM Low-overhead loop pass, which performs 41 /// the final transformation to a DLSTP or WLSTP tail-predicated loop. 42 43 #include "ARM.h" 44 #include "ARMSubtarget.h" 45 #include "ARMTargetTransformInfo.h" 46 #include "llvm/Analysis/LoopInfo.h" 47 #include "llvm/Analysis/LoopPass.h" 48 #include "llvm/Analysis/ScalarEvolution.h" 49 #include "llvm/Analysis/ScalarEvolutionExpressions.h" 50 #include "llvm/Analysis/TargetLibraryInfo.h" 51 #include "llvm/Analysis/TargetTransformInfo.h" 52 #include "llvm/CodeGen/TargetPassConfig.h" 53 #include "llvm/IR/IRBuilder.h" 54 #include "llvm/IR/Instructions.h" 55 #include "llvm/IR/IntrinsicsARM.h" 56 #include "llvm/IR/PatternMatch.h" 57 #include "llvm/InitializePasses.h" 58 #include "llvm/Support/Debug.h" 59 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 60 #include "llvm/Transforms/Utils/LoopUtils.h" 61 #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h" 62 63 using namespace llvm; 64 65 #define DEBUG_TYPE "mve-tail-predication" 66 #define DESC "Transform predicated vector loops to use MVE tail predication" 67 68 cl::opt<TailPredication::Mode> EnableTailPredication( 69 "tail-predication", cl::desc("MVE tail-predication options"), 70 cl::init(TailPredication::Disabled), 71 cl::values(clEnumValN(TailPredication::Disabled, "disabled", 72 "Don't tail-predicate loops"), 73 clEnumValN(TailPredication::EnabledNoReductions, 74 "enabled-no-reductions", 75 "Enable tail-predication, but not for reduction loops"), 76 clEnumValN(TailPredication::Enabled, 77 "enabled", 78 "Enable tail-predication, including reduction loops"), 79 clEnumValN(TailPredication::ForceEnabledNoReductions, 80 "force-enabled-no-reductions", 81 "Enable tail-predication, but not for reduction loops, " 82 "and force this which might be unsafe"), 83 clEnumValN(TailPredication::ForceEnabled, 84 "force-enabled", 85 "Enable tail-predication, including reduction loops, " 86 "and force this which might be unsafe"))); 87 88 89 namespace { 90 91 class MVETailPredication : public LoopPass { 92 SmallVector<IntrinsicInst*, 4> MaskedInsts; 93 Loop *L = nullptr; 94 ScalarEvolution *SE = nullptr; 95 TargetTransformInfo *TTI = nullptr; 96 const ARMSubtarget *ST = nullptr; 97 98 public: 99 static char ID; 100 101 MVETailPredication() : LoopPass(ID) { } 102 103 void getAnalysisUsage(AnalysisUsage &AU) const override { 104 AU.addRequired<ScalarEvolutionWrapperPass>(); 105 AU.addRequired<LoopInfoWrapperPass>(); 106 AU.addRequired<TargetPassConfig>(); 107 AU.addRequired<TargetTransformInfoWrapperPass>(); 108 AU.addPreserved<LoopInfoWrapperPass>(); 109 AU.setPreservesCFG(); 110 } 111 112 bool runOnLoop(Loop *L, LPPassManager&) override; 113 114 private: 115 /// Perform the relevant checks on the loop and convert if possible. 116 bool TryConvert(Value *TripCount); 117 118 /// Return whether this is a vectorized loop, that contains masked 119 /// load/stores. 120 bool IsPredicatedVectorLoop(); 121 122 /// Perform checks on the arguments of @llvm.get.active.lane.mask 123 /// intrinsic: check if the first is a loop induction variable, and for the 124 /// the second check that no overflow can occur in the expression that use 125 /// this backedge-taken count. 126 bool IsSafeActiveMask(IntrinsicInst *ActiveLaneMask, Value *TripCount, 127 FixedVectorType *VecTy); 128 129 /// Insert the intrinsic to represent the effect of tail predication. 130 void InsertVCTPIntrinsic(IntrinsicInst *ActiveLaneMask, Value *TripCount, 131 FixedVectorType *VecTy); 132 133 /// Rematerialize the iteration count in exit blocks, which enables 134 /// ARMLowOverheadLoops to better optimise away loop update statements inside 135 /// hardware-loops. 136 void RematerializeIterCount(); 137 }; 138 139 } // end namespace 140 141 static bool IsDecrement(Instruction &I) { 142 auto *Call = dyn_cast<IntrinsicInst>(&I); 143 if (!Call) 144 return false; 145 146 Intrinsic::ID ID = Call->getIntrinsicID(); 147 return ID == Intrinsic::loop_decrement_reg; 148 } 149 150 static bool IsMasked(Instruction *I) { 151 auto *Call = dyn_cast<IntrinsicInst>(I); 152 if (!Call) 153 return false; 154 155 Intrinsic::ID ID = Call->getIntrinsicID(); 156 // TODO: Support gather/scatter expand/compress operations. 157 return ID == Intrinsic::masked_store || ID == Intrinsic::masked_load; 158 } 159 160 bool MVETailPredication::runOnLoop(Loop *L, LPPassManager&) { 161 if (skipLoop(L) || !EnableTailPredication) 162 return false; 163 164 MaskedInsts.clear(); 165 Function &F = *L->getHeader()->getParent(); 166 auto &TPC = getAnalysis<TargetPassConfig>(); 167 auto &TM = TPC.getTM<TargetMachine>(); 168 ST = &TM.getSubtarget<ARMSubtarget>(F); 169 TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 170 SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE(); 171 this->L = L; 172 173 // The MVE and LOB extensions are combined to enable tail-predication, but 174 // there's nothing preventing us from generating VCTP instructions for v8.1m. 175 if (!ST->hasMVEIntegerOps() || !ST->hasV8_1MMainlineOps()) { 176 LLVM_DEBUG(dbgs() << "ARM TP: Not a v8.1m.main+mve target.\n"); 177 return false; 178 } 179 180 BasicBlock *Preheader = L->getLoopPreheader(); 181 if (!Preheader) 182 return false; 183 184 auto FindLoopIterations = [](BasicBlock *BB) -> IntrinsicInst* { 185 for (auto &I : *BB) { 186 auto *Call = dyn_cast<IntrinsicInst>(&I); 187 if (!Call) 188 continue; 189 190 Intrinsic::ID ID = Call->getIntrinsicID(); 191 if (ID == Intrinsic::set_loop_iterations || 192 ID == Intrinsic::test_set_loop_iterations) 193 return cast<IntrinsicInst>(&I); 194 } 195 return nullptr; 196 }; 197 198 // Look for the hardware loop intrinsic that sets the iteration count. 199 IntrinsicInst *Setup = FindLoopIterations(Preheader); 200 201 // The test.set iteration could live in the pre-preheader. 202 if (!Setup) { 203 if (!Preheader->getSinglePredecessor()) 204 return false; 205 Setup = FindLoopIterations(Preheader->getSinglePredecessor()); 206 if (!Setup) 207 return false; 208 } 209 210 // Search for the hardware loop intrinic that decrements the loop counter. 211 IntrinsicInst *Decrement = nullptr; 212 for (auto *BB : L->getBlocks()) { 213 for (auto &I : *BB) { 214 if (IsDecrement(I)) { 215 Decrement = cast<IntrinsicInst>(&I); 216 break; 217 } 218 } 219 } 220 221 if (!Decrement) 222 return false; 223 224 LLVM_DEBUG(dbgs() << "ARM TP: Running on Loop: " << *L << *Setup << "\n" 225 << *Decrement << "\n"); 226 227 if (!TryConvert(Setup->getArgOperand(0))) { 228 LLVM_DEBUG(dbgs() << "ARM TP: Can't tail-predicate this loop.\n"); 229 return false; 230 } 231 232 return true; 233 } 234 235 static FixedVectorType *getVectorType(IntrinsicInst *I) { 236 unsigned TypeOp = I->getIntrinsicID() == Intrinsic::masked_load ? 0 : 1; 237 auto *PtrTy = cast<PointerType>(I->getOperand(TypeOp)->getType()); 238 auto *VecTy = cast<FixedVectorType>(PtrTy->getElementType()); 239 assert(VecTy && "No scalable vectors expected here"); 240 return VecTy; 241 } 242 243 bool MVETailPredication::IsPredicatedVectorLoop() { 244 // Check that the loop contains at least one masked load/store intrinsic. 245 // We only support 'normal' vector instructions - other than masked 246 // load/stores. 247 bool ActiveLaneMask = false; 248 for (auto *BB : L->getBlocks()) { 249 for (auto &I : *BB) { 250 auto *Int = dyn_cast<IntrinsicInst>(&I); 251 if (!Int) 252 continue; 253 254 switch (Int->getIntrinsicID()) { 255 case Intrinsic::get_active_lane_mask: 256 ActiveLaneMask = true; 257 LLVM_FALLTHROUGH; 258 case Intrinsic::sadd_sat: 259 case Intrinsic::uadd_sat: 260 case Intrinsic::ssub_sat: 261 case Intrinsic::usub_sat: 262 continue; 263 case Intrinsic::fma: 264 case Intrinsic::trunc: 265 case Intrinsic::rint: 266 case Intrinsic::round: 267 case Intrinsic::floor: 268 case Intrinsic::ceil: 269 case Intrinsic::fabs: 270 if (ST->hasMVEFloatOps()) 271 continue; 272 LLVM_FALLTHROUGH; 273 default: 274 break; 275 } 276 277 if (IsMasked(&I)) { 278 auto *VecTy = getVectorType(Int); 279 unsigned Lanes = VecTy->getNumElements(); 280 unsigned ElementWidth = VecTy->getScalarSizeInBits(); 281 // MVE vectors are 128-bit, but don't support 128 x i1. 282 // TODO: Can we support vectors larger than 128-bits? 283 unsigned MaxWidth = TTI->getRegisterBitWidth(true); 284 if (Lanes * ElementWidth > MaxWidth || Lanes == MaxWidth) 285 return false; 286 MaskedInsts.push_back(cast<IntrinsicInst>(&I)); 287 continue; 288 } 289 290 for (const Use &U : Int->args()) { 291 if (isa<VectorType>(U->getType())) 292 return false; 293 } 294 } 295 } 296 297 if (!ActiveLaneMask) { 298 LLVM_DEBUG(dbgs() << "ARM TP: No get.active.lane.mask intrinsic found.\n"); 299 return false; 300 } 301 return !MaskedInsts.empty(); 302 } 303 304 // Look through the exit block to see whether there's a duplicate predicate 305 // instruction. This can happen when we need to perform a select on values 306 // from the last and previous iteration. Instead of doing a straight 307 // replacement of that predicate with the vctp, clone the vctp and place it 308 // in the block. This means that the VPR doesn't have to be live into the 309 // exit block which should make it easier to convert this loop into a proper 310 // tail predicated loop. 311 static void Cleanup(SetVector<Instruction*> &MaybeDead, Loop *L) { 312 BasicBlock *Exit = L->getUniqueExitBlock(); 313 if (!Exit) { 314 LLVM_DEBUG(dbgs() << "ARM TP: can't find loop exit block\n"); 315 return; 316 } 317 318 // Drop references and add operands to check for dead. 319 SmallPtrSet<Instruction*, 4> Dead; 320 while (!MaybeDead.empty()) { 321 auto *I = MaybeDead.front(); 322 MaybeDead.remove(I); 323 if (I->hasNUsesOrMore(1)) 324 continue; 325 326 for (auto &U : I->operands()) 327 if (auto *OpI = dyn_cast<Instruction>(U)) 328 MaybeDead.insert(OpI); 329 330 Dead.insert(I); 331 } 332 333 for (auto *I : Dead) { 334 LLVM_DEBUG(dbgs() << "ARM TP: removing dead insn: "; I->dump()); 335 I->eraseFromParent(); 336 } 337 338 for (auto I : L->blocks()) 339 DeleteDeadPHIs(I); 340 } 341 342 // The active lane intrinsic has this form: 343 // 344 // @llvm.get.active.lane.mask(IV, BTC) 345 // 346 // Here we perform checks that this intrinsic behaves as expected, 347 // which means: 348 // 349 // 1) The element count, which is calculated with BTC + 1, cannot overflow. 350 // 2) The element count needs to be sufficiently large that the decrement of 351 // element counter doesn't overflow, which means that we need to prove: 352 // ceil(ElementCount / VectorWidth) >= TripCount 353 // by rounding up ElementCount up: 354 // ((ElementCount + (VectorWidth - 1)) / VectorWidth 355 // and evaluate if expression isKnownNonNegative: 356 // (((ElementCount + (VectorWidth - 1)) / VectorWidth) - TripCount 357 // 3) The IV must be an induction phi with an increment equal to the 358 // vector width. 359 bool MVETailPredication::IsSafeActiveMask(IntrinsicInst *ActiveLaneMask, 360 Value *TripCount, FixedVectorType *VecTy) { 361 bool ForceTailPredication = 362 EnableTailPredication == TailPredication::ForceEnabledNoReductions || 363 EnableTailPredication == TailPredication::ForceEnabled; 364 // 1) Test whether entry to the loop is protected by a conditional 365 // BTC + 1 < 0. In other words, if the scalar trip count overflows, 366 // becomes negative, we shouldn't enter the loop and creating 367 // tripcount expression BTC + 1 is not safe. So, check that BTC 368 // isn't max. This is evaluated in unsigned, because the semantics 369 // of @get.active.lane.mask is a ULE comparison. 370 371 int VectorWidth = VecTy->getNumElements(); 372 auto *BackedgeTakenCount = ActiveLaneMask->getOperand(1); 373 auto *BTC = SE->getSCEV(BackedgeTakenCount); 374 375 if (!llvm::cannotBeMaxInLoop(BTC, L, *SE, false /*Signed*/) && 376 !ForceTailPredication) { 377 LLVM_DEBUG(dbgs() << "ARM TP: Overflow possible, BTC can be max: "; 378 BTC->dump()); 379 return false; 380 } 381 382 // 2) Prove that the sub expression is non-negative, i.e. it doesn't overflow: 383 // 384 // (((ElementCount + (VectorWidth - 1)) / VectorWidth) - TripCount 385 // 386 // 2.1) First prove overflow can't happen in: 387 // 388 // ElementCount + (VectorWidth - 1) 389 // 390 // Because of a lack of context, it is difficult to get a useful bounds on 391 // this expression. But since ElementCount uses the same variables as the 392 // TripCount (TC), for which we can find meaningful value ranges, we use that 393 // instead and assert that: 394 // 395 // upperbound(TC) <= UINT_MAX - VectorWidth 396 // 397 auto *TC = SE->getSCEV(TripCount); 398 unsigned SizeInBits = TripCount->getType()->getScalarSizeInBits(); 399 auto Diff = APInt(SizeInBits, ~0) - APInt(SizeInBits, VectorWidth); 400 uint64_t MaxMinusVW = Diff.getZExtValue(); 401 uint64_t UpperboundTC = SE->getSignedRange(TC).getUpper().getZExtValue(); 402 403 if (UpperboundTC > MaxMinusVW && !ForceTailPredication) { 404 LLVM_DEBUG(dbgs() << "ARM TP: Overflow possible in tripcount rounding:\n"; 405 dbgs() << "upperbound(TC) <= UINT_MAX - VectorWidth\n"; 406 dbgs() << UpperboundTC << " <= " << MaxMinusVW << "== false\n";); 407 return false; 408 } 409 410 // 2.2) Make sure overflow doesn't happen in final expression: 411 // (((ElementCount + (VectorWidth - 1)) / VectorWidth) - TripCount, 412 // To do this, compare the full ranges of these subexpressions: 413 // 414 // Range(Ceil) <= Range(TC) 415 // 416 // where Ceil = ElementCount + (VW-1) / VW. If Ceil and TC are runtime 417 // values (and not constants), we have to compensate for the lowerbound value 418 // range to be off by 1. The reason is that BTC lives in the preheader in 419 // this form: 420 // 421 // %trip.count.minus = add nsw nuw i32 %N, -1 422 // 423 // For the loop to be executed, %N has to be >= 1 and as a result the value 424 // range of %trip.count.minus has a lower bound of 0. Value %TC has this form: 425 // 426 // %5 = add nuw nsw i32 %4, 1 427 // call void @llvm.set.loop.iterations.i32(i32 %5) 428 // 429 // where %5 is some expression using %N, which needs to have a lower bound of 430 // 1. Thus, if the ranges of Ceil and TC are not a single constant but a set, 431 // we first add 0 to TC such that we can do the <= comparison on both sets. 432 // 433 auto *One = SE->getOne(TripCount->getType()); 434 // ElementCount = BTC + 1 435 auto *ElementCount = SE->getAddExpr(BTC, One); 436 // Tmp = ElementCount + (VW-1) 437 auto *ECPlusVWMinus1 = SE->getAddExpr(ElementCount, 438 SE->getSCEV(ConstantInt::get(TripCount->getType(), VectorWidth - 1))); 439 // Ceil = ElementCount + (VW-1) / VW 440 auto *Ceil = SE->getUDivExpr(ECPlusVWMinus1, 441 SE->getSCEV(ConstantInt::get(TripCount->getType(), VectorWidth))); 442 443 ConstantRange RangeCeil = SE->getSignedRange(Ceil) ; 444 ConstantRange RangeTC = SE->getSignedRange(TC) ; 445 if (!RangeTC.isSingleElement()) { 446 auto ZeroRange = 447 ConstantRange(APInt(TripCount->getType()->getScalarSizeInBits(), 0)); 448 RangeTC = RangeTC.unionWith(ZeroRange); 449 } 450 if (!RangeTC.contains(RangeCeil) && !ForceTailPredication) { 451 LLVM_DEBUG(dbgs() << "ARM TP: Overflow possible in sub\n"); 452 return false; 453 } 454 455 // 3) Find out if IV is an induction phi. Note that We can't use Loop 456 // helpers here to get the induction variable, because the hardware loop is 457 // no longer in loopsimplify form, and also the hwloop intrinsic use a 458 // different counter. Using SCEV, we check that the induction is of the 459 // form i = i + 4, where the increment must be equal to the VectorWidth. 460 auto *IV = ActiveLaneMask->getOperand(0); 461 auto *IVExpr = SE->getSCEV(IV); 462 auto *AddExpr = dyn_cast<SCEVAddRecExpr>(IVExpr); 463 if (!AddExpr) { 464 LLVM_DEBUG(dbgs() << "ARM TP: induction not an add expr: "; IVExpr->dump()); 465 return false; 466 } 467 // Check that this AddRec is associated with this loop. 468 if (AddExpr->getLoop() != L) { 469 LLVM_DEBUG(dbgs() << "ARM TP: phi not part of this loop\n"); 470 return false; 471 } 472 auto *Step = dyn_cast<SCEVConstant>(AddExpr->getOperand(1)); 473 if (!Step) { 474 LLVM_DEBUG(dbgs() << "ARM TP: induction step is not a constant: "; 475 AddExpr->getOperand(1)->dump()); 476 return false; 477 } 478 auto StepValue = Step->getValue()->getSExtValue(); 479 if (VectorWidth == StepValue) 480 return true; 481 482 LLVM_DEBUG(dbgs() << "ARM TP: Step value " << StepValue << " doesn't match " 483 "vector width " << VectorWidth << "\n"); 484 485 return false; 486 } 487 488 // Materialize NumElements in the preheader block. 489 static Value *getNumElements(BasicBlock *Preheader, Value *BTC) { 490 // First, check the preheader if it not already exist: 491 // 492 // preheader: 493 // %BTC = add i32 %N, -1 494 // .. 495 // vector.body: 496 // 497 // if %BTC already exists. We don't need to emit %NumElems = %BTC + 1, 498 // but instead can just return %N. 499 for (auto &I : *Preheader) { 500 if (I.getOpcode() != Instruction::Add || &I != BTC) 501 continue; 502 ConstantInt *MinusOne = nullptr; 503 if (!(MinusOne = dyn_cast<ConstantInt>(I.getOperand(1)))) 504 continue; 505 if (MinusOne->getSExtValue() == -1) { 506 LLVM_DEBUG(dbgs() << "ARM TP: Found num elems: " << I << "\n"); 507 return I.getOperand(0); 508 } 509 } 510 511 // But we do need to materialise BTC if it is not already there, 512 // e.g. if it is a constant. 513 IRBuilder<> Builder(Preheader->getTerminator()); 514 Value *NumElements = Builder.CreateAdd(BTC, 515 ConstantInt::get(BTC->getType(), 1), "num.elements"); 516 LLVM_DEBUG(dbgs() << "ARM TP: Created num elems: " << *NumElements << "\n"); 517 return NumElements; 518 } 519 520 void MVETailPredication::InsertVCTPIntrinsic(IntrinsicInst *ActiveLaneMask, 521 Value *TripCount, FixedVectorType *VecTy) { 522 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator()); 523 Module *M = L->getHeader()->getModule(); 524 Type *Ty = IntegerType::get(M->getContext(), 32); 525 unsigned VectorWidth = VecTy->getNumElements(); 526 527 // The backedge-taken count in @llvm.get.active.lane.mask, its 2nd operand, 528 // is one less than the trip count. So we need to find or create 529 // %num.elements = %BTC + 1 in the preheader. 530 Value *BTC = ActiveLaneMask->getOperand(1); 531 Builder.SetInsertPoint(L->getLoopPreheader()->getTerminator()); 532 Value *NumElements = getNumElements(L->getLoopPreheader(), BTC); 533 534 // Insert a phi to count the number of elements processed by the loop. 535 Builder.SetInsertPoint(L->getHeader()->getFirstNonPHI() ); 536 PHINode *Processed = Builder.CreatePHI(Ty, 2); 537 Processed->addIncoming(NumElements, L->getLoopPreheader()); 538 539 // Replace @llvm.get.active.mask() with the ARM specific VCTP intrinic, and thus 540 // represent the effect of tail predication. 541 Builder.SetInsertPoint(ActiveLaneMask); 542 ConstantInt *Factor = 543 ConstantInt::get(cast<IntegerType>(Ty), VectorWidth); 544 545 Intrinsic::ID VCTPID; 546 switch (VectorWidth) { 547 default: 548 llvm_unreachable("unexpected number of lanes"); 549 case 4: VCTPID = Intrinsic::arm_mve_vctp32; break; 550 case 8: VCTPID = Intrinsic::arm_mve_vctp16; break; 551 case 16: VCTPID = Intrinsic::arm_mve_vctp8; break; 552 553 // FIXME: vctp64 currently not supported because the predicate 554 // vector wants to be <2 x i1>, but v2i1 is not a legal MVE 555 // type, so problems happen at isel time. 556 // Intrinsic::arm_mve_vctp64 exists for ACLE intrinsics 557 // purposes, but takes a v4i1 instead of a v2i1. 558 } 559 Function *VCTP = Intrinsic::getDeclaration(M, VCTPID); 560 Value *VCTPCall = Builder.CreateCall(VCTP, Processed); 561 ActiveLaneMask->replaceAllUsesWith(VCTPCall); 562 563 // Add the incoming value to the new phi. 564 // TODO: This add likely already exists in the loop. 565 Value *Remaining = Builder.CreateSub(Processed, Factor); 566 Processed->addIncoming(Remaining, L->getLoopLatch()); 567 LLVM_DEBUG(dbgs() << "ARM TP: Insert processed elements phi: " 568 << *Processed << "\n" 569 << "ARM TP: Inserted VCTP: " << *VCTPCall << "\n"); 570 } 571 572 bool MVETailPredication::TryConvert(Value *TripCount) { 573 if (!IsPredicatedVectorLoop()) { 574 LLVM_DEBUG(dbgs() << "ARM TP: no masked instructions in loop.\n"); 575 return false; 576 } 577 578 LLVM_DEBUG(dbgs() << "ARM TP: Found predicated vector loop.\n"); 579 SetVector<Instruction*> Predicates; 580 581 // Walk through the masked intrinsics and try to find whether the predicate 582 // operand is generated by intrinsic @llvm.get.active.lane.mask(). 583 for (auto *I : MaskedInsts) { 584 unsigned PredOp = I->getIntrinsicID() == Intrinsic::masked_load ? 2 : 3; 585 auto *Predicate = dyn_cast<Instruction>(I->getArgOperand(PredOp)); 586 if (!Predicate || Predicates.count(Predicate)) 587 continue; 588 589 auto *ActiveLaneMask = dyn_cast<IntrinsicInst>(Predicate); 590 if (!ActiveLaneMask || 591 ActiveLaneMask->getIntrinsicID() != Intrinsic::get_active_lane_mask) 592 continue; 593 594 Predicates.insert(Predicate); 595 LLVM_DEBUG(dbgs() << "ARM TP: Found active lane mask: " 596 << *ActiveLaneMask << "\n"); 597 598 auto *VecTy = getVectorType(I); 599 if (!IsSafeActiveMask(ActiveLaneMask, TripCount, VecTy)) { 600 LLVM_DEBUG(dbgs() << "ARM TP: Not safe to insert VCTP.\n"); 601 return false; 602 } 603 LLVM_DEBUG(dbgs() << "ARM TP: Safe to insert VCTP.\n"); 604 InsertVCTPIntrinsic(ActiveLaneMask, TripCount, VecTy); 605 } 606 607 Cleanup(Predicates, L); 608 return true; 609 } 610 611 Pass *llvm::createMVETailPredicationPass() { 612 return new MVETailPredication(); 613 } 614 615 char MVETailPredication::ID = 0; 616 617 INITIALIZE_PASS_BEGIN(MVETailPredication, DEBUG_TYPE, DESC, false, false) 618 INITIALIZE_PASS_END(MVETailPredication, DEBUG_TYPE, DESC, false, false) 619