1 //===- ARMTargetTransformInfo.cpp - ARM specific TTI ----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "ARMTargetTransformInfo.h" 10 #include "ARMSubtarget.h" 11 #include "MCTargetDesc/ARMAddressingModes.h" 12 #include "llvm/ADT/APInt.h" 13 #include "llvm/ADT/SmallVector.h" 14 #include "llvm/Analysis/LoopInfo.h" 15 #include "llvm/CodeGen/CostTable.h" 16 #include "llvm/CodeGen/ISDOpcodes.h" 17 #include "llvm/CodeGen/ValueTypes.h" 18 #include "llvm/IR/BasicBlock.h" 19 #include "llvm/IR/DataLayout.h" 20 #include "llvm/IR/DerivedTypes.h" 21 #include "llvm/IR/Instruction.h" 22 #include "llvm/IR/Instructions.h" 23 #include "llvm/IR/Intrinsics.h" 24 #include "llvm/IR/IntrinsicInst.h" 25 #include "llvm/IR/IntrinsicsARM.h" 26 #include "llvm/IR/PatternMatch.h" 27 #include "llvm/IR/Type.h" 28 #include "llvm/MC/SubtargetFeature.h" 29 #include "llvm/Support/Casting.h" 30 #include "llvm/Support/KnownBits.h" 31 #include "llvm/Support/MachineValueType.h" 32 #include "llvm/Target/TargetMachine.h" 33 #include "llvm/Transforms/InstCombine/InstCombiner.h" 34 #include "llvm/Transforms/Utils/Local.h" 35 #include "llvm/Transforms/Utils/LoopUtils.h" 36 #include <algorithm> 37 #include <cassert> 38 #include <cstdint> 39 #include <utility> 40 41 using namespace llvm; 42 43 #define DEBUG_TYPE "armtti" 44 45 static cl::opt<bool> EnableMaskedLoadStores( 46 "enable-arm-maskedldst", cl::Hidden, cl::init(true), 47 cl::desc("Enable the generation of masked loads and stores")); 48 49 static cl::opt<bool> DisableLowOverheadLoops( 50 "disable-arm-loloops", cl::Hidden, cl::init(false), 51 cl::desc("Disable the generation of low-overhead loops")); 52 53 static cl::opt<bool> 54 AllowWLSLoops("allow-arm-wlsloops", cl::Hidden, cl::init(true), 55 cl::desc("Enable the generation of WLS loops")); 56 57 extern cl::opt<TailPredication::Mode> EnableTailPredication; 58 59 extern cl::opt<bool> EnableMaskedGatherScatters; 60 61 extern cl::opt<unsigned> MVEMaxSupportedInterleaveFactor; 62 63 /// Convert a vector load intrinsic into a simple llvm load instruction. 64 /// This is beneficial when the underlying object being addressed comes 65 /// from a constant, since we get constant-folding for free. 66 static Value *simplifyNeonVld1(const IntrinsicInst &II, unsigned MemAlign, 67 InstCombiner::BuilderTy &Builder) { 68 auto *IntrAlign = dyn_cast<ConstantInt>(II.getArgOperand(1)); 69 70 if (!IntrAlign) 71 return nullptr; 72 73 unsigned Alignment = IntrAlign->getLimitedValue() < MemAlign 74 ? MemAlign 75 : IntrAlign->getLimitedValue(); 76 77 if (!isPowerOf2_32(Alignment)) 78 return nullptr; 79 80 auto *BCastInst = Builder.CreateBitCast(II.getArgOperand(0), 81 PointerType::get(II.getType(), 0)); 82 return Builder.CreateAlignedLoad(II.getType(), BCastInst, Align(Alignment)); 83 } 84 85 bool ARMTTIImpl::areInlineCompatible(const Function *Caller, 86 const Function *Callee) const { 87 const TargetMachine &TM = getTLI()->getTargetMachine(); 88 const FeatureBitset &CallerBits = 89 TM.getSubtargetImpl(*Caller)->getFeatureBits(); 90 const FeatureBitset &CalleeBits = 91 TM.getSubtargetImpl(*Callee)->getFeatureBits(); 92 93 // To inline a callee, all features not in the allowed list must match exactly. 94 bool MatchExact = (CallerBits & ~InlineFeaturesAllowed) == 95 (CalleeBits & ~InlineFeaturesAllowed); 96 // For features in the allowed list, the callee's features must be a subset of 97 // the callers'. 98 bool MatchSubset = ((CallerBits & CalleeBits) & InlineFeaturesAllowed) == 99 (CalleeBits & InlineFeaturesAllowed); 100 return MatchExact && MatchSubset; 101 } 102 103 TTI::AddressingModeKind 104 ARMTTIImpl::getPreferredAddressingMode(const Loop *L, 105 ScalarEvolution *SE) const { 106 if (ST->hasMVEIntegerOps()) 107 return TTI::AMK_PostIndexed; 108 109 if (L->getHeader()->getParent()->hasOptSize()) 110 return TTI::AMK_None; 111 112 if (ST->isMClass() && ST->isThumb2() && 113 L->getNumBlocks() == 1) 114 return TTI::AMK_PreIndexed; 115 116 return TTI::AMK_None; 117 } 118 119 Optional<Instruction *> 120 ARMTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { 121 using namespace PatternMatch; 122 Intrinsic::ID IID = II.getIntrinsicID(); 123 switch (IID) { 124 default: 125 break; 126 case Intrinsic::arm_neon_vld1: { 127 Align MemAlign = 128 getKnownAlignment(II.getArgOperand(0), IC.getDataLayout(), &II, 129 &IC.getAssumptionCache(), &IC.getDominatorTree()); 130 if (Value *V = simplifyNeonVld1(II, MemAlign.value(), IC.Builder)) { 131 return IC.replaceInstUsesWith(II, V); 132 } 133 break; 134 } 135 136 case Intrinsic::arm_neon_vld2: 137 case Intrinsic::arm_neon_vld3: 138 case Intrinsic::arm_neon_vld4: 139 case Intrinsic::arm_neon_vld2lane: 140 case Intrinsic::arm_neon_vld3lane: 141 case Intrinsic::arm_neon_vld4lane: 142 case Intrinsic::arm_neon_vst1: 143 case Intrinsic::arm_neon_vst2: 144 case Intrinsic::arm_neon_vst3: 145 case Intrinsic::arm_neon_vst4: 146 case Intrinsic::arm_neon_vst2lane: 147 case Intrinsic::arm_neon_vst3lane: 148 case Intrinsic::arm_neon_vst4lane: { 149 Align MemAlign = 150 getKnownAlignment(II.getArgOperand(0), IC.getDataLayout(), &II, 151 &IC.getAssumptionCache(), &IC.getDominatorTree()); 152 unsigned AlignArg = II.arg_size() - 1; 153 Value *AlignArgOp = II.getArgOperand(AlignArg); 154 MaybeAlign Align = cast<ConstantInt>(AlignArgOp)->getMaybeAlignValue(); 155 if (Align && *Align < MemAlign) { 156 return IC.replaceOperand( 157 II, AlignArg, 158 ConstantInt::get(Type::getInt32Ty(II.getContext()), MemAlign.value(), 159 false)); 160 } 161 break; 162 } 163 164 case Intrinsic::arm_mve_pred_i2v: { 165 Value *Arg = II.getArgOperand(0); 166 Value *ArgArg; 167 if (match(Arg, PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>( 168 PatternMatch::m_Value(ArgArg))) && 169 II.getType() == ArgArg->getType()) { 170 return IC.replaceInstUsesWith(II, ArgArg); 171 } 172 Constant *XorMask; 173 if (match(Arg, m_Xor(PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>( 174 PatternMatch::m_Value(ArgArg)), 175 PatternMatch::m_Constant(XorMask))) && 176 II.getType() == ArgArg->getType()) { 177 if (auto *CI = dyn_cast<ConstantInt>(XorMask)) { 178 if (CI->getValue().trunc(16).isAllOnes()) { 179 auto TrueVector = IC.Builder.CreateVectorSplat( 180 cast<FixedVectorType>(II.getType())->getNumElements(), 181 IC.Builder.getTrue()); 182 return BinaryOperator::Create(Instruction::Xor, ArgArg, TrueVector); 183 } 184 } 185 } 186 KnownBits ScalarKnown(32); 187 if (IC.SimplifyDemandedBits(&II, 0, APInt::getLowBitsSet(32, 16), 188 ScalarKnown, 0)) { 189 return &II; 190 } 191 break; 192 } 193 case Intrinsic::arm_mve_pred_v2i: { 194 Value *Arg = II.getArgOperand(0); 195 Value *ArgArg; 196 if (match(Arg, PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_i2v>( 197 PatternMatch::m_Value(ArgArg)))) { 198 return IC.replaceInstUsesWith(II, ArgArg); 199 } 200 if (!II.getMetadata(LLVMContext::MD_range)) { 201 Type *IntTy32 = Type::getInt32Ty(II.getContext()); 202 Metadata *M[] = { 203 ConstantAsMetadata::get(ConstantInt::get(IntTy32, 0)), 204 ConstantAsMetadata::get(ConstantInt::get(IntTy32, 0x10000))}; 205 II.setMetadata(LLVMContext::MD_range, MDNode::get(II.getContext(), M)); 206 return &II; 207 } 208 break; 209 } 210 case Intrinsic::arm_mve_vadc: 211 case Intrinsic::arm_mve_vadc_predicated: { 212 unsigned CarryOp = 213 (II.getIntrinsicID() == Intrinsic::arm_mve_vadc_predicated) ? 3 : 2; 214 assert(II.getArgOperand(CarryOp)->getType()->getScalarSizeInBits() == 32 && 215 "Bad type for intrinsic!"); 216 217 KnownBits CarryKnown(32); 218 if (IC.SimplifyDemandedBits(&II, CarryOp, APInt::getOneBitSet(32, 29), 219 CarryKnown)) { 220 return &II; 221 } 222 break; 223 } 224 case Intrinsic::arm_mve_vmldava: { 225 Instruction *I = cast<Instruction>(&II); 226 if (I->hasOneUse()) { 227 auto *User = cast<Instruction>(*I->user_begin()); 228 Value *OpZ; 229 if (match(User, m_c_Add(m_Specific(I), m_Value(OpZ))) && 230 match(I->getOperand(3), m_Zero())) { 231 Value *OpX = I->getOperand(4); 232 Value *OpY = I->getOperand(5); 233 Type *OpTy = OpX->getType(); 234 235 IC.Builder.SetInsertPoint(User); 236 Value *V = 237 IC.Builder.CreateIntrinsic(Intrinsic::arm_mve_vmldava, {OpTy}, 238 {I->getOperand(0), I->getOperand(1), 239 I->getOperand(2), OpZ, OpX, OpY}); 240 241 IC.replaceInstUsesWith(*User, V); 242 return IC.eraseInstFromFunction(*User); 243 } 244 } 245 return None; 246 } 247 } 248 return None; 249 } 250 251 Optional<Value *> ARMTTIImpl::simplifyDemandedVectorEltsIntrinsic( 252 InstCombiner &IC, IntrinsicInst &II, APInt OrigDemandedElts, 253 APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3, 254 std::function<void(Instruction *, unsigned, APInt, APInt &)> 255 SimplifyAndSetOp) const { 256 257 // Compute the demanded bits for a narrowing MVE intrinsic. The TopOpc is the 258 // opcode specifying a Top/Bottom instruction, which can change between 259 // instructions. 260 auto SimplifyNarrowInstrTopBottom =[&](unsigned TopOpc) { 261 unsigned NumElts = cast<FixedVectorType>(II.getType())->getNumElements(); 262 unsigned IsTop = cast<ConstantInt>(II.getOperand(TopOpc))->getZExtValue(); 263 264 // The only odd/even lanes of operand 0 will only be demanded depending 265 // on whether this is a top/bottom instruction. 266 APInt DemandedElts = 267 APInt::getSplat(NumElts, IsTop ? APInt::getLowBitsSet(2, 1) 268 : APInt::getHighBitsSet(2, 1)); 269 SimplifyAndSetOp(&II, 0, OrigDemandedElts & DemandedElts, UndefElts); 270 // The other lanes will be defined from the inserted elements. 271 UndefElts &= APInt::getSplat(NumElts, !IsTop ? APInt::getLowBitsSet(2, 1) 272 : APInt::getHighBitsSet(2, 1)); 273 return None; 274 }; 275 276 switch (II.getIntrinsicID()) { 277 default: 278 break; 279 case Intrinsic::arm_mve_vcvt_narrow: 280 SimplifyNarrowInstrTopBottom(2); 281 break; 282 case Intrinsic::arm_mve_vqmovn: 283 SimplifyNarrowInstrTopBottom(4); 284 break; 285 case Intrinsic::arm_mve_vshrn: 286 SimplifyNarrowInstrTopBottom(7); 287 break; 288 } 289 290 return None; 291 } 292 293 InstructionCost ARMTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty, 294 TTI::TargetCostKind CostKind) { 295 assert(Ty->isIntegerTy()); 296 297 unsigned Bits = Ty->getPrimitiveSizeInBits(); 298 if (Bits == 0 || Imm.getActiveBits() >= 64) 299 return 4; 300 301 int64_t SImmVal = Imm.getSExtValue(); 302 uint64_t ZImmVal = Imm.getZExtValue(); 303 if (!ST->isThumb()) { 304 if ((SImmVal >= 0 && SImmVal < 65536) || 305 (ARM_AM::getSOImmVal(ZImmVal) != -1) || 306 (ARM_AM::getSOImmVal(~ZImmVal) != -1)) 307 return 1; 308 return ST->hasV6T2Ops() ? 2 : 3; 309 } 310 if (ST->isThumb2()) { 311 if ((SImmVal >= 0 && SImmVal < 65536) || 312 (ARM_AM::getT2SOImmVal(ZImmVal) != -1) || 313 (ARM_AM::getT2SOImmVal(~ZImmVal) != -1)) 314 return 1; 315 return ST->hasV6T2Ops() ? 2 : 3; 316 } 317 // Thumb1, any i8 imm cost 1. 318 if (Bits == 8 || (SImmVal >= 0 && SImmVal < 256)) 319 return 1; 320 if ((~SImmVal < 256) || ARM_AM::isThumbImmShiftedVal(ZImmVal)) 321 return 2; 322 // Load from constantpool. 323 return 3; 324 } 325 326 // Constants smaller than 256 fit in the immediate field of 327 // Thumb1 instructions so we return a zero cost and 1 otherwise. 328 InstructionCost ARMTTIImpl::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx, 329 const APInt &Imm, Type *Ty) { 330 if (Imm.isNonNegative() && Imm.getLimitedValue() < 256) 331 return 0; 332 333 return 1; 334 } 335 336 // Checks whether Inst is part of a min(max()) or max(min()) pattern 337 // that will match to an SSAT instruction. Returns the instruction being 338 // saturated, or null if no saturation pattern was found. 339 static Value *isSSATMinMaxPattern(Instruction *Inst, const APInt &Imm) { 340 Value *LHS, *RHS; 341 ConstantInt *C; 342 SelectPatternFlavor InstSPF = matchSelectPattern(Inst, LHS, RHS).Flavor; 343 344 if (InstSPF == SPF_SMAX && 345 PatternMatch::match(RHS, PatternMatch::m_ConstantInt(C)) && 346 C->getValue() == Imm && Imm.isNegative() && Imm.isNegatedPowerOf2()) { 347 348 auto isSSatMin = [&](Value *MinInst) { 349 if (isa<SelectInst>(MinInst)) { 350 Value *MinLHS, *MinRHS; 351 ConstantInt *MinC; 352 SelectPatternFlavor MinSPF = 353 matchSelectPattern(MinInst, MinLHS, MinRHS).Flavor; 354 if (MinSPF == SPF_SMIN && 355 PatternMatch::match(MinRHS, PatternMatch::m_ConstantInt(MinC)) && 356 MinC->getValue() == ((-Imm) - 1)) 357 return true; 358 } 359 return false; 360 }; 361 362 if (isSSatMin(Inst->getOperand(1))) 363 return cast<Instruction>(Inst->getOperand(1))->getOperand(1); 364 if (Inst->hasNUses(2) && 365 (isSSatMin(*Inst->user_begin()) || isSSatMin(*(++Inst->user_begin())))) 366 return Inst->getOperand(1); 367 } 368 return nullptr; 369 } 370 371 // Look for a FP Saturation pattern, where the instruction can be simplified to 372 // a fptosi.sat. max(min(fptosi)). The constant in this case is always free. 373 static bool isFPSatMinMaxPattern(Instruction *Inst, const APInt &Imm) { 374 if (Imm.getBitWidth() != 64 || 375 Imm != APInt::getHighBitsSet(64, 33)) // -2147483648 376 return false; 377 Value *FP = isSSATMinMaxPattern(Inst, Imm); 378 if (!FP && isa<ICmpInst>(Inst) && Inst->hasOneUse()) 379 FP = isSSATMinMaxPattern(cast<Instruction>(*Inst->user_begin()), Imm); 380 if (!FP) 381 return false; 382 return isa<FPToSIInst>(FP); 383 } 384 385 InstructionCost ARMTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx, 386 const APInt &Imm, Type *Ty, 387 TTI::TargetCostKind CostKind, 388 Instruction *Inst) { 389 // Division by a constant can be turned into multiplication, but only if we 390 // know it's constant. So it's not so much that the immediate is cheap (it's 391 // not), but that the alternative is worse. 392 // FIXME: this is probably unneeded with GlobalISel. 393 if ((Opcode == Instruction::SDiv || Opcode == Instruction::UDiv || 394 Opcode == Instruction::SRem || Opcode == Instruction::URem) && 395 Idx == 1) 396 return 0; 397 398 // Leave any gep offsets for the CodeGenPrepare, which will do a better job at 399 // splitting any large offsets. 400 if (Opcode == Instruction::GetElementPtr && Idx != 0) 401 return 0; 402 403 if (Opcode == Instruction::And) { 404 // UXTB/UXTH 405 if (Imm == 255 || Imm == 65535) 406 return 0; 407 // Conversion to BIC is free, and means we can use ~Imm instead. 408 return std::min(getIntImmCost(Imm, Ty, CostKind), 409 getIntImmCost(~Imm, Ty, CostKind)); 410 } 411 412 if (Opcode == Instruction::Add) 413 // Conversion to SUB is free, and means we can use -Imm instead. 414 return std::min(getIntImmCost(Imm, Ty, CostKind), 415 getIntImmCost(-Imm, Ty, CostKind)); 416 417 if (Opcode == Instruction::ICmp && Imm.isNegative() && 418 Ty->getIntegerBitWidth() == 32) { 419 int64_t NegImm = -Imm.getSExtValue(); 420 if (ST->isThumb2() && NegImm < 1<<12) 421 // icmp X, #-C -> cmn X, #C 422 return 0; 423 if (ST->isThumb() && NegImm < 1<<8) 424 // icmp X, #-C -> adds X, #C 425 return 0; 426 } 427 428 // xor a, -1 can always be folded to MVN 429 if (Opcode == Instruction::Xor && Imm.isAllOnes()) 430 return 0; 431 432 // Ensures negative constant of min(max()) or max(min()) patterns that 433 // match to SSAT instructions don't get hoisted 434 if (Inst && ((ST->hasV6Ops() && !ST->isThumb()) || ST->isThumb2()) && 435 Ty->getIntegerBitWidth() <= 32) { 436 if (isSSATMinMaxPattern(Inst, Imm) || 437 (isa<ICmpInst>(Inst) && Inst->hasOneUse() && 438 isSSATMinMaxPattern(cast<Instruction>(*Inst->user_begin()), Imm))) 439 return 0; 440 } 441 442 if (Inst && ST->hasVFP2Base() && isFPSatMinMaxPattern(Inst, Imm)) 443 return 0; 444 445 // We can convert <= -1 to < 0, which is generally quite cheap. 446 if (Inst && Opcode == Instruction::ICmp && Idx == 1 && Imm.isAllOnesValue()) { 447 ICmpInst::Predicate Pred = cast<ICmpInst>(Inst)->getPredicate(); 448 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE) 449 return std::min(getIntImmCost(Imm, Ty, CostKind), 450 getIntImmCost(Imm + 1, Ty, CostKind)); 451 } 452 453 return getIntImmCost(Imm, Ty, CostKind); 454 } 455 456 InstructionCost ARMTTIImpl::getCFInstrCost(unsigned Opcode, 457 TTI::TargetCostKind CostKind, 458 const Instruction *I) { 459 if (CostKind == TTI::TCK_RecipThroughput && 460 (ST->hasNEON() || ST->hasMVEIntegerOps())) { 461 // FIXME: The vectorizer is highly sensistive to the cost of these 462 // instructions, which suggests that it may be using the costs incorrectly. 463 // But, for now, just make them free to avoid performance regressions for 464 // vector targets. 465 return 0; 466 } 467 return BaseT::getCFInstrCost(Opcode, CostKind, I); 468 } 469 470 InstructionCost ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, 471 Type *Src, 472 TTI::CastContextHint CCH, 473 TTI::TargetCostKind CostKind, 474 const Instruction *I) { 475 int ISD = TLI->InstructionOpcodeToISD(Opcode); 476 assert(ISD && "Invalid opcode"); 477 478 // TODO: Allow non-throughput costs that aren't binary. 479 auto AdjustCost = [&CostKind](InstructionCost Cost) -> InstructionCost { 480 if (CostKind != TTI::TCK_RecipThroughput) 481 return Cost == 0 ? 0 : 1; 482 return Cost; 483 }; 484 auto IsLegalFPType = [this](EVT VT) { 485 EVT EltVT = VT.getScalarType(); 486 return (EltVT == MVT::f32 && ST->hasVFP2Base()) || 487 (EltVT == MVT::f64 && ST->hasFP64()) || 488 (EltVT == MVT::f16 && ST->hasFullFP16()); 489 }; 490 491 EVT SrcTy = TLI->getValueType(DL, Src); 492 EVT DstTy = TLI->getValueType(DL, Dst); 493 494 if (!SrcTy.isSimple() || !DstTy.isSimple()) 495 return AdjustCost( 496 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I)); 497 498 // Extending masked load/Truncating masked stores is expensive because we 499 // currently don't split them. This means that we'll likely end up 500 // loading/storing each element individually (hence the high cost). 501 if ((ST->hasMVEIntegerOps() && 502 (Opcode == Instruction::Trunc || Opcode == Instruction::ZExt || 503 Opcode == Instruction::SExt)) || 504 (ST->hasMVEFloatOps() && 505 (Opcode == Instruction::FPExt || Opcode == Instruction::FPTrunc) && 506 IsLegalFPType(SrcTy) && IsLegalFPType(DstTy))) 507 if (CCH == TTI::CastContextHint::Masked && DstTy.getSizeInBits() > 128) 508 return 2 * DstTy.getVectorNumElements() * 509 ST->getMVEVectorCostFactor(CostKind); 510 511 // The extend of other kinds of load is free 512 if (CCH == TTI::CastContextHint::Normal || 513 CCH == TTI::CastContextHint::Masked) { 514 static const TypeConversionCostTblEntry LoadConversionTbl[] = { 515 {ISD::SIGN_EXTEND, MVT::i32, MVT::i16, 0}, 516 {ISD::ZERO_EXTEND, MVT::i32, MVT::i16, 0}, 517 {ISD::SIGN_EXTEND, MVT::i32, MVT::i8, 0}, 518 {ISD::ZERO_EXTEND, MVT::i32, MVT::i8, 0}, 519 {ISD::SIGN_EXTEND, MVT::i16, MVT::i8, 0}, 520 {ISD::ZERO_EXTEND, MVT::i16, MVT::i8, 0}, 521 {ISD::SIGN_EXTEND, MVT::i64, MVT::i32, 1}, 522 {ISD::ZERO_EXTEND, MVT::i64, MVT::i32, 1}, 523 {ISD::SIGN_EXTEND, MVT::i64, MVT::i16, 1}, 524 {ISD::ZERO_EXTEND, MVT::i64, MVT::i16, 1}, 525 {ISD::SIGN_EXTEND, MVT::i64, MVT::i8, 1}, 526 {ISD::ZERO_EXTEND, MVT::i64, MVT::i8, 1}, 527 }; 528 if (const auto *Entry = ConvertCostTableLookup( 529 LoadConversionTbl, ISD, DstTy.getSimpleVT(), SrcTy.getSimpleVT())) 530 return AdjustCost(Entry->Cost); 531 532 static const TypeConversionCostTblEntry MVELoadConversionTbl[] = { 533 {ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i16, 0}, 534 {ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i16, 0}, 535 {ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i8, 0}, 536 {ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i8, 0}, 537 {ISD::SIGN_EXTEND, MVT::v8i16, MVT::v8i8, 0}, 538 {ISD::ZERO_EXTEND, MVT::v8i16, MVT::v8i8, 0}, 539 // The following extend from a legal type to an illegal type, so need to 540 // split the load. This introduced an extra load operation, but the 541 // extend is still "free". 542 {ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 1}, 543 {ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 1}, 544 {ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8, 3}, 545 {ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8, 3}, 546 {ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 1}, 547 {ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 1}, 548 }; 549 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) { 550 if (const auto *Entry = 551 ConvertCostTableLookup(MVELoadConversionTbl, ISD, 552 DstTy.getSimpleVT(), SrcTy.getSimpleVT())) 553 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind); 554 } 555 556 static const TypeConversionCostTblEntry MVEFLoadConversionTbl[] = { 557 // FPExtends are similar but also require the VCVT instructions. 558 {ISD::FP_EXTEND, MVT::v4f32, MVT::v4f16, 1}, 559 {ISD::FP_EXTEND, MVT::v8f32, MVT::v8f16, 3}, 560 }; 561 if (SrcTy.isVector() && ST->hasMVEFloatOps()) { 562 if (const auto *Entry = 563 ConvertCostTableLookup(MVEFLoadConversionTbl, ISD, 564 DstTy.getSimpleVT(), SrcTy.getSimpleVT())) 565 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind); 566 } 567 568 // The truncate of a store is free. This is the mirror of extends above. 569 static const TypeConversionCostTblEntry MVEStoreConversionTbl[] = { 570 {ISD::TRUNCATE, MVT::v4i32, MVT::v4i16, 0}, 571 {ISD::TRUNCATE, MVT::v4i32, MVT::v4i8, 0}, 572 {ISD::TRUNCATE, MVT::v8i16, MVT::v8i8, 0}, 573 {ISD::TRUNCATE, MVT::v8i32, MVT::v8i16, 1}, 574 {ISD::TRUNCATE, MVT::v8i32, MVT::v8i8, 1}, 575 {ISD::TRUNCATE, MVT::v16i32, MVT::v16i8, 3}, 576 {ISD::TRUNCATE, MVT::v16i16, MVT::v16i8, 1}, 577 }; 578 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) { 579 if (const auto *Entry = 580 ConvertCostTableLookup(MVEStoreConversionTbl, ISD, 581 SrcTy.getSimpleVT(), DstTy.getSimpleVT())) 582 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind); 583 } 584 585 static const TypeConversionCostTblEntry MVEFStoreConversionTbl[] = { 586 {ISD::FP_ROUND, MVT::v4f32, MVT::v4f16, 1}, 587 {ISD::FP_ROUND, MVT::v8f32, MVT::v8f16, 3}, 588 }; 589 if (SrcTy.isVector() && ST->hasMVEFloatOps()) { 590 if (const auto *Entry = 591 ConvertCostTableLookup(MVEFStoreConversionTbl, ISD, 592 SrcTy.getSimpleVT(), DstTy.getSimpleVT())) 593 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind); 594 } 595 } 596 597 // NEON vector operations that can extend their inputs. 598 if ((ISD == ISD::SIGN_EXTEND || ISD == ISD::ZERO_EXTEND) && 599 I && I->hasOneUse() && ST->hasNEON() && SrcTy.isVector()) { 600 static const TypeConversionCostTblEntry NEONDoubleWidthTbl[] = { 601 // vaddl 602 { ISD::ADD, MVT::v4i32, MVT::v4i16, 0 }, 603 { ISD::ADD, MVT::v8i16, MVT::v8i8, 0 }, 604 // vsubl 605 { ISD::SUB, MVT::v4i32, MVT::v4i16, 0 }, 606 { ISD::SUB, MVT::v8i16, MVT::v8i8, 0 }, 607 // vmull 608 { ISD::MUL, MVT::v4i32, MVT::v4i16, 0 }, 609 { ISD::MUL, MVT::v8i16, MVT::v8i8, 0 }, 610 // vshll 611 { ISD::SHL, MVT::v4i32, MVT::v4i16, 0 }, 612 { ISD::SHL, MVT::v8i16, MVT::v8i8, 0 }, 613 }; 614 615 auto *User = cast<Instruction>(*I->user_begin()); 616 int UserISD = TLI->InstructionOpcodeToISD(User->getOpcode()); 617 if (auto *Entry = ConvertCostTableLookup(NEONDoubleWidthTbl, UserISD, 618 DstTy.getSimpleVT(), 619 SrcTy.getSimpleVT())) { 620 return AdjustCost(Entry->Cost); 621 } 622 } 623 624 // Single to/from double precision conversions. 625 if (Src->isVectorTy() && ST->hasNEON() && 626 ((ISD == ISD::FP_ROUND && SrcTy.getScalarType() == MVT::f64 && 627 DstTy.getScalarType() == MVT::f32) || 628 (ISD == ISD::FP_EXTEND && SrcTy.getScalarType() == MVT::f32 && 629 DstTy.getScalarType() == MVT::f64))) { 630 static const CostTblEntry NEONFltDblTbl[] = { 631 // Vector fptrunc/fpext conversions. 632 {ISD::FP_ROUND, MVT::v2f64, 2}, 633 {ISD::FP_EXTEND, MVT::v2f32, 2}, 634 {ISD::FP_EXTEND, MVT::v4f32, 4}}; 635 636 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Src); 637 if (const auto *Entry = CostTableLookup(NEONFltDblTbl, ISD, LT.second)) 638 return AdjustCost(LT.first * Entry->Cost); 639 } 640 641 // Some arithmetic, load and store operations have specific instructions 642 // to cast up/down their types automatically at no extra cost. 643 // TODO: Get these tables to know at least what the related operations are. 644 static const TypeConversionCostTblEntry NEONVectorConversionTbl[] = { 645 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i16, 1 }, 646 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i16, 1 }, 647 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i32, 1 }, 648 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i32, 1 }, 649 { ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, 0 }, 650 { ISD::TRUNCATE, MVT::v4i16, MVT::v4i32, 1 }, 651 652 // The number of vmovl instructions for the extension. 653 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v8i8, 1 }, 654 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v8i8, 1 }, 655 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i8, 2 }, 656 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i8, 2 }, 657 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i8, 3 }, 658 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i8, 3 }, 659 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i16, 2 }, 660 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i16, 2 }, 661 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i16, 3 }, 662 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i16, 3 }, 663 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i8, 3 }, 664 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i8, 3 }, 665 { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i8, 7 }, 666 { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i8, 7 }, 667 { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i16, 6 }, 668 { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i16, 6 }, 669 { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8, 6 }, 670 { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8, 6 }, 671 672 // Operations that we legalize using splitting. 673 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, 6 }, 674 { ISD::TRUNCATE, MVT::v8i8, MVT::v8i32, 3 }, 675 676 // Vector float <-> i32 conversions. 677 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 1 }, 678 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 1 }, 679 680 { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i8, 3 }, 681 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i8, 3 }, 682 { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i16, 2 }, 683 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i16, 2 }, 684 { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i32, 1 }, 685 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, 1 }, 686 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i1, 3 }, 687 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i1, 3 }, 688 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i8, 3 }, 689 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i8, 3 }, 690 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i16, 2 }, 691 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i16, 2 }, 692 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, 4 }, 693 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, 4 }, 694 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i32, 2 }, 695 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i32, 2 }, 696 { ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i16, 8 }, 697 { ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i16, 8 }, 698 { ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i32, 4 }, 699 { ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i32, 4 }, 700 701 { ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f32, 1 }, 702 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, 1 }, 703 { ISD::FP_TO_SINT, MVT::v4i8, MVT::v4f32, 3 }, 704 { ISD::FP_TO_UINT, MVT::v4i8, MVT::v4f32, 3 }, 705 { ISD::FP_TO_SINT, MVT::v4i16, MVT::v4f32, 2 }, 706 { ISD::FP_TO_UINT, MVT::v4i16, MVT::v4f32, 2 }, 707 708 // Vector double <-> i32 conversions. 709 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 }, 710 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 }, 711 712 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i8, 4 }, 713 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i8, 4 }, 714 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i16, 3 }, 715 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i16, 3 }, 716 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 }, 717 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 }, 718 719 { ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f64, 2 }, 720 { ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f64, 2 }, 721 { ISD::FP_TO_SINT, MVT::v8i16, MVT::v8f32, 4 }, 722 { ISD::FP_TO_UINT, MVT::v8i16, MVT::v8f32, 4 }, 723 { ISD::FP_TO_SINT, MVT::v16i16, MVT::v16f32, 8 }, 724 { ISD::FP_TO_UINT, MVT::v16i16, MVT::v16f32, 8 } 725 }; 726 727 if (SrcTy.isVector() && ST->hasNEON()) { 728 if (const auto *Entry = ConvertCostTableLookup(NEONVectorConversionTbl, ISD, 729 DstTy.getSimpleVT(), 730 SrcTy.getSimpleVT())) 731 return AdjustCost(Entry->Cost); 732 } 733 734 // Scalar float to integer conversions. 735 static const TypeConversionCostTblEntry NEONFloatConversionTbl[] = { 736 { ISD::FP_TO_SINT, MVT::i1, MVT::f32, 2 }, 737 { ISD::FP_TO_UINT, MVT::i1, MVT::f32, 2 }, 738 { ISD::FP_TO_SINT, MVT::i1, MVT::f64, 2 }, 739 { ISD::FP_TO_UINT, MVT::i1, MVT::f64, 2 }, 740 { ISD::FP_TO_SINT, MVT::i8, MVT::f32, 2 }, 741 { ISD::FP_TO_UINT, MVT::i8, MVT::f32, 2 }, 742 { ISD::FP_TO_SINT, MVT::i8, MVT::f64, 2 }, 743 { ISD::FP_TO_UINT, MVT::i8, MVT::f64, 2 }, 744 { ISD::FP_TO_SINT, MVT::i16, MVT::f32, 2 }, 745 { ISD::FP_TO_UINT, MVT::i16, MVT::f32, 2 }, 746 { ISD::FP_TO_SINT, MVT::i16, MVT::f64, 2 }, 747 { ISD::FP_TO_UINT, MVT::i16, MVT::f64, 2 }, 748 { ISD::FP_TO_SINT, MVT::i32, MVT::f32, 2 }, 749 { ISD::FP_TO_UINT, MVT::i32, MVT::f32, 2 }, 750 { ISD::FP_TO_SINT, MVT::i32, MVT::f64, 2 }, 751 { ISD::FP_TO_UINT, MVT::i32, MVT::f64, 2 }, 752 { ISD::FP_TO_SINT, MVT::i64, MVT::f32, 10 }, 753 { ISD::FP_TO_UINT, MVT::i64, MVT::f32, 10 }, 754 { ISD::FP_TO_SINT, MVT::i64, MVT::f64, 10 }, 755 { ISD::FP_TO_UINT, MVT::i64, MVT::f64, 10 } 756 }; 757 if (SrcTy.isFloatingPoint() && ST->hasNEON()) { 758 if (const auto *Entry = ConvertCostTableLookup(NEONFloatConversionTbl, ISD, 759 DstTy.getSimpleVT(), 760 SrcTy.getSimpleVT())) 761 return AdjustCost(Entry->Cost); 762 } 763 764 // Scalar integer to float conversions. 765 static const TypeConversionCostTblEntry NEONIntegerConversionTbl[] = { 766 { ISD::SINT_TO_FP, MVT::f32, MVT::i1, 2 }, 767 { ISD::UINT_TO_FP, MVT::f32, MVT::i1, 2 }, 768 { ISD::SINT_TO_FP, MVT::f64, MVT::i1, 2 }, 769 { ISD::UINT_TO_FP, MVT::f64, MVT::i1, 2 }, 770 { ISD::SINT_TO_FP, MVT::f32, MVT::i8, 2 }, 771 { ISD::UINT_TO_FP, MVT::f32, MVT::i8, 2 }, 772 { ISD::SINT_TO_FP, MVT::f64, MVT::i8, 2 }, 773 { ISD::UINT_TO_FP, MVT::f64, MVT::i8, 2 }, 774 { ISD::SINT_TO_FP, MVT::f32, MVT::i16, 2 }, 775 { ISD::UINT_TO_FP, MVT::f32, MVT::i16, 2 }, 776 { ISD::SINT_TO_FP, MVT::f64, MVT::i16, 2 }, 777 { ISD::UINT_TO_FP, MVT::f64, MVT::i16, 2 }, 778 { ISD::SINT_TO_FP, MVT::f32, MVT::i32, 2 }, 779 { ISD::UINT_TO_FP, MVT::f32, MVT::i32, 2 }, 780 { ISD::SINT_TO_FP, MVT::f64, MVT::i32, 2 }, 781 { ISD::UINT_TO_FP, MVT::f64, MVT::i32, 2 }, 782 { ISD::SINT_TO_FP, MVT::f32, MVT::i64, 10 }, 783 { ISD::UINT_TO_FP, MVT::f32, MVT::i64, 10 }, 784 { ISD::SINT_TO_FP, MVT::f64, MVT::i64, 10 }, 785 { ISD::UINT_TO_FP, MVT::f64, MVT::i64, 10 } 786 }; 787 788 if (SrcTy.isInteger() && ST->hasNEON()) { 789 if (const auto *Entry = ConvertCostTableLookup(NEONIntegerConversionTbl, 790 ISD, DstTy.getSimpleVT(), 791 SrcTy.getSimpleVT())) 792 return AdjustCost(Entry->Cost); 793 } 794 795 // MVE extend costs, taken from codegen tests. i8->i16 or i16->i32 is one 796 // instruction, i8->i32 is two. i64 zexts are an VAND with a constant, sext 797 // are linearised so take more. 798 static const TypeConversionCostTblEntry MVEVectorConversionTbl[] = { 799 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v8i8, 1 }, 800 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v8i8, 1 }, 801 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i8, 2 }, 802 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i8, 2 }, 803 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i8, 10 }, 804 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i8, 2 }, 805 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i16, 1 }, 806 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i16, 1 }, 807 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i16, 10 }, 808 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i16, 2 }, 809 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i32, 8 }, 810 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i32, 2 }, 811 }; 812 813 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) { 814 if (const auto *Entry = ConvertCostTableLookup(MVEVectorConversionTbl, 815 ISD, DstTy.getSimpleVT(), 816 SrcTy.getSimpleVT())) 817 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind); 818 } 819 820 if (ISD == ISD::FP_ROUND || ISD == ISD::FP_EXTEND) { 821 // As general rule, fp converts that were not matched above are scalarized 822 // and cost 1 vcvt for each lane, so long as the instruction is available. 823 // If not it will become a series of function calls. 824 const InstructionCost CallCost = 825 getCallInstrCost(nullptr, Dst, {Src}, CostKind); 826 int Lanes = 1; 827 if (SrcTy.isFixedLengthVector()) 828 Lanes = SrcTy.getVectorNumElements(); 829 830 if (IsLegalFPType(SrcTy) && IsLegalFPType(DstTy)) 831 return Lanes; 832 else 833 return Lanes * CallCost; 834 } 835 836 if (ISD == ISD::TRUNCATE && ST->hasMVEIntegerOps() && 837 SrcTy.isFixedLengthVector()) { 838 // Treat a truncate with larger than legal source (128bits for MVE) as 839 // expensive, 2 instructions per lane. 840 if ((SrcTy.getScalarType() == MVT::i8 || 841 SrcTy.getScalarType() == MVT::i16 || 842 SrcTy.getScalarType() == MVT::i32) && 843 SrcTy.getSizeInBits() > 128 && 844 SrcTy.getSizeInBits() > DstTy.getSizeInBits()) 845 return SrcTy.getVectorNumElements() * 2; 846 } 847 848 // Scalar integer conversion costs. 849 static const TypeConversionCostTblEntry ARMIntegerConversionTbl[] = { 850 // i16 -> i64 requires two dependent operations. 851 { ISD::SIGN_EXTEND, MVT::i64, MVT::i16, 2 }, 852 853 // Truncates on i64 are assumed to be free. 854 { ISD::TRUNCATE, MVT::i32, MVT::i64, 0 }, 855 { ISD::TRUNCATE, MVT::i16, MVT::i64, 0 }, 856 { ISD::TRUNCATE, MVT::i8, MVT::i64, 0 }, 857 { ISD::TRUNCATE, MVT::i1, MVT::i64, 0 } 858 }; 859 860 if (SrcTy.isInteger()) { 861 if (const auto *Entry = ConvertCostTableLookup(ARMIntegerConversionTbl, ISD, 862 DstTy.getSimpleVT(), 863 SrcTy.getSimpleVT())) 864 return AdjustCost(Entry->Cost); 865 } 866 867 int BaseCost = ST->hasMVEIntegerOps() && Src->isVectorTy() 868 ? ST->getMVEVectorCostFactor(CostKind) 869 : 1; 870 return AdjustCost( 871 BaseCost * BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I)); 872 } 873 874 InstructionCost ARMTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy, 875 unsigned Index) { 876 // Penalize inserting into an D-subregister. We end up with a three times 877 // lower estimated throughput on swift. 878 if (ST->hasSlowLoadDSubregister() && Opcode == Instruction::InsertElement && 879 ValTy->isVectorTy() && ValTy->getScalarSizeInBits() <= 32) 880 return 3; 881 882 if (ST->hasNEON() && (Opcode == Instruction::InsertElement || 883 Opcode == Instruction::ExtractElement)) { 884 // Cross-class copies are expensive on many microarchitectures, 885 // so assume they are expensive by default. 886 if (cast<VectorType>(ValTy)->getElementType()->isIntegerTy()) 887 return 3; 888 889 // Even if it's not a cross class copy, this likely leads to mixing 890 // of NEON and VFP code and should be therefore penalized. 891 if (ValTy->isVectorTy() && 892 ValTy->getScalarSizeInBits() <= 32) 893 return std::max<InstructionCost>( 894 BaseT::getVectorInstrCost(Opcode, ValTy, Index), 2U); 895 } 896 897 if (ST->hasMVEIntegerOps() && (Opcode == Instruction::InsertElement || 898 Opcode == Instruction::ExtractElement)) { 899 // Integer cross-lane moves are more expensive than float, which can 900 // sometimes just be vmovs. Integer involve being passes to GPR registers, 901 // causing more of a delay. 902 std::pair<InstructionCost, MVT> LT = 903 getTLI()->getTypeLegalizationCost(DL, ValTy->getScalarType()); 904 return LT.first * (ValTy->getScalarType()->isIntegerTy() ? 4 : 1); 905 } 906 907 return BaseT::getVectorInstrCost(Opcode, ValTy, Index); 908 } 909 910 InstructionCost ARMTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, 911 Type *CondTy, 912 CmpInst::Predicate VecPred, 913 TTI::TargetCostKind CostKind, 914 const Instruction *I) { 915 int ISD = TLI->InstructionOpcodeToISD(Opcode); 916 917 // Thumb scalar code size cost for select. 918 if (CostKind == TTI::TCK_CodeSize && ISD == ISD::SELECT && 919 ST->isThumb() && !ValTy->isVectorTy()) { 920 // Assume expensive structs. 921 if (TLI->getValueType(DL, ValTy, true) == MVT::Other) 922 return TTI::TCC_Expensive; 923 924 // Select costs can vary because they: 925 // - may require one or more conditional mov (including an IT), 926 // - can't operate directly on immediates, 927 // - require live flags, which we can't copy around easily. 928 InstructionCost Cost = TLI->getTypeLegalizationCost(DL, ValTy).first; 929 930 // Possible IT instruction for Thumb2, or more for Thumb1. 931 ++Cost; 932 933 // i1 values may need rematerialising by using mov immediates and/or 934 // flag setting instructions. 935 if (ValTy->isIntegerTy(1)) 936 ++Cost; 937 938 return Cost; 939 } 940 941 // If this is a vector min/max/abs, use the cost of that intrinsic directly 942 // instead. Hopefully when min/max intrinsics are more prevalent this code 943 // will not be needed. 944 const Instruction *Sel = I; 945 if ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) && Sel && 946 Sel->hasOneUse()) 947 Sel = cast<Instruction>(Sel->user_back()); 948 if (Sel && ValTy->isVectorTy() && 949 (ValTy->isIntOrIntVectorTy() || ValTy->isFPOrFPVectorTy())) { 950 const Value *LHS, *RHS; 951 SelectPatternFlavor SPF = matchSelectPattern(Sel, LHS, RHS).Flavor; 952 unsigned IID = 0; 953 switch (SPF) { 954 case SPF_ABS: 955 IID = Intrinsic::abs; 956 break; 957 case SPF_SMIN: 958 IID = Intrinsic::smin; 959 break; 960 case SPF_SMAX: 961 IID = Intrinsic::smax; 962 break; 963 case SPF_UMIN: 964 IID = Intrinsic::umin; 965 break; 966 case SPF_UMAX: 967 IID = Intrinsic::umax; 968 break; 969 case SPF_FMINNUM: 970 IID = Intrinsic::minnum; 971 break; 972 case SPF_FMAXNUM: 973 IID = Intrinsic::maxnum; 974 break; 975 default: 976 break; 977 } 978 if (IID) { 979 // The ICmp is free, the select gets the cost of the min/max/etc 980 if (Sel != I) 981 return 0; 982 IntrinsicCostAttributes CostAttrs(IID, ValTy, {ValTy, ValTy}); 983 return getIntrinsicInstrCost(CostAttrs, CostKind); 984 } 985 } 986 987 // On NEON a vector select gets lowered to vbsl. 988 if (ST->hasNEON() && ValTy->isVectorTy() && ISD == ISD::SELECT && CondTy) { 989 // Lowering of some vector selects is currently far from perfect. 990 static const TypeConversionCostTblEntry NEONVectorSelectTbl[] = { 991 { ISD::SELECT, MVT::v4i1, MVT::v4i64, 4*4 + 1*2 + 1 }, 992 { ISD::SELECT, MVT::v8i1, MVT::v8i64, 50 }, 993 { ISD::SELECT, MVT::v16i1, MVT::v16i64, 100 } 994 }; 995 996 EVT SelCondTy = TLI->getValueType(DL, CondTy); 997 EVT SelValTy = TLI->getValueType(DL, ValTy); 998 if (SelCondTy.isSimple() && SelValTy.isSimple()) { 999 if (const auto *Entry = ConvertCostTableLookup(NEONVectorSelectTbl, ISD, 1000 SelCondTy.getSimpleVT(), 1001 SelValTy.getSimpleVT())) 1002 return Entry->Cost; 1003 } 1004 1005 std::pair<InstructionCost, MVT> LT = 1006 TLI->getTypeLegalizationCost(DL, ValTy); 1007 return LT.first; 1008 } 1009 1010 if (ST->hasMVEIntegerOps() && ValTy->isVectorTy() && 1011 (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) && 1012 cast<FixedVectorType>(ValTy)->getNumElements() > 1) { 1013 FixedVectorType *VecValTy = cast<FixedVectorType>(ValTy); 1014 FixedVectorType *VecCondTy = dyn_cast_or_null<FixedVectorType>(CondTy); 1015 if (!VecCondTy) 1016 VecCondTy = cast<FixedVectorType>(CmpInst::makeCmpResultType(VecValTy)); 1017 1018 // If we don't have mve.fp any fp operations will need to be scalarized. 1019 if (Opcode == Instruction::FCmp && !ST->hasMVEFloatOps()) { 1020 // One scalaization insert, one scalarization extract and the cost of the 1021 // fcmps. 1022 return BaseT::getScalarizationOverhead(VecValTy, false, true) + 1023 BaseT::getScalarizationOverhead(VecCondTy, true, false) + 1024 VecValTy->getNumElements() * 1025 getCmpSelInstrCost(Opcode, ValTy->getScalarType(), 1026 VecCondTy->getScalarType(), VecPred, CostKind, 1027 I); 1028 } 1029 1030 std::pair<InstructionCost, MVT> LT = 1031 TLI->getTypeLegalizationCost(DL, ValTy); 1032 int BaseCost = ST->getMVEVectorCostFactor(CostKind); 1033 // There are two types - the input that specifies the type of the compare 1034 // and the output vXi1 type. Because we don't know how the output will be 1035 // split, we may need an expensive shuffle to get two in sync. This has the 1036 // effect of making larger than legal compares (v8i32 for example) 1037 // expensive. 1038 if (LT.second.getVectorNumElements() > 2) { 1039 if (LT.first > 1) 1040 return LT.first * BaseCost + 1041 BaseT::getScalarizationOverhead(VecCondTy, true, false); 1042 return BaseCost; 1043 } 1044 } 1045 1046 // Default to cheap (throughput/size of 1 instruction) but adjust throughput 1047 // for "multiple beats" potentially needed by MVE instructions. 1048 int BaseCost = 1; 1049 if (ST->hasMVEIntegerOps() && ValTy->isVectorTy()) 1050 BaseCost = ST->getMVEVectorCostFactor(CostKind); 1051 1052 return BaseCost * 1053 BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); 1054 } 1055 1056 InstructionCost ARMTTIImpl::getAddressComputationCost(Type *Ty, 1057 ScalarEvolution *SE, 1058 const SCEV *Ptr) { 1059 // Address computations in vectorized code with non-consecutive addresses will 1060 // likely result in more instructions compared to scalar code where the 1061 // computation can more often be merged into the index mode. The resulting 1062 // extra micro-ops can significantly decrease throughput. 1063 unsigned NumVectorInstToHideOverhead = 10; 1064 int MaxMergeDistance = 64; 1065 1066 if (ST->hasNEON()) { 1067 if (Ty->isVectorTy() && SE && 1068 !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MaxMergeDistance + 1)) 1069 return NumVectorInstToHideOverhead; 1070 1071 // In many cases the address computation is not merged into the instruction 1072 // addressing mode. 1073 return 1; 1074 } 1075 return BaseT::getAddressComputationCost(Ty, SE, Ptr); 1076 } 1077 1078 bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) { 1079 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) { 1080 // If a VCTP is part of a chain, it's already profitable and shouldn't be 1081 // optimized, else LSR may block tail-predication. 1082 switch (II->getIntrinsicID()) { 1083 case Intrinsic::arm_mve_vctp8: 1084 case Intrinsic::arm_mve_vctp16: 1085 case Intrinsic::arm_mve_vctp32: 1086 case Intrinsic::arm_mve_vctp64: 1087 return true; 1088 default: 1089 break; 1090 } 1091 } 1092 return false; 1093 } 1094 1095 bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment) { 1096 if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps()) 1097 return false; 1098 1099 if (auto *VecTy = dyn_cast<FixedVectorType>(DataTy)) { 1100 // Don't support v2i1 yet. 1101 if (VecTy->getNumElements() == 2) 1102 return false; 1103 1104 // We don't support extending fp types. 1105 unsigned VecWidth = DataTy->getPrimitiveSizeInBits(); 1106 if (VecWidth != 128 && VecTy->getElementType()->isFloatingPointTy()) 1107 return false; 1108 } 1109 1110 unsigned EltWidth = DataTy->getScalarSizeInBits(); 1111 return (EltWidth == 32 && Alignment >= 4) || 1112 (EltWidth == 16 && Alignment >= 2) || (EltWidth == 8); 1113 } 1114 1115 bool ARMTTIImpl::isLegalMaskedGather(Type *Ty, Align Alignment) { 1116 if (!EnableMaskedGatherScatters || !ST->hasMVEIntegerOps()) 1117 return false; 1118 1119 unsigned EltWidth = Ty->getScalarSizeInBits(); 1120 return ((EltWidth == 32 && Alignment >= 4) || 1121 (EltWidth == 16 && Alignment >= 2) || EltWidth == 8); 1122 } 1123 1124 /// Given a memcpy/memset/memmove instruction, return the number of memory 1125 /// operations performed, via querying findOptimalMemOpLowering. Returns -1 if a 1126 /// call is used. 1127 int ARMTTIImpl::getNumMemOps(const IntrinsicInst *I) const { 1128 MemOp MOp; 1129 unsigned DstAddrSpace = ~0u; 1130 unsigned SrcAddrSpace = ~0u; 1131 const Function *F = I->getParent()->getParent(); 1132 1133 if (const auto *MC = dyn_cast<MemTransferInst>(I)) { 1134 ConstantInt *C = dyn_cast<ConstantInt>(MC->getLength()); 1135 // If 'size' is not a constant, a library call will be generated. 1136 if (!C) 1137 return -1; 1138 1139 const unsigned Size = C->getValue().getZExtValue(); 1140 const Align DstAlign = *MC->getDestAlign(); 1141 const Align SrcAlign = *MC->getSourceAlign(); 1142 1143 MOp = MemOp::Copy(Size, /*DstAlignCanChange*/ false, DstAlign, SrcAlign, 1144 /*IsVolatile*/ false); 1145 DstAddrSpace = MC->getDestAddressSpace(); 1146 SrcAddrSpace = MC->getSourceAddressSpace(); 1147 } 1148 else if (const auto *MS = dyn_cast<MemSetInst>(I)) { 1149 ConstantInt *C = dyn_cast<ConstantInt>(MS->getLength()); 1150 // If 'size' is not a constant, a library call will be generated. 1151 if (!C) 1152 return -1; 1153 1154 const unsigned Size = C->getValue().getZExtValue(); 1155 const Align DstAlign = *MS->getDestAlign(); 1156 1157 MOp = MemOp::Set(Size, /*DstAlignCanChange*/ false, DstAlign, 1158 /*IsZeroMemset*/ false, /*IsVolatile*/ false); 1159 DstAddrSpace = MS->getDestAddressSpace(); 1160 } 1161 else 1162 llvm_unreachable("Expected a memcpy/move or memset!"); 1163 1164 unsigned Limit, Factor = 2; 1165 switch(I->getIntrinsicID()) { 1166 case Intrinsic::memcpy: 1167 Limit = TLI->getMaxStoresPerMemcpy(F->hasMinSize()); 1168 break; 1169 case Intrinsic::memmove: 1170 Limit = TLI->getMaxStoresPerMemmove(F->hasMinSize()); 1171 break; 1172 case Intrinsic::memset: 1173 Limit = TLI->getMaxStoresPerMemset(F->hasMinSize()); 1174 Factor = 1; 1175 break; 1176 default: 1177 llvm_unreachable("Expected a memcpy/move or memset!"); 1178 } 1179 1180 // MemOps will be poplulated with a list of data types that needs to be 1181 // loaded and stored. That's why we multiply the number of elements by 2 to 1182 // get the cost for this memcpy. 1183 std::vector<EVT> MemOps; 1184 if (getTLI()->findOptimalMemOpLowering( 1185 MemOps, Limit, MOp, DstAddrSpace, 1186 SrcAddrSpace, F->getAttributes())) 1187 return MemOps.size() * Factor; 1188 1189 // If we can't find an optimal memop lowering, return the default cost 1190 return -1; 1191 } 1192 1193 InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) { 1194 int NumOps = getNumMemOps(cast<IntrinsicInst>(I)); 1195 1196 // To model the cost of a library call, we assume 1 for the call, and 1197 // 3 for the argument setup. 1198 if (NumOps == -1) 1199 return 4; 1200 return NumOps; 1201 } 1202 1203 InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, 1204 VectorType *Tp, ArrayRef<int> Mask, 1205 int Index, VectorType *SubTp) { 1206 Kind = improveShuffleKindFromMask(Kind, Mask); 1207 if (ST->hasNEON()) { 1208 if (Kind == TTI::SK_Broadcast) { 1209 static const CostTblEntry NEONDupTbl[] = { 1210 // VDUP handles these cases. 1211 {ISD::VECTOR_SHUFFLE, MVT::v2i32, 1}, 1212 {ISD::VECTOR_SHUFFLE, MVT::v2f32, 1}, 1213 {ISD::VECTOR_SHUFFLE, MVT::v2i64, 1}, 1214 {ISD::VECTOR_SHUFFLE, MVT::v2f64, 1}, 1215 {ISD::VECTOR_SHUFFLE, MVT::v4i16, 1}, 1216 {ISD::VECTOR_SHUFFLE, MVT::v8i8, 1}, 1217 1218 {ISD::VECTOR_SHUFFLE, MVT::v4i32, 1}, 1219 {ISD::VECTOR_SHUFFLE, MVT::v4f32, 1}, 1220 {ISD::VECTOR_SHUFFLE, MVT::v8i16, 1}, 1221 {ISD::VECTOR_SHUFFLE, MVT::v16i8, 1}}; 1222 1223 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp); 1224 if (const auto *Entry = 1225 CostTableLookup(NEONDupTbl, ISD::VECTOR_SHUFFLE, LT.second)) 1226 return LT.first * Entry->Cost; 1227 } 1228 if (Kind == TTI::SK_Reverse) { 1229 static const CostTblEntry NEONShuffleTbl[] = { 1230 // Reverse shuffle cost one instruction if we are shuffling within a 1231 // double word (vrev) or two if we shuffle a quad word (vrev, vext). 1232 {ISD::VECTOR_SHUFFLE, MVT::v2i32, 1}, 1233 {ISD::VECTOR_SHUFFLE, MVT::v2f32, 1}, 1234 {ISD::VECTOR_SHUFFLE, MVT::v2i64, 1}, 1235 {ISD::VECTOR_SHUFFLE, MVT::v2f64, 1}, 1236 {ISD::VECTOR_SHUFFLE, MVT::v4i16, 1}, 1237 {ISD::VECTOR_SHUFFLE, MVT::v8i8, 1}, 1238 1239 {ISD::VECTOR_SHUFFLE, MVT::v4i32, 2}, 1240 {ISD::VECTOR_SHUFFLE, MVT::v4f32, 2}, 1241 {ISD::VECTOR_SHUFFLE, MVT::v8i16, 2}, 1242 {ISD::VECTOR_SHUFFLE, MVT::v16i8, 2}}; 1243 1244 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp); 1245 if (const auto *Entry = 1246 CostTableLookup(NEONShuffleTbl, ISD::VECTOR_SHUFFLE, LT.second)) 1247 return LT.first * Entry->Cost; 1248 } 1249 if (Kind == TTI::SK_Select) { 1250 static const CostTblEntry NEONSelShuffleTbl[] = { 1251 // Select shuffle cost table for ARM. Cost is the number of 1252 // instructions 1253 // required to create the shuffled vector. 1254 1255 {ISD::VECTOR_SHUFFLE, MVT::v2f32, 1}, 1256 {ISD::VECTOR_SHUFFLE, MVT::v2i64, 1}, 1257 {ISD::VECTOR_SHUFFLE, MVT::v2f64, 1}, 1258 {ISD::VECTOR_SHUFFLE, MVT::v2i32, 1}, 1259 1260 {ISD::VECTOR_SHUFFLE, MVT::v4i32, 2}, 1261 {ISD::VECTOR_SHUFFLE, MVT::v4f32, 2}, 1262 {ISD::VECTOR_SHUFFLE, MVT::v4i16, 2}, 1263 1264 {ISD::VECTOR_SHUFFLE, MVT::v8i16, 16}, 1265 1266 {ISD::VECTOR_SHUFFLE, MVT::v16i8, 32}}; 1267 1268 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp); 1269 if (const auto *Entry = CostTableLookup(NEONSelShuffleTbl, 1270 ISD::VECTOR_SHUFFLE, LT.second)) 1271 return LT.first * Entry->Cost; 1272 } 1273 } 1274 if (ST->hasMVEIntegerOps()) { 1275 if (Kind == TTI::SK_Broadcast) { 1276 static const CostTblEntry MVEDupTbl[] = { 1277 // VDUP handles these cases. 1278 {ISD::VECTOR_SHUFFLE, MVT::v4i32, 1}, 1279 {ISD::VECTOR_SHUFFLE, MVT::v8i16, 1}, 1280 {ISD::VECTOR_SHUFFLE, MVT::v16i8, 1}, 1281 {ISD::VECTOR_SHUFFLE, MVT::v4f32, 1}, 1282 {ISD::VECTOR_SHUFFLE, MVT::v8f16, 1}}; 1283 1284 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp); 1285 if (const auto *Entry = CostTableLookup(MVEDupTbl, ISD::VECTOR_SHUFFLE, 1286 LT.second)) 1287 return LT.first * Entry->Cost * 1288 ST->getMVEVectorCostFactor(TTI::TCK_RecipThroughput); 1289 } 1290 1291 if (!Mask.empty()) { 1292 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp); 1293 if (Mask.size() <= LT.second.getVectorNumElements() && 1294 (isVREVMask(Mask, LT.second, 16) || isVREVMask(Mask, LT.second, 32) || 1295 isVREVMask(Mask, LT.second, 64))) 1296 return ST->getMVEVectorCostFactor(TTI::TCK_RecipThroughput) * LT.first; 1297 } 1298 } 1299 1300 int BaseCost = ST->hasMVEIntegerOps() && Tp->isVectorTy() 1301 ? ST->getMVEVectorCostFactor(TTI::TCK_RecipThroughput) 1302 : 1; 1303 return BaseCost * BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp); 1304 } 1305 1306 InstructionCost ARMTTIImpl::getArithmeticInstrCost( 1307 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, 1308 TTI::OperandValueKind Op1Info, TTI::OperandValueKind Op2Info, 1309 TTI::OperandValueProperties Opd1PropInfo, 1310 TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args, 1311 const Instruction *CxtI) { 1312 int ISDOpcode = TLI->InstructionOpcodeToISD(Opcode); 1313 if (ST->isThumb() && CostKind == TTI::TCK_CodeSize && Ty->isIntegerTy(1)) { 1314 // Make operations on i1 relatively expensive as this often involves 1315 // combining predicates. AND and XOR should be easier to handle with IT 1316 // blocks. 1317 switch (ISDOpcode) { 1318 default: 1319 break; 1320 case ISD::AND: 1321 case ISD::XOR: 1322 return 2; 1323 case ISD::OR: 1324 return 3; 1325 } 1326 } 1327 1328 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty); 1329 1330 if (ST->hasNEON()) { 1331 const unsigned FunctionCallDivCost = 20; 1332 const unsigned ReciprocalDivCost = 10; 1333 static const CostTblEntry CostTbl[] = { 1334 // Division. 1335 // These costs are somewhat random. Choose a cost of 20 to indicate that 1336 // vectorizing devision (added function call) is going to be very expensive. 1337 // Double registers types. 1338 { ISD::SDIV, MVT::v1i64, 1 * FunctionCallDivCost}, 1339 { ISD::UDIV, MVT::v1i64, 1 * FunctionCallDivCost}, 1340 { ISD::SREM, MVT::v1i64, 1 * FunctionCallDivCost}, 1341 { ISD::UREM, MVT::v1i64, 1 * FunctionCallDivCost}, 1342 { ISD::SDIV, MVT::v2i32, 2 * FunctionCallDivCost}, 1343 { ISD::UDIV, MVT::v2i32, 2 * FunctionCallDivCost}, 1344 { ISD::SREM, MVT::v2i32, 2 * FunctionCallDivCost}, 1345 { ISD::UREM, MVT::v2i32, 2 * FunctionCallDivCost}, 1346 { ISD::SDIV, MVT::v4i16, ReciprocalDivCost}, 1347 { ISD::UDIV, MVT::v4i16, ReciprocalDivCost}, 1348 { ISD::SREM, MVT::v4i16, 4 * FunctionCallDivCost}, 1349 { ISD::UREM, MVT::v4i16, 4 * FunctionCallDivCost}, 1350 { ISD::SDIV, MVT::v8i8, ReciprocalDivCost}, 1351 { ISD::UDIV, MVT::v8i8, ReciprocalDivCost}, 1352 { ISD::SREM, MVT::v8i8, 8 * FunctionCallDivCost}, 1353 { ISD::UREM, MVT::v8i8, 8 * FunctionCallDivCost}, 1354 // Quad register types. 1355 { ISD::SDIV, MVT::v2i64, 2 * FunctionCallDivCost}, 1356 { ISD::UDIV, MVT::v2i64, 2 * FunctionCallDivCost}, 1357 { ISD::SREM, MVT::v2i64, 2 * FunctionCallDivCost}, 1358 { ISD::UREM, MVT::v2i64, 2 * FunctionCallDivCost}, 1359 { ISD::SDIV, MVT::v4i32, 4 * FunctionCallDivCost}, 1360 { ISD::UDIV, MVT::v4i32, 4 * FunctionCallDivCost}, 1361 { ISD::SREM, MVT::v4i32, 4 * FunctionCallDivCost}, 1362 { ISD::UREM, MVT::v4i32, 4 * FunctionCallDivCost}, 1363 { ISD::SDIV, MVT::v8i16, 8 * FunctionCallDivCost}, 1364 { ISD::UDIV, MVT::v8i16, 8 * FunctionCallDivCost}, 1365 { ISD::SREM, MVT::v8i16, 8 * FunctionCallDivCost}, 1366 { ISD::UREM, MVT::v8i16, 8 * FunctionCallDivCost}, 1367 { ISD::SDIV, MVT::v16i8, 16 * FunctionCallDivCost}, 1368 { ISD::UDIV, MVT::v16i8, 16 * FunctionCallDivCost}, 1369 { ISD::SREM, MVT::v16i8, 16 * FunctionCallDivCost}, 1370 { ISD::UREM, MVT::v16i8, 16 * FunctionCallDivCost}, 1371 // Multiplication. 1372 }; 1373 1374 if (const auto *Entry = CostTableLookup(CostTbl, ISDOpcode, LT.second)) 1375 return LT.first * Entry->Cost; 1376 1377 InstructionCost Cost = BaseT::getArithmeticInstrCost( 1378 Opcode, Ty, CostKind, Op1Info, Op2Info, Opd1PropInfo, Opd2PropInfo); 1379 1380 // This is somewhat of a hack. The problem that we are facing is that SROA 1381 // creates a sequence of shift, and, or instructions to construct values. 1382 // These sequences are recognized by the ISel and have zero-cost. Not so for 1383 // the vectorized code. Because we have support for v2i64 but not i64 those 1384 // sequences look particularly beneficial to vectorize. 1385 // To work around this we increase the cost of v2i64 operations to make them 1386 // seem less beneficial. 1387 if (LT.second == MVT::v2i64 && 1388 Op2Info == TargetTransformInfo::OK_UniformConstantValue) 1389 Cost += 4; 1390 1391 return Cost; 1392 } 1393 1394 // If this operation is a shift on arm/thumb2, it might well be folded into 1395 // the following instruction, hence having a cost of 0. 1396 auto LooksLikeAFreeShift = [&]() { 1397 if (ST->isThumb1Only() || Ty->isVectorTy()) 1398 return false; 1399 1400 if (!CxtI || !CxtI->hasOneUse() || !CxtI->isShift()) 1401 return false; 1402 if (Op2Info != TargetTransformInfo::OK_UniformConstantValue) 1403 return false; 1404 1405 // Folded into a ADC/ADD/AND/BIC/CMP/EOR/MVN/ORR/ORN/RSB/SBC/SUB 1406 switch (cast<Instruction>(CxtI->user_back())->getOpcode()) { 1407 case Instruction::Add: 1408 case Instruction::Sub: 1409 case Instruction::And: 1410 case Instruction::Xor: 1411 case Instruction::Or: 1412 case Instruction::ICmp: 1413 return true; 1414 default: 1415 return false; 1416 } 1417 }; 1418 if (LooksLikeAFreeShift()) 1419 return 0; 1420 1421 // Default to cheap (throughput/size of 1 instruction) but adjust throughput 1422 // for "multiple beats" potentially needed by MVE instructions. 1423 int BaseCost = 1; 1424 if (ST->hasMVEIntegerOps() && Ty->isVectorTy()) 1425 BaseCost = ST->getMVEVectorCostFactor(CostKind); 1426 1427 // The rest of this mostly follows what is done in BaseT::getArithmeticInstrCost, 1428 // without treating floats as more expensive that scalars or increasing the 1429 // costs for custom operations. The results is also multiplied by the 1430 // MVEVectorCostFactor where appropriate. 1431 if (TLI->isOperationLegalOrCustomOrPromote(ISDOpcode, LT.second)) 1432 return LT.first * BaseCost; 1433 1434 // Else this is expand, assume that we need to scalarize this op. 1435 if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) { 1436 unsigned Num = VTy->getNumElements(); 1437 InstructionCost Cost = 1438 getArithmeticInstrCost(Opcode, Ty->getScalarType(), CostKind); 1439 // Return the cost of multiple scalar invocation plus the cost of 1440 // inserting and extracting the values. 1441 SmallVector<Type *> Tys(Args.size(), Ty); 1442 return BaseT::getScalarizationOverhead(VTy, Args, Tys) + Num * Cost; 1443 } 1444 1445 return BaseCost; 1446 } 1447 1448 InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, 1449 MaybeAlign Alignment, 1450 unsigned AddressSpace, 1451 TTI::TargetCostKind CostKind, 1452 const Instruction *I) { 1453 // TODO: Handle other cost kinds. 1454 if (CostKind != TTI::TCK_RecipThroughput) 1455 return 1; 1456 1457 // Type legalization can't handle structs 1458 if (TLI->getValueType(DL, Src, true) == MVT::Other) 1459 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, 1460 CostKind); 1461 1462 if (ST->hasNEON() && Src->isVectorTy() && 1463 (Alignment && *Alignment != Align(16)) && 1464 cast<VectorType>(Src)->getElementType()->isDoubleTy()) { 1465 // Unaligned loads/stores are extremely inefficient. 1466 // We need 4 uops for vst.1/vld.1 vs 1uop for vldr/vstr. 1467 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Src); 1468 return LT.first * 4; 1469 } 1470 1471 // MVE can optimize a fpext(load(4xhalf)) using an extending integer load. 1472 // Same for stores. 1473 if (ST->hasMVEFloatOps() && isa<FixedVectorType>(Src) && I && 1474 ((Opcode == Instruction::Load && I->hasOneUse() && 1475 isa<FPExtInst>(*I->user_begin())) || 1476 (Opcode == Instruction::Store && isa<FPTruncInst>(I->getOperand(0))))) { 1477 FixedVectorType *SrcVTy = cast<FixedVectorType>(Src); 1478 Type *DstTy = 1479 Opcode == Instruction::Load 1480 ? (*I->user_begin())->getType() 1481 : cast<Instruction>(I->getOperand(0))->getOperand(0)->getType(); 1482 if (SrcVTy->getNumElements() == 4 && SrcVTy->getScalarType()->isHalfTy() && 1483 DstTy->getScalarType()->isFloatTy()) 1484 return ST->getMVEVectorCostFactor(CostKind); 1485 } 1486 1487 int BaseCost = ST->hasMVEIntegerOps() && Src->isVectorTy() 1488 ? ST->getMVEVectorCostFactor(CostKind) 1489 : 1; 1490 return BaseCost * BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, 1491 CostKind, I); 1492 } 1493 1494 InstructionCost 1495 ARMTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, 1496 unsigned AddressSpace, 1497 TTI::TargetCostKind CostKind) { 1498 if (ST->hasMVEIntegerOps()) { 1499 if (Opcode == Instruction::Load && isLegalMaskedLoad(Src, Alignment)) 1500 return ST->getMVEVectorCostFactor(CostKind); 1501 if (Opcode == Instruction::Store && isLegalMaskedStore(Src, Alignment)) 1502 return ST->getMVEVectorCostFactor(CostKind); 1503 } 1504 if (!isa<FixedVectorType>(Src)) 1505 return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, 1506 CostKind); 1507 // Scalar cost, which is currently very high due to the efficiency of the 1508 // generated code. 1509 return cast<FixedVectorType>(Src)->getNumElements() * 8; 1510 } 1511 1512 InstructionCost ARMTTIImpl::getInterleavedMemoryOpCost( 1513 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, 1514 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, 1515 bool UseMaskForCond, bool UseMaskForGaps) { 1516 assert(Factor >= 2 && "Invalid interleave factor"); 1517 assert(isa<VectorType>(VecTy) && "Expect a vector type"); 1518 1519 // vldN/vstN doesn't support vector types of i64/f64 element. 1520 bool EltIs64Bits = DL.getTypeSizeInBits(VecTy->getScalarType()) == 64; 1521 1522 if (Factor <= TLI->getMaxSupportedInterleaveFactor() && !EltIs64Bits && 1523 !UseMaskForCond && !UseMaskForGaps) { 1524 unsigned NumElts = cast<FixedVectorType>(VecTy)->getNumElements(); 1525 auto *SubVecTy = 1526 FixedVectorType::get(VecTy->getScalarType(), NumElts / Factor); 1527 1528 // vldN/vstN only support legal vector types of size 64 or 128 in bits. 1529 // Accesses having vector types that are a multiple of 128 bits can be 1530 // matched to more than one vldN/vstN instruction. 1531 int BaseCost = 1532 ST->hasMVEIntegerOps() ? ST->getMVEVectorCostFactor(CostKind) : 1; 1533 if (NumElts % Factor == 0 && 1534 TLI->isLegalInterleavedAccessType(Factor, SubVecTy, Alignment, DL)) 1535 return Factor * BaseCost * TLI->getNumInterleavedAccesses(SubVecTy, DL); 1536 1537 // Some smaller than legal interleaved patterns are cheap as we can make 1538 // use of the vmovn or vrev patterns to interleave a standard load. This is 1539 // true for v4i8, v8i8 and v4i16 at least (but not for v4f16 as it is 1540 // promoted differently). The cost of 2 here is then a load and vrev or 1541 // vmovn. 1542 if (ST->hasMVEIntegerOps() && Factor == 2 && NumElts / Factor > 2 && 1543 VecTy->isIntOrIntVectorTy() && 1544 DL.getTypeSizeInBits(SubVecTy).getFixedSize() <= 64) 1545 return 2 * BaseCost; 1546 } 1547 1548 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, 1549 Alignment, AddressSpace, CostKind, 1550 UseMaskForCond, UseMaskForGaps); 1551 } 1552 1553 InstructionCost ARMTTIImpl::getGatherScatterOpCost( 1554 unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, 1555 Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) { 1556 using namespace PatternMatch; 1557 if (!ST->hasMVEIntegerOps() || !EnableMaskedGatherScatters) 1558 return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, 1559 Alignment, CostKind, I); 1560 1561 assert(DataTy->isVectorTy() && "Can't do gather/scatters on scalar!"); 1562 auto *VTy = cast<FixedVectorType>(DataTy); 1563 1564 // TODO: Splitting, once we do that. 1565 1566 unsigned NumElems = VTy->getNumElements(); 1567 unsigned EltSize = VTy->getScalarSizeInBits(); 1568 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, DataTy); 1569 1570 // For now, it is assumed that for the MVE gather instructions the loads are 1571 // all effectively serialised. This means the cost is the scalar cost 1572 // multiplied by the number of elements being loaded. This is possibly very 1573 // conservative, but even so we still end up vectorising loops because the 1574 // cost per iteration for many loops is lower than for scalar loops. 1575 InstructionCost VectorCost = 1576 NumElems * LT.first * ST->getMVEVectorCostFactor(CostKind); 1577 // The scalarization cost should be a lot higher. We use the number of vector 1578 // elements plus the scalarization overhead. 1579 InstructionCost ScalarCost = 1580 NumElems * LT.first + BaseT::getScalarizationOverhead(VTy, true, false) + 1581 BaseT::getScalarizationOverhead(VTy, false, true); 1582 1583 if (EltSize < 8 || Alignment < EltSize / 8) 1584 return ScalarCost; 1585 1586 unsigned ExtSize = EltSize; 1587 // Check whether there's a single user that asks for an extended type 1588 if (I != nullptr) { 1589 // Dependent of the caller of this function, a gather instruction will 1590 // either have opcode Instruction::Load or be a call to the masked_gather 1591 // intrinsic 1592 if ((I->getOpcode() == Instruction::Load || 1593 match(I, m_Intrinsic<Intrinsic::masked_gather>())) && 1594 I->hasOneUse()) { 1595 const User *Us = *I->users().begin(); 1596 if (isa<ZExtInst>(Us) || isa<SExtInst>(Us)) { 1597 // only allow valid type combinations 1598 unsigned TypeSize = 1599 cast<Instruction>(Us)->getType()->getScalarSizeInBits(); 1600 if (((TypeSize == 32 && (EltSize == 8 || EltSize == 16)) || 1601 (TypeSize == 16 && EltSize == 8)) && 1602 TypeSize * NumElems == 128) { 1603 ExtSize = TypeSize; 1604 } 1605 } 1606 } 1607 // Check whether the input data needs to be truncated 1608 TruncInst *T; 1609 if ((I->getOpcode() == Instruction::Store || 1610 match(I, m_Intrinsic<Intrinsic::masked_scatter>())) && 1611 (T = dyn_cast<TruncInst>(I->getOperand(0)))) { 1612 // Only allow valid type combinations 1613 unsigned TypeSize = T->getOperand(0)->getType()->getScalarSizeInBits(); 1614 if (((EltSize == 16 && TypeSize == 32) || 1615 (EltSize == 8 && (TypeSize == 32 || TypeSize == 16))) && 1616 TypeSize * NumElems == 128) 1617 ExtSize = TypeSize; 1618 } 1619 } 1620 1621 if (ExtSize * NumElems != 128 || NumElems < 4) 1622 return ScalarCost; 1623 1624 // Any (aligned) i32 gather will not need to be scalarised. 1625 if (ExtSize == 32) 1626 return VectorCost; 1627 // For smaller types, we need to ensure that the gep's inputs are correctly 1628 // extended from a small enough value. Other sizes (including i64) are 1629 // scalarized for now. 1630 if (ExtSize != 8 && ExtSize != 16) 1631 return ScalarCost; 1632 1633 if (const auto *BC = dyn_cast<BitCastInst>(Ptr)) 1634 Ptr = BC->getOperand(0); 1635 if (const auto *GEP = dyn_cast<GetElementPtrInst>(Ptr)) { 1636 if (GEP->getNumOperands() != 2) 1637 return ScalarCost; 1638 unsigned Scale = DL.getTypeAllocSize(GEP->getResultElementType()); 1639 // Scale needs to be correct (which is only relevant for i16s). 1640 if (Scale != 1 && Scale * 8 != ExtSize) 1641 return ScalarCost; 1642 // And we need to zext (not sext) the indexes from a small enough type. 1643 if (const auto *ZExt = dyn_cast<ZExtInst>(GEP->getOperand(1))) { 1644 if (ZExt->getOperand(0)->getType()->getScalarSizeInBits() <= ExtSize) 1645 return VectorCost; 1646 } 1647 return ScalarCost; 1648 } 1649 return ScalarCost; 1650 } 1651 1652 InstructionCost 1653 ARMTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy, 1654 Optional<FastMathFlags> FMF, 1655 TTI::TargetCostKind CostKind) { 1656 if (TTI::requiresOrderedReduction(FMF)) 1657 return BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind); 1658 1659 EVT ValVT = TLI->getValueType(DL, ValTy); 1660 int ISD = TLI->InstructionOpcodeToISD(Opcode); 1661 if (!ST->hasMVEIntegerOps() || !ValVT.isSimple() || ISD != ISD::ADD) 1662 return BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind); 1663 1664 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, ValTy); 1665 1666 static const CostTblEntry CostTblAdd[]{ 1667 {ISD::ADD, MVT::v16i8, 1}, 1668 {ISD::ADD, MVT::v8i16, 1}, 1669 {ISD::ADD, MVT::v4i32, 1}, 1670 }; 1671 if (const auto *Entry = CostTableLookup(CostTblAdd, ISD, LT.second)) 1672 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind) * LT.first; 1673 1674 return BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind); 1675 } 1676 1677 InstructionCost 1678 ARMTTIImpl::getExtendedAddReductionCost(bool IsMLA, bool IsUnsigned, 1679 Type *ResTy, VectorType *ValTy, 1680 TTI::TargetCostKind CostKind) { 1681 EVT ValVT = TLI->getValueType(DL, ValTy); 1682 EVT ResVT = TLI->getValueType(DL, ResTy); 1683 1684 if (ST->hasMVEIntegerOps() && ValVT.isSimple() && ResVT.isSimple()) { 1685 std::pair<InstructionCost, MVT> LT = 1686 TLI->getTypeLegalizationCost(DL, ValTy); 1687 1688 // The legal cases are: 1689 // VADDV u/s 8/16/32 1690 // VMLAV u/s 8/16/32 1691 // VADDLV u/s 32 1692 // VMLALV u/s 16/32 1693 // Codegen currently cannot always handle larger than legal vectors very 1694 // well, especially for predicated reductions where the mask needs to be 1695 // split, so restrict to 128bit or smaller input types. 1696 unsigned RevVTSize = ResVT.getSizeInBits(); 1697 if (ValVT.getSizeInBits() <= 128 && 1698 ((LT.second == MVT::v16i8 && RevVTSize <= 32) || 1699 (LT.second == MVT::v8i16 && RevVTSize <= (IsMLA ? 64u : 32u)) || 1700 (LT.second == MVT::v4i32 && RevVTSize <= 64))) 1701 return ST->getMVEVectorCostFactor(CostKind) * LT.first; 1702 } 1703 1704 return BaseT::getExtendedAddReductionCost(IsMLA, IsUnsigned, ResTy, ValTy, 1705 CostKind); 1706 } 1707 1708 InstructionCost 1709 ARMTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, 1710 TTI::TargetCostKind CostKind) { 1711 switch (ICA.getID()) { 1712 case Intrinsic::get_active_lane_mask: 1713 // Currently we make a somewhat optimistic assumption that 1714 // active_lane_mask's are always free. In reality it may be freely folded 1715 // into a tail predicated loop, expanded into a VCPT or expanded into a lot 1716 // of add/icmp code. We may need to improve this in the future, but being 1717 // able to detect if it is free or not involves looking at a lot of other 1718 // code. We currently assume that the vectorizer inserted these, and knew 1719 // what it was doing in adding one. 1720 if (ST->hasMVEIntegerOps()) 1721 return 0; 1722 break; 1723 case Intrinsic::sadd_sat: 1724 case Intrinsic::ssub_sat: 1725 case Intrinsic::uadd_sat: 1726 case Intrinsic::usub_sat: { 1727 if (!ST->hasMVEIntegerOps()) 1728 break; 1729 Type *VT = ICA.getReturnType(); 1730 1731 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, VT); 1732 if (LT.second == MVT::v4i32 || LT.second == MVT::v8i16 || 1733 LT.second == MVT::v16i8) { 1734 // This is a base cost of 1 for the vqadd, plus 3 extract shifts if we 1735 // need to extend the type, as it uses shr(qadd(shl, shl)). 1736 unsigned Instrs = 1737 LT.second.getScalarSizeInBits() == VT->getScalarSizeInBits() ? 1 : 4; 1738 return LT.first * ST->getMVEVectorCostFactor(CostKind) * Instrs; 1739 } 1740 break; 1741 } 1742 case Intrinsic::abs: 1743 case Intrinsic::smin: 1744 case Intrinsic::smax: 1745 case Intrinsic::umin: 1746 case Intrinsic::umax: { 1747 if (!ST->hasMVEIntegerOps()) 1748 break; 1749 Type *VT = ICA.getReturnType(); 1750 1751 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, VT); 1752 if (LT.second == MVT::v4i32 || LT.second == MVT::v8i16 || 1753 LT.second == MVT::v16i8) 1754 return LT.first * ST->getMVEVectorCostFactor(CostKind); 1755 break; 1756 } 1757 case Intrinsic::minnum: 1758 case Intrinsic::maxnum: { 1759 if (!ST->hasMVEFloatOps()) 1760 break; 1761 Type *VT = ICA.getReturnType(); 1762 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, VT); 1763 if (LT.second == MVT::v4f32 || LT.second == MVT::v8f16) 1764 return LT.first * ST->getMVEVectorCostFactor(CostKind); 1765 break; 1766 } 1767 } 1768 1769 return BaseT::getIntrinsicInstrCost(ICA, CostKind); 1770 } 1771 1772 bool ARMTTIImpl::isLoweredToCall(const Function *F) { 1773 if (!F->isIntrinsic()) 1774 BaseT::isLoweredToCall(F); 1775 1776 // Assume all Arm-specific intrinsics map to an instruction. 1777 if (F->getName().startswith("llvm.arm")) 1778 return false; 1779 1780 switch (F->getIntrinsicID()) { 1781 default: break; 1782 case Intrinsic::powi: 1783 case Intrinsic::sin: 1784 case Intrinsic::cos: 1785 case Intrinsic::pow: 1786 case Intrinsic::log: 1787 case Intrinsic::log10: 1788 case Intrinsic::log2: 1789 case Intrinsic::exp: 1790 case Intrinsic::exp2: 1791 return true; 1792 case Intrinsic::sqrt: 1793 case Intrinsic::fabs: 1794 case Intrinsic::copysign: 1795 case Intrinsic::floor: 1796 case Intrinsic::ceil: 1797 case Intrinsic::trunc: 1798 case Intrinsic::rint: 1799 case Intrinsic::nearbyint: 1800 case Intrinsic::round: 1801 case Intrinsic::canonicalize: 1802 case Intrinsic::lround: 1803 case Intrinsic::llround: 1804 case Intrinsic::lrint: 1805 case Intrinsic::llrint: 1806 if (F->getReturnType()->isDoubleTy() && !ST->hasFP64()) 1807 return true; 1808 if (F->getReturnType()->isHalfTy() && !ST->hasFullFP16()) 1809 return true; 1810 // Some operations can be handled by vector instructions and assume 1811 // unsupported vectors will be expanded into supported scalar ones. 1812 // TODO Handle scalar operations properly. 1813 return !ST->hasFPARMv8Base() && !ST->hasVFP2Base(); 1814 case Intrinsic::masked_store: 1815 case Intrinsic::masked_load: 1816 case Intrinsic::masked_gather: 1817 case Intrinsic::masked_scatter: 1818 return !ST->hasMVEIntegerOps(); 1819 case Intrinsic::sadd_with_overflow: 1820 case Intrinsic::uadd_with_overflow: 1821 case Intrinsic::ssub_with_overflow: 1822 case Intrinsic::usub_with_overflow: 1823 case Intrinsic::sadd_sat: 1824 case Intrinsic::uadd_sat: 1825 case Intrinsic::ssub_sat: 1826 case Intrinsic::usub_sat: 1827 return false; 1828 } 1829 1830 return BaseT::isLoweredToCall(F); 1831 } 1832 1833 bool ARMTTIImpl::maybeLoweredToCall(Instruction &I) { 1834 unsigned ISD = TLI->InstructionOpcodeToISD(I.getOpcode()); 1835 EVT VT = TLI->getValueType(DL, I.getType(), true); 1836 if (TLI->getOperationAction(ISD, VT) == TargetLowering::LibCall) 1837 return true; 1838 1839 // Check if an intrinsic will be lowered to a call and assume that any 1840 // other CallInst will generate a bl. 1841 if (auto *Call = dyn_cast<CallInst>(&I)) { 1842 if (auto *II = dyn_cast<IntrinsicInst>(Call)) { 1843 switch(II->getIntrinsicID()) { 1844 case Intrinsic::memcpy: 1845 case Intrinsic::memset: 1846 case Intrinsic::memmove: 1847 return getNumMemOps(II) == -1; 1848 default: 1849 if (const Function *F = Call->getCalledFunction()) 1850 return isLoweredToCall(F); 1851 } 1852 } 1853 return true; 1854 } 1855 1856 // FPv5 provides conversions between integer, double-precision, 1857 // single-precision, and half-precision formats. 1858 switch (I.getOpcode()) { 1859 default: 1860 break; 1861 case Instruction::FPToSI: 1862 case Instruction::FPToUI: 1863 case Instruction::SIToFP: 1864 case Instruction::UIToFP: 1865 case Instruction::FPTrunc: 1866 case Instruction::FPExt: 1867 return !ST->hasFPARMv8Base(); 1868 } 1869 1870 // FIXME: Unfortunately the approach of checking the Operation Action does 1871 // not catch all cases of Legalization that use library calls. Our 1872 // Legalization step categorizes some transformations into library calls as 1873 // Custom, Expand or even Legal when doing type legalization. So for now 1874 // we have to special case for instance the SDIV of 64bit integers and the 1875 // use of floating point emulation. 1876 if (VT.isInteger() && VT.getSizeInBits() >= 64) { 1877 switch (ISD) { 1878 default: 1879 break; 1880 case ISD::SDIV: 1881 case ISD::UDIV: 1882 case ISD::SREM: 1883 case ISD::UREM: 1884 case ISD::SDIVREM: 1885 case ISD::UDIVREM: 1886 return true; 1887 } 1888 } 1889 1890 // Assume all other non-float operations are supported. 1891 if (!VT.isFloatingPoint()) 1892 return false; 1893 1894 // We'll need a library call to handle most floats when using soft. 1895 if (TLI->useSoftFloat()) { 1896 switch (I.getOpcode()) { 1897 default: 1898 return true; 1899 case Instruction::Alloca: 1900 case Instruction::Load: 1901 case Instruction::Store: 1902 case Instruction::Select: 1903 case Instruction::PHI: 1904 return false; 1905 } 1906 } 1907 1908 // We'll need a libcall to perform double precision operations on a single 1909 // precision only FPU. 1910 if (I.getType()->isDoubleTy() && !ST->hasFP64()) 1911 return true; 1912 1913 // Likewise for half precision arithmetic. 1914 if (I.getType()->isHalfTy() && !ST->hasFullFP16()) 1915 return true; 1916 1917 return false; 1918 } 1919 1920 bool ARMTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE, 1921 AssumptionCache &AC, 1922 TargetLibraryInfo *LibInfo, 1923 HardwareLoopInfo &HWLoopInfo) { 1924 // Low-overhead branches are only supported in the 'low-overhead branch' 1925 // extension of v8.1-m. 1926 if (!ST->hasLOB() || DisableLowOverheadLoops) { 1927 LLVM_DEBUG(dbgs() << "ARMHWLoops: Disabled\n"); 1928 return false; 1929 } 1930 1931 if (!SE.hasLoopInvariantBackedgeTakenCount(L)) { 1932 LLVM_DEBUG(dbgs() << "ARMHWLoops: No BETC\n"); 1933 return false; 1934 } 1935 1936 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); 1937 if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) { 1938 LLVM_DEBUG(dbgs() << "ARMHWLoops: Uncomputable BETC\n"); 1939 return false; 1940 } 1941 1942 const SCEV *TripCountSCEV = 1943 SE.getAddExpr(BackedgeTakenCount, 1944 SE.getOne(BackedgeTakenCount->getType())); 1945 1946 // We need to store the trip count in LR, a 32-bit register. 1947 if (SE.getUnsignedRangeMax(TripCountSCEV).getBitWidth() > 32) { 1948 LLVM_DEBUG(dbgs() << "ARMHWLoops: Trip count does not fit into 32bits\n"); 1949 return false; 1950 } 1951 1952 // Making a call will trash LR and clear LO_BRANCH_INFO, so there's little 1953 // point in generating a hardware loop if that's going to happen. 1954 1955 auto IsHardwareLoopIntrinsic = [](Instruction &I) { 1956 if (auto *Call = dyn_cast<IntrinsicInst>(&I)) { 1957 switch (Call->getIntrinsicID()) { 1958 default: 1959 break; 1960 case Intrinsic::start_loop_iterations: 1961 case Intrinsic::test_start_loop_iterations: 1962 case Intrinsic::loop_decrement: 1963 case Intrinsic::loop_decrement_reg: 1964 return true; 1965 } 1966 } 1967 return false; 1968 }; 1969 1970 // Scan the instructions to see if there's any that we know will turn into a 1971 // call or if this loop is already a low-overhead loop or will become a tail 1972 // predicated loop. 1973 bool IsTailPredLoop = false; 1974 auto ScanLoop = [&](Loop *L) { 1975 for (auto *BB : L->getBlocks()) { 1976 for (auto &I : *BB) { 1977 if (maybeLoweredToCall(I) || IsHardwareLoopIntrinsic(I) || 1978 isa<InlineAsm>(I)) { 1979 LLVM_DEBUG(dbgs() << "ARMHWLoops: Bad instruction: " << I << "\n"); 1980 return false; 1981 } 1982 if (auto *II = dyn_cast<IntrinsicInst>(&I)) 1983 IsTailPredLoop |= 1984 II->getIntrinsicID() == Intrinsic::get_active_lane_mask || 1985 II->getIntrinsicID() == Intrinsic::arm_mve_vctp8 || 1986 II->getIntrinsicID() == Intrinsic::arm_mve_vctp16 || 1987 II->getIntrinsicID() == Intrinsic::arm_mve_vctp32 || 1988 II->getIntrinsicID() == Intrinsic::arm_mve_vctp64; 1989 } 1990 } 1991 return true; 1992 }; 1993 1994 // Visit inner loops. 1995 for (auto Inner : *L) 1996 if (!ScanLoop(Inner)) 1997 return false; 1998 1999 if (!ScanLoop(L)) 2000 return false; 2001 2002 // TODO: Check whether the trip count calculation is expensive. If L is the 2003 // inner loop but we know it has a low trip count, calculating that trip 2004 // count (in the parent loop) may be detrimental. 2005 2006 LLVMContext &C = L->getHeader()->getContext(); 2007 HWLoopInfo.CounterInReg = true; 2008 HWLoopInfo.IsNestingLegal = false; 2009 HWLoopInfo.PerformEntryTest = AllowWLSLoops && !IsTailPredLoop; 2010 HWLoopInfo.CountType = Type::getInt32Ty(C); 2011 HWLoopInfo.LoopDecrement = ConstantInt::get(HWLoopInfo.CountType, 1); 2012 return true; 2013 } 2014 2015 static bool canTailPredicateInstruction(Instruction &I, int &ICmpCount) { 2016 // We don't allow icmp's, and because we only look at single block loops, 2017 // we simply count the icmps, i.e. there should only be 1 for the backedge. 2018 if (isa<ICmpInst>(&I) && ++ICmpCount > 1) 2019 return false; 2020 // FIXME: This is a workaround for poor cost modelling. Min/Max intrinsics are 2021 // not currently canonical, but soon will be. Code without them uses icmp, and 2022 // so is not tail predicated as per the condition above. In order to get the 2023 // same performance we treat min and max the same as an icmp for tailpred 2024 // purposes for the moment (we often rely on non-tailpred and higher VF's to 2025 // pick more optimial instructions like VQDMULH. They need to be recognized 2026 // directly by the vectorizer). 2027 if (auto *II = dyn_cast<IntrinsicInst>(&I)) 2028 if ((II->getIntrinsicID() == Intrinsic::smin || 2029 II->getIntrinsicID() == Intrinsic::smax || 2030 II->getIntrinsicID() == Intrinsic::umin || 2031 II->getIntrinsicID() == Intrinsic::umax) && 2032 ++ICmpCount > 1) 2033 return false; 2034 2035 if (isa<FCmpInst>(&I)) 2036 return false; 2037 2038 // We could allow extending/narrowing FP loads/stores, but codegen is 2039 // too inefficient so reject this for now. 2040 if (isa<FPExtInst>(&I) || isa<FPTruncInst>(&I)) 2041 return false; 2042 2043 // Extends have to be extending-loads 2044 if (isa<SExtInst>(&I) || isa<ZExtInst>(&I) ) 2045 if (!I.getOperand(0)->hasOneUse() || !isa<LoadInst>(I.getOperand(0))) 2046 return false; 2047 2048 // Truncs have to be narrowing-stores 2049 if (isa<TruncInst>(&I) ) 2050 if (!I.hasOneUse() || !isa<StoreInst>(*I.user_begin())) 2051 return false; 2052 2053 return true; 2054 } 2055 2056 // To set up a tail-predicated loop, we need to know the total number of 2057 // elements processed by that loop. Thus, we need to determine the element 2058 // size and: 2059 // 1) it should be uniform for all operations in the vector loop, so we 2060 // e.g. don't want any widening/narrowing operations. 2061 // 2) it should be smaller than i64s because we don't have vector operations 2062 // that work on i64s. 2063 // 3) we don't want elements to be reversed or shuffled, to make sure the 2064 // tail-predication masks/predicates the right lanes. 2065 // 2066 static bool canTailPredicateLoop(Loop *L, LoopInfo *LI, ScalarEvolution &SE, 2067 const DataLayout &DL, 2068 const LoopAccessInfo *LAI) { 2069 LLVM_DEBUG(dbgs() << "Tail-predication: checking allowed instructions\n"); 2070 2071 // If there are live-out values, it is probably a reduction. We can predicate 2072 // most reduction operations freely under MVE using a combination of 2073 // prefer-predicated-reduction-select and inloop reductions. We limit this to 2074 // floating point and integer reductions, but don't check for operators 2075 // specifically here. If the value ends up not being a reduction (and so the 2076 // vectorizer cannot tailfold the loop), we should fall back to standard 2077 // vectorization automatically. 2078 SmallVector< Instruction *, 8 > LiveOuts; 2079 LiveOuts = llvm::findDefsUsedOutsideOfLoop(L); 2080 bool ReductionsDisabled = 2081 EnableTailPredication == TailPredication::EnabledNoReductions || 2082 EnableTailPredication == TailPredication::ForceEnabledNoReductions; 2083 2084 for (auto *I : LiveOuts) { 2085 if (!I->getType()->isIntegerTy() && !I->getType()->isFloatTy() && 2086 !I->getType()->isHalfTy()) { 2087 LLVM_DEBUG(dbgs() << "Don't tail-predicate loop with non-integer/float " 2088 "live-out value\n"); 2089 return false; 2090 } 2091 if (ReductionsDisabled) { 2092 LLVM_DEBUG(dbgs() << "Reductions not enabled\n"); 2093 return false; 2094 } 2095 } 2096 2097 // Next, check that all instructions can be tail-predicated. 2098 PredicatedScalarEvolution PSE = LAI->getPSE(); 2099 SmallVector<Instruction *, 16> LoadStores; 2100 int ICmpCount = 0; 2101 2102 for (BasicBlock *BB : L->blocks()) { 2103 for (Instruction &I : BB->instructionsWithoutDebug()) { 2104 if (isa<PHINode>(&I)) 2105 continue; 2106 if (!canTailPredicateInstruction(I, ICmpCount)) { 2107 LLVM_DEBUG(dbgs() << "Instruction not allowed: "; I.dump()); 2108 return false; 2109 } 2110 2111 Type *T = I.getType(); 2112 if (T->getScalarSizeInBits() > 32) { 2113 LLVM_DEBUG(dbgs() << "Unsupported Type: "; T->dump()); 2114 return false; 2115 } 2116 if (isa<StoreInst>(I) || isa<LoadInst>(I)) { 2117 Value *Ptr = getLoadStorePointerOperand(&I); 2118 Type *AccessTy = getLoadStoreType(&I); 2119 int64_t NextStride = getPtrStride(PSE, AccessTy, Ptr, L); 2120 if (NextStride == 1) { 2121 // TODO: for now only allow consecutive strides of 1. We could support 2122 // other strides as long as it is uniform, but let's keep it simple 2123 // for now. 2124 continue; 2125 } else if (NextStride == -1 || 2126 (NextStride == 2 && MVEMaxSupportedInterleaveFactor >= 2) || 2127 (NextStride == 4 && MVEMaxSupportedInterleaveFactor >= 4)) { 2128 LLVM_DEBUG(dbgs() 2129 << "Consecutive strides of 2 found, vld2/vstr2 can't " 2130 "be tail-predicated\n."); 2131 return false; 2132 // TODO: don't tail predicate if there is a reversed load? 2133 } else if (EnableMaskedGatherScatters) { 2134 // Gather/scatters do allow loading from arbitrary strides, at 2135 // least if they are loop invariant. 2136 // TODO: Loop variant strides should in theory work, too, but 2137 // this requires further testing. 2138 const SCEV *PtrScev = PSE.getSE()->getSCEV(Ptr); 2139 if (auto AR = dyn_cast<SCEVAddRecExpr>(PtrScev)) { 2140 const SCEV *Step = AR->getStepRecurrence(*PSE.getSE()); 2141 if (PSE.getSE()->isLoopInvariant(Step, L)) 2142 continue; 2143 } 2144 } 2145 LLVM_DEBUG(dbgs() << "Bad stride found, can't " 2146 "tail-predicate\n."); 2147 return false; 2148 } 2149 } 2150 } 2151 2152 LLVM_DEBUG(dbgs() << "tail-predication: all instructions allowed!\n"); 2153 return true; 2154 } 2155 2156 bool ARMTTIImpl::preferPredicateOverEpilogue(Loop *L, LoopInfo *LI, 2157 ScalarEvolution &SE, 2158 AssumptionCache &AC, 2159 TargetLibraryInfo *TLI, 2160 DominatorTree *DT, 2161 const LoopAccessInfo *LAI) { 2162 if (!EnableTailPredication) { 2163 LLVM_DEBUG(dbgs() << "Tail-predication not enabled.\n"); 2164 return false; 2165 } 2166 2167 // Creating a predicated vector loop is the first step for generating a 2168 // tail-predicated hardware loop, for which we need the MVE masked 2169 // load/stores instructions: 2170 if (!ST->hasMVEIntegerOps()) 2171 return false; 2172 2173 // For now, restrict this to single block loops. 2174 if (L->getNumBlocks() > 1) { 2175 LLVM_DEBUG(dbgs() << "preferPredicateOverEpilogue: not a single block " 2176 "loop.\n"); 2177 return false; 2178 } 2179 2180 assert(L->isInnermost() && "preferPredicateOverEpilogue: inner-loop expected"); 2181 2182 HardwareLoopInfo HWLoopInfo(L); 2183 if (!HWLoopInfo.canAnalyze(*LI)) { 2184 LLVM_DEBUG(dbgs() << "preferPredicateOverEpilogue: hardware-loop is not " 2185 "analyzable.\n"); 2186 return false; 2187 } 2188 2189 // This checks if we have the low-overhead branch architecture 2190 // extension, and if we will create a hardware-loop: 2191 if (!isHardwareLoopProfitable(L, SE, AC, TLI, HWLoopInfo)) { 2192 LLVM_DEBUG(dbgs() << "preferPredicateOverEpilogue: hardware-loop is not " 2193 "profitable.\n"); 2194 return false; 2195 } 2196 2197 if (!HWLoopInfo.isHardwareLoopCandidate(SE, *LI, *DT)) { 2198 LLVM_DEBUG(dbgs() << "preferPredicateOverEpilogue: hardware-loop is not " 2199 "a candidate.\n"); 2200 return false; 2201 } 2202 2203 return canTailPredicateLoop(L, LI, SE, DL, LAI); 2204 } 2205 2206 bool ARMTTIImpl::emitGetActiveLaneMask() const { 2207 if (!ST->hasMVEIntegerOps() || !EnableTailPredication) 2208 return false; 2209 2210 // Intrinsic @llvm.get.active.lane.mask is supported. 2211 // It is used in the MVETailPredication pass, which requires the number of 2212 // elements processed by this vector loop to setup the tail-predicated 2213 // loop. 2214 return true; 2215 } 2216 void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, 2217 TTI::UnrollingPreferences &UP, 2218 OptimizationRemarkEmitter *ORE) { 2219 // Enable Upper bound unrolling universally, not dependant upon the conditions 2220 // below. 2221 UP.UpperBound = true; 2222 2223 // Only currently enable these preferences for M-Class cores. 2224 if (!ST->isMClass()) 2225 return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE); 2226 2227 // Disable loop unrolling for Oz and Os. 2228 UP.OptSizeThreshold = 0; 2229 UP.PartialOptSizeThreshold = 0; 2230 if (L->getHeader()->getParent()->hasOptSize()) 2231 return; 2232 2233 SmallVector<BasicBlock*, 4> ExitingBlocks; 2234 L->getExitingBlocks(ExitingBlocks); 2235 LLVM_DEBUG(dbgs() << "Loop has:\n" 2236 << "Blocks: " << L->getNumBlocks() << "\n" 2237 << "Exit blocks: " << ExitingBlocks.size() << "\n"); 2238 2239 // Only allow another exit other than the latch. This acts as an early exit 2240 // as it mirrors the profitability calculation of the runtime unroller. 2241 if (ExitingBlocks.size() > 2) 2242 return; 2243 2244 // Limit the CFG of the loop body for targets with a branch predictor. 2245 // Allowing 4 blocks permits if-then-else diamonds in the body. 2246 if (ST->hasBranchPredictor() && L->getNumBlocks() > 4) 2247 return; 2248 2249 // Don't unroll vectorized loops, including the remainder loop 2250 if (getBooleanLoopAttribute(L, "llvm.loop.isvectorized")) 2251 return; 2252 2253 // Scan the loop: don't unroll loops with calls as this could prevent 2254 // inlining. 2255 InstructionCost Cost = 0; 2256 for (auto *BB : L->getBlocks()) { 2257 for (auto &I : *BB) { 2258 // Don't unroll vectorised loop. MVE does not benefit from it as much as 2259 // scalar code. 2260 if (I.getType()->isVectorTy()) 2261 return; 2262 2263 if (isa<CallInst>(I) || isa<InvokeInst>(I)) { 2264 if (const Function *F = cast<CallBase>(I).getCalledFunction()) { 2265 if (!isLoweredToCall(F)) 2266 continue; 2267 } 2268 return; 2269 } 2270 2271 SmallVector<const Value*, 4> Operands(I.operand_values()); 2272 Cost += 2273 getUserCost(&I, Operands, TargetTransformInfo::TCK_SizeAndLatency); 2274 } 2275 } 2276 2277 // On v6m cores, there are very few registers available. We can easily end up 2278 // spilling and reloading more registers in an unrolled loop. Look at the 2279 // number of LCSSA phis as a rough measure of how many registers will need to 2280 // be live out of the loop, reducing the default unroll count if more than 1 2281 // value is needed. In the long run, all of this should be being learnt by a 2282 // machine. 2283 unsigned UnrollCount = 4; 2284 if (ST->isThumb1Only()) { 2285 unsigned ExitingValues = 0; 2286 SmallVector<BasicBlock *, 4> ExitBlocks; 2287 L->getExitBlocks(ExitBlocks); 2288 for (auto *Exit : ExitBlocks) { 2289 // Count the number of LCSSA phis. Exclude values coming from GEP's as 2290 // only the last is expected to be needed for address operands. 2291 unsigned LiveOuts = count_if(Exit->phis(), [](auto &PH) { 2292 return PH.getNumOperands() != 1 || 2293 !isa<GetElementPtrInst>(PH.getOperand(0)); 2294 }); 2295 ExitingValues = ExitingValues < LiveOuts ? LiveOuts : ExitingValues; 2296 } 2297 if (ExitingValues) 2298 UnrollCount /= ExitingValues; 2299 if (UnrollCount <= 1) 2300 return; 2301 } 2302 2303 LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n"); 2304 LLVM_DEBUG(dbgs() << "Default Runtime Unroll Count: " << UnrollCount << "\n"); 2305 2306 UP.Partial = true; 2307 UP.Runtime = true; 2308 UP.UnrollRemainder = true; 2309 UP.DefaultUnrollRuntimeCount = UnrollCount; 2310 UP.UnrollAndJam = true; 2311 UP.UnrollAndJamInnerLoopThreshold = 60; 2312 2313 // Force unrolling small loops can be very useful because of the branch 2314 // taken cost of the backedge. 2315 if (Cost < 12) 2316 UP.Force = true; 2317 } 2318 2319 void ARMTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, 2320 TTI::PeelingPreferences &PP) { 2321 BaseT::getPeelingPreferences(L, SE, PP); 2322 } 2323 2324 bool ARMTTIImpl::preferInLoopReduction(unsigned Opcode, Type *Ty, 2325 TTI::ReductionFlags Flags) const { 2326 if (!ST->hasMVEIntegerOps()) 2327 return false; 2328 2329 unsigned ScalarBits = Ty->getScalarSizeInBits(); 2330 switch (Opcode) { 2331 case Instruction::Add: 2332 return ScalarBits <= 64; 2333 default: 2334 return false; 2335 } 2336 } 2337 2338 bool ARMTTIImpl::preferPredicatedReductionSelect( 2339 unsigned Opcode, Type *Ty, TTI::ReductionFlags Flags) const { 2340 if (!ST->hasMVEIntegerOps()) 2341 return false; 2342 return true; 2343 } 2344