1 //===-- AArch64TargetTransformInfo.cpp - AArch64 specific TTI -------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "AArch64TargetTransformInfo.h" 10 #include "AArch64ExpandImm.h" 11 #include "MCTargetDesc/AArch64AddressingModes.h" 12 #include "llvm/Analysis/LoopInfo.h" 13 #include "llvm/Analysis/TargetTransformInfo.h" 14 #include "llvm/CodeGen/BasicTTIImpl.h" 15 #include "llvm/CodeGen/CostTable.h" 16 #include "llvm/CodeGen/TargetLowering.h" 17 #include "llvm/IR/IntrinsicInst.h" 18 #include "llvm/IR/IntrinsicsAArch64.h" 19 #include "llvm/IR/PatternMatch.h" 20 #include "llvm/Support/Debug.h" 21 #include "llvm/Transforms/InstCombine/InstCombiner.h" 22 #include <algorithm> 23 using namespace llvm; 24 using namespace llvm::PatternMatch; 25 26 #define DEBUG_TYPE "aarch64tti" 27 28 static cl::opt<bool> EnableFalkorHWPFUnrollFix("enable-falkor-hwpf-unroll-fix", 29 cl::init(true), cl::Hidden); 30 31 bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, 32 const Function *Callee) const { 33 const TargetMachine &TM = getTLI()->getTargetMachine(); 34 35 const FeatureBitset &CallerBits = 36 TM.getSubtargetImpl(*Caller)->getFeatureBits(); 37 const FeatureBitset &CalleeBits = 38 TM.getSubtargetImpl(*Callee)->getFeatureBits(); 39 40 // Inline a callee if its target-features are a subset of the callers 41 // target-features. 42 return (CallerBits & CalleeBits) == CalleeBits; 43 } 44 45 /// Calculate the cost of materializing a 64-bit value. This helper 46 /// method might only calculate a fraction of a larger immediate. Therefore it 47 /// is valid to return a cost of ZERO. 48 InstructionCost AArch64TTIImpl::getIntImmCost(int64_t Val) { 49 // Check if the immediate can be encoded within an instruction. 50 if (Val == 0 || AArch64_AM::isLogicalImmediate(Val, 64)) 51 return 0; 52 53 if (Val < 0) 54 Val = ~Val; 55 56 // Calculate how many moves we will need to materialize this constant. 57 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn; 58 AArch64_IMM::expandMOVImm(Val, 64, Insn); 59 return Insn.size(); 60 } 61 62 /// Calculate the cost of materializing the given constant. 63 InstructionCost AArch64TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty, 64 TTI::TargetCostKind CostKind) { 65 assert(Ty->isIntegerTy()); 66 67 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 68 if (BitSize == 0) 69 return ~0U; 70 71 // Sign-extend all constants to a multiple of 64-bit. 72 APInt ImmVal = Imm; 73 if (BitSize & 0x3f) 74 ImmVal = Imm.sext((BitSize + 63) & ~0x3fU); 75 76 // Split the constant into 64-bit chunks and calculate the cost for each 77 // chunk. 78 InstructionCost Cost = 0; 79 for (unsigned ShiftVal = 0; ShiftVal < BitSize; ShiftVal += 64) { 80 APInt Tmp = ImmVal.ashr(ShiftVal).sextOrTrunc(64); 81 int64_t Val = Tmp.getSExtValue(); 82 Cost += getIntImmCost(Val); 83 } 84 // We need at least one instruction to materialze the constant. 85 return std::max<InstructionCost>(1, Cost); 86 } 87 88 InstructionCost AArch64TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx, 89 const APInt &Imm, Type *Ty, 90 TTI::TargetCostKind CostKind, 91 Instruction *Inst) { 92 assert(Ty->isIntegerTy()); 93 94 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 95 // There is no cost model for constants with a bit size of 0. Return TCC_Free 96 // here, so that constant hoisting will ignore this constant. 97 if (BitSize == 0) 98 return TTI::TCC_Free; 99 100 unsigned ImmIdx = ~0U; 101 switch (Opcode) { 102 default: 103 return TTI::TCC_Free; 104 case Instruction::GetElementPtr: 105 // Always hoist the base address of a GetElementPtr. 106 if (Idx == 0) 107 return 2 * TTI::TCC_Basic; 108 return TTI::TCC_Free; 109 case Instruction::Store: 110 ImmIdx = 0; 111 break; 112 case Instruction::Add: 113 case Instruction::Sub: 114 case Instruction::Mul: 115 case Instruction::UDiv: 116 case Instruction::SDiv: 117 case Instruction::URem: 118 case Instruction::SRem: 119 case Instruction::And: 120 case Instruction::Or: 121 case Instruction::Xor: 122 case Instruction::ICmp: 123 ImmIdx = 1; 124 break; 125 // Always return TCC_Free for the shift value of a shift instruction. 126 case Instruction::Shl: 127 case Instruction::LShr: 128 case Instruction::AShr: 129 if (Idx == 1) 130 return TTI::TCC_Free; 131 break; 132 case Instruction::Trunc: 133 case Instruction::ZExt: 134 case Instruction::SExt: 135 case Instruction::IntToPtr: 136 case Instruction::PtrToInt: 137 case Instruction::BitCast: 138 case Instruction::PHI: 139 case Instruction::Call: 140 case Instruction::Select: 141 case Instruction::Ret: 142 case Instruction::Load: 143 break; 144 } 145 146 if (Idx == ImmIdx) { 147 int NumConstants = (BitSize + 63) / 64; 148 InstructionCost Cost = AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind); 149 return (Cost <= NumConstants * TTI::TCC_Basic) 150 ? static_cast<int>(TTI::TCC_Free) 151 : Cost; 152 } 153 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind); 154 } 155 156 InstructionCost 157 AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, 158 const APInt &Imm, Type *Ty, 159 TTI::TargetCostKind CostKind) { 160 assert(Ty->isIntegerTy()); 161 162 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 163 // There is no cost model for constants with a bit size of 0. Return TCC_Free 164 // here, so that constant hoisting will ignore this constant. 165 if (BitSize == 0) 166 return TTI::TCC_Free; 167 168 // Most (all?) AArch64 intrinsics do not support folding immediates into the 169 // selected instruction, so we compute the materialization cost for the 170 // immediate directly. 171 if (IID >= Intrinsic::aarch64_addg && IID <= Intrinsic::aarch64_udiv) 172 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind); 173 174 switch (IID) { 175 default: 176 return TTI::TCC_Free; 177 case Intrinsic::sadd_with_overflow: 178 case Intrinsic::uadd_with_overflow: 179 case Intrinsic::ssub_with_overflow: 180 case Intrinsic::usub_with_overflow: 181 case Intrinsic::smul_with_overflow: 182 case Intrinsic::umul_with_overflow: 183 if (Idx == 1) { 184 int NumConstants = (BitSize + 63) / 64; 185 InstructionCost Cost = AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind); 186 return (Cost <= NumConstants * TTI::TCC_Basic) 187 ? static_cast<int>(TTI::TCC_Free) 188 : Cost; 189 } 190 break; 191 case Intrinsic::experimental_stackmap: 192 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) 193 return TTI::TCC_Free; 194 break; 195 case Intrinsic::experimental_patchpoint_void: 196 case Intrinsic::experimental_patchpoint_i64: 197 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) 198 return TTI::TCC_Free; 199 break; 200 case Intrinsic::experimental_gc_statepoint: 201 if ((Idx < 5) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) 202 return TTI::TCC_Free; 203 break; 204 } 205 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind); 206 } 207 208 TargetTransformInfo::PopcntSupportKind 209 AArch64TTIImpl::getPopcntSupport(unsigned TyWidth) { 210 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2"); 211 if (TyWidth == 32 || TyWidth == 64) 212 return TTI::PSK_FastHardware; 213 // TODO: AArch64TargetLowering::LowerCTPOP() supports 128bit popcount. 214 return TTI::PSK_Software; 215 } 216 217 InstructionCost 218 AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, 219 TTI::TargetCostKind CostKind) { 220 auto *RetTy = ICA.getReturnType(); 221 switch (ICA.getID()) { 222 case Intrinsic::umin: 223 case Intrinsic::umax: { 224 auto LT = TLI->getTypeLegalizationCost(DL, RetTy); 225 // umin(x,y) -> sub(x,usubsat(x,y)) 226 // umax(x,y) -> add(x,usubsat(y,x)) 227 if (LT.second == MVT::v2i64) 228 return LT.first * 2; 229 LLVM_FALLTHROUGH; 230 } 231 case Intrinsic::smin: 232 case Intrinsic::smax: { 233 static const auto ValidMinMaxTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16, 234 MVT::v8i16, MVT::v2i32, MVT::v4i32}; 235 auto LT = TLI->getTypeLegalizationCost(DL, RetTy); 236 if (any_of(ValidMinMaxTys, [<](MVT M) { return M == LT.second; })) 237 return LT.first; 238 break; 239 } 240 case Intrinsic::sadd_sat: 241 case Intrinsic::ssub_sat: 242 case Intrinsic::uadd_sat: 243 case Intrinsic::usub_sat: { 244 static const auto ValidSatTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16, 245 MVT::v8i16, MVT::v2i32, MVT::v4i32, 246 MVT::v2i64}; 247 auto LT = TLI->getTypeLegalizationCost(DL, RetTy); 248 // This is a base cost of 1 for the vadd, plus 3 extract shifts if we 249 // need to extend the type, as it uses shr(qadd(shl, shl)). 250 unsigned Instrs = 251 LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits() ? 1 : 4; 252 if (any_of(ValidSatTys, [<](MVT M) { return M == LT.second; })) 253 return LT.first * Instrs; 254 break; 255 } 256 case Intrinsic::abs: { 257 static const auto ValidAbsTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16, 258 MVT::v8i16, MVT::v2i32, MVT::v4i32, 259 MVT::v2i64}; 260 auto LT = TLI->getTypeLegalizationCost(DL, RetTy); 261 if (any_of(ValidAbsTys, [<](MVT M) { return M == LT.second; })) 262 return LT.first; 263 break; 264 } 265 case Intrinsic::experimental_stepvector: { 266 InstructionCost Cost = 1; // Cost of the `index' instruction 267 auto LT = TLI->getTypeLegalizationCost(DL, RetTy); 268 // Legalisation of illegal vectors involves an `index' instruction plus 269 // (LT.first - 1) vector adds. 270 if (LT.first > 1) { 271 Type *LegalVTy = EVT(LT.second).getTypeForEVT(RetTy->getContext()); 272 InstructionCost AddCost = 273 getArithmeticInstrCost(Instruction::Add, LegalVTy, CostKind); 274 Cost += AddCost * (LT.first - 1); 275 } 276 return Cost; 277 } 278 case Intrinsic::bitreverse: { 279 static const CostTblEntry BitreverseTbl[] = { 280 {Intrinsic::bitreverse, MVT::i32, 1}, 281 {Intrinsic::bitreverse, MVT::i64, 1}, 282 {Intrinsic::bitreverse, MVT::v8i8, 1}, 283 {Intrinsic::bitreverse, MVT::v16i8, 1}, 284 {Intrinsic::bitreverse, MVT::v4i16, 2}, 285 {Intrinsic::bitreverse, MVT::v8i16, 2}, 286 {Intrinsic::bitreverse, MVT::v2i32, 2}, 287 {Intrinsic::bitreverse, MVT::v4i32, 2}, 288 {Intrinsic::bitreverse, MVT::v1i64, 2}, 289 {Intrinsic::bitreverse, MVT::v2i64, 2}, 290 }; 291 const auto LegalisationCost = TLI->getTypeLegalizationCost(DL, RetTy); 292 const auto *Entry = 293 CostTableLookup(BitreverseTbl, ICA.getID(), LegalisationCost.second); 294 // Cost Model is using the legal type(i32) that i8 and i16 will be converted 295 // to +1 so that we match the actual lowering cost 296 if (TLI->getValueType(DL, RetTy, true) == MVT::i8 || 297 TLI->getValueType(DL, RetTy, true) == MVT::i16) 298 return LegalisationCost.first * Entry->Cost + 1; 299 if (Entry) 300 return LegalisationCost.first * Entry->Cost; 301 break; 302 } 303 case Intrinsic::ctpop: { 304 static const CostTblEntry CtpopCostTbl[] = { 305 {ISD::CTPOP, MVT::v2i64, 4}, 306 {ISD::CTPOP, MVT::v4i32, 3}, 307 {ISD::CTPOP, MVT::v8i16, 2}, 308 {ISD::CTPOP, MVT::v16i8, 1}, 309 {ISD::CTPOP, MVT::i64, 4}, 310 {ISD::CTPOP, MVT::v2i32, 3}, 311 {ISD::CTPOP, MVT::v4i16, 2}, 312 {ISD::CTPOP, MVT::v8i8, 1}, 313 {ISD::CTPOP, MVT::i32, 5}, 314 }; 315 auto LT = TLI->getTypeLegalizationCost(DL, RetTy); 316 MVT MTy = LT.second; 317 if (const auto *Entry = CostTableLookup(CtpopCostTbl, ISD::CTPOP, MTy)) { 318 // Extra cost of +1 when illegal vector types are legalized by promoting 319 // the integer type. 320 int ExtraCost = MTy.isVector() && MTy.getScalarSizeInBits() != 321 RetTy->getScalarSizeInBits() 322 ? 1 323 : 0; 324 return LT.first * Entry->Cost + ExtraCost; 325 } 326 break; 327 } 328 default: 329 break; 330 } 331 return BaseT::getIntrinsicInstrCost(ICA, CostKind); 332 } 333 334 /// The function will remove redundant reinterprets casting in the presence 335 /// of the control flow 336 static Optional<Instruction *> processPhiNode(InstCombiner &IC, 337 IntrinsicInst &II) { 338 SmallVector<Instruction *, 32> Worklist; 339 auto RequiredType = II.getType(); 340 341 auto *PN = dyn_cast<PHINode>(II.getArgOperand(0)); 342 assert(PN && "Expected Phi Node!"); 343 344 // Don't create a new Phi unless we can remove the old one. 345 if (!PN->hasOneUse()) 346 return None; 347 348 for (Value *IncValPhi : PN->incoming_values()) { 349 auto *Reinterpret = dyn_cast<IntrinsicInst>(IncValPhi); 350 if (!Reinterpret || 351 Reinterpret->getIntrinsicID() != 352 Intrinsic::aarch64_sve_convert_to_svbool || 353 RequiredType != Reinterpret->getArgOperand(0)->getType()) 354 return None; 355 } 356 357 // Create the new Phi 358 LLVMContext &Ctx = PN->getContext(); 359 IRBuilder<> Builder(Ctx); 360 Builder.SetInsertPoint(PN); 361 PHINode *NPN = Builder.CreatePHI(RequiredType, PN->getNumIncomingValues()); 362 Worklist.push_back(PN); 363 364 for (unsigned I = 0; I < PN->getNumIncomingValues(); I++) { 365 auto *Reinterpret = cast<Instruction>(PN->getIncomingValue(I)); 366 NPN->addIncoming(Reinterpret->getOperand(0), PN->getIncomingBlock(I)); 367 Worklist.push_back(Reinterpret); 368 } 369 370 // Cleanup Phi Node and reinterprets 371 return IC.replaceInstUsesWith(II, NPN); 372 } 373 374 static Optional<Instruction *> instCombineConvertFromSVBool(InstCombiner &IC, 375 IntrinsicInst &II) { 376 // If the reinterpret instruction operand is a PHI Node 377 if (isa<PHINode>(II.getArgOperand(0))) 378 return processPhiNode(IC, II); 379 380 SmallVector<Instruction *, 32> CandidatesForRemoval; 381 Value *Cursor = II.getOperand(0), *EarliestReplacement = nullptr; 382 383 const auto *IVTy = cast<VectorType>(II.getType()); 384 385 // Walk the chain of conversions. 386 while (Cursor) { 387 // If the type of the cursor has fewer lanes than the final result, zeroing 388 // must take place, which breaks the equivalence chain. 389 const auto *CursorVTy = cast<VectorType>(Cursor->getType()); 390 if (CursorVTy->getElementCount().getKnownMinValue() < 391 IVTy->getElementCount().getKnownMinValue()) 392 break; 393 394 // If the cursor has the same type as I, it is a viable replacement. 395 if (Cursor->getType() == IVTy) 396 EarliestReplacement = Cursor; 397 398 auto *IntrinsicCursor = dyn_cast<IntrinsicInst>(Cursor); 399 400 // If this is not an SVE conversion intrinsic, this is the end of the chain. 401 if (!IntrinsicCursor || !(IntrinsicCursor->getIntrinsicID() == 402 Intrinsic::aarch64_sve_convert_to_svbool || 403 IntrinsicCursor->getIntrinsicID() == 404 Intrinsic::aarch64_sve_convert_from_svbool)) 405 break; 406 407 CandidatesForRemoval.insert(CandidatesForRemoval.begin(), IntrinsicCursor); 408 Cursor = IntrinsicCursor->getOperand(0); 409 } 410 411 // If no viable replacement in the conversion chain was found, there is 412 // nothing to do. 413 if (!EarliestReplacement) 414 return None; 415 416 return IC.replaceInstUsesWith(II, EarliestReplacement); 417 } 418 419 static Optional<Instruction *> instCombineSVEDup(InstCombiner &IC, 420 IntrinsicInst &II) { 421 IntrinsicInst *Pg = dyn_cast<IntrinsicInst>(II.getArgOperand(1)); 422 if (!Pg) 423 return None; 424 425 if (Pg->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue) 426 return None; 427 428 const auto PTruePattern = 429 cast<ConstantInt>(Pg->getOperand(0))->getZExtValue(); 430 if (PTruePattern != AArch64SVEPredPattern::vl1) 431 return None; 432 433 // The intrinsic is inserting into lane zero so use an insert instead. 434 auto *IdxTy = Type::getInt64Ty(II.getContext()); 435 auto *Insert = InsertElementInst::Create( 436 II.getArgOperand(0), II.getArgOperand(2), ConstantInt::get(IdxTy, 0)); 437 Insert->insertBefore(&II); 438 Insert->takeName(&II); 439 440 return IC.replaceInstUsesWith(II, Insert); 441 } 442 443 static Optional<Instruction *> instCombineSVECmpNE(InstCombiner &IC, 444 IntrinsicInst &II) { 445 LLVMContext &Ctx = II.getContext(); 446 IRBuilder<> Builder(Ctx); 447 Builder.SetInsertPoint(&II); 448 449 // Check that the predicate is all active 450 auto *Pg = dyn_cast<IntrinsicInst>(II.getArgOperand(0)); 451 if (!Pg || Pg->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue) 452 return None; 453 454 const auto PTruePattern = 455 cast<ConstantInt>(Pg->getOperand(0))->getZExtValue(); 456 if (PTruePattern != AArch64SVEPredPattern::all) 457 return None; 458 459 // Check that we have a compare of zero.. 460 auto *DupX = dyn_cast<IntrinsicInst>(II.getArgOperand(2)); 461 if (!DupX || DupX->getIntrinsicID() != Intrinsic::aarch64_sve_dup_x) 462 return None; 463 464 auto *DupXArg = dyn_cast<ConstantInt>(DupX->getArgOperand(0)); 465 if (!DupXArg || !DupXArg->isZero()) 466 return None; 467 468 // ..against a dupq 469 auto *DupQLane = dyn_cast<IntrinsicInst>(II.getArgOperand(1)); 470 if (!DupQLane || 471 DupQLane->getIntrinsicID() != Intrinsic::aarch64_sve_dupq_lane) 472 return None; 473 474 // Where the dupq is a lane 0 replicate of a vector insert 475 if (!cast<ConstantInt>(DupQLane->getArgOperand(1))->isZero()) 476 return None; 477 478 auto *VecIns = dyn_cast<IntrinsicInst>(DupQLane->getArgOperand(0)); 479 if (!VecIns || 480 VecIns->getIntrinsicID() != Intrinsic::experimental_vector_insert) 481 return None; 482 483 // Where the vector insert is a fixed constant vector insert into undef at 484 // index zero 485 if (!isa<UndefValue>(VecIns->getArgOperand(0))) 486 return None; 487 488 if (!cast<ConstantInt>(VecIns->getArgOperand(2))->isZero()) 489 return None; 490 491 auto *ConstVec = dyn_cast<Constant>(VecIns->getArgOperand(1)); 492 if (!ConstVec) 493 return None; 494 495 auto *VecTy = dyn_cast<FixedVectorType>(ConstVec->getType()); 496 auto *OutTy = dyn_cast<ScalableVectorType>(II.getType()); 497 if (!VecTy || !OutTy || VecTy->getNumElements() != OutTy->getMinNumElements()) 498 return None; 499 500 unsigned NumElts = VecTy->getNumElements(); 501 unsigned PredicateBits = 0; 502 503 // Expand intrinsic operands to a 16-bit byte level predicate 504 for (unsigned I = 0; I < NumElts; ++I) { 505 auto *Arg = dyn_cast<ConstantInt>(ConstVec->getAggregateElement(I)); 506 if (!Arg) 507 return None; 508 if (!Arg->isZero()) 509 PredicateBits |= 1 << (I * (16 / NumElts)); 510 } 511 512 // If all bits are zero bail early with an empty predicate 513 if (PredicateBits == 0) { 514 auto *PFalse = Constant::getNullValue(II.getType()); 515 PFalse->takeName(&II); 516 return IC.replaceInstUsesWith(II, PFalse); 517 } 518 519 // Calculate largest predicate type used (where byte predicate is largest) 520 unsigned Mask = 8; 521 for (unsigned I = 0; I < 16; ++I) 522 if ((PredicateBits & (1 << I)) != 0) 523 Mask |= (I % 8); 524 525 unsigned PredSize = Mask & -Mask; 526 auto *PredType = ScalableVectorType::get( 527 Type::getInt1Ty(Ctx), AArch64::SVEBitsPerBlock / (PredSize * 8)); 528 529 // Ensure all relevant bits are set 530 for (unsigned I = 0; I < 16; I += PredSize) 531 if ((PredicateBits & (1 << I)) == 0) 532 return None; 533 534 auto *PTruePat = 535 ConstantInt::get(Type::getInt32Ty(Ctx), AArch64SVEPredPattern::all); 536 auto *PTrue = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue, 537 {PredType}, {PTruePat}); 538 auto *ConvertToSVBool = Builder.CreateIntrinsic( 539 Intrinsic::aarch64_sve_convert_to_svbool, {PredType}, {PTrue}); 540 auto *ConvertFromSVBool = 541 Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, 542 {II.getType()}, {ConvertToSVBool}); 543 544 ConvertFromSVBool->takeName(&II); 545 return IC.replaceInstUsesWith(II, ConvertFromSVBool); 546 } 547 548 static Optional<Instruction *> instCombineSVELast(InstCombiner &IC, 549 IntrinsicInst &II) { 550 Value *Pg = II.getArgOperand(0); 551 Value *Vec = II.getArgOperand(1); 552 bool IsAfter = II.getIntrinsicID() == Intrinsic::aarch64_sve_lasta; 553 554 // lastX(splat(X)) --> X 555 if (auto *SplatVal = getSplatValue(Vec)) 556 return IC.replaceInstUsesWith(II, SplatVal); 557 558 auto *C = dyn_cast<Constant>(Pg); 559 if (IsAfter && C && C->isNullValue()) { 560 // The intrinsic is extracting lane 0 so use an extract instead. 561 auto *IdxTy = Type::getInt64Ty(II.getContext()); 562 auto *Extract = ExtractElementInst::Create(Vec, ConstantInt::get(IdxTy, 0)); 563 Extract->insertBefore(&II); 564 Extract->takeName(&II); 565 return IC.replaceInstUsesWith(II, Extract); 566 } 567 568 auto *IntrPG = dyn_cast<IntrinsicInst>(Pg); 569 if (!IntrPG) 570 return None; 571 572 if (IntrPG->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue) 573 return None; 574 575 const auto PTruePattern = 576 cast<ConstantInt>(IntrPG->getOperand(0))->getZExtValue(); 577 578 // Can the intrinsic's predicate be converted to a known constant index? 579 unsigned Idx; 580 switch (PTruePattern) { 581 default: 582 return None; 583 case AArch64SVEPredPattern::vl1: 584 Idx = 0; 585 break; 586 case AArch64SVEPredPattern::vl2: 587 Idx = 1; 588 break; 589 case AArch64SVEPredPattern::vl3: 590 Idx = 2; 591 break; 592 case AArch64SVEPredPattern::vl4: 593 Idx = 3; 594 break; 595 case AArch64SVEPredPattern::vl5: 596 Idx = 4; 597 break; 598 case AArch64SVEPredPattern::vl6: 599 Idx = 5; 600 break; 601 case AArch64SVEPredPattern::vl7: 602 Idx = 6; 603 break; 604 case AArch64SVEPredPattern::vl8: 605 Idx = 7; 606 break; 607 case AArch64SVEPredPattern::vl16: 608 Idx = 15; 609 break; 610 } 611 612 // Increment the index if extracting the element after the last active 613 // predicate element. 614 if (IsAfter) 615 ++Idx; 616 617 // Ignore extracts whose index is larger than the known minimum vector 618 // length. NOTE: This is an artificial constraint where we prefer to 619 // maintain what the user asked for until an alternative is proven faster. 620 auto *PgVTy = cast<ScalableVectorType>(Pg->getType()); 621 if (Idx >= PgVTy->getMinNumElements()) 622 return None; 623 624 // The intrinsic is extracting a fixed lane so use an extract instead. 625 auto *IdxTy = Type::getInt64Ty(II.getContext()); 626 auto *Extract = ExtractElementInst::Create(Vec, ConstantInt::get(IdxTy, Idx)); 627 Extract->insertBefore(&II); 628 Extract->takeName(&II); 629 return IC.replaceInstUsesWith(II, Extract); 630 } 631 632 static Optional<Instruction *> instCombineRDFFR(InstCombiner &IC, 633 IntrinsicInst &II) { 634 LLVMContext &Ctx = II.getContext(); 635 IRBuilder<> Builder(Ctx); 636 Builder.SetInsertPoint(&II); 637 // Replace rdffr with predicated rdffr.z intrinsic, so that optimizePTestInstr 638 // can work with RDFFR_PP for ptest elimination. 639 auto *AllPat = 640 ConstantInt::get(Type::getInt32Ty(Ctx), AArch64SVEPredPattern::all); 641 auto *PTrue = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue, 642 {II.getType()}, {AllPat}); 643 auto *RDFFR = 644 Builder.CreateIntrinsic(Intrinsic::aarch64_sve_rdffr_z, {}, {PTrue}); 645 RDFFR->takeName(&II); 646 return IC.replaceInstUsesWith(II, RDFFR); 647 } 648 649 static Optional<Instruction *> 650 instCombineSVECntElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts) { 651 const auto Pattern = cast<ConstantInt>(II.getArgOperand(0))->getZExtValue(); 652 653 if (Pattern == AArch64SVEPredPattern::all) { 654 LLVMContext &Ctx = II.getContext(); 655 IRBuilder<> Builder(Ctx); 656 Builder.SetInsertPoint(&II); 657 658 Constant *StepVal = ConstantInt::get(II.getType(), NumElts); 659 auto *VScale = Builder.CreateVScale(StepVal); 660 VScale->takeName(&II); 661 return IC.replaceInstUsesWith(II, VScale); 662 } 663 664 unsigned MinNumElts = 0; 665 switch (Pattern) { 666 default: 667 return None; 668 case AArch64SVEPredPattern::vl1: 669 case AArch64SVEPredPattern::vl2: 670 case AArch64SVEPredPattern::vl3: 671 case AArch64SVEPredPattern::vl4: 672 case AArch64SVEPredPattern::vl5: 673 case AArch64SVEPredPattern::vl6: 674 case AArch64SVEPredPattern::vl7: 675 case AArch64SVEPredPattern::vl8: 676 MinNumElts = Pattern; 677 break; 678 case AArch64SVEPredPattern::vl16: 679 MinNumElts = 16; 680 break; 681 } 682 683 return NumElts >= MinNumElts 684 ? Optional<Instruction *>(IC.replaceInstUsesWith( 685 II, ConstantInt::get(II.getType(), MinNumElts))) 686 : None; 687 } 688 689 static Optional<Instruction *> instCombineSVEPTest(InstCombiner &IC, 690 IntrinsicInst &II) { 691 IntrinsicInst *Op1 = dyn_cast<IntrinsicInst>(II.getArgOperand(0)); 692 IntrinsicInst *Op2 = dyn_cast<IntrinsicInst>(II.getArgOperand(1)); 693 694 if (Op1 && Op2 && 695 Op1->getIntrinsicID() == Intrinsic::aarch64_sve_convert_to_svbool && 696 Op2->getIntrinsicID() == Intrinsic::aarch64_sve_convert_to_svbool && 697 Op1->getArgOperand(0)->getType() == Op2->getArgOperand(0)->getType()) { 698 699 IRBuilder<> Builder(II.getContext()); 700 Builder.SetInsertPoint(&II); 701 702 Value *Ops[] = {Op1->getArgOperand(0), Op2->getArgOperand(0)}; 703 Type *Tys[] = {Op1->getArgOperand(0)->getType()}; 704 705 auto *PTest = Builder.CreateIntrinsic(II.getIntrinsicID(), Tys, Ops); 706 707 PTest->takeName(&II); 708 return IC.replaceInstUsesWith(II, PTest); 709 } 710 711 return None; 712 } 713 714 static Optional<Instruction *> instCombineSVEVectorMul(InstCombiner &IC, 715 IntrinsicInst &II) { 716 auto *OpPredicate = II.getOperand(0); 717 auto *OpMultiplicand = II.getOperand(1); 718 auto *OpMultiplier = II.getOperand(2); 719 720 IRBuilder<> Builder(II.getContext()); 721 Builder.SetInsertPoint(&II); 722 723 // Return true if a given instruction is an aarch64_sve_dup_x intrinsic call 724 // with a unit splat value, false otherwise. 725 auto IsUnitDupX = [](auto *I) { 726 auto *IntrI = dyn_cast<IntrinsicInst>(I); 727 if (!IntrI || IntrI->getIntrinsicID() != Intrinsic::aarch64_sve_dup_x) 728 return false; 729 730 auto *SplatValue = IntrI->getOperand(0); 731 return match(SplatValue, m_FPOne()) || match(SplatValue, m_One()); 732 }; 733 734 // Return true if a given instruction is an aarch64_sve_dup intrinsic call 735 // with a unit splat value, false otherwise. 736 auto IsUnitDup = [](auto *I) { 737 auto *IntrI = dyn_cast<IntrinsicInst>(I); 738 if (!IntrI || IntrI->getIntrinsicID() != Intrinsic::aarch64_sve_dup) 739 return false; 740 741 auto *SplatValue = IntrI->getOperand(2); 742 return match(SplatValue, m_FPOne()) || match(SplatValue, m_One()); 743 }; 744 745 // The OpMultiplier variable should always point to the dup (if any), so 746 // swap if necessary. 747 if (IsUnitDup(OpMultiplicand) || IsUnitDupX(OpMultiplicand)) 748 std::swap(OpMultiplier, OpMultiplicand); 749 750 if (IsUnitDupX(OpMultiplier)) { 751 // [f]mul pg (dupx 1) %n => %n 752 OpMultiplicand->takeName(&II); 753 return IC.replaceInstUsesWith(II, OpMultiplicand); 754 } else if (IsUnitDup(OpMultiplier)) { 755 // [f]mul pg (dup pg 1) %n => %n 756 auto *DupInst = cast<IntrinsicInst>(OpMultiplier); 757 auto *DupPg = DupInst->getOperand(1); 758 // TODO: this is naive. The optimization is still valid if DupPg 759 // 'encompasses' OpPredicate, not only if they're the same predicate. 760 if (OpPredicate == DupPg) { 761 OpMultiplicand->takeName(&II); 762 return IC.replaceInstUsesWith(II, OpMultiplicand); 763 } 764 } 765 766 return None; 767 } 768 769 static Optional<Instruction *> instCombineSVETBL(InstCombiner &IC, 770 IntrinsicInst &II) { 771 auto *OpVal = II.getOperand(0); 772 auto *OpIndices = II.getOperand(1); 773 VectorType *VTy = cast<VectorType>(II.getType()); 774 775 // Check whether OpIndices is an aarch64_sve_dup_x intrinsic call with 776 // constant splat value < minimal element count of result. 777 auto *DupXIntrI = dyn_cast<IntrinsicInst>(OpIndices); 778 if (!DupXIntrI || DupXIntrI->getIntrinsicID() != Intrinsic::aarch64_sve_dup_x) 779 return None; 780 781 auto *SplatValue = dyn_cast<ConstantInt>(DupXIntrI->getOperand(0)); 782 if (!SplatValue || 783 SplatValue->getValue().uge(VTy->getElementCount().getKnownMinValue())) 784 return None; 785 786 // Convert sve_tbl(OpVal sve_dup_x(SplatValue)) to 787 // splat_vector(extractelement(OpVal, SplatValue)) for further optimization. 788 IRBuilder<> Builder(II.getContext()); 789 Builder.SetInsertPoint(&II); 790 auto *Extract = Builder.CreateExtractElement(OpVal, SplatValue); 791 auto *VectorSplat = 792 Builder.CreateVectorSplat(VTy->getElementCount(), Extract); 793 794 VectorSplat->takeName(&II); 795 return IC.replaceInstUsesWith(II, VectorSplat); 796 } 797 798 Optional<Instruction *> 799 AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC, 800 IntrinsicInst &II) const { 801 Intrinsic::ID IID = II.getIntrinsicID(); 802 switch (IID) { 803 default: 804 break; 805 case Intrinsic::aarch64_sve_convert_from_svbool: 806 return instCombineConvertFromSVBool(IC, II); 807 case Intrinsic::aarch64_sve_dup: 808 return instCombineSVEDup(IC, II); 809 case Intrinsic::aarch64_sve_cmpne: 810 case Intrinsic::aarch64_sve_cmpne_wide: 811 return instCombineSVECmpNE(IC, II); 812 case Intrinsic::aarch64_sve_rdffr: 813 return instCombineRDFFR(IC, II); 814 case Intrinsic::aarch64_sve_lasta: 815 case Intrinsic::aarch64_sve_lastb: 816 return instCombineSVELast(IC, II); 817 case Intrinsic::aarch64_sve_cntd: 818 return instCombineSVECntElts(IC, II, 2); 819 case Intrinsic::aarch64_sve_cntw: 820 return instCombineSVECntElts(IC, II, 4); 821 case Intrinsic::aarch64_sve_cnth: 822 return instCombineSVECntElts(IC, II, 8); 823 case Intrinsic::aarch64_sve_cntb: 824 return instCombineSVECntElts(IC, II, 16); 825 case Intrinsic::aarch64_sve_ptest_any: 826 case Intrinsic::aarch64_sve_ptest_first: 827 case Intrinsic::aarch64_sve_ptest_last: 828 return instCombineSVEPTest(IC, II); 829 case Intrinsic::aarch64_sve_mul: 830 case Intrinsic::aarch64_sve_fmul: 831 return instCombineSVEVectorMul(IC, II); 832 case Intrinsic::aarch64_sve_tbl: 833 return instCombineSVETBL(IC, II); 834 } 835 836 return None; 837 } 838 839 bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode, 840 ArrayRef<const Value *> Args) { 841 842 // A helper that returns a vector type from the given type. The number of 843 // elements in type Ty determine the vector width. 844 auto toVectorTy = [&](Type *ArgTy) { 845 return VectorType::get(ArgTy->getScalarType(), 846 cast<VectorType>(DstTy)->getElementCount()); 847 }; 848 849 // Exit early if DstTy is not a vector type whose elements are at least 850 // 16-bits wide. 851 if (!DstTy->isVectorTy() || DstTy->getScalarSizeInBits() < 16) 852 return false; 853 854 // Determine if the operation has a widening variant. We consider both the 855 // "long" (e.g., usubl) and "wide" (e.g., usubw) versions of the 856 // instructions. 857 // 858 // TODO: Add additional widening operations (e.g., mul, shl, etc.) once we 859 // verify that their extending operands are eliminated during code 860 // generation. 861 switch (Opcode) { 862 case Instruction::Add: // UADDL(2), SADDL(2), UADDW(2), SADDW(2). 863 case Instruction::Sub: // USUBL(2), SSUBL(2), USUBW(2), SSUBW(2). 864 break; 865 default: 866 return false; 867 } 868 869 // To be a widening instruction (either the "wide" or "long" versions), the 870 // second operand must be a sign- or zero extend having a single user. We 871 // only consider extends having a single user because they may otherwise not 872 // be eliminated. 873 if (Args.size() != 2 || 874 (!isa<SExtInst>(Args[1]) && !isa<ZExtInst>(Args[1])) || 875 !Args[1]->hasOneUse()) 876 return false; 877 auto *Extend = cast<CastInst>(Args[1]); 878 879 // Legalize the destination type and ensure it can be used in a widening 880 // operation. 881 auto DstTyL = TLI->getTypeLegalizationCost(DL, DstTy); 882 unsigned DstElTySize = DstTyL.second.getScalarSizeInBits(); 883 if (!DstTyL.second.isVector() || DstElTySize != DstTy->getScalarSizeInBits()) 884 return false; 885 886 // Legalize the source type and ensure it can be used in a widening 887 // operation. 888 auto *SrcTy = toVectorTy(Extend->getSrcTy()); 889 auto SrcTyL = TLI->getTypeLegalizationCost(DL, SrcTy); 890 unsigned SrcElTySize = SrcTyL.second.getScalarSizeInBits(); 891 if (!SrcTyL.second.isVector() || SrcElTySize != SrcTy->getScalarSizeInBits()) 892 return false; 893 894 // Get the total number of vector elements in the legalized types. 895 InstructionCost NumDstEls = 896 DstTyL.first * DstTyL.second.getVectorMinNumElements(); 897 InstructionCost NumSrcEls = 898 SrcTyL.first * SrcTyL.second.getVectorMinNumElements(); 899 900 // Return true if the legalized types have the same number of vector elements 901 // and the destination element type size is twice that of the source type. 902 return NumDstEls == NumSrcEls && 2 * SrcElTySize == DstElTySize; 903 } 904 905 InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, 906 Type *Src, 907 TTI::CastContextHint CCH, 908 TTI::TargetCostKind CostKind, 909 const Instruction *I) { 910 int ISD = TLI->InstructionOpcodeToISD(Opcode); 911 assert(ISD && "Invalid opcode"); 912 913 // If the cast is observable, and it is used by a widening instruction (e.g., 914 // uaddl, saddw, etc.), it may be free. 915 if (I && I->hasOneUse()) { 916 auto *SingleUser = cast<Instruction>(*I->user_begin()); 917 SmallVector<const Value *, 4> Operands(SingleUser->operand_values()); 918 if (isWideningInstruction(Dst, SingleUser->getOpcode(), Operands)) { 919 // If the cast is the second operand, it is free. We will generate either 920 // a "wide" or "long" version of the widening instruction. 921 if (I == SingleUser->getOperand(1)) 922 return 0; 923 // If the cast is not the second operand, it will be free if it looks the 924 // same as the second operand. In this case, we will generate a "long" 925 // version of the widening instruction. 926 if (auto *Cast = dyn_cast<CastInst>(SingleUser->getOperand(1))) 927 if (I->getOpcode() == unsigned(Cast->getOpcode()) && 928 cast<CastInst>(I)->getSrcTy() == Cast->getSrcTy()) 929 return 0; 930 } 931 } 932 933 // TODO: Allow non-throughput costs that aren't binary. 934 auto AdjustCost = [&CostKind](InstructionCost Cost) -> InstructionCost { 935 if (CostKind != TTI::TCK_RecipThroughput) 936 return Cost == 0 ? 0 : 1; 937 return Cost; 938 }; 939 940 EVT SrcTy = TLI->getValueType(DL, Src); 941 EVT DstTy = TLI->getValueType(DL, Dst); 942 943 if (!SrcTy.isSimple() || !DstTy.isSimple()) 944 return AdjustCost( 945 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I)); 946 947 static const TypeConversionCostTblEntry 948 ConversionTbl[] = { 949 { ISD::TRUNCATE, MVT::v4i16, MVT::v4i32, 1 }, 950 { ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, 0 }, 951 { ISD::TRUNCATE, MVT::v8i8, MVT::v8i32, 3 }, 952 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, 6 }, 953 954 // Truncations on nxvmiN 955 { ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i16, 1 }, 956 { ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i32, 1 }, 957 { ISD::TRUNCATE, MVT::nxv2i1, MVT::nxv2i64, 1 }, 958 { ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i16, 1 }, 959 { ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i32, 1 }, 960 { ISD::TRUNCATE, MVT::nxv4i1, MVT::nxv4i64, 2 }, 961 { ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i16, 1 }, 962 { ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i32, 3 }, 963 { ISD::TRUNCATE, MVT::nxv8i1, MVT::nxv8i64, 5 }, 964 { ISD::TRUNCATE, MVT::nxv16i1, MVT::nxv16i8, 1 }, 965 { ISD::TRUNCATE, MVT::nxv2i16, MVT::nxv2i32, 1 }, 966 { ISD::TRUNCATE, MVT::nxv2i32, MVT::nxv2i64, 1 }, 967 { ISD::TRUNCATE, MVT::nxv4i16, MVT::nxv4i32, 1 }, 968 { ISD::TRUNCATE, MVT::nxv4i32, MVT::nxv4i64, 2 }, 969 { ISD::TRUNCATE, MVT::nxv8i16, MVT::nxv8i32, 3 }, 970 { ISD::TRUNCATE, MVT::nxv8i32, MVT::nxv8i64, 6 }, 971 972 // The number of shll instructions for the extension. 973 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i16, 3 }, 974 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i16, 3 }, 975 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, 2 }, 976 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, 2 }, 977 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i8, 3 }, 978 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i8, 3 }, 979 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 2 }, 980 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 2 }, 981 { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i8, 7 }, 982 { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i8, 7 }, 983 { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i16, 6 }, 984 { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i16, 6 }, 985 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 2 }, 986 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 2 }, 987 { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8, 6 }, 988 { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8, 6 }, 989 990 // LowerVectorINT_TO_FP: 991 { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i32, 1 }, 992 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 1 }, 993 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i64, 1 }, 994 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, 1 }, 995 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 1 }, 996 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, 1 }, 997 998 // Complex: to v2f32 999 { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i8, 3 }, 1000 { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i16, 3 }, 1001 { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i64, 2 }, 1002 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i8, 3 }, 1003 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i16, 3 }, 1004 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i64, 2 }, 1005 1006 // Complex: to v4f32 1007 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i8, 4 }, 1008 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i16, 2 }, 1009 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i8, 3 }, 1010 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i16, 2 }, 1011 1012 // Complex: to v8f32 1013 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i8, 10 }, 1014 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, 4 }, 1015 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i8, 10 }, 1016 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, 4 }, 1017 1018 // Complex: to v16f32 1019 { ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i8, 21 }, 1020 { ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i8, 21 }, 1021 1022 // Complex: to v2f64 1023 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i8, 4 }, 1024 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i16, 4 }, 1025 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 }, 1026 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i8, 4 }, 1027 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i16, 4 }, 1028 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 }, 1029 1030 1031 // LowerVectorFP_TO_INT 1032 { ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f32, 1 }, 1033 { ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f32, 1 }, 1034 { ISD::FP_TO_SINT, MVT::v2i64, MVT::v2f64, 1 }, 1035 { ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f32, 1 }, 1036 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, 1 }, 1037 { ISD::FP_TO_UINT, MVT::v2i64, MVT::v2f64, 1 }, 1038 1039 // Complex, from v2f32: legal type is v2i32 (no cost) or v2i64 (1 ext). 1040 { ISD::FP_TO_SINT, MVT::v2i64, MVT::v2f32, 2 }, 1041 { ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f32, 1 }, 1042 { ISD::FP_TO_SINT, MVT::v2i8, MVT::v2f32, 1 }, 1043 { ISD::FP_TO_UINT, MVT::v2i64, MVT::v2f32, 2 }, 1044 { ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f32, 1 }, 1045 { ISD::FP_TO_UINT, MVT::v2i8, MVT::v2f32, 1 }, 1046 1047 // Complex, from v4f32: legal type is v4i16, 1 narrowing => ~2 1048 { ISD::FP_TO_SINT, MVT::v4i16, MVT::v4f32, 2 }, 1049 { ISD::FP_TO_SINT, MVT::v4i8, MVT::v4f32, 2 }, 1050 { ISD::FP_TO_UINT, MVT::v4i16, MVT::v4f32, 2 }, 1051 { ISD::FP_TO_UINT, MVT::v4i8, MVT::v4f32, 2 }, 1052 1053 // Complex, from nxv2f32. 1054 { ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f32, 1 }, 1055 { ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f32, 1 }, 1056 { ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f32, 1 }, 1057 { ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f32, 1 }, 1058 { ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f32, 1 }, 1059 { ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f32, 1 }, 1060 { ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f32, 1 }, 1061 { ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f32, 1 }, 1062 1063 // Complex, from v2f64: legal type is v2i32, 1 narrowing => ~2. 1064 { ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f64, 2 }, 1065 { ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f64, 2 }, 1066 { ISD::FP_TO_SINT, MVT::v2i8, MVT::v2f64, 2 }, 1067 { ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f64, 2 }, 1068 { ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f64, 2 }, 1069 { ISD::FP_TO_UINT, MVT::v2i8, MVT::v2f64, 2 }, 1070 1071 // Complex, from nxv2f64. 1072 { ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f64, 1 }, 1073 { ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f64, 1 }, 1074 { ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f64, 1 }, 1075 { ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f64, 1 }, 1076 { ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f64, 1 }, 1077 { ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f64, 1 }, 1078 { ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f64, 1 }, 1079 { ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f64, 1 }, 1080 1081 // Complex, from nxv4f32. 1082 { ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f32, 4 }, 1083 { ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f32, 1 }, 1084 { ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f32, 1 }, 1085 { ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f32, 1 }, 1086 { ISD::FP_TO_UINT, MVT::nxv4i64, MVT::nxv4f32, 4 }, 1087 { ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f32, 1 }, 1088 { ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f32, 1 }, 1089 { ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f32, 1 }, 1090 1091 // Complex, from nxv8f64. Illegal -> illegal conversions not required. 1092 { ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f64, 7 }, 1093 { ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f64, 7 }, 1094 { ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f64, 7 }, 1095 { ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f64, 7 }, 1096 1097 // Complex, from nxv4f64. Illegal -> illegal conversions not required. 1098 { ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f64, 3 }, 1099 { ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f64, 3 }, 1100 { ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f64, 3 }, 1101 { ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f64, 3 }, 1102 { ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f64, 3 }, 1103 { ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f64, 3 }, 1104 1105 // Complex, from nxv8f32. Illegal -> illegal conversions not required. 1106 { ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f32, 3 }, 1107 { ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f32, 3 }, 1108 { ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f32, 3 }, 1109 { ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f32, 3 }, 1110 1111 // Complex, from nxv8f16. 1112 { ISD::FP_TO_SINT, MVT::nxv8i64, MVT::nxv8f16, 10 }, 1113 { ISD::FP_TO_SINT, MVT::nxv8i32, MVT::nxv8f16, 4 }, 1114 { ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f16, 1 }, 1115 { ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f16, 1 }, 1116 { ISD::FP_TO_UINT, MVT::nxv8i64, MVT::nxv8f16, 10 }, 1117 { ISD::FP_TO_UINT, MVT::nxv8i32, MVT::nxv8f16, 4 }, 1118 { ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f16, 1 }, 1119 { ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f16, 1 }, 1120 1121 // Complex, from nxv4f16. 1122 { ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f16, 4 }, 1123 { ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f16, 1 }, 1124 { ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f16, 1 }, 1125 { ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f16, 1 }, 1126 { ISD::FP_TO_UINT, MVT::nxv4i64, MVT::nxv4f16, 4 }, 1127 { ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f16, 1 }, 1128 { ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f16, 1 }, 1129 { ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f16, 1 }, 1130 1131 // Complex, from nxv2f16. 1132 { ISD::FP_TO_SINT, MVT::nxv2i64, MVT::nxv2f16, 1 }, 1133 { ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f16, 1 }, 1134 { ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f16, 1 }, 1135 { ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f16, 1 }, 1136 { ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f16, 1 }, 1137 { ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f16, 1 }, 1138 { ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f16, 1 }, 1139 { ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f16, 1 }, 1140 1141 // Truncate from nxvmf32 to nxvmf16. 1142 { ISD::FP_ROUND, MVT::nxv2f16, MVT::nxv2f32, 1 }, 1143 { ISD::FP_ROUND, MVT::nxv4f16, MVT::nxv4f32, 1 }, 1144 { ISD::FP_ROUND, MVT::nxv8f16, MVT::nxv8f32, 3 }, 1145 1146 // Truncate from nxvmf64 to nxvmf16. 1147 { ISD::FP_ROUND, MVT::nxv2f16, MVT::nxv2f64, 1 }, 1148 { ISD::FP_ROUND, MVT::nxv4f16, MVT::nxv4f64, 3 }, 1149 { ISD::FP_ROUND, MVT::nxv8f16, MVT::nxv8f64, 7 }, 1150 1151 // Truncate from nxvmf64 to nxvmf32. 1152 { ISD::FP_ROUND, MVT::nxv2f32, MVT::nxv2f64, 1 }, 1153 { ISD::FP_ROUND, MVT::nxv4f32, MVT::nxv4f64, 3 }, 1154 { ISD::FP_ROUND, MVT::nxv8f32, MVT::nxv8f64, 6 }, 1155 1156 // Extend from nxvmf16 to nxvmf32. 1157 { ISD::FP_EXTEND, MVT::nxv2f32, MVT::nxv2f16, 1}, 1158 { ISD::FP_EXTEND, MVT::nxv4f32, MVT::nxv4f16, 1}, 1159 { ISD::FP_EXTEND, MVT::nxv8f32, MVT::nxv8f16, 2}, 1160 1161 // Extend from nxvmf16 to nxvmf64. 1162 { ISD::FP_EXTEND, MVT::nxv2f64, MVT::nxv2f16, 1}, 1163 { ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4f16, 2}, 1164 { ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8f16, 4}, 1165 1166 // Extend from nxvmf32 to nxvmf64. 1167 { ISD::FP_EXTEND, MVT::nxv2f64, MVT::nxv2f32, 1}, 1168 { ISD::FP_EXTEND, MVT::nxv4f64, MVT::nxv4f32, 2}, 1169 { ISD::FP_EXTEND, MVT::nxv8f64, MVT::nxv8f32, 6}, 1170 1171 }; 1172 1173 if (const auto *Entry = ConvertCostTableLookup(ConversionTbl, ISD, 1174 DstTy.getSimpleVT(), 1175 SrcTy.getSimpleVT())) 1176 return AdjustCost(Entry->Cost); 1177 1178 return AdjustCost( 1179 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I)); 1180 } 1181 1182 InstructionCost AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, 1183 Type *Dst, 1184 VectorType *VecTy, 1185 unsigned Index) { 1186 1187 // Make sure we were given a valid extend opcode. 1188 assert((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) && 1189 "Invalid opcode"); 1190 1191 // We are extending an element we extract from a vector, so the source type 1192 // of the extend is the element type of the vector. 1193 auto *Src = VecTy->getElementType(); 1194 1195 // Sign- and zero-extends are for integer types only. 1196 assert(isa<IntegerType>(Dst) && isa<IntegerType>(Src) && "Invalid type"); 1197 1198 // Get the cost for the extract. We compute the cost (if any) for the extend 1199 // below. 1200 InstructionCost Cost = 1201 getVectorInstrCost(Instruction::ExtractElement, VecTy, Index); 1202 1203 // Legalize the types. 1204 auto VecLT = TLI->getTypeLegalizationCost(DL, VecTy); 1205 auto DstVT = TLI->getValueType(DL, Dst); 1206 auto SrcVT = TLI->getValueType(DL, Src); 1207 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; 1208 1209 // If the resulting type is still a vector and the destination type is legal, 1210 // we may get the extension for free. If not, get the default cost for the 1211 // extend. 1212 if (!VecLT.second.isVector() || !TLI->isTypeLegal(DstVT)) 1213 return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None, 1214 CostKind); 1215 1216 // The destination type should be larger than the element type. If not, get 1217 // the default cost for the extend. 1218 if (DstVT.getFixedSizeInBits() < SrcVT.getFixedSizeInBits()) 1219 return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None, 1220 CostKind); 1221 1222 switch (Opcode) { 1223 default: 1224 llvm_unreachable("Opcode should be either SExt or ZExt"); 1225 1226 // For sign-extends, we only need a smov, which performs the extension 1227 // automatically. 1228 case Instruction::SExt: 1229 return Cost; 1230 1231 // For zero-extends, the extend is performed automatically by a umov unless 1232 // the destination type is i64 and the element type is i8 or i16. 1233 case Instruction::ZExt: 1234 if (DstVT.getSizeInBits() != 64u || SrcVT.getSizeInBits() == 32u) 1235 return Cost; 1236 } 1237 1238 // If we are unable to perform the extend for free, get the default cost. 1239 return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None, 1240 CostKind); 1241 } 1242 1243 InstructionCost AArch64TTIImpl::getCFInstrCost(unsigned Opcode, 1244 TTI::TargetCostKind CostKind, 1245 const Instruction *I) { 1246 if (CostKind != TTI::TCK_RecipThroughput) 1247 return Opcode == Instruction::PHI ? 0 : 1; 1248 assert(CostKind == TTI::TCK_RecipThroughput && "unexpected CostKind"); 1249 // Branches are assumed to be predicted. 1250 return 0; 1251 } 1252 1253 InstructionCost AArch64TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, 1254 unsigned Index) { 1255 assert(Val->isVectorTy() && "This must be a vector type"); 1256 1257 if (Index != -1U) { 1258 // Legalize the type. 1259 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Val); 1260 1261 // This type is legalized to a scalar type. 1262 if (!LT.second.isVector()) 1263 return 0; 1264 1265 // The type may be split. Normalize the index to the new type. 1266 unsigned Width = LT.second.getVectorNumElements(); 1267 Index = Index % Width; 1268 1269 // The element at index zero is already inside the vector. 1270 if (Index == 0) 1271 return 0; 1272 } 1273 1274 // All other insert/extracts cost this much. 1275 return ST->getVectorInsertExtractBaseCost(); 1276 } 1277 1278 InstructionCost AArch64TTIImpl::getArithmeticInstrCost( 1279 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, 1280 TTI::OperandValueKind Opd1Info, TTI::OperandValueKind Opd2Info, 1281 TTI::OperandValueProperties Opd1PropInfo, 1282 TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args, 1283 const Instruction *CxtI) { 1284 // TODO: Handle more cost kinds. 1285 if (CostKind != TTI::TCK_RecipThroughput) 1286 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info, 1287 Opd2Info, Opd1PropInfo, 1288 Opd2PropInfo, Args, CxtI); 1289 1290 // Legalize the type. 1291 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty); 1292 1293 // If the instruction is a widening instruction (e.g., uaddl, saddw, etc.), 1294 // add in the widening overhead specified by the sub-target. Since the 1295 // extends feeding widening instructions are performed automatically, they 1296 // aren't present in the generated code and have a zero cost. By adding a 1297 // widening overhead here, we attach the total cost of the combined operation 1298 // to the widening instruction. 1299 InstructionCost Cost = 0; 1300 if (isWideningInstruction(Ty, Opcode, Args)) 1301 Cost += ST->getWideningBaseCost(); 1302 1303 int ISD = TLI->InstructionOpcodeToISD(Opcode); 1304 1305 switch (ISD) { 1306 default: 1307 return Cost + BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info, 1308 Opd2Info, 1309 Opd1PropInfo, Opd2PropInfo); 1310 case ISD::SDIV: 1311 if (Opd2Info == TargetTransformInfo::OK_UniformConstantValue && 1312 Opd2PropInfo == TargetTransformInfo::OP_PowerOf2) { 1313 // On AArch64, scalar signed division by constants power-of-two are 1314 // normally expanded to the sequence ADD + CMP + SELECT + SRA. 1315 // The OperandValue properties many not be same as that of previous 1316 // operation; conservatively assume OP_None. 1317 Cost += getArithmeticInstrCost(Instruction::Add, Ty, CostKind, 1318 Opd1Info, Opd2Info, 1319 TargetTransformInfo::OP_None, 1320 TargetTransformInfo::OP_None); 1321 Cost += getArithmeticInstrCost(Instruction::Sub, Ty, CostKind, 1322 Opd1Info, Opd2Info, 1323 TargetTransformInfo::OP_None, 1324 TargetTransformInfo::OP_None); 1325 Cost += getArithmeticInstrCost(Instruction::Select, Ty, CostKind, 1326 Opd1Info, Opd2Info, 1327 TargetTransformInfo::OP_None, 1328 TargetTransformInfo::OP_None); 1329 Cost += getArithmeticInstrCost(Instruction::AShr, Ty, CostKind, 1330 Opd1Info, Opd2Info, 1331 TargetTransformInfo::OP_None, 1332 TargetTransformInfo::OP_None); 1333 return Cost; 1334 } 1335 LLVM_FALLTHROUGH; 1336 case ISD::UDIV: 1337 if (Opd2Info == TargetTransformInfo::OK_UniformConstantValue) { 1338 auto VT = TLI->getValueType(DL, Ty); 1339 if (TLI->isOperationLegalOrCustom(ISD::MULHU, VT)) { 1340 // Vector signed division by constant are expanded to the 1341 // sequence MULHS + ADD/SUB + SRA + SRL + ADD, and unsigned division 1342 // to MULHS + SUB + SRL + ADD + SRL. 1343 InstructionCost MulCost = getArithmeticInstrCost( 1344 Instruction::Mul, Ty, CostKind, Opd1Info, Opd2Info, 1345 TargetTransformInfo::OP_None, TargetTransformInfo::OP_None); 1346 InstructionCost AddCost = getArithmeticInstrCost( 1347 Instruction::Add, Ty, CostKind, Opd1Info, Opd2Info, 1348 TargetTransformInfo::OP_None, TargetTransformInfo::OP_None); 1349 InstructionCost ShrCost = getArithmeticInstrCost( 1350 Instruction::AShr, Ty, CostKind, Opd1Info, Opd2Info, 1351 TargetTransformInfo::OP_None, TargetTransformInfo::OP_None); 1352 return MulCost * 2 + AddCost * 2 + ShrCost * 2 + 1; 1353 } 1354 } 1355 1356 Cost += BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info, 1357 Opd2Info, 1358 Opd1PropInfo, Opd2PropInfo); 1359 if (Ty->isVectorTy()) { 1360 // On AArch64, vector divisions are not supported natively and are 1361 // expanded into scalar divisions of each pair of elements. 1362 Cost += getArithmeticInstrCost(Instruction::ExtractElement, Ty, CostKind, 1363 Opd1Info, Opd2Info, Opd1PropInfo, 1364 Opd2PropInfo); 1365 Cost += getArithmeticInstrCost(Instruction::InsertElement, Ty, CostKind, 1366 Opd1Info, Opd2Info, Opd1PropInfo, 1367 Opd2PropInfo); 1368 // TODO: if one of the arguments is scalar, then it's not necessary to 1369 // double the cost of handling the vector elements. 1370 Cost += Cost; 1371 } 1372 return Cost; 1373 1374 case ISD::MUL: 1375 if (LT.second != MVT::v2i64) 1376 return (Cost + 1) * LT.first; 1377 // Since we do not have a MUL.2d instruction, a mul <2 x i64> is expensive 1378 // as elements are extracted from the vectors and the muls scalarized. 1379 // As getScalarizationOverhead is a bit too pessimistic, we estimate the 1380 // cost for a i64 vector directly here, which is: 1381 // - four i64 extracts, 1382 // - two i64 inserts, and 1383 // - two muls. 1384 // So, for a v2i64 with LT.First = 1 the cost is 8, and for a v4i64 with 1385 // LT.first = 2 the cost is 16. 1386 return LT.first * 8; 1387 case ISD::ADD: 1388 case ISD::XOR: 1389 case ISD::OR: 1390 case ISD::AND: 1391 // These nodes are marked as 'custom' for combining purposes only. 1392 // We know that they are legal. See LowerAdd in ISelLowering. 1393 return (Cost + 1) * LT.first; 1394 1395 case ISD::FADD: 1396 // These nodes are marked as 'custom' just to lower them to SVE. 1397 // We know said lowering will incur no additional cost. 1398 if (isa<FixedVectorType>(Ty) && !Ty->getScalarType()->isFP128Ty()) 1399 return (Cost + 2) * LT.first; 1400 1401 return Cost + BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info, 1402 Opd2Info, 1403 Opd1PropInfo, Opd2PropInfo); 1404 } 1405 } 1406 1407 InstructionCost AArch64TTIImpl::getAddressComputationCost(Type *Ty, 1408 ScalarEvolution *SE, 1409 const SCEV *Ptr) { 1410 // Address computations in vectorized code with non-consecutive addresses will 1411 // likely result in more instructions compared to scalar code where the 1412 // computation can more often be merged into the index mode. The resulting 1413 // extra micro-ops can significantly decrease throughput. 1414 unsigned NumVectorInstToHideOverhead = 10; 1415 int MaxMergeDistance = 64; 1416 1417 if (Ty->isVectorTy() && SE && 1418 !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MaxMergeDistance + 1)) 1419 return NumVectorInstToHideOverhead; 1420 1421 // In many cases the address computation is not merged into the instruction 1422 // addressing mode. 1423 return 1; 1424 } 1425 1426 InstructionCost AArch64TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, 1427 Type *CondTy, 1428 CmpInst::Predicate VecPred, 1429 TTI::TargetCostKind CostKind, 1430 const Instruction *I) { 1431 // TODO: Handle other cost kinds. 1432 if (CostKind != TTI::TCK_RecipThroughput) 1433 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, 1434 I); 1435 1436 int ISD = TLI->InstructionOpcodeToISD(Opcode); 1437 // We don't lower some vector selects well that are wider than the register 1438 // width. 1439 if (isa<FixedVectorType>(ValTy) && ISD == ISD::SELECT) { 1440 // We would need this many instructions to hide the scalarization happening. 1441 const int AmortizationCost = 20; 1442 1443 // If VecPred is not set, check if we can get a predicate from the context 1444 // instruction, if its type matches the requested ValTy. 1445 if (VecPred == CmpInst::BAD_ICMP_PREDICATE && I && I->getType() == ValTy) { 1446 CmpInst::Predicate CurrentPred; 1447 if (match(I, m_Select(m_Cmp(CurrentPred, m_Value(), m_Value()), m_Value(), 1448 m_Value()))) 1449 VecPred = CurrentPred; 1450 } 1451 // Check if we have a compare/select chain that can be lowered using CMxx & 1452 // BFI pair. 1453 if (CmpInst::isIntPredicate(VecPred)) { 1454 static const auto ValidMinMaxTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16, 1455 MVT::v8i16, MVT::v2i32, MVT::v4i32, 1456 MVT::v2i64}; 1457 auto LT = TLI->getTypeLegalizationCost(DL, ValTy); 1458 if (any_of(ValidMinMaxTys, [<](MVT M) { return M == LT.second; })) 1459 return LT.first; 1460 } 1461 1462 static const TypeConversionCostTblEntry 1463 VectorSelectTbl[] = { 1464 { ISD::SELECT, MVT::v16i1, MVT::v16i16, 16 }, 1465 { ISD::SELECT, MVT::v8i1, MVT::v8i32, 8 }, 1466 { ISD::SELECT, MVT::v16i1, MVT::v16i32, 16 }, 1467 { ISD::SELECT, MVT::v4i1, MVT::v4i64, 4 * AmortizationCost }, 1468 { ISD::SELECT, MVT::v8i1, MVT::v8i64, 8 * AmortizationCost }, 1469 { ISD::SELECT, MVT::v16i1, MVT::v16i64, 16 * AmortizationCost } 1470 }; 1471 1472 EVT SelCondTy = TLI->getValueType(DL, CondTy); 1473 EVT SelValTy = TLI->getValueType(DL, ValTy); 1474 if (SelCondTy.isSimple() && SelValTy.isSimple()) { 1475 if (const auto *Entry = ConvertCostTableLookup(VectorSelectTbl, ISD, 1476 SelCondTy.getSimpleVT(), 1477 SelValTy.getSimpleVT())) 1478 return Entry->Cost; 1479 } 1480 } 1481 // The base case handles scalable vectors fine for now, since it treats the 1482 // cost as 1 * legalization cost. 1483 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); 1484 } 1485 1486 AArch64TTIImpl::TTI::MemCmpExpansionOptions 1487 AArch64TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const { 1488 TTI::MemCmpExpansionOptions Options; 1489 if (ST->requiresStrictAlign()) { 1490 // TODO: Add cost modeling for strict align. Misaligned loads expand to 1491 // a bunch of instructions when strict align is enabled. 1492 return Options; 1493 } 1494 Options.AllowOverlappingLoads = true; 1495 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize); 1496 Options.NumLoadsPerBlock = Options.MaxNumLoads; 1497 // TODO: Though vector loads usually perform well on AArch64, in some targets 1498 // they may wake up the FP unit, which raises the power consumption. Perhaps 1499 // they could be used with no holds barred (-O3). 1500 Options.LoadSizes = {8, 4, 2, 1}; 1501 return Options; 1502 } 1503 1504 InstructionCost 1505 AArch64TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, 1506 Align Alignment, unsigned AddressSpace, 1507 TTI::TargetCostKind CostKind) { 1508 if (!isa<ScalableVectorType>(Src)) 1509 return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, 1510 CostKind); 1511 auto LT = TLI->getTypeLegalizationCost(DL, Src); 1512 if (!LT.first.isValid()) 1513 return InstructionCost::getInvalid(); 1514 1515 // The code-generator is currently not able to handle scalable vectors 1516 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting 1517 // it. This change will be removed when code-generation for these types is 1518 // sufficiently reliable. 1519 if (cast<VectorType>(Src)->getElementCount() == ElementCount::getScalable(1)) 1520 return InstructionCost::getInvalid(); 1521 1522 return LT.first * 2; 1523 } 1524 1525 InstructionCost AArch64TTIImpl::getGatherScatterOpCost( 1526 unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, 1527 Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) { 1528 1529 if (!isa<ScalableVectorType>(DataTy)) 1530 return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, 1531 Alignment, CostKind, I); 1532 auto *VT = cast<VectorType>(DataTy); 1533 auto LT = TLI->getTypeLegalizationCost(DL, DataTy); 1534 if (!LT.first.isValid()) 1535 return InstructionCost::getInvalid(); 1536 1537 // The code-generator is currently not able to handle scalable vectors 1538 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting 1539 // it. This change will be removed when code-generation for these types is 1540 // sufficiently reliable. 1541 if (cast<VectorType>(DataTy)->getElementCount() == 1542 ElementCount::getScalable(1)) 1543 return InstructionCost::getInvalid(); 1544 1545 ElementCount LegalVF = LT.second.getVectorElementCount(); 1546 InstructionCost MemOpCost = 1547 getMemoryOpCost(Opcode, VT->getElementType(), Alignment, 0, CostKind, I); 1548 return LT.first * MemOpCost * getMaxNumElements(LegalVF); 1549 } 1550 1551 bool AArch64TTIImpl::useNeonVector(const Type *Ty) const { 1552 return isa<FixedVectorType>(Ty) && !ST->useSVEForFixedLengthVectors(); 1553 } 1554 1555 InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty, 1556 MaybeAlign Alignment, 1557 unsigned AddressSpace, 1558 TTI::TargetCostKind CostKind, 1559 const Instruction *I) { 1560 EVT VT = TLI->getValueType(DL, Ty, true); 1561 // Type legalization can't handle structs 1562 if (VT == MVT::Other) 1563 return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace, 1564 CostKind); 1565 1566 auto LT = TLI->getTypeLegalizationCost(DL, Ty); 1567 if (!LT.first.isValid()) 1568 return InstructionCost::getInvalid(); 1569 1570 // The code-generator is currently not able to handle scalable vectors 1571 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting 1572 // it. This change will be removed when code-generation for these types is 1573 // sufficiently reliable. 1574 if (auto *VTy = dyn_cast<ScalableVectorType>(Ty)) 1575 if (VTy->getElementCount() == ElementCount::getScalable(1)) 1576 return InstructionCost::getInvalid(); 1577 1578 // TODO: consider latency as well for TCK_SizeAndLatency. 1579 if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency) 1580 return LT.first; 1581 1582 if (CostKind != TTI::TCK_RecipThroughput) 1583 return 1; 1584 1585 if (ST->isMisaligned128StoreSlow() && Opcode == Instruction::Store && 1586 LT.second.is128BitVector() && (!Alignment || *Alignment < Align(16))) { 1587 // Unaligned stores are extremely inefficient. We don't split all 1588 // unaligned 128-bit stores because the negative impact that has shown in 1589 // practice on inlined block copy code. 1590 // We make such stores expensive so that we will only vectorize if there 1591 // are 6 other instructions getting vectorized. 1592 const int AmortizationCost = 6; 1593 1594 return LT.first * 2 * AmortizationCost; 1595 } 1596 1597 // Check truncating stores and extending loads. 1598 if (useNeonVector(Ty) && 1599 Ty->getScalarSizeInBits() != LT.second.getScalarSizeInBits()) { 1600 // v4i8 types are lowered to scalar a load/store and sshll/xtn. 1601 if (VT == MVT::v4i8) 1602 return 2; 1603 // Otherwise we need to scalarize. 1604 return cast<FixedVectorType>(Ty)->getNumElements() * 2; 1605 } 1606 1607 return LT.first; 1608 } 1609 1610 InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost( 1611 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, 1612 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, 1613 bool UseMaskForCond, bool UseMaskForGaps) { 1614 assert(Factor >= 2 && "Invalid interleave factor"); 1615 auto *VecVTy = cast<FixedVectorType>(VecTy); 1616 1617 if (!UseMaskForCond && !UseMaskForGaps && 1618 Factor <= TLI->getMaxSupportedInterleaveFactor()) { 1619 unsigned NumElts = VecVTy->getNumElements(); 1620 auto *SubVecTy = 1621 FixedVectorType::get(VecTy->getScalarType(), NumElts / Factor); 1622 1623 // ldN/stN only support legal vector types of size 64 or 128 in bits. 1624 // Accesses having vector types that are a multiple of 128 bits can be 1625 // matched to more than one ldN/stN instruction. 1626 if (NumElts % Factor == 0 && 1627 TLI->isLegalInterleavedAccessType(SubVecTy, DL)) 1628 return Factor * TLI->getNumInterleavedAccesses(SubVecTy, DL); 1629 } 1630 1631 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, 1632 Alignment, AddressSpace, CostKind, 1633 UseMaskForCond, UseMaskForGaps); 1634 } 1635 1636 InstructionCost 1637 AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) { 1638 InstructionCost Cost = 0; 1639 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; 1640 for (auto *I : Tys) { 1641 if (!I->isVectorTy()) 1642 continue; 1643 if (I->getScalarSizeInBits() * cast<FixedVectorType>(I)->getNumElements() == 1644 128) 1645 Cost += getMemoryOpCost(Instruction::Store, I, Align(128), 0, CostKind) + 1646 getMemoryOpCost(Instruction::Load, I, Align(128), 0, CostKind); 1647 } 1648 return Cost; 1649 } 1650 1651 unsigned AArch64TTIImpl::getMaxInterleaveFactor(unsigned VF) { 1652 return ST->getMaxInterleaveFactor(); 1653 } 1654 1655 // For Falkor, we want to avoid having too many strided loads in a loop since 1656 // that can exhaust the HW prefetcher resources. We adjust the unroller 1657 // MaxCount preference below to attempt to ensure unrolling doesn't create too 1658 // many strided loads. 1659 static void 1660 getFalkorUnrollingPreferences(Loop *L, ScalarEvolution &SE, 1661 TargetTransformInfo::UnrollingPreferences &UP) { 1662 enum { MaxStridedLoads = 7 }; 1663 auto countStridedLoads = [](Loop *L, ScalarEvolution &SE) { 1664 int StridedLoads = 0; 1665 // FIXME? We could make this more precise by looking at the CFG and 1666 // e.g. not counting loads in each side of an if-then-else diamond. 1667 for (const auto BB : L->blocks()) { 1668 for (auto &I : *BB) { 1669 LoadInst *LMemI = dyn_cast<LoadInst>(&I); 1670 if (!LMemI) 1671 continue; 1672 1673 Value *PtrValue = LMemI->getPointerOperand(); 1674 if (L->isLoopInvariant(PtrValue)) 1675 continue; 1676 1677 const SCEV *LSCEV = SE.getSCEV(PtrValue); 1678 const SCEVAddRecExpr *LSCEVAddRec = dyn_cast<SCEVAddRecExpr>(LSCEV); 1679 if (!LSCEVAddRec || !LSCEVAddRec->isAffine()) 1680 continue; 1681 1682 // FIXME? We could take pairing of unrolled load copies into account 1683 // by looking at the AddRec, but we would probably have to limit this 1684 // to loops with no stores or other memory optimization barriers. 1685 ++StridedLoads; 1686 // We've seen enough strided loads that seeing more won't make a 1687 // difference. 1688 if (StridedLoads > MaxStridedLoads / 2) 1689 return StridedLoads; 1690 } 1691 } 1692 return StridedLoads; 1693 }; 1694 1695 int StridedLoads = countStridedLoads(L, SE); 1696 LLVM_DEBUG(dbgs() << "falkor-hwpf: detected " << StridedLoads 1697 << " strided loads\n"); 1698 // Pick the largest power of 2 unroll count that won't result in too many 1699 // strided loads. 1700 if (StridedLoads) { 1701 UP.MaxCount = 1 << Log2_32(MaxStridedLoads / StridedLoads); 1702 LLVM_DEBUG(dbgs() << "falkor-hwpf: setting unroll MaxCount to " 1703 << UP.MaxCount << '\n'); 1704 } 1705 } 1706 1707 void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, 1708 TTI::UnrollingPreferences &UP) { 1709 // Enable partial unrolling and runtime unrolling. 1710 BaseT::getUnrollingPreferences(L, SE, UP); 1711 1712 // For inner loop, it is more likely to be a hot one, and the runtime check 1713 // can be promoted out from LICM pass, so the overhead is less, let's try 1714 // a larger threshold to unroll more loops. 1715 if (L->getLoopDepth() > 1) 1716 UP.PartialThreshold *= 2; 1717 1718 // Disable partial & runtime unrolling on -Os. 1719 UP.PartialOptSizeThreshold = 0; 1720 1721 if (ST->getProcFamily() == AArch64Subtarget::Falkor && 1722 EnableFalkorHWPFUnrollFix) 1723 getFalkorUnrollingPreferences(L, SE, UP); 1724 1725 // Scan the loop: don't unroll loops with calls as this could prevent 1726 // inlining. Don't unroll vector loops either, as they don't benefit much from 1727 // unrolling. 1728 for (auto *BB : L->getBlocks()) { 1729 for (auto &I : *BB) { 1730 // Don't unroll vectorised loop. 1731 if (I.getType()->isVectorTy()) 1732 return; 1733 1734 if (isa<CallInst>(I) || isa<InvokeInst>(I)) { 1735 if (const Function *F = cast<CallBase>(I).getCalledFunction()) { 1736 if (!isLoweredToCall(F)) 1737 continue; 1738 } 1739 return; 1740 } 1741 } 1742 } 1743 1744 // Enable runtime unrolling for in-order models 1745 // If mcpu is omitted, getProcFamily() returns AArch64Subtarget::Others, so by 1746 // checking for that case, we can ensure that the default behaviour is 1747 // unchanged 1748 if (ST->getProcFamily() != AArch64Subtarget::Others && 1749 !ST->getSchedModel().isOutOfOrder()) { 1750 UP.Runtime = true; 1751 UP.Partial = true; 1752 UP.UpperBound = true; 1753 UP.UnrollRemainder = true; 1754 UP.DefaultUnrollRuntimeCount = 4; 1755 1756 UP.UnrollAndJam = true; 1757 UP.UnrollAndJamInnerLoopThreshold = 60; 1758 } 1759 } 1760 1761 void AArch64TTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, 1762 TTI::PeelingPreferences &PP) { 1763 BaseT::getPeelingPreferences(L, SE, PP); 1764 } 1765 1766 Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, 1767 Type *ExpectedType) { 1768 switch (Inst->getIntrinsicID()) { 1769 default: 1770 return nullptr; 1771 case Intrinsic::aarch64_neon_st2: 1772 case Intrinsic::aarch64_neon_st3: 1773 case Intrinsic::aarch64_neon_st4: { 1774 // Create a struct type 1775 StructType *ST = dyn_cast<StructType>(ExpectedType); 1776 if (!ST) 1777 return nullptr; 1778 unsigned NumElts = Inst->getNumArgOperands() - 1; 1779 if (ST->getNumElements() != NumElts) 1780 return nullptr; 1781 for (unsigned i = 0, e = NumElts; i != e; ++i) { 1782 if (Inst->getArgOperand(i)->getType() != ST->getElementType(i)) 1783 return nullptr; 1784 } 1785 Value *Res = UndefValue::get(ExpectedType); 1786 IRBuilder<> Builder(Inst); 1787 for (unsigned i = 0, e = NumElts; i != e; ++i) { 1788 Value *L = Inst->getArgOperand(i); 1789 Res = Builder.CreateInsertValue(Res, L, i); 1790 } 1791 return Res; 1792 } 1793 case Intrinsic::aarch64_neon_ld2: 1794 case Intrinsic::aarch64_neon_ld3: 1795 case Intrinsic::aarch64_neon_ld4: 1796 if (Inst->getType() == ExpectedType) 1797 return Inst; 1798 return nullptr; 1799 } 1800 } 1801 1802 bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst, 1803 MemIntrinsicInfo &Info) { 1804 switch (Inst->getIntrinsicID()) { 1805 default: 1806 break; 1807 case Intrinsic::aarch64_neon_ld2: 1808 case Intrinsic::aarch64_neon_ld3: 1809 case Intrinsic::aarch64_neon_ld4: 1810 Info.ReadMem = true; 1811 Info.WriteMem = false; 1812 Info.PtrVal = Inst->getArgOperand(0); 1813 break; 1814 case Intrinsic::aarch64_neon_st2: 1815 case Intrinsic::aarch64_neon_st3: 1816 case Intrinsic::aarch64_neon_st4: 1817 Info.ReadMem = false; 1818 Info.WriteMem = true; 1819 Info.PtrVal = Inst->getArgOperand(Inst->getNumArgOperands() - 1); 1820 break; 1821 } 1822 1823 switch (Inst->getIntrinsicID()) { 1824 default: 1825 return false; 1826 case Intrinsic::aarch64_neon_ld2: 1827 case Intrinsic::aarch64_neon_st2: 1828 Info.MatchingId = VECTOR_LDST_TWO_ELEMENTS; 1829 break; 1830 case Intrinsic::aarch64_neon_ld3: 1831 case Intrinsic::aarch64_neon_st3: 1832 Info.MatchingId = VECTOR_LDST_THREE_ELEMENTS; 1833 break; 1834 case Intrinsic::aarch64_neon_ld4: 1835 case Intrinsic::aarch64_neon_st4: 1836 Info.MatchingId = VECTOR_LDST_FOUR_ELEMENTS; 1837 break; 1838 } 1839 return true; 1840 } 1841 1842 /// See if \p I should be considered for address type promotion. We check if \p 1843 /// I is a sext with right type and used in memory accesses. If it used in a 1844 /// "complex" getelementptr, we allow it to be promoted without finding other 1845 /// sext instructions that sign extended the same initial value. A getelementptr 1846 /// is considered as "complex" if it has more than 2 operands. 1847 bool AArch64TTIImpl::shouldConsiderAddressTypePromotion( 1848 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) { 1849 bool Considerable = false; 1850 AllowPromotionWithoutCommonHeader = false; 1851 if (!isa<SExtInst>(&I)) 1852 return false; 1853 Type *ConsideredSExtType = 1854 Type::getInt64Ty(I.getParent()->getParent()->getContext()); 1855 if (I.getType() != ConsideredSExtType) 1856 return false; 1857 // See if the sext is the one with the right type and used in at least one 1858 // GetElementPtrInst. 1859 for (const User *U : I.users()) { 1860 if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(U)) { 1861 Considerable = true; 1862 // A getelementptr is considered as "complex" if it has more than 2 1863 // operands. We will promote a SExt used in such complex GEP as we 1864 // expect some computation to be merged if they are done on 64 bits. 1865 if (GEPInst->getNumOperands() > 2) { 1866 AllowPromotionWithoutCommonHeader = true; 1867 break; 1868 } 1869 } 1870 } 1871 return Considerable; 1872 } 1873 1874 bool AArch64TTIImpl::isLegalToVectorizeReduction( 1875 const RecurrenceDescriptor &RdxDesc, ElementCount VF) const { 1876 if (!VF.isScalable()) 1877 return true; 1878 1879 Type *Ty = RdxDesc.getRecurrenceType(); 1880 if (Ty->isBFloatTy() || !isElementTypeLegalForScalableVector(Ty)) 1881 return false; 1882 1883 switch (RdxDesc.getRecurrenceKind()) { 1884 case RecurKind::Add: 1885 case RecurKind::FAdd: 1886 case RecurKind::And: 1887 case RecurKind::Or: 1888 case RecurKind::Xor: 1889 case RecurKind::SMin: 1890 case RecurKind::SMax: 1891 case RecurKind::UMin: 1892 case RecurKind::UMax: 1893 case RecurKind::FMin: 1894 case RecurKind::FMax: 1895 return true; 1896 default: 1897 return false; 1898 } 1899 } 1900 1901 InstructionCost 1902 AArch64TTIImpl::getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy, 1903 bool IsUnsigned, 1904 TTI::TargetCostKind CostKind) { 1905 if (!isa<ScalableVectorType>(Ty)) 1906 return BaseT::getMinMaxReductionCost(Ty, CondTy, IsUnsigned, CostKind); 1907 assert((isa<ScalableVectorType>(Ty) && isa<ScalableVectorType>(CondTy)) && 1908 "Both vector needs to be scalable"); 1909 1910 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty); 1911 InstructionCost LegalizationCost = 0; 1912 if (LT.first > 1) { 1913 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Ty->getContext()); 1914 unsigned CmpOpcode = 1915 Ty->isFPOrFPVectorTy() ? Instruction::FCmp : Instruction::ICmp; 1916 LegalizationCost = 1917 getCmpSelInstrCost(CmpOpcode, LegalVTy, LegalVTy, 1918 CmpInst::BAD_ICMP_PREDICATE, CostKind) + 1919 getCmpSelInstrCost(Instruction::Select, LegalVTy, LegalVTy, 1920 CmpInst::BAD_ICMP_PREDICATE, CostKind); 1921 LegalizationCost *= LT.first - 1; 1922 } 1923 1924 return LegalizationCost + /*Cost of horizontal reduction*/ 2; 1925 } 1926 1927 InstructionCost AArch64TTIImpl::getArithmeticReductionCostSVE( 1928 unsigned Opcode, VectorType *ValTy, TTI::TargetCostKind CostKind) { 1929 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, ValTy); 1930 InstructionCost LegalizationCost = 0; 1931 if (LT.first > 1) { 1932 Type *LegalVTy = EVT(LT.second).getTypeForEVT(ValTy->getContext()); 1933 LegalizationCost = getArithmeticInstrCost(Opcode, LegalVTy, CostKind); 1934 LegalizationCost *= LT.first - 1; 1935 } 1936 1937 int ISD = TLI->InstructionOpcodeToISD(Opcode); 1938 assert(ISD && "Invalid opcode"); 1939 // Add the final reduction cost for the legal horizontal reduction 1940 switch (ISD) { 1941 case ISD::ADD: 1942 case ISD::AND: 1943 case ISD::OR: 1944 case ISD::XOR: 1945 case ISD::FADD: 1946 return LegalizationCost + 2; 1947 default: 1948 return InstructionCost::getInvalid(); 1949 } 1950 } 1951 1952 InstructionCost 1953 AArch64TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy, 1954 Optional<FastMathFlags> FMF, 1955 TTI::TargetCostKind CostKind) { 1956 if (TTI::requiresOrderedReduction(FMF)) { 1957 if (!isa<ScalableVectorType>(ValTy)) 1958 return BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind); 1959 1960 if (Opcode != Instruction::FAdd) 1961 return InstructionCost::getInvalid(); 1962 1963 auto *VTy = cast<ScalableVectorType>(ValTy); 1964 InstructionCost Cost = 1965 getArithmeticInstrCost(Opcode, VTy->getScalarType(), CostKind); 1966 Cost *= getMaxNumElements(VTy->getElementCount()); 1967 return Cost; 1968 } 1969 1970 if (isa<ScalableVectorType>(ValTy)) 1971 return getArithmeticReductionCostSVE(Opcode, ValTy, CostKind); 1972 1973 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, ValTy); 1974 MVT MTy = LT.second; 1975 int ISD = TLI->InstructionOpcodeToISD(Opcode); 1976 assert(ISD && "Invalid opcode"); 1977 1978 // Horizontal adds can use the 'addv' instruction. We model the cost of these 1979 // instructions as twice a normal vector add, plus 1 for each legalization 1980 // step (LT.first). This is the only arithmetic vector reduction operation for 1981 // which we have an instruction. 1982 // OR, XOR and AND costs should match the codegen from: 1983 // OR: llvm/test/CodeGen/AArch64/reduce-or.ll 1984 // XOR: llvm/test/CodeGen/AArch64/reduce-xor.ll 1985 // AND: llvm/test/CodeGen/AArch64/reduce-and.ll 1986 static const CostTblEntry CostTblNoPairwise[]{ 1987 {ISD::ADD, MVT::v8i8, 2}, 1988 {ISD::ADD, MVT::v16i8, 2}, 1989 {ISD::ADD, MVT::v4i16, 2}, 1990 {ISD::ADD, MVT::v8i16, 2}, 1991 {ISD::ADD, MVT::v4i32, 2}, 1992 {ISD::OR, MVT::v8i8, 15}, 1993 {ISD::OR, MVT::v16i8, 17}, 1994 {ISD::OR, MVT::v4i16, 7}, 1995 {ISD::OR, MVT::v8i16, 9}, 1996 {ISD::OR, MVT::v2i32, 3}, 1997 {ISD::OR, MVT::v4i32, 5}, 1998 {ISD::OR, MVT::v2i64, 3}, 1999 {ISD::XOR, MVT::v8i8, 15}, 2000 {ISD::XOR, MVT::v16i8, 17}, 2001 {ISD::XOR, MVT::v4i16, 7}, 2002 {ISD::XOR, MVT::v8i16, 9}, 2003 {ISD::XOR, MVT::v2i32, 3}, 2004 {ISD::XOR, MVT::v4i32, 5}, 2005 {ISD::XOR, MVT::v2i64, 3}, 2006 {ISD::AND, MVT::v8i8, 15}, 2007 {ISD::AND, MVT::v16i8, 17}, 2008 {ISD::AND, MVT::v4i16, 7}, 2009 {ISD::AND, MVT::v8i16, 9}, 2010 {ISD::AND, MVT::v2i32, 3}, 2011 {ISD::AND, MVT::v4i32, 5}, 2012 {ISD::AND, MVT::v2i64, 3}, 2013 }; 2014 switch (ISD) { 2015 default: 2016 break; 2017 case ISD::ADD: 2018 if (const auto *Entry = CostTableLookup(CostTblNoPairwise, ISD, MTy)) 2019 return (LT.first - 1) + Entry->Cost; 2020 break; 2021 case ISD::XOR: 2022 case ISD::AND: 2023 case ISD::OR: 2024 const auto *Entry = CostTableLookup(CostTblNoPairwise, ISD, MTy); 2025 if (!Entry) 2026 break; 2027 auto *ValVTy = cast<FixedVectorType>(ValTy); 2028 if (!ValVTy->getElementType()->isIntegerTy(1) && 2029 MTy.getVectorNumElements() <= ValVTy->getNumElements() && 2030 isPowerOf2_32(ValVTy->getNumElements())) { 2031 InstructionCost ExtraCost = 0; 2032 if (LT.first != 1) { 2033 // Type needs to be split, so there is an extra cost of LT.first - 1 2034 // arithmetic ops. 2035 auto *Ty = FixedVectorType::get(ValTy->getElementType(), 2036 MTy.getVectorNumElements()); 2037 ExtraCost = getArithmeticInstrCost(Opcode, Ty, CostKind); 2038 ExtraCost *= LT.first - 1; 2039 } 2040 return Entry->Cost + ExtraCost; 2041 } 2042 break; 2043 } 2044 return BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind); 2045 } 2046 2047 InstructionCost AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index) { 2048 static const CostTblEntry ShuffleTbl[] = { 2049 { TTI::SK_Splice, MVT::nxv16i8, 1 }, 2050 { TTI::SK_Splice, MVT::nxv8i16, 1 }, 2051 { TTI::SK_Splice, MVT::nxv4i32, 1 }, 2052 { TTI::SK_Splice, MVT::nxv2i64, 1 }, 2053 { TTI::SK_Splice, MVT::nxv2f16, 1 }, 2054 { TTI::SK_Splice, MVT::nxv4f16, 1 }, 2055 { TTI::SK_Splice, MVT::nxv8f16, 1 }, 2056 { TTI::SK_Splice, MVT::nxv2bf16, 1 }, 2057 { TTI::SK_Splice, MVT::nxv4bf16, 1 }, 2058 { TTI::SK_Splice, MVT::nxv8bf16, 1 }, 2059 { TTI::SK_Splice, MVT::nxv2f32, 1 }, 2060 { TTI::SK_Splice, MVT::nxv4f32, 1 }, 2061 { TTI::SK_Splice, MVT::nxv2f64, 1 }, 2062 }; 2063 2064 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp); 2065 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Tp->getContext()); 2066 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; 2067 EVT PromotedVT = LT.second.getScalarType() == MVT::i1 2068 ? TLI->getPromotedVTForPredicate(EVT(LT.second)) 2069 : LT.second; 2070 Type *PromotedVTy = EVT(PromotedVT).getTypeForEVT(Tp->getContext()); 2071 InstructionCost LegalizationCost = 0; 2072 if (Index < 0) { 2073 LegalizationCost = 2074 getCmpSelInstrCost(Instruction::ICmp, PromotedVTy, PromotedVTy, 2075 CmpInst::BAD_ICMP_PREDICATE, CostKind) + 2076 getCmpSelInstrCost(Instruction::Select, PromotedVTy, LegalVTy, 2077 CmpInst::BAD_ICMP_PREDICATE, CostKind); 2078 } 2079 2080 // Predicated splice are promoted when lowering. See AArch64ISelLowering.cpp 2081 // Cost performed on a promoted type. 2082 if (LT.second.getScalarType() == MVT::i1) { 2083 LegalizationCost += 2084 getCastInstrCost(Instruction::ZExt, PromotedVTy, LegalVTy, 2085 TTI::CastContextHint::None, CostKind) + 2086 getCastInstrCost(Instruction::Trunc, LegalVTy, PromotedVTy, 2087 TTI::CastContextHint::None, CostKind); 2088 } 2089 const auto *Entry = 2090 CostTableLookup(ShuffleTbl, TTI::SK_Splice, PromotedVT.getSimpleVT()); 2091 assert(Entry && "Illegal Type for Splice"); 2092 LegalizationCost += Entry->Cost; 2093 return LegalizationCost * LT.first; 2094 } 2095 2096 InstructionCost AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, 2097 VectorType *Tp, 2098 ArrayRef<int> Mask, int Index, 2099 VectorType *SubTp) { 2100 Kind = improveShuffleKindFromMask(Kind, Mask); 2101 if (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Transpose || 2102 Kind == TTI::SK_Select || Kind == TTI::SK_PermuteSingleSrc || 2103 Kind == TTI::SK_Reverse) { 2104 static const CostTblEntry ShuffleTbl[] = { 2105 // Broadcast shuffle kinds can be performed with 'dup'. 2106 { TTI::SK_Broadcast, MVT::v8i8, 1 }, 2107 { TTI::SK_Broadcast, MVT::v16i8, 1 }, 2108 { TTI::SK_Broadcast, MVT::v4i16, 1 }, 2109 { TTI::SK_Broadcast, MVT::v8i16, 1 }, 2110 { TTI::SK_Broadcast, MVT::v2i32, 1 }, 2111 { TTI::SK_Broadcast, MVT::v4i32, 1 }, 2112 { TTI::SK_Broadcast, MVT::v2i64, 1 }, 2113 { TTI::SK_Broadcast, MVT::v2f32, 1 }, 2114 { TTI::SK_Broadcast, MVT::v4f32, 1 }, 2115 { TTI::SK_Broadcast, MVT::v2f64, 1 }, 2116 // Transpose shuffle kinds can be performed with 'trn1/trn2' and 2117 // 'zip1/zip2' instructions. 2118 { TTI::SK_Transpose, MVT::v8i8, 1 }, 2119 { TTI::SK_Transpose, MVT::v16i8, 1 }, 2120 { TTI::SK_Transpose, MVT::v4i16, 1 }, 2121 { TTI::SK_Transpose, MVT::v8i16, 1 }, 2122 { TTI::SK_Transpose, MVT::v2i32, 1 }, 2123 { TTI::SK_Transpose, MVT::v4i32, 1 }, 2124 { TTI::SK_Transpose, MVT::v2i64, 1 }, 2125 { TTI::SK_Transpose, MVT::v2f32, 1 }, 2126 { TTI::SK_Transpose, MVT::v4f32, 1 }, 2127 { TTI::SK_Transpose, MVT::v2f64, 1 }, 2128 // Select shuffle kinds. 2129 // TODO: handle vXi8/vXi16. 2130 { TTI::SK_Select, MVT::v2i32, 1 }, // mov. 2131 { TTI::SK_Select, MVT::v4i32, 2 }, // rev+trn (or similar). 2132 { TTI::SK_Select, MVT::v2i64, 1 }, // mov. 2133 { TTI::SK_Select, MVT::v2f32, 1 }, // mov. 2134 { TTI::SK_Select, MVT::v4f32, 2 }, // rev+trn (or similar). 2135 { TTI::SK_Select, MVT::v2f64, 1 }, // mov. 2136 // PermuteSingleSrc shuffle kinds. 2137 { TTI::SK_PermuteSingleSrc, MVT::v2i32, 1 }, // mov. 2138 { TTI::SK_PermuteSingleSrc, MVT::v4i32, 3 }, // perfectshuffle worst case. 2139 { TTI::SK_PermuteSingleSrc, MVT::v2i64, 1 }, // mov. 2140 { TTI::SK_PermuteSingleSrc, MVT::v2f32, 1 }, // mov. 2141 { TTI::SK_PermuteSingleSrc, MVT::v4f32, 3 }, // perfectshuffle worst case. 2142 { TTI::SK_PermuteSingleSrc, MVT::v2f64, 1 }, // mov. 2143 { TTI::SK_PermuteSingleSrc, MVT::v4i16, 3 }, // perfectshuffle worst case. 2144 { TTI::SK_PermuteSingleSrc, MVT::v4f16, 3 }, // perfectshuffle worst case. 2145 { TTI::SK_PermuteSingleSrc, MVT::v4bf16, 3 }, // perfectshuffle worst case. 2146 { TTI::SK_PermuteSingleSrc, MVT::v8i16, 8 }, // constpool + load + tbl 2147 { TTI::SK_PermuteSingleSrc, MVT::v8f16, 8 }, // constpool + load + tbl 2148 { TTI::SK_PermuteSingleSrc, MVT::v8bf16, 8 }, // constpool + load + tbl 2149 { TTI::SK_PermuteSingleSrc, MVT::v8i8, 8 }, // constpool + load + tbl 2150 { TTI::SK_PermuteSingleSrc, MVT::v16i8, 8 }, // constpool + load + tbl 2151 // Reverse can be lowered with `rev`. 2152 { TTI::SK_Reverse, MVT::v2i32, 1 }, // mov. 2153 { TTI::SK_Reverse, MVT::v4i32, 2 }, // REV64; EXT 2154 { TTI::SK_Reverse, MVT::v2i64, 1 }, // mov. 2155 { TTI::SK_Reverse, MVT::v2f32, 1 }, // mov. 2156 { TTI::SK_Reverse, MVT::v4f32, 2 }, // REV64; EXT 2157 { TTI::SK_Reverse, MVT::v2f64, 1 }, // mov. 2158 // Broadcast shuffle kinds for scalable vectors 2159 { TTI::SK_Broadcast, MVT::nxv16i8, 1 }, 2160 { TTI::SK_Broadcast, MVT::nxv8i16, 1 }, 2161 { TTI::SK_Broadcast, MVT::nxv4i32, 1 }, 2162 { TTI::SK_Broadcast, MVT::nxv2i64, 1 }, 2163 { TTI::SK_Broadcast, MVT::nxv2f16, 1 }, 2164 { TTI::SK_Broadcast, MVT::nxv4f16, 1 }, 2165 { TTI::SK_Broadcast, MVT::nxv8f16, 1 }, 2166 { TTI::SK_Broadcast, MVT::nxv2bf16, 1 }, 2167 { TTI::SK_Broadcast, MVT::nxv4bf16, 1 }, 2168 { TTI::SK_Broadcast, MVT::nxv8bf16, 1 }, 2169 { TTI::SK_Broadcast, MVT::nxv2f32, 1 }, 2170 { TTI::SK_Broadcast, MVT::nxv4f32, 1 }, 2171 { TTI::SK_Broadcast, MVT::nxv2f64, 1 }, 2172 { TTI::SK_Broadcast, MVT::nxv16i1, 1 }, 2173 { TTI::SK_Broadcast, MVT::nxv8i1, 1 }, 2174 { TTI::SK_Broadcast, MVT::nxv4i1, 1 }, 2175 { TTI::SK_Broadcast, MVT::nxv2i1, 1 }, 2176 // Handle the cases for vector.reverse with scalable vectors 2177 { TTI::SK_Reverse, MVT::nxv16i8, 1 }, 2178 { TTI::SK_Reverse, MVT::nxv8i16, 1 }, 2179 { TTI::SK_Reverse, MVT::nxv4i32, 1 }, 2180 { TTI::SK_Reverse, MVT::nxv2i64, 1 }, 2181 { TTI::SK_Reverse, MVT::nxv2f16, 1 }, 2182 { TTI::SK_Reverse, MVT::nxv4f16, 1 }, 2183 { TTI::SK_Reverse, MVT::nxv8f16, 1 }, 2184 { TTI::SK_Reverse, MVT::nxv2bf16, 1 }, 2185 { TTI::SK_Reverse, MVT::nxv4bf16, 1 }, 2186 { TTI::SK_Reverse, MVT::nxv8bf16, 1 }, 2187 { TTI::SK_Reverse, MVT::nxv2f32, 1 }, 2188 { TTI::SK_Reverse, MVT::nxv4f32, 1 }, 2189 { TTI::SK_Reverse, MVT::nxv2f64, 1 }, 2190 { TTI::SK_Reverse, MVT::nxv16i1, 1 }, 2191 { TTI::SK_Reverse, MVT::nxv8i1, 1 }, 2192 { TTI::SK_Reverse, MVT::nxv4i1, 1 }, 2193 { TTI::SK_Reverse, MVT::nxv2i1, 1 }, 2194 }; 2195 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp); 2196 if (const auto *Entry = CostTableLookup(ShuffleTbl, Kind, LT.second)) 2197 return LT.first * Entry->Cost; 2198 } 2199 if (Kind == TTI::SK_Splice && isa<ScalableVectorType>(Tp)) 2200 return getSpliceCost(Tp, Index); 2201 return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp); 2202 } 2203