1 //===-- PPCTargetTransformInfo.cpp - PPC specific TTI ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "PPCTargetTransformInfo.h" 10 #include "llvm/Analysis/CodeMetrics.h" 11 #include "llvm/Analysis/TargetLibraryInfo.h" 12 #include "llvm/Analysis/TargetTransformInfo.h" 13 #include "llvm/CodeGen/BasicTTIImpl.h" 14 #include "llvm/CodeGen/CostTable.h" 15 #include "llvm/CodeGen/TargetLowering.h" 16 #include "llvm/CodeGen/TargetSchedule.h" 17 #include "llvm/IR/IntrinsicsPowerPC.h" 18 #include "llvm/Support/CommandLine.h" 19 #include "llvm/Support/Debug.h" 20 #include "llvm/Support/KnownBits.h" 21 #include "llvm/Transforms/InstCombine/InstCombiner.h" 22 #include "llvm/Transforms/Utils/Local.h" 23 24 using namespace llvm; 25 26 #define DEBUG_TYPE "ppctti" 27 28 static cl::opt<bool> DisablePPCConstHoist("disable-ppc-constant-hoisting", 29 cl::desc("disable constant hoisting on PPC"), cl::init(false), cl::Hidden); 30 31 // This is currently only used for the data prefetch pass 32 static cl::opt<unsigned> 33 CacheLineSize("ppc-loop-prefetch-cache-line", cl::Hidden, cl::init(64), 34 cl::desc("The loop prefetch cache line size")); 35 36 static cl::opt<bool> 37 EnablePPCColdCC("ppc-enable-coldcc", cl::Hidden, cl::init(false), 38 cl::desc("Enable using coldcc calling conv for cold " 39 "internal functions")); 40 41 static cl::opt<bool> 42 LsrNoInsnsCost("ppc-lsr-no-insns-cost", cl::Hidden, cl::init(false), 43 cl::desc("Do not add instruction count to lsr cost model")); 44 45 // The latency of mtctr is only justified if there are more than 4 46 // comparisons that will be removed as a result. 47 static cl::opt<unsigned> 48 SmallCTRLoopThreshold("min-ctr-loop-threshold", cl::init(4), cl::Hidden, 49 cl::desc("Loops with a constant trip count smaller than " 50 "this value will not use the count register.")); 51 52 //===----------------------------------------------------------------------===// 53 // 54 // PPC cost model. 55 // 56 //===----------------------------------------------------------------------===// 57 58 TargetTransformInfo::PopcntSupportKind 59 PPCTTIImpl::getPopcntSupport(unsigned TyWidth) { 60 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2"); 61 if (ST->hasPOPCNTD() != PPCSubtarget::POPCNTD_Unavailable && TyWidth <= 64) 62 return ST->hasPOPCNTD() == PPCSubtarget::POPCNTD_Slow ? 63 TTI::PSK_SlowHardware : TTI::PSK_FastHardware; 64 return TTI::PSK_Software; 65 } 66 67 Optional<Instruction *> 68 PPCTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { 69 Intrinsic::ID IID = II.getIntrinsicID(); 70 switch (IID) { 71 default: 72 break; 73 case Intrinsic::ppc_altivec_lvx: 74 case Intrinsic::ppc_altivec_lvxl: 75 // Turn PPC lvx -> load if the pointer is known aligned. 76 if (getOrEnforceKnownAlignment( 77 II.getArgOperand(0), Align(16), IC.getDataLayout(), &II, 78 &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) { 79 Value *Ptr = IC.Builder.CreateBitCast( 80 II.getArgOperand(0), PointerType::getUnqual(II.getType())); 81 return new LoadInst(II.getType(), Ptr, "", false, Align(16)); 82 } 83 break; 84 case Intrinsic::ppc_vsx_lxvw4x: 85 case Intrinsic::ppc_vsx_lxvd2x: { 86 // Turn PPC VSX loads into normal loads. 87 Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(0), 88 PointerType::getUnqual(II.getType())); 89 return new LoadInst(II.getType(), Ptr, Twine(""), false, Align(1)); 90 } 91 case Intrinsic::ppc_altivec_stvx: 92 case Intrinsic::ppc_altivec_stvxl: 93 // Turn stvx -> store if the pointer is known aligned. 94 if (getOrEnforceKnownAlignment( 95 II.getArgOperand(1), Align(16), IC.getDataLayout(), &II, 96 &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) { 97 Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType()); 98 Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy); 99 return new StoreInst(II.getArgOperand(0), Ptr, false, Align(16)); 100 } 101 break; 102 case Intrinsic::ppc_vsx_stxvw4x: 103 case Intrinsic::ppc_vsx_stxvd2x: { 104 // Turn PPC VSX stores into normal stores. 105 Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType()); 106 Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy); 107 return new StoreInst(II.getArgOperand(0), Ptr, false, Align(1)); 108 } 109 case Intrinsic::ppc_altivec_vperm: 110 // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant. 111 // Note that ppc_altivec_vperm has a big-endian bias, so when creating 112 // a vectorshuffle for little endian, we must undo the transformation 113 // performed on vec_perm in altivec.h. That is, we must complement 114 // the permutation mask with respect to 31 and reverse the order of 115 // V1 and V2. 116 if (Constant *Mask = dyn_cast<Constant>(II.getArgOperand(2))) { 117 assert(cast<FixedVectorType>(Mask->getType())->getNumElements() == 16 && 118 "Bad type for intrinsic!"); 119 120 // Check that all of the elements are integer constants or undefs. 121 bool AllEltsOk = true; 122 for (unsigned i = 0; i != 16; ++i) { 123 Constant *Elt = Mask->getAggregateElement(i); 124 if (!Elt || !(isa<ConstantInt>(Elt) || isa<UndefValue>(Elt))) { 125 AllEltsOk = false; 126 break; 127 } 128 } 129 130 if (AllEltsOk) { 131 // Cast the input vectors to byte vectors. 132 Value *Op0 = 133 IC.Builder.CreateBitCast(II.getArgOperand(0), Mask->getType()); 134 Value *Op1 = 135 IC.Builder.CreateBitCast(II.getArgOperand(1), Mask->getType()); 136 Value *Result = UndefValue::get(Op0->getType()); 137 138 // Only extract each element once. 139 Value *ExtractedElts[32]; 140 memset(ExtractedElts, 0, sizeof(ExtractedElts)); 141 142 for (unsigned i = 0; i != 16; ++i) { 143 if (isa<UndefValue>(Mask->getAggregateElement(i))) 144 continue; 145 unsigned Idx = 146 cast<ConstantInt>(Mask->getAggregateElement(i))->getZExtValue(); 147 Idx &= 31; // Match the hardware behavior. 148 if (DL.isLittleEndian()) 149 Idx = 31 - Idx; 150 151 if (!ExtractedElts[Idx]) { 152 Value *Op0ToUse = (DL.isLittleEndian()) ? Op1 : Op0; 153 Value *Op1ToUse = (DL.isLittleEndian()) ? Op0 : Op1; 154 ExtractedElts[Idx] = IC.Builder.CreateExtractElement( 155 Idx < 16 ? Op0ToUse : Op1ToUse, IC.Builder.getInt32(Idx & 15)); 156 } 157 158 // Insert this value into the result vector. 159 Result = IC.Builder.CreateInsertElement(Result, ExtractedElts[Idx], 160 IC.Builder.getInt32(i)); 161 } 162 return CastInst::Create(Instruction::BitCast, Result, II.getType()); 163 } 164 } 165 break; 166 } 167 return None; 168 } 169 170 InstructionCost PPCTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty, 171 TTI::TargetCostKind CostKind) { 172 if (DisablePPCConstHoist) 173 return BaseT::getIntImmCost(Imm, Ty, CostKind); 174 175 assert(Ty->isIntegerTy()); 176 177 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 178 if (BitSize == 0) 179 return ~0U; 180 181 if (Imm == 0) 182 return TTI::TCC_Free; 183 184 if (Imm.getBitWidth() <= 64) { 185 if (isInt<16>(Imm.getSExtValue())) 186 return TTI::TCC_Basic; 187 188 if (isInt<32>(Imm.getSExtValue())) { 189 // A constant that can be materialized using lis. 190 if ((Imm.getZExtValue() & 0xFFFF) == 0) 191 return TTI::TCC_Basic; 192 193 return 2 * TTI::TCC_Basic; 194 } 195 } 196 197 return 4 * TTI::TCC_Basic; 198 } 199 200 InstructionCost PPCTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, 201 const APInt &Imm, Type *Ty, 202 TTI::TargetCostKind CostKind) { 203 if (DisablePPCConstHoist) 204 return BaseT::getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind); 205 206 assert(Ty->isIntegerTy()); 207 208 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 209 if (BitSize == 0) 210 return ~0U; 211 212 switch (IID) { 213 default: 214 return TTI::TCC_Free; 215 case Intrinsic::sadd_with_overflow: 216 case Intrinsic::uadd_with_overflow: 217 case Intrinsic::ssub_with_overflow: 218 case Intrinsic::usub_with_overflow: 219 if ((Idx == 1) && Imm.getBitWidth() <= 64 && isInt<16>(Imm.getSExtValue())) 220 return TTI::TCC_Free; 221 break; 222 case Intrinsic::experimental_stackmap: 223 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) 224 return TTI::TCC_Free; 225 break; 226 case Intrinsic::experimental_patchpoint_void: 227 case Intrinsic::experimental_patchpoint_i64: 228 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) 229 return TTI::TCC_Free; 230 break; 231 } 232 return PPCTTIImpl::getIntImmCost(Imm, Ty, CostKind); 233 } 234 235 InstructionCost PPCTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx, 236 const APInt &Imm, Type *Ty, 237 TTI::TargetCostKind CostKind, 238 Instruction *Inst) { 239 if (DisablePPCConstHoist) 240 return BaseT::getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind, Inst); 241 242 assert(Ty->isIntegerTy()); 243 244 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 245 if (BitSize == 0) 246 return ~0U; 247 248 unsigned ImmIdx = ~0U; 249 bool ShiftedFree = false, RunFree = false, UnsignedFree = false, 250 ZeroFree = false; 251 switch (Opcode) { 252 default: 253 return TTI::TCC_Free; 254 case Instruction::GetElementPtr: 255 // Always hoist the base address of a GetElementPtr. This prevents the 256 // creation of new constants for every base constant that gets constant 257 // folded with the offset. 258 if (Idx == 0) 259 return 2 * TTI::TCC_Basic; 260 return TTI::TCC_Free; 261 case Instruction::And: 262 RunFree = true; // (for the rotate-and-mask instructions) 263 LLVM_FALLTHROUGH; 264 case Instruction::Add: 265 case Instruction::Or: 266 case Instruction::Xor: 267 ShiftedFree = true; 268 LLVM_FALLTHROUGH; 269 case Instruction::Sub: 270 case Instruction::Mul: 271 case Instruction::Shl: 272 case Instruction::LShr: 273 case Instruction::AShr: 274 ImmIdx = 1; 275 break; 276 case Instruction::ICmp: 277 UnsignedFree = true; 278 ImmIdx = 1; 279 // Zero comparisons can use record-form instructions. 280 LLVM_FALLTHROUGH; 281 case Instruction::Select: 282 ZeroFree = true; 283 break; 284 case Instruction::PHI: 285 case Instruction::Call: 286 case Instruction::Ret: 287 case Instruction::Load: 288 case Instruction::Store: 289 break; 290 } 291 292 if (ZeroFree && Imm == 0) 293 return TTI::TCC_Free; 294 295 if (Idx == ImmIdx && Imm.getBitWidth() <= 64) { 296 if (isInt<16>(Imm.getSExtValue())) 297 return TTI::TCC_Free; 298 299 if (RunFree) { 300 if (Imm.getBitWidth() <= 32 && 301 (isShiftedMask_32(Imm.getZExtValue()) || 302 isShiftedMask_32(~Imm.getZExtValue()))) 303 return TTI::TCC_Free; 304 305 if (ST->isPPC64() && 306 (isShiftedMask_64(Imm.getZExtValue()) || 307 isShiftedMask_64(~Imm.getZExtValue()))) 308 return TTI::TCC_Free; 309 } 310 311 if (UnsignedFree && isUInt<16>(Imm.getZExtValue())) 312 return TTI::TCC_Free; 313 314 if (ShiftedFree && (Imm.getZExtValue() & 0xFFFF) == 0) 315 return TTI::TCC_Free; 316 } 317 318 return PPCTTIImpl::getIntImmCost(Imm, Ty, CostKind); 319 } 320 321 // Check if the current Type is an MMA vector type. Valid MMA types are 322 // v256i1 and v512i1 respectively. 323 static bool isMMAType(Type *Ty) { 324 return Ty->isVectorTy() && (Ty->getScalarSizeInBits() == 1) && 325 (Ty->getPrimitiveSizeInBits() > 128); 326 } 327 328 InstructionCost PPCTTIImpl::getUserCost(const User *U, 329 ArrayRef<const Value *> Operands, 330 TTI::TargetCostKind CostKind) { 331 // We already implement getCastInstrCost and getMemoryOpCost where we perform 332 // the vector adjustment there. 333 if (isa<CastInst>(U) || isa<LoadInst>(U) || isa<StoreInst>(U)) 334 return BaseT::getUserCost(U, Operands, CostKind); 335 336 if (U->getType()->isVectorTy()) { 337 // Instructions that need to be split should cost more. 338 std::pair<InstructionCost, MVT> LT = 339 TLI->getTypeLegalizationCost(DL, U->getType()); 340 return LT.first * BaseT::getUserCost(U, Operands, CostKind); 341 } 342 343 return BaseT::getUserCost(U, Operands, CostKind); 344 } 345 346 // Determining the address of a TLS variable results in a function call in 347 // certain TLS models. 348 static bool memAddrUsesCTR(const Value *MemAddr, const PPCTargetMachine &TM, 349 SmallPtrSetImpl<const Value *> &Visited) { 350 // No need to traverse again if we already checked this operand. 351 if (!Visited.insert(MemAddr).second) 352 return false; 353 const auto *GV = dyn_cast<GlobalValue>(MemAddr); 354 if (!GV) { 355 // Recurse to check for constants that refer to TLS global variables. 356 if (const auto *CV = dyn_cast<Constant>(MemAddr)) 357 for (const auto &CO : CV->operands()) 358 if (memAddrUsesCTR(CO, TM, Visited)) 359 return true; 360 return false; 361 } 362 363 if (!GV->isThreadLocal()) 364 return false; 365 TLSModel::Model Model = TM.getTLSModel(GV); 366 return Model == TLSModel::GeneralDynamic || Model == TLSModel::LocalDynamic; 367 } 368 369 bool PPCTTIImpl::mightUseCTR(BasicBlock *BB, TargetLibraryInfo *LibInfo, 370 SmallPtrSetImpl<const Value *> &Visited) { 371 const PPCTargetMachine &TM = ST->getTargetMachine(); 372 373 // Loop through the inline asm constraints and look for something that 374 // clobbers ctr. 375 auto asmClobbersCTR = [](InlineAsm *IA) { 376 InlineAsm::ConstraintInfoVector CIV = IA->ParseConstraints(); 377 for (const InlineAsm::ConstraintInfo &C : CIV) { 378 if (C.Type != InlineAsm::isInput) 379 for (const auto &Code : C.Codes) 380 if (StringRef(Code).equals_insensitive("{ctr}")) 381 return true; 382 } 383 return false; 384 }; 385 386 auto isLargeIntegerTy = [](bool Is32Bit, Type *Ty) { 387 if (IntegerType *ITy = dyn_cast<IntegerType>(Ty)) 388 return ITy->getBitWidth() > (Is32Bit ? 32U : 64U); 389 390 return false; 391 }; 392 393 auto supportedHalfPrecisionOp = [](Instruction *Inst) { 394 switch (Inst->getOpcode()) { 395 default: 396 return false; 397 case Instruction::FPTrunc: 398 case Instruction::FPExt: 399 case Instruction::Load: 400 case Instruction::Store: 401 case Instruction::FPToUI: 402 case Instruction::UIToFP: 403 case Instruction::FPToSI: 404 case Instruction::SIToFP: 405 return true; 406 } 407 }; 408 409 for (BasicBlock::iterator J = BB->begin(), JE = BB->end(); 410 J != JE; ++J) { 411 // There are no direct operations on half precision so assume that 412 // anything with that type requires a call except for a few select 413 // operations with Power9. 414 if (Instruction *CurrInst = dyn_cast<Instruction>(J)) { 415 for (const auto &Op : CurrInst->operands()) { 416 if (Op->getType()->getScalarType()->isHalfTy() || 417 CurrInst->getType()->getScalarType()->isHalfTy()) 418 return !(ST->isISA3_0() && supportedHalfPrecisionOp(CurrInst)); 419 } 420 } 421 if (CallInst *CI = dyn_cast<CallInst>(J)) { 422 // Inline ASM is okay, unless it clobbers the ctr register. 423 if (InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledOperand())) { 424 if (asmClobbersCTR(IA)) 425 return true; 426 continue; 427 } 428 429 if (Function *F = CI->getCalledFunction()) { 430 // Most intrinsics don't become function calls, but some might. 431 // sin, cos, exp and log are always calls. 432 unsigned Opcode = 0; 433 if (F->getIntrinsicID() != Intrinsic::not_intrinsic) { 434 switch (F->getIntrinsicID()) { 435 default: continue; 436 // If we have a call to loop_decrement or set_loop_iterations, 437 // we're definitely using CTR. 438 case Intrinsic::set_loop_iterations: 439 case Intrinsic::loop_decrement: 440 return true; 441 442 // Binary operations on 128-bit value will use CTR. 443 case Intrinsic::experimental_constrained_fadd: 444 case Intrinsic::experimental_constrained_fsub: 445 case Intrinsic::experimental_constrained_fmul: 446 case Intrinsic::experimental_constrained_fdiv: 447 case Intrinsic::experimental_constrained_frem: 448 if (F->getType()->getScalarType()->isFP128Ty() || 449 F->getType()->getScalarType()->isPPC_FP128Ty()) 450 return true; 451 break; 452 453 case Intrinsic::experimental_constrained_fptosi: 454 case Intrinsic::experimental_constrained_fptoui: 455 case Intrinsic::experimental_constrained_sitofp: 456 case Intrinsic::experimental_constrained_uitofp: { 457 Type *SrcType = CI->getArgOperand(0)->getType()->getScalarType(); 458 Type *DstType = CI->getType()->getScalarType(); 459 if (SrcType->isPPC_FP128Ty() || DstType->isPPC_FP128Ty() || 460 isLargeIntegerTy(!TM.isPPC64(), SrcType) || 461 isLargeIntegerTy(!TM.isPPC64(), DstType)) 462 return true; 463 break; 464 } 465 466 // Exclude eh_sjlj_setjmp; we don't need to exclude eh_sjlj_longjmp 467 // because, although it does clobber the counter register, the 468 // control can't then return to inside the loop unless there is also 469 // an eh_sjlj_setjmp. 470 case Intrinsic::eh_sjlj_setjmp: 471 472 case Intrinsic::memcpy: 473 case Intrinsic::memmove: 474 case Intrinsic::memset: 475 case Intrinsic::powi: 476 case Intrinsic::log: 477 case Intrinsic::log2: 478 case Intrinsic::log10: 479 case Intrinsic::exp: 480 case Intrinsic::exp2: 481 case Intrinsic::pow: 482 case Intrinsic::sin: 483 case Intrinsic::cos: 484 case Intrinsic::experimental_constrained_powi: 485 case Intrinsic::experimental_constrained_log: 486 case Intrinsic::experimental_constrained_log2: 487 case Intrinsic::experimental_constrained_log10: 488 case Intrinsic::experimental_constrained_exp: 489 case Intrinsic::experimental_constrained_exp2: 490 case Intrinsic::experimental_constrained_pow: 491 case Intrinsic::experimental_constrained_sin: 492 case Intrinsic::experimental_constrained_cos: 493 return true; 494 // There is no corresponding FMA instruction for PPC double double. 495 // Thus, we need to disable CTR loop generation for this type. 496 case Intrinsic::fmuladd: 497 case Intrinsic::copysign: 498 if (CI->getArgOperand(0)->getType()->getScalarType()-> 499 isPPC_FP128Ty()) 500 return true; 501 else 502 continue; // ISD::FCOPYSIGN is never a library call. 503 case Intrinsic::fma: Opcode = ISD::FMA; break; 504 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break; 505 case Intrinsic::floor: Opcode = ISD::FFLOOR; break; 506 case Intrinsic::ceil: Opcode = ISD::FCEIL; break; 507 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break; 508 case Intrinsic::rint: Opcode = ISD::FRINT; break; 509 case Intrinsic::lrint: Opcode = ISD::LRINT; break; 510 case Intrinsic::llrint: Opcode = ISD::LLRINT; break; 511 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break; 512 case Intrinsic::round: Opcode = ISD::FROUND; break; 513 case Intrinsic::lround: Opcode = ISD::LROUND; break; 514 case Intrinsic::llround: Opcode = ISD::LLROUND; break; 515 case Intrinsic::minnum: Opcode = ISD::FMINNUM; break; 516 case Intrinsic::maxnum: Opcode = ISD::FMAXNUM; break; 517 case Intrinsic::experimental_constrained_fcmp: 518 Opcode = ISD::STRICT_FSETCC; 519 break; 520 case Intrinsic::experimental_constrained_fcmps: 521 Opcode = ISD::STRICT_FSETCCS; 522 break; 523 case Intrinsic::experimental_constrained_fma: 524 Opcode = ISD::STRICT_FMA; 525 break; 526 case Intrinsic::experimental_constrained_sqrt: 527 Opcode = ISD::STRICT_FSQRT; 528 break; 529 case Intrinsic::experimental_constrained_floor: 530 Opcode = ISD::STRICT_FFLOOR; 531 break; 532 case Intrinsic::experimental_constrained_ceil: 533 Opcode = ISD::STRICT_FCEIL; 534 break; 535 case Intrinsic::experimental_constrained_trunc: 536 Opcode = ISD::STRICT_FTRUNC; 537 break; 538 case Intrinsic::experimental_constrained_rint: 539 Opcode = ISD::STRICT_FRINT; 540 break; 541 case Intrinsic::experimental_constrained_lrint: 542 Opcode = ISD::STRICT_LRINT; 543 break; 544 case Intrinsic::experimental_constrained_llrint: 545 Opcode = ISD::STRICT_LLRINT; 546 break; 547 case Intrinsic::experimental_constrained_nearbyint: 548 Opcode = ISD::STRICT_FNEARBYINT; 549 break; 550 case Intrinsic::experimental_constrained_round: 551 Opcode = ISD::STRICT_FROUND; 552 break; 553 case Intrinsic::experimental_constrained_lround: 554 Opcode = ISD::STRICT_LROUND; 555 break; 556 case Intrinsic::experimental_constrained_llround: 557 Opcode = ISD::STRICT_LLROUND; 558 break; 559 case Intrinsic::experimental_constrained_minnum: 560 Opcode = ISD::STRICT_FMINNUM; 561 break; 562 case Intrinsic::experimental_constrained_maxnum: 563 Opcode = ISD::STRICT_FMAXNUM; 564 break; 565 case Intrinsic::umul_with_overflow: Opcode = ISD::UMULO; break; 566 case Intrinsic::smul_with_overflow: Opcode = ISD::SMULO; break; 567 } 568 } 569 570 // PowerPC does not use [US]DIVREM or other library calls for 571 // operations on regular types which are not otherwise library calls 572 // (i.e. soft float or atomics). If adapting for targets that do, 573 // additional care is required here. 574 575 LibFunc Func; 576 if (!F->hasLocalLinkage() && F->hasName() && LibInfo && 577 LibInfo->getLibFunc(F->getName(), Func) && 578 LibInfo->hasOptimizedCodeGen(Func)) { 579 // Non-read-only functions are never treated as intrinsics. 580 if (!CI->onlyReadsMemory()) 581 return true; 582 583 // Conversion happens only for FP calls. 584 if (!CI->getArgOperand(0)->getType()->isFloatingPointTy()) 585 return true; 586 587 switch (Func) { 588 default: return true; 589 case LibFunc_copysign: 590 case LibFunc_copysignf: 591 continue; // ISD::FCOPYSIGN is never a library call. 592 case LibFunc_copysignl: 593 return true; 594 case LibFunc_fabs: 595 case LibFunc_fabsf: 596 case LibFunc_fabsl: 597 continue; // ISD::FABS is never a library call. 598 case LibFunc_sqrt: 599 case LibFunc_sqrtf: 600 case LibFunc_sqrtl: 601 Opcode = ISD::FSQRT; break; 602 case LibFunc_floor: 603 case LibFunc_floorf: 604 case LibFunc_floorl: 605 Opcode = ISD::FFLOOR; break; 606 case LibFunc_nearbyint: 607 case LibFunc_nearbyintf: 608 case LibFunc_nearbyintl: 609 Opcode = ISD::FNEARBYINT; break; 610 case LibFunc_ceil: 611 case LibFunc_ceilf: 612 case LibFunc_ceill: 613 Opcode = ISD::FCEIL; break; 614 case LibFunc_rint: 615 case LibFunc_rintf: 616 case LibFunc_rintl: 617 Opcode = ISD::FRINT; break; 618 case LibFunc_round: 619 case LibFunc_roundf: 620 case LibFunc_roundl: 621 Opcode = ISD::FROUND; break; 622 case LibFunc_trunc: 623 case LibFunc_truncf: 624 case LibFunc_truncl: 625 Opcode = ISD::FTRUNC; break; 626 case LibFunc_fmin: 627 case LibFunc_fminf: 628 case LibFunc_fminl: 629 Opcode = ISD::FMINNUM; break; 630 case LibFunc_fmax: 631 case LibFunc_fmaxf: 632 case LibFunc_fmaxl: 633 Opcode = ISD::FMAXNUM; break; 634 } 635 } 636 637 if (Opcode) { 638 EVT EVTy = 639 TLI->getValueType(DL, CI->getArgOperand(0)->getType(), true); 640 641 if (EVTy == MVT::Other) 642 return true; 643 644 if (TLI->isOperationLegalOrCustom(Opcode, EVTy)) 645 continue; 646 else if (EVTy.isVector() && 647 TLI->isOperationLegalOrCustom(Opcode, EVTy.getScalarType())) 648 continue; 649 650 return true; 651 } 652 } 653 654 return true; 655 } else if ((J->getType()->getScalarType()->isFP128Ty() || 656 J->getType()->getScalarType()->isPPC_FP128Ty())) { 657 // Most operations on f128 or ppc_f128 values become calls. 658 return true; 659 } else if (isa<FCmpInst>(J) && 660 J->getOperand(0)->getType()->getScalarType()->isFP128Ty()) { 661 return true; 662 } else if ((isa<FPTruncInst>(J) || isa<FPExtInst>(J)) && 663 (cast<CastInst>(J)->getSrcTy()->getScalarType()->isFP128Ty() || 664 cast<CastInst>(J)->getDestTy()->getScalarType()->isFP128Ty())) { 665 return true; 666 } else if (isa<UIToFPInst>(J) || isa<SIToFPInst>(J) || 667 isa<FPToUIInst>(J) || isa<FPToSIInst>(J)) { 668 CastInst *CI = cast<CastInst>(J); 669 if (CI->getSrcTy()->getScalarType()->isPPC_FP128Ty() || 670 CI->getDestTy()->getScalarType()->isPPC_FP128Ty() || 671 isLargeIntegerTy(!TM.isPPC64(), CI->getSrcTy()->getScalarType()) || 672 isLargeIntegerTy(!TM.isPPC64(), CI->getDestTy()->getScalarType())) 673 return true; 674 } else if (isLargeIntegerTy(!TM.isPPC64(), 675 J->getType()->getScalarType()) && 676 (J->getOpcode() == Instruction::UDiv || 677 J->getOpcode() == Instruction::SDiv || 678 J->getOpcode() == Instruction::URem || 679 J->getOpcode() == Instruction::SRem)) { 680 return true; 681 } else if (!TM.isPPC64() && 682 isLargeIntegerTy(false, J->getType()->getScalarType()) && 683 (J->getOpcode() == Instruction::Shl || 684 J->getOpcode() == Instruction::AShr || 685 J->getOpcode() == Instruction::LShr)) { 686 // Only on PPC32, for 128-bit integers (specifically not 64-bit 687 // integers), these might be runtime calls. 688 return true; 689 } else if (isa<IndirectBrInst>(J) || isa<InvokeInst>(J)) { 690 // On PowerPC, indirect jumps use the counter register. 691 return true; 692 } else if (SwitchInst *SI = dyn_cast<SwitchInst>(J)) { 693 if (SI->getNumCases() + 1 >= (unsigned)TLI->getMinimumJumpTableEntries()) 694 return true; 695 } 696 697 // FREM is always a call. 698 if (J->getOpcode() == Instruction::FRem) 699 return true; 700 701 if (ST->useSoftFloat()) { 702 switch(J->getOpcode()) { 703 case Instruction::FAdd: 704 case Instruction::FSub: 705 case Instruction::FMul: 706 case Instruction::FDiv: 707 case Instruction::FPTrunc: 708 case Instruction::FPExt: 709 case Instruction::FPToUI: 710 case Instruction::FPToSI: 711 case Instruction::UIToFP: 712 case Instruction::SIToFP: 713 case Instruction::FCmp: 714 return true; 715 } 716 } 717 718 for (Value *Operand : J->operands()) 719 if (memAddrUsesCTR(Operand, TM, Visited)) 720 return true; 721 } 722 723 return false; 724 } 725 726 bool PPCTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE, 727 AssumptionCache &AC, 728 TargetLibraryInfo *LibInfo, 729 HardwareLoopInfo &HWLoopInfo) { 730 const PPCTargetMachine &TM = ST->getTargetMachine(); 731 TargetSchedModel SchedModel; 732 SchedModel.init(ST); 733 734 // Do not convert small short loops to CTR loop. 735 unsigned ConstTripCount = SE.getSmallConstantTripCount(L); 736 if (ConstTripCount && ConstTripCount < SmallCTRLoopThreshold) { 737 SmallPtrSet<const Value *, 32> EphValues; 738 CodeMetrics::collectEphemeralValues(L, &AC, EphValues); 739 CodeMetrics Metrics; 740 for (BasicBlock *BB : L->blocks()) 741 Metrics.analyzeBasicBlock(BB, *this, EphValues); 742 // 6 is an approximate latency for the mtctr instruction. 743 if (Metrics.NumInsts <= (6 * SchedModel.getIssueWidth())) 744 return false; 745 } 746 747 // We don't want to spill/restore the counter register, and so we don't 748 // want to use the counter register if the loop contains calls. 749 SmallPtrSet<const Value *, 4> Visited; 750 for (Loop::block_iterator I = L->block_begin(), IE = L->block_end(); 751 I != IE; ++I) 752 if (mightUseCTR(*I, LibInfo, Visited)) 753 return false; 754 755 SmallVector<BasicBlock*, 4> ExitingBlocks; 756 L->getExitingBlocks(ExitingBlocks); 757 758 // If there is an exit edge known to be frequently taken, 759 // we should not transform this loop. 760 for (auto &BB : ExitingBlocks) { 761 Instruction *TI = BB->getTerminator(); 762 if (!TI) continue; 763 764 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { 765 uint64_t TrueWeight = 0, FalseWeight = 0; 766 if (!BI->isConditional() || 767 !BI->extractProfMetadata(TrueWeight, FalseWeight)) 768 continue; 769 770 // If the exit path is more frequent than the loop path, 771 // we return here without further analysis for this loop. 772 bool TrueIsExit = !L->contains(BI->getSuccessor(0)); 773 if (( TrueIsExit && FalseWeight < TrueWeight) || 774 (!TrueIsExit && FalseWeight > TrueWeight)) 775 return false; 776 } 777 } 778 779 // If an exit block has a PHI that accesses a TLS variable as one of the 780 // incoming values from the loop, we cannot produce a CTR loop because the 781 // address for that value will be computed in the loop. 782 SmallVector<BasicBlock *, 4> ExitBlocks; 783 L->getExitBlocks(ExitBlocks); 784 for (auto &BB : ExitBlocks) { 785 for (auto &PHI : BB->phis()) { 786 for (int Idx = 0, EndIdx = PHI.getNumIncomingValues(); Idx < EndIdx; 787 Idx++) { 788 const BasicBlock *IncomingBB = PHI.getIncomingBlock(Idx); 789 const Value *IncomingValue = PHI.getIncomingValue(Idx); 790 if (L->contains(IncomingBB) && 791 memAddrUsesCTR(IncomingValue, TM, Visited)) 792 return false; 793 } 794 } 795 } 796 797 LLVMContext &C = L->getHeader()->getContext(); 798 HWLoopInfo.CountType = TM.isPPC64() ? 799 Type::getInt64Ty(C) : Type::getInt32Ty(C); 800 HWLoopInfo.LoopDecrement = ConstantInt::get(HWLoopInfo.CountType, 1); 801 return true; 802 } 803 804 void PPCTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, 805 TTI::UnrollingPreferences &UP, 806 OptimizationRemarkEmitter *ORE) { 807 if (ST->getCPUDirective() == PPC::DIR_A2) { 808 // The A2 is in-order with a deep pipeline, and concatenation unrolling 809 // helps expose latency-hiding opportunities to the instruction scheduler. 810 UP.Partial = UP.Runtime = true; 811 812 // We unroll a lot on the A2 (hundreds of instructions), and the benefits 813 // often outweigh the cost of a division to compute the trip count. 814 UP.AllowExpensiveTripCount = true; 815 } 816 817 BaseT::getUnrollingPreferences(L, SE, UP, ORE); 818 } 819 820 void PPCTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, 821 TTI::PeelingPreferences &PP) { 822 BaseT::getPeelingPreferences(L, SE, PP); 823 } 824 // This function returns true to allow using coldcc calling convention. 825 // Returning true results in coldcc being used for functions which are cold at 826 // all call sites when the callers of the functions are not calling any other 827 // non coldcc functions. 828 bool PPCTTIImpl::useColdCCForColdCall(Function &F) { 829 return EnablePPCColdCC; 830 } 831 832 bool PPCTTIImpl::enableAggressiveInterleaving(bool LoopHasReductions) { 833 // On the A2, always unroll aggressively. 834 if (ST->getCPUDirective() == PPC::DIR_A2) 835 return true; 836 837 return LoopHasReductions; 838 } 839 840 PPCTTIImpl::TTI::MemCmpExpansionOptions 841 PPCTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const { 842 TTI::MemCmpExpansionOptions Options; 843 Options.LoadSizes = {8, 4, 2, 1}; 844 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize); 845 return Options; 846 } 847 848 bool PPCTTIImpl::enableInterleavedAccessVectorization() { 849 return true; 850 } 851 852 unsigned PPCTTIImpl::getNumberOfRegisters(unsigned ClassID) const { 853 assert(ClassID == GPRRC || ClassID == FPRRC || 854 ClassID == VRRC || ClassID == VSXRC); 855 if (ST->hasVSX()) { 856 assert(ClassID == GPRRC || ClassID == VSXRC || ClassID == VRRC); 857 return ClassID == VSXRC ? 64 : 32; 858 } 859 assert(ClassID == GPRRC || ClassID == FPRRC || ClassID == VRRC); 860 return 32; 861 } 862 863 unsigned PPCTTIImpl::getRegisterClassForType(bool Vector, Type *Ty) const { 864 if (Vector) 865 return ST->hasVSX() ? VSXRC : VRRC; 866 else if (Ty && (Ty->getScalarType()->isFloatTy() || 867 Ty->getScalarType()->isDoubleTy())) 868 return ST->hasVSX() ? VSXRC : FPRRC; 869 else if (Ty && (Ty->getScalarType()->isFP128Ty() || 870 Ty->getScalarType()->isPPC_FP128Ty())) 871 return VRRC; 872 else if (Ty && Ty->getScalarType()->isHalfTy()) 873 return VSXRC; 874 else 875 return GPRRC; 876 } 877 878 const char* PPCTTIImpl::getRegisterClassName(unsigned ClassID) const { 879 880 switch (ClassID) { 881 default: 882 llvm_unreachable("unknown register class"); 883 return "PPC::unknown register class"; 884 case GPRRC: return "PPC::GPRRC"; 885 case FPRRC: return "PPC::FPRRC"; 886 case VRRC: return "PPC::VRRC"; 887 case VSXRC: return "PPC::VSXRC"; 888 } 889 } 890 891 TypeSize 892 PPCTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const { 893 switch (K) { 894 case TargetTransformInfo::RGK_Scalar: 895 return TypeSize::getFixed(ST->isPPC64() ? 64 : 32); 896 case TargetTransformInfo::RGK_FixedWidthVector: 897 return TypeSize::getFixed(ST->hasAltivec() ? 128 : 0); 898 case TargetTransformInfo::RGK_ScalableVector: 899 return TypeSize::getScalable(0); 900 } 901 902 llvm_unreachable("Unsupported register kind"); 903 } 904 905 unsigned PPCTTIImpl::getCacheLineSize() const { 906 // Check first if the user specified a custom line size. 907 if (CacheLineSize.getNumOccurrences() > 0) 908 return CacheLineSize; 909 910 // Starting with P7 we have a cache line size of 128. 911 unsigned Directive = ST->getCPUDirective(); 912 // Assume that Future CPU has the same cache line size as the others. 913 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 || 914 Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR10 || 915 Directive == PPC::DIR_PWR_FUTURE) 916 return 128; 917 918 // On other processors return a default of 64 bytes. 919 return 64; 920 } 921 922 unsigned PPCTTIImpl::getPrefetchDistance() const { 923 return 300; 924 } 925 926 unsigned PPCTTIImpl::getMaxInterleaveFactor(unsigned VF) { 927 unsigned Directive = ST->getCPUDirective(); 928 // The 440 has no SIMD support, but floating-point instructions 929 // have a 5-cycle latency, so unroll by 5x for latency hiding. 930 if (Directive == PPC::DIR_440) 931 return 5; 932 933 // The A2 has no SIMD support, but floating-point instructions 934 // have a 6-cycle latency, so unroll by 6x for latency hiding. 935 if (Directive == PPC::DIR_A2) 936 return 6; 937 938 // FIXME: For lack of any better information, do no harm... 939 if (Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) 940 return 1; 941 942 // For P7 and P8, floating-point instructions have a 6-cycle latency and 943 // there are two execution units, so unroll by 12x for latency hiding. 944 // FIXME: the same for P9 as previous gen until POWER9 scheduling is ready 945 // FIXME: the same for P10 as previous gen until POWER10 scheduling is ready 946 // Assume that future is the same as the others. 947 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 || 948 Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR10 || 949 Directive == PPC::DIR_PWR_FUTURE) 950 return 12; 951 952 // For most things, modern systems have two execution units (and 953 // out-of-order execution). 954 return 2; 955 } 956 957 // Returns a cost adjustment factor to adjust the cost of vector instructions 958 // on targets which there is overlap between the vector and scalar units, 959 // thereby reducing the overall throughput of vector code wrt. scalar code. 960 // An invalid instruction cost is returned if the type is an MMA vector type. 961 InstructionCost PPCTTIImpl::vectorCostAdjustmentFactor(unsigned Opcode, 962 Type *Ty1, Type *Ty2) { 963 // If the vector type is of an MMA type (v256i1, v512i1), an invalid 964 // instruction cost is returned. This is to signify to other cost computing 965 // functions to return the maximum instruction cost in order to prevent any 966 // opportunities for the optimizer to produce MMA types within the IR. 967 if (isMMAType(Ty1)) 968 return InstructionCost::getInvalid(); 969 970 if (!ST->vectorsUseTwoUnits() || !Ty1->isVectorTy()) 971 return InstructionCost(1); 972 973 std::pair<InstructionCost, MVT> LT1 = TLI->getTypeLegalizationCost(DL, Ty1); 974 // If type legalization involves splitting the vector, we don't want to 975 // double the cost at every step - only the last step. 976 if (LT1.first != 1 || !LT1.second.isVector()) 977 return InstructionCost(1); 978 979 int ISD = TLI->InstructionOpcodeToISD(Opcode); 980 if (TLI->isOperationExpand(ISD, LT1.second)) 981 return InstructionCost(1); 982 983 if (Ty2) { 984 std::pair<InstructionCost, MVT> LT2 = TLI->getTypeLegalizationCost(DL, Ty2); 985 if (LT2.first != 1 || !LT2.second.isVector()) 986 return InstructionCost(1); 987 } 988 989 return InstructionCost(2); 990 } 991 992 InstructionCost PPCTTIImpl::getArithmeticInstrCost( 993 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, 994 TTI::OperandValueKind Op1Info, TTI::OperandValueKind Op2Info, 995 TTI::OperandValueProperties Opd1PropInfo, 996 TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args, 997 const Instruction *CxtI) { 998 assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode"); 999 1000 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Ty, nullptr); 1001 if (!CostFactor.isValid()) 1002 return InstructionCost::getMax(); 1003 1004 // TODO: Handle more cost kinds. 1005 if (CostKind != TTI::TCK_RecipThroughput) 1006 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, 1007 Op2Info, Opd1PropInfo, 1008 Opd2PropInfo, Args, CxtI); 1009 1010 // Fallback to the default implementation. 1011 InstructionCost Cost = BaseT::getArithmeticInstrCost( 1012 Opcode, Ty, CostKind, Op1Info, Op2Info, Opd1PropInfo, Opd2PropInfo); 1013 return Cost * CostFactor; 1014 } 1015 1016 InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, 1017 ArrayRef<int> Mask, int Index, 1018 Type *SubTp) { 1019 1020 InstructionCost CostFactor = 1021 vectorCostAdjustmentFactor(Instruction::ShuffleVector, Tp, nullptr); 1022 if (!CostFactor.isValid()) 1023 return InstructionCost::getMax(); 1024 1025 // Legalize the type. 1026 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp); 1027 1028 // PPC, for both Altivec/VSX, support cheap arbitrary permutations 1029 // (at least in the sense that there need only be one non-loop-invariant 1030 // instruction). We need one such shuffle instruction for each actual 1031 // register (this is not true for arbitrary shuffles, but is true for the 1032 // structured types of shuffles covered by TTI::ShuffleKind). 1033 return LT.first * CostFactor; 1034 } 1035 1036 InstructionCost PPCTTIImpl::getCFInstrCost(unsigned Opcode, 1037 TTI::TargetCostKind CostKind, 1038 const Instruction *I) { 1039 if (CostKind != TTI::TCK_RecipThroughput) 1040 return Opcode == Instruction::PHI ? 0 : 1; 1041 // Branches are assumed to be predicted. 1042 return 0; 1043 } 1044 1045 InstructionCost PPCTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, 1046 Type *Src, 1047 TTI::CastContextHint CCH, 1048 TTI::TargetCostKind CostKind, 1049 const Instruction *I) { 1050 assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode"); 1051 1052 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Dst, Src); 1053 if (!CostFactor.isValid()) 1054 return InstructionCost::getMax(); 1055 1056 InstructionCost Cost = 1057 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); 1058 Cost *= CostFactor; 1059 // TODO: Allow non-throughput costs that aren't binary. 1060 if (CostKind != TTI::TCK_RecipThroughput) 1061 return Cost == 0 ? 0 : 1; 1062 return Cost; 1063 } 1064 1065 InstructionCost PPCTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, 1066 Type *CondTy, 1067 CmpInst::Predicate VecPred, 1068 TTI::TargetCostKind CostKind, 1069 const Instruction *I) { 1070 InstructionCost CostFactor = 1071 vectorCostAdjustmentFactor(Opcode, ValTy, nullptr); 1072 if (!CostFactor.isValid()) 1073 return InstructionCost::getMax(); 1074 1075 InstructionCost Cost = 1076 BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); 1077 // TODO: Handle other cost kinds. 1078 if (CostKind != TTI::TCK_RecipThroughput) 1079 return Cost; 1080 return Cost * CostFactor; 1081 } 1082 1083 InstructionCost PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, 1084 unsigned Index) { 1085 assert(Val->isVectorTy() && "This must be a vector type"); 1086 1087 int ISD = TLI->InstructionOpcodeToISD(Opcode); 1088 assert(ISD && "Invalid opcode"); 1089 1090 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Val, nullptr); 1091 if (!CostFactor.isValid()) 1092 return InstructionCost::getMax(); 1093 1094 InstructionCost Cost = BaseT::getVectorInstrCost(Opcode, Val, Index); 1095 Cost *= CostFactor; 1096 1097 if (ST->hasVSX() && Val->getScalarType()->isDoubleTy()) { 1098 // Double-precision scalars are already located in index #0 (or #1 if LE). 1099 if (ISD == ISD::EXTRACT_VECTOR_ELT && 1100 Index == (ST->isLittleEndian() ? 1 : 0)) 1101 return 0; 1102 1103 return Cost; 1104 1105 } else if (Val->getScalarType()->isIntegerTy() && Index != -1U) { 1106 if (ST->hasP9Altivec()) { 1107 if (ISD == ISD::INSERT_VECTOR_ELT) 1108 // A move-to VSR and a permute/insert. Assume vector operation cost 1109 // for both (cost will be 2x on P9). 1110 return 2 * CostFactor; 1111 1112 // It's an extract. Maybe we can do a cheap move-from VSR. 1113 unsigned EltSize = Val->getScalarSizeInBits(); 1114 if (EltSize == 64) { 1115 unsigned MfvsrdIndex = ST->isLittleEndian() ? 1 : 0; 1116 if (Index == MfvsrdIndex) 1117 return 1; 1118 } else if (EltSize == 32) { 1119 unsigned MfvsrwzIndex = ST->isLittleEndian() ? 2 : 1; 1120 if (Index == MfvsrwzIndex) 1121 return 1; 1122 } 1123 1124 // We need a vector extract (or mfvsrld). Assume vector operation cost. 1125 // The cost of the load constant for a vector extract is disregarded 1126 // (invariant, easily schedulable). 1127 return CostFactor; 1128 1129 } else if (ST->hasDirectMove()) 1130 // Assume permute has standard cost. 1131 // Assume move-to/move-from VSR have 2x standard cost. 1132 return 3; 1133 } 1134 1135 // Estimated cost of a load-hit-store delay. This was obtained 1136 // experimentally as a minimum needed to prevent unprofitable 1137 // vectorization for the paq8p benchmark. It may need to be 1138 // raised further if other unprofitable cases remain. 1139 unsigned LHSPenalty = 2; 1140 if (ISD == ISD::INSERT_VECTOR_ELT) 1141 LHSPenalty += 7; 1142 1143 // Vector element insert/extract with Altivec is very expensive, 1144 // because they require store and reload with the attendant 1145 // processor stall for load-hit-store. Until VSX is available, 1146 // these need to be estimated as very costly. 1147 if (ISD == ISD::EXTRACT_VECTOR_ELT || 1148 ISD == ISD::INSERT_VECTOR_ELT) 1149 return LHSPenalty + Cost; 1150 1151 return Cost; 1152 } 1153 1154 InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, 1155 MaybeAlign Alignment, 1156 unsigned AddressSpace, 1157 TTI::TargetCostKind CostKind, 1158 const Instruction *I) { 1159 1160 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Src, nullptr); 1161 if (!CostFactor.isValid()) 1162 return InstructionCost::getMax(); 1163 1164 if (TLI->getValueType(DL, Src, true) == MVT::Other) 1165 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, 1166 CostKind); 1167 // Legalize the type. 1168 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Src); 1169 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) && 1170 "Invalid Opcode"); 1171 1172 InstructionCost Cost = 1173 BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); 1174 // TODO: Handle other cost kinds. 1175 if (CostKind != TTI::TCK_RecipThroughput) 1176 return Cost; 1177 1178 Cost *= CostFactor; 1179 1180 bool IsAltivecType = ST->hasAltivec() && 1181 (LT.second == MVT::v16i8 || LT.second == MVT::v8i16 || 1182 LT.second == MVT::v4i32 || LT.second == MVT::v4f32); 1183 bool IsVSXType = ST->hasVSX() && 1184 (LT.second == MVT::v2f64 || LT.second == MVT::v2i64); 1185 1186 // VSX has 32b/64b load instructions. Legalization can handle loading of 1187 // 32b/64b to VSR correctly and cheaply. But BaseT::getMemoryOpCost and 1188 // PPCTargetLowering can't compute the cost appropriately. So here we 1189 // explicitly check this case. 1190 unsigned MemBytes = Src->getPrimitiveSizeInBits(); 1191 if (Opcode == Instruction::Load && ST->hasVSX() && IsAltivecType && 1192 (MemBytes == 64 || (ST->hasP8Vector() && MemBytes == 32))) 1193 return 1; 1194 1195 // Aligned loads and stores are easy. 1196 unsigned SrcBytes = LT.second.getStoreSize(); 1197 if (!SrcBytes || !Alignment || *Alignment >= SrcBytes) 1198 return Cost; 1199 1200 // If we can use the permutation-based load sequence, then this is also 1201 // relatively cheap (not counting loop-invariant instructions): one load plus 1202 // one permute (the last load in a series has extra cost, but we're 1203 // neglecting that here). Note that on the P7, we could do unaligned loads 1204 // for Altivec types using the VSX instructions, but that's more expensive 1205 // than using the permutation-based load sequence. On the P8, that's no 1206 // longer true. 1207 if (Opcode == Instruction::Load && (!ST->hasP8Vector() && IsAltivecType) && 1208 *Alignment >= LT.second.getScalarType().getStoreSize()) 1209 return Cost + LT.first; // Add the cost of the permutations. 1210 1211 // For VSX, we can do unaligned loads and stores on Altivec/VSX types. On the 1212 // P7, unaligned vector loads are more expensive than the permutation-based 1213 // load sequence, so that might be used instead, but regardless, the net cost 1214 // is about the same (not counting loop-invariant instructions). 1215 if (IsVSXType || (ST->hasVSX() && IsAltivecType)) 1216 return Cost; 1217 1218 // Newer PPC supports unaligned memory access. 1219 if (TLI->allowsMisalignedMemoryAccesses(LT.second, 0)) 1220 return Cost; 1221 1222 // PPC in general does not support unaligned loads and stores. They'll need 1223 // to be decomposed based on the alignment factor. 1224 1225 // Add the cost of each scalar load or store. 1226 assert(Alignment); 1227 Cost += LT.first * ((SrcBytes / Alignment->value()) - 1); 1228 1229 // For a vector type, there is also scalarization overhead (only for 1230 // stores, loads are expanded using the vector-load + permutation sequence, 1231 // which is much less expensive). 1232 if (Src->isVectorTy() && Opcode == Instruction::Store) 1233 for (int i = 0, e = cast<FixedVectorType>(Src)->getNumElements(); i < e; 1234 ++i) 1235 Cost += getVectorInstrCost(Instruction::ExtractElement, Src, i); 1236 1237 return Cost; 1238 } 1239 1240 InstructionCost PPCTTIImpl::getInterleavedMemoryOpCost( 1241 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, 1242 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, 1243 bool UseMaskForCond, bool UseMaskForGaps) { 1244 InstructionCost CostFactor = 1245 vectorCostAdjustmentFactor(Opcode, VecTy, nullptr); 1246 if (!CostFactor.isValid()) 1247 return InstructionCost::getMax(); 1248 1249 if (UseMaskForCond || UseMaskForGaps) 1250 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, 1251 Alignment, AddressSpace, CostKind, 1252 UseMaskForCond, UseMaskForGaps); 1253 1254 assert(isa<VectorType>(VecTy) && 1255 "Expect a vector type for interleaved memory op"); 1256 1257 // Legalize the type. 1258 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, VecTy); 1259 1260 // Firstly, the cost of load/store operation. 1261 InstructionCost Cost = getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), 1262 AddressSpace, CostKind); 1263 1264 // PPC, for both Altivec/VSX, support cheap arbitrary permutations 1265 // (at least in the sense that there need only be one non-loop-invariant 1266 // instruction). For each result vector, we need one shuffle per incoming 1267 // vector (except that the first shuffle can take two incoming vectors 1268 // because it does not need to take itself). 1269 Cost += Factor*(LT.first-1); 1270 1271 return Cost; 1272 } 1273 1274 InstructionCost 1275 PPCTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, 1276 TTI::TargetCostKind CostKind) { 1277 return BaseT::getIntrinsicInstrCost(ICA, CostKind); 1278 } 1279 1280 bool PPCTTIImpl::areTypesABICompatible(const Function *Caller, 1281 const Function *Callee, 1282 const ArrayRef<Type *> &Types) const { 1283 1284 // We need to ensure that argument promotion does not 1285 // attempt to promote pointers to MMA types (__vector_pair 1286 // and __vector_quad) since these types explicitly cannot be 1287 // passed as arguments. Both of these types are larger than 1288 // the 128-bit Altivec vectors and have a scalar size of 1 bit. 1289 if (!BaseT::areTypesABICompatible(Caller, Callee, Types)) 1290 return false; 1291 1292 return llvm::none_of(Types, [](Type *Ty) { 1293 if (Ty->isSized()) 1294 return Ty->isIntOrIntVectorTy(1) && Ty->getPrimitiveSizeInBits() > 128; 1295 return false; 1296 }); 1297 } 1298 1299 bool PPCTTIImpl::canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, 1300 LoopInfo *LI, DominatorTree *DT, 1301 AssumptionCache *AC, TargetLibraryInfo *LibInfo) { 1302 // Process nested loops first. 1303 for (Loop *I : *L) 1304 if (canSaveCmp(I, BI, SE, LI, DT, AC, LibInfo)) 1305 return false; // Stop search. 1306 1307 HardwareLoopInfo HWLoopInfo(L); 1308 1309 if (!HWLoopInfo.canAnalyze(*LI)) 1310 return false; 1311 1312 if (!isHardwareLoopProfitable(L, *SE, *AC, LibInfo, HWLoopInfo)) 1313 return false; 1314 1315 if (!HWLoopInfo.isHardwareLoopCandidate(*SE, *LI, *DT)) 1316 return false; 1317 1318 *BI = HWLoopInfo.ExitBranch; 1319 return true; 1320 } 1321 1322 bool PPCTTIImpl::isLSRCostLess(TargetTransformInfo::LSRCost &C1, 1323 TargetTransformInfo::LSRCost &C2) { 1324 // PowerPC default behaviour here is "instruction number 1st priority". 1325 // If LsrNoInsnsCost is set, call default implementation. 1326 if (!LsrNoInsnsCost) 1327 return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost, C1.NumIVMuls, 1328 C1.NumBaseAdds, C1.ScaleCost, C1.ImmCost, C1.SetupCost) < 1329 std::tie(C2.Insns, C2.NumRegs, C2.AddRecCost, C2.NumIVMuls, 1330 C2.NumBaseAdds, C2.ScaleCost, C2.ImmCost, C2.SetupCost); 1331 else 1332 return TargetTransformInfoImplBase::isLSRCostLess(C1, C2); 1333 } 1334 1335 bool PPCTTIImpl::isNumRegsMajorCostOfLSR() { 1336 return false; 1337 } 1338 1339 bool PPCTTIImpl::shouldBuildRelLookupTables() const { 1340 const PPCTargetMachine &TM = ST->getTargetMachine(); 1341 // XCOFF hasn't implemented lowerRelativeReference, disable non-ELF for now. 1342 if (!TM.isELFv2ABI()) 1343 return false; 1344 return BaseT::shouldBuildRelLookupTables(); 1345 } 1346 1347 bool PPCTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst, 1348 MemIntrinsicInfo &Info) { 1349 switch (Inst->getIntrinsicID()) { 1350 case Intrinsic::ppc_altivec_lvx: 1351 case Intrinsic::ppc_altivec_lvxl: 1352 case Intrinsic::ppc_altivec_lvebx: 1353 case Intrinsic::ppc_altivec_lvehx: 1354 case Intrinsic::ppc_altivec_lvewx: 1355 case Intrinsic::ppc_vsx_lxvd2x: 1356 case Intrinsic::ppc_vsx_lxvw4x: 1357 case Intrinsic::ppc_vsx_lxvd2x_be: 1358 case Intrinsic::ppc_vsx_lxvw4x_be: 1359 case Intrinsic::ppc_vsx_lxvl: 1360 case Intrinsic::ppc_vsx_lxvll: 1361 case Intrinsic::ppc_vsx_lxvp: { 1362 Info.PtrVal = Inst->getArgOperand(0); 1363 Info.ReadMem = true; 1364 Info.WriteMem = false; 1365 return true; 1366 } 1367 case Intrinsic::ppc_altivec_stvx: 1368 case Intrinsic::ppc_altivec_stvxl: 1369 case Intrinsic::ppc_altivec_stvebx: 1370 case Intrinsic::ppc_altivec_stvehx: 1371 case Intrinsic::ppc_altivec_stvewx: 1372 case Intrinsic::ppc_vsx_stxvd2x: 1373 case Intrinsic::ppc_vsx_stxvw4x: 1374 case Intrinsic::ppc_vsx_stxvd2x_be: 1375 case Intrinsic::ppc_vsx_stxvw4x_be: 1376 case Intrinsic::ppc_vsx_stxvl: 1377 case Intrinsic::ppc_vsx_stxvll: 1378 case Intrinsic::ppc_vsx_stxvp: { 1379 Info.PtrVal = Inst->getArgOperand(1); 1380 Info.ReadMem = false; 1381 Info.WriteMem = true; 1382 return true; 1383 } 1384 default: 1385 break; 1386 } 1387 1388 return false; 1389 } 1390 1391 bool PPCTTIImpl::hasActiveVectorLength(unsigned Opcode, Type *DataType, 1392 Align Alignment) const { 1393 // Only load and stores instructions can have variable vector length on Power. 1394 if (Opcode != Instruction::Load && Opcode != Instruction::Store) 1395 return false; 1396 // Loads/stores with length instructions use bits 0-7 of the GPR operand and 1397 // therefore cannot be used in 32-bit mode. 1398 if ((!ST->hasP9Vector() && !ST->hasP10Vector()) || !ST->isPPC64()) 1399 return false; 1400 if (isa<FixedVectorType>(DataType)) { 1401 unsigned VecWidth = DataType->getPrimitiveSizeInBits(); 1402 return VecWidth == 128; 1403 } 1404 Type *ScalarTy = DataType->getScalarType(); 1405 1406 if (ScalarTy->isPointerTy()) 1407 return true; 1408 1409 if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy()) 1410 return true; 1411 1412 if (!ScalarTy->isIntegerTy()) 1413 return false; 1414 1415 unsigned IntWidth = ScalarTy->getIntegerBitWidth(); 1416 return IntWidth == 8 || IntWidth == 16 || IntWidth == 32 || IntWidth == 64; 1417 } 1418 1419 InstructionCost PPCTTIImpl::getVPMemoryOpCost(unsigned Opcode, Type *Src, 1420 Align Alignment, 1421 unsigned AddressSpace, 1422 TTI::TargetCostKind CostKind, 1423 const Instruction *I) { 1424 InstructionCost Cost = BaseT::getVPMemoryOpCost(Opcode, Src, Alignment, 1425 AddressSpace, CostKind, I); 1426 if (TLI->getValueType(DL, Src, true) == MVT::Other) 1427 return Cost; 1428 // TODO: Handle other cost kinds. 1429 if (CostKind != TTI::TCK_RecipThroughput) 1430 return Cost; 1431 1432 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) && 1433 "Invalid Opcode"); 1434 1435 auto *SrcVTy = dyn_cast<FixedVectorType>(Src); 1436 assert(SrcVTy && "Expected a vector type for VP memory operations"); 1437 1438 if (hasActiveVectorLength(Opcode, Src, Alignment)) { 1439 std::pair<InstructionCost, MVT> LT = 1440 TLI->getTypeLegalizationCost(DL, SrcVTy); 1441 1442 InstructionCost CostFactor = 1443 vectorCostAdjustmentFactor(Opcode, Src, nullptr); 1444 if (!CostFactor.isValid()) 1445 return InstructionCost::getMax(); 1446 1447 InstructionCost Cost = LT.first * CostFactor; 1448 assert(Cost.isValid() && "Expected valid cost"); 1449 1450 // On P9 but not on P10, if the op is misaligned then it will cause a 1451 // pipeline flush. Otherwise the VSX masked memops cost the same as unmasked 1452 // ones. 1453 const Align DesiredAlignment(16); 1454 if (Alignment >= DesiredAlignment || ST->getCPUDirective() != PPC::DIR_PWR9) 1455 return Cost; 1456 1457 // Since alignment may be under estimated, we try to compute the probability 1458 // that the actual address is aligned to the desired boundary. For example 1459 // an 8-byte aligned load is assumed to be actually 16-byte aligned half the 1460 // time, while a 4-byte aligned load has a 25% chance of being 16-byte 1461 // aligned. 1462 float AlignmentProb = ((float)Alignment.value()) / DesiredAlignment.value(); 1463 float MisalignmentProb = 1.0 - AlignmentProb; 1464 return (MisalignmentProb * P9PipelineFlushEstimate) + 1465 (AlignmentProb * *Cost.getValue()); 1466 } 1467 1468 // Usually we should not get to this point, but the following is an attempt to 1469 // model the cost of legalization. Currently we can only lower intrinsics with 1470 // evl but no mask, on Power 9/10. Otherwise, we must scalarize. 1471 return getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); 1472 } 1473