1 //===- AMDGPUTargetTransformInfo.cpp - AMDGPU specific TTI pass -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // \file 10 // This file implements a TargetTransformInfo analysis pass specific to the 11 // AMDGPU target machine. It uses the target's detailed information to provide 12 // more precise answers to certain TTI queries, while letting the target 13 // independent and default TTI implementations handle the rest. 14 // 15 //===----------------------------------------------------------------------===// 16 17 #include "AMDGPUTargetTransformInfo.h" 18 #include "AMDGPUTargetMachine.h" 19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 20 #include "llvm/Analysis/LoopInfo.h" 21 #include "llvm/Analysis/ValueTracking.h" 22 #include "llvm/IR/IRBuilder.h" 23 #include "llvm/IR/IntrinsicsAMDGPU.h" 24 #include "llvm/IR/PatternMatch.h" 25 #include "llvm/Support/KnownBits.h" 26 27 using namespace llvm; 28 29 #define DEBUG_TYPE "AMDGPUtti" 30 31 static cl::opt<unsigned> UnrollThresholdPrivate( 32 "amdgpu-unroll-threshold-private", 33 cl::desc("Unroll threshold for AMDGPU if private memory used in a loop"), 34 cl::init(2700), cl::Hidden); 35 36 static cl::opt<unsigned> UnrollThresholdLocal( 37 "amdgpu-unroll-threshold-local", 38 cl::desc("Unroll threshold for AMDGPU if local memory used in a loop"), 39 cl::init(1000), cl::Hidden); 40 41 static cl::opt<unsigned> UnrollThresholdIf( 42 "amdgpu-unroll-threshold-if", 43 cl::desc("Unroll threshold increment for AMDGPU for each if statement inside loop"), 44 cl::init(200), cl::Hidden); 45 46 static cl::opt<bool> UnrollRuntimeLocal( 47 "amdgpu-unroll-runtime-local", 48 cl::desc("Allow runtime unroll for AMDGPU if local memory used in a loop"), 49 cl::init(true), cl::Hidden); 50 51 static cl::opt<bool> UseLegacyDA( 52 "amdgpu-use-legacy-divergence-analysis", 53 cl::desc("Enable legacy divergence analysis for AMDGPU"), 54 cl::init(false), cl::Hidden); 55 56 static cl::opt<unsigned> UnrollMaxBlockToAnalyze( 57 "amdgpu-unroll-max-block-to-analyze", 58 cl::desc("Inner loop block size threshold to analyze in unroll for AMDGPU"), 59 cl::init(32), cl::Hidden); 60 61 static cl::opt<unsigned> ArgAllocaCost("amdgpu-inline-arg-alloca-cost", 62 cl::Hidden, cl::init(4000), 63 cl::desc("Cost of alloca argument")); 64 65 // If the amount of scratch memory to eliminate exceeds our ability to allocate 66 // it into registers we gain nothing by aggressively inlining functions for that 67 // heuristic. 68 static cl::opt<unsigned> 69 ArgAllocaCutoff("amdgpu-inline-arg-alloca-cutoff", cl::Hidden, 70 cl::init(256), 71 cl::desc("Maximum alloca size to use for inline cost")); 72 73 // Inliner constraint to achieve reasonable compilation time. 74 static cl::opt<size_t> InlineMaxBB( 75 "amdgpu-inline-max-bb", cl::Hidden, cl::init(1100), 76 cl::desc("Maximum number of BBs allowed in a function after inlining" 77 " (compile time constraint)")); 78 79 static bool dependsOnLocalPhi(const Loop *L, const Value *Cond, 80 unsigned Depth = 0) { 81 const Instruction *I = dyn_cast<Instruction>(Cond); 82 if (!I) 83 return false; 84 85 for (const Value *V : I->operand_values()) { 86 if (!L->contains(I)) 87 continue; 88 if (const PHINode *PHI = dyn_cast<PHINode>(V)) { 89 if (llvm::none_of(L->getSubLoops(), [PHI](const Loop* SubLoop) { 90 return SubLoop->contains(PHI); })) 91 return true; 92 } else if (Depth < 10 && dependsOnLocalPhi(L, V, Depth+1)) 93 return true; 94 } 95 return false; 96 } 97 98 AMDGPUTTIImpl::AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F) 99 : BaseT(TM, F.getParent()->getDataLayout()), 100 TargetTriple(TM->getTargetTriple()), 101 ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))), 102 TLI(ST->getTargetLowering()) {} 103 104 void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, 105 TTI::UnrollingPreferences &UP, 106 OptimizationRemarkEmitter *ORE) { 107 const Function &F = *L->getHeader()->getParent(); 108 UP.Threshold = AMDGPU::getIntegerAttribute(F, "amdgpu-unroll-threshold", 300); 109 UP.MaxCount = std::numeric_limits<unsigned>::max(); 110 UP.Partial = true; 111 112 // Conditional branch in a loop back edge needs 3 additional exec 113 // manipulations in average. 114 UP.BEInsns += 3; 115 116 // TODO: Do we want runtime unrolling? 117 118 // Maximum alloca size than can fit registers. Reserve 16 registers. 119 const unsigned MaxAlloca = (256 - 16) * 4; 120 unsigned ThresholdPrivate = UnrollThresholdPrivate; 121 unsigned ThresholdLocal = UnrollThresholdLocal; 122 123 // If this loop has the amdgpu.loop.unroll.threshold metadata we will use the 124 // provided threshold value as the default for Threshold 125 if (MDNode *LoopUnrollThreshold = 126 findOptionMDForLoop(L, "amdgpu.loop.unroll.threshold")) { 127 if (LoopUnrollThreshold->getNumOperands() == 2) { 128 ConstantInt *MetaThresholdValue = mdconst::extract_or_null<ConstantInt>( 129 LoopUnrollThreshold->getOperand(1)); 130 if (MetaThresholdValue) { 131 // We will also use the supplied value for PartialThreshold for now. 132 // We may introduce additional metadata if it becomes necessary in the 133 // future. 134 UP.Threshold = MetaThresholdValue->getSExtValue(); 135 UP.PartialThreshold = UP.Threshold; 136 ThresholdPrivate = std::min(ThresholdPrivate, UP.Threshold); 137 ThresholdLocal = std::min(ThresholdLocal, UP.Threshold); 138 } 139 } 140 } 141 142 unsigned MaxBoost = std::max(ThresholdPrivate, ThresholdLocal); 143 for (const BasicBlock *BB : L->getBlocks()) { 144 const DataLayout &DL = BB->getModule()->getDataLayout(); 145 unsigned LocalGEPsSeen = 0; 146 147 if (llvm::any_of(L->getSubLoops(), [BB](const Loop* SubLoop) { 148 return SubLoop->contains(BB); })) 149 continue; // Block belongs to an inner loop. 150 151 for (const Instruction &I : *BB) { 152 // Unroll a loop which contains an "if" statement whose condition 153 // defined by a PHI belonging to the loop. This may help to eliminate 154 // if region and potentially even PHI itself, saving on both divergence 155 // and registers used for the PHI. 156 // Add a small bonus for each of such "if" statements. 157 if (const BranchInst *Br = dyn_cast<BranchInst>(&I)) { 158 if (UP.Threshold < MaxBoost && Br->isConditional()) { 159 BasicBlock *Succ0 = Br->getSuccessor(0); 160 BasicBlock *Succ1 = Br->getSuccessor(1); 161 if ((L->contains(Succ0) && L->isLoopExiting(Succ0)) || 162 (L->contains(Succ1) && L->isLoopExiting(Succ1))) 163 continue; 164 if (dependsOnLocalPhi(L, Br->getCondition())) { 165 UP.Threshold += UnrollThresholdIf; 166 LLVM_DEBUG(dbgs() << "Set unroll threshold " << UP.Threshold 167 << " for loop:\n" 168 << *L << " due to " << *Br << '\n'); 169 if (UP.Threshold >= MaxBoost) 170 return; 171 } 172 } 173 continue; 174 } 175 176 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I); 177 if (!GEP) 178 continue; 179 180 unsigned AS = GEP->getAddressSpace(); 181 unsigned Threshold = 0; 182 if (AS == AMDGPUAS::PRIVATE_ADDRESS) 183 Threshold = ThresholdPrivate; 184 else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) 185 Threshold = ThresholdLocal; 186 else 187 continue; 188 189 if (UP.Threshold >= Threshold) 190 continue; 191 192 if (AS == AMDGPUAS::PRIVATE_ADDRESS) { 193 const Value *Ptr = GEP->getPointerOperand(); 194 const AllocaInst *Alloca = 195 dyn_cast<AllocaInst>(getUnderlyingObject(Ptr)); 196 if (!Alloca || !Alloca->isStaticAlloca()) 197 continue; 198 Type *Ty = Alloca->getAllocatedType(); 199 unsigned AllocaSize = Ty->isSized() ? DL.getTypeAllocSize(Ty) : 0; 200 if (AllocaSize > MaxAlloca) 201 continue; 202 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || 203 AS == AMDGPUAS::REGION_ADDRESS) { 204 LocalGEPsSeen++; 205 // Inhibit unroll for local memory if we have seen addressing not to 206 // a variable, most likely we will be unable to combine it. 207 // Do not unroll too deep inner loops for local memory to give a chance 208 // to unroll an outer loop for a more important reason. 209 if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2 || 210 (!isa<GlobalVariable>(GEP->getPointerOperand()) && 211 !isa<Argument>(GEP->getPointerOperand()))) 212 continue; 213 LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n" 214 << *L << " due to LDS use.\n"); 215 UP.Runtime = UnrollRuntimeLocal; 216 } 217 218 // Check if GEP depends on a value defined by this loop itself. 219 bool HasLoopDef = false; 220 for (const Value *Op : GEP->operands()) { 221 const Instruction *Inst = dyn_cast<Instruction>(Op); 222 if (!Inst || L->isLoopInvariant(Op)) 223 continue; 224 225 if (llvm::any_of(L->getSubLoops(), [Inst](const Loop* SubLoop) { 226 return SubLoop->contains(Inst); })) 227 continue; 228 HasLoopDef = true; 229 break; 230 } 231 if (!HasLoopDef) 232 continue; 233 234 // We want to do whatever we can to limit the number of alloca 235 // instructions that make it through to the code generator. allocas 236 // require us to use indirect addressing, which is slow and prone to 237 // compiler bugs. If this loop does an address calculation on an 238 // alloca ptr, then we want to use a higher than normal loop unroll 239 // threshold. This will give SROA a better chance to eliminate these 240 // allocas. 241 // 242 // We also want to have more unrolling for local memory to let ds 243 // instructions with different offsets combine. 244 // 245 // Don't use the maximum allowed value here as it will make some 246 // programs way too big. 247 UP.Threshold = Threshold; 248 LLVM_DEBUG(dbgs() << "Set unroll threshold " << Threshold 249 << " for loop:\n" 250 << *L << " due to " << *GEP << '\n'); 251 if (UP.Threshold >= MaxBoost) 252 return; 253 } 254 255 // If we got a GEP in a small BB from inner loop then increase max trip 256 // count to analyze for better estimation cost in unroll 257 if (L->isInnermost() && BB->size() < UnrollMaxBlockToAnalyze) 258 UP.MaxIterationsCountToAnalyze = 32; 259 } 260 } 261 262 void AMDGPUTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, 263 TTI::PeelingPreferences &PP) { 264 BaseT::getPeelingPreferences(L, SE, PP); 265 } 266 267 const FeatureBitset GCNTTIImpl::InlineFeatureIgnoreList = { 268 // Codegen control options which don't matter. 269 AMDGPU::FeatureEnableLoadStoreOpt, AMDGPU::FeatureEnableSIScheduler, 270 AMDGPU::FeatureEnableUnsafeDSOffsetFolding, AMDGPU::FeatureFlatForGlobal, 271 AMDGPU::FeaturePromoteAlloca, AMDGPU::FeatureUnalignedScratchAccess, 272 AMDGPU::FeatureUnalignedAccessMode, 273 274 AMDGPU::FeatureAutoWaitcntBeforeBarrier, 275 276 // Property of the kernel/environment which can't actually differ. 277 AMDGPU::FeatureSGPRInitBug, AMDGPU::FeatureXNACK, 278 AMDGPU::FeatureTrapHandler, 279 280 // The default assumption needs to be ecc is enabled, but no directly 281 // exposed operations depend on it, so it can be safely inlined. 282 AMDGPU::FeatureSRAMECC, 283 284 // Perf-tuning features 285 AMDGPU::FeatureFastFMAF32, AMDGPU::HalfRate64Ops}; 286 287 GCNTTIImpl::GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F) 288 : BaseT(TM, F.getParent()->getDataLayout()), 289 ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))), 290 TLI(ST->getTargetLowering()), CommonTTI(TM, F), 291 IsGraphics(AMDGPU::isGraphics(F.getCallingConv())), 292 MaxVGPRs(ST->getMaxNumVGPRs( 293 std::max(ST->getWavesPerEU(F).first, 294 ST->getWavesPerEUForWorkGroup( 295 ST->getFlatWorkGroupSizes(F).second)))) { 296 AMDGPU::SIModeRegisterDefaults Mode(F); 297 HasFP32Denormals = Mode.allFP32Denormals(); 298 HasFP64FP16Denormals = Mode.allFP64FP16Denormals(); 299 } 300 301 unsigned GCNTTIImpl::getHardwareNumberOfRegisters(bool Vec) const { 302 // The concept of vector registers doesn't really exist. Some packed vector 303 // operations operate on the normal 32-bit registers. 304 return MaxVGPRs; 305 } 306 307 unsigned GCNTTIImpl::getNumberOfRegisters(bool Vec) const { 308 // This is really the number of registers to fill when vectorizing / 309 // interleaving loops, so we lie to avoid trying to use all registers. 310 return getHardwareNumberOfRegisters(Vec) >> 3; 311 } 312 313 unsigned GCNTTIImpl::getNumberOfRegisters(unsigned RCID) const { 314 const SIRegisterInfo *TRI = ST->getRegisterInfo(); 315 const TargetRegisterClass *RC = TRI->getRegClass(RCID); 316 unsigned NumVGPRs = (TRI->getRegSizeInBits(*RC) + 31) / 32; 317 return getHardwareNumberOfRegisters(false) / NumVGPRs; 318 } 319 320 TypeSize 321 GCNTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const { 322 switch (K) { 323 case TargetTransformInfo::RGK_Scalar: 324 return TypeSize::getFixed(32); 325 case TargetTransformInfo::RGK_FixedWidthVector: 326 return TypeSize::getFixed(ST->hasPackedFP32Ops() ? 64 : 32); 327 case TargetTransformInfo::RGK_ScalableVector: 328 return TypeSize::getScalable(0); 329 } 330 llvm_unreachable("Unsupported register kind"); 331 } 332 333 unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const { 334 return 32; 335 } 336 337 unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const { 338 if (Opcode == Instruction::Load || Opcode == Instruction::Store) 339 return 32 * 4 / ElemWidth; 340 return (ElemWidth == 16 && ST->has16BitInsts()) ? 2 341 : (ElemWidth == 32 && ST->hasPackedFP32Ops()) ? 2 342 : 1; 343 } 344 345 unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize, 346 unsigned ChainSizeInBytes, 347 VectorType *VecTy) const { 348 unsigned VecRegBitWidth = VF * LoadSize; 349 if (VecRegBitWidth > 128 && VecTy->getScalarSizeInBits() < 32) 350 // TODO: Support element-size less than 32bit? 351 return 128 / LoadSize; 352 353 return VF; 354 } 355 356 unsigned GCNTTIImpl::getStoreVectorFactor(unsigned VF, unsigned StoreSize, 357 unsigned ChainSizeInBytes, 358 VectorType *VecTy) const { 359 unsigned VecRegBitWidth = VF * StoreSize; 360 if (VecRegBitWidth > 128) 361 return 128 / StoreSize; 362 363 return VF; 364 } 365 366 unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const { 367 if (AddrSpace == AMDGPUAS::GLOBAL_ADDRESS || 368 AddrSpace == AMDGPUAS::CONSTANT_ADDRESS || 369 AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT || 370 AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER) { 371 return 512; 372 } 373 374 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) 375 return 8 * ST->getMaxPrivateElementSize(); 376 377 // Common to flat, global, local and region. Assume for unknown addrspace. 378 return 128; 379 } 380 381 bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, 382 Align Alignment, 383 unsigned AddrSpace) const { 384 // We allow vectorization of flat stores, even though we may need to decompose 385 // them later if they may access private memory. We don't have enough context 386 // here, and legalization can handle it. 387 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) { 388 return (Alignment >= 4 || ST->hasUnalignedScratchAccess()) && 389 ChainSizeInBytes <= ST->getMaxPrivateElementSize(); 390 } 391 return true; 392 } 393 394 bool GCNTTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, 395 Align Alignment, 396 unsigned AddrSpace) const { 397 return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace); 398 } 399 400 bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, 401 Align Alignment, 402 unsigned AddrSpace) const { 403 return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace); 404 } 405 406 // FIXME: Really we would like to issue multiple 128-bit loads and stores per 407 // iteration. Should we report a larger size and let it legalize? 408 // 409 // FIXME: Should we use narrower types for local/region, or account for when 410 // unaligned access is legal? 411 // 412 // FIXME: This could use fine tuning and microbenchmarks. 413 Type *GCNTTIImpl::getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, 414 unsigned SrcAddrSpace, 415 unsigned DestAddrSpace, 416 unsigned SrcAlign, 417 unsigned DestAlign) const { 418 unsigned MinAlign = std::min(SrcAlign, DestAlign); 419 420 // A (multi-)dword access at an address == 2 (mod 4) will be decomposed by the 421 // hardware into byte accesses. If you assume all alignments are equally 422 // probable, it's more efficient on average to use short accesses for this 423 // case. 424 if (MinAlign == 2) 425 return Type::getInt16Ty(Context); 426 427 // Not all subtargets have 128-bit DS instructions, and we currently don't 428 // form them by default. 429 if (SrcAddrSpace == AMDGPUAS::LOCAL_ADDRESS || 430 SrcAddrSpace == AMDGPUAS::REGION_ADDRESS || 431 DestAddrSpace == AMDGPUAS::LOCAL_ADDRESS || 432 DestAddrSpace == AMDGPUAS::REGION_ADDRESS) { 433 return FixedVectorType::get(Type::getInt32Ty(Context), 2); 434 } 435 436 // Global memory works best with 16-byte accesses. Private memory will also 437 // hit this, although they'll be decomposed. 438 return FixedVectorType::get(Type::getInt32Ty(Context), 4); 439 } 440 441 void GCNTTIImpl::getMemcpyLoopResidualLoweringType( 442 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, 443 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, 444 unsigned SrcAlign, unsigned DestAlign) const { 445 assert(RemainingBytes < 16); 446 447 unsigned MinAlign = std::min(SrcAlign, DestAlign); 448 449 if (MinAlign != 2) { 450 Type *I64Ty = Type::getInt64Ty(Context); 451 while (RemainingBytes >= 8) { 452 OpsOut.push_back(I64Ty); 453 RemainingBytes -= 8; 454 } 455 456 Type *I32Ty = Type::getInt32Ty(Context); 457 while (RemainingBytes >= 4) { 458 OpsOut.push_back(I32Ty); 459 RemainingBytes -= 4; 460 } 461 } 462 463 Type *I16Ty = Type::getInt16Ty(Context); 464 while (RemainingBytes >= 2) { 465 OpsOut.push_back(I16Ty); 466 RemainingBytes -= 2; 467 } 468 469 Type *I8Ty = Type::getInt8Ty(Context); 470 while (RemainingBytes) { 471 OpsOut.push_back(I8Ty); 472 --RemainingBytes; 473 } 474 } 475 476 unsigned GCNTTIImpl::getMaxInterleaveFactor(unsigned VF) { 477 // Disable unrolling if the loop is not vectorized. 478 // TODO: Enable this again. 479 if (VF == 1) 480 return 1; 481 482 return 8; 483 } 484 485 bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst, 486 MemIntrinsicInfo &Info) const { 487 switch (Inst->getIntrinsicID()) { 488 case Intrinsic::amdgcn_atomic_inc: 489 case Intrinsic::amdgcn_atomic_dec: 490 case Intrinsic::amdgcn_ds_ordered_add: 491 case Intrinsic::amdgcn_ds_ordered_swap: 492 case Intrinsic::amdgcn_ds_fadd: 493 case Intrinsic::amdgcn_ds_fmin: 494 case Intrinsic::amdgcn_ds_fmax: { 495 auto *Ordering = dyn_cast<ConstantInt>(Inst->getArgOperand(2)); 496 auto *Volatile = dyn_cast<ConstantInt>(Inst->getArgOperand(4)); 497 if (!Ordering || !Volatile) 498 return false; // Invalid. 499 500 unsigned OrderingVal = Ordering->getZExtValue(); 501 if (OrderingVal > static_cast<unsigned>(AtomicOrdering::SequentiallyConsistent)) 502 return false; 503 504 Info.PtrVal = Inst->getArgOperand(0); 505 Info.Ordering = static_cast<AtomicOrdering>(OrderingVal); 506 Info.ReadMem = true; 507 Info.WriteMem = true; 508 Info.IsVolatile = !Volatile->isZero(); 509 return true; 510 } 511 default: 512 return false; 513 } 514 } 515 516 InstructionCost GCNTTIImpl::getArithmeticInstrCost( 517 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, 518 TTI::OperandValueKind Opd1Info, TTI::OperandValueKind Opd2Info, 519 TTI::OperandValueProperties Opd1PropInfo, 520 TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args, 521 const Instruction *CxtI) { 522 523 // Legalize the type. 524 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty); 525 int ISD = TLI->InstructionOpcodeToISD(Opcode); 526 527 // Because we don't have any legal vector operations, but the legal types, we 528 // need to account for split vectors. 529 unsigned NElts = LT.second.isVector() ? 530 LT.second.getVectorNumElements() : 1; 531 532 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy; 533 534 switch (ISD) { 535 case ISD::SHL: 536 case ISD::SRL: 537 case ISD::SRA: 538 if (SLT == MVT::i64) 539 return get64BitInstrCost(CostKind) * LT.first * NElts; 540 541 if (ST->has16BitInsts() && SLT == MVT::i16) 542 NElts = (NElts + 1) / 2; 543 544 // i32 545 return getFullRateInstrCost() * LT.first * NElts; 546 case ISD::ADD: 547 case ISD::SUB: 548 case ISD::AND: 549 case ISD::OR: 550 case ISD::XOR: 551 if (SLT == MVT::i64) { 552 // and, or and xor are typically split into 2 VALU instructions. 553 return 2 * getFullRateInstrCost() * LT.first * NElts; 554 } 555 556 if (ST->has16BitInsts() && SLT == MVT::i16) 557 NElts = (NElts + 1) / 2; 558 559 return LT.first * NElts * getFullRateInstrCost(); 560 case ISD::MUL: { 561 const int QuarterRateCost = getQuarterRateInstrCost(CostKind); 562 if (SLT == MVT::i64) { 563 const int FullRateCost = getFullRateInstrCost(); 564 return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts; 565 } 566 567 if (ST->has16BitInsts() && SLT == MVT::i16) 568 NElts = (NElts + 1) / 2; 569 570 // i32 571 return QuarterRateCost * NElts * LT.first; 572 } 573 case ISD::FMUL: 574 // Check possible fuse {fadd|fsub}(a,fmul(b,c)) and return zero cost for 575 // fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole 576 // fused operation. 577 if (CxtI && CxtI->hasOneUse()) 578 if (const auto *FAdd = dyn_cast<BinaryOperator>(*CxtI->user_begin())) { 579 const int OPC = TLI->InstructionOpcodeToISD(FAdd->getOpcode()); 580 if (OPC == ISD::FADD || OPC == ISD::FSUB) { 581 if (ST->hasMadMacF32Insts() && SLT == MVT::f32 && !HasFP32Denormals) 582 return TargetTransformInfo::TCC_Free; 583 if (ST->has16BitInsts() && SLT == MVT::f16 && !HasFP64FP16Denormals) 584 return TargetTransformInfo::TCC_Free; 585 586 // Estimate all types may be fused with contract/unsafe flags 587 const TargetOptions &Options = TLI->getTargetMachine().Options; 588 if (Options.AllowFPOpFusion == FPOpFusion::Fast || 589 Options.UnsafeFPMath || 590 (FAdd->hasAllowContract() && CxtI->hasAllowContract())) 591 return TargetTransformInfo::TCC_Free; 592 } 593 } 594 LLVM_FALLTHROUGH; 595 case ISD::FADD: 596 case ISD::FSUB: 597 if (ST->hasPackedFP32Ops() && SLT == MVT::f32) 598 NElts = (NElts + 1) / 2; 599 if (SLT == MVT::f64) 600 return LT.first * NElts * get64BitInstrCost(CostKind); 601 602 if (ST->has16BitInsts() && SLT == MVT::f16) 603 NElts = (NElts + 1) / 2; 604 605 if (SLT == MVT::f32 || SLT == MVT::f16) 606 return LT.first * NElts * getFullRateInstrCost(); 607 break; 608 case ISD::FDIV: 609 case ISD::FREM: 610 // FIXME: frem should be handled separately. The fdiv in it is most of it, 611 // but the current lowering is also not entirely correct. 612 if (SLT == MVT::f64) { 613 int Cost = 7 * get64BitInstrCost(CostKind) + 614 getQuarterRateInstrCost(CostKind) + 615 3 * getHalfRateInstrCost(CostKind); 616 // Add cost of workaround. 617 if (!ST->hasUsableDivScaleConditionOutput()) 618 Cost += 3 * getFullRateInstrCost(); 619 620 return LT.first * Cost * NElts; 621 } 622 623 if (!Args.empty() && match(Args[0], PatternMatch::m_FPOne())) { 624 // TODO: This is more complicated, unsafe flags etc. 625 if ((SLT == MVT::f32 && !HasFP32Denormals) || 626 (SLT == MVT::f16 && ST->has16BitInsts())) { 627 return LT.first * getQuarterRateInstrCost(CostKind) * NElts; 628 } 629 } 630 631 if (SLT == MVT::f16 && ST->has16BitInsts()) { 632 // 2 x v_cvt_f32_f16 633 // f32 rcp 634 // f32 fmul 635 // v_cvt_f16_f32 636 // f16 div_fixup 637 int Cost = 638 4 * getFullRateInstrCost() + 2 * getQuarterRateInstrCost(CostKind); 639 return LT.first * Cost * NElts; 640 } 641 642 if (SLT == MVT::f32 || SLT == MVT::f16) { 643 // 4 more v_cvt_* insts without f16 insts support 644 int Cost = (SLT == MVT::f16 ? 14 : 10) * getFullRateInstrCost() + 645 1 * getQuarterRateInstrCost(CostKind); 646 647 if (!HasFP32Denormals) { 648 // FP mode switches. 649 Cost += 2 * getFullRateInstrCost(); 650 } 651 652 return LT.first * NElts * Cost; 653 } 654 break; 655 case ISD::FNEG: 656 // Use the backend' estimation. If fneg is not free each element will cost 657 // one additional instruction. 658 return TLI->isFNegFree(SLT) ? 0 : NElts; 659 default: 660 break; 661 } 662 663 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info, Opd2Info, 664 Opd1PropInfo, Opd2PropInfo, Args, CxtI); 665 } 666 667 // Return true if there's a potential benefit from using v2f16/v2i16 668 // instructions for an intrinsic, even if it requires nontrivial legalization. 669 static bool intrinsicHasPackedVectorBenefit(Intrinsic::ID ID) { 670 switch (ID) { 671 case Intrinsic::fma: // TODO: fmuladd 672 // There's a small benefit to using vector ops in the legalized code. 673 case Intrinsic::round: 674 case Intrinsic::uadd_sat: 675 case Intrinsic::usub_sat: 676 case Intrinsic::sadd_sat: 677 case Intrinsic::ssub_sat: 678 return true; 679 default: 680 return false; 681 } 682 } 683 684 InstructionCost 685 GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, 686 TTI::TargetCostKind CostKind) { 687 if (ICA.getID() == Intrinsic::fabs) 688 return 0; 689 690 if (!intrinsicHasPackedVectorBenefit(ICA.getID())) 691 return BaseT::getIntrinsicInstrCost(ICA, CostKind); 692 693 Type *RetTy = ICA.getReturnType(); 694 695 // Legalize the type. 696 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, RetTy); 697 698 unsigned NElts = LT.second.isVector() ? 699 LT.second.getVectorNumElements() : 1; 700 701 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy; 702 703 if (SLT == MVT::f64) 704 return LT.first * NElts * get64BitInstrCost(CostKind); 705 706 if ((ST->has16BitInsts() && SLT == MVT::f16) || 707 (ST->hasPackedFP32Ops() && SLT == MVT::f32)) 708 NElts = (NElts + 1) / 2; 709 710 // TODO: Get more refined intrinsic costs? 711 unsigned InstRate = getQuarterRateInstrCost(CostKind); 712 713 switch (ICA.getID()) { 714 case Intrinsic::fma: 715 InstRate = ST->hasFastFMAF32() ? getHalfRateInstrCost(CostKind) 716 : getQuarterRateInstrCost(CostKind); 717 break; 718 case Intrinsic::uadd_sat: 719 case Intrinsic::usub_sat: 720 case Intrinsic::sadd_sat: 721 case Intrinsic::ssub_sat: 722 static const auto ValidSatTys = {MVT::v2i16, MVT::v4i16}; 723 if (any_of(ValidSatTys, [<](MVT M) { return M == LT.second; })) 724 NElts = 1; 725 break; 726 } 727 728 return LT.first * NElts * InstRate; 729 } 730 731 InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode, 732 TTI::TargetCostKind CostKind, 733 const Instruction *I) { 734 assert((I == nullptr || I->getOpcode() == Opcode) && 735 "Opcode should reflect passed instruction."); 736 const bool SCost = 737 (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency); 738 const int CBrCost = SCost ? 5 : 7; 739 switch (Opcode) { 740 case Instruction::Br: { 741 // Branch instruction takes about 4 slots on gfx900. 742 auto BI = dyn_cast_or_null<BranchInst>(I); 743 if (BI && BI->isUnconditional()) 744 return SCost ? 1 : 4; 745 // Suppose conditional branch takes additional 3 exec manipulations 746 // instructions in average. 747 return CBrCost; 748 } 749 case Instruction::Switch: { 750 auto SI = dyn_cast_or_null<SwitchInst>(I); 751 // Each case (including default) takes 1 cmp + 1 cbr instructions in 752 // average. 753 return (SI ? (SI->getNumCases() + 1) : 4) * (CBrCost + 1); 754 } 755 case Instruction::Ret: 756 return SCost ? 1 : 10; 757 } 758 return BaseT::getCFInstrCost(Opcode, CostKind, I); 759 } 760 761 InstructionCost 762 GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, 763 Optional<FastMathFlags> FMF, 764 TTI::TargetCostKind CostKind) { 765 if (TTI::requiresOrderedReduction(FMF)) 766 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind); 767 768 EVT OrigTy = TLI->getValueType(DL, Ty); 769 770 // Computes cost on targets that have packed math instructions(which support 771 // 16-bit types only). 772 if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16) 773 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind); 774 775 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty); 776 return LT.first * getFullRateInstrCost(); 777 } 778 779 InstructionCost 780 GCNTTIImpl::getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy, 781 bool IsUnsigned, 782 TTI::TargetCostKind CostKind) { 783 EVT OrigTy = TLI->getValueType(DL, Ty); 784 785 // Computes cost on targets that have packed math instructions(which support 786 // 16-bit types only). 787 if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16) 788 return BaseT::getMinMaxReductionCost(Ty, CondTy, IsUnsigned, CostKind); 789 790 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty); 791 return LT.first * getHalfRateInstrCost(CostKind); 792 } 793 794 InstructionCost GCNTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy, 795 unsigned Index) { 796 switch (Opcode) { 797 case Instruction::ExtractElement: 798 case Instruction::InsertElement: { 799 unsigned EltSize 800 = DL.getTypeSizeInBits(cast<VectorType>(ValTy)->getElementType()); 801 if (EltSize < 32) { 802 if (EltSize == 16 && Index == 0 && ST->has16BitInsts()) 803 return 0; 804 return BaseT::getVectorInstrCost(Opcode, ValTy, Index); 805 } 806 807 // Extracts are just reads of a subregister, so are free. Inserts are 808 // considered free because we don't want to have any cost for scalarizing 809 // operations, and we don't have to copy into a different register class. 810 811 // Dynamic indexing isn't free and is best avoided. 812 return Index == ~0u ? 2 : 0; 813 } 814 default: 815 return BaseT::getVectorInstrCost(Opcode, ValTy, Index); 816 } 817 } 818 819 /// Analyze if the results of inline asm are divergent. If \p Indices is empty, 820 /// this is analyzing the collective result of all output registers. Otherwise, 821 /// this is only querying a specific result index if this returns multiple 822 /// registers in a struct. 823 bool GCNTTIImpl::isInlineAsmSourceOfDivergence( 824 const CallInst *CI, ArrayRef<unsigned> Indices) const { 825 // TODO: Handle complex extract indices 826 if (Indices.size() > 1) 827 return true; 828 829 const DataLayout &DL = CI->getModule()->getDataLayout(); 830 const SIRegisterInfo *TRI = ST->getRegisterInfo(); 831 TargetLowering::AsmOperandInfoVector TargetConstraints = 832 TLI->ParseConstraints(DL, ST->getRegisterInfo(), *CI); 833 834 const int TargetOutputIdx = Indices.empty() ? -1 : Indices[0]; 835 836 int OutputIdx = 0; 837 for (auto &TC : TargetConstraints) { 838 if (TC.Type != InlineAsm::isOutput) 839 continue; 840 841 // Skip outputs we don't care about. 842 if (TargetOutputIdx != -1 && TargetOutputIdx != OutputIdx++) 843 continue; 844 845 TLI->ComputeConstraintToUse(TC, SDValue()); 846 847 const TargetRegisterClass *RC = TLI->getRegForInlineAsmConstraint( 848 TRI, TC.ConstraintCode, TC.ConstraintVT).second; 849 850 // For AGPR constraints null is returned on subtargets without AGPRs, so 851 // assume divergent for null. 852 if (!RC || !TRI->isSGPRClass(RC)) 853 return true; 854 } 855 856 return false; 857 } 858 859 /// \returns true if the new GPU divergence analysis is enabled. 860 bool GCNTTIImpl::useGPUDivergenceAnalysis() const { 861 return !UseLegacyDA; 862 } 863 864 /// \returns true if the result of the value could potentially be 865 /// different across workitems in a wavefront. 866 bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const { 867 if (const Argument *A = dyn_cast<Argument>(V)) 868 return !AMDGPU::isArgPassedInSGPR(A); 869 870 // Loads from the private and flat address spaces are divergent, because 871 // threads can execute the load instruction with the same inputs and get 872 // different results. 873 // 874 // All other loads are not divergent, because if threads issue loads with the 875 // same arguments, they will always get the same result. 876 if (const LoadInst *Load = dyn_cast<LoadInst>(V)) 877 return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS || 878 Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS; 879 880 // Atomics are divergent because they are executed sequentially: when an 881 // atomic operation refers to the same address in each thread, then each 882 // thread after the first sees the value written by the previous thread as 883 // original value. 884 if (isa<AtomicRMWInst>(V) || isa<AtomicCmpXchgInst>(V)) 885 return true; 886 887 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) 888 return AMDGPU::isIntrinsicSourceOfDivergence(Intrinsic->getIntrinsicID()); 889 890 // Assume all function calls are a source of divergence. 891 if (const CallInst *CI = dyn_cast<CallInst>(V)) { 892 if (CI->isInlineAsm()) 893 return isInlineAsmSourceOfDivergence(CI); 894 return true; 895 } 896 897 // Assume all function calls are a source of divergence. 898 if (isa<InvokeInst>(V)) 899 return true; 900 901 return false; 902 } 903 904 bool GCNTTIImpl::isAlwaysUniform(const Value *V) const { 905 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) { 906 switch (Intrinsic->getIntrinsicID()) { 907 default: 908 return false; 909 case Intrinsic::amdgcn_readfirstlane: 910 case Intrinsic::amdgcn_readlane: 911 case Intrinsic::amdgcn_icmp: 912 case Intrinsic::amdgcn_fcmp: 913 case Intrinsic::amdgcn_ballot: 914 case Intrinsic::amdgcn_if_break: 915 return true; 916 } 917 } 918 919 if (const CallInst *CI = dyn_cast<CallInst>(V)) { 920 if (CI->isInlineAsm()) 921 return !isInlineAsmSourceOfDivergence(CI); 922 return false; 923 } 924 925 const ExtractValueInst *ExtValue = dyn_cast<ExtractValueInst>(V); 926 if (!ExtValue) 927 return false; 928 929 const CallInst *CI = dyn_cast<CallInst>(ExtValue->getOperand(0)); 930 if (!CI) 931 return false; 932 933 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(CI)) { 934 switch (Intrinsic->getIntrinsicID()) { 935 default: 936 return false; 937 case Intrinsic::amdgcn_if: 938 case Intrinsic::amdgcn_else: { 939 ArrayRef<unsigned> Indices = ExtValue->getIndices(); 940 return Indices.size() == 1 && Indices[0] == 1; 941 } 942 } 943 } 944 945 // If we have inline asm returning mixed SGPR and VGPR results, we inferred 946 // divergent for the overall struct return. We need to override it in the 947 // case we're extracting an SGPR component here. 948 if (CI->isInlineAsm()) 949 return !isInlineAsmSourceOfDivergence(CI, ExtValue->getIndices()); 950 951 return false; 952 } 953 954 bool GCNTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes, 955 Intrinsic::ID IID) const { 956 switch (IID) { 957 case Intrinsic::amdgcn_atomic_inc: 958 case Intrinsic::amdgcn_atomic_dec: 959 case Intrinsic::amdgcn_ds_fadd: 960 case Intrinsic::amdgcn_ds_fmin: 961 case Intrinsic::amdgcn_ds_fmax: 962 case Intrinsic::amdgcn_is_shared: 963 case Intrinsic::amdgcn_is_private: 964 OpIndexes.push_back(0); 965 return true; 966 default: 967 return false; 968 } 969 } 970 971 Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, 972 Value *OldV, 973 Value *NewV) const { 974 auto IntrID = II->getIntrinsicID(); 975 switch (IntrID) { 976 case Intrinsic::amdgcn_atomic_inc: 977 case Intrinsic::amdgcn_atomic_dec: 978 case Intrinsic::amdgcn_ds_fadd: 979 case Intrinsic::amdgcn_ds_fmin: 980 case Intrinsic::amdgcn_ds_fmax: { 981 const ConstantInt *IsVolatile = cast<ConstantInt>(II->getArgOperand(4)); 982 if (!IsVolatile->isZero()) 983 return nullptr; 984 Module *M = II->getParent()->getParent()->getParent(); 985 Type *DestTy = II->getType(); 986 Type *SrcTy = NewV->getType(); 987 Function *NewDecl = 988 Intrinsic::getDeclaration(M, II->getIntrinsicID(), {DestTy, SrcTy}); 989 II->setArgOperand(0, NewV); 990 II->setCalledFunction(NewDecl); 991 return II; 992 } 993 case Intrinsic::amdgcn_is_shared: 994 case Intrinsic::amdgcn_is_private: { 995 unsigned TrueAS = IntrID == Intrinsic::amdgcn_is_shared ? 996 AMDGPUAS::LOCAL_ADDRESS : AMDGPUAS::PRIVATE_ADDRESS; 997 unsigned NewAS = NewV->getType()->getPointerAddressSpace(); 998 LLVMContext &Ctx = NewV->getType()->getContext(); 999 ConstantInt *NewVal = (TrueAS == NewAS) ? 1000 ConstantInt::getTrue(Ctx) : ConstantInt::getFalse(Ctx); 1001 return NewVal; 1002 } 1003 case Intrinsic::ptrmask: { 1004 unsigned OldAS = OldV->getType()->getPointerAddressSpace(); 1005 unsigned NewAS = NewV->getType()->getPointerAddressSpace(); 1006 Value *MaskOp = II->getArgOperand(1); 1007 Type *MaskTy = MaskOp->getType(); 1008 1009 bool DoTruncate = false; 1010 1011 const GCNTargetMachine &TM = 1012 static_cast<const GCNTargetMachine &>(getTLI()->getTargetMachine()); 1013 if (!TM.isNoopAddrSpaceCast(OldAS, NewAS)) { 1014 // All valid 64-bit to 32-bit casts work by chopping off the high 1015 // bits. Any masking only clearing the low bits will also apply in the new 1016 // address space. 1017 if (DL.getPointerSizeInBits(OldAS) != 64 || 1018 DL.getPointerSizeInBits(NewAS) != 32) 1019 return nullptr; 1020 1021 // TODO: Do we need to thread more context in here? 1022 KnownBits Known = computeKnownBits(MaskOp, DL, 0, nullptr, II); 1023 if (Known.countMinLeadingOnes() < 32) 1024 return nullptr; 1025 1026 DoTruncate = true; 1027 } 1028 1029 IRBuilder<> B(II); 1030 if (DoTruncate) { 1031 MaskTy = B.getInt32Ty(); 1032 MaskOp = B.CreateTrunc(MaskOp, MaskTy); 1033 } 1034 1035 return B.CreateIntrinsic(Intrinsic::ptrmask, {NewV->getType(), MaskTy}, 1036 {NewV, MaskOp}); 1037 } 1038 default: 1039 return nullptr; 1040 } 1041 } 1042 1043 InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, 1044 VectorType *VT, ArrayRef<int> Mask, 1045 int Index, VectorType *SubTp) { 1046 Kind = improveShuffleKindFromMask(Kind, Mask); 1047 if (ST->hasVOP3PInsts()) { 1048 if (cast<FixedVectorType>(VT)->getNumElements() == 2 && 1049 DL.getTypeSizeInBits(VT->getElementType()) == 16) { 1050 // With op_sel VOP3P instructions freely can access the low half or high 1051 // half of a register, so any swizzle is free. 1052 1053 switch (Kind) { 1054 case TTI::SK_Broadcast: 1055 case TTI::SK_Reverse: 1056 case TTI::SK_PermuteSingleSrc: 1057 return 0; 1058 default: 1059 break; 1060 } 1061 } 1062 } 1063 1064 return BaseT::getShuffleCost(Kind, VT, Mask, Index, SubTp); 1065 } 1066 1067 bool GCNTTIImpl::areInlineCompatible(const Function *Caller, 1068 const Function *Callee) const { 1069 const TargetMachine &TM = getTLI()->getTargetMachine(); 1070 const GCNSubtarget *CallerST 1071 = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Caller)); 1072 const GCNSubtarget *CalleeST 1073 = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Callee)); 1074 1075 const FeatureBitset &CallerBits = CallerST->getFeatureBits(); 1076 const FeatureBitset &CalleeBits = CalleeST->getFeatureBits(); 1077 1078 FeatureBitset RealCallerBits = CallerBits & ~InlineFeatureIgnoreList; 1079 FeatureBitset RealCalleeBits = CalleeBits & ~InlineFeatureIgnoreList; 1080 if ((RealCallerBits & RealCalleeBits) != RealCalleeBits) 1081 return false; 1082 1083 // FIXME: dx10_clamp can just take the caller setting, but there seems to be 1084 // no way to support merge for backend defined attributes. 1085 AMDGPU::SIModeRegisterDefaults CallerMode(*Caller); 1086 AMDGPU::SIModeRegisterDefaults CalleeMode(*Callee); 1087 if (!CallerMode.isInlineCompatible(CalleeMode)) 1088 return false; 1089 1090 if (Callee->hasFnAttribute(Attribute::AlwaysInline) || 1091 Callee->hasFnAttribute(Attribute::InlineHint)) 1092 return true; 1093 1094 // Hack to make compile times reasonable. 1095 if (InlineMaxBB) { 1096 // Single BB does not increase total BB amount. 1097 if (Callee->size() == 1) 1098 return true; 1099 size_t BBSize = Caller->size() + Callee->size() - 1; 1100 return BBSize <= InlineMaxBB; 1101 } 1102 1103 return true; 1104 } 1105 1106 unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const { 1107 // If we have a pointer to private array passed into a function 1108 // it will not be optimized out, leaving scratch usage. 1109 // Increase the inline threshold to allow inlining in this case. 1110 uint64_t AllocaSize = 0; 1111 SmallPtrSet<const AllocaInst *, 8> AIVisited; 1112 for (Value *PtrArg : CB->args()) { 1113 PointerType *Ty = dyn_cast<PointerType>(PtrArg->getType()); 1114 if (!Ty || (Ty->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS && 1115 Ty->getAddressSpace() != AMDGPUAS::FLAT_ADDRESS)) 1116 continue; 1117 1118 PtrArg = getUnderlyingObject(PtrArg); 1119 if (const AllocaInst *AI = dyn_cast<AllocaInst>(PtrArg)) { 1120 if (!AI->isStaticAlloca() || !AIVisited.insert(AI).second) 1121 continue; 1122 AllocaSize += DL.getTypeAllocSize(AI->getAllocatedType()); 1123 // If the amount of stack memory is excessive we will not be able 1124 // to get rid of the scratch anyway, bail out. 1125 if (AllocaSize > ArgAllocaCutoff) { 1126 AllocaSize = 0; 1127 break; 1128 } 1129 } 1130 } 1131 if (AllocaSize) 1132 return ArgAllocaCost; 1133 return 0; 1134 } 1135 1136 void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, 1137 TTI::UnrollingPreferences &UP, 1138 OptimizationRemarkEmitter *ORE) { 1139 CommonTTI.getUnrollingPreferences(L, SE, UP, ORE); 1140 } 1141 1142 void GCNTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, 1143 TTI::PeelingPreferences &PP) { 1144 CommonTTI.getPeelingPreferences(L, SE, PP); 1145 } 1146 1147 int GCNTTIImpl::get64BitInstrCost(TTI::TargetCostKind CostKind) const { 1148 return ST->hasFullRate64Ops() 1149 ? getFullRateInstrCost() 1150 : ST->hasHalfRate64Ops() ? getHalfRateInstrCost(CostKind) 1151 : getQuarterRateInstrCost(CostKind); 1152 } 1153