Lines Matching +full:function +full:- +full:off
1 //===-- AMDGPULowerBufferFatPointers.cpp ---------------------------=//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
15 // Address space 7 (the buffer fat pointer) is a 160-bit pointer that consists
16 // of a 128-bit buffer descriptor and a 32-bit offset into that descriptor.
24 // often encountered in the context of SPIR-V kernels).
26 // However, because of their non-power-of-2 size, these fat pointers cannot be
32 // and the offset part (the 32-bit integer) as separate arguments. In addition,
40 // %x.rsrc` and `i32 %x.off`, which will be combined into `{ptr addrspace(8),
41 // i32} %x = {%x.rsrc, %x.off}` if needed. Similarly, `vector<Nxp7>` becomes
67 // hardware does not include a 160-bit atomic.
84 // that do take such pointers have their basic blocks moved to a new function
97 // In the first pass through each function that is being lowered, the splitter
98 // inserts new instructions to implement the split-structures behavior, which is
126 // "non-uniform" resource argument.
129 // post-processes all encountered conditionals. Given a PHI node or select,
147 // `%z.off = select i1 %cond, ptr i32 %x.off, i32 %y.off`
153 // After conditionals have been cleaned up, the IR for each function is
157 // produces a resource-and-offset struct after type remapping) is
166 // 3. If a user of the original struct-valued result remains, the structure
167 // needed for the new types to work is constructed out of the newly-defined
187 // %y.off = add i32 %x.off, 1
189 // %x.off, ...)
191 // %x.rsrc, i32 %y.off, ...)
194 //===----------------------------------------------------------------------===//
228 #define DEBUG_TYPE "amdgpu-lower-buffer-fat-pointers"
284 if (PT->getAddressSpace() == AMDGPUAS::BUFFER_FAT_POINTER) { in remapTypeImpl()
289 auto *PT = dyn_cast<PointerType>(VT->getElementType()); in remapTypeImpl()
290 if (PT && PT->getAddressSpace() == AMDGPUAS::BUFFER_FAT_POINTER) { in remapTypeImpl()
295 // Whether the type is one that is structurally uniqued - that is, if it is in remapTypeImpl()
299 bool IsUniqued = !TyAsStruct || TyAsStruct->isLiteral(); in remapTypeImpl()
302 if (Ty->getNumContainedTypes() == 0 && IsUniqued) in remapTypeImpl()
307 StructType *Placeholder = StructType::create(Ty->getContext()); in remapTypeImpl()
312 SmallVector<Type *> ElementTypes(Ty->getNumContainedTypes(), nullptr); in remapTypeImpl()
313 for (unsigned int I = 0, E = Ty->getNumContainedTypes(); I < E; ++I) { in remapTypeImpl()
314 Type *OldElem = Ty->getContainedType(I); in remapTypeImpl()
325 return *Entry = ArrayType::get(ElementTypes[0], ArrTy->getNumElements()); in remapTypeImpl()
329 FnTy->isVarArg()); in remapTypeImpl()
332 if (STy->isOpaque()) in remapTypeImpl()
334 bool IsPacked = STy->isPacked(); in remapTypeImpl()
336 return *Entry = StructType::get(Ty->getContext(), ElementTypes, IsPacked); in remapTypeImpl()
337 SmallString<16> Name(STy->getName()); in remapTypeImpl()
338 STy->setName(""); in remapTypeImpl()
342 Placeholder->setBody(ElementTypes, IsPacked); in remapTypeImpl()
343 Placeholder->setName(Name); in remapTypeImpl()
346 return *Entry = StructType::create(Ty->getContext(), ElementTypes, Name, in remapTypeImpl()
358 LLVMContext &Ctx = PT->getContext(); in remapScalar()
364 ElementCount EC = VT->getElementCount(); in remapVector()
365 LLVMContext &Ctx = VT->getContext(); in remapVector()
373 if (auto *PT = dyn_cast<PointerType>(Ty->getScalarType())) in isBufferFatPtrOrVector()
374 return PT->getAddressSpace() == AMDGPUAS::BUFFER_FAT_POINTER; in isBufferFatPtrOrVector()
384 if (!ST->isLiteral() || ST->getNumElements() != 2) in isSplitFatPtr()
387 dyn_cast<PointerType>(ST->getElementType(0)->getScalarType()); in isSplitFatPtr()
389 dyn_cast<IntegerType>(ST->getElementType(1)->getScalarType()); in isSplitFatPtr()
391 MaybeRsrc->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE && in isSplitFatPtr()
392 MaybeOff->getBitWidth() == BufferOffsetWidth; in isSplitFatPtr()
397 Type *T = C->getType(); in isBufferFatPtrConst()
398 return isBufferFatPtrOrVector(T) || any_of(C->operands(), [](const Use &U) { in isBufferFatPtrConst()
399 return isBufferFatPtrOrVector(U.get()->getType()); in isBufferFatPtrConst()
431 bool processFunction(Function &F);
447 return Find->second; in fatPtrsToInts()
453 if (From->getNumContainedTypes() == 0) in fatPtrsToInts()
458 Type *FromPart = AT->getArrayElementType(); in fatPtrsToInts()
459 Type *ToPart = cast<ArrayType>(To)->getElementType(); in fatPtrsToInts()
460 for (uint64_t I = 0, E = AT->getArrayNumElements(); I < E; ++I) { in fatPtrsToInts()
468 enumerate(From->subtypes(), To->subtypes())) { in fatPtrsToInts()
487 if (From->getNumContainedTypes() == 0) in intsToFatPtrs()
492 Type *FromPart = AT->getArrayElementType(); in intsToFatPtrs()
493 Type *ToPart = cast<ArrayType>(To)->getElementType(); in intsToFatPtrs()
494 for (uint64_t I = 0, E = AT->getArrayNumElements(); I < E; ++I) { in intsToFatPtrs()
502 enumerate(From->subtypes(), To->subtypes())) { in intsToFatPtrs()
512 bool StoreFatPtrsAsIntsVisitor::processFunction(Function &F) { in processFunction()
525 Type *NewTy = TypeMap->remapType(Ty); in visitAllocaInst()
534 Type *NewTy = TypeMap->remapType(Ty); in visitGetElementPtrInst()
540 I.setResultElementType(TypeMap->remapType(I.getResultElementType())); in visitGetElementPtrInst()
546 Type *IntTy = TypeMap->remapType(Ty); in visitLoadInst()
552 NLI->mutateType(IntTy); in visitLoadInst()
555 NLI->takeName(&LI); in visitLoadInst()
557 Value *CastBack = intsToFatPtrs(NLI, IntTy, Ty, NLI->getName()); in visitLoadInst()
565 Type *Ty = V->getType(); in visitStoreInst()
566 Type *IntTy = TypeMap->remapType(Ty); in visitStoreInst()
571 Value *IntV = fatPtrsToInts(V, Ty, IntTy, V->getName()); in visitStoreInst()
573 Dbg->setValue(IntV); in visitStoreInst()
583 assert(isSplitFatPtr(C->getType()) && "Not a split fat buffer pointer"); in splitLoweredFatBufferConst()
584 return std::make_pair(C->getAggregateElement(0u), C->getAggregateElement(1u)); in splitLoweredFatBufferConst()
613 Type *SrcTy = C->getType(); in materializeBufferFatPtrConst()
614 auto *NewTy = dyn_cast<StructType>(TypeMap->remapType(SrcTy)); in materializeBufferFatPtrConst()
615 if (C->isNullValue()) in materializeBufferFatPtrConst()
619 {PoisonValue::get(NewTy->getElementType(0)), in materializeBufferFatPtrConst()
620 PoisonValue::get(NewTy->getElementType(1))}); in materializeBufferFatPtrConst()
624 {UndefValue::get(NewTy->getElementType(0)), in materializeBufferFatPtrConst()
625 UndefValue::get(NewTy->getElementType(1))}); in materializeBufferFatPtrConst()
629 if (Constant *S = VC->getSplatValue()) { in materializeBufferFatPtrConst()
633 auto [Rsrc, Off] = splitLoweredFatBufferConst(NewS); in materializeBufferFatPtrConst()
634 auto EC = VC->getType()->getElementCount(); in materializeBufferFatPtrConst()
636 ConstantVector::getSplat(EC, Off)}); in materializeBufferFatPtrConst()
640 for (Value *Op : VC->operand_values()) { in materializeBufferFatPtrConst()
644 auto [Rsrc, Off] = splitLoweredFatBufferConst(NewOp); in materializeBufferFatPtrConst()
646 Offs.push_back(Off); in materializeBufferFatPtrConst()
687 // set, as that replacement will be handled in a post-visit step. However,
719 // of `Roots` and `Roots - Seen` contains one element, the resource part of
730 // Also, kill the temporary extractvalue operations produced by the two-stage
745 void processFunction(Function &F);
778 DestI->copyMetadata(*SrcI); in copyMetadata()
782 assert(isSplitFatPtr(V->getType()) && "it's not meaningful to get the parts " in getPtrParts()
790 auto [Rsrc, Off] = splitLoweredFatBufferConst(C); in getPtrParts()
791 return {*RsrcEntry = Rsrc, *OffEntry = Off}; in getPtrParts()
797 auto [Rsrc, Off] = visit(*I); in getPtrParts()
798 if (Rsrc && Off) in getPtrParts()
799 return {*RsrcEntry = Rsrc, *OffEntry = Off}; in getPtrParts()
802 IRB.SetInsertPoint(*I->getInsertionPointAfterDef()); in getPtrParts()
803 IRB.SetCurrentDebugLocation(I->getDebugLoc()); in getPtrParts()
805 IRB.SetInsertPointPastAllocas(A->getParent()); in getPtrParts()
808 Value *Rsrc = IRB.CreateExtractValue(V, 0, V->getName() + ".rsrc"); in getPtrParts()
809 Value *Off = IRB.CreateExtractValue(V, 1, V->getName() + ".off"); in getPtrParts() local
810 return {*RsrcEntry = Rsrc, *OffEntry = Off}; in getPtrParts()
823 V = GEP->getPointerOperand(); in rsrcPartRoot()
825 V = ASC->getPointerOperand(); in rsrcPartRoot()
835 for (Value *In : PHI->incoming_values()) { in getPossibleRsrcRoots()
844 Value *TrueVal = rsrcPartRoot(SI->getTrueValue()); in getPossibleRsrcRoots()
845 Value *FalseVal = rsrcPartRoot(SI->getFalseValue()); in getPossibleRsrcRoots()
864 Value *Off = OffParts[I]; in processConditionals() local
865 assert(Rsrc && Off && "must have visited conditionals by now"); in processConditionals()
870 MaybeRsrc = MaybeFoundRsrc->second; in processConditionals()
896 if (isSplitFatPtr(RootVal->getType())) in processConditionals()
906 StructType *PHITy = cast<StructType>(PHI->getType()); in processConditionals()
907 IRB.SetInsertPoint(*PHI->getInsertionPointAfterDef()); in processConditionals()
908 IRB.SetCurrentDebugLocation(PHI->getDebugLoc()); in processConditionals()
912 Type *RsrcTy = PHITy->getElementType(0); in processConditionals()
913 auto *RsrcPHI = IRB.CreatePHI(RsrcTy, PHI->getNumIncomingValues()); in processConditionals()
914 RsrcPHI->takeName(Rsrc); in processConditionals()
915 for (auto [V, BB] : llvm::zip(PHI->incoming_values(), PHI->blocks())) { in processConditionals()
917 RsrcPHI->addIncoming(VRsrc, BB); in processConditionals()
923 Type *OffTy = PHITy->getElementType(1); in processConditionals()
924 auto *NewOff = IRB.CreatePHI(OffTy, PHI->getNumIncomingValues()); in processConditionals()
925 NewOff->takeName(Off); in processConditionals()
926 for (auto [V, BB] : llvm::zip(PHI->incoming_values(), PHI->blocks())) { in processConditionals()
929 NewOff->addIncoming(VOff, BB); in processConditionals()
938 ConditionalTemps.push_back(cast<Instruction>(Off)); in processConditionals()
939 Rsrc->replaceAllUsesWith(NewRsrc); in processConditionals()
940 Off->replaceAllUsesWith(NewOff); in processConditionals()
942 // Save on recomputing the cycle traversals in known-root cases. in processConditionals()
949 Rsrc->replaceAllUsesWith(*MaybeRsrc); in processConditionals()
962 I->eraseFromParent(); in killAndReplaceSplitInstructions()
972 auto &DL = I->getDataLayout(); in killAndReplaceSplitInstructions()
973 assert(isSplitFatPtr(I->getType()) && in killAndReplaceSplitInstructions()
975 auto *OffDbg = cast<DbgValueInst>(Dbg->clone()); in killAndReplaceSplitInstructions()
977 auto [Rsrc, Off] = getPtrParts(I); in killAndReplaceSplitInstructions()
979 int64_t RsrcSz = DL.getTypeSizeInBits(Rsrc->getType()); in killAndReplaceSplitInstructions()
980 int64_t OffSz = DL.getTypeSizeInBits(Off->getType()); in killAndReplaceSplitInstructions()
983 DIExpression::createFragmentExpression(Dbg->getExpression(), 0, in killAndReplaceSplitInstructions()
986 DIExpression::createFragmentExpression(Dbg->getExpression(), RsrcSz, in killAndReplaceSplitInstructions()
989 OffDbg->setExpression(*OffExpr); in killAndReplaceSplitInstructions()
990 OffDbg->replaceVariableLocationOp(I, Off); in killAndReplaceSplitInstructions()
993 OffDbg->deleteValue(); in killAndReplaceSplitInstructions()
996 Dbg->setExpression(*RsrcExpr); in killAndReplaceSplitInstructions()
997 Dbg->replaceVariableLocationOp(I, Rsrc); in killAndReplaceSplitInstructions()
999 Dbg->replaceVariableLocationOp(I, UndefValue::get(I->getType())); in killAndReplaceSplitInstructions()
1003 Value *Poison = PoisonValue::get(I->getType()); in killAndReplaceSplitInstructions()
1004 I->replaceUsesWithIf(Poison, [&](const Use &U) -> bool { in killAndReplaceSplitInstructions()
1010 if (I->use_empty()) { in killAndReplaceSplitInstructions()
1011 I->eraseFromParent(); in killAndReplaceSplitInstructions()
1014 IRB.SetInsertPoint(*I->getInsertionPointAfterDef()); in killAndReplaceSplitInstructions()
1015 IRB.SetCurrentDebugLocation(I->getDebugLoc()); in killAndReplaceSplitInstructions()
1016 auto [Rsrc, Off] = getPtrParts(I); in killAndReplaceSplitInstructions()
1017 Value *Struct = PoisonValue::get(I->getType()); in killAndReplaceSplitInstructions()
1019 Struct = IRB.CreateInsertValue(Struct, Off, 1); in killAndReplaceSplitInstructions()
1021 Struct->takeName(I); in killAndReplaceSplitInstructions()
1022 I->replaceAllUsesWith(Struct); in killAndReplaceSplitInstructions()
1023 I->eraseFromParent(); in killAndReplaceSplitInstructions()
1028 LLVMContext &Ctx = Intr->getContext(); in setAlign()
1029 Intr->addParamAttr(RsrcArgIdx, Attribute::getWithAlignment(Ctx, A)); in setAlign()
1064 auto [Rsrc, Off] = getPtrParts(Ptr); in handleMemoryInst()
1069 Args.push_back(Off); in handleMemoryInst()
1078 (isa<LoadInst>(I) && I->getMetadata(LLVMContext::MD_invariant_load)); in handleMemoryInst()
1079 bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal); in handleMemoryInst()
1080 // Atomic loads and stores need glc, atomic read-modify-write doesn't. in handleMemoryInst()
1087 if (isa<LoadInst>(I) && ST->getGeneration() == AMDGPUSubtarget::GFX10) in handleMemoryInst()
1101 switch (RMW->getOperation()) { in handleMemoryInst()
1163 Call->takeName(I); in handleMemoryInst()
1169 I->replaceAllUsesWith(Call); in handleMemoryInst()
1190 handleMemoryInst(&SI, Arg, SI.getPointerOperand(), Arg->getType(), in visitStoreInst()
1197 if (!isSplitFatPtr(AI.getPointerOperand()->getType())) in visitAtomicRMWInst()
1200 handleMemoryInst(&AI, Arg, AI.getPointerOperand(), Arg->getType(), in visitAtomicRMWInst()
1210 if (!isSplitFatPtr(Ptr->getType())) in visitAtomicCmpXchgInst()
1214 Type *Ty = AI.getNewValOperand()->getType(); in visitAtomicCmpXchgInst()
1219 auto [Rsrc, Off] = getPtrParts(Ptr); in visitAtomicCmpXchgInst()
1230 Off, IRB.getInt32(0), IRB.getInt32(Aux)}); in visitAtomicCmpXchgInst()
1233 Call->takeName(&AI); in visitAtomicCmpXchgInst()
1250 if (!isSplitFatPtr(Ptr->getType())) in visitGetElementPtrInst()
1254 auto [Rsrc, Off] = getPtrParts(Ptr); in visitGetElementPtrInst()
1261 if (auto *VT = dyn_cast<VectorType>(Off->getType())) in visitGetElementPtrInst()
1262 FatPtrTy = VectorType::get(FatPtrTy, VT->getElementCount()); in visitGetElementPtrInst()
1265 GEP.mutateType(Ptr->getType()); in visitGetElementPtrInst()
1266 if (match(OffAccum, m_Zero())) { // Constant-zero offset in visitGetElementPtrInst()
1268 return {Rsrc, Off}; in visitGetElementPtrInst()
1273 HasNonNegativeOff = !CI->isNegative(); in visitGetElementPtrInst()
1276 if (match(Off, m_Zero())) { in visitGetElementPtrInst()
1279 NewOff = IRB.CreateAdd(Off, OffAccum, "", in visitGetElementPtrInst()
1284 NewOff->takeName(&GEP); in visitGetElementPtrInst()
1291 if (!isSplitFatPtr(Ptr->getType())) in visitPtrToIntInst()
1296 unsigned Width = ResTy->getScalarSizeInBits(); in visitPtrToIntInst()
1298 auto [Rsrc, Off] = getPtrParts(Ptr); in visitPtrToIntInst()
1304 Res = IRB.CreateIntCast(Off, ResTy, /*isSigned=*/false, in visitPtrToIntInst()
1305 PI.getName() + ".off"); in visitPtrToIntInst()
1312 Value *OffCast = IRB.CreateIntCast(Off, ResTy, /*isSigned=*/false, in visitPtrToIntInst()
1313 PI.getName() + ".off"); in visitPtrToIntInst()
1318 Res->takeName(&PI); in visitPtrToIntInst()
1331 Type *IntTy = Int->getType(); in visitIntToPtrInst()
1332 Type *RsrcIntTy = IntTy->getWithNewBitWidth(RsrcPtrWidth); in visitIntToPtrInst()
1333 unsigned Width = IntTy->getScalarSizeInBits(); in visitIntToPtrInst()
1336 Type *RsrcTy = RetTy->getElementType(0); in visitIntToPtrInst()
1337 Type *OffTy = RetTy->getElementType(1); in visitIntToPtrInst()
1343 Value *Off = in visitIntToPtrInst() local
1344 IRB.CreateIntCast(Int, OffTy, /*IsSigned=*/false, IP.getName() + ".off"); in visitIntToPtrInst()
1348 return {Rsrc, Off}; in visitIntToPtrInst()
1356 // No-op casts preserve parts in visitAddrSpaceCastInst()
1357 if (In->getType() == I.getType()) { in visitAddrSpaceCastInst()
1358 auto [Rsrc, Off] = getPtrParts(In); in visitAddrSpaceCastInst()
1360 return {Rsrc, Off}; in visitAddrSpaceCastInst()
1365 Type *OffTy = cast<StructType>(I.getType())->getElementType(1); in visitAddrSpaceCastInst()
1373 if (!isSplitFatPtr(Lhs->getType())) in visitICmpInst()
1386 Value *OffCmp = IRB.CreateICmp(Pred, LhsOff, RhsOff, Cmp.getName() + ".off"); in visitICmpInst()
1395 Res->takeName(&Cmp); in visitICmpInst()
1405 auto [Rsrc, Off] = getPtrParts(I.getOperand(0)); in visitFreezeInst()
1409 Value *OffRes = IRB.CreateFreeze(Off, I.getName() + ".off"); in visitFreezeInst()
1421 auto [Rsrc, Off] = getPtrParts(Vec); in visitExtractElementInst()
1425 Value *OffRes = IRB.CreateExtractElement(Off, Idx, I.getName() + ".off"); in visitExtractElementInst()
1447 IRB.CreateInsertElement(VecOff, ElemOff, Idx, I.getName() + ".off"); in visitInsertElementInst()
1469 IRB.CreateShuffleVector(V1Off, V2Off, Mask, I.getName() + ".off"); in visitShuffleVectorInst()
1479 // Phi nodes will be handled in post-processing after we've visited every in visitPHINode()
1485 Value *TmpOff = IRB.CreateExtractValue(&PHI, 1, PHI.getName() + ".off"); in visitPHINode()
1507 IRB.CreateSelect(Cond, TrueOff, FalseOff, SI.getName() + ".off", &SI); in visitSelectInst()
1537 if (!isSplitFatPtr(Ptr->getType())) in visitIntrinsicInst()
1541 auto [Rsrc, Off] = getPtrParts(Ptr); in visitIntrinsicInst()
1542 if (Mask->getType() != Off->getType()) in visitIntrinsicInst()
1545 Value *OffRes = IRB.CreateAnd(Off, Mask, I.getName() + ".off"); in visitIntrinsicInst()
1550 // Pointer annotation intrinsics that, given their object-wide nature in visitIntrinsicInst()
1554 if (!isSplitFatPtr(Ptr->getType())) in visitIntrinsicInst()
1557 auto [Rsrc, Off] = getPtrParts(Ptr); in visitIntrinsicInst()
1561 NewRsrc->takeName(&I); in visitIntrinsicInst()
1568 if (!isSplitFatPtr(RealPtr->getType())) in visitIntrinsicInst()
1574 Value *NewRsrc = IRB.CreateIntrinsic(IID, {RealRsrc->getType()}, in visitIntrinsicInst()
1577 NewRsrc->takeName(&I); in visitIntrinsicInst()
1585 if (!isSplitFatPtr(Ptr->getType())) in visitIntrinsicInst()
1588 auto [Rsrc, Off] = getPtrParts(Ptr); in visitIntrinsicInst()
1589 Value *NewRsrc = IRB.CreateIntrinsic(IID, {Rsrc->getType()}, {Rsrc}); in visitIntrinsicInst()
1591 NewRsrc->takeName(&I); in visitIntrinsicInst()
1593 return {NewRsrc, Off}; in visitIntrinsicInst()
1599 void SplitPtrStructs::processFunction(Function &F) { in processFunction()
1600 ST = &TM->getSubtarget<GCNSubtarget>(F); in processFunction()
1602 LLVM_DEBUG(dbgs() << "Splitting pointer structs in function: " << F.getName() in processFunction()
1607 auto [Rsrc, Off] = visit(I); in processFunction()
1608 assert(((Rsrc && Off) || (!Rsrc && !Off)) && in processFunction()
1612 if (Off) in processFunction()
1613 OffParts[I] = Off; in processFunction()
1644 /// which means we'll need to perform rewrites on this function. As a side
1646 static bool containsBufferFatPointers(const Function &F, in containsBufferFatPointers()
1651 HasFatPointers |= (I.getType() != TypeMap->remapType(I.getType())); in containsBufferFatPointers()
1655 static bool hasFatPointerInterface(const Function &F, in hasFatPointerInterface()
1658 return Ty != TypeMap->remapType(Ty); in hasFatPointerInterface()
1661 /// Move the body of `OldF` into a new function, returning it.
1662 static Function *moveFunctionAdaptingType(Function *OldF, FunctionType *NewTy, in moveFunctionAdaptingType()
1664 bool IsIntrinsic = OldF->isIntrinsic(); in moveFunctionAdaptingType()
1665 Function *NewF = in moveFunctionAdaptingType()
1666 Function::Create(NewTy, OldF->getLinkage(), OldF->getAddressSpace()); in moveFunctionAdaptingType()
1667 NewF->IsNewDbgInfoFormat = OldF->IsNewDbgInfoFormat; in moveFunctionAdaptingType()
1668 NewF->copyAttributesFrom(OldF); in moveFunctionAdaptingType()
1669 NewF->copyMetadata(OldF, 0); in moveFunctionAdaptingType()
1670 NewF->takeName(OldF); in moveFunctionAdaptingType()
1671 NewF->updateAfterNameChange(); in moveFunctionAdaptingType()
1672 NewF->setDLLStorageClass(OldF->getDLLStorageClass()); in moveFunctionAdaptingType()
1673 OldF->getParent()->getFunctionList().insertAfter(OldF->getIterator(), NewF); in moveFunctionAdaptingType()
1675 while (!OldF->empty()) { in moveFunctionAdaptingType()
1676 BasicBlock *BB = &OldF->front(); in moveFunctionAdaptingType()
1677 BB->removeFromParent(); in moveFunctionAdaptingType()
1678 BB->insertInto(NewF); in moveFunctionAdaptingType()
1694 AttributeList OldAttrs = OldF->getAttributes(); in moveFunctionAdaptingType()
1696 for (auto [I, OldArg, NewArg] : enumerate(OldF->args(), NewF->args())) { in moveFunctionAdaptingType()
1708 ArgAttr = ArgAttr.removeAttributes(NewF->getContext(), PtrOnlyAttrs); in moveFunctionAdaptingType()
1712 if (OldF->getReturnType() != NewF->getReturnType() && !IsIntrinsic) in moveFunctionAdaptingType()
1713 RetAttrs = RetAttrs.removeAttributes(NewF->getContext(), PtrOnlyAttrs); in moveFunctionAdaptingType()
1714 NewF->setAttributes(AttributeList::get( in moveFunctionAdaptingType()
1715 NewF->getContext(), OldAttrs.getFnAttrs(), RetAttrs, ArgAttrs)); in moveFunctionAdaptingType()
1719 static void makeCloneInPraceMap(Function *F, ValueToValueMapTy &CloneMap) { in makeCloneInPraceMap()
1720 for (Argument &A : F->args()) in makeCloneInPraceMap()
1733 // The second element of the pair indicates whether the function has to have in run()
1735 SmallVector<std::pair<Function *, bool>> NeedsRemap; in run()
1751 // Collect all constant exprs and aggregates referenced by any function. in run()
1753 for (Function &F : M.functions()) in run()
1766 if (isBufferFatPtrOrVector(C->getType())) in run()
1768 for (Value *Op : C->operands()) in run()
1781 for (Function &F : M.functions()) { in run()
1791 SmallVector<Function *> NeedsPostProcess; in run()
1792 SmallVector<Function *> Intrinsics; in run()
1799 Function *NewF = F; in run()
1802 F, cast<FunctionType>(StructTM.remapType(F->getFunctionType())), in run()
1807 if (NewF->isIntrinsic()) in run()
1812 F->replaceAllUsesWith(NewF); in run()
1813 F->eraseFromParent(); in run()
1822 for (Function *F : NeedsPostProcess) in run()
1824 for (Function *F : Intrinsics) { in run()
1825 if (isRemovablePointerIntrinsic(F->getIntrinsicID())) { in run()
1826 F->eraseFromParent(); in run()
1828 std::optional<Function *> NewF = Intrinsic::remangleIntrinsicFunction(F); in run()
1830 F->replaceAllUsesWith(*NewF); in run()