Lines Matching +full:current +full:- +full:rotate
1 //===-- SPIRVPrepareFunctions.cpp - modify function signatures --*- C++ -*-===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
12 // restore correct SPIR-V types of function arguments and return values.
14 // generated functions (as the Khronos LLVM/SPIR-V Translator does).
16 // NOTE: this pass is a module-level one due to the necessity to modify
19 //===----------------------------------------------------------------------===//
68 INITIALIZE_PASS(SPIRVPrepareFunctions, "prepare-functions",
72 Function *IntrinsicFunc = II->getCalledFunction(); in lowerLLVMIntrinsicName()
74 std::string FuncName = IntrinsicFunc->getName().str(); in lowerLLVMIntrinsicName()
84 Function *F = M->getFunction(Name); in getOrCreateFunction()
85 if (F && F->getFunctionType() == FT) in getOrCreateFunction()
89 NewF->setDSOLocal(F->isDSOLocal()); in getOrCreateFunction()
90 NewF->setCallingConv(CallingConv::SPIR_FUNC); in getOrCreateFunction()
100 if (isa<Constant>(MSI->getValue()) && isa<ConstantInt>(MSI->getLength())) in lowerIntrinsicToFunction()
103 Module *M = Intrinsic->getModule(); in lowerIntrinsicToFunction()
105 if (Intrinsic->isVolatile()) in lowerIntrinsicToFunction()
108 Function *F = M->getFunction(FuncName); in lowerIntrinsicToFunction()
110 Intrinsic->setCalledFunction(F); in lowerIntrinsicToFunction()
115 M->getOrInsertFunction(FuncName, Intrinsic->getFunctionType()); in lowerIntrinsicToFunction()
116 auto IntrinsicID = Intrinsic->getIntrinsicID(); in lowerIntrinsicToFunction()
117 Intrinsic->setCalledFunction(FC); in lowerIntrinsicToFunction()
125 Argument *Dest = F->getArg(0); in lowerIntrinsicToFunction()
126 Argument *Val = F->getArg(1); in lowerIntrinsicToFunction()
127 Argument *Len = F->getArg(2); in lowerIntrinsicToFunction()
128 Argument *IsVolatile = F->getArg(3); in lowerIntrinsicToFunction()
129 Dest->setName("dest"); in lowerIntrinsicToFunction()
130 Val->setName("val"); in lowerIntrinsicToFunction()
131 Len->setName("len"); in lowerIntrinsicToFunction()
132 IsVolatile->setName("isvolatile"); in lowerIntrinsicToFunction()
133 BasicBlock *EntryBB = BasicBlock::Create(M->getContext(), "entry", F); in lowerIntrinsicToFunction()
135 auto *MemSet = IRB.CreateMemSet(Dest, Val, Len, MSI->getDestAlign(), in lowerIntrinsicToFunction()
136 MSI->isVolatile()); in lowerIntrinsicToFunction()
139 MemSet->eraseFromParent(); in lowerIntrinsicToFunction()
143 BasicBlock *EntryBB = BasicBlock::Create(M->getContext(), "entry", F); in lowerIntrinsicToFunction()
145 auto *BSwap = IRB.CreateIntrinsic(Intrinsic::bswap, Intrinsic->getType(), in lowerIntrinsicToFunction()
146 F->getArg(0)); in lowerIntrinsicToFunction()
148 IntrinsicLowering IL(M->getDataLayout()); in lowerIntrinsicToFunction()
160 AnnoVal = Ref->getOperand(0); in getAnnotation()
162 OptAnnoVal = Ref->getOperand(0); in getAnnotation()
173 C && C->getNumOperands()) { in getAnnotation()
174 Value *MaybeStruct = C->getOperand(0); in getAnnotation()
176 for (unsigned I = 0, E = Struct->getNumOperands(); I != E; ++I) { in getAnnotation()
177 if (auto *CInt = dyn_cast<ConstantInt>(Struct->getOperand(I))) in getAnnotation()
179 std::to_string(CInt->getType()->getIntegerBitWidth() == 1 in getAnnotation()
180 ? CInt->getZExtValue() in getAnnotation()
181 : CInt->getSExtValue()); in getAnnotation()
185 for (unsigned I = 0, E = Struct->getType()->getStructNumElements(); in getAnnotation()
209 if (It->position() != Pos) in parseAnnotation()
211 Pos = It->position() + It->length(); in parseAnnotation()
220 Item = Item.substr(1, Item.length() - 2); in parseAnnotation()
249 LLVMContext &Ctx = II->getContext(); in lowerPtrAnnotation()
254 if (auto *BI = dyn_cast<BitCastInst>(II->getArgOperand(0))) in lowerPtrAnnotation()
255 PtrArg = BI->getOperand(0); in lowerPtrAnnotation()
257 PtrArg = II->getOperand(0); in lowerPtrAnnotation()
259 getAnnotation(II->getArgOperand(1), in lowerPtrAnnotation()
260 4 < II->arg_size() ? II->getArgOperand(4) : nullptr); in lowerPtrAnnotation()
276 IRBuilder<> IRB(II->getParent()); in lowerPtrAnnotation()
279 Intrinsic::spv_assign_decoration, {PtrArg->getType()}, in lowerPtrAnnotation()
281 II->replaceAllUsesWith(II->getOperand(0)); in lowerPtrAnnotation()
285 // Get a separate function - otherwise, we'd have to rework the CFG of the in lowerFunnelShifts()
286 // current one. Then simply replace the intrinsic uses with a call to the new in lowerFunnelShifts()
289 Module *M = FSHIntrinsic->getModule(); in lowerFunnelShifts()
290 FunctionType *FSHFuncTy = FSHIntrinsic->getFunctionType(); in lowerFunnelShifts()
291 Type *FSHRetTy = FSHFuncTy->getReturnType(); in lowerFunnelShifts()
294 getOrCreateFunction(M, FSHRetTy, FSHFuncTy->params(), FuncName); in lowerFunnelShifts()
296 if (!FSHFunc->empty()) { in lowerFunnelShifts()
297 FSHIntrinsic->setCalledFunction(FSHFunc); in lowerFunnelShifts()
300 BasicBlock *RotateBB = BasicBlock::Create(M->getContext(), "rotate", FSHFunc); in lowerFunnelShifts()
302 Type *Ty = FSHFunc->getReturnType(); in lowerFunnelShifts()
303 // Build the actual funnel shift rotate logic. in lowerFunnelShifts()
307 Type *IntTy = VectorTy ? VectorTy->getElementType() : Ty; in lowerFunnelShifts()
308 unsigned BitWidth = IntTy->getIntegerBitWidth(); in lowerFunnelShifts()
312 ? IRB.CreateVectorSplat(VectorTy->getNumElements(), BitWidthConstant) in lowerFunnelShifts()
315 IRB.CreateURem(/*Rotate*/ FSHFunc->getArg(2), BitWidthForInsts); in lowerFunnelShifts()
317 if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) { in lowerFunnelShifts()
318 // Shift the less significant number right, the "rotate" number of bits in lowerFunnelShifts()
319 // will be 0-filled on the left as a result of this regular shift. in lowerFunnelShifts()
320 FirstShift = IRB.CreateLShr(FSHFunc->getArg(1), RotateModVal); in lowerFunnelShifts()
322 // Shift the more significant number left, the "rotate" number of bits in lowerFunnelShifts()
323 // will be 0-filled on the right as a result of this regular shift. in lowerFunnelShifts()
324 FirstShift = IRB.CreateShl(FSHFunc->getArg(0), RotateModVal); in lowerFunnelShifts()
326 // We want the "rotate" number of the more significant int's LSBs (MSBs) to in lowerFunnelShifts()
328 // Therefore, subtract the "rotate" number from the integer bitsize... in lowerFunnelShifts()
330 if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) { in lowerFunnelShifts()
331 // ...and left-shift the more significant int by this number, zero-filling in lowerFunnelShifts()
333 SecShift = IRB.CreateShl(FSHFunc->getArg(0), SubRotateVal); in lowerFunnelShifts()
335 // ...and right-shift the less significant int by this number, zero-filling in lowerFunnelShifts()
337 SecShift = IRB.CreateLShr(FSHFunc->getArg(1), SubRotateVal); in lowerFunnelShifts()
342 FSHIntrinsic->setCalledFunction(FSHFunc); in lowerFunnelShifts()
347 if (!UMulFunc->empty()) in buildUMulWithOverflowFunc()
350 BasicBlock *EntryBB = BasicBlock::Create(UMulFunc->getParent()->getContext(), in buildUMulWithOverflowFunc()
357 Value *Mul = IRB.CreateNUWMul(UMulFunc->getArg(0), UMulFunc->getArg(1)); in buildUMulWithOverflowFunc()
358 Value *Div = IRB.CreateUDiv(Mul, UMulFunc->getArg(0)); in buildUMulWithOverflowFunc()
359 Value *Overflow = IRB.CreateICmpNE(UMulFunc->getArg(0), Div); in buildUMulWithOverflowFunc()
363 Type *StructTy = UMulFunc->getReturnType(); in buildUMulWithOverflowFunc()
372 // Otherwise we should lower the intrinsic to the corresponding SPIR-V in lowerExpectAssume()
377 // We need to lower this into a builtin and then the builtin into a SPIR-V in lowerExpectAssume()
379 if (II->getIntrinsicID() == Intrinsic::assume) { in lowerExpectAssume()
381 II->getModule(), Intrinsic::SPVIntrinsics::spv_assume); in lowerExpectAssume()
382 II->setCalledFunction(F); in lowerExpectAssume()
383 } else if (II->getIntrinsicID() == Intrinsic::expect) { in lowerExpectAssume()
385 II->getModule(), Intrinsic::SPVIntrinsics::spv_expect, in lowerExpectAssume()
386 {II->getOperand(0)->getType()}); in lowerExpectAssume()
387 II->setCalledFunction(F); in lowerExpectAssume()
399 F = Intrinsic::getDeclaration(II->getModule(), NewID); in toSpvOverloadedIntrinsic()
403 Tys.push_back(II->getOperand(OpNo)->getType()); in toSpvOverloadedIntrinsic()
404 F = Intrinsic::getDeclaration(II->getModule(), NewID, Tys); in toSpvOverloadedIntrinsic()
406 II->setCalledFunction(F); in toSpvOverloadedIntrinsic()
411 // Get a separate function - otherwise, we'd have to rework the CFG of the in lowerUMulWithOverflow()
412 // current one. Then simply replace the intrinsic uses with a call to the new in lowerUMulWithOverflow()
414 Module *M = UMulIntrinsic->getModule(); in lowerUMulWithOverflow()
415 FunctionType *UMulFuncTy = UMulIntrinsic->getFunctionType(); in lowerUMulWithOverflow()
416 Type *FSHLRetTy = UMulFuncTy->getReturnType(); in lowerUMulWithOverflow()
419 getOrCreateFunction(M, FSHLRetTy, UMulFuncTy->params(), FuncName); in lowerUMulWithOverflow()
421 UMulIntrinsic->setCalledFunction(UMulFunc); in lowerUMulWithOverflow()
424 // Substitutes calls to LLVM intrinsics with either calls to SPIR-V intrinsics
433 Function *CF = Call->getCalledFunction(); in substituteIntrinsicCalls()
434 if (!CF || !CF->isIntrinsic()) in substituteIntrinsicCalls()
437 switch (II->getIntrinsicID()) { in substituteIntrinsicCalls()
481 IRBuilder<> B(F->getContext()); in removeAggregateTypesFromSignature()
483 bool IsRetAggr = F->getReturnType()->isAggregateType(); in removeAggregateTypesFromSignature()
485 std::any_of(F->arg_begin(), F->arg_end(), [](Argument &Arg) { in removeAggregateTypesFromSignature()
486 return Arg.getType()->isAggregateType(); in removeAggregateTypesFromSignature()
492 Type *RetType = IsRetAggr ? B.getInt32Ty() : F->getReturnType(); in removeAggregateTypesFromSignature()
494 ChangedTypes.push_back(std::pair<int, Type *>(-1, F->getReturnType())); in removeAggregateTypesFromSignature()
496 for (const auto &Arg : F->args()) { in removeAggregateTypesFromSignature()
497 if (Arg.getType()->isAggregateType()) { in removeAggregateTypesFromSignature()
505 FunctionType::get(RetType, ArgTypes, F->getFunctionType()->isVarArg()); in removeAggregateTypesFromSignature()
507 Function::Create(NewFTy, F->getLinkage(), F->getName(), *F->getParent()); in removeAggregateTypesFromSignature()
510 auto NewFArgIt = NewF->arg_begin(); in removeAggregateTypesFromSignature()
511 for (auto &Arg : F->args()) { in removeAggregateTypesFromSignature()
513 NewFArgIt->setName(ArgName); in removeAggregateTypesFromSignature()
520 NewF->takeName(F); in removeAggregateTypesFromSignature()
523 F->getParent()->getOrInsertNamedMetadata("spv.cloned_funcs"); in removeAggregateTypesFromSignature()
525 MDArgs.push_back(MDString::get(B.getContext(), NewF->getName())); in removeAggregateTypesFromSignature()
532 FuncMD->addOperand(ThisFuncMD); in removeAggregateTypesFromSignature()
534 for (auto *U : make_early_inc_range(F->users())) { in removeAggregateTypesFromSignature()
536 CI->mutateFunctionType(NewF->getFunctionType()); in removeAggregateTypesFromSignature()
537 U->replaceUsesOfWith(F, NewF); in removeAggregateTypesFromSignature()
541 if (RetType != F->getReturnType()) in removeAggregateTypesFromSignature()
542 TM.getSubtarget<SPIRVSubtarget>(*F).getSPIRVGlobalRegistry()->addMutated( in removeAggregateTypesFromSignature()
543 NewF, F->getReturnType()); in removeAggregateTypesFromSignature()
560 F->eraseFromParent(); in runOnModule()