Lines Matching +full:pic +full:- +full:base +full:- +full:vec
1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file implements the auto-upgrade helper functions.
13 //===----------------------------------------------------------------------===//
49 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
52 static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); } in rename()
60 Type *Arg0Type = F->getFunctionType()->getParamType(0); in upgradePTESTIntrinsic()
61 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4)) in upgradePTESTIntrinsic()
66 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradePTESTIntrinsic()
70 // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
75 Type *LastArgType = F->getFunctionType()->getParamType( in upgradeX86IntrinsicsWith8BitMask()
76 F->getFunctionType()->getNumParams() - 1); in upgradeX86IntrinsicsWith8BitMask()
77 if (!LastArgType->isIntegerTy(32)) in upgradeX86IntrinsicsWith8BitMask()
82 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86IntrinsicsWith8BitMask()
91 if (F->getReturnType()->isVectorTy()) in upgradeX86MaskedFPCompare()
95 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86MaskedFPCompare()
101 if (F->getReturnType()->getScalarType()->isBFloatTy()) in upgradeX86BF16Intrinsic()
105 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86BF16Intrinsic()
111 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy()) in upgradeX86BF16DPIntrinsic()
115 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86BF16DPIntrinsic()
499 if (F->getFunctionType()->getNumParams() == 0) in upgradeX86IntrinsicFunction()
503 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeX86IntrinsicFunction()
588 auto Idx = F->getFunctionType()->getParamType(2); in upgradeX86IntrinsicFunction()
589 if (Idx->isFPOrFPVectorTy()) { in upgradeX86IntrinsicFunction()
590 unsigned IdxSize = Idx->getPrimitiveSizeInBits(); in upgradeX86IntrinsicFunction()
591 unsigned EltSize = Idx->getScalarSizeInBits(); in upgradeX86IntrinsicFunction()
601 } else if (F->arg_size() == 2) in upgradeX86IntrinsicFunction()
610 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeX86IntrinsicFunction()
617 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp); in upgradeX86IntrinsicFunction()
631 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse, in upgradeArmOrAarch64IntrinsicFunction()
632 F->arg_begin()->getType()); in upgradeArmOrAarch64IntrinsicFunction()
639 Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer); in upgradeArmOrAarch64IntrinsicFunction()
657 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits(); in upgradeArmOrAarch64IntrinsicFunction()
660 LLVMContext &Ctx = F->getParent()->getContext(); in upgradeArmOrAarch64IntrinsicFunction()
662 {F->getReturnType(), in upgradeArmOrAarch64IntrinsicFunction()
664 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); in upgradeArmOrAarch64IntrinsicFunction()
689 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeArmOrAarch64IntrinsicFunction()
713 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, in upgradeArmOrAarch64IntrinsicFunction()
714 F->arg_begin()->getType()); in upgradeArmOrAarch64IntrinsicFunction()
720 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$"); in upgradeArmOrAarch64IntrinsicFunction()
731 auto fArgs = F->getFunctionType()->params(); in upgradeArmOrAarch64IntrinsicFunction()
734 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
735 StoreInts[fArgs.size() - 3], Tys); in upgradeArmOrAarch64IntrinsicFunction()
738 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys); in upgradeArmOrAarch64IntrinsicFunction()
750 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) { in upgradeArmOrAarch64IntrinsicFunction()
770 if (Name.consume_front("base.")) { in upgradeArmOrAarch64IntrinsicFunction()
773 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)? in upgradeArmOrAarch64IntrinsicFunction()
811 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, in upgradeArmOrAarch64IntrinsicFunction()
812 F->arg_begin()->getType()); in upgradeArmOrAarch64IntrinsicFunction()
818 if (F->arg_size() != 2) in upgradeArmOrAarch64IntrinsicFunction()
820 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType()); in upgradeArmOrAarch64IntrinsicFunction()
821 if (Ty && Ty->getElementType()->isFloatingPointTy()) { in upgradeArmOrAarch64IntrinsicFunction()
822 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
841 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeArmOrAarch64IntrinsicFunction()
851 if (!F->getReturnType()->isFPOrFPVectorTy()) in upgradeArmOrAarch64IntrinsicFunction()
854 auto Args = F->getFunctionType()->params(); in upgradeArmOrAarch64IntrinsicFunction()
855 Type *Tys[] = {F->getReturnType(), Args[1]}; in upgradeArmOrAarch64IntrinsicFunction()
856 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
863 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)"); in upgradeArmOrAarch64IntrinsicFunction()
866 cast<VectorType>(F->getReturnType())->getElementType(); in upgradeArmOrAarch64IntrinsicFunction()
868 cast<VectorType>(F->arg_begin()->getType())->getElementCount(); in upgradeArmOrAarch64IntrinsicFunction()
875 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
876 LoadIDs[Name[0] - '2'], Ty); in upgradeArmOrAarch64IntrinsicFunction()
886 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()}; in upgradeArmOrAarch64IntrinsicFunction()
887 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
894 auto Args = F->getFunctionType()->params(); in upgradeArmOrAarch64IntrinsicFunction()
896 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
901 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)"); in upgradeArmOrAarch64IntrinsicFunction()
904 auto Args = F->getFunctionType()->params(); in upgradeArmOrAarch64IntrinsicFunction()
905 Type *Tys[] = {F->getReturnType(), Args[1]}; in upgradeArmOrAarch64IntrinsicFunction()
906 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeArmOrAarch64IntrinsicFunction()
1000 assert(F && "Illegal to upgrade a non-existent Function."); in upgradeIntrinsicFunction1()
1002 StringRef Name = F->getName(); in upgradeIntrinsicFunction1()
1021 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::fshr, in upgradeIntrinsicFunction1()
1022 {F->getReturnType()}); in upgradeIntrinsicFunction1()
1047 F->getParent(), Intrinsic::ldexp, in upgradeIntrinsicFunction1()
1048 {F->getReturnType(), F->getArg(1)->getType()}); in upgradeIntrinsicFunction1()
1057 if (F->arg_size() == 1) { in upgradeIntrinsicFunction1()
1064 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, in upgradeIntrinsicFunction1()
1065 F->arg_begin()->getType()); in upgradeIntrinsicFunction1()
1070 if (F->arg_size() == 2 && Name == "coro.end") { in upgradeIntrinsicFunction1()
1072 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::coro_end); in upgradeIntrinsicFunction1()
1082 F->getParent()->IsNewDbgInfoFormat) { in upgradeIntrinsicFunction1()
1093 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) { in upgradeIntrinsicFunction1()
1095 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value); in upgradeIntrinsicFunction1()
1113 const auto *FT = F->getFunctionType(); in upgradeIntrinsicFunction1()
1118 Tys.push_back(FT->getReturnType()); in upgradeIntrinsicFunction1()
1120 Tys.push_back(FT->getParamType(0)); in upgradeIntrinsicFunction1()
1123 Tys.push_back(FT->getParamType(1)); in upgradeIntrinsicFunction1()
1125 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); in upgradeIntrinsicFunction1()
1131 static const Regex R("^([a-z]+)\\.[a-z][0-9]+"); in upgradeIntrinsicFunction1()
1149 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+"); in upgradeIntrinsicFunction1()
1160 auto Args = F->getFunctionType()->params(); in upgradeIntrinsicFunction1()
1162 Intrinsic::getDeclaration(F->getParent(), ID, {Args[V2 ? 1 : 0]}); in upgradeIntrinsicFunction1()
1173 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::get_rounding); in upgradeIntrinsicFunction1()
1180 auto Args = F->getFunctionType()->params(); in upgradeIntrinsicFunction1()
1183 NewFn = Intrinsic::getDeclaration(F->getParent(), in upgradeIntrinsicFunction1()
1196 if (F->arg_size() == 5) { in upgradeIntrinsicFunction1()
1200 F->getFunctionType()->params().slice(0, 3); in upgradeIntrinsicFunction1()
1201 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ParamTypes); in upgradeIntrinsicFunction1()
1205 if (Name.starts_with("memset.") && F->arg_size() == 5) { in upgradeIntrinsicFunction1()
1208 const auto *FT = F->getFunctionType(); in upgradeIntrinsicFunction1()
1210 FT->getParamType(0), // Dest in upgradeIntrinsicFunction1()
1211 FT->getParamType(2) // len in upgradeIntrinsicFunction1()
1213 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset, in upgradeIntrinsicFunction1()
1222 if (F->arg_size() == 1) { in upgradeIntrinsicFunction1()
1230 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, in upgradeIntrinsicFunction1()
1231 {F->getReturnType()}); in upgradeIntrinsicFunction1()
1237 if (!F->getReturnType()->getScalarType()->isBFloatTy()) { in upgradeIntrinsicFunction1()
1277 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() }; in upgradeIntrinsicFunction1()
1278 if (F->arg_size() == 2 || F->arg_size() == 3 || in upgradeIntrinsicFunction1()
1279 F->getName() != in upgradeIntrinsicFunction1()
1280 Intrinsic::getName(Intrinsic::objectsize, Tys, F->getParent())) { in upgradeIntrinsicFunction1()
1282 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize, in upgradeIntrinsicFunction1()
1290 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) { in upgradeIntrinsicFunction1()
1293 F->getParent(), Intrinsic::ptr_annotation, in upgradeIntrinsicFunction1()
1294 {F->arg_begin()->getType(), F->getArg(1)->getType()}); in upgradeIntrinsicFunction1()
1309 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) { in upgradeIntrinsicFunction1()
1311 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeIntrinsicFunction1()
1322 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) || in upgradeIntrinsicFunction1()
1323 F->getFunctionType()->getReturnType()->isIntegerTy(64)) { in upgradeIntrinsicFunction1()
1325 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeIntrinsicFunction1()
1340 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) { in upgradeIntrinsicFunction1()
1342 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeIntrinsicFunction1()
1359 if (Name == "var.annotation" && F->arg_size() == 4) { in upgradeIntrinsicFunction1()
1362 F->getParent(), Intrinsic::var_annotation, in upgradeIntrinsicFunction1()
1363 {{F->arg_begin()->getType(), F->getArg(1)->getType()}}); in upgradeIntrinsicFunction1()
1380 Intrinsic::getDeclaration(F->getParent(), ID, F->getReturnType()); in upgradeIntrinsicFunction1()
1392 NewFn = Intrinsic::getDeclaration(F->getParent(), ID); in upgradeIntrinsicFunction1()
1406 auto *ST = dyn_cast<StructType>(F->getReturnType()); in upgradeIntrinsicFunction1()
1407 if (ST && (!ST->isLiteral() || ST->isPacked()) && in upgradeIntrinsicFunction1()
1408 F->getIntrinsicID() != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1409 // Replace return type with literal non-packed struct. Only do this for in upgradeIntrinsicFunction1()
1414 Intrinsic::getIntrinsicInfoTableEntries(F->getIntrinsicID(), Desc); in upgradeIntrinsicFunction1()
1416 auto *FT = F->getFunctionType(); in upgradeIntrinsicFunction1()
1417 auto *NewST = StructType::get(ST->getContext(), ST->elements()); in upgradeIntrinsicFunction1()
1418 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg()); in upgradeIntrinsicFunction1()
1419 std::string Name = F->getName().str(); in upgradeIntrinsicFunction1()
1421 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(), in upgradeIntrinsicFunction1()
1422 Name, F->getParent()); in upgradeIntrinsicFunction1()
1455 if (Intrinsic::ID id = F->getIntrinsicID()) in UpgradeIntrinsicFunction()
1456 F->setAttributes(Intrinsic::getAttributes(F->getContext(), id)); in UpgradeIntrinsicFunction()
1461 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" || in UpgradeGlobalVariable()
1462 GV->getName() == "llvm.global_dtors")) || in UpgradeGlobalVariable()
1463 !GV->hasInitializer()) in UpgradeGlobalVariable()
1465 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType()); in UpgradeGlobalVariable()
1468 StructType *STy = dyn_cast<StructType>(ATy->getElementType()); in UpgradeGlobalVariable()
1469 if (!STy || STy->getNumElements() != 2) in UpgradeGlobalVariable()
1472 LLVMContext &C = GV->getContext(); in UpgradeGlobalVariable()
1474 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1), in UpgradeGlobalVariable()
1476 Constant *Init = GV->getInitializer(); in UpgradeGlobalVariable()
1477 unsigned N = Init->getNumOperands(); in UpgradeGlobalVariable()
1480 auto Ctor = cast<Constant>(Init->getOperand(i)); in UpgradeGlobalVariable()
1481 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u), in UpgradeGlobalVariable()
1482 Ctor->getAggregateElement(1), in UpgradeGlobalVariable()
1487 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(), in UpgradeGlobalVariable()
1488 NewInit, GV->getName()); in UpgradeGlobalVariable()
1495 auto *ResultTy = cast<FixedVectorType>(Op->getType()); in upgradeX86PSLLDQIntrinsics()
1496 unsigned NumElts = ResultTy->getNumElements() * 8; in upgradeX86PSLLDQIntrinsics()
1498 // Bitcast from a 64-bit element type to a byte element type. in upgradeX86PSLLDQIntrinsics()
1509 // 256/512-bit version is split into 2/4 16-byte lanes. in upgradeX86PSLLDQIntrinsics()
1512 unsigned Idx = NumElts + i - Shift; in upgradeX86PSLLDQIntrinsics()
1514 Idx -= NumElts - 16; // end of lane, switch operand. in upgradeX86PSLLDQIntrinsics()
1521 // Bitcast back to a 64-bit element type. in upgradeX86PSLLDQIntrinsics()
1529 auto *ResultTy = cast<FixedVectorType>(Op->getType()); in upgradeX86PSRLDQIntrinsics()
1530 unsigned NumElts = ResultTy->getNumElements() * 8; in upgradeX86PSRLDQIntrinsics()
1532 // Bitcast from a 64-bit element type to a byte element type. in upgradeX86PSRLDQIntrinsics()
1543 // 256/512-bit version is split into 2/4 16-byte lanes. in upgradeX86PSRLDQIntrinsics()
1548 Idx += NumElts - 16; // end of lane, switch operand. in upgradeX86PSRLDQIntrinsics()
1555 // Bitcast back to a 64-bit element type. in upgradeX86PSRLDQIntrinsics()
1561 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements"); in getX86MaskVec()
1563 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth()); in getX86MaskVec()
1583 if (C->isAllOnesValue()) in emitX86Select()
1587 cast<FixedVectorType>(Op0->getType())->getNumElements()); in emitX86Select()
1595 if (C->isAllOnesValue()) in emitX86ScalarSelect()
1599 Mask->getType()->getIntegerBitWidth()); in emitX86ScalarSelect()
1607 // so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
1612 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue(); in upgradeX86ALIGNIntrinsics()
1614 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements(); in upgradeX86ALIGNIntrinsics()
1621 ShiftVal &= (NumElts - 1); in upgradeX86ALIGNIntrinsics()
1626 return llvm::Constant::getNullValue(Op0->getType()); in upgradeX86ALIGNIntrinsics()
1631 ShiftVal -= 16; in upgradeX86ALIGNIntrinsics()
1633 Op0 = llvm::Constant::getNullValue(Op0->getType()); in upgradeX86ALIGNIntrinsics()
1637 // 256-bit palignr operates on 128-bit lanes so we need to handle that in upgradeX86ALIGNIntrinsics()
1642 Idx += NumElts - 16; // End of lane, switch operand. in upgradeX86ALIGNIntrinsics()
1656 unsigned VecWidth = Ty->getPrimitiveSizeInBits(); in upgradeX86VPERMT2Intrinsics()
1657 unsigned EltWidth = Ty->getScalarSizeInBits(); in upgradeX86VPERMT2Intrinsics()
1658 bool IsFloat = Ty->isFPOrFPVectorTy(); in upgradeX86VPERMT2Intrinsics()
1737 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so in upgradeX86Rotate()
1739 if (Amt->getType() != Ty) { in upgradeX86Rotate()
1740 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements(); in upgradeX86Rotate()
1741 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false); in upgradeX86Rotate()
1807 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so in upgradeX86ConcatShift()
1809 if (Amt->getType() != Ty) { in upgradeX86ConcatShift()
1810 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements(); in upgradeX86ConcatShift()
1811 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false); in upgradeX86ConcatShift()
1824 Value *Mask = CI.getOperand(NumArgs - 1); in upgradeX86ConcatShift()
1834 llvm::PointerType::getUnqual(Data->getType())); in upgradeMaskedStore()
1837 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8) in upgradeMaskedStore()
1842 if (C->isAllOnesValue()) in upgradeMaskedStore()
1846 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements(); in upgradeMaskedStore()
1853 Type *ValTy = Passthru->getType(); in upgradeMaskedLoad()
1859 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() / in upgradeMaskedLoad()
1865 if (C->isAllOnesValue()) in upgradeMaskedLoad()
1869 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements(); in upgradeMaskedLoad()
1914 static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, in applyX86MaskOn1BitsVec() argument
1916 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements(); in applyX86MaskOn1BitsVec()
1919 if (!C || !C->isAllOnesValue()) in applyX86MaskOn1BitsVec()
1920 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts)); in applyX86MaskOn1BitsVec()
1929 Vec = Builder.CreateShuffleVector(Vec, in applyX86MaskOn1BitsVec()
1930 Constant::getNullValue(Vec->getType()), in applyX86MaskOn1BitsVec()
1933 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U))); in applyX86MaskOn1BitsVec()
1939 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements(); in upgradeMaskedCompare()
1962 Value *Mask = CI.getArgOperand(CI.arg_size() - 1); in upgradeMaskedCompare()
1993 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements(); in upgradeMaskToInt()
2003 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits(); in upgradeAVX512MaskToSelect()
2004 unsigned EltWidth = CI.getType()->getScalarSizeInBits(); in upgradeAVX512MaskToSelect()
2144 bool IsFloat = CI.getType()->isFPOrFPVectorTy(); in upgradeAVX512MaskToSelect()
2232 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep, in upgradeAVX512MaskToSelect()
2233 CI.getArgOperand(NumArgs - 2)); in upgradeAVX512MaskToSelect()
2241 if (AsmStr->find("mov\tfp") == 0 && in UpgradeInlineAsmString()
2242 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos && in UpgradeInlineAsmString()
2243 (Pos = AsmStr->find("# marker")) != std::string::npos) { in UpgradeInlineAsmString()
2244 AsmStr->replace(Pos, 1, ";"); in UpgradeInlineAsmString()
2250 LLVMContext &C = F->getContext(); in upgradeX86IntrinsicCall()
2259 Value *Arg0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2260 Value *Arg1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2264 Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType(); in upgradeX86IntrinsicCall()
2271 SI->setMetadata(LLVMContext::MD_nontemporal, Node); in upgradeX86IntrinsicCall()
2279 Value *Arg0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2280 Value *Arg1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2284 Arg0, PointerType::getUnqual(Arg1->getType()), "cast"); in upgradeX86IntrinsicCall()
2287 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)); in upgradeX86IntrinsicCall()
2288 SI->setMetadata(LLVMContext::MD_nontemporal, Node); in upgradeX86IntrinsicCall()
2290 Value *Arg0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2291 Value *Arg1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2297 Arg0, PointerType::getUnqual(Elt->getType()), "cast"); in upgradeX86IntrinsicCall()
2302 Value *Arg0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2303 Value *Arg1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2305 Arg0 = Builder.CreateBitCast(Arg0, PointerType::getUnqual(Arg1->getType()), in upgradeX86IntrinsicCall()
2309 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1)); in upgradeX86IntrinsicCall()
2310 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2315 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2316 CI->getArgOperand(2), Aligned); in upgradeX86IntrinsicCall()
2322 CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2323 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); in upgradeX86IntrinsicCall()
2326 if (CI->getOperand(0)->getType()->isIntegerTy(8)) in upgradeX86IntrinsicCall()
2328 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / in upgradeX86IntrinsicCall()
2329 ExtTy->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
2330 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy); in upgradeX86IntrinsicCall()
2333 Value *Vec = CI->getArgOperand(0); in upgradeX86IntrinsicCall() local
2334 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0); in upgradeX86IntrinsicCall()
2335 Function *Intr = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sqrt, in upgradeX86IntrinsicCall()
2336 Elt0->getType()); in upgradeX86IntrinsicCall()
2338 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0); in upgradeX86IntrinsicCall()
2344 F->getParent(), Intrinsic::sqrt, CI->getType()), in upgradeX86IntrinsicCall()
2345 {CI->getArgOperand(0)}); in upgradeX86IntrinsicCall()
2347 if (CI->arg_size() == 4 && in upgradeX86IntrinsicCall()
2348 (!isa<ConstantInt>(CI->getArgOperand(3)) || in upgradeX86IntrinsicCall()
2349 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { in upgradeX86IntrinsicCall()
2353 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)}; in upgradeX86IntrinsicCall()
2354 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
2357 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), in upgradeX86IntrinsicCall()
2359 CI->getType()), in upgradeX86IntrinsicCall()
2360 {CI->getArgOperand(0)}); in upgradeX86IntrinsicCall()
2363 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2366 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2367 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2368 Value *Mask = CI->getArgOperand(2); in upgradeX86IntrinsicCall()
2370 llvm::Type *Ty = Op0->getType(); in upgradeX86IntrinsicCall()
2378 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType()) in upgradeX86IntrinsicCall()
2379 ->getNumElements(); in upgradeX86IntrinsicCall()
2380 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0)); in upgradeX86IntrinsicCall()
2382 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2384 unsigned NumElts = CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
2385 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts); in upgradeX86IntrinsicCall()
2386 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts); in upgradeX86IntrinsicCall()
2398 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2400 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2401 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2403 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2405 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2406 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2409 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2411 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2412 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2414 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2416 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2417 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2419 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2421 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2422 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2425 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2427 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2429 Rep = Builder.CreateBitCast(Rep, CI->getType()); in upgradeX86IntrinsicCall()
2431 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); in upgradeX86IntrinsicCall()
2432 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); in upgradeX86IntrinsicCall()
2447 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2449 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2460 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp, in upgradeX86IntrinsicCall()
2467 Type *OpTy = CI->getArgOperand(0)->getType(); in upgradeX86IntrinsicCall()
2468 unsigned VecWidth = OpTy->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
2484 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
2485 {CI->getOperand(0), CI->getArgOperand(1)}); in upgradeX86IntrinsicCall()
2486 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
2488 Type *OpTy = CI->getArgOperand(0)->getType(); in upgradeX86IntrinsicCall()
2489 unsigned VecWidth = OpTy->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
2490 unsigned EltWidth = OpTy->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
2507 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
2508 {CI->getOperand(0), CI->getArgOperand(1)}); in upgradeX86IntrinsicCall()
2509 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
2511 SmallVector<Value *, 4> Args(CI->args()); in upgradeX86IntrinsicCall()
2512 Type *OpTy = Args[0]->getType(); in upgradeX86IntrinsicCall()
2513 unsigned VecWidth = OpTy->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
2514 unsigned EltWidth = OpTy->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
2531 Value *Mask = Constant::getAllOnesValue(CI->getType()); in upgradeX86IntrinsicCall()
2536 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
2540 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2543 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2549 Value *Op = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2550 Value *Zero = llvm::Constant::getNullValue(Op->getType()); in upgradeX86IntrinsicCall()
2584 Builder.CreateSIToFP(CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2585 cast<VectorType>(CI->getType())->getElementType()); in upgradeX86IntrinsicCall()
2586 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in upgradeX86IntrinsicCall()
2589 Builder.CreateUIToFP(CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2590 cast<VectorType>(CI->getType())->getElementType()); in upgradeX86IntrinsicCall()
2591 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in upgradeX86IntrinsicCall()
2593 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0); in upgradeX86IntrinsicCall()
2595 Rep, cast<VectorType>(CI->getType())->getElementType()); in upgradeX86IntrinsicCall()
2596 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in upgradeX86IntrinsicCall()
2612 auto *DstTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2613 Rep = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2614 auto *SrcTy = cast<FixedVectorType>(Rep->getType()); in upgradeX86IntrinsicCall()
2616 unsigned NumDstElts = DstTy->getNumElements(); in upgradeX86IntrinsicCall()
2617 if (NumDstElts < SrcTy->getNumElements()) { in upgradeX86IntrinsicCall()
2622 bool IsPS2PD = SrcTy->getElementType()->isFloatTy(); in upgradeX86IntrinsicCall()
2626 else if (CI->arg_size() == 4 && in upgradeX86IntrinsicCall()
2627 (!isa<ConstantInt>(CI->getArgOperand(3)) || in upgradeX86IntrinsicCall()
2628 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { in upgradeX86IntrinsicCall()
2632 Intrinsic::getDeclaration(CI->getModule(), IID, {DstTy, SrcTy}); in upgradeX86IntrinsicCall()
2633 Rep = Builder.CreateCall(F, {Rep, CI->getArgOperand(3)}); in upgradeX86IntrinsicCall()
2639 if (CI->arg_size() >= 3) in upgradeX86IntrinsicCall()
2640 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in upgradeX86IntrinsicCall()
2641 CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2644 auto *DstTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2645 Rep = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2646 auto *SrcTy = cast<FixedVectorType>(Rep->getType()); in upgradeX86IntrinsicCall()
2647 unsigned NumDstElts = DstTy->getNumElements(); in upgradeX86IntrinsicCall()
2648 if (NumDstElts != SrcTy->getNumElements()) { in upgradeX86IntrinsicCall()
2655 if (CI->arg_size() >= 3) in upgradeX86IntrinsicCall()
2656 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in upgradeX86IntrinsicCall()
2657 CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2661 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2662 CI->getArgOperand(2), Aligned); in upgradeX86IntrinsicCall()
2664 auto *ResultTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2665 Type *PtrTy = ResultTy->getElementType(); in upgradeX86IntrinsicCall()
2668 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), in upgradeX86IntrinsicCall()
2671 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), in upgradeX86IntrinsicCall()
2672 ResultTy->getNumElements()); in upgradeX86IntrinsicCall()
2675 F->getParent(), Intrinsic::masked_expandload, ResultTy); in upgradeX86IntrinsicCall()
2676 Rep = Builder.CreateCall(ELd, {Ptr, MaskVec, CI->getOperand(1)}); in upgradeX86IntrinsicCall()
2678 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType()); in upgradeX86IntrinsicCall()
2679 Type *PtrTy = ResultTy->getElementType(); in upgradeX86IntrinsicCall()
2682 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), in upgradeX86IntrinsicCall()
2686 getX86MaskVec(Builder, CI->getArgOperand(2), in upgradeX86IntrinsicCall()
2687 cast<FixedVectorType>(ResultTy)->getNumElements()); in upgradeX86IntrinsicCall()
2690 F->getParent(), Intrinsic::masked_compressstore, ResultTy); in upgradeX86IntrinsicCall()
2691 Rep = Builder.CreateCall(CSt, {CI->getArgOperand(1), Ptr, MaskVec}); in upgradeX86IntrinsicCall()
2694 auto *ResultTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2696 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), in upgradeX86IntrinsicCall()
2697 ResultTy->getNumElements()); in upgradeX86IntrinsicCall()
2702 Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy); in upgradeX86IntrinsicCall()
2704 {CI->getOperand(0), CI->getOperand(1), MaskVec}); in upgradeX86IntrinsicCall()
2717 if (CI->arg_size() == 3) { in upgradeX86IntrinsicCall()
2718 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2743 Value *Sel = CI->getArgOperand(2); in upgradeX86IntrinsicCall()
2745 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel); in upgradeX86IntrinsicCall()
2746 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel); in upgradeX86IntrinsicCall()
2766 F->getParent(), Intrinsic::x86_sse42_crc32_32_8); in upgradeX86IntrinsicCall()
2768 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C)); in upgradeX86IntrinsicCall()
2769 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)}); in upgradeX86IntrinsicCall()
2770 Rep = Builder.CreateZExt(Rep, CI->getType(), ""); in upgradeX86IntrinsicCall()
2774 auto *VecTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2775 Type *EltTy = VecTy->getElementType(); in upgradeX86IntrinsicCall()
2776 unsigned EltNum = VecTy->getNumElements(); in upgradeX86IntrinsicCall()
2777 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0)); in upgradeX86IntrinsicCall()
2788 auto *DstTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2789 unsigned NumDstElts = DstTy->getNumElements(); in upgradeX86IntrinsicCall()
2796 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask); in upgradeX86IntrinsicCall()
2802 if (CI->arg_size() == 3) in upgradeX86IntrinsicCall()
2803 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in upgradeX86IntrinsicCall()
2804 CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2809 Type *Ty = CI->getArgOperand(1)->getType(); in upgradeX86IntrinsicCall()
2810 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty); in upgradeX86IntrinsicCall()
2812 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2816 Type *EltTy = cast<VectorType>(CI->getType())->getElementType(); in upgradeX86IntrinsicCall()
2817 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
2819 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2829 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2830 Type *VT = CI->getType(); in upgradeX86IntrinsicCall()
2831 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128; in upgradeX86IntrinsicCall()
2832 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
2833 unsigned ControlBitsMask = NumLanes - 1; in upgradeX86IntrinsicCall()
2845 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2846 CI->getArgOperand(1), ShuffleMask); in upgradeX86IntrinsicCall()
2848 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3)); in upgradeX86IntrinsicCall()
2851 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType()) in upgradeX86IntrinsicCall()
2852 ->getNumElements(); in upgradeX86IntrinsicCall()
2854 cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
2860 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2861 CI->getArgOperand(0), ShuffleMask); in upgradeX86IntrinsicCall()
2863 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2869 Value *Op = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2870 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount(); in upgradeX86IntrinsicCall()
2876 if (CI->arg_size() == 3) in upgradeX86IntrinsicCall()
2877 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep, in upgradeX86IntrinsicCall()
2878 CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
2898 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2899 CI->getArgOperand(1), CI->getArgOperand(2), in upgradeX86IntrinsicCall()
2900 CI->getArgOperand(3), CI->getArgOperand(4), in upgradeX86IntrinsicCall()
2904 Builder, CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
2905 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true); in upgradeX86IntrinsicCall()
2907 // 128/256-bit shift left specified in bits. in upgradeX86IntrinsicCall()
2908 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
2909 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2912 // 128/256-bit shift right specified in bits. in upgradeX86IntrinsicCall()
2913 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
2914 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), in upgradeX86IntrinsicCall()
2918 // 128/256/512-bit shift left specified in bytes. in upgradeX86IntrinsicCall()
2919 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
2920 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); in upgradeX86IntrinsicCall()
2923 // 128/256/512-bit shift right specified in bytes. in upgradeX86IntrinsicCall()
2924 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
2925 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); in upgradeX86IntrinsicCall()
2929 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2930 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2931 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2932 auto *VecTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
2933 unsigned NumElts = VecTy->getNumElements(); in upgradeX86IntrinsicCall()
2943 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2944 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
2945 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
2947 cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
2949 cast<FixedVectorType>(Op1->getType())->getNumElements(); in upgradeX86IntrinsicCall()
2970 // Example of shuffle mask for 32-bit elements: in upgradeX86IntrinsicCall()
2983 if (CI->arg_size() == 5) in upgradeX86IntrinsicCall()
2984 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, in upgradeX86IntrinsicCall()
2985 CI->getArgOperand(3)); in upgradeX86IntrinsicCall()
2989 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
2990 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
2992 cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
2994 cast<FixedVectorType>(Op0->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3008 if (CI->arg_size() == 4) in upgradeX86IntrinsicCall()
3009 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in upgradeX86IntrinsicCall()
3010 CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3013 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3014 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
3015 auto *VecTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3016 unsigned NumElts = VecTy->getNumElements(); in upgradeX86IntrinsicCall()
3024 if (CI->arg_size() == 4) in upgradeX86IntrinsicCall()
3025 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in upgradeX86IntrinsicCall()
3026 CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3029 // [1:0] - select 128 bits from sources for low half of destination in upgradeX86IntrinsicCall()
3030 // [2] - ignore in upgradeX86IntrinsicCall()
3031 // [3] - zero low half of destination in upgradeX86IntrinsicCall()
3032 // [5:4] - select 128 bits from sources for high half of destination in upgradeX86IntrinsicCall()
3033 // [6] - ignore in upgradeX86IntrinsicCall()
3034 // [7] - zero high half of destination in upgradeX86IntrinsicCall()
3036 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
3038 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3043 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3044 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3047 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0; in upgradeX86IntrinsicCall()
3048 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1; in upgradeX86IntrinsicCall()
3065 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3066 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
3067 auto *VecTy = cast<FixedVectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3068 unsigned NumElts = VecTy->getNumElements(); in upgradeX86IntrinsicCall()
3070 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3071 unsigned IdxMask = ((1 << IdxSize) - 1); in upgradeX86IntrinsicCall()
3075 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need in upgradeX86IntrinsicCall()
3082 if (CI->arg_size() == 4) in upgradeX86IntrinsicCall()
3083 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in upgradeX86IntrinsicCall()
3084 CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3087 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3088 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
3089 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3101 if (CI->arg_size() == 4) in upgradeX86IntrinsicCall()
3102 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in upgradeX86IntrinsicCall()
3103 CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3106 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3107 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in upgradeX86IntrinsicCall()
3108 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3120 if (CI->arg_size() == 4) in upgradeX86IntrinsicCall()
3121 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, in upgradeX86IntrinsicCall()
3122 CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3124 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3125 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
3126 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); in upgradeX86IntrinsicCall()
3127 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3129 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3134 // Base index is the starting element of the lane. in upgradeX86IntrinsicCall()
3135 Idxs[i] = i - (i % NumLaneElts); in upgradeX86IntrinsicCall()
3140 // the immediate. Wrapping around the immediate every 8-bits. in upgradeX86IntrinsicCall()
3141 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1); in upgradeX86IntrinsicCall()
3147 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3)); in upgradeX86IntrinsicCall()
3151 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3152 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3153 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3169 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3172 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3173 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
3174 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3175 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3185 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3188 Value *Op0 = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3189 Value *Op1 = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
3190 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3191 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3201 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3204 VectorType *FTy = cast<VectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3206 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in upgradeX86IntrinsicCall()
3207 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in upgradeX86IntrinsicCall()
3210 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3213 VectorType *FTy = cast<VectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3215 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy)); in upgradeX86IntrinsicCall()
3217 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in upgradeX86IntrinsicCall()
3220 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3223 VectorType *FTy = cast<VectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3225 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in upgradeX86IntrinsicCall()
3226 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in upgradeX86IntrinsicCall()
3229 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3232 VectorType *FTy = cast<VectorType>(CI->getType()); in upgradeX86IntrinsicCall()
3234 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy), in upgradeX86IntrinsicCall()
3235 Builder.CreateBitCast(CI->getArgOperand(1), ITy)); in upgradeX86IntrinsicCall()
3238 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3240 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3242 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3244 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3246 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3248 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3250 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3260 Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3261 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3263 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3266 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3276 Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3277 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3279 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3282 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3292 Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3293 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3295 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3298 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3308 Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3309 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3311 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3314 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3326 Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3327 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3329 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); in upgradeX86IntrinsicCall()
3333 F->getParent(), Intrinsic::ctlz, CI->getType()), in upgradeX86IntrinsicCall()
3334 {CI->getArgOperand(0), Builder.getInt1(false)}); in upgradeX86IntrinsicCall()
3336 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); in upgradeX86IntrinsicCall()
3544 Value *Ptr = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3548 Ptr, PointerType::getUnqual(CI->getType()), "cast"); in upgradeX86IntrinsicCall()
3550 CI->getType(), BC, in upgradeX86IntrinsicCall()
3551 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)); in upgradeX86IntrinsicCall()
3552 LI->setMetadata(LLVMContext::MD_nontemporal, Node); in upgradeX86IntrinsicCall()
3562 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3563 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3578 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in upgradeX86IntrinsicCall()
3580 Ops[0]->getType()), in upgradeX86IntrinsicCall()
3584 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); in upgradeX86IntrinsicCall()
3586 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3587 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3593 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in upgradeX86IntrinsicCall()
3595 Ops[0]->getType()), in upgradeX86IntrinsicCall()
3598 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()), in upgradeX86IntrinsicCall()
3612 Value *A = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3613 Value *B = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
3614 Value *C = CI->getArgOperand(2); in upgradeX86IntrinsicCall()
3627 if (!isa<ConstantInt>(CI->getArgOperand(4)) || in upgradeX86IntrinsicCall()
3628 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) { in upgradeX86IntrinsicCall()
3629 Value *Ops[] = {A, B, C, CI->getArgOperand(4)}; in upgradeX86IntrinsicCall()
3636 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID); in upgradeX86IntrinsicCall()
3639 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in upgradeX86IntrinsicCall()
3640 A->getType()); in upgradeX86IntrinsicCall()
3644 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType()) in upgradeX86IntrinsicCall()
3652 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0); in upgradeX86IntrinsicCall()
3654 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3655 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep, in upgradeX86IntrinsicCall()
3671 Value *A = CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3672 Value *B = CI->getArgOperand(1); in upgradeX86IntrinsicCall()
3673 Value *C = CI->getArgOperand(2); in upgradeX86IntrinsicCall()
3682 if (CI->arg_size() == 5 && in upgradeX86IntrinsicCall()
3683 (!isa<ConstantInt>(CI->getArgOperand(4)) || in upgradeX86IntrinsicCall()
3684 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) { in upgradeX86IntrinsicCall()
3687 if (Name[Name.size() - 5] == 's') in upgradeX86IntrinsicCall()
3692 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3693 {A, B, C, CI->getArgOperand(4)}); in upgradeX86IntrinsicCall()
3695 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in upgradeX86IntrinsicCall()
3696 A->getType()); in upgradeX86IntrinsicCall()
3700 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) in upgradeX86IntrinsicCall()
3701 : IsMask3 ? CI->getArgOperand(2) in upgradeX86IntrinsicCall()
3702 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3704 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3706 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
3707 unsigned EltWidth = CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3720 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3721 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3724 Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), Ops); in upgradeX86IntrinsicCall()
3734 if (CI->arg_size() == 5) { in upgradeX86IntrinsicCall()
3737 if (Name[Name.size() - 5] == 's') in upgradeX86IntrinsicCall()
3742 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3743 CI->getArgOperand(2), CI->getArgOperand(4)}; in upgradeX86IntrinsicCall()
3747 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in upgradeX86IntrinsicCall()
3750 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in upgradeX86IntrinsicCall()
3752 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3753 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3755 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in upgradeX86IntrinsicCall()
3756 Ops[0]->getType()); in upgradeX86IntrinsicCall()
3771 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) in upgradeX86IntrinsicCall()
3772 : IsMask3 ? CI->getArgOperand(2) in upgradeX86IntrinsicCall()
3773 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3775 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3779 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
3780 unsigned EltWidth = CI->getType()->getScalarSizeInBits(); in upgradeX86IntrinsicCall()
3797 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3798 CI->getArgOperand(2), CI->getArgOperand(3)}; in upgradeX86IntrinsicCall()
3799 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3801 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in upgradeX86IntrinsicCall()
3802 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3803 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru); in upgradeX86IntrinsicCall()
3808 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
3825 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3826 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3827 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3829 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in upgradeX86IntrinsicCall()
3830 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3831 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3844 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
3861 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3862 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3863 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3865 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in upgradeX86IntrinsicCall()
3866 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3867 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3874 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); in upgradeX86IntrinsicCall()
3891 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3892 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3893 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in upgradeX86IntrinsicCall()
3895 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) in upgradeX86IntrinsicCall()
3896 : CI->getArgOperand(0); in upgradeX86IntrinsicCall()
3897 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); in upgradeX86IntrinsicCall()
3914 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), in upgradeX86IntrinsicCall()
3915 CI->getArgOperand(2)}; in upgradeX86IntrinsicCall()
3917 Intrinsic::getDeclaration(CI->getModule(), IID), Args); in upgradeX86IntrinsicCall()
3923 CI->getArgOperand(3), llvm::PointerType::getUnqual(Data->getType())); in upgradeX86IntrinsicCall()
3928 CI->replaceAllUsesWith(CF); in upgradeX86IntrinsicCall()
3941 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the in upgradeARMIntrinsicCall()
3944 Intrinsic::getDeclaration(F->getParent(), Intrinsic::arm_mve_vctp64), in upgradeARMIntrinsicCall()
3945 CI->getArgOperand(0), CI->getName()); in upgradeARMIntrinsicCall()
3948 F->getParent(), Intrinsic::arm_mve_pred_v2i, in upgradeARMIntrinsicCall()
3953 F->getParent(), Intrinsic::arm_mve_pred_i2v, in upgradeARMIntrinsicCall()
3958 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" || in upgradeARMIntrinsicCall()
3959 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" || in upgradeARMIntrinsicCall()
3963 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" || in upgradeARMIntrinsicCall()
3964 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" || in upgradeARMIntrinsicCall()
3975 unsigned ID = CI->getIntrinsicID(); in upgradeARMIntrinsicCall()
3981 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
3986 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(), in upgradeARMIntrinsicCall()
3990 Tys = {CI->getType(), CI->getOperand(0)->getType(), in upgradeARMIntrinsicCall()
3991 CI->getOperand(1)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
3994 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(), in upgradeARMIntrinsicCall()
3995 CI->getOperand(2)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
4003 Tys = {CI->getOperand(1)->getType(), V2I1Ty}; in upgradeARMIntrinsicCall()
4010 for (Value *Op : CI->args()) { in upgradeARMIntrinsicCall()
4011 Type *Ty = Op->getType(); in upgradeARMIntrinsicCall()
4012 if (Ty->getScalarSizeInBits() == 1) { in upgradeARMIntrinsicCall()
4015 F->getParent(), Intrinsic::arm_mve_pred_v2i, in upgradeARMIntrinsicCall()
4019 Intrinsic::getDeclaration(F->getParent(), in upgradeARMIntrinsicCall()
4026 Function *Fn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); in upgradeARMIntrinsicCall()
4027 return Builder.CreateCall(Fn, Ops, CI->getName()); in upgradeARMIntrinsicCall()
4047 unsigned NumOperands = CI->getNumOperands(); in upgradeAMDGCNIntrinsicCall()
4051 Value *Ptr = CI->getArgOperand(0); in upgradeAMDGCNIntrinsicCall()
4052 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType()); in upgradeAMDGCNIntrinsicCall()
4056 Value *Val = CI->getArgOperand(1); in upgradeAMDGCNIntrinsicCall()
4057 if (Val->getType() != CI->getType()) // Malformed. in upgradeAMDGCNIntrinsicCall()
4066 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2)); in upgradeAMDGCNIntrinsicCall()
4071 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4)); in upgradeAMDGCNIntrinsicCall()
4072 IsVolatile = !VolatileArg || !VolatileArg->isZero(); in upgradeAMDGCNIntrinsicCall()
4076 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue())) in upgradeAMDGCNIntrinsicCall()
4077 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue()); in upgradeAMDGCNIntrinsicCall()
4081 LLVMContext &Ctx = F->getContext(); in upgradeAMDGCNIntrinsicCall()
4084 Type *RetTy = CI->getType(); in upgradeAMDGCNIntrinsicCall()
4086 if (VT->getElementType()->isIntegerTy(16)) { in upgradeAMDGCNIntrinsicCall()
4088 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount()); in upgradeAMDGCNIntrinsicCall()
4099 if (PtrTy->getAddressSpace() != 3) { in upgradeAMDGCNIntrinsicCall()
4100 RMW->setMetadata("amdgpu.no.fine.grained.memory", in upgradeAMDGCNIntrinsicCall()
4101 MDNode::get(F->getContext(), {})); in upgradeAMDGCNIntrinsicCall()
4105 RMW->setVolatile(true); in upgradeAMDGCNIntrinsicCall()
4113 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) in unwrapMAVOp()
4114 return dyn_cast<MDType>(MAV->getMetadata()); in unwrapMAVOp()
4118 /// Convert debug intrinsic calls to non-instruction debug records.
4119 /// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
4120 /// \p CI - The debug intrinsic call.
4124 DR = new DbgLabelRecord(unwrapMAVOp<DILabel>(CI, 0), CI->getDebugLoc()); in upgradeDbgIntrinsicToDbgRecord()
4130 CI->getDebugLoc()); in upgradeDbgIntrinsicToDbgRecord()
4134 unwrapMAVOp<DIExpression>(CI, 2), CI->getDebugLoc(), in upgradeDbgIntrinsicToDbgRecord()
4142 CI->getDebugLoc()); in upgradeDbgIntrinsicToDbgRecord()
4147 if (CI->arg_size() == 4) { in upgradeDbgIntrinsicToDbgRecord()
4148 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)); in upgradeDbgIntrinsicToDbgRecord()
4150 if (!Offset || !Offset->isZeroValue()) in upgradeDbgIntrinsicToDbgRecord()
4157 unwrapMAVOp<DIExpression>(CI, ExprOp), CI->getDebugLoc()); in upgradeDbgIntrinsicToDbgRecord()
4160 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator()); in upgradeDbgIntrinsicToDbgRecord()
4169 Function *F = dyn_cast<Function>(CI->getCalledOperand()); in UpgradeIntrinsicCall()
4173 LLVMContext &C = CI->getContext(); in UpgradeIntrinsicCall()
4175 Builder.SetInsertPoint(CI->getParent(), CI->getIterator()); in UpgradeIntrinsicCall()
4180 StringRef Name = F->getName(); in UpgradeIntrinsicCall()
4195 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4198 Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond"); in UpgradeIntrinsicCall()
4202 Value *Ptr = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4203 Value *Val = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4209 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4210 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4218 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4219 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4226 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4228 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, in UpgradeIntrinsicCall()
4229 {Arg->getType()}), in UpgradeIntrinsicCall()
4235 Value *Arg = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4237 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, in UpgradeIntrinsicCall()
4238 {Arg->getType()}), in UpgradeIntrinsicCall()
4245 F->getParent(), Intrinsic::convert_from_fp16, in UpgradeIntrinsicCall()
4247 CI->getArgOperand(0), "h2f"); in UpgradeIntrinsicCall()
4251 !F->getReturnType()->getScalarType()->isBFloatTy()) { in UpgradeIntrinsicCall()
4253 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in UpgradeIntrinsicCall()
4255 for (size_t I = 0; I < NewFn->arg_size(); ++I) { in UpgradeIntrinsicCall()
4256 Value *Arg = CI->getArgOperand(I); in UpgradeIntrinsicCall()
4257 Type *OldType = Arg->getType(); in UpgradeIntrinsicCall()
4258 Type *NewType = NewFn->getArg(I)->getType(); in UpgradeIntrinsicCall()
4259 Args.push_back((OldType->isIntegerTy() && in UpgradeIntrinsicCall()
4260 NewType->getScalarType()->isBFloatTy()) in UpgradeIntrinsicCall()
4265 if (F->getReturnType()->isIntegerTy()) in UpgradeIntrinsicCall()
4266 Rep = Builder.CreateBitCast(Rep, F->getReturnType()); in UpgradeIntrinsicCall()
4280 if (!CI->getModule()->IsNewDbgInfoFormat) { in UpgradeIntrinsicCall()
4282 upgradeIntrinsicFunction1(CI->getCalledFunction(), NewFn, false); in UpgradeIntrinsicCall()
4295 CI->replaceAllUsesWith(Rep); in UpgradeIntrinsicCall()
4296 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4301 const auto &DefaultCase = [&]() -> void { in UpgradeIntrinsicCall()
4302 if (CI->getFunctionType() == NewFn->getFunctionType()) { in UpgradeIntrinsicCall()
4305 (CI->getCalledFunction()->getName() != NewFn->getName()) && in UpgradeIntrinsicCall()
4307 CI->setCalledFunction(NewFn); in UpgradeIntrinsicCall()
4312 if (auto *OldST = dyn_cast<StructType>(CI->getType())) { in UpgradeIntrinsicCall()
4313 assert(OldST != NewFn->getReturnType() && in UpgradeIntrinsicCall()
4315 assert(OldST->getNumElements() == in UpgradeIntrinsicCall()
4316 cast<StructType>(NewFn->getReturnType())->getNumElements() && in UpgradeIntrinsicCall()
4319 SmallVector<Value *> Args(CI->args()); in UpgradeIntrinsicCall()
4322 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) { in UpgradeIntrinsicCall()
4326 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4327 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4333 CI->setCalledOperand( in UpgradeIntrinsicCall()
4334 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType())); in UpgradeIntrinsicCall()
4338 switch (NewFn->getIntrinsicID()) { in UpgradeIntrinsicCall()
4350 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4357 LLVMContext &Ctx = F->getParent()->getContext(); in UpgradeIntrinsicCall()
4358 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4360 cast<ConstantInt>(Args[3])->getZExtValue()); in UpgradeIntrinsicCall()
4367 StringRef Name = F->getName(); in UpgradeIntrinsicCall()
4374 auto *RetTy = cast<ScalableVectorType>(F->getReturnType()); in UpgradeIntrinsicCall()
4375 unsigned MinElts = RetTy->getMinNumElements() / N; in UpgradeIntrinsicCall()
4376 SmallVector<Value *, 2> Args(CI->args()); in UpgradeIntrinsicCall()
4389 SmallVector<Value *, 3> Args(CI->args()); in UpgradeIntrinsicCall()
4390 Args.push_back(ConstantTokenNone::get(CI->getContext())); in UpgradeIntrinsicCall()
4396 StringRef Name = F->getName(); in UpgradeIntrinsicCall()
4402 auto *RetTy = cast<ScalableVectorType>(F->getReturnType()); in UpgradeIntrinsicCall()
4403 unsigned MinElts = RetTy->getMinNumElements(); in UpgradeIntrinsicCall()
4404 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
4406 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx}); in UpgradeIntrinsicCall()
4411 StringRef Name = F->getName(); in UpgradeIntrinsicCall()
4418 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); in UpgradeIntrinsicCall()
4419 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType()); in UpgradeIntrinsicCall()
4421 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements()); in UpgradeIntrinsicCall()
4423 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx}); in UpgradeIntrinsicCall()
4432 assert(N > 1 && "Create is expected to be between 2-4"); in UpgradeIntrinsicCall()
4433 auto *RetTy = cast<ScalableVectorType>(F->getReturnType()); in UpgradeIntrinsicCall()
4435 unsigned MinElts = RetTy->getMinNumElements() / N; in UpgradeIntrinsicCall()
4438 Value *V = CI->getArgOperand(I); in UpgradeIntrinsicCall()
4455 assert(CI->arg_size() == 3 && in UpgradeIntrinsicCall()
4458 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits(); in UpgradeIntrinsicCall()
4462 auto Iter = CI->args().begin(); in UpgradeIntrinsicCall()
4471 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
4476 assert(CI->arg_size() == 1 && in UpgradeIntrinsicCall()
4479 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()}); in UpgradeIntrinsicCall()
4484 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2); in UpgradeIntrinsicCall()
4486 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3); in UpgradeIntrinsicCall()
4488 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic}); in UpgradeIntrinsicCall()
4493 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
4497 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); in UpgradeIntrinsicCall()
4501 StringRef Name = F->getName(); in UpgradeIntrinsicCall()
4506 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata()); in UpgradeIntrinsicCall()
4509 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4515 assert(CI->arg_size() == 4); in UpgradeIntrinsicCall()
4517 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1))) in UpgradeIntrinsicCall()
4518 if (Offset->isZeroValue()) { in UpgradeIntrinsicCall()
4521 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)}); in UpgradeIntrinsicCall()
4524 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4530 if (CI->arg_size() != 4) { in UpgradeIntrinsicCall()
4537 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4538 CI->getArgOperand(2), CI->getArgOperand(3), in UpgradeIntrinsicCall()
4540 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4541 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
4542 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4547 if (CI->arg_size() != 4) { in UpgradeIntrinsicCall()
4553 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4554 CI->getArgOperand(2), CI->getArgOperand(3), in UpgradeIntrinsicCall()
4556 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4557 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
4558 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4569 Value *Arg2 = CI->getArgOperand(2); in UpgradeIntrinsicCall()
4570 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64)) in UpgradeIntrinsicCall()
4573 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4574 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4575 if (CI->getType()->isIntegerTy(64)) { in UpgradeIntrinsicCall()
4581 cast<ConstantInt>(Arg2)->getZExtValue()); in UpgradeIntrinsicCall()
4585 if (Res->getType() != CI->getType()) in UpgradeIntrinsicCall()
4586 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true); in UpgradeIntrinsicCall()
4587 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4588 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4589 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4600 if (!CI->getType()->isIntegerTy(64)) in UpgradeIntrinsicCall()
4604 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty()); in UpgradeIntrinsicCall()
4608 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true); in UpgradeIntrinsicCall()
4609 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4610 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4611 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4617 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)}); in UpgradeIntrinsicCall()
4624 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4625 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType()); in UpgradeIntrinsicCall()
4639 Value *Arg0 = CI->getArgOperand(0); in UpgradeIntrinsicCall()
4640 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4)) in UpgradeIntrinsicCall()
4644 Value *Arg1 = CI->getArgOperand(1); in UpgradeIntrinsicCall()
4658 if (CI->getNumOperands() == 0) in UpgradeIntrinsicCall()
4665 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0), in UpgradeIntrinsicCall()
4666 llvm::PointerType::getUnqual(Data->getType())); in UpgradeIntrinsicCall()
4671 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4672 CI->replaceAllUsesWith(TSC); in UpgradeIntrinsicCall()
4673 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4683 // Need to truncate the last argument from i32 to i8 -- this argument models in UpgradeIntrinsicCall()
4684 // an inherently 8-bit immediate operand to these x86 instructions. in UpgradeIntrinsicCall()
4685 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4699 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4701 cast<FixedVectorType>(Args[0]->getType())->getNumElements(); in UpgradeIntrinsicCall()
4707 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4708 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4709 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4719 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4720 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements(); in UpgradeIntrinsicCall()
4721 if (NewFn->getIntrinsicID() == in UpgradeIntrinsicCall()
4730 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4731 CI->replaceAllUsesWith(Res); in UpgradeIntrinsicCall()
4732 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4738 SmallVector<Value *, 4> Args(CI->args()); in UpgradeIntrinsicCall()
4740 cast<FixedVectorType>(CI->getType())->getNumElements() * 2; in UpgradeIntrinsicCall()
4761 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1) in UpgradeIntrinsicCall()
4763 // -> @llvm.memset...(i8*, i8, i[32|64], i1) in UpgradeIntrinsicCall()
4765 if (CI->arg_size() != 5) { in UpgradeIntrinsicCall()
4771 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1), in UpgradeIntrinsicCall()
4772 CI->getArgOperand(2), CI->getArgOperand(4)}; in UpgradeIntrinsicCall()
4774 AttributeList OldAttrs = CI->getAttributes(); in UpgradeIntrinsicCall()
4779 NewCall->setAttributes(NewAttrs); in UpgradeIntrinsicCall()
4782 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3)); in UpgradeIntrinsicCall()
4783 MemCI->setDestAlignment(Align->getMaybeAlignValue()); in UpgradeIntrinsicCall()
4786 MTI->setSourceAlignment(Align->getMaybeAlignValue()); in UpgradeIntrinsicCall()
4792 NewCall->takeName(CI); in UpgradeIntrinsicCall()
4793 CI->replaceAllUsesWith(NewCall); in UpgradeIntrinsicCall()
4794 CI->eraseFromParent(); in UpgradeIntrinsicCall()
4798 assert(F && "Illegal attempt to upgrade a non-existent intrinsic."); in UpgradeCallsToIntrinsic()
4806 for (User *U : make_early_inc_range(F->users())) in UpgradeCallsToIntrinsic()
4811 F->eraseFromParent(); in UpgradeCallsToIntrinsic()
4820 // Check if the tag uses struct-path aware TBAA format. in UpgradeTBAANode()
4847 Type *SrcTy = V->getType(); in UpgradeBitCastInst()
4848 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() && in UpgradeBitCastInst()
4849 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) { in UpgradeBitCastInst()
4850 LLVMContext &Context = V->getContext(); in UpgradeBitCastInst()
4867 Type *SrcTy = C->getType(); in UpgradeBitCastExpr()
4868 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() && in UpgradeBitCastExpr()
4869 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) { in UpgradeBitCastExpr()
4870 LLVMContext &Context = C->getContext(); in UpgradeBitCastExpr()
4883 /// Check the debug info version number, if it is out-dated, drop the debug
4919 MDNode *Op = ModRetainReleaseMarker->getOperand(0); in upgradeRetainReleaseMarker()
4921 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0)); in upgradeRetainReleaseMarker()
4924 ID->getString().split(ValueComp, "#"); in upgradeRetainReleaseMarker()
4950 for (User *U : make_early_inc_range(Fn->users())) { in UpgradeARCRuntime()
4952 if (!CI || CI->getCalledFunction() != Fn) in UpgradeARCRuntime()
4955 IRBuilder<> Builder(CI->getParent(), CI->getIterator()); in UpgradeARCRuntime()
4956 FunctionType *NewFuncTy = NewFn->getFunctionType(); in UpgradeARCRuntime()
4961 if (NewFuncTy->getReturnType() != CI->getType() && in UpgradeARCRuntime()
4963 NewFuncTy->getReturnType())) in UpgradeARCRuntime()
4968 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) { in UpgradeARCRuntime()
4969 Value *Arg = CI->getArgOperand(I); in UpgradeARCRuntime()
4973 if (I < NewFuncTy->getNumParams()) { in UpgradeARCRuntime()
4977 NewFuncTy->getParamType(I))) { in UpgradeARCRuntime()
4981 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I)); in UpgradeARCRuntime()
4991 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind()); in UpgradeARCRuntime()
4992 NewCall->takeName(CI); in UpgradeARCRuntime()
4995 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType()); in UpgradeARCRuntime()
4997 if (!CI->use_empty()) in UpgradeARCRuntime()
4998 CI->replaceAllUsesWith(NewRetVal); in UpgradeARCRuntime()
4999 CI->eraseFromParent(); in UpgradeARCRuntime()
5002 if (Fn->use_empty()) in UpgradeARCRuntime()
5003 Fn->eraseFromParent(); in UpgradeARCRuntime()
5071 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) { in UpgradeModuleFlags()
5072 MDNode *Op = ModFlags->getOperand(I); in UpgradeModuleFlags()
5073 if (Op->getNumOperands() != 3) in UpgradeModuleFlags()
5075 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1)); in UpgradeModuleFlags()
5081 MDString::get(M.getContext(), ID->getString()), in UpgradeModuleFlags()
5082 Op->getOperand(2)}; in UpgradeModuleFlags()
5083 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops)); in UpgradeModuleFlags()
5087 if (ID->getString() == "Objective-C Image Info Version") in UpgradeModuleFlags()
5089 if (ID->getString() == "Objective-C Class Properties") in UpgradeModuleFlags()
5091 // Upgrade PIC from Error/Max to Min. in UpgradeModuleFlags()
5092 if (ID->getString() == "PIC Level") { in UpgradeModuleFlags()
5094 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) { in UpgradeModuleFlags()
5095 uint64_t V = Behavior->getLimitedValue(); in UpgradeModuleFlags()
5101 if (ID->getString() == "PIE Level") in UpgradeModuleFlags()
5103 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) in UpgradeModuleFlags()
5104 if (Behavior->getLimitedValue() == Module::Error) in UpgradeModuleFlags()
5109 if (ID->getString() == "branch-target-enforcement" || in UpgradeModuleFlags()
5110 ID->getString().starts_with("sign-return-address")) { in UpgradeModuleFlags()
5112 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) { in UpgradeModuleFlags()
5113 if (Behavior->getLimitedValue() == Module::Error) { in UpgradeModuleFlags()
5117 Op->getOperand(1), Op->getOperand(2)}; in UpgradeModuleFlags()
5118 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops)); in UpgradeModuleFlags()
5124 // Upgrade Objective-C Image Info Section. Removed the whitespce in the in UpgradeModuleFlags()
5125 // section name so that llvm-lto will not complain about mismatching in UpgradeModuleFlags()
5127 if (ID->getString() == "Objective-C Image Info Section") { in UpgradeModuleFlags()
5128 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) { in UpgradeModuleFlags()
5130 Value->getString().split(ValueComp, " "); in UpgradeModuleFlags()
5135 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1), in UpgradeModuleFlags()
5137 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops)); in UpgradeModuleFlags()
5143 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value. in UpgradeModuleFlags()
5145 if (ID->getString() == "Objective-C Garbage Collection") { in UpgradeModuleFlags()
5146 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2)); in UpgradeModuleFlags()
5148 assert(Md->getValue() && "Expected non-empty metadata"); in UpgradeModuleFlags()
5149 auto Type = Md->getValue()->getType(); in UpgradeModuleFlags()
5152 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue(); in UpgradeModuleFlags()
5161 Op->getOperand(1), in UpgradeModuleFlags()
5163 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops)); in UpgradeModuleFlags()
5168 if (ID->getString() == "amdgpu_code_object_version") { in UpgradeModuleFlags()
5170 Op->getOperand(0), in UpgradeModuleFlags()
5172 Op->getOperand(2)}; in UpgradeModuleFlags()
5173 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops)); in UpgradeModuleFlags()
5178 // "Objective-C Class Properties" is recently added for Objective-C. We in UpgradeModuleFlags()
5179 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module in UpgradeModuleFlags()
5184 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties", in UpgradeModuleFlags()
5203 auto TrimSpaces = [](StringRef Section) -> std::string { in UpgradeSectionAttributes()
5272 // Older versions of LLVM treated an "implicit-section-name" attribute in UpgradeFunctionAttributes()
5274 if (Attribute A = F.getFnAttribute("implicit-section-name"); in UpgradeFunctionAttributes()
5277 F.removeFnAttr("implicit-section-name"); in UpgradeFunctionAttributes()
5285 if (T->getNumOperands() < 1) in isOldLoopArgument()
5287 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0)); in isOldLoopArgument()
5290 return S->getString().starts_with("llvm.vectorizer."); in isOldLoopArgument()
5309 if (T->getNumOperands() < 1) in upgradeLoopArgument()
5311 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0)); in upgradeLoopArgument()
5314 if (!OldTag->getString().starts_with("llvm.vectorizer.")) in upgradeLoopArgument()
5319 Ops.reserve(T->getNumOperands()); in upgradeLoopArgument()
5320 Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString())); in upgradeLoopArgument()
5321 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I) in upgradeLoopArgument()
5322 Ops.push_back(T->getOperand(I)); in upgradeLoopArgument()
5324 return MDTuple::get(T->getContext(), Ops); in upgradeLoopArgument()
5332 if (none_of(T->operands(), isOldLoopArgument)) in upgradeInstructionLoopAttachment()
5336 Ops.reserve(T->getNumOperands()); in upgradeInstructionLoopAttachment()
5337 for (Metadata *MD : T->operands()) in upgradeInstructionLoopAttachment()
5340 return MDTuple::get(T->getContext(), Ops); in upgradeInstructionLoopAttachment()
5345 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting in UpgradeDataLayoutString()
5349 !DL.contains("-G") && !DL.starts_with("G")) { in UpgradeDataLayoutString()
5350 return DL.empty() ? std::string("G1") : (DL + "-G1").str(); in UpgradeDataLayoutString()
5354 // Make i32 a native type for 64-bit LoongArch and RISC-V. in UpgradeDataLayoutString()
5355 auto I = DL.find("-n64-"); in UpgradeDataLayoutString()
5357 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str(); in UpgradeDataLayoutString()
5365 if (!DL.contains("-G") && !DL.starts_with("G")) in UpgradeDataLayoutString()
5366 Res.append(Res.empty() ? "G1" : "-G1"); in UpgradeDataLayoutString()
5368 // Add missing non-integral declarations. in UpgradeDataLayoutString()
5371 if (!DL.contains("-ni") && !DL.starts_with("ni")) in UpgradeDataLayoutString()
5372 Res.append("-ni:7:8:9"); in UpgradeDataLayoutString()
5381 if (!DL.contains("-p7") && !DL.starts_with("p7")) in UpgradeDataLayoutString()
5382 Res.append("-p7:160:256:256:32"); in UpgradeDataLayoutString()
5383 if (!DL.contains("-p8") && !DL.starts_with("p8")) in UpgradeDataLayoutString()
5384 Res.append("-p8:128:128"); in UpgradeDataLayoutString()
5385 if (!DL.contains("-p9") && !DL.starts_with("p9")) in UpgradeDataLayoutString()
5386 Res.append("-p9:192:256:256:32"); in UpgradeDataLayoutString()
5393 // Add "-Fn32" in UpgradeDataLayoutString()
5394 if (!DL.empty() && !DL.contains("-Fn32")) in UpgradeDataLayoutString()
5395 Res.append("-Fn32"); in UpgradeDataLayoutString()
5404 std::string AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64"; in UpgradeDataLayoutString()
5407 Regex R("(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)"); in UpgradeDataLayoutString()
5412 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc in UpgradeDataLayoutString()
5417 // Intel MCU is an exception and uses 4-byte-alignment. in UpgradeDataLayoutString()
5419 std::string I128 = "-i128:128"; in UpgradeDataLayoutString()
5422 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$"); in UpgradeDataLayoutString()
5428 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes. in UpgradeDataLayoutString()
5433 auto I = Ref.find("-f80:32-"); in UpgradeDataLayoutString()
5435 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str(); in UpgradeDataLayoutString()
5443 Attribute A = B.getAttribute("no-frame-pointer-elim"); in UpgradeAttributes()
5447 B.removeAttribute("no-frame-pointer-elim"); in UpgradeAttributes()
5449 if (B.contains("no-frame-pointer-elim-non-leaf")) { in UpgradeAttributes()
5450 // The value is ignored. "no-frame-pointer-elim"="true" takes priority. in UpgradeAttributes()
5452 FramePointer = "non-leaf"; in UpgradeAttributes()
5453 B.removeAttribute("no-frame-pointer-elim-non-leaf"); in UpgradeAttributes()
5456 B.addAttribute("frame-pointer", FramePointer); in UpgradeAttributes()
5458 A = B.getAttribute("null-pointer-is-valid"); in UpgradeAttributes()
5462 B.removeAttribute("null-pointer-is-valid"); in UpgradeAttributes()