Lines Matching refs:MIRBuilder
107 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()), in LegalizerHelper()
114 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI), in LegalizerHelper()
122 MIRBuilder.setInstrAndDebugLoc(MI); in legalizeInstrStep()
171 MIRBuilder.buildMergeLikeInstr(DstReg, PartRegs); in insertParts()
176 MIRBuilder.buildConcatVectors(DstReg, PartRegs); in insertParts()
178 MIRBuilder.buildBuildVector(DstReg, PartRegs); in insertParts()
204 MIRBuilder, MRI); in appendVectorElts()
221 MIRBuilder.buildMergeLikeInstr(DstReg, AllElts); in mergeMixedSubvectors()
245 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); in extractGCDType()
273 PadReg = MIRBuilder.buildConstant(GCDTy, 0).getReg(0); in buildLCMMergePieces()
275 PadReg = MIRBuilder.buildUndef(GCDTy).getReg(0); in buildLCMMergePieces()
281 MIRBuilder.buildConstant(LLT::scalar(64), GCDTy.getSizeInBits() - 1); in buildLCMMergePieces()
282 PadReg = MIRBuilder.buildAShr(GCDTy, VRegs.back(), ShiftAmt).getReg(0); in buildLCMMergePieces()
320 AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0); in buildLCMMergePieces()
322 AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0); in buildLCMMergePieces()
338 Remerge[I] = MIRBuilder.buildMergeLikeInstr(NarrowTy, SubMerge).getReg(0); in buildLCMMergePieces()
357 MIRBuilder.buildMergeLikeInstr(DstReg, RemergeRegs); in buildWidenedRemergeToDst()
361 auto Remerge = MIRBuilder.buildMergeLikeInstr(LCMTy, RemergeRegs); in buildWidenedRemergeToDst()
363 MIRBuilder.buildTrunc(DstReg, Remerge); in buildWidenedRemergeToDst()
374 MIRBuilder.buildUnmerge(UnmergeDefs, in buildWidenedRemergeToDst()
375 MIRBuilder.buildMergeLikeInstr(LCMTy, RemergeRegs)); in buildWidenedRemergeToDst()
565 llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, in createLibcall() argument
570 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); in createLibcall()
579 Result.Ty == MIRBuilder.getMF().getFunction().getReturnType()) && in createLibcall()
580 isLibCallInTailPosition(Result, *MI, MIRBuilder.getTII(), in createLibcall()
581 *MIRBuilder.getMRI()); in createLibcall()
584 if (!CLI.lowerCall(MIRBuilder, Info)) in createLibcall()
612 llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, in createLibcall() argument
616 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); in createLibcall()
621 return createLibcall(MIRBuilder, Name, Result, Args, CC, LocObserver, MI); in createLibcall()
626 simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, in simpleLibcall() argument
634 return createLibcall(MIRBuilder, Libcall, in simpleLibcall()
640 llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, in createMemLibcall() argument
642 auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); in createMemLibcall()
659 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); in createMemLibcall()
660 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); in createMemLibcall()
687 << MIRBuilder.getTII().getName(Opc) << "\n"); in createMemLibcall()
697 isLibCallInTailPosition(Info.OrigRet, MI, MIRBuilder.getTII(), MRI); in createMemLibcall()
700 if (!CLI.lowerCall(MIRBuilder, Info)) in createMemLibcall()
777 createAtomicLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI) { in createAtomicLibcall() argument
778 auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); in createAtomicLibcall()
816 MIRBuilder.buildXor(ValLLT, MIRBuilder.buildConstant(ValLLT, -1), Val) in createAtomicLibcall()
820 MIRBuilder.buildSub(ValLLT, MIRBuilder.buildConstant(ValLLT, 0), Val) in createAtomicLibcall()
830 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); in createAtomicLibcall()
831 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); in createAtomicLibcall()
838 << MIRBuilder.getTII().getName(Opc) << "\n"); in createAtomicLibcall()
848 if (!CLI.lowerCall(MIRBuilder, Info)) in createAtomicLibcall()
877 conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, in conversionLibcall() argument
881 MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType, 0}, in conversionLibcall()
924 LegalizerHelper::createGetStateLibcall(MachineIRBuilder &MIRBuilder, in createGetStateLibcall() argument
927 const DataLayout &DL = MIRBuilder.getDataLayout(); in createGetStateLibcall()
928 auto &MF = MIRBuilder.getMF(); in createGetStateLibcall()
929 auto &MRI = *MIRBuilder.getMRI(); in createGetStateLibcall()
945 createLibcall(MIRBuilder, RTLibcall, in createGetStateLibcall()
955 MIRBuilder.buildLoadInstr(TargetOpcode::G_LOAD, Dst, Temp, *MMO); in createGetStateLibcall()
964 LegalizerHelper::createSetStateLibcall(MachineIRBuilder &MIRBuilder, in createSetStateLibcall() argument
967 const DataLayout &DL = MIRBuilder.getDataLayout(); in createSetStateLibcall()
968 auto &MF = MIRBuilder.getMF(); in createSetStateLibcall()
969 auto &MRI = *MIRBuilder.getMRI(); in createSetStateLibcall()
983 MIRBuilder.buildStore(Src, Temp, *MMO); in createSetStateLibcall()
989 return createLibcall(MIRBuilder, RTLibcall, in createSetStateLibcall()
1001 LegalizerHelper::createResetStateLibcall(MachineIRBuilder &MIRBuilder, in createResetStateLibcall() argument
1004 const DataLayout &DL = MIRBuilder.getDataLayout(); in createResetStateLibcall()
1005 auto &MF = MIRBuilder.getMF(); in createResetStateLibcall()
1013 auto DefValue = MIRBuilder.buildConstant(LLT::scalar(PtrSize), -1LL); in createResetStateLibcall()
1015 MIRBuilder.buildIntToPtr(Dest, DefValue); in createResetStateLibcall()
1018 return createLibcall(MIRBuilder, RTLibcall, in createResetStateLibcall()
1026 auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); in libcall()
1040 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy, LocObserver); in libcall()
1083 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy, LocObserver); in libcall()
1101 createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ITy, 0}, in libcall()
1123 createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), HLTy, 0}, in libcall()
1136 conversionLibcall(MI, MIRBuilder, ToTy, FromTy, LocObserver); in libcall()
1150 MI, MIRBuilder, Type::getIntNTy(Ctx, ToSize), FromTy, LocObserver); in libcall()
1163 MI, MIRBuilder, ToTy, Type::getIntNTy(Ctx, FromSize), LocObserver); in libcall()
1176 auto Status = createAtomicLibcall(MIRBuilder, MI); in libcall()
1186 createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI, LocObserver); in libcall()
1194 LegalizeResult Result = createGetStateLibcall(MIRBuilder, MI, LocObserver); in libcall()
1201 LegalizeResult Result = createSetStateLibcall(MIRBuilder, MI, LocObserver); in libcall()
1209 createResetStateLibcall(MIRBuilder, MI, LocObserver); in libcall()
1244 Register ImplicitReg = MIRBuilder.buildUndef(ImplicitTy).getReg(0); in narrowScalar()
1245 MIRBuilder.buildAnyExt(DstReg, ImplicitReg); in narrowScalar()
1255 DstRegs.push_back(MIRBuilder.buildUndef(NarrowTy).getReg(0)); in narrowScalar()
1258 MIRBuilder.buildBuildVector(DstReg, DstRegs); in narrowScalar()
1260 MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs); in narrowScalar()
1274 auto K = MIRBuilder.buildConstant(NarrowTy, in narrowScalar()
1284 auto K = MIRBuilder.buildConstant( in narrowScalar()
1310 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); in narrowScalar()
1311 MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0)); in narrowScalar()
1325 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1).getReg()); in narrowScalar()
1329 MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, {Unmerge.getReg(i)}) in narrowScalar()
1333 MIRBuilder.buildMergeLikeInstr(MI.getOperand(0).getReg(), Parts); in narrowScalar()
1364 MIRBuilder.buildLoad(TmpReg, LoadMI.getPointerReg(), LoadMI.getMMO()); in narrowScalar()
1365 MIRBuilder.buildAnyExt(DstReg, TmpReg); in narrowScalar()
1383 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); in narrowScalar()
1385 MIRBuilder.buildLoadInstr(LoadMI.getOpcode(), TmpReg, PtrReg, MMO); in narrowScalar()
1392 MIRBuilder.buildZExt(DstReg, TmpReg); in narrowScalar()
1394 MIRBuilder.buildSExt(DstReg, TmpReg); in narrowScalar()
1415 MIRBuilder.buildTrunc(TmpReg, SrcReg); in narrowScalar()
1416 MIRBuilder.buildStore(TmpReg, StoreMI.getPointerReg(), StoreMI.getMMO()); in narrowScalar()
1494 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminatorForward()); in narrowScalar()
1496 SrcRegs[i / 2], MIRBuilder, MRI); in narrowScalar()
1499 MIRBuilder.setInsertPt(MBB, MI); in narrowScalar()
1503 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]); in narrowScalar()
1507 MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI()); in narrowScalar()
1508 MIRBuilder.buildMergeLikeInstr(MI.getOperand(0), DstRegs); in narrowScalar()
1538 LHSLeftoverRegs, MIRBuilder, MRI)) in narrowScalar()
1544 RHSPartRegs, RHSLeftoverRegs, MIRBuilder, MRI)) in narrowScalar()
1555 auto Zero = MIRBuilder.buildConstant(NarrowTy, 0); in narrowScalar()
1560 auto Xor = MIRBuilder.buildXor(NarrowTy, LHS, RHS).getReg(0); in narrowScalar()
1570 auto Xor = MIRBuilder.buildXor(LeftoverTy, LHS, RHS).getReg(0); in narrowScalar()
1581 auto Or = MIRBuilder.buildOr(NarrowTy, Xors[0], Xors[1]); in narrowScalar()
1583 Or = MIRBuilder.buildOr(NarrowTy, Or, Xors[I]); in narrowScalar()
1584 MIRBuilder.buildICmp(Pred, Dst, Or, Zero); in narrowScalar()
1593 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH); in narrowScalar()
1595 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH); in narrowScalar()
1596 MachineInstrBuilder CmpLU = MIRBuilder.buildICmp( in narrowScalar()
1598 MIRBuilder.buildSelect(Dst, CmpHEQ, CmpLU, CmpH); in narrowScalar()
1625 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1); in narrowScalar()
1630 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in narrowScalar()
1631 MIRBuilder.buildSExt(MO2, DstExt); in narrowScalar()
1658 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1)); in narrowScalar()
1661 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1) in narrowScalar()
1679 MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg) in narrowScalar()
1684 MIRBuilder in narrowScalar()
1695 MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs); in narrowScalar()
1708 MIRBuilder, MRI); in narrowScalar()
1711 auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, in narrowScalar()
1716 MIRBuilder.buildMergeLikeInstr(MI.getOperand(0), DstRegs); in narrowScalar()
1750 auto VScaleBase = MIRBuilder.buildVScale(NarrowTy, One); in narrowScalar()
1751 auto ZExt = MIRBuilder.buildZExt(Ty, VScaleBase); in narrowScalar()
1752 auto C = MIRBuilder.buildConstant(Ty, *MI.getOperand(1).getCImm()); in narrowScalar()
1753 MIRBuilder.buildMul(Dst, ZExt, C); in narrowScalar()
1766 const DataLayout &DL = MIRBuilder.getDataLayout(); in coerceToScalar()
1771 return MIRBuilder.buildPtrToInt(NewTy, Val).getReg(0); in coerceToScalar()
1778 NewVal = MIRBuilder.buildPtrToInt(NewTy, NewVal).getReg(0); in coerceToScalar()
1779 return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0); in coerceToScalar()
1785 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO}); in widenScalarSrc()
1792 auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO); in narrowScalarSrc()
1800 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in widenScalarDst()
1801 MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt}); in widenScalarDst()
1809 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in narrowScalarDst()
1810 MIRBuilder.buildInstr(ExtOpcode, {MO}, {DstTrunc}); in narrowScalarDst()
1817 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in moreElementsVectorDst()
1821 MIRBuilder.buildDeleteTrailingVectorElements(Dst, DstExt); in moreElementsVectorDst()
1828 MO.setReg(MIRBuilder.buildPadVectorWithUndefElements(MoreTy, MO).getReg(0)); in moreElementsVectorSrc()
1833 Op.setReg(MIRBuilder.buildBitcast(CastTy, Op).getReg(0)); in bitcastSrc()
1839 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in bitcastDst()
1840 MIRBuilder.buildBitcast(MO, CastDst); in bitcastDst()
1866 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1Reg).getReg(0); in widenScalarMergeValues()
1874 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); in widenScalarMergeValues()
1879 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); in widenScalarMergeValues()
1880 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); in widenScalarMergeValues()
1881 MIRBuilder.buildOr(NextResult, ResultReg, Shl); in widenScalarMergeValues()
1886 MIRBuilder.buildTrunc(DstReg, ResultReg); in widenScalarMergeValues()
1888 MIRBuilder.buildIntToPtr(DstReg, ResultReg); in widenScalarMergeValues()
1929 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); in widenScalarMergeValues()
1937 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0); in widenScalarMergeValues()
1948 MIRBuilder.buildMergeLikeInstr(WideTy, Slicer.take_front(PartsPerGCD)); in widenScalarMergeValues()
1955 MIRBuilder.buildMergeLikeInstr(DstReg, NewMergeRegs); in widenScalarMergeValues()
1957 auto FinalMerge = MIRBuilder.buildMergeLikeInstr(WideDstTy, NewMergeRegs); in widenScalarMergeValues()
1958 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0)); in widenScalarMergeValues()
1984 const DataLayout &DL = MIRBuilder.getDataLayout(); in widenScalarUnmergeValues()
1992 SrcReg = MIRBuilder.buildPtrToInt(SrcTy, SrcReg).getReg(0); in widenScalarUnmergeValues()
2000 SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0); in widenScalarUnmergeValues()
2007 MIRBuilder.buildTrunc(Dst0Reg, SrcReg); in widenScalarUnmergeValues()
2009 auto ShiftAmt = MIRBuilder.buildConstant(SrcTy, DstSize * I); in widenScalarUnmergeValues()
2010 auto Shr = MIRBuilder.buildLShr(SrcTy, SrcReg, ShiftAmt); in widenScalarUnmergeValues()
2011 MIRBuilder.buildTrunc(MI.getOperand(I), Shr); in widenScalarUnmergeValues()
2029 WideSrc = MIRBuilder.buildAnyExt(LCMTy, WideSrc).getReg(0); in widenScalarUnmergeValues()
2032 auto Unmerge = MIRBuilder.buildUnmerge(WideTy, WideSrc); in widenScalarUnmergeValues()
2059 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); in widenScalarUnmergeValues()
2085 MIRBuilder.buildMergeLikeInstr(MI.getOperand(I).getReg(), RemergeParts); in widenScalarUnmergeValues()
2108 const DataLayout &DL = MIRBuilder.getDataLayout(); in widenScalarExtract()
2113 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src); in widenScalarExtract()
2122 MIRBuilder.buildTrunc(DstReg, in widenScalarExtract()
2123 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src)); in widenScalarExtract()
2131 Src = MIRBuilder.buildAnyExt(WideTy, Src); in widenScalarExtract()
2135 auto LShr = MIRBuilder.buildLShr( in widenScalarExtract()
2136 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset)); in widenScalarExtract()
2137 MIRBuilder.buildTrunc(DstReg, LShr); in widenScalarExtract()
2228 unsigned BoolExtOp = MIRBuilder.getBoolExtOp(WideTy.isVector(), false); in widenScalarAddSubOverflow()
2239 auto LHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(2)}); in widenScalarAddSubOverflow()
2240 auto RHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(3)}); in widenScalarAddSubOverflow()
2245 NewOp = MIRBuilder in widenScalarAddSubOverflow()
2250 NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSExt, RHSExt}).getReg(0); in widenScalarAddSubOverflow()
2253 auto TruncOp = MIRBuilder.buildTrunc(OrigTy, NewOp); in widenScalarAddSubOverflow()
2254 auto ExtOp = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {TruncOp}); in widenScalarAddSubOverflow()
2256 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, ExtOp); in widenScalarAddSubOverflow()
2258 MIRBuilder.buildTrunc(MI.getOperand(0), NewOp); in widenScalarAddSubOverflow()
2287 auto LHS = MIRBuilder.buildAnyExt(WideTy, MI.getOperand(1)); in widenScalarAddSubShlSat()
2288 auto RHS = IsShift ? MIRBuilder.buildZExt(WideTy, MI.getOperand(2)) in widenScalarAddSubShlSat()
2289 : MIRBuilder.buildAnyExt(WideTy, MI.getOperand(2)); in widenScalarAddSubShlSat()
2290 auto ShiftK = MIRBuilder.buildConstant(WideTy, SHLAmount); in widenScalarAddSubShlSat()
2291 auto ShiftL = MIRBuilder.buildShl(WideTy, LHS, ShiftK); in widenScalarAddSubShlSat()
2292 auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(WideTy, RHS, ShiftK); in widenScalarAddSubShlSat()
2294 auto WideInst = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, in widenScalarAddSubShlSat()
2299 auto Result = IsSigned ? MIRBuilder.buildAShr(WideTy, WideInst, ShiftK) in widenScalarAddSubShlSat()
2300 : MIRBuilder.buildLShr(WideTy, WideInst, ShiftK); in widenScalarAddSubShlSat()
2302 MIRBuilder.buildTrunc(DstReg, Result); in widenScalarAddSubShlSat()
2328 auto LeftOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {LHS}); in widenScalarMulo()
2329 auto RightOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {RHS}); in widenScalarMulo()
2340 Mulo = MIRBuilder.buildInstr(MulOpc, {WideTy, OverflowTy}, in widenScalarMulo()
2343 Mulo = MIRBuilder.buildInstr(MulOpc, {WideTy}, {LeftOperand, RightOperand}); in widenScalarMulo()
2346 MIRBuilder.buildTrunc(Result, Mul); in widenScalarMulo()
2355 ExtResult = MIRBuilder.buildSExtInReg(WideTy, Mul, SrcBitWidth); in widenScalarMulo()
2359 ExtResult = MIRBuilder.buildZExtInReg(WideTy, Mul, SrcBitWidth); in widenScalarMulo()
2364 MIRBuilder.buildICmp(CmpInst::ICMP_NE, OverflowTy, Mul, ExtResult); in widenScalarMulo()
2366 MIRBuilder.buildOr(OriginalOverflow, Mulo->getOperand(1), Overflow); in widenScalarMulo()
2368 MIRBuilder.buildICmp(CmpInst::ICMP_NE, OriginalOverflow, Mul, ExtResult); in widenScalarMulo()
2464 auto MIBSrc = MIRBuilder.buildInstr(ExtOpc, {WideTy}, {SrcReg}); in widenScalar()
2473 MIBSrc = MIRBuilder.buildOr( in widenScalar()
2474 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit)); in widenScalar()
2486 MIBSrc = MIRBuilder.buildShl(WideTy, MIBSrc, in widenScalar()
2487 MIRBuilder.buildConstant(WideTy, SizeDiff)); in widenScalar()
2491 auto MIBNewOp = MIRBuilder.buildInstr(NewOpc, {WideTy}, {MIBSrc}); in widenScalar()
2495 MIBNewOp = MIRBuilder.buildSub( in widenScalar()
2496 WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)); in widenScalar()
2499 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp); in widenScalar()
2514 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in widenScalar()
2518 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits); in widenScalar()
2519 MIRBuilder.buildLShr(ShrReg, DstExt, ShiftAmtReg); in widenScalar()
2521 MIRBuilder.buildTrunc(DstReg, ShrReg); in widenScalar()
2535 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in widenScalar()
2537 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits); in widenScalar()
2538 auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt); in widenScalar()
2539 MIRBuilder.buildTrunc(DstReg, Shift); in widenScalar()
2687 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false)); in widenScalar()
2753 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); in widenScalar()
2775 MIRBuilder.setInstrAndDebugLoc(MI); in widenScalar()
2776 auto IntCst = MIRBuilder.buildConstant(MI.getOperand(0).getReg(), Val); in widenScalar()
2789 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false)); in widenScalar()
2832 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminatorForward()); in widenScalar()
2837 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); in widenScalar()
3014 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BUILD_VECTOR_TRUNC)); in widenScalar()
3060 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); in widenScalar()
3090 MachineIRBuilder &MIRBuilder) { in emitLoadFromConstantPool() argument
3091 MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); in emitLoadFromConstantPool()
3092 MachineFunction &MF = MIRBuilder.getMF(); in emitLoadFromConstantPool()
3093 const DataLayout &DL = MIRBuilder.getDataLayout(); in emitLoadFromConstantPool()
3100 auto Addr = MIRBuilder.buildConstantPool( in emitLoadFromConstantPool()
3108 MIRBuilder.buildLoadInstr(TargetOpcode::G_LOAD, DstReg, Addr, *MMO); in emitLoadFromConstantPool()
3116 emitLoadFromConstantPool(MI.getOperand(0).getReg(), ConstantVal, MIRBuilder); in lowerConstant()
3127 emitLoadFromConstantPool(MI.getOperand(0).getReg(), ConstantVal, MIRBuilder); in lowerFConstant()
3175 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcPartTy); in lowerBitcast()
3177 SrcReg = MIRBuilder.buildBitcast(DstCastTy, SrcReg).getReg(0); in lowerBitcast()
3179 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcEltTy); in lowerBitcast()
3181 MIRBuilder.buildMergeLikeInstr(Dst, SrcRegs); in lowerBitcast()
3188 getUnmergePieces(SrcRegs, MIRBuilder, Src, DstTy.getElementType()); in lowerBitcast()
3189 MIRBuilder.buildMergeLikeInstr(Dst, SrcRegs); in lowerBitcast()
3238 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0); in bitcastExtractVectorElt()
3261 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(IdxTy, NewEltsPerOldElt); in bitcastExtractVectorElt()
3264 auto NewBaseIdx = MIRBuilder.buildMul(IdxTy, Idx, NewEltsPerOldEltK); in bitcastExtractVectorElt()
3267 auto IdxOffset = MIRBuilder.buildConstant(IdxTy, I); in bitcastExtractVectorElt()
3268 auto TmpIdx = MIRBuilder.buildAdd(IdxTy, NewBaseIdx, IdxOffset); in bitcastExtractVectorElt()
3269 auto Elt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, TmpIdx); in bitcastExtractVectorElt()
3273 auto NewVec = MIRBuilder.buildBuildVector(MidTy, NewOps); in bitcastExtractVectorElt()
3274 MIRBuilder.buildBitcast(Dst, NewVec); in bitcastExtractVectorElt()
3303 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio); in bitcastExtractVectorElt()
3306 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio); in bitcastExtractVectorElt()
3310 WideElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, in bitcastExtractVectorElt()
3316 MIRBuilder, Idx, NewEltSize, OldEltSize); in bitcastExtractVectorElt()
3319 auto ExtractedBits = MIRBuilder.buildLShr(NewEltTy, WideElt, OffsetBits); in bitcastExtractVectorElt()
3320 MIRBuilder.buildTrunc(Dst, ExtractedBits); in bitcastExtractVectorElt()
3379 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0); in bitcastInsertVectorElt()
3391 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio); in bitcastInsertVectorElt()
3394 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio); in bitcastInsertVectorElt()
3398 ExtractedElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, in bitcastInsertVectorElt()
3404 MIRBuilder, Idx, NewEltSize, OldEltSize); in bitcastInsertVectorElt()
3406 Register InsertedElt = buildBitFieldInsert(MIRBuilder, ExtractedElt, in bitcastInsertVectorElt()
3409 InsertedElt = MIRBuilder.buildInsertVectorElement( in bitcastInsertVectorElt()
3413 MIRBuilder.buildBitcast(Dst, InsertedElt); in bitcastInsertVectorElt()
3456 MIRBuilder.buildBitcast(SrcScalTy, ConcatMI->getSourceReg(i)) in bitcastConcatVector()
3462 MIRBuilder.buildBuildVector(CastTy, BitcastRegs).getReg(0); in bitcastConcatVector()
3463 MIRBuilder.buildBitcast(DstReg, BuildReg); in bitcastConcatVector()
3476 MachineFunction &MF = MIRBuilder.getMF(); in lowerLoad()
3502 auto NewLoad = MIRBuilder.buildLoad(LoadTy, PtrReg, *NewMMO); in lowerLoad()
3503 MIRBuilder.buildSExtInReg(LoadReg, NewLoad, MemSizeInBits); in lowerLoad()
3505 auto NewLoad = MIRBuilder.buildLoad(LoadTy, PtrReg, *NewMMO); in lowerLoad()
3508 MIRBuilder.buildAssertZExt(LoadReg, NewLoad, MemSizeInBits); in lowerLoad()
3510 MIRBuilder.buildLoad(LoadReg, PtrReg, *NewMMO); in lowerLoad()
3514 MIRBuilder.buildTrunc(DstReg, LoadReg); in lowerLoad()
3521 if (MIRBuilder.getDataLayout().isBigEndian()) in lowerLoad()
3551 if (TLI.allowsMemoryAccess(Ctx, MIRBuilder.getDataLayout(), MemTy, MMO)) in lowerLoad()
3575 auto LargeLoad = MIRBuilder.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, AnyExtTy, in lowerLoad()
3578 auto OffsetCst = MIRBuilder.buildConstant(LLT::scalar(PtrTy.getSizeInBits()), in lowerLoad()
3581 auto SmallPtr = MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst); in lowerLoad()
3582 auto SmallLoad = MIRBuilder.buildLoadInstr(LoadMI.getOpcode(), AnyExtTy, in lowerLoad()
3585 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize); in lowerLoad()
3586 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt); in lowerLoad()
3589 MIRBuilder.buildOr(DstReg, Shift, LargeLoad); in lowerLoad()
3591 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad); in lowerLoad()
3592 MIRBuilder.buildTrunc(DstReg, {Or}); in lowerLoad()
3595 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad); in lowerLoad()
3599 MIRBuilder.buildIntToPtr(DstReg, Or); in lowerLoad()
3616 MachineFunction &MF = MIRBuilder.getMF(); in lowerStore()
3634 SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0); in lowerStore()
3638 auto ZextInReg = MIRBuilder.buildZExtInReg(SrcTy, SrcReg, StoreWidth); in lowerStore()
3642 MIRBuilder.buildStore(ZextInReg, PtrReg, *NewMMO); in lowerStore()
3665 if (TLI.allowsMemoryAccess(Ctx, MIRBuilder.getDataLayout(), MemTy, MMO)) in lowerStore()
3679 SrcReg = MIRBuilder.buildPtrToInt(IntPtrTy, SrcReg).getReg(0); in lowerStore()
3682 auto ExtVal = MIRBuilder.buildAnyExtOrTrunc(NewSrcTy, SrcReg); in lowerStore()
3685 auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, LargeSplitSize); in lowerStore()
3686 auto SmallVal = MIRBuilder.buildLShr(NewSrcTy, ExtVal, ShiftAmt); in lowerStore()
3690 auto OffsetCst = MIRBuilder.buildConstant( in lowerStore()
3693 MIRBuilder.buildPtrAdd(PtrTy, PtrReg, OffsetCst); in lowerStore()
3699 MIRBuilder.buildStore(ExtVal, PtrReg, *LargeMMO); in lowerStore()
3700 MIRBuilder.buildStore(SmallVal, SmallPtr, *SmallMMO); in lowerStore()
3783 MI.setDesc(MIRBuilder.getTII().get(NewOpcode)); in changeOpcode()
3802 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, {Ty}, in lower()
3805 auto Prod = MIRBuilder.buildMul(Ty, Quot, MI.getOperand(2)); in lower()
3806 MIRBuilder.buildSub(MI.getOperand(0), MI.getOperand(1), Prod); in lower()
3828 const auto &TII = MIRBuilder.getTII(); in lower()
3833 auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS}); in lower()
3834 auto Zero = MIRBuilder.buildConstant(Ty, 0); in lower()
3837 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); in lower()
3842 auto ShiftAmt = MIRBuilder.buildConstant(Ty, Ty.getSizeInBits() - 1); in lower()
3843 auto Shifted = MIRBuilder.buildAShr(Ty, Res, ShiftAmt); in lower()
3844 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted); in lower()
3846 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero); in lower()
3859 MIRBuilder.buildConstant(Ty, APInt::getSignMask(Ty.getSizeInBits())); in lower()
3860 MIRBuilder.buildXor(Res, SubByReg, SignMask); in lower()
3870 auto Neg = MIRBuilder.buildFNeg(Ty, RHS); in lower()
3873 MIRBuilder.buildStrictFAdd(Res, LHS, Neg, MI.getFlags()); in lower()
3875 MIRBuilder.buildFAdd(Res, LHS, Neg, MI.getFlags()); in lower()
3895 MIRBuilder.buildAtomicCmpXchg(NewOldValRes, Addr, CmpVal, NewVal, in lower()
3897 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, NewOldValRes, CmpVal); in lower()
3898 MIRBuilder.buildCopy(OldValRes, NewOldValRes); in lower()
3919 MIRBuilder.buildAdd(NewRes, LHS, RHS); in lower()
3920 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, NewRes, RHS); in lower()
3922 MIRBuilder.buildCopy(Res, NewRes); in lower()
3935 auto TmpRes = MIRBuilder.buildAdd(Ty, LHS, RHS); in lower()
3938 auto Carry = MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CondTy, TmpRes, LHS); in lower()
3941 auto ZExtCarryIn = MIRBuilder.buildZExt(Ty, CarryIn); in lower()
3942 MIRBuilder.buildAdd(NewRes, TmpRes, ZExtCarryIn); in lower()
3946 auto Zero = MIRBuilder.buildConstant(Ty, 0); in lower()
3948 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, NewRes, Zero); in lower()
3949 auto Carry2 = MIRBuilder.buildAnd(CondTy, ResEqZero, CarryIn); in lower()
3950 MIRBuilder.buildOr(CarryOut, Carry, Carry2); in lower()
3952 MIRBuilder.buildCopy(Res, NewRes); in lower()
3960 MIRBuilder.buildSub(Res, LHS, RHS); in lower()
3961 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS); in lower()
3972 auto TmpRes = MIRBuilder.buildSub(Ty, LHS, RHS); in lower()
3975 auto Borrow = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, CondTy, TmpRes, LHS); in lower()
3978 auto ZExtBorrowIn = MIRBuilder.buildZExt(Ty, BorrowIn); in lower()
3979 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn); in lower()
3983 auto Zero = MIRBuilder.buildConstant(Ty, 0); in lower()
3985 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, TmpRes, Zero); in lower()
3986 auto Borrow2 = MIRBuilder.buildAnd(CondTy, TmpResEqZero, BorrowIn); in lower()
3987 MIRBuilder.buildOr(BorrowOut, Borrow, Borrow2); in lower()
4029 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits); in lower()
4030 MIRBuilder.buildShl(TmpRes, SrcReg, MIBSz->getOperand(0)); in lower()
4031 MIRBuilder.buildAShr(DstReg, TmpRes, MIBSz->getOperand(0)); in lower()
4130 MachineFunction &MF = MIRBuilder.getMF(); in createStackTemporary()
4131 const DataLayout &DL = MIRBuilder.getDataLayout(); in createStackTemporary()
4138 return MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx); in createStackTemporary()
4171 Index = clampVectorIndex(MIRBuilder, Index, VecTy); in getVectorElementPointer()
4174 const DataLayout &DL = MIRBuilder.getDataLayout(); in getVectorElementPointer()
4179 Index = MIRBuilder.buildSExtOrTrunc(IdxTy, Index).getReg(0); in getVectorElementPointer()
4181 auto Mul = MIRBuilder.buildMul(IdxTy, Index, in getVectorElementPointer()
4182 MIRBuilder.buildConstant(IdxTy, EltSize)); in getVectorElementPointer()
4185 return MIRBuilder.buildPtrAdd(PtrTy, VecPtr, Mul).getReg(0); in getVectorElementPointer()
4311 extractVectorParts(MI.getReg(UseIdx), NumElts, SplitPieces, MIRBuilder, in fewerElementsVectorMultiEltType()
4331 auto I = MIRBuilder.buildInstr(MI.getOpcode(), Defs, Uses, MI.getFlags()); in fewerElementsVectorMultiEltType()
4342 MIRBuilder.buildMergeLikeInstr(MI.getReg(i), OutputRegs[i]); in fewerElementsVectorMultiEltType()
4367 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminatorForward()); in fewerElementsVectorPhi()
4369 MIRBuilder, MRI); in fewerElementsVectorPhi()
4374 MIRBuilder.setInsertPt(*MI.getParent(), MI); in fewerElementsVectorPhi()
4376 auto Phi = MIRBuilder.buildInstr(TargetOpcode::G_PHI); in fewerElementsVectorPhi()
4389 MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI()); in fewerElementsVectorPhi()
4395 MIRBuilder.buildMergeLikeInstr(MI.getReg(0), OutputRegs); in fewerElementsVectorPhi()
4435 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, SrcReg); in fewerElementsVectorUnmergeValues()
4440 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); in fewerElementsVectorUnmergeValues()
4485 auto Unmerge = MIRBuilder.buildUnmerge(EltTy, MI.getOperand(i).getReg()); in fewerElementsVectorMerge()
4497 MIRBuilder.buildMergeLikeInstr(NarrowTy, Pieces).getReg(0)); in fewerElementsVectorMerge()
4500 MIRBuilder.buildMergeLikeInstr(DstReg, NarrowTyElts); in fewerElementsVectorMerge()
4529 MIRBuilder.buildMergeLikeInstr(NarrowTy, Sources).getReg(0)); in fewerElementsVectorMerge()
4532 MIRBuilder.buildMergeLikeInstr(DstReg, NarrowTyElts); in fewerElementsVectorMerge()
4565 MIRBuilder.buildUndef(DstReg); in fewerElementsVectorExtractInsertVectorElt()
4582 MIRBuilder.buildConstant(IdxTy, IdxVal - NewNumElts * PartIdx); in fewerElementsVectorExtractInsertVectorElt()
4588 auto InsertPart = MIRBuilder.buildInsertVectorElement( in fewerElementsVectorExtractInsertVectorElt()
4596 MIRBuilder.buildExtractVectorElement(DstReg, VecParts[PartIdx], NewIdx); in fewerElementsVectorExtractInsertVectorElt()
4642 NarrowLeftoverRegs, MIRBuilder, MRI)) { in reduceLoadStoreWidth()
4660 bool isBigEndian = MIRBuilder.getDataLayout().isBigEndian(); in reduceLoadStoreWidth()
4664 MachineFunction &MF = MIRBuilder.getMF(); in reduceLoadStoreWidth()
4671 MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset); in reduceLoadStoreWidth()
4679 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO); in reduceLoadStoreWidth()
4681 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO); in reduceLoadStoreWidth()
4898 MIRBuilder.buildBitcast(NarrowTy, SrcVRegs[i]).getReg(0)); in fewerElementsBitcast()
4900 MIRBuilder.buildMergeLikeInstr(DstReg, BitcastVRegs); in fewerElementsBitcast()
4930 extractParts(Src1Reg, NarrowTy, 2, SplitSrc1Regs, MIRBuilder, MRI); in fewerElementsVectorShuffle()
4931 extractParts(Src2Reg, NarrowTy, 2, SplitSrc2Regs, MIRBuilder, MRI); in fewerElementsVectorShuffle()
5005 SVOps.push_back(MIRBuilder.buildUndef(EltTy).getReg(0)); in fewerElementsVectorShuffle()
5013 SVOps.push_back(MIRBuilder in fewerElementsVectorShuffle()
5016 MIRBuilder.buildConstant(LLT::scalar(32), Idx)) in fewerElementsVectorShuffle()
5021 Output = MIRBuilder.buildBuildVector(NarrowTy, SVOps).getReg(0); in fewerElementsVectorShuffle()
5024 Output = MIRBuilder.buildUndef(NarrowTy).getReg(0); in fewerElementsVectorShuffle()
5029 ? MIRBuilder.buildUndef(NarrowTy).getReg(0) in fewerElementsVectorShuffle()
5032 Output = MIRBuilder.buildShuffleVector(NarrowTy, Op0, Op1, Ops).getReg(0); in fewerElementsVectorShuffle()
5038 MIRBuilder.buildConcatVectors(DstReg, {Lo, Hi}); in fewerElementsVectorShuffle()
5065 extractParts(SrcReg, NarrowTy, NumParts, SplitSrcs, MIRBuilder, MRI); in fewerElementsVectorReductions()
5077 MIRBuilder in fewerElementsVectorReductions()
5087 MIRBuilder.buildCopy(DstReg, SplitSrcs[0]); in fewerElementsVectorReductions()
5094 Acc = MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {Acc, SplitSrcs[Idx]}) in fewerElementsVectorReductions()
5096 MIRBuilder.buildCopy(DstReg, Acc); in fewerElementsVectorReductions()
5103 MIRBuilder.buildInstr(RdxMI.getOpcode(), {DstTy}, {SplitSrcs[Part]}) in fewerElementsVectorReductions()
5117 MIRBuilder.buildInstr(ScalarOpc, {DstReg}, in fewerElementsVectorReductions()
5120 Acc = MIRBuilder in fewerElementsVectorReductions()
5148 extractParts(SrcReg, NarrowTy, NumParts, SplitSrcs, MIRBuilder, MRI); in fewerElementsVectorSeqReductions()
5151 Acc = MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {Acc, SplitSrcs[i]}) in fewerElementsVectorSeqReductions()
5154 MIRBuilder.buildCopy(DstReg, Acc); in fewerElementsVectorSeqReductions()
5167 MIRBuilder, MRI); in tryNarrowPow2Reduction()
5177 MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {LHS, RHS}).getReg(0); in tryNarrowPow2Reduction()
5195 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); in narrowScalarShiftByConstant()
5198 MIRBuilder.buildMergeLikeInstr(MI.getOperand(0), {InL, InH}); in narrowScalarShiftByConstant()
5210 Lo = Hi = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
5212 Lo = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
5213 Hi = MIRBuilder.buildShl(NVT, InL, in narrowScalarShiftByConstant()
5214 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); in narrowScalarShiftByConstant()
5216 Lo = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
5219 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt)); in narrowScalarShiftByConstant()
5221 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt)); in narrowScalarShiftByConstant()
5222 auto OrRHS = MIRBuilder.buildLShr( in narrowScalarShiftByConstant()
5223 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); in narrowScalarShiftByConstant()
5224 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); in narrowScalarShiftByConstant()
5228 Lo = Hi = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
5230 Lo = MIRBuilder.buildLShr(NVT, InH, in narrowScalarShiftByConstant()
5231 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); in narrowScalarShiftByConstant()
5232 Hi = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
5235 Hi = MIRBuilder.buildConstant(NVT, 0); in narrowScalarShiftByConstant()
5237 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); in narrowScalarShiftByConstant()
5239 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); in narrowScalarShiftByConstant()
5240 auto OrRHS = MIRBuilder.buildShl( in narrowScalarShiftByConstant()
5241 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); in narrowScalarShiftByConstant()
5243 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); in narrowScalarShiftByConstant()
5244 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst); in narrowScalarShiftByConstant()
5248 Hi = Lo = MIRBuilder.buildAShr( in narrowScalarShiftByConstant()
5249 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); in narrowScalarShiftByConstant()
5251 Lo = MIRBuilder.buildAShr(NVT, InH, in narrowScalarShiftByConstant()
5252 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); in narrowScalarShiftByConstant()
5253 Hi = MIRBuilder.buildAShr(NVT, InH, in narrowScalarShiftByConstant()
5254 MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); in narrowScalarShiftByConstant()
5257 Hi = MIRBuilder.buildAShr(NVT, InH, in narrowScalarShiftByConstant()
5258 MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); in narrowScalarShiftByConstant()
5260 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); in narrowScalarShiftByConstant()
5262 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); in narrowScalarShiftByConstant()
5263 auto OrRHS = MIRBuilder.buildShl( in narrowScalarShiftByConstant()
5264 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); in narrowScalarShiftByConstant()
5266 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); in narrowScalarShiftByConstant()
5267 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst); in narrowScalarShiftByConstant()
5271 MIRBuilder.buildMergeLikeInstr(MI.getOperand(0), {Lo, Hi}); in narrowScalarShiftByConstant()
5314 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize); in narrowScalarShift()
5318 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); in narrowScalarShift()
5320 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits); in narrowScalarShift()
5321 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt); in narrowScalarShift()
5323 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0); in narrowScalarShift()
5324 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits); in narrowScalarShift()
5325 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero); in narrowScalarShift()
5331 auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt); in narrowScalarShift()
5333 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack); in narrowScalarShift()
5334 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt); in narrowScalarShift()
5335 auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); in narrowScalarShift()
5338 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero. in narrowScalarShift()
5339 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part. in narrowScalarShift()
5341 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL); in narrowScalarShift()
5342 auto Hi = MIRBuilder.buildSelect( in narrowScalarShift()
5343 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL)); in narrowScalarShift()
5352 auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt}); in narrowScalarShift()
5354 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt); in narrowScalarShift()
5355 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack); in narrowScalarShift()
5356 auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); in narrowScalarShift()
5361 HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero. in narrowScalarShift()
5363 auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1); in narrowScalarShift()
5364 HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part. in narrowScalarShift()
5366 auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, in narrowScalarShift()
5369 auto Lo = MIRBuilder.buildSelect( in narrowScalarShift()
5370 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL)); in narrowScalarShift()
5372 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL); in narrowScalarShift()
5382 MIRBuilder.buildMergeLikeInstr(DstReg, ResultRegs); in narrowScalarShift()
5395 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); in moreElementsVectorPhi()
5400 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); in moreElementsVectorPhi()
5407 unsigned Opcode, MachineIRBuilder &MIRBuilder, LLT Ty) { in getNeutralElementForVecReduce() argument
5418 return MIRBuilder.buildConstant(Ty, 0); in getNeutralElementForVecReduce()
5420 return MIRBuilder.buildConstant(Ty, 1); in getNeutralElementForVecReduce()
5423 return MIRBuilder.buildConstant( in getNeutralElementForVecReduce()
5426 return MIRBuilder.buildConstant( in getNeutralElementForVecReduce()
5429 return MIRBuilder.buildConstant( in getNeutralElementForVecReduce()
5432 return MIRBuilder.buildFConstant(Ty, -0.0); in getNeutralElementForVecReduce()
5434 return MIRBuilder.buildFConstant(Ty, 1.0); in getNeutralElementForVecReduce()
5555 auto ShufSplat = MIRBuilder.buildShuffleSplat(MoreTy, CondReg); in moreElementsVector()
5585 Elts.push_back(MIRBuilder.buildUndef(MoreTy.getScalarType())); in moreElementsVector()
5588 MIRBuilder.buildDeleteTrailingVectorElements( in moreElementsVector()
5589 MI.getOperand(0).getReg(), MIRBuilder.buildInstr(Opc, {MoreTy}, Elts)); in moreElementsVector()
5671 auto NewVec = MIRBuilder.buildPadVectorWithUndefElements(MoreTy, MO); in moreElementsVector()
5673 MI.getOpcode(), MIRBuilder, MoreTy.getElementType()); in moreElementsVector()
5675 LLT IdxTy(TLI.getVectorIdxTy(MIRBuilder.getDataLayout())); in moreElementsVector()
5678 auto Idx = MIRBuilder.buildConstant(IdxTy, i); in moreElementsVector()
5679 NewVec = MIRBuilder.buildInsertVectorElement(MoreTy, NewVec, in moreElementsVector()
5713 MIRBuilder.setInstrAndDebugLoc(MI); in equalizeVectorShuffleLengths()
5714 MIRBuilder.buildShuffleVector(MI.getOperand(0).getReg(), in equalizeVectorShuffleLengths()
5728 auto Undef = MIRBuilder.buildUndef(SrcTy); in equalizeVectorShuffleLengths()
5734 auto Src1 = MIRBuilder.buildConcatVectors(PaddedTy, MOps1); in equalizeVectorShuffleLengths()
5735 auto Src2 = MIRBuilder.buildConcatVectors(PaddedTy, MOps2); in equalizeVectorShuffleLengths()
5749 MIRBuilder.buildShuffleVector(PaddedTy, Src1, Src2, MappedOps); in equalizeVectorShuffleLengths()
5754 MIRBuilder.buildExtractVectorElementConstant(DestEltTy, Shuffle, I) in equalizeVectorShuffleLengths()
5757 MIRBuilder.buildBuildVector(DstReg, Elts); in equalizeVectorShuffleLengths()
5759 MIRBuilder.buildShuffleVector(DstReg, Src1, Src2, MappedOps); in equalizeVectorShuffleLengths()
5801 MIRBuilder.setInstrAndDebugLoc(MI); in moreElementsVectorShuffle()
5802 MIRBuilder.buildShuffleVector(MI.getOperand(0).getReg(), in moreElementsVectorShuffle()
5813 MachineIRBuilder &B = MIRBuilder; in multiplyRegisters()
5927 MIRBuilder, MRI); in narrowScalarAddSub()
5928 extractParts(Src2, RegTy, NarrowTy, DummyTy, Src2Regs, Src2Left, MIRBuilder, in narrowScalarAddSub()
5947 MIRBuilder.buildInstr(OpO, {DstReg, CarryOut}, in narrowScalarAddSub()
5950 MIRBuilder.buildInstr(OpF, {DstReg, CarryOut}, in narrowScalarAddSub()
5953 MIRBuilder.buildInstr(OpE, {DstReg, CarryOut}, in narrowScalarAddSub()
5987 extractParts(Src1, NarrowTy, NumParts, Src1Parts, MIRBuilder, MRI); in narrowScalarMul()
5988 extractParts(Src2, NarrowTy, NumParts, Src2Parts, MIRBuilder, MRI); in narrowScalarMul()
5993 MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs); in narrowScalarMul()
6041 MIRBuilder, MRI); in narrowScalarExtract()
6074 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset); in narrowScalarExtract()
6082 MIRBuilder.buildBuildVector(DstReg, DstRegs); in narrowScalarExtract()
6084 MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs); in narrowScalarExtract()
6086 MIRBuilder.buildCopy(DstReg, DstRegs[0]); in narrowScalarExtract()
6103 LeftoverRegs, MIRBuilder, MRI); in narrowScalarInsert()
6126 MIRBuilder.buildAnyExt(SrcReg, SrcRegs[I]); in narrowScalarInsert()
6154 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset); in narrowScalarInsert()
6158 MIRBuilder.buildInsert(DstReg, SrcReg, SegReg, InsertOffset); in narrowScalarInsert()
6166 MIRBuilder.buildMergeLikeInstr(MergeReg, DstRegs); in narrowScalarInsert()
6167 MIRBuilder.buildTrunc(DstReg, MergeReg); in narrowScalarInsert()
6169 MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs); in narrowScalarInsert()
6188 Src0Regs, Src0LeftoverRegs, MIRBuilder, MRI)) in narrowScalarBasic()
6193 Src1Regs, Src1LeftoverRegs, MIRBuilder, MRI)) in narrowScalarBasic()
6197 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, in narrowScalarBasic()
6203 auto Inst = MIRBuilder.buildInstr( in narrowScalarBasic()
6256 Src1Regs, Src1LeftoverRegs, MIRBuilder, MRI)) in narrowScalarSelect()
6261 Src2Regs, Src2LeftoverRegs, MIRBuilder, MRI)) in narrowScalarSelect()
6265 auto Select = MIRBuilder.buildSelect(NarrowTy, in narrowScalarSelect()
6271 auto Select = MIRBuilder.buildSelect( in narrowScalarSelect()
6295 MachineIRBuilder &B = MIRBuilder; in narrowScalarCTLZ()
6328 MachineIRBuilder &B = MIRBuilder; in narrowScalarCTTZ()
6359 auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); in narrowScalarCTPOP()
6361 auto LoCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(0)); in narrowScalarCTPOP()
6362 auto HiCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(1)); in narrowScalarCTPOP()
6363 MIRBuilder.buildAdd(DstReg, HiCTPOP, LoCTPOP); in narrowScalarCTPOP()
6378 MachineIRBuilder &B = MIRBuilder; in narrowScalarFLDEXP()
6400 const auto &TII = MIRBuilder.getTII(); in lowerBitCount()
6421 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_UNDEF(DstTy, SrcReg); in lowerBitCount()
6422 auto ZeroSrc = MIRBuilder.buildConstant(SrcTy, 0); in lowerBitCount()
6423 auto ICmp = MIRBuilder.buildICmp( in lowerBitCount()
6425 auto LenConst = MIRBuilder.buildConstant(DstTy, Len); in lowerBitCount()
6426 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CtlzZU); in lowerBitCount()
6444 auto MIBShiftAmt = MIRBuilder.buildConstant(SrcTy, 1ULL << i); in lowerBitCount()
6445 auto MIBOp = MIRBuilder.buildOr( in lowerBitCount()
6446 SrcTy, Op, MIRBuilder.buildLShr(SrcTy, Op, MIBShiftAmt)); in lowerBitCount()
6449 auto MIBPop = MIRBuilder.buildCTPOP(DstTy, Op); in lowerBitCount()
6450 MIRBuilder.buildSub(MI.getOperand(0), MIRBuilder.buildConstant(DstTy, Len), in lowerBitCount()
6469 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_UNDEF(DstTy, SrcReg); in lowerBitCount()
6470 auto Zero = MIRBuilder.buildConstant(SrcTy, 0); in lowerBitCount()
6471 auto ICmp = MIRBuilder.buildICmp( in lowerBitCount()
6473 auto LenConst = MIRBuilder.buildConstant(DstTy, Len); in lowerBitCount()
6474 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CttzZU); in lowerBitCount()
6482 auto MIBCstNeg1 = MIRBuilder.buildConstant(SrcTy, -1); in lowerBitCount()
6483 auto MIBNot = MIRBuilder.buildXor(SrcTy, SrcReg, MIBCstNeg1); in lowerBitCount()
6484 auto MIBTmp = MIRBuilder.buildAnd( in lowerBitCount()
6485 SrcTy, MIBNot, MIRBuilder.buildAdd(SrcTy, SrcReg, MIBCstNeg1)); in lowerBitCount()
6488 auto MIBCstLen = MIRBuilder.buildConstant(SrcTy, Len); in lowerBitCount()
6489 MIRBuilder.buildSub(MI.getOperand(0), MIBCstLen, in lowerBitCount()
6490 MIRBuilder.buildCTLZ(SrcTy, MIBTmp)); in lowerBitCount()
6504 MachineIRBuilder &B = MIRBuilder; in lowerBitCount()
6600 auto Zero = MIRBuilder.buildConstant(ShTy, 0); in lowerFunnelShiftWithInverse()
6601 Z = MIRBuilder.buildSub(Ty, Zero, Z).getReg(0); in lowerFunnelShiftWithInverse()
6605 auto One = MIRBuilder.buildConstant(ShTy, 1); in lowerFunnelShiftWithInverse()
6607 Y = MIRBuilder.buildInstr(RevOpcode, {Ty}, {X, Y, One}).getReg(0); in lowerFunnelShiftWithInverse()
6608 X = MIRBuilder.buildLShr(Ty, X, One).getReg(0); in lowerFunnelShiftWithInverse()
6610 X = MIRBuilder.buildInstr(RevOpcode, {Ty}, {X, Y, One}).getReg(0); in lowerFunnelShiftWithInverse()
6611 Y = MIRBuilder.buildShl(Ty, Y, One).getReg(0); in lowerFunnelShiftWithInverse()
6614 Z = MIRBuilder.buildNot(ShTy, Z).getReg(0); in lowerFunnelShiftWithInverse()
6617 MIRBuilder.buildInstr(RevOpcode, {Dst}, {X, Y, Z}); in lowerFunnelShiftWithInverse()
6639 auto BitWidthC = MIRBuilder.buildConstant(ShTy, BW); in lowerFunnelShiftAsShifts()
6640 ShAmt = MIRBuilder.buildURem(ShTy, Z, BitWidthC).getReg(0); in lowerFunnelShiftAsShifts()
6641 InvShAmt = MIRBuilder.buildSub(ShTy, BitWidthC, ShAmt).getReg(0); in lowerFunnelShiftAsShifts()
6642 ShX = MIRBuilder.buildShl(Ty, X, IsFSHL ? ShAmt : InvShAmt).getReg(0); in lowerFunnelShiftAsShifts()
6643 ShY = MIRBuilder.buildLShr(Ty, Y, IsFSHL ? InvShAmt : ShAmt).getReg(0); in lowerFunnelShiftAsShifts()
6647 auto Mask = MIRBuilder.buildConstant(ShTy, BW - 1); in lowerFunnelShiftAsShifts()
6650 ShAmt = MIRBuilder.buildAnd(ShTy, Z, Mask).getReg(0); in lowerFunnelShiftAsShifts()
6652 auto NotZ = MIRBuilder.buildNot(ShTy, Z); in lowerFunnelShiftAsShifts()
6653 InvShAmt = MIRBuilder.buildAnd(ShTy, NotZ, Mask).getReg(0); in lowerFunnelShiftAsShifts()
6655 auto BitWidthC = MIRBuilder.buildConstant(ShTy, BW); in lowerFunnelShiftAsShifts()
6656 ShAmt = MIRBuilder.buildURem(ShTy, Z, BitWidthC).getReg(0); in lowerFunnelShiftAsShifts()
6657 InvShAmt = MIRBuilder.buildSub(ShTy, Mask, ShAmt).getReg(0); in lowerFunnelShiftAsShifts()
6660 auto One = MIRBuilder.buildConstant(ShTy, 1); in lowerFunnelShiftAsShifts()
6662 ShX = MIRBuilder.buildShl(Ty, X, ShAmt).getReg(0); in lowerFunnelShiftAsShifts()
6663 auto ShY1 = MIRBuilder.buildLShr(Ty, Y, One); in lowerFunnelShiftAsShifts()
6664 ShY = MIRBuilder.buildLShr(Ty, ShY1, InvShAmt).getReg(0); in lowerFunnelShiftAsShifts()
6666 auto ShX1 = MIRBuilder.buildShl(Ty, X, One); in lowerFunnelShiftAsShifts()
6667 ShX = MIRBuilder.buildShl(Ty, ShX1, InvShAmt).getReg(0); in lowerFunnelShiftAsShifts()
6668 ShY = MIRBuilder.buildLShr(Ty, Y, ShAmt).getReg(0); in lowerFunnelShiftAsShifts()
6672 MIRBuilder.buildOr(Dst, ShX, ShY); in lowerFunnelShiftAsShifts()
6720 auto NewExt = MIRBuilder.buildInstr(MI.getOpcode(), {MidTy}, {Src}); in lowerEXT()
6724 auto UnmergeSrc = MIRBuilder.buildUnmerge(EltTy, NewExt); in lowerEXT()
6729 auto ZExtRes1 = MIRBuilder.buildInstr(MI.getOpcode(), {ZExtResTy}, in lowerEXT()
6731 auto ZExtRes2 = MIRBuilder.buildInstr(MI.getOpcode(), {ZExtResTy}, in lowerEXT()
6735 MIRBuilder.buildMergeLikeInstr(Dst, {ZExtRes1, ZExtRes2}); in lowerEXT()
6745 MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); in lowerTRUNC()
6771 extractParts(SrcReg, SplitSrcTy, 2, SplitSrcs, MIRBuilder, MRI); in lowerTRUNC()
6780 SplitSrcs[I] = MIRBuilder.buildTrunc(InterTy, SplitSrcs[I]).getReg(0); in lowerTRUNC()
6784 auto Merge = MIRBuilder.buildMergeLikeInstr( in lowerTRUNC()
6789 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), Merge.getReg(0)); in lowerTRUNC()
6791 MIRBuilder.buildCopy(MI.getOperand(0).getReg(), Merge.getReg(0)); in lowerTRUNC()
6803 auto Zero = MIRBuilder.buildConstant(AmtTy, 0); in lowerRotateWithReverseRotate()
6806 auto Neg = MIRBuilder.buildSub(AmtTy, Zero, Amt); in lowerRotateWithReverseRotate()
6807 MIRBuilder.buildInstr(RevRot, {Dst}, {Src, Neg}); in lowerRotateWithReverseRotate()
6818 MIRBuilder.setInstrAndDebugLoc(MI); in lowerRotate()
6834 MIRBuilder.buildInstr(Opc, {R1}, {R2, R2, R3}); in lowerRotate()
6842 Amt = MIRBuilder.buildNeg(DstTy, Amt).getReg(0); in lowerRotate()
6847 auto Zero = MIRBuilder.buildConstant(AmtTy, 0); in lowerRotate()
6850 auto BitWidthMinusOneC = MIRBuilder.buildConstant(AmtTy, EltSizeInBits - 1); in lowerRotate()
6856 auto NegAmt = MIRBuilder.buildSub(AmtTy, Zero, Amt); in lowerRotate()
6857 auto ShAmt = MIRBuilder.buildAnd(AmtTy, Amt, BitWidthMinusOneC); in lowerRotate()
6858 ShVal = MIRBuilder.buildInstr(ShOpc, {DstTy}, {Src, ShAmt}).getReg(0); in lowerRotate()
6859 auto RevAmt = MIRBuilder.buildAnd(AmtTy, NegAmt, BitWidthMinusOneC); in lowerRotate()
6861 MIRBuilder.buildInstr(RevShiftOpc, {DstTy}, {Src, RevAmt}).getReg(0); in lowerRotate()
6865 auto BitWidthC = MIRBuilder.buildConstant(AmtTy, EltSizeInBits); in lowerRotate()
6866 auto ShAmt = MIRBuilder.buildURem(AmtTy, Amt, BitWidthC); in lowerRotate()
6867 ShVal = MIRBuilder.buildInstr(ShOpc, {DstTy}, {Src, ShAmt}).getReg(0); in lowerRotate()
6868 auto RevAmt = MIRBuilder.buildSub(AmtTy, BitWidthMinusOneC, ShAmt); in lowerRotate()
6869 auto One = MIRBuilder.buildConstant(AmtTy, 1); in lowerRotate()
6870 auto Inner = MIRBuilder.buildInstr(RevShiftOpc, {DstTy}, {Src, One}); in lowerRotate()
6872 MIRBuilder.buildInstr(RevShiftOpc, {DstTy}, {Inner, RevAmt}).getReg(0); in lowerRotate()
6874 MIRBuilder.buildOr(Dst, ShVal, RevShiftVal); in lowerRotate()
6900 auto Zero32 = MIRBuilder.buildConstant(S32, 0); in lowerU64ToF32BitOps()
6901 auto Zero64 = MIRBuilder.buildConstant(S64, 0); in lowerU64ToF32BitOps()
6903 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src); in lowerU64ToF32BitOps()
6905 auto K = MIRBuilder.buildConstant(S32, 127U + 63U); in lowerU64ToF32BitOps()
6906 auto Sub = MIRBuilder.buildSub(S32, K, LZ); in lowerU64ToF32BitOps()
6908 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64); in lowerU64ToF32BitOps()
6909 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32); in lowerU64ToF32BitOps()
6911 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1); in lowerU64ToF32BitOps()
6912 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ); in lowerU64ToF32BitOps()
6914 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0); in lowerU64ToF32BitOps()
6916 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL); in lowerU64ToF32BitOps()
6917 auto T = MIRBuilder.buildAnd(S64, U, Mask1); in lowerU64ToF32BitOps()
6919 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40)); in lowerU64ToF32BitOps()
6920 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23)); in lowerU64ToF32BitOps()
6921 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl)); in lowerU64ToF32BitOps()
6923 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL); in lowerU64ToF32BitOps()
6924 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C); in lowerU64ToF32BitOps()
6925 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C); in lowerU64ToF32BitOps()
6926 auto One = MIRBuilder.buildConstant(S32, 1); in lowerU64ToF32BitOps()
6928 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One); in lowerU64ToF32BitOps()
6929 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32); in lowerU64ToF32BitOps()
6930 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0); in lowerU64ToF32BitOps()
6931 MIRBuilder.buildAdd(Dst, V, R); in lowerU64ToF32BitOps()
6941 auto True = MIRBuilder.buildFConstant(DstTy, 1.0); in lowerUITOFP()
6942 auto False = MIRBuilder.buildFConstant(DstTy, 0.0); in lowerUITOFP()
6943 MIRBuilder.buildSelect(Dst, Src, True, False); in lowerUITOFP()
6970 auto True = MIRBuilder.buildFConstant(DstTy, -1.0); in lowerSITOFP()
6971 auto False = MIRBuilder.buildFConstant(DstTy, 0.0); in lowerSITOFP()
6972 MIRBuilder.buildSelect(Dst, Src, True, False); in lowerSITOFP()
6987 auto SignBit = MIRBuilder.buildConstant(S64, 63); in lowerSITOFP()
6988 auto S = MIRBuilder.buildAShr(S64, L, SignBit); in lowerSITOFP()
6990 auto LPlusS = MIRBuilder.buildAdd(S64, L, S); in lowerSITOFP()
6991 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S); in lowerSITOFP()
6992 auto R = MIRBuilder.buildUITOFP(S32, Xor); in lowerSITOFP()
6994 auto RNeg = MIRBuilder.buildFNeg(S32, R); in lowerSITOFP()
6995 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S, in lowerSITOFP()
6996 MIRBuilder.buildConstant(S64, 0)); in lowerSITOFP()
6997 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R); in lowerSITOFP()
7025 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src); in lowerFPTOUI()
7027 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP); in lowerFPTOUI()
7030 MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold); in lowerFPTOUI()
7031 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub); in lowerFPTOUI()
7032 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt); in lowerFPTOUI()
7033 MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit); in lowerFPTOUI()
7038 MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, S1, Src, Threshold); in lowerFPTOUI()
7039 MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res); in lowerFPTOUI()
7060 auto ExponentMask = MIRBuilder.buildConstant(SrcTy, 0x7F800000); in lowerFPTOSI()
7061 auto ExponentLoBit = MIRBuilder.buildConstant(SrcTy, 23); in lowerFPTOSI()
7063 auto AndExpMask = MIRBuilder.buildAnd(SrcTy, Src, ExponentMask); in lowerFPTOSI()
7064 auto ExponentBits = MIRBuilder.buildLShr(SrcTy, AndExpMask, ExponentLoBit); in lowerFPTOSI()
7066 auto SignMask = MIRBuilder.buildConstant(SrcTy, in lowerFPTOSI()
7068 auto AndSignMask = MIRBuilder.buildAnd(SrcTy, Src, SignMask); in lowerFPTOSI()
7069 auto SignLowBit = MIRBuilder.buildConstant(SrcTy, SrcEltBits - 1); in lowerFPTOSI()
7070 auto Sign = MIRBuilder.buildAShr(SrcTy, AndSignMask, SignLowBit); in lowerFPTOSI()
7071 Sign = MIRBuilder.buildSExt(DstTy, Sign); in lowerFPTOSI()
7073 auto MantissaMask = MIRBuilder.buildConstant(SrcTy, 0x007FFFFF); in lowerFPTOSI()
7074 auto AndMantissaMask = MIRBuilder.buildAnd(SrcTy, Src, MantissaMask); in lowerFPTOSI()
7075 auto K = MIRBuilder.buildConstant(SrcTy, 0x00800000); in lowerFPTOSI()
7077 auto R = MIRBuilder.buildOr(SrcTy, AndMantissaMask, K); in lowerFPTOSI()
7078 R = MIRBuilder.buildZExt(DstTy, R); in lowerFPTOSI()
7080 auto Bias = MIRBuilder.buildConstant(SrcTy, 127); in lowerFPTOSI()
7081 auto Exponent = MIRBuilder.buildSub(SrcTy, ExponentBits, Bias); in lowerFPTOSI()
7082 auto SubExponent = MIRBuilder.buildSub(SrcTy, Exponent, ExponentLoBit); in lowerFPTOSI()
7083 auto ExponentSub = MIRBuilder.buildSub(SrcTy, ExponentLoBit, Exponent); in lowerFPTOSI()
7085 auto Shl = MIRBuilder.buildShl(DstTy, R, SubExponent); in lowerFPTOSI()
7086 auto Srl = MIRBuilder.buildLShr(DstTy, R, ExponentSub); in lowerFPTOSI()
7089 auto CmpGt = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, in lowerFPTOSI()
7092 R = MIRBuilder.buildSelect(DstTy, CmpGt, Shl, Srl); in lowerFPTOSI()
7094 auto XorSign = MIRBuilder.buildXor(DstTy, R, Sign); in lowerFPTOSI()
7095 auto Ret = MIRBuilder.buildSub(DstTy, XorSign, Sign); in lowerFPTOSI()
7097 auto ZeroSrcTy = MIRBuilder.buildConstant(SrcTy, 0); in lowerFPTOSI()
7099 auto ExponentLt0 = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, in lowerFPTOSI()
7102 auto ZeroDstTy = MIRBuilder.buildConstant(DstTy, 0); in lowerFPTOSI()
7103 MIRBuilder.buildSelect(Dst, ExponentLt0, ZeroDstTy, Ret); in lowerFPTOSI()
7122 if (MIRBuilder.getMF().getTarget().Options.UnsafeFPMath) { in lowerFPTRUNC_F64_TO_F16()
7124 auto Src32 = MIRBuilder.buildFPTrunc(S32, Src, Flags); in lowerFPTRUNC_F64_TO_F16()
7125 MIRBuilder.buildFPTrunc(Dst, Src32, Flags); in lowerFPTRUNC_F64_TO_F16()
7134 auto Unmerge = MIRBuilder.buildUnmerge(S32, Src); in lowerFPTRUNC_F64_TO_F16()
7138 auto E = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 20)); in lowerFPTRUNC_F64_TO_F16()
7139 E = MIRBuilder.buildAnd(S32, E, MIRBuilder.buildConstant(S32, ExpMask)); in lowerFPTRUNC_F64_TO_F16()
7143 E = MIRBuilder.buildAdd( in lowerFPTRUNC_F64_TO_F16()
7144 S32, E, MIRBuilder.buildConstant(S32, -ExpBiasf64 + ExpBiasf16)); in lowerFPTRUNC_F64_TO_F16()
7146 auto M = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 8)); in lowerFPTRUNC_F64_TO_F16()
7147 M = MIRBuilder.buildAnd(S32, M, MIRBuilder.buildConstant(S32, 0xffe)); in lowerFPTRUNC_F64_TO_F16()
7149 auto MaskedSig = MIRBuilder.buildAnd(S32, UH, in lowerFPTRUNC_F64_TO_F16()
7150 MIRBuilder.buildConstant(S32, 0x1ff)); in lowerFPTRUNC_F64_TO_F16()
7151 MaskedSig = MIRBuilder.buildOr(S32, MaskedSig, U); in lowerFPTRUNC_F64_TO_F16()
7153 auto Zero = MIRBuilder.buildConstant(S32, 0); in lowerFPTRUNC_F64_TO_F16()
7154 auto SigCmpNE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, MaskedSig, Zero); in lowerFPTRUNC_F64_TO_F16()
7155 auto Lo40Set = MIRBuilder.buildZExt(S32, SigCmpNE0); in lowerFPTRUNC_F64_TO_F16()
7156 M = MIRBuilder.buildOr(S32, M, Lo40Set); in lowerFPTRUNC_F64_TO_F16()
7159 auto Bits0x200 = MIRBuilder.buildConstant(S32, 0x0200); in lowerFPTRUNC_F64_TO_F16()
7160 auto CmpM_NE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, M, Zero); in lowerFPTRUNC_F64_TO_F16()
7161 auto SelectCC = MIRBuilder.buildSelect(S32, CmpM_NE0, Bits0x200, Zero); in lowerFPTRUNC_F64_TO_F16()
7163 auto Bits0x7c00 = MIRBuilder.buildConstant(S32, 0x7c00); in lowerFPTRUNC_F64_TO_F16()
7164 auto I = MIRBuilder.buildOr(S32, SelectCC, Bits0x7c00); in lowerFPTRUNC_F64_TO_F16()
7167 auto EShl12 = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 12)); in lowerFPTRUNC_F64_TO_F16()
7168 auto N = MIRBuilder.buildOr(S32, M, EShl12); in lowerFPTRUNC_F64_TO_F16()
7171 auto One = MIRBuilder.buildConstant(S32, 1); in lowerFPTRUNC_F64_TO_F16()
7172 auto OneSubExp = MIRBuilder.buildSub(S32, One, E); in lowerFPTRUNC_F64_TO_F16()
7173 auto B = MIRBuilder.buildSMax(S32, OneSubExp, Zero); in lowerFPTRUNC_F64_TO_F16()
7174 B = MIRBuilder.buildSMin(S32, B, MIRBuilder.buildConstant(S32, 13)); in lowerFPTRUNC_F64_TO_F16()
7176 auto SigSetHigh = MIRBuilder.buildOr(S32, M, in lowerFPTRUNC_F64_TO_F16()
7177 MIRBuilder.buildConstant(S32, 0x1000)); in lowerFPTRUNC_F64_TO_F16()
7179 auto D = MIRBuilder.buildLShr(S32, SigSetHigh, B); in lowerFPTRUNC_F64_TO_F16()
7180 auto D0 = MIRBuilder.buildShl(S32, D, B); in lowerFPTRUNC_F64_TO_F16()
7182 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, in lowerFPTRUNC_F64_TO_F16()
7184 auto D1 = MIRBuilder.buildZExt(S32, D0_NE_SigSetHigh); in lowerFPTRUNC_F64_TO_F16()
7185 D = MIRBuilder.buildOr(S32, D, D1); in lowerFPTRUNC_F64_TO_F16()
7187 auto CmpELtOne = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, S1, E, One); in lowerFPTRUNC_F64_TO_F16()
7188 auto V = MIRBuilder.buildSelect(S32, CmpELtOne, D, N); in lowerFPTRUNC_F64_TO_F16()
7190 auto VLow3 = MIRBuilder.buildAnd(S32, V, MIRBuilder.buildConstant(S32, 7)); in lowerFPTRUNC_F64_TO_F16()
7191 V = MIRBuilder.buildLShr(S32, V, MIRBuilder.buildConstant(S32, 2)); in lowerFPTRUNC_F64_TO_F16()
7193 auto VLow3Eq3 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, VLow3, in lowerFPTRUNC_F64_TO_F16()
7194 MIRBuilder.buildConstant(S32, 3)); in lowerFPTRUNC_F64_TO_F16()
7195 auto V0 = MIRBuilder.buildZExt(S32, VLow3Eq3); in lowerFPTRUNC_F64_TO_F16()
7197 auto VLow3Gt5 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, VLow3, in lowerFPTRUNC_F64_TO_F16()
7198 MIRBuilder.buildConstant(S32, 5)); in lowerFPTRUNC_F64_TO_F16()
7199 auto V1 = MIRBuilder.buildZExt(S32, VLow3Gt5); in lowerFPTRUNC_F64_TO_F16()
7201 V1 = MIRBuilder.buildOr(S32, V0, V1); in lowerFPTRUNC_F64_TO_F16()
7202 V = MIRBuilder.buildAdd(S32, V, V1); in lowerFPTRUNC_F64_TO_F16()
7204 auto CmpEGt30 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, in lowerFPTRUNC_F64_TO_F16()
7205 E, MIRBuilder.buildConstant(S32, 30)); in lowerFPTRUNC_F64_TO_F16()
7206 V = MIRBuilder.buildSelect(S32, CmpEGt30, in lowerFPTRUNC_F64_TO_F16()
7207 MIRBuilder.buildConstant(S32, 0x7c00), V); in lowerFPTRUNC_F64_TO_F16()
7209 auto CmpEGt1039 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, in lowerFPTRUNC_F64_TO_F16()
7210 E, MIRBuilder.buildConstant(S32, 1039)); in lowerFPTRUNC_F64_TO_F16()
7211 V = MIRBuilder.buildSelect(S32, CmpEGt1039, I, V); in lowerFPTRUNC_F64_TO_F16()
7214 auto Sign = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 16)); in lowerFPTRUNC_F64_TO_F16()
7215 Sign = MIRBuilder.buildAnd(S32, Sign, MIRBuilder.buildConstant(S32, 0x8000)); in lowerFPTRUNC_F64_TO_F16()
7218 V = MIRBuilder.buildOr(S32, Sign, V); in lowerFPTRUNC_F64_TO_F16()
7220 MIRBuilder.buildTrunc(Dst, V); in lowerFPTRUNC_F64_TO_F16()
7241 auto CvtSrc1 = MIRBuilder.buildSITOFP(Ty, Src1); in lowerFPOWI()
7242 MIRBuilder.buildFPow(Dst, Src0, CvtSrc1, MI.getFlags()); in lowerFPOWI()
7268 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1); in lowerMinMax()
7269 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1); in lowerMinMax()
7290 auto One = MIRBuilder.buildConstant(DstTy, 1); in lowerThreewayCompare()
7291 auto Zero = MIRBuilder.buildConstant(DstTy, 0); in lowerThreewayCompare()
7292 auto IsGT = MIRBuilder.buildICmp(GTPredicate, CmpTy, Cmp->getLHSReg(), in lowerThreewayCompare()
7294 auto SelectZeroOrOne = MIRBuilder.buildSelect(DstTy, IsGT, One, Zero); in lowerThreewayCompare()
7296 auto MinusOne = MIRBuilder.buildConstant(DstTy, -1); in lowerThreewayCompare()
7297 auto IsLT = MIRBuilder.buildICmp(LTPredicate, CmpTy, Cmp->getLHSReg(), in lowerThreewayCompare()
7299 MIRBuilder.buildSelect(Dst, IsLT, MinusOne, SelectZeroOrOne); in lowerThreewayCompare()
7311 auto SignBitMask = MIRBuilder.buildConstant( in lowerFCopySign()
7314 auto NotSignBitMask = MIRBuilder.buildConstant( in lowerFCopySign()
7317 Register And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask).getReg(0); in lowerFCopySign()
7320 And1 = MIRBuilder.buildAnd(Src1Ty, Src1, SignBitMask).getReg(0); in lowerFCopySign()
7322 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size); in lowerFCopySign()
7323 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1); in lowerFCopySign()
7324 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt); in lowerFCopySign()
7325 And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask).getReg(0); in lowerFCopySign()
7327 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size); in lowerFCopySign()
7328 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt); in lowerFCopySign()
7329 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift); in lowerFCopySign()
7330 And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask).getReg(0); in lowerFCopySign()
7341 MIRBuilder.buildOr(Dst, And0, And1, Flags); in lowerFCopySign()
7363 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0); in lowerFMinNumMaxNum()
7366 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0); in lowerFMinNumMaxNum()
7371 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags()); in lowerFMinNumMaxNum()
7382 auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2), in lowerFMad()
7384 MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags); in lowerFMad()
7402 auto T = MIRBuilder.buildIntrinsicTrunc(Ty, X, Flags); in lowerIntrinsicRound()
7404 auto Diff = MIRBuilder.buildFSub(Ty, X, T, Flags); in lowerIntrinsicRound()
7405 auto AbsDiff = MIRBuilder.buildFAbs(Ty, Diff, Flags); in lowerIntrinsicRound()
7407 auto Half = MIRBuilder.buildFConstant(Ty, 0.5); in lowerIntrinsicRound()
7409 MIRBuilder.buildFCmp(CmpInst::FCMP_OGE, CondTy, AbsDiff, Half, Flags); in lowerIntrinsicRound()
7412 auto One = MIRBuilder.buildFConstant(Ty, 1.0); in lowerIntrinsicRound()
7413 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0); in lowerIntrinsicRound()
7414 auto BoolFP = MIRBuilder.buildSelect(Ty, Cmp, One, Zero); in lowerIntrinsicRound()
7415 auto SignedOffset = MIRBuilder.buildFCopysign(Ty, BoolFP, X); in lowerIntrinsicRound()
7417 MIRBuilder.buildFAdd(DstReg, T, SignedOffset, Flags); in lowerIntrinsicRound()
7433 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Ty, SrcReg, Flags); in lowerFFloor()
7434 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0); in lowerFFloor()
7436 auto Lt0 = MIRBuilder.buildFCmp(CmpInst::FCMP_OLT, CondTy, in lowerFFloor()
7438 auto NeTrunc = MIRBuilder.buildFCmp(CmpInst::FCMP_ONE, CondTy, in lowerFFloor()
7440 auto And = MIRBuilder.buildAnd(CondTy, Lt0, NeTrunc); in lowerFFloor()
7441 auto AddVal = MIRBuilder.buildSITOFP(Ty, And); in lowerFFloor()
7443 MIRBuilder.buildFAdd(DstReg, Trunc, AddVal, Flags); in lowerFFloor()
7455 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src0Reg).getReg(0); in lowerMergeValues()
7461 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); in lowerMergeValues()
7466 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); in lowerMergeValues()
7467 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); in lowerMergeValues()
7468 MIRBuilder.buildOr(NextResult, ResultReg, Shl); in lowerMergeValues()
7473 if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace( in lowerMergeValues()
7479 MIRBuilder.buildIntToPtr(DstReg, ResultReg); in lowerMergeValues()
7502 MIRBuilder.buildTrunc(Dst0Reg, SrcReg); in lowerUnmergeValues()
7507 auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset); in lowerUnmergeValues()
7508 auto Shift = MIRBuilder.buildLShr(IntTy, SrcReg, ShiftAmt); in lowerUnmergeValues()
7509 MIRBuilder.buildTrunc(MI.getOperand(I), Shift); in lowerUnmergeValues()
7543 extractParts(SrcVec, EltTy, NumElts, SrcRegs, MIRBuilder, MRI); in lowerExtractInsertVectorElt()
7547 MIRBuilder.buildMergeLikeInstr(DstReg, SrcRegs); in lowerExtractInsertVectorElt()
7549 MIRBuilder.buildCopy(DstReg, SrcRegs[IdxVal]); in lowerExtractInsertVectorElt()
7568 MIRBuilder.buildStore(SrcVec, StackTemp, PtrInfo, VecAlign); in lowerExtractInsertVectorElt()
7586 MIRBuilder.buildStore(InsertVal, EltPtr, PtrInfo, EltAlign); in lowerExtractInsertVectorElt()
7589 MIRBuilder.buildLoad(DstReg, StackTemp, PtrInfo, VecAlign); in lowerExtractInsertVectorElt()
7591 MIRBuilder.buildLoad(DstReg, EltPtr, PtrInfo, EltAlign); in lowerExtractInsertVectorElt()
7612 Undef = MIRBuilder.buildUndef(EltTy).getReg(0); in lowerShuffleVector()
7623 auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx); in lowerShuffleVector()
7624 auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK); in lowerShuffleVector()
7630 MIRBuilder.buildCopy(DstReg, BuildVec[0]); in lowerShuffleVector()
7632 MIRBuilder.buildBuildVector(DstReg, BuildVec); in lowerShuffleVector()
7658 auto OutPos = MIRBuilder.buildConstant(IdxTy, 0); in lowerVECTOR_COMPRESS()
7664 MIRBuilder.buildStore(Passthru, StackPtr, PtrInfo, VecAlign); in lowerVECTOR_COMPRESS()
7672 MIRBuilder.buildConstant(ValTy, PassthruSplatVal.value()).getReg(0); in lowerVECTOR_COMPRESS()
7674 auto Popcount = MIRBuilder.buildZExt(MaskTy.changeElementSize(32), Mask); in lowerVECTOR_COMPRESS()
7675 Popcount = MIRBuilder.buildInstr(TargetOpcode::G_VECREDUCE_ADD, in lowerVECTOR_COMPRESS()
7681 MIRBuilder.buildLoad(ValTy, LastElmtPtr, ValPtrInfo, ValAlign) in lowerVECTOR_COMPRESS()
7687 auto Idx = MIRBuilder.buildConstant(IdxTy, I); in lowerVECTOR_COMPRESS()
7688 auto Val = MIRBuilder.buildExtractVectorElement(ValTy, Vec, Idx); in lowerVECTOR_COMPRESS()
7691 MIRBuilder.buildStore(Val, ElmtPtr, ValPtrInfo, ValAlign); in lowerVECTOR_COMPRESS()
7694 auto MaskI = MIRBuilder.buildExtractVectorElement(MaskITy, Mask, Idx); in lowerVECTOR_COMPRESS()
7696 MaskI = MIRBuilder.buildTrunc(LLT::scalar(1), MaskI); in lowerVECTOR_COMPRESS()
7698 MaskI = MIRBuilder.buildZExt(IdxTy, MaskI); in lowerVECTOR_COMPRESS()
7699 OutPos = MIRBuilder.buildAdd(IdxTy, OutPos, MaskI); in lowerVECTOR_COMPRESS()
7703 MIRBuilder.buildConstant(IdxTy, VecTy.getNumElements() - 1); in lowerVECTOR_COMPRESS()
7704 auto AllLanesSelected = MIRBuilder.buildICmp( in lowerVECTOR_COMPRESS()
7706 OutPos = MIRBuilder.buildInstr(TargetOpcode::G_UMIN, {IdxTy}, in lowerVECTOR_COMPRESS()
7711 MIRBuilder.buildSelect(ValTy, AllLanesSelected, Val, LastWriteVal) in lowerVECTOR_COMPRESS()
7713 MIRBuilder.buildStore(LastWriteVal, ElmtPtr, ValPtrInfo, ValAlign); in lowerVECTOR_COMPRESS()
7718 MIRBuilder.buildLoad(Dst, StackPtr, PtrInfo, VecAlign); in lowerVECTOR_COMPRESS()
7730 auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg); in getDynStackAllocTargetPtr()
7731 SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp); in getDynStackAllocTargetPtr()
7736 auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize); in getDynStackAllocTargetPtr()
7740 auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask); in getDynStackAllocTargetPtr()
7741 Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst); in getDynStackAllocTargetPtr()
7744 return MIRBuilder.buildCast(PtrTy, Alloc).getReg(0); in getDynStackAllocTargetPtr()
7763 MIRBuilder.buildCopy(SPReg, SPTmp); in lowerDynStackAlloc()
7764 MIRBuilder.buildCopy(Dst, SPTmp); in lowerDynStackAlloc()
7776 MIRBuilder.buildCopy(MI.getOperand(0), StackPtr); in lowerStackSave()
7787 MIRBuilder.buildCopy(StackPtr, MI.getOperand(0)); in lowerStackRestore()
7805 auto Unmerge = MIRBuilder.buildUnmerge(SrcTy.getElementType(), SrcReg); in lowerExtract()
7814 MIRBuilder.buildCopy(DstReg, SubVectorElts[0]); in lowerExtract()
7816 MIRBuilder.buildMergeLikeInstr(DstReg, SubVectorElts); in lowerExtract()
7829 SrcReg = MIRBuilder.buildBitcast(SrcIntTy, SrcReg).getReg(0); in lowerExtract()
7833 MIRBuilder.buildTrunc(DstReg, SrcReg); in lowerExtract()
7835 auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset); in lowerExtract()
7836 auto Shr = MIRBuilder.buildLShr(SrcIntTy, SrcReg, ShiftAmt); in lowerExtract()
7837 MIRBuilder.buildTrunc(DstReg, Shr); in lowerExtract()
7862 auto UnmergeSrc = MIRBuilder.buildUnmerge(EltTy, Src); in lowerInsert()
7872 auto UnmergeInsertSrc = MIRBuilder.buildUnmerge(EltTy, InsertSrc); in lowerInsert()
7887 MIRBuilder.buildMergeLikeInstr(Dst, DstElts); in lowerInsert()
7897 const DataLayout &DL = MIRBuilder.getDataLayout(); in lowerInsert()
7910 Src = MIRBuilder.buildCast(IntDstTy, Src).getReg(0); in lowerInsert()
7915 InsertSrc = MIRBuilder.buildPtrToInt(IntInsertTy, InsertSrc).getReg(0); in lowerInsert()
7918 Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0); in lowerInsert()
7920 auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset); in lowerInsert()
7921 ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0); in lowerInsert()
7927 auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal); in lowerInsert()
7928 auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask); in lowerInsert()
7929 auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc); in lowerInsert()
7931 MIRBuilder.buildCast(Dst, Or); in lowerInsert()
7948 MIRBuilder.buildAdd(NewDst0, LHS, RHS); in lowerSADDO_SSUBO()
7950 MIRBuilder.buildSub(NewDst0, LHS, RHS); in lowerSADDO_SSUBO()
7954 auto Zero = MIRBuilder.buildConstant(Ty, 0); in lowerSADDO_SSUBO()
7963 MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, NewDst0, LHS); in lowerSADDO_SSUBO()
7964 auto ConditionRHS = MIRBuilder.buildICmp( in lowerSADDO_SSUBO()
7967 MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS); in lowerSADDO_SSUBO()
7969 MIRBuilder.buildCopy(Dst0, NewDst0); in lowerSADDO_SSUBO()
8020 MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(NumBits)); in lowerAddSubSatToMinMax()
8022 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits)); in lowerAddSubSatToMinMax()
8025 auto Zero = MIRBuilder.buildConstant(Ty, 0); in lowerAddSubSatToMinMax()
8026 Hi = MIRBuilder.buildSub(Ty, MaxVal, MIRBuilder.buildSMax(Ty, LHS, Zero)); in lowerAddSubSatToMinMax()
8027 Lo = MIRBuilder.buildSub(Ty, MinVal, MIRBuilder.buildSMin(Ty, LHS, Zero)); in lowerAddSubSatToMinMax()
8029 auto NegOne = MIRBuilder.buildConstant(Ty, -1); in lowerAddSubSatToMinMax()
8030 Lo = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMax(Ty, LHS, NegOne), in lowerAddSubSatToMinMax()
8032 Hi = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMin(Ty, LHS, NegOne), in lowerAddSubSatToMinMax()
8036 MIRBuilder.buildSMin(Ty, MIRBuilder.buildSMax(Ty, Lo, RHS), Hi); in lowerAddSubSatToMinMax()
8037 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, RHSClamped}); in lowerAddSubSatToMinMax()
8041 Register Not = IsAdd ? MIRBuilder.buildNot(Ty, LHS).getReg(0) : LHS; in lowerAddSubSatToMinMax()
8042 auto Min = MIRBuilder.buildUMin(Ty, Not, RHS); in lowerAddSubSatToMinMax()
8043 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, Min}); in lowerAddSubSatToMinMax()
8084 MIRBuilder.buildInstr(OverflowOp, {Ty, BoolTy}, {LHS, RHS}); in lowerAddSubSatToAddoSubo()
8096 auto ShiftAmount = MIRBuilder.buildConstant(Ty, NumBits - 1); in lowerAddSubSatToAddoSubo()
8097 auto Sign = MIRBuilder.buildAShr(Ty, Tmp, ShiftAmount); in lowerAddSubSatToAddoSubo()
8099 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits)); in lowerAddSubSatToAddoSubo()
8100 Clamp = MIRBuilder.buildAdd(Ty, Sign, MinVal); in lowerAddSubSatToAddoSubo()
8108 Clamp = MIRBuilder.buildConstant(Ty, IsAdd ? -1 : 0); in lowerAddSubSatToAddoSubo()
8110 MIRBuilder.buildSelect(Res, Ov, Clamp, Tmp); in lowerAddSubSatToAddoSubo()
8127 auto Result = MIRBuilder.buildShl(Ty, LHS, RHS); in lowerShlSat()
8128 auto Orig = IsSigned ? MIRBuilder.buildAShr(Ty, Result, RHS) in lowerShlSat()
8129 : MIRBuilder.buildLShr(Ty, Result, RHS); in lowerShlSat()
8133 auto SatMin = MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(BW)); in lowerShlSat()
8134 auto SatMax = MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(BW)); in lowerShlSat()
8135 auto Cmp = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, LHS, in lowerShlSat()
8136 MIRBuilder.buildConstant(Ty, 0)); in lowerShlSat()
8137 SatVal = MIRBuilder.buildSelect(Ty, Cmp, SatMin, SatMax); in lowerShlSat()
8139 SatVal = MIRBuilder.buildConstant(Ty, APInt::getMaxValue(BW)); in lowerShlSat()
8141 auto Ov = MIRBuilder.buildICmp(CmpInst::ICMP_NE, BoolTy, LHS, Orig); in lowerShlSat()
8142 MIRBuilder.buildSelect(Res, Ov, SatVal, Result); in lowerShlSat()
8155 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt); in lowerBswap()
8156 auto LSByteShiftedLeft = MIRBuilder.buildShl(Ty, Src, ShiftAmt); in lowerBswap()
8157 auto MSByteShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); in lowerBswap()
8158 auto Res = MIRBuilder.buildOr(Ty, MSByteShiftedRight, LSByteShiftedLeft); in lowerBswap()
8164 auto Mask = MIRBuilder.buildConstant(Ty, APMask); in lowerBswap()
8165 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt - 16 * i); in lowerBswap()
8167 auto LoByte = MIRBuilder.buildAnd(Ty, Src, Mask); in lowerBswap()
8168 auto LoShiftedLeft = MIRBuilder.buildShl(Ty, LoByte, ShiftAmt); in lowerBswap()
8169 Res = MIRBuilder.buildOr(Ty, Res, LoShiftedLeft); in lowerBswap()
8171 auto SrcShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); in lowerBswap()
8172 auto HiShiftedRight = MIRBuilder.buildAnd(Ty, SrcShiftedRight, Mask); in lowerBswap()
8173 Res = MIRBuilder.buildOr(Ty, Res, HiShiftedRight); in lowerBswap()
8200 MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src}); in lowerBitreverse()
8206 SwapN(4, Ty, MIRBuilder, BSWAP, APInt::getSplat(Size, APInt(8, 0xF0))); in lowerBitreverse()
8212 SwapN(2, Ty, MIRBuilder, Swap4, APInt::getSplat(Size, APInt(8, 0xCC))); in lowerBitreverse()
8218 SwapN(1, Dst, MIRBuilder, Swap2, APInt::getSplat(Size, APInt(8, 0xAA))); in lowerBitreverse()
8225 auto ShAmt = MIRBuilder.buildConstant(Ty, J - I); in lowerBitreverse()
8226 Tmp2 = MIRBuilder.buildShl(Ty, Src, ShAmt); in lowerBitreverse()
8228 auto ShAmt = MIRBuilder.buildConstant(Ty, I - J); in lowerBitreverse()
8229 Tmp2 = MIRBuilder.buildLShr(Ty, Src, ShAmt); in lowerBitreverse()
8232 auto Mask = MIRBuilder.buildConstant(Ty, 1ULL << J); in lowerBitreverse()
8233 Tmp2 = MIRBuilder.buildAnd(Ty, Tmp2, Mask); in lowerBitreverse()
8237 Tmp = MIRBuilder.buildOr(Ty, Tmp, Tmp2); in lowerBitreverse()
8239 MIRBuilder.buildCopy(Dst, Tmp); in lowerBitreverse()
8248 MachineFunction &MF = MIRBuilder.getMF(); in lowerReadWriteRegister()
8264 MIRBuilder.buildCopy(ValReg, PhysReg); in lowerReadWriteRegister()
8266 MIRBuilder.buildCopy(PhysReg, ValReg); in lowerReadWriteRegister()
8281 auto LHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(1)}); in lowerSMULH_UMULH()
8282 auto RHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(2)}); in lowerSMULH_UMULH()
8283 auto Mul = MIRBuilder.buildMul(WideTy, LHS, RHS); in lowerSMULH_UMULH()
8286 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, SizeInBits); in lowerSMULH_UMULH()
8287 auto Shifted = MIRBuilder.buildInstr(ShiftOp, {WideTy}, {Mul, ShiftAmt}); in lowerSMULH_UMULH()
8288 MIRBuilder.buildTrunc(Result, Shifted); in lowerSMULH_UMULH()
8300 MIRBuilder.buildConstant(DstReg, 0); in lowerISFPCLASS()
8305 MIRBuilder.buildConstant(DstReg, 1); in lowerISFPCLASS()
8319 auto AsInt = MIRBuilder.buildCopy(IntTy, SrcReg); in lowerISFPCLASS()
8331 auto SignBitC = MIRBuilder.buildConstant(IntTy, SignBit); in lowerISFPCLASS()
8332 auto ValueMaskC = MIRBuilder.buildConstant(IntTy, ValueMask); in lowerISFPCLASS()
8333 auto InfC = MIRBuilder.buildConstant(IntTy, Inf); in lowerISFPCLASS()
8334 auto ExpMaskC = MIRBuilder.buildConstant(IntTy, ExpMask); in lowerISFPCLASS()
8335 auto ZeroC = MIRBuilder.buildConstant(IntTy, 0); in lowerISFPCLASS()
8337 auto Abs = MIRBuilder.buildAnd(IntTy, AsInt, ValueMaskC); in lowerISFPCLASS()
8339 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_NE, DstTy, AsInt, Abs); in lowerISFPCLASS()
8341 auto Res = MIRBuilder.buildConstant(DstTy, 0); in lowerISFPCLASS()
8345 Res = MIRBuilder.buildOr(DstTyCopy, Res, ToAppend); in lowerISFPCLASS()
8351 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, Abs, in lowerISFPCLASS()
8356 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, AsInt, in lowerISFPCLASS()
8361 auto Cmp = MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, Abs, in lowerISFPCLASS()
8363 auto And = MIRBuilder.buildAnd(DstTy, Cmp, Sign); in lowerISFPCLASS()
8373 auto ExpBits = MIRBuilder.buildAnd(IntTy, AsInt, ExpMaskC); in lowerISFPCLASS()
8374 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, in lowerISFPCLASS()
8383 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, in lowerISFPCLASS()
8387 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, Abs, ZeroC)); in lowerISFPCLASS()
8389 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, in lowerISFPCLASS()
8397 auto OneC = MIRBuilder.buildConstant(IntTy, 1); in lowerISFPCLASS()
8398 auto VMinusOne = MIRBuilder.buildSub(IntTy, V, OneC); in lowerISFPCLASS()
8400 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, VMinusOne, in lowerISFPCLASS()
8401 MIRBuilder.buildConstant(IntTy, AllOneMantissa)); in lowerISFPCLASS()
8403 SubnormalRes = MIRBuilder.buildAnd(DstTy, SubnormalRes, Sign); in lowerISFPCLASS()
8409 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, in lowerISFPCLASS()
8413 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, Abs, InfC)); in lowerISFPCLASS()
8416 auto NegInfC = MIRBuilder.buildConstant(IntTy, NegInf); in lowerISFPCLASS()
8417 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, in lowerISFPCLASS()
8423 auto InfWithQnanBitC = MIRBuilder.buildConstant(IntTy, Inf | QNaNBitMask); in lowerISFPCLASS()
8427 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, Abs, InfC)); in lowerISFPCLASS()
8430 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGE, DstTy, Abs, in lowerISFPCLASS()
8436 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, Abs, InfC); in lowerISFPCLASS()
8437 auto IsNotQnan = MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, in lowerISFPCLASS()
8439 appendToRes(MIRBuilder.buildAnd(DstTy, IsNan, IsNotQnan)); in lowerISFPCLASS()
8447 auto ExpMinusOne = MIRBuilder.buildSub( in lowerISFPCLASS()
8448 IntTy, Abs, MIRBuilder.buildConstant(IntTy, ExpLSB)); in lowerISFPCLASS()
8451 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, ExpMinusOne, in lowerISFPCLASS()
8452 MIRBuilder.buildConstant(IntTy, MaxExpMinusOne)); in lowerISFPCLASS()
8454 NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, Sign); in lowerISFPCLASS()
8456 auto PosSign = MIRBuilder.buildXor( in lowerISFPCLASS()
8457 DstTy, Sign, MIRBuilder.buildConstant(DstTy, InvertionMask)); in lowerISFPCLASS()
8458 NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, PosSign); in lowerISFPCLASS()
8463 MIRBuilder.buildCopy(DstReg, Res); in lowerISFPCLASS()
8477 Op1Reg = MIRBuilder.buildPtrToInt(NewTy, Op1Reg).getReg(0); in lowerSelect()
8478 Op2Reg = MIRBuilder.buildPtrToInt(NewTy, Op2Reg).getReg(0); in lowerSelect()
8490 MaskElt = MIRBuilder.buildSExtInReg(MaskTy, MaskElt, 1).getReg(0); in lowerSelect()
8494 MIRBuilder.buildSExtOrTrunc(DstTy.getScalarType(), MaskElt).getReg(0); in lowerSelect()
8498 auto ShufSplat = MIRBuilder.buildShuffleSplat(DstTy, MaskElt); in lowerSelect()
8513 auto NotMask = MIRBuilder.buildNot(MaskTy, MaskReg); in lowerSelect()
8514 auto NewOp1 = MIRBuilder.buildAnd(MaskTy, Op1Reg, MaskReg); in lowerSelect()
8515 auto NewOp2 = MIRBuilder.buildAnd(MaskTy, Op2Reg, NotMask); in lowerSelect()
8517 auto Or = MIRBuilder.buildOr(DstTy, NewOp1, NewOp2); in lowerSelect()
8518 MIRBuilder.buildIntToPtr(DstReg, Or); in lowerSelect()
8520 MIRBuilder.buildOr(DstReg, NewOp1, NewOp2); in lowerSelect()
8530 MIRBuilder.buildInstr( in lowerDIVREM()
8534 MIRBuilder.buildInstr( in lowerDIVREM()
8551 MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - 1); in lowerAbsToAddXor()
8552 auto Shift = MIRBuilder.buildAShr(DstTy, OpReg, ShiftAmt); in lowerAbsToAddXor()
8553 auto Add = MIRBuilder.buildAdd(DstTy, OpReg, Shift); in lowerAbsToAddXor()
8554 MIRBuilder.buildXor(MI.getOperand(0).getReg(), Add, Shift); in lowerAbsToAddXor()
8567 auto Zero = MIRBuilder.buildConstant(Ty, 0); in lowerAbsToMaxNeg()
8568 auto Sub = MIRBuilder.buildSub(Ty, Zero, SrcReg); in lowerAbsToMaxNeg()
8569 MIRBuilder.buildSMax(MI.getOperand(0), SrcReg, Sub); in lowerAbsToMaxNeg()
8579 auto Zero = MIRBuilder.buildConstant(Ty, 0).getReg(0); in lowerAbsToCNeg()
8580 auto Sub = MIRBuilder.buildSub(Ty, Zero, SrcReg).getReg(0); in lowerAbsToCNeg()
8581 auto ICmp = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, IType, SrcReg, Zero); in lowerAbsToCNeg()
8582 MIRBuilder.buildSelect(DestReg, ICmp, SrcReg, Sub); in lowerAbsToCNeg()
8599 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::COPY)); in lowerVectorReduction()
8608 const DataLayout &DL = MIRBuilder.getDataLayout(); in lowerVAArg()
8618 auto VAList = MIRBuilder.buildLoad(PtrTy, ListPtr, *PtrLoadMMO).getReg(0); in lowerVAArg()
8624 MIRBuilder.buildConstant(PtrTyAsScalarTy, A.value() - 1).getReg(0); in lowerVAArg()
8625 auto AddDst = MIRBuilder.buildPtrAdd(PtrTy, VAList, AlignAmt); in lowerVAArg()
8626 auto AndDst = MIRBuilder.buildMaskLowPtrBits(PtrTy, AddDst, Log2(A)); in lowerVAArg()
8637 MIRBuilder.buildConstant(PtrTyAsScalarTy, DL.getTypeAllocSize(Ty)); in lowerVAArg()
8638 auto Succ = MIRBuilder.buildPtrAdd(PtrTy, VAList, IncAmt); in lowerVAArg()
8643 MIRBuilder.buildStore(Succ, ListPtr, *StoreMMO); in lowerVAArg()
8648 MIRBuilder.buildLoad(Dst, VAList, *EltLoadMMO); in lowerVAArg()