Lines Matching refs:DAG
2605 SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, in emitStackGuardXorFP() argument
2607 EVT PtrTy = getPointerTy(DAG.getDataLayout()); in emitStackGuardXorFP()
2609 MachineSDNode *Node = DAG.getMachineNode(XorOp, DL, PtrTy, Val); in emitStackGuardXorFP()
2750 SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const { in getReturnAddressFrameIndex()
2751 MachineFunction &MF = DAG.getMachineFunction(); in getReturnAddressFrameIndex()
2765 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy(DAG.getDataLayout())); in getReturnAddressFrameIndex()
2840 SelectionDAG &DAG) { in TranslateX86CC() argument
2845 RHS = DAG.getConstant(0, DL, RHS.getValueType()); in TranslateX86CC()
2858 RHS = DAG.getConstant(0, DL, RHS.getValueType()); in TranslateX86CC()
3260 const SelectionDAG &DAG, in isLoadBitCastBeneficial() argument
3274 return TargetLowering::isLoadBitCastBeneficial(LoadVT, BitcastVT, DAG, MMO); in isLoadBitCastBeneficial()
3345 SelectionDAG &DAG) const { in shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd()
3348 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG)) in shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd()
3355 if (DAG.isSplatValue(Y, /*AllowUndefs=*/true)) in shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd()
3491 SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const { in preferredShiftLegalizationStrategy() argument
3492 if (DAG.getMachineFunction().getFunction().hasMinSize() && in preferredShiftLegalizationStrategy()
3495 return TargetLowering::preferredShiftLegalizationStrategy(DAG, N, in preferredShiftLegalizationStrategy()
3786 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG, in getConstVector() argument
3794 bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64); in getConstVector()
3803 SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) : in getConstVector()
3804 DAG.getConstant(Values[i], dl, EltVT); in getConstVector()
3807 Ops.push_back(IsUndef ? DAG.getUNDEF(EltVT) : in getConstVector()
3808 DAG.getConstant(0, dl, EltVT)); in getConstVector()
3810 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops); in getConstVector()
3812 ConstsNode = DAG.getBitcast(VT, ConstsNode); in getConstVector()
3817 MVT VT, SelectionDAG &DAG, const SDLoc &dl) { in getConstVector() argument
3825 bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64); in getConstVector()
3834 Ops.append(Split ? 2 : 1, DAG.getUNDEF(EltVT)); in getConstVector()
3840 Ops.push_back(DAG.getConstant(V.trunc(32), dl, EltVT)); in getConstVector()
3841 Ops.push_back(DAG.getConstant(V.lshr(32).trunc(32), dl, EltVT)); in getConstVector()
3844 Ops.push_back(DAG.getConstantFP(FV, dl, EltVT)); in getConstVector()
3847 Ops.push_back(DAG.getConstantFP(FV, dl, EltVT)); in getConstVector()
3849 Ops.push_back(DAG.getConstant(V, dl, EltVT)); in getConstVector()
3853 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops); in getConstVector()
3854 return DAG.getBitcast(VT, ConstsNode); in getConstVector()
3858 SelectionDAG &DAG, const SDLoc &dl) { in getConstVector() argument
3860 return getConstVector(Bits, Undefs, VT, DAG, dl); in getConstVector()
3865 SelectionDAG &DAG, const SDLoc &dl) { in getZeroVector() argument
3874 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getZeroVector()
3876 Vec = DAG.getConstantFP(+0.0, dl, MVT::v4f32); in getZeroVector()
3879 Vec = DAG.getConstantFP(+0.0, dl, VT); in getZeroVector()
3883 Vec = DAG.getConstant(0, dl, VT); in getZeroVector()
3886 Vec = DAG.getConstant(0, dl, MVT::getVectorVT(MVT::i32, Num32BitElts)); in getZeroVector()
3888 return DAG.getBitcast(VT, Vec); in getZeroVector()
3914 static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, in extractSubVector() argument
3919 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT, in extractSubVector()
3932 return DAG.getBuildVector(ResultVT, dl, in extractSubVector()
3939 return DAG.getUNDEF(ResultVT); in extractSubVector()
3941 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl); in extractSubVector()
3942 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx); in extractSubVector()
3952 SelectionDAG &DAG, const SDLoc &dl) { in extract128BitVector() argument
3955 return extractSubVector(Vec, IdxVal, DAG, dl, 128); in extract128BitVector()
3960 SelectionDAG &DAG, const SDLoc &dl) { in extract256BitVector() argument
3962 return extractSubVector(Vec, IdxVal, DAG, dl, 256); in extract256BitVector()
3966 SelectionDAG &DAG, const SDLoc &dl, in insertSubVector() argument
3985 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl); in insertSubVector()
3986 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx); in insertSubVector()
3996 SelectionDAG &DAG, const SDLoc &dl) { in insert128BitVector() argument
3998 return insertSubVector(Result, Vec, IdxVal, DAG, dl, 128); in insert128BitVector()
4004 const X86Subtarget &Subtarget, SelectionDAG &DAG, in widenSubVector() argument
4009 SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl) in widenSubVector()
4010 : DAG.getUNDEF(VT); in widenSubVector()
4011 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, Vec, in widenSubVector()
4012 DAG.getIntPtrConstant(0, dl)); in widenSubVector()
4018 const X86Subtarget &Subtarget, SelectionDAG &DAG, in widenSubVector() argument
4026 return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl); in widenSubVector()
4042 const X86Subtarget &Subtarget, SelectionDAG &DAG, in widenMaskVector() argument
4045 return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl); in widenMaskVector()
4052 SelectionDAG &DAG) { in collectConcatOps() argument
4071 Ops.push_back(DAG.getUNDEF(SubVT)); in collectConcatOps()
4083 if (collectConcatOps(Lo.getNode(), LoOps, DAG) && in collectConcatOps()
4084 collectConcatOps(Hi.getNode(), HiOps, DAG) && in collectConcatOps()
4102 Ops.push_back(DAG.getUNDEF(SubVT)); in collectConcatOps()
4116 SelectionDAG &DAG) { in isUpperSubvectorUndef() argument
4118 if (!collectConcatOps(V.getNode(), SubOps, DAG)) in isUpperSubvectorUndef()
4129 EVT HalfVT = V.getValueType().getHalfNumVectorElementsVT(*DAG.getContext()); in isUpperSubvectorUndef()
4131 return DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, LowerOps); in isUpperSubvectorUndef()
4136 static bool isFreeToSplitVector(SDNode *N, SelectionDAG &DAG) { in isFreeToSplitVector() argument
4138 return collectConcatOps(N, Ops, DAG); in isFreeToSplitVector()
4141 static std::pair<SDValue, SDValue> splitVector(SDValue Op, SelectionDAG &DAG, in splitVector() argument
4151 SDValue Lo = extractSubVector(Op, 0, DAG, dl, SizeInBits / 2); in splitVector()
4152 if (DAG.isSplatValue(Op, /*AllowUndefs*/ false)) in splitVector()
4155 SDValue Hi = extractSubVector(Op, NumElems / 2, DAG, dl, SizeInBits / 2); in splitVector()
4160 static SDValue splitVectorOp(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) { in splitVectorOp() argument
4173 std::tie(LoOps[I], HiOps[I]) = splitVector(SrcOp, DAG, dl); in splitVectorOp()
4177 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in splitVectorOp()
4178 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, in splitVectorOp()
4179 DAG.getNode(Op.getOpcode(), dl, LoVT, LoOps), in splitVectorOp()
4180 DAG.getNode(Op.getOpcode(), dl, HiVT, HiOps)); in splitVectorOp()
4185 static SDValue splitVectorIntUnary(SDValue Op, SelectionDAG &DAG, in splitVectorIntUnary() argument
4196 return splitVectorOp(Op, DAG, dl); in splitVectorIntUnary()
4201 static SDValue splitVectorIntBinary(SDValue Op, SelectionDAG &DAG, in splitVectorIntBinary() argument
4208 return splitVectorOp(Op, DAG, dl); in splitVectorIntBinary()
4219 SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget, in SplitOpsAndApply() argument
4243 return Builder(DAG, DL, Ops); in SplitOpsAndApply()
4252 SubOps.push_back(extractSubVector(Op, i * NumSubElts, DAG, DL, SizeSub)); in SplitOpsAndApply()
4254 Subs.push_back(Builder(DAG, DL, SubOps)); in SplitOpsAndApply()
4256 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs); in SplitOpsAndApply()
4262 ArrayRef<SDValue> Ops, SelectionDAG &DAG, in getAVX512Node() argument
4274 !DAG.getTargetLoweringInfo().isTypeLegal(SVT)) in getAVX512Node()
4286 return DAG.getConstant(SplatValue, DL, DstVT); in getAVX512Node()
4313 Op = widenSubVector(Op, false, Subtarget, DAG, DL, 512); in getAVX512Node()
4316 SDValue Res = DAG.getNode(Opcode, DL, DstVT, SrcOps); in getAVX512Node()
4320 Res = extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits()); in getAVX512Node()
4325 static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG, in insert1BitVector() argument
4343 SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl); in insert1BitVector()
4352 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, in insert1BitVector()
4353 DAG.getConstant(0, dl, WideOpVT), in insert1BitVector()
4355 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx); in insert1BitVector()
4364 SDValue Undef = DAG.getUNDEF(WideOpVT); in insert1BitVector()
4368 SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8); in insert1BitVector()
4369 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, in insert1BitVector()
4371 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits); in insert1BitVector()
4372 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits); in insert1BitVector()
4374 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, in insert1BitVector()
4375 DAG.getConstant(0, dl, WideOpVT), in insert1BitVector()
4377 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec); in insert1BitVector()
4378 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx); in insert1BitVector()
4381 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, in insert1BitVector()
4386 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
4387 DAG.getTargetConstant(IdxVal, dl, MVT::i8)); in insert1BitVector()
4388 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx); in insert1BitVector()
4396 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
4397 DAG.getTargetConstant(IdxVal, dl, MVT::i8)); in insert1BitVector()
4402 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
4403 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8)); in insert1BitVector()
4405 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec, in insert1BitVector()
4406 DAG.getTargetConstant(ShiftRight, dl, MVT::i8)); in insert1BitVector()
4408 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx); in insert1BitVector()
4413 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
4414 DAG.getTargetConstant(IdxVal, dl, MVT::i8)); in insert1BitVector()
4418 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVecVT, Vec, ZeroIdx); in insert1BitVector()
4419 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, in insert1BitVector()
4420 DAG.getConstant(0, dl, WideOpVT), in insert1BitVector()
4424 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, in insert1BitVector()
4427 SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8); in insert1BitVector()
4428 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits); in insert1BitVector()
4429 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits); in insert1BitVector()
4431 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec); in insert1BitVector()
4432 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx); in insert1BitVector()
4440 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx); in insert1BitVector()
4449 SDValue CMask0 = DAG.getConstant(Mask0, dl, MVT::getIntegerVT(NumElems)); in insert1BitVector()
4450 SDValue VMask0 = DAG.getNode(ISD::BITCAST, dl, WideOpVT, CMask0); in insert1BitVector()
4451 Vec = DAG.getNode(ISD::AND, dl, WideOpVT, Vec, VMask0); in insert1BitVector()
4452 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
4453 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8)); in insert1BitVector()
4454 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec, in insert1BitVector()
4455 DAG.getTargetConstant(ShiftRight, dl, MVT::i8)); in insert1BitVector()
4456 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec); in insert1BitVector()
4459 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx); in insert1BitVector()
4463 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec, in insert1BitVector()
4464 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8)); in insert1BitVector()
4465 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec, in insert1BitVector()
4466 DAG.getTargetConstant(ShiftRight, dl, MVT::i8)); in insert1BitVector()
4470 SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, in insert1BitVector()
4471 DAG.getTargetConstant(LowShift, dl, MVT::i8)); in insert1BitVector()
4472 Low = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Low, in insert1BitVector()
4473 DAG.getTargetConstant(LowShift, dl, MVT::i8)); in insert1BitVector()
4477 SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, in insert1BitVector()
4478 DAG.getTargetConstant(HighShift, dl, MVT::i8)); in insert1BitVector()
4479 High = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, High, in insert1BitVector()
4480 DAG.getTargetConstant(HighShift, dl, MVT::i8)); in insert1BitVector()
4483 Vec = DAG.getNode(ISD::OR, dl, WideOpVT, Low, High); in insert1BitVector()
4484 SubVec = DAG.getNode(ISD::OR, dl, WideOpVT, SubVec, Vec); in insert1BitVector()
4487 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx); in insert1BitVector()
4490 static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG, in concatSubVectors() argument
4497 EVT VT = EVT::getVectorVT(*DAG.getContext(), SubSVT, 2 * SubNumElts); in concatSubVectors()
4498 SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth); in concatSubVectors()
4499 return insertSubVector(V, V2, SubNumElts, DAG, dl, SubVectorWidth); in concatSubVectors()
4505 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) { in getOnesVector() argument
4509 SDValue Vec = DAG.getAllOnesConstant(dl, MVT::getVectorVT(MVT::i32, NumElts)); in getOnesVector()
4510 return DAG.getBitcast(VT, Vec); in getOnesVector()
4514 SDValue In, SelectionDAG &DAG) { in getEXTEND_VECTOR_INREG() argument
4527 In = extractSubVector(In, 0, DAG, DL, in getEXTEND_VECTOR_INREG()
4533 Opcode = DAG.getOpcode_EXTEND_VECTOR_INREG(Opcode); in getEXTEND_VECTOR_INREG()
4535 return DAG.getNode(Opcode, DL, VT, In); in getEXTEND_VECTOR_INREG()
4540 SDValue Mask, SelectionDAG &DAG) { in getBitSelect() argument
4541 LHS = DAG.getNode(ISD::AND, DL, VT, LHS, Mask); in getBitSelect()
4542 RHS = DAG.getNode(X86ISD::ANDNP, DL, VT, Mask, RHS); in getBitSelect()
4543 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in getBitSelect()
4578 static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl, in getVectorShuffle() argument
4582 SmallVector<SDValue> Ops(Mask.size(), DAG.getUNDEF(VT.getScalarType())); in getVectorShuffle()
4592 return DAG.getBuildVector(VT, dl, Ops); in getVectorShuffle()
4595 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask); in getVectorShuffle()
4599 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT, in getUnpackl() argument
4603 return getVectorShuffle(DAG, VT, dl, V1, V2, Mask); in getUnpackl()
4607 static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT, in getUnpackh() argument
4611 return getVectorShuffle(DAG, VT, dl, V1, V2, Mask); in getUnpackh()
4617 static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget, in getPack() argument
4641 return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, LHS), in getPack()
4642 DAG.getBitcast(VT, RHS), PackMask); in getPack()
4648 DAG.computeKnownBits(LHS).countMaxActiveBits() <= EltSizeInBits && in getPack()
4649 DAG.computeKnownBits(RHS).countMaxActiveBits() <= EltSizeInBits) in getPack()
4650 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS); in getPack()
4652 if (DAG.ComputeMaxSignificantBits(LHS) <= EltSizeInBits && in getPack()
4653 DAG.ComputeMaxSignificantBits(RHS) <= EltSizeInBits) in getPack()
4654 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS); in getPack()
4658 SDValue Amt = DAG.getTargetConstant(EltSizeInBits, dl, MVT::i8); in getPack()
4661 LHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, LHS, Amt); in getPack()
4662 RHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, RHS, Amt); in getPack()
4664 SDValue Mask = DAG.getConstant((1ULL << EltSizeInBits) - 1, dl, OpVT); in getPack()
4665 LHS = DAG.getNode(ISD::AND, dl, OpVT, LHS, Mask); in getPack()
4666 RHS = DAG.getNode(ISD::AND, dl, OpVT, RHS, Mask); in getPack()
4668 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS); in getPack()
4672 LHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, LHS, Amt); in getPack()
4673 RHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, RHS, Amt); in getPack()
4675 LHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, LHS, Amt); in getPack()
4676 RHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, RHS, Amt); in getPack()
4677 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS); in getPack()
4687 SelectionDAG &DAG) { in getShuffleVectorZeroOrUndef() argument
4690 ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT); in getShuffleVectorZeroOrUndef()
4696 return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, MaskVec); in getShuffleVectorZeroOrUndef()
5112 static SDValue IsNOT(SDValue V, SelectionDAG &DAG) { in IsNOT() argument
5120 if (SDValue Not = IsNOT(V.getOperand(0), DAG)) { in IsNOT()
5121 Not = DAG.getBitcast(V.getOperand(0).getValueType(), Not); in IsNOT()
5122 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Not), V.getValueType(), in IsNOT()
5144 return DAG.getNode(X86ISD::PCMPGT, DL, VT, V.getOperand(1), in IsNOT()
5145 getConstVector(EltBits, UndefElts, VT, DAG, DL)); in IsNOT()
5150 if (collectConcatOps(V.getNode(), CatOps, DAG)) { in IsNOT()
5152 SDValue NotCat = IsNOT(CatOp, DAG); in IsNOT()
5154 CatOp = DAG.getBitcast(CatOp.getValueType(), NotCat); in IsNOT()
5156 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(V), V.getValueType(), CatOps); in IsNOT()
5798 const SelectionDAG &DAG, unsigned Depth,
5808 const SelectionDAG &DAG, unsigned Depth, in getFauxShuffleMask() argument
5873 if (!getTargetShuffleInputs(N0, Demand0, SrcInputs0, SrcMask0, DAG, in getFauxShuffleMask()
5875 !getTargetShuffleInputs(N1, Demand1, SrcInputs1, SrcMask1, DAG, in getFauxShuffleMask()
5958 if (!getTargetShuffleInputs(SubSrc, SubDemand, SubInputs, SubMask, DAG, in getFauxShuffleMask()
6099 DAG.ComputeNumSignBits(N0, EltsLHS, Depth + 1) <= NumBitsPerElt) || in getFauxShuffleMask()
6101 DAG.ComputeNumSignBits(N1, EltsRHS, Depth + 1) <= NumBitsPerElt)) in getFauxShuffleMask()
6119 !DAG.MaskedValueIsZero(N0, ZeroMask, EltsLHS, Depth + 1)) || in getFauxShuffleMask()
6121 !DAG.MaskedValueIsZero(N1, ZeroMask, EltsRHS, Depth + 1))) in getFauxShuffleMask()
6242 if (DAG.ComputeNumSignBits(Src, DemandedSrcElts) != NumBitsPerSrcElt) in getFauxShuffleMask()
6325 const SelectionDAG &DAG, unsigned Depth, in getTargetShuffleInputs() argument
6339 if (getFauxShuffleMask(Op, DemandedElts, Mask, Inputs, DAG, Depth, in getTargetShuffleInputs()
6350 const SelectionDAG &DAG, unsigned Depth, in getTargetShuffleInputs() argument
6354 KnownZero, DAG, Depth, ResolveKnownElts); in getTargetShuffleInputs()
6359 const SelectionDAG &DAG, unsigned Depth = 0, in getTargetShuffleInputs() argument
6367 return getTargetShuffleInputs(Op, DemandedElts, Inputs, Mask, DAG, Depth, in getTargetShuffleInputs()
6374 SelectionDAG &DAG) { in getBROADCAST_LOAD() argument
6383 SDValue Ptr = DAG.getMemBasePlusOffset(Mem->getBasePtr(), in getBROADCAST_LOAD()
6385 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in getBROADCAST_LOAD()
6387 SDValue BcstLd = DAG.getMemIntrinsicNode( in getBROADCAST_LOAD()
6389 DAG.getMachineFunction().getMachineMemOperand( in getBROADCAST_LOAD()
6391 DAG.makeEquivalentMemoryOrdering(SDValue(Mem, 1), BcstLd.getValue(1)); in getBROADCAST_LOAD()
6398 SelectionDAG &DAG, unsigned Depth) { in getShuffleScalarElt() argument
6411 return DAG.getUNDEF(VT.getVectorElementType()); in getShuffleScalarElt()
6414 return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1); in getShuffleScalarElt()
6429 return ShufSVT.isInteger() ? DAG.getConstant(0, SDLoc(Op), ShufSVT) in getShuffleScalarElt()
6430 : DAG.getConstantFP(+0.0, SDLoc(Op), ShufSVT); in getShuffleScalarElt()
6432 return DAG.getUNDEF(ShufSVT); in getShuffleScalarElt()
6436 return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1); in getShuffleScalarElt()
6447 return getShuffleScalarElt(Sub, Index - SubIdx, DAG, Depth + 1); in getShuffleScalarElt()
6448 return getShuffleScalarElt(Vec, Index, DAG, Depth + 1); in getShuffleScalarElt()
6457 return getShuffleScalarElt(Op.getOperand(SubIdx), SubElt, DAG, Depth + 1); in getShuffleScalarElt()
6464 return getShuffleScalarElt(Src, Index + SrcIdx, DAG, Depth + 1); in getShuffleScalarElt()
6472 return getShuffleScalarElt(Src, Index, DAG, Depth + 1); in getShuffleScalarElt()
6484 return getShuffleScalarElt(Op.getOperand(0), Index, DAG, Depth + 1); in getShuffleScalarElt()
6489 : DAG.getUNDEF(VT.getVectorElementType()); in getShuffleScalarElt()
6501 SelectionDAG &DAG, in LowerBuildVectorAsInsert() argument
6523 V = getZeroVector(VT, Subtarget, DAG, DL); in LowerBuildVectorAsInsert()
6526 V = DAG.getAnyExtOrTrunc(Op.getOperand(i), DL, MVT::i32); in LowerBuildVectorAsInsert()
6527 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, V); in LowerBuildVectorAsInsert()
6528 V = DAG.getBitcast(VT, V); in LowerBuildVectorAsInsert()
6532 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, V, Op.getOperand(i), in LowerBuildVectorAsInsert()
6533 DAG.getIntPtrConstant(i, DL)); in LowerBuildVectorAsInsert()
6543 SelectionDAG &DAG, in LowerBuildVectorv16i8() argument
6551 DAG, Subtarget); in LowerBuildVectorv16i8()
6562 SDValue Elt = DAG.getZExtOrTrunc(Op.getOperand(I), DL, MVT::i32); in LowerBuildVectorv16i8()
6564 Elt = DAG.getNode(ISD::SHL, DL, MVT::i32, Elt, in LowerBuildVectorv16i8()
6565 DAG.getConstant(I * 8, DL, MVT::i8)); in LowerBuildVectorv16i8()
6566 V = V ? DAG.getNode(ISD::OR, DL, MVT::i32, V, Elt) : Elt; in LowerBuildVectorv16i8()
6569 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, V); in LowerBuildVectorv16i8()
6570 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v4i32, V); in LowerBuildVectorv16i8()
6571 V = DAG.getBitcast(MVT::v8i16, V); in LowerBuildVectorv16i8()
6582 Elt = DAG.getZExtOrTrunc(Op.getOperand(i), DL, MVT::i32); in LowerBuildVectorv16i8()
6584 Elt = DAG.getAnyExtOrTrunc(Op.getOperand(i), DL, MVT::i32); in LowerBuildVectorv16i8()
6590 NextElt = DAG.getZExtOrTrunc(NextElt, DL, MVT::i32); in LowerBuildVectorv16i8()
6592 NextElt = DAG.getAnyExtOrTrunc(NextElt, DL, MVT::i32); in LowerBuildVectorv16i8()
6593 NextElt = DAG.getNode(ISD::SHL, DL, MVT::i32, NextElt, in LowerBuildVectorv16i8()
6594 DAG.getConstant(8, DL, MVT::i8)); in LowerBuildVectorv16i8()
6596 Elt = DAG.getNode(ISD::OR, DL, MVT::i32, NextElt, Elt); in LowerBuildVectorv16i8()
6606 V = getZeroVector(MVT::v8i16, Subtarget, DAG, DL); in LowerBuildVectorv16i8()
6608 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, Elt); in LowerBuildVectorv16i8()
6609 V = DAG.getBitcast(MVT::v8i16, V); in LowerBuildVectorv16i8()
6613 Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt); in LowerBuildVectorv16i8()
6614 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v8i16, V, Elt, in LowerBuildVectorv16i8()
6615 DAG.getIntPtrConstant(i / 2, DL)); in LowerBuildVectorv16i8()
6618 return DAG.getBitcast(MVT::v16i8, V); in LowerBuildVectorv16i8()
6625 SelectionDAG &DAG, in LowerBuildVectorv8i16() argument
6631 return LowerBuildVectorAsInsert(Op, DL, NonZeroMask, NumNonZero, NumZero, DAG, in LowerBuildVectorv8i16()
6637 SelectionDAG &DAG, in LowerBuildVectorv4x32() argument
6652 DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) }; in LowerBuildVectorv4x32()
6653 SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops)); in LowerBuildVectorv4x32()
6654 SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV); in LowerBuildVectorv4x32()
6655 return DAG.getBitcast(VT, Dup); in LowerBuildVectorv4x32()
6715 ? DAG.getUNDEF(VT) in LowerBuildVectorv4x32()
6716 : getZeroVector(VT, Subtarget, DAG, DL); in LowerBuildVectorv4x32()
6718 V1 = DAG.getBitcast(VT, V1); in LowerBuildVectorv4x32()
6719 return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZeroOrUndef, Mask); in LowerBuildVectorv4x32()
6747 V1 = DAG.getBitcast(MVT::v4f32, V1); in LowerBuildVectorv4x32()
6749 V2 = DAG.getBitcast(MVT::v4f32, V2); in LowerBuildVectorv4x32()
6756 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, in LowerBuildVectorv4x32()
6757 DAG.getIntPtrConstant(InsertPSMask, DL, true)); in LowerBuildVectorv4x32()
6758 return DAG.getBitcast(VT, Result); in LowerBuildVectorv4x32()
6763 SelectionDAG &DAG, const TargetLowering &TLI, in getVShift() argument
6768 SrcOp = DAG.getBitcast(ShVT, SrcOp); in getVShift()
6770 SDValue ShiftVal = DAG.getTargetConstant(NumBits / 8, dl, MVT::i8); in getVShift()
6771 return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal)); in getVShift()
6775 SelectionDAG &DAG) { in LowerAsSplatVectorLoad() argument
6793 } else if (DAG.isBaseWithConstantOffset(Ptr) && in LowerAsSplatVectorLoad()
6807 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); in LowerAsSplatVectorLoad()
6808 MaybeAlign InferredAlign = DAG.InferPtrAlign(Ptr); in LowerAsSplatVectorLoad()
6829 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, in LowerAsSplatVectorLoad()
6830 DAG.getConstant(StartOffset, DL, Ptr.getValueType())); in LowerAsSplatVectorLoad()
6836 EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems); in LowerAsSplatVectorLoad()
6837 SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr, in LowerAsSplatVectorLoad()
6842 return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), Mask); in LowerAsSplatVectorLoad()
6897 const SDLoc &DL, SelectionDAG &DAG, in EltsFromConsecutiveLoads() argument
6949 return DAG.getUNDEF(VT); in EltsFromConsecutiveLoads()
6951 return VT.isInteger() ? DAG.getConstant(0, DL, VT) in EltsFromConsecutiveLoads()
6952 : DAG.getConstantFP(0.0, DL, VT); in EltsFromConsecutiveLoads()
6954 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in EltsFromConsecutiveLoads()
6982 return DAG.areNonVolatileConsecutiveLoads(Ld, Base, BaseSizeInBytes, in EltsFromConsecutiveLoads()
7003 auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) { in EltsFromConsecutiveLoads()
7008 DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), in EltsFromConsecutiveLoads()
7013 DAG.makeEquivalentMemoryOrdering(LD, NewLd); in EltsFromConsecutiveLoads()
7019 VT.getSizeInBits() / 8, *DAG.getContext(), DAG.getDataLayout()); in EltsFromConsecutiveLoads()
7039 return DAG.getBitcast(VT, Elts[FirstLoadedElt]); in EltsFromConsecutiveLoads()
7059 SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT) in EltsFromConsecutiveLoads()
7060 : DAG.getConstantFP(0.0, DL, VT); in EltsFromConsecutiveLoads()
7061 return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask); in EltsFromConsecutiveLoads()
7071 EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), HalfNumElems); in EltsFromConsecutiveLoads()
7074 DAG, Subtarget, IsAfterLegalize); in EltsFromConsecutiveLoads()
7076 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), in EltsFromConsecutiveLoads()
7077 HalfLD, DAG.getIntPtrConstant(0, DL)); in EltsFromConsecutiveLoads()
7094 SDVTList Tys = DAG.getVTList(VecVT, MVT::Other); in EltsFromConsecutiveLoads()
7096 SDValue ResNode = DAG.getMemIntrinsicNode( in EltsFromConsecutiveLoads()
7101 DAG.makeEquivalentMemoryOrdering(LD, ResNode); in EltsFromConsecutiveLoads()
7102 return DAG.getBitcast(VT, ResNode); in EltsFromConsecutiveLoads()
7122 SmallVector<SDValue, 8> RepeatedLoads(SubElems, DAG.getUNDEF(EltBaseVT)); in EltsFromConsecutiveLoads()
7141 ? EVT::getIntegerVT(*DAG.getContext(), ScalarSize) in EltsFromConsecutiveLoads()
7144 RepeatVT = EVT::getVectorVT(*DAG.getContext(), RepeatVT, in EltsFromConsecutiveLoads()
7147 EVT::getVectorVT(*DAG.getContext(), RepeatVT.getScalarType(), in EltsFromConsecutiveLoads()
7151 RepeatVT, RepeatedLoads, DL, DAG, Subtarget, IsAfterLegalize)) { in EltsFromConsecutiveLoads()
7155 Broadcast = concatSubVectors(Broadcast, Broadcast, DAG, DL); in EltsFromConsecutiveLoads()
7164 DAG.getNode(X86ISD::VBROADCAST, DL, BroadcastVT, RepeatLoad); in EltsFromConsecutiveLoads()
7166 return DAG.getBitcast(VT, Broadcast); in EltsFromConsecutiveLoads()
7179 SelectionDAG &DAG, in combineToConsecutiveLoads() argument
7184 if (SDValue Elt = getShuffleScalarElt(Op, i, DAG, 0)) { in combineToConsecutiveLoads()
7191 return EltsFromConsecutiveLoads(VT, Elts, DL, DAG, Subtarget, in combineToConsecutiveLoads()
7281 SelectionDAG &DAG) { in lowerBuildVectorAsBroadcast() argument
7290 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in lowerBuildVectorAsBroadcast()
7333 SDValue Bcst = DAG.getNode(X86ISD::VBROADCASTM, dl, BcstVT, BOperand); in lowerBuildVectorAsBroadcast()
7335 Bcst = extractSubVector(Bcst, 0, DAG, dl, VT.getSizeInBits()); in lowerBuildVectorAsBroadcast()
7336 return DAG.getBitcast(VT, Bcst); in lowerBuildVectorAsBroadcast()
7355 LLVMContext *Ctx = DAG.getContext(); in lowerBuildVectorAsBroadcast()
7356 MVT PVT = TLI.getPointerTy(DAG.getDataLayout()); in lowerBuildVectorAsBroadcast()
7362 SDValue CP = DAG.getConstantPool(C, PVT); in lowerBuildVectorAsBroadcast()
7366 SDVTList Tys = DAG.getVTList(MVT::getVectorVT(CVT, Repeat), MVT::Other); in lowerBuildVectorAsBroadcast()
7367 SDValue Ops[] = {DAG.getEntryNode(), CP}; in lowerBuildVectorAsBroadcast()
7369 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()); in lowerBuildVectorAsBroadcast()
7371 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, CVT, in lowerBuildVectorAsBroadcast()
7373 return DAG.getBitcast(VT, Brdcst); in lowerBuildVectorAsBroadcast()
7378 SDValue VCP = DAG.getConstantPool(VecC, PVT); in lowerBuildVectorAsBroadcast()
7382 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
7383 SDValue Ops[] = {DAG.getEntryNode(), VCP}; in lowerBuildVectorAsBroadcast()
7385 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()); in lowerBuildVectorAsBroadcast()
7386 return DAG.getMemIntrinsicNode(X86ISD::SUBV_BROADCAST_LOAD, dl, Tys, in lowerBuildVectorAsBroadcast()
7424 bool OptForSize = DAG.shouldOptForSize(); in lowerBuildVectorAsBroadcast()
7453 DAG.getConstantPool(C, TLI.getPointerTy(DAG.getDataLayout())); in lowerBuildVectorAsBroadcast()
7456 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
7457 SDValue Ops[] = {DAG.getEntryNode(), CP}; in lowerBuildVectorAsBroadcast()
7459 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()); in lowerBuildVectorAsBroadcast()
7460 return DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, CVT, in lowerBuildVectorAsBroadcast()
7468 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); in lowerBuildVectorAsBroadcast()
7481 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
7484 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, in lowerBuildVectorAsBroadcast()
7486 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1)); in lowerBuildVectorAsBroadcast()
7495 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
7498 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, in lowerBuildVectorAsBroadcast()
7500 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1)); in lowerBuildVectorAsBroadcast()
7505 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); in lowerBuildVectorAsBroadcast()
7547 SelectionDAG &DAG) { in buildFromShuffleMostly() argument
7551 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in buildFromShuffleMostly()
7607 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT); in buildFromShuffleMostly()
7608 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask); in buildFromShuffleMostly()
7611 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx), in buildFromShuffleMostly()
7612 DAG.getIntPtrConstant(Idx, DL)); in buildFromShuffleMostly()
7618 static SDValue LowerBUILD_VECTORvXbf16(SDValue Op, SelectionDAG &DAG, in LowerBUILD_VECTORvXbf16() argument
7625 NewOps.push_back(DAG.getBitcast(Subtarget.hasFP16() ? MVT::f16 : MVT::i16, in LowerBUILD_VECTORvXbf16()
7627 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(), IVT, NewOps); in LowerBUILD_VECTORvXbf16()
7628 return DAG.getBitcast(VT, Res); in LowerBUILD_VECTORvXbf16()
7633 SelectionDAG &DAG, in LowerBUILD_VECTORvXi1() argument
7673 Cond = DAG.getNode(ISD::AND, dl, MVT::i8, Cond, in LowerBUILD_VECTORvXi1()
7674 DAG.getConstant(1, dl, MVT::i8)); in LowerBUILD_VECTORvXi1()
7678 SDValue Select = DAG.getSelect(dl, MVT::i32, Cond, in LowerBUILD_VECTORvXi1()
7679 DAG.getAllOnesConstant(dl, MVT::i32), in LowerBUILD_VECTORvXi1()
7680 DAG.getConstant(0, dl, MVT::i32)); in LowerBUILD_VECTORvXi1()
7681 Select = DAG.getBitcast(MVT::v32i1, Select); in LowerBUILD_VECTORvXi1()
7682 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Select, Select); in LowerBUILD_VECTORvXi1()
7685 SDValue Select = DAG.getSelect(dl, ImmVT, Cond, in LowerBUILD_VECTORvXi1()
7686 DAG.getAllOnesConstant(dl, ImmVT), in LowerBUILD_VECTORvXi1()
7687 DAG.getConstant(0, dl, ImmVT)); in LowerBUILD_VECTORvXi1()
7689 Select = DAG.getBitcast(VecVT, Select); in LowerBUILD_VECTORvXi1()
7690 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Select, in LowerBUILD_VECTORvXi1()
7691 DAG.getIntPtrConstant(0, dl)); in LowerBUILD_VECTORvXi1()
7699 SDValue ImmL = DAG.getConstant(Lo_32(Immediate), dl, MVT::i32); in LowerBUILD_VECTORvXi1()
7700 SDValue ImmH = DAG.getConstant(Hi_32(Immediate), dl, MVT::i32); in LowerBUILD_VECTORvXi1()
7701 ImmL = DAG.getBitcast(MVT::v32i1, ImmL); in LowerBUILD_VECTORvXi1()
7702 ImmH = DAG.getBitcast(MVT::v32i1, ImmH); in LowerBUILD_VECTORvXi1()
7703 DstVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, ImmL, ImmH); in LowerBUILD_VECTORvXi1()
7706 SDValue Imm = DAG.getConstant(Immediate, dl, ImmVT); in LowerBUILD_VECTORvXi1()
7708 DstVec = DAG.getBitcast(VecVT, Imm); in LowerBUILD_VECTORvXi1()
7709 DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, DstVec, in LowerBUILD_VECTORvXi1()
7710 DAG.getIntPtrConstant(0, dl)); in LowerBUILD_VECTORvXi1()
7713 DstVec = DAG.getUNDEF(VT); in LowerBUILD_VECTORvXi1()
7716 DstVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, in LowerBUILD_VECTORvXi1()
7718 DAG.getIntPtrConstant(InsertIdx, dl)); in LowerBUILD_VECTORvXi1()
7757 const SDLoc &DL, SelectionDAG &DAG, in isHorizontalBinOpPart() argument
7770 V0 = DAG.getUNDEF(VT); in isHorizontalBinOpPart()
7771 V1 = DAG.getUNDEF(VT); in isHorizontalBinOpPart()
7871 const SDLoc &DL, SelectionDAG &DAG, in ExpandHorizontalBinOp() argument
7879 SDValue V0_LO = extract128BitVector(V0, 0, DAG, DL); in ExpandHorizontalBinOp()
7880 SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL); in ExpandHorizontalBinOp()
7881 SDValue V1_LO = extract128BitVector(V1, 0, DAG, DL); in ExpandHorizontalBinOp()
7882 SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL); in ExpandHorizontalBinOp()
7885 SDValue LO = DAG.getUNDEF(NewVT); in ExpandHorizontalBinOp()
7886 SDValue HI = DAG.getUNDEF(NewVT); in ExpandHorizontalBinOp()
7891 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI); in ExpandHorizontalBinOp()
7893 HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI); in ExpandHorizontalBinOp()
7897 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO); in ExpandHorizontalBinOp()
7900 HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI); in ExpandHorizontalBinOp()
7903 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI); in ExpandHorizontalBinOp()
7912 const X86Subtarget &Subtarget, SelectionDAG &DAG, in isAddSubOrSubAdd() argument
7922 SDValue InVec0 = DAG.getUNDEF(VT); in isAddSubOrSubAdd()
7923 SDValue InVec1 = DAG.getUNDEF(VT); in isAddSubOrSubAdd()
8039 SelectionDAG &DAG, in isFMAddSubOrFMSubAdd() argument
8050 const TargetOptions &Options = DAG.getTarget().Options; in isFMAddSubOrFMSubAdd()
8069 SelectionDAG &DAG) { in lowerToAddSubOrFMAddSub() argument
8073 if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, NumExtracts, in lowerToAddSubOrFMAddSub()
8081 if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, NumExtracts)) { in lowerToAddSubOrFMAddSub()
8083 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2); in lowerToAddSubOrFMAddSub()
8098 SDValue Sub = DAG.getNode(ISD::FSUB, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
8099 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
8100 return DAG.getVectorShuffle(VT, DL, Sub, Add, Mask); in lowerToAddSubOrFMAddSub()
8103 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
8106 static bool isHopBuildVector(const BuildVectorSDNode *BV, SelectionDAG &DAG, in isHopBuildVector() argument
8111 V0 = DAG.getUNDEF(VT); in isHopBuildVector()
8112 V1 = DAG.getUNDEF(VT); in isHopBuildVector()
8196 const SDLoc &DL, SelectionDAG &DAG, in getHopForBuildVector() argument
8204 V0 = extractSubVector(V0, 0, DAG, DL, Width); in getHopForBuildVector()
8206 V0 = insertSubVector(DAG.getUNDEF(VT), V0, 0, DAG, DL, Width); in getHopForBuildVector()
8209 V1 = extractSubVector(V1, 0, DAG, DL, Width); in getHopForBuildVector()
8211 V1 = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, DL, Width); in getHopForBuildVector()
8223 V0 = extractSubVector(V0, 0, DAG, DL, 128); in getHopForBuildVector()
8224 V1 = extractSubVector(V1, 0, DAG, DL, 128); in getHopForBuildVector()
8225 SDValue Half = DAG.getNode(HOpcode, DL, HalfVT, V0, V1); in getHopForBuildVector()
8226 return insertSubVector(DAG.getUNDEF(VT), Half, 0, DAG, DL, 256); in getHopForBuildVector()
8229 return DAG.getNode(HOpcode, DL, VT, V0, V1); in getHopForBuildVector()
8235 SelectionDAG &DAG) { in LowerToHorizontalOp() argument
8252 if (isHopBuildVector(BV, DAG, HOpcode, V0, V1)) in LowerToHorizontalOp()
8253 return getHopForBuildVector(BV, DL, DAG, HOpcode, V0, V1); in LowerToHorizontalOp()
8279 if (isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, 0, Half, InVec0, InVec1) && in LowerToHorizontalOp()
8280 isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, Half, NumElts, InVec2, in LowerToHorizontalOp()
8285 else if (isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, 0, Half, InVec0, in LowerToHorizontalOp()
8287 isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, Half, NumElts, InVec2, in LowerToHorizontalOp()
8309 return ExpandHorizontalBinOp(V0, V1, DL, DAG, X86Opcode, false, isUndefLO, in LowerToHorizontalOp()
8317 if (isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, 0, NumElts, InVec0, in LowerToHorizontalOp()
8320 else if (isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, 0, NumElts, InVec0, in LowerToHorizontalOp()
8323 else if (isHorizontalBinOpPart(BV, ISD::FADD, DL, DAG, 0, NumElts, InVec0, in LowerToHorizontalOp()
8326 else if (isHorizontalBinOpPart(BV, ISD::FSUB, DL, DAG, 0, NumElts, InVec0, in LowerToHorizontalOp()
8341 return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true, in LowerToHorizontalOp()
8349 SelectionDAG &DAG);
8359 SelectionDAG &DAG) { in lowerBuildVectorToBitOp() argument
8362 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in lowerBuildVectorToBitOp()
8406 RHS = DAG.getZExtOrTrunc(RHS, DL, VT.getScalarType()); in lowerBuildVectorToBitOp()
8419 SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts); in lowerBuildVectorToBitOp()
8420 SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts); in lowerBuildVectorToBitOp()
8421 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS); in lowerBuildVectorToBitOp()
8428 return LowerShift(Res, Subtarget, DAG); in lowerBuildVectorToBitOp()
8435 SelectionDAG &DAG, in materializeVectorConstant() argument
8450 return getOnesVector(VT, DAG, DL); in materializeVectorConstant()
8460 const SDLoc &DL, SelectionDAG &DAG, in createVariablePermute() argument
8473 IndicesVec = extractSubVector(IndicesVec, 0, DAG, SDLoc(IndicesVec), in createVariablePermute()
8476 IndicesVec = widenSubVector(IndicesVec, false, Subtarget, DAG, in createVariablePermute()
8480 IndicesVec = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(IndicesVec), in createVariablePermute()
8483 IndicesVec = DAG.getZExtOrTrunc(IndicesVec, SDLoc(IndicesVec), IndicesVT); in createVariablePermute()
8493 Subtarget, DAG, SDLoc(IndicesVec)); in createVariablePermute()
8495 createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget); in createVariablePermute()
8497 return extractSubVector(NewSrcVec, 0, DAG, DL, SizeInBits); in createVariablePermute()
8501 SrcVec = widenSubVector(VT, SrcVec, false, Subtarget, DAG, SDLoc(SrcVec)); in createVariablePermute()
8506 auto ScaleIndices = [&DAG](SDValue Idx, uint64_t Scale) { in createVariablePermute()
8523 Idx = DAG.getNode(ISD::MUL, SDLoc(Idx), SrcVT, Idx, in createVariablePermute()
8524 DAG.getConstant(IndexScale, SDLoc(Idx), SrcVT)); in createVariablePermute()
8525 Idx = DAG.getNode(ISD::ADD, SDLoc(Idx), SrcVT, Idx, in createVariablePermute()
8526 DAG.getConstant(IndexOffset, SDLoc(Idx), SrcVT)); in createVariablePermute()
8560 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec); in createVariablePermute()
8565 return DAG.getSelectCC( in createVariablePermute()
8567 getZeroVector(IndicesVT.getSimpleVT(), Subtarget, DAG, DL), in createVariablePermute()
8568 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {0, 0}), in createVariablePermute()
8569 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {1, 1}), in createVariablePermute()
8577 SDValue LoSrc = extract128BitVector(SrcVec, 0, DAG, DL); in createVariablePermute()
8578 SDValue HiSrc = extract128BitVector(SrcVec, 16, DAG, DL); in createVariablePermute()
8579 SDValue LoIdx = extract128BitVector(IndicesVec, 0, DAG, DL); in createVariablePermute()
8580 SDValue HiIdx = extract128BitVector(IndicesVec, 16, DAG, DL); in createVariablePermute()
8581 return DAG.getNode( in createVariablePermute()
8583 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, LoIdx), in createVariablePermute()
8584 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, HiIdx)); in createVariablePermute()
8586 SDValue Lo = extract128BitVector(SrcVec, 0, DAG, DL); in createVariablePermute()
8587 SDValue Hi = extract128BitVector(SrcVec, 16, DAG, DL); in createVariablePermute()
8588 SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo); in createVariablePermute()
8589 SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi); in createVariablePermute()
8590 auto PSHUFBBuilder = [](SelectionDAG &DAG, const SDLoc &DL, in createVariablePermute()
8597 return DAG.getSelectCC(DL, Idx, DAG.getConstant(15, DL, VT), in createVariablePermute()
8598 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx), in createVariablePermute()
8599 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx), in createVariablePermute()
8603 return SplitOpsAndApply(DAG, Subtarget, DL, MVT::v32i8, Ops, in createVariablePermute()
8613 return DAG.getBitcast( in createVariablePermute()
8615 MVT::v32i8, DAG.getBitcast(MVT::v32i8, SrcVec), in createVariablePermute()
8616 DAG.getBitcast(MVT::v32i8, IndicesVec), DL, DAG, Subtarget)); in createVariablePermute()
8624 SrcVec = DAG.getBitcast(MVT::v8f32, SrcVec); in createVariablePermute()
8625 SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec, in createVariablePermute()
8627 SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec, in createVariablePermute()
8630 return DAG.getBitcast( in createVariablePermute()
8631 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi, in createVariablePermute()
8632 IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8))); in createVariablePermute()
8635 SDValue Res = DAG.getSelectCC( in createVariablePermute()
8636 DL, IndicesVec, DAG.getConstant(3, DL, MVT::v8i32), in createVariablePermute()
8637 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, HiHi, IndicesVec), in createVariablePermute()
8638 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, LoLo, IndicesVec), in createVariablePermute()
8640 return DAG.getBitcast(VT, Res); in createVariablePermute()
8648 SrcVec = widenSubVector(WidenSrcVT, SrcVec, false, Subtarget, DAG, in createVariablePermute()
8651 DAG, SDLoc(IndicesVec)); in createVariablePermute()
8653 DAG, Subtarget); in createVariablePermute()
8654 return extract256BitVector(Res, 0, DAG, DL); in createVariablePermute()
8658 SrcVec = DAG.getBitcast(MVT::v4f64, SrcVec); in createVariablePermute()
8660 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {0, 1, 0, 1}); in createVariablePermute()
8662 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {2, 3, 2, 3}); in createVariablePermute()
8664 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec); in createVariablePermute()
8666 return DAG.getBitcast( in createVariablePermute()
8667 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi, in createVariablePermute()
8668 IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8))); in createVariablePermute()
8671 SDValue Res = DAG.getSelectCC( in createVariablePermute()
8672 DL, IndicesVec, DAG.getConstant(2, DL, MVT::v4i64), in createVariablePermute()
8673 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, HiHi, IndicesVec), in createVariablePermute()
8674 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, LoLo, IndicesVec), in createVariablePermute()
8676 return DAG.getBitcast(VT, Res); in createVariablePermute()
8707 IndicesVec = DAG.getBitcast(ShuffleIdxVT, IndicesVec); in createVariablePermute()
8709 SrcVec = DAG.getBitcast(ShuffleVT, SrcVec); in createVariablePermute()
8711 ? DAG.getNode(Opcode, DL, ShuffleVT, IndicesVec, SrcVec) in createVariablePermute()
8712 : DAG.getNode(Opcode, DL, ShuffleVT, SrcVec, IndicesVec); in createVariablePermute()
8713 return DAG.getBitcast(VT, Res); in createVariablePermute()
8729 SelectionDAG &DAG, in LowerBUILD_VECTORAsVariablePermute() argument
8769 return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget); in LowerBUILD_VECTORAsVariablePermute()
8773 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { in LowerBUILD_VECTOR()
8783 return LowerBUILD_VECTORvXi1(Op, dl, DAG, Subtarget); in LowerBUILD_VECTOR()
8787 return LowerBUILD_VECTORvXbf16(Op, DAG, Subtarget); in LowerBUILD_VECTOR()
8789 if (SDValue VectorCst = materializeVectorConstant(Op, dl, DAG, Subtarget)) in LowerBUILD_VECTOR()
8826 return DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
8830 return DAG.getFreeze(DAG.getUNDEF(VT)); in LowerBUILD_VECTOR()
8834 return getZeroVector(VT, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
8844 SmallVector<SDValue, 16> Elts(NumElems, DAG.getUNDEF(OpEltVT)); in LowerBUILD_VECTOR()
8856 SDValue EltsBV = DAG.getBuildVector(VT, dl, Elts); in LowerBUILD_VECTOR()
8857 SDValue FrozenUndefElt = DAG.getFreeze(DAG.getUNDEF(OpEltVT)); in LowerBUILD_VECTOR()
8858 SDValue FrozenUndefBV = DAG.getSplatBuildVector(VT, dl, FrozenUndefElt); in LowerBUILD_VECTOR()
8859 return DAG.getVectorShuffle(VT, dl, EltsBV, FrozenUndefBV, BlendMask); in LowerBUILD_VECTOR()
8880 DAG.getBuildVector(LowerVT, dl, Op->ops().drop_back(UpperElems)); in LowerBUILD_VECTOR()
8881 return widenSubVector(VT, NewBV, !UndefUpper, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
8885 if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, dl, Subtarget, DAG)) in LowerBUILD_VECTOR()
8887 if (SDValue HorizontalOp = LowerToHorizontalOp(BV, dl, Subtarget, DAG)) in LowerBUILD_VECTOR()
8889 if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, dl, Subtarget, DAG)) in LowerBUILD_VECTOR()
8891 if (SDValue BitOp = lowerBuildVectorToBitOp(BV, dl, Subtarget, DAG)) in LowerBUILD_VECTOR()
8910 LLVMContext &Context = *DAG.getContext(); in LowerBUILD_VECTOR()
8925 InsIndex = DAG.getVectorIdxConstant(i, dl); in LowerBUILD_VECTOR()
8929 SDValue DAGConstVec = DAG.getConstantPool(CV, VT); in LowerBUILD_VECTOR()
8937 SDValue LegalDAGConstVec = LowerConstantPool(DAGConstVec, DAG); in LowerBUILD_VECTOR()
8938 MachineFunction &MF = DAG.getMachineFunction(); in LowerBUILD_VECTOR()
8940 SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI); in LowerBUILD_VECTOR()
8944 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ld, VarElt, InsIndex); in LowerBUILD_VECTOR()
8954 SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt); in LowerBUILD_VECTOR()
8955 return DAG.getVectorShuffle(VT, dl, Ld, S2V, ShuffleMask); in LowerBUILD_VECTOR()
8969 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
8977 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
8980 return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); in LowerBUILD_VECTOR()
8986 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
8988 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, Item); in LowerBUILD_VECTOR()
8989 Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); in LowerBUILD_VECTOR()
8990 return DAG.getBitcast(VT, Item); in LowerBUILD_VECTOR()
9000 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in LowerBUILD_VECTOR()
9002 NumBits/2, DAG, *this, dl); in LowerBUILD_VECTOR()
9014 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
9015 return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG); in LowerBUILD_VECTOR()
9029 return LowerAsSplatVectorLoad(Item, VT, dl, DAG); in LowerBUILD_VECTOR()
9039 if (SDValue V = LowerBUILD_VECTORAsVariablePermute(Op, dl, DAG, Subtarget)) in LowerBUILD_VECTOR()
9046 EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false)) in LowerBUILD_VECTOR()
9055 DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) }; in LowerBUILD_VECTOR()
9067 SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2), in LowerBUILD_VECTOR()
9068 DAG.getBuildVector(NarrowVT, dl, Ops)); in LowerBUILD_VECTOR()
9071 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT, in LowerBUILD_VECTOR()
9083 DAG.getBuildVector(HVT, dl, Op->ops().slice(0, NumElems / 2)); in LowerBUILD_VECTOR()
9084 SDValue Upper = DAG.getBuildVector( in LowerBUILD_VECTOR()
9088 return concatSubVectors(Lower, Upper, DAG, dl); in LowerBUILD_VECTOR()
9096 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, in LowerBUILD_VECTOR()
9098 return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG); in LowerBUILD_VECTOR()
9106 NumZero, DAG, Subtarget)) in LowerBUILD_VECTOR()
9111 NumZero, DAG, Subtarget)) in LowerBUILD_VECTOR()
9116 if (SDValue V = LowerBuildVectorv4x32(Op, dl, DAG, Subtarget)) in LowerBUILD_VECTOR()
9125 Ops[i] = getZeroVector(VT, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
9127 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
9137 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2+1], Ops[i*2]); in LowerBUILD_VECTOR()
9140 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2], Ops[i*2+1]); in LowerBUILD_VECTOR()
9143 Ops[i] = getUnpackl(DAG, dl, VT, Ops[i*2], Ops[i*2+1]); in LowerBUILD_VECTOR()
9156 return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], MaskVec); in LowerBUILD_VECTOR()
9162 if (SDValue Sh = buildFromShuffleMostly(Op, dl, DAG)) in LowerBUILD_VECTOR()
9169 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0)); in LowerBUILD_VECTOR()
9171 Result = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
9175 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result, in LowerBUILD_VECTOR()
9176 Op.getOperand(i), DAG.getIntPtrConstant(i, dl)); in LowerBUILD_VECTOR()
9187 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
9189 Ops[i] = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
9206 Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2*i], Ops[(2*i)+1], Mask); in LowerBUILD_VECTOR()
9214 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, in LowerAVXCONCAT_VECTORS() argument
9251 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, in LowerAVXCONCAT_VECTORS()
9253 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, in LowerAVXCONCAT_VECTORS()
9255 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); in LowerAVXCONCAT_VECTORS()
9259 SDValue Vec = NumZero ? getZeroVector(ResVT, Subtarget, DAG, dl) in LowerAVXCONCAT_VECTORS()
9260 : (NumFreezeUndef ? DAG.getFreeze(DAG.getUNDEF(ResVT)) in LowerAVXCONCAT_VECTORS()
9261 : DAG.getUNDEF(ResVT)); in LowerAVXCONCAT_VECTORS()
9269 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, in LowerAVXCONCAT_VECTORS()
9271 DAG.getIntPtrConstant(i * NumSubElems, dl)); in LowerAVXCONCAT_VECTORS()
9283 SelectionDAG & DAG) { in LowerCONCAT_VECTORSvXi1() argument
9315 Op = widenSubVector(ShiftVT, SubVec, false, Subtarget, DAG, dl); in LowerCONCAT_VECTORSvXi1()
9316 Op = DAG.getNode(X86ISD::KSHIFTL, dl, ShiftVT, Op, in LowerCONCAT_VECTORSvXi1()
9317 DAG.getTargetConstant(Idx * SubVecNumElts, dl, MVT::i8)); in LowerCONCAT_VECTORSvXi1()
9318 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, Op, in LowerCONCAT_VECTORSvXi1()
9319 DAG.getIntPtrConstant(0, dl)); in LowerCONCAT_VECTORSvXi1()
9324 SDValue Vec = Zeros ? DAG.getConstant(0, dl, ResVT) : DAG.getUNDEF(ResVT); in LowerCONCAT_VECTORSvXi1()
9330 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, SubVec, in LowerCONCAT_VECTORSvXi1()
9331 DAG.getIntPtrConstant(Idx * SubVecNumElts, dl)); in LowerCONCAT_VECTORSvXi1()
9337 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, in LowerCONCAT_VECTORSvXi1()
9339 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, in LowerCONCAT_VECTORSvXi1()
9341 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); in LowerCONCAT_VECTORSvXi1()
9349 SDValue Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, in LowerCONCAT_VECTORSvXi1()
9350 DAG.getUNDEF(ResVT), Op.getOperand(0), in LowerCONCAT_VECTORSvXi1()
9351 DAG.getIntPtrConstant(0, dl)); in LowerCONCAT_VECTORSvXi1()
9352 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Op.getOperand(1), in LowerCONCAT_VECTORSvXi1()
9353 DAG.getIntPtrConstant(NumElems/2, dl)); in LowerCONCAT_VECTORSvXi1()
9358 SelectionDAG &DAG) { in LowerCONCAT_VECTORS() argument
9361 return LowerCONCAT_VECTORSvXi1(Op, Subtarget, DAG); in LowerCONCAT_VECTORS()
9371 return LowerAVXCONCAT_VECTORS(Op, DAG, Subtarget); in LowerCONCAT_VECTORS()
9656 const SelectionDAG &DAG, in isTargetShuffleEquivalent() argument
9709 return (ZeroV1.isZero() || DAG.MaskedVectorIsZero(V1, ZeroV1)) && in isTargetShuffleEquivalent()
9710 (ZeroV2.isZero() || DAG.MaskedVectorIsZero(V2, ZeroV2)); in isTargetShuffleEquivalent()
9716 const SelectionDAG &DAG) { in isUnpackWdShuffleMask() argument
9726 bool IsUnpackwdMask = (isTargetShuffleEquivalent(VT, Mask, Unpcklwd, DAG) || in isUnpackWdShuffleMask()
9727 isTargetShuffleEquivalent(VT, Mask, Unpckhwd, DAG)); in isUnpackWdShuffleMask()
9732 const SelectionDAG &DAG) { in is128BitUnpackShuffleMask() argument
9745 if (isTargetShuffleEquivalent(VT, Mask, UnpackMask, DAG) || in is128BitUnpackShuffleMask()
9746 isTargetShuffleEquivalent(VT, CommutedMask, UnpackMask, DAG)) in is128BitUnpackShuffleMask()
9799 SelectionDAG &DAG) { in getV4X86ShuffleImm8ForMask() argument
9800 return DAG.getTargetConstant(getV4X86ShuffleImm(Mask), DL, MVT::i8); in getV4X86ShuffleImm8ForMask()
9840 SelectionDAG &DAG) { in lowerShuffleWithPSHUFB() argument
9852 SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8); in lowerShuffleWithPSHUFB()
9858 PSHUFBMask[i] = DAG.getUNDEF(MVT::i8); in lowerShuffleWithPSHUFB()
9879 PSHUFBMask[i] = DAG.getConstant(M, DL, MVT::i8); in lowerShuffleWithPSHUFB()
9884 return DAG.getBitcast( in lowerShuffleWithPSHUFB()
9885 VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V), in lowerShuffleWithPSHUFB()
9886 DAG.getBuildVector(I8VT, DL, PSHUFBMask))); in lowerShuffleWithPSHUFB()
9890 const X86Subtarget &Subtarget, SelectionDAG &DAG,
9897 SDValue &V2, SelectionDAG &DAG, in lowerShuffleToEXPAND() argument
9906 SDValue MaskNode = DAG.getConstant(VEXPANDMask, DL, IntegerType); in lowerShuffleToEXPAND()
9911 Subtarget, DAG, DL); in lowerShuffleToEXPAND()
9912 SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleToEXPAND()
9914 return DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector, ZeroVector, VMask); in lowerShuffleToEXPAND()
9920 SelectionDAG &DAG, in matchShuffleWithUNPCK() argument
9939 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG, V1, in matchShuffleWithUNPCK()
9942 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2)); in matchShuffleWithUNPCK()
9943 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1); in matchShuffleWithUNPCK()
9948 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG, V1, in matchShuffleWithUNPCK()
9951 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2)); in matchShuffleWithUNPCK()
9952 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1); in matchShuffleWithUNPCK()
9978 V2 = Zero2 ? getZeroVector(VT, Subtarget, DAG, DL) : V1; in matchShuffleWithUNPCK()
9979 V1 = Zero1 ? getZeroVector(VT, Subtarget, DAG, DL) : V1; in matchShuffleWithUNPCK()
9987 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG)) { in matchShuffleWithUNPCK()
9994 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG)) { in matchShuffleWithUNPCK()
10008 SelectionDAG &DAG) { in lowerShuffleWithUNPCK() argument
10012 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2); in lowerShuffleWithUNPCK()
10017 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2); in lowerShuffleWithUNPCK()
10022 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1); in lowerShuffleWithUNPCK()
10026 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1); in lowerShuffleWithUNPCK()
10035 SDValue V2, SelectionDAG &DAG) { in lowerShuffleWithUNPCK256() argument
10051 V1 = DAG.getVectorShuffle(MVT::v4f64, DL, DAG.getBitcast(MVT::v4f64, V1), in lowerShuffleWithUNPCK256()
10052 DAG.getUNDEF(MVT::v4f64), {0, 2, 1, 3}); in lowerShuffleWithUNPCK256()
10053 V1 = DAG.getBitcast(VT, V1); in lowerShuffleWithUNPCK256()
10054 return DAG.getNode(UnpackOpcode, DL, VT, V1, V1); in lowerShuffleWithUNPCK256()
10099 SelectionDAG &DAG, bool ZeroUppers) { in getAVX512TruncNode() argument
10106 if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT)) in getAVX512TruncNode()
10111 return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Src); in getAVX512TruncNode()
10115 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src); in getAVX512TruncNode()
10116 return extractSubVector(Trunc, 0, DAG, DL, DstVT.getSizeInBits()); in getAVX512TruncNode()
10121 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src); in getAVX512TruncNode()
10122 return widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL, in getAVX512TruncNode()
10129 SDValue NewSrc = widenSubVector(Src, ZeroUppers, Subtarget, DAG, DL, 512); in getAVX512TruncNode()
10130 return getAVX512TruncNode(DL, DstVT, NewSrc, Subtarget, DAG, ZeroUppers); in getAVX512TruncNode()
10135 SDValue Trunc = DAG.getNode(X86ISD::VTRUNC, DL, TruncVT, Src); in getAVX512TruncNode()
10137 Trunc = widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL, in getAVX512TruncNode()
10162 SelectionDAG &DAG) { in lowerShuffleWithVPMOV() argument
10187 Src = DAG.getBitcast(SrcVT, Src); in lowerShuffleWithVPMOV()
10190 ((DAG.ComputeNumSignBits(Src) > EltSizeInBits) || in lowerShuffleWithVPMOV()
10191 (DAG.computeKnownBits(Src).countMinLeadingZeros() >= EltSizeInBits))) in lowerShuffleWithVPMOV()
10201 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers); in lowerShuffleWithVPMOV()
10212 SelectionDAG &DAG) { in lowerShuffleAsVTRUNC() argument
10254 return DAG.areNonVolatileConsecutiveLoads( in lowerShuffleAsVTRUNC()
10266 SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2); in lowerShuffleAsVTRUNC()
10270 Src = DAG.getBitcast(SrcVT, Src); in lowerShuffleAsVTRUNC()
10275 Src = DAG.getNode( in lowerShuffleAsVTRUNC()
10277 DAG.getTargetConstant(Offset * EltSizeInBits, DL, MVT::i8)); in lowerShuffleAsVTRUNC()
10279 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers); in lowerShuffleAsVTRUNC()
10364 const SelectionDAG &DAG, in matchShuffleWithPACK() argument
10386 if ((N1.isUndef() || IsZero1 || DAG.MaskedValueIsZero(N1, ZeroMask)) && in matchShuffleWithPACK()
10387 (N2.isUndef() || IsZero2 || DAG.MaskedValueIsZero(N2, ZeroMask))) { in matchShuffleWithPACK()
10398 DAG.ComputeNumSignBits(N1) > NumPackedBits) && in matchShuffleWithPACK()
10400 DAG.ComputeNumSignBits(N2) > NumPackedBits)) { in matchShuffleWithPACK()
10418 if (isTargetShuffleEquivalent(VT, TargetMask, BinaryMask, DAG, V1, V2)) in matchShuffleWithPACK()
10425 if (isTargetShuffleEquivalent(VT, TargetMask, UnaryMask, DAG, V1)) in matchShuffleWithPACK()
10434 SDValue V1, SDValue V2, SelectionDAG &DAG, in lowerShuffleWithPACK() argument
10441 if (!matchShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG, in lowerShuffleWithPACK()
10468 Res = DAG.getNode(PackOpcode, DL, DstVT, DAG.getBitcast(SrcVT, V1), in lowerShuffleWithPACK()
10469 DAG.getBitcast(SrcVT, V2)); in lowerShuffleWithPACK()
10486 SelectionDAG &DAG) { in lowerShuffleAsBitMask() argument
10498 Zero = DAG.getConstantFP(0.0, DL, EltVT); in lowerShuffleAsBitMask()
10501 AllOnes = DAG.getConstantFP(AllOnesValue, DL, EltVT); in lowerShuffleAsBitMask()
10505 Zero = DAG.getConstant(0, DL, EltVT); in lowerShuffleAsBitMask()
10506 AllOnes = DAG.getAllOnesConstant(DL, EltVT); in lowerShuffleAsBitMask()
10526 SDValue VMask = DAG.getBuildVector(MaskVT, DL, VMaskOps); in lowerShuffleAsBitMask()
10527 VMask = DAG.getBitcast(LogicVT, VMask); in lowerShuffleAsBitMask()
10528 V = DAG.getBitcast(LogicVT, V); in lowerShuffleAsBitMask()
10529 SDValue And = DAG.getNode(ISD::AND, DL, LogicVT, V, VMask); in lowerShuffleAsBitMask()
10530 return DAG.getBitcast(VT, And); in lowerShuffleAsBitMask()
10540 SelectionDAG &DAG) { in lowerShuffleAsBitBlend() argument
10543 SDValue Zero = DAG.getConstant(0, DL, EltVT); in lowerShuffleAsBitBlend()
10544 SDValue AllOnes = DAG.getAllOnesConstant(DL, EltVT); in lowerShuffleAsBitBlend()
10552 SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps); in lowerShuffleAsBitBlend()
10553 return getBitSelect(DL, VT, V1, V2, V1Mask, DAG); in lowerShuffleAsBitBlend()
10559 SelectionDAG &DAG);
10649 SelectionDAG &DAG) { in lowerShuffleAsBlend() argument
10659 V1 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleAsBlend()
10661 V2 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleAsBlend()
10680 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2, in lowerShuffleAsBlend()
10681 DAG.getTargetConstant(BlendMask, DL, MVT::i8)); in lowerShuffleAsBlend()
10692 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, in lowerShuffleAsBlend()
10693 DAG.getTargetConstant(BlendMask, DL, MVT::i8)); in lowerShuffleAsBlend()
10701 SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, in lowerShuffleAsBlend()
10702 DAG.getTargetConstant(LoMask, DL, MVT::i8)); in lowerShuffleAsBlend()
10703 SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, in lowerShuffleAsBlend()
10704 DAG.getTargetConstant(HiMask, DL, MVT::i8)); in lowerShuffleAsBlend()
10705 return DAG.getVectorShuffle( in lowerShuffleAsBlend()
10719 Subtarget, DAG)) in lowerShuffleAsBlend()
10724 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType); in lowerShuffleAsBlend()
10725 return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG); in lowerShuffleAsBlend()
10731 lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG)) in lowerShuffleAsBlend()
10765 Mask[i] < 0 ? DAG.getUNDEF(MVT::i8) in lowerShuffleAsBlend()
10766 : DAG.getConstant(Mask[i] < Size ? -1 : 0, DL, in lowerShuffleAsBlend()
10769 V1 = DAG.getBitcast(BlendVT, V1); in lowerShuffleAsBlend()
10770 V2 = DAG.getBitcast(BlendVT, V2); in lowerShuffleAsBlend()
10771 return DAG.getBitcast( in lowerShuffleAsBlend()
10773 DAG.getSelect(DL, BlendVT, DAG.getBuildVector(BlendVT, DL, VSELECTMask), in lowerShuffleAsBlend()
10783 bool OptForSize = DAG.shouldOptForSize(); in lowerShuffleAsBlend()
10786 Subtarget, DAG)) in lowerShuffleAsBlend()
10793 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType); in lowerShuffleAsBlend()
10794 return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG); in lowerShuffleAsBlend()
10809 SelectionDAG &DAG, in lowerShuffleAsBlendAndPermute() argument
10836 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask); in lowerShuffleAsBlendAndPermute()
10837 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask); in lowerShuffleAsBlendAndPermute()
10848 SelectionDAG &DAG) { in lowerShuffleAsUNPCKAndPermute() argument
10855 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)}; in lowerShuffleAsUNPCKAndPermute()
10915 SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops); in lowerShuffleAsUNPCKAndPermute()
10916 return DAG.getVectorShuffle(VT, DL, Unpck, DAG.getUNDEF(VT), PermuteMask); in lowerShuffleAsUNPCKAndPermute()
10932 SelectionDAG &DAG) { in lowerShuffleAsPermuteAndUnpack() argument
10977 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask); in lowerShuffleAsPermuteAndUnpack()
10978 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask); in lowerShuffleAsPermuteAndUnpack()
10983 V1 = DAG.getBitcast(UnpackVT, V1); in lowerShuffleAsPermuteAndUnpack()
10984 V2 = DAG.getBitcast(UnpackVT, V2); in lowerShuffleAsPermuteAndUnpack()
10987 return DAG.getBitcast( in lowerShuffleAsPermuteAndUnpack()
10988 VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL, in lowerShuffleAsPermuteAndUnpack()
11027 return DAG.getVectorShuffle( in lowerShuffleAsPermuteAndUnpack()
11029 DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL, DL, VT, in lowerShuffleAsPermuteAndUnpack()
11031 DAG.getUNDEF(VT), PermMask); in lowerShuffleAsPermuteAndUnpack()
11041 const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerShuffleAsByteRotateAndPermute() argument
11096 SDValue Rotate = DAG.getBitcast( in lowerShuffleAsByteRotateAndPermute()
11097 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi), in lowerShuffleAsByteRotateAndPermute()
11098 DAG.getBitcast(ByteVT, Lo), in lowerShuffleAsByteRotateAndPermute()
11099 DAG.getTargetConstant(Scale * RotAmt, DL, MVT::i8))); in lowerShuffleAsByteRotateAndPermute()
11112 return DAG.getVectorShuffle(VT, DL, Rotate, DAG.getUNDEF(VT), PermMask); in lowerShuffleAsByteRotateAndPermute()
11158 const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerShuffleAsDecomposedShuffleMerge() argument
11186 &DAG](SDValue &Input, in lowerShuffleAsDecomposedShuffleMerge()
11196 Input = DAG.getNode(X86ISD::VBROADCAST, DL, VT, Input); in lowerShuffleAsDecomposedShuffleMerge()
11222 DAG, true)) in lowerShuffleAsDecomposedShuffleMerge()
11233 lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask, DAG)) in lowerShuffleAsDecomposedShuffleMerge()
11236 DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerShuffleAsDecomposedShuffleMerge()
11240 DAG)) in lowerShuffleAsDecomposedShuffleMerge()
11244 DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerShuffleAsDecomposedShuffleMerge()
11269 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask); in lowerShuffleAsDecomposedShuffleMerge()
11270 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask); in lowerShuffleAsDecomposedShuffleMerge()
11271 return DAG.getVectorShuffle(VT, DL, V1, V2, FinalMask); in lowerShuffleAsDecomposedShuffleMerge()
11297 SelectionDAG &DAG) { in lowerShuffleAsBitRotate() argument
11320 V1 = DAG.getBitcast(RotateVT, V1); in lowerShuffleAsBitRotate()
11321 SDValue SHL = DAG.getNode(X86ISD::VSHLI, DL, RotateVT, V1, in lowerShuffleAsBitRotate()
11322 DAG.getTargetConstant(ShlAmt, DL, MVT::i8)); in lowerShuffleAsBitRotate()
11323 SDValue SRL = DAG.getNode(X86ISD::VSRLI, DL, RotateVT, V1, in lowerShuffleAsBitRotate()
11324 DAG.getTargetConstant(SrlAmt, DL, MVT::i8)); in lowerShuffleAsBitRotate()
11325 SDValue Rot = DAG.getNode(ISD::OR, DL, RotateVT, SHL, SRL); in lowerShuffleAsBitRotate()
11326 return DAG.getBitcast(VT, Rot); in lowerShuffleAsBitRotate()
11330 DAG.getNode(X86ISD::VROTLI, DL, RotateVT, DAG.getBitcast(RotateVT, V1), in lowerShuffleAsBitRotate()
11331 DAG.getTargetConstant(RotateAmt, DL, MVT::i8)); in lowerShuffleAsBitRotate()
11332 return DAG.getBitcast(VT, Rot); in lowerShuffleAsBitRotate()
11448 SelectionDAG &DAG) { in lowerShuffleAsByteRotate() argument
11459 Lo = DAG.getBitcast(ByteVT, Lo); in lowerShuffleAsByteRotate()
11460 Hi = DAG.getBitcast(ByteVT, Hi); in lowerShuffleAsByteRotate()
11466 return DAG.getBitcast( in lowerShuffleAsByteRotate()
11467 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi, in lowerShuffleAsByteRotate()
11468 DAG.getTargetConstant(ByteRotation, DL, MVT::i8))); in lowerShuffleAsByteRotate()
11483 DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo, in lowerShuffleAsByteRotate()
11484 DAG.getTargetConstant(LoByteShift, DL, MVT::i8)); in lowerShuffleAsByteRotate()
11486 DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi, in lowerShuffleAsByteRotate()
11487 DAG.getTargetConstant(HiByteShift, DL, MVT::i8)); in lowerShuffleAsByteRotate()
11488 return DAG.getBitcast(VT, in lowerShuffleAsByteRotate()
11489 DAG.getNode(ISD::OR, DL, MVT::v16i8, LoShift, HiShift)); in lowerShuffleAsByteRotate()
11506 SelectionDAG &DAG) { in lowerShuffleAsVALIGN() argument
11517 return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi, in lowerShuffleAsVALIGN()
11518 DAG.getTargetConstant(Rotation, DL, MVT::i8)); in lowerShuffleAsVALIGN()
11535 return DAG.getNode(X86ISD::VALIGN, DL, VT, Src, in lowerShuffleAsVALIGN()
11536 getZeroVector(VT, Subtarget, DAG, DL), in lowerShuffleAsVALIGN()
11537 DAG.getTargetConstant(NumElts - ZeroLo, DL, MVT::i8)); in lowerShuffleAsVALIGN()
11544 return DAG.getNode(X86ISD::VALIGN, DL, VT, in lowerShuffleAsVALIGN()
11545 getZeroVector(VT, Subtarget, DAG, DL), Src, in lowerShuffleAsVALIGN()
11546 DAG.getTargetConstant(ZeroHi, DL, MVT::i8)); in lowerShuffleAsVALIGN()
11557 SelectionDAG &DAG) { in lowerShuffleAsByteShiftMask() argument
11580 Res = DAG.getBitcast(MVT::v16i8, Res); in lowerShuffleAsByteShiftMask()
11589 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
11590 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8)); in lowerShuffleAsByteShiftMask()
11591 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
11592 DAG.getTargetConstant(Scale * ZeroHi, DL, MVT::i8)); in lowerShuffleAsByteShiftMask()
11595 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
11596 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8)); in lowerShuffleAsByteShiftMask()
11597 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
11598 DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8)); in lowerShuffleAsByteShiftMask()
11604 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
11605 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8)); in lowerShuffleAsByteShiftMask()
11607 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
11608 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8)); in lowerShuffleAsByteShiftMask()
11609 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res, in lowerShuffleAsByteShiftMask()
11610 DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8)); in lowerShuffleAsByteShiftMask()
11614 return DAG.getBitcast(VT, Res); in lowerShuffleAsByteShiftMask()
11706 SelectionDAG &DAG, bool BitwiseOnly) { in lowerShuffleAsShift() argument
11731 assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) && in lowerShuffleAsShift()
11733 V = DAG.getBitcast(ShiftVT, V); in lowerShuffleAsShift()
11734 V = DAG.getNode(Opcode, DL, ShiftVT, V, in lowerShuffleAsShift()
11735 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8)); in lowerShuffleAsShift()
11736 return DAG.getBitcast(VT, V); in lowerShuffleAsShift()
11865 const APInt &Zeroable, SelectionDAG &DAG) { in lowerShuffleWithSSE4A() argument
11868 return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1, in lowerShuffleWithSSE4A()
11869 DAG.getTargetConstant(BitLen, DL, MVT::i8), in lowerShuffleWithSSE4A()
11870 DAG.getTargetConstant(BitIdx, DL, MVT::i8)); in lowerShuffleWithSSE4A()
11873 return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT), in lowerShuffleWithSSE4A()
11874 V2 ? V2 : DAG.getUNDEF(VT), in lowerShuffleWithSSE4A()
11875 DAG.getTargetConstant(BitLen, DL, MVT::i8), in lowerShuffleWithSSE4A()
11876 DAG.getTargetConstant(BitIdx, DL, MVT::i8)); in lowerShuffleWithSSE4A()
11892 ArrayRef<int> Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerShuffleAsSpecificZeroOrAnyExtend() argument
11920 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), ShMask); in lowerShuffleAsSpecificZeroOrAnyExtend()
11932 InputV = DAG.getBitcast(VT, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
11935 DL, ExtVT, InputV, DAG); in lowerShuffleAsSpecificZeroOrAnyExtend()
11936 return DAG.getBitcast(VT, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
11940 InputV = DAG.getBitcast(VT, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
11947 return DAG.getBitcast( in lowerShuffleAsSpecificZeroOrAnyExtend()
11948 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerShuffleAsSpecificZeroOrAnyExtend()
11949 DAG.getBitcast(MVT::v4i32, InputV), in lowerShuffleAsSpecificZeroOrAnyExtend()
11950 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG))); in lowerShuffleAsSpecificZeroOrAnyExtend()
11955 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerShuffleAsSpecificZeroOrAnyExtend()
11956 DAG.getBitcast(MVT::v4i32, InputV), in lowerShuffleAsSpecificZeroOrAnyExtend()
11957 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)); in lowerShuffleAsSpecificZeroOrAnyExtend()
11960 return DAG.getBitcast( in lowerShuffleAsSpecificZeroOrAnyExtend()
11961 VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16, in lowerShuffleAsSpecificZeroOrAnyExtend()
11962 DAG.getBitcast(MVT::v8i16, InputV), in lowerShuffleAsSpecificZeroOrAnyExtend()
11963 getV4X86ShuffleImm8ForMask(PSHUFWMask, DL, DAG))); in lowerShuffleAsSpecificZeroOrAnyExtend()
11973 SDValue Lo = DAG.getBitcast( in lowerShuffleAsSpecificZeroOrAnyExtend()
11974 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
11975 DAG.getTargetConstant(EltBits, DL, MVT::i8), in lowerShuffleAsSpecificZeroOrAnyExtend()
11976 DAG.getTargetConstant(LoIdx, DL, MVT::i8))); in lowerShuffleAsSpecificZeroOrAnyExtend()
11979 return DAG.getBitcast(VT, Lo); in lowerShuffleAsSpecificZeroOrAnyExtend()
11982 SDValue Hi = DAG.getBitcast( in lowerShuffleAsSpecificZeroOrAnyExtend()
11983 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
11984 DAG.getTargetConstant(EltBits, DL, MVT::i8), in lowerShuffleAsSpecificZeroOrAnyExtend()
11985 DAG.getTargetConstant(HiIdx, DL, MVT::i8))); in lowerShuffleAsSpecificZeroOrAnyExtend()
11986 return DAG.getBitcast(VT, in lowerShuffleAsSpecificZeroOrAnyExtend()
11987 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi)); in lowerShuffleAsSpecificZeroOrAnyExtend()
11999 PSHUFBMask[i] = DAG.getConstant(Idx, DL, MVT::i8); in lowerShuffleAsSpecificZeroOrAnyExtend()
12003 AnyExt ? DAG.getUNDEF(MVT::i8) : DAG.getConstant(0x80, DL, MVT::i8); in lowerShuffleAsSpecificZeroOrAnyExtend()
12005 InputV = DAG.getBitcast(MVT::v16i8, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
12006 return DAG.getBitcast( in lowerShuffleAsSpecificZeroOrAnyExtend()
12007 VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
12008 DAG.getBuildVector(MVT::v16i8, DL, PSHUFBMask))); in lowerShuffleAsSpecificZeroOrAnyExtend()
12018 InputV = DAG.getVectorShuffle(VT, DL, InputV, DAG.getUNDEF(VT), ShMask); in lowerShuffleAsSpecificZeroOrAnyExtend()
12031 SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT) in lowerShuffleAsSpecificZeroOrAnyExtend()
12032 : getZeroVector(InputVT, Subtarget, DAG, DL); in lowerShuffleAsSpecificZeroOrAnyExtend()
12033 InputV = DAG.getBitcast(InputVT, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
12034 InputV = DAG.getNode(UnpackLoHi, DL, InputVT, InputV, Ext); in lowerShuffleAsSpecificZeroOrAnyExtend()
12039 return DAG.getBitcast(VT, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
12057 SelectionDAG &DAG) { in lowerShuffleAsZeroOrAnyExtend() argument
12126 InputV, Mask, Subtarget, DAG); in lowerShuffleAsZeroOrAnyExtend()
12161 V = DAG.getBitcast(MVT::v2i64, V); in lowerShuffleAsZeroOrAnyExtend()
12162 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V); in lowerShuffleAsZeroOrAnyExtend()
12163 return DAG.getBitcast(VT, V); in lowerShuffleAsZeroOrAnyExtend()
12174 SelectionDAG &DAG) { in getScalarValueForVectorElement() argument
12191 return DAG.getBitcast(EltVT, S); in getScalarValueForVectorElement()
12219 SelectionDAG &DAG) { in lowerShuffleAsElementInsertion() argument
12253 DAG); in lowerShuffleAsElementInsertion()
12254 if (V2S && DAG.getTargetLoweringInfo().isTypeLegal(V2S.getValueType())) { in lowerShuffleAsElementInsertion()
12256 V2S = DAG.getBitcast(EltVT, V2S); in lowerShuffleAsElementInsertion()
12266 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S); in lowerShuffleAsElementInsertion()
12273 SDValue BitMask = getConstVector(Bits, VT, DAG, DL); in lowerShuffleAsElementInsertion()
12274 V1 = DAG.getNode(ISD::AND, DL, VT, V1, BitMask); in lowerShuffleAsElementInsertion()
12275 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S); in lowerShuffleAsElementInsertion()
12276 V2 = DAG.getBitcast(VT, DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2)); in lowerShuffleAsElementInsertion()
12277 return DAG.getNode(ISD::OR, DL, VT, V1, V2); in lowerShuffleAsElementInsertion()
12280 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S); in lowerShuffleAsElementInsertion()
12307 return DAG.getNode(MovOpc, DL, ExtVT, V1, V2); in lowerShuffleAsElementInsertion()
12314 V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2); in lowerShuffleAsElementInsertion()
12316 V2 = DAG.getBitcast(VT, V2); in lowerShuffleAsElementInsertion()
12326 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle); in lowerShuffleAsElementInsertion()
12328 V2 = DAG.getBitcast(MVT::v16i8, V2); in lowerShuffleAsElementInsertion()
12329 V2 = DAG.getNode( in lowerShuffleAsElementInsertion()
12331 DAG.getTargetConstant(V2Index * EltBits / 8, DL, MVT::i8)); in lowerShuffleAsElementInsertion()
12332 V2 = DAG.getBitcast(VT, V2); in lowerShuffleAsElementInsertion()
12345 SelectionDAG &DAG) { in lowerShuffleAsTruncBroadcast() argument
12384 Scalar = DAG.getNode(ISD::SRL, DL, Scalar.getValueType(), Scalar, in lowerShuffleAsTruncBroadcast()
12385 DAG.getConstant(OffsetIdx * EltSize, DL, MVT::i8)); in lowerShuffleAsTruncBroadcast()
12387 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in lowerShuffleAsTruncBroadcast()
12388 DAG.getNode(ISD::TRUNCATE, DL, EltVT, Scalar)); in lowerShuffleAsTruncBroadcast()
12433 SelectionDAG &DAG) { in lowerShuffleOfExtractsAsVperm() argument
12466 (isSingleSHUFPSMask(NewMask) || is128BitUnpackShuffleMask(NewMask, DAG))) in lowerShuffleOfExtractsAsVperm()
12473 SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT), in lowerShuffleOfExtractsAsVperm()
12476 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf, in lowerShuffleOfExtractsAsVperm()
12477 DAG.getIntPtrConstant(0, DL)); in lowerShuffleOfExtractsAsVperm()
12488 SelectionDAG &DAG) { in lowerShuffleAsBroadcast() argument
12570 DL, VT, V, BroadcastIdx, Subtarget, DAG)) in lowerShuffleAsBroadcast()
12595 DAG.getMemBasePlusOffset(BaseAddr, TypeSize::getFixed(Offset), DL); in lowerShuffleAsBroadcast()
12601 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerShuffleAsBroadcast()
12603 V = DAG.getMemIntrinsicNode( in lowerShuffleAsBroadcast()
12605 DAG.getMachineFunction().getMachineMemOperand( in lowerShuffleAsBroadcast()
12607 DAG.makeEquivalentMemoryOrdering(Ld, V); in lowerShuffleAsBroadcast()
12608 return DAG.getBitcast(VT, V); in lowerShuffleAsBroadcast()
12611 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr, in lowerShuffleAsBroadcast()
12612 DAG.getMachineFunction().getMachineMemOperand( in lowerShuffleAsBroadcast()
12614 DAG.makeEquivalentMemoryOrdering(Ld, V); in lowerShuffleAsBroadcast()
12638 V = extract128BitVector(V, ExtractIdx, DAG, DL); in lowerShuffleAsBroadcast()
12643 V = DAG.getBitcast(MVT::f64, V); in lowerShuffleAsBroadcast()
12645 V = DAG.getNode(X86ISD::VBROADCAST, DL, MVT::v2f64, V); in lowerShuffleAsBroadcast()
12646 return DAG.getBitcast(VT, V); in lowerShuffleAsBroadcast()
12648 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V); in lowerShuffleAsBroadcast()
12657 return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V)); in lowerShuffleAsBroadcast()
12664 V = extract128BitVector(peekThroughBitcasts(V), 0, DAG, DL); in lowerShuffleAsBroadcast()
12670 return DAG.getNode(Opcode, DL, VT, DAG.getBitcast(CastVT, V)); in lowerShuffleAsBroadcast()
12682 ArrayRef<int> Mask, SelectionDAG &DAG) { in matchShuffleAsInsertPS() argument
12743 VA = DAG.getUNDEF(MVT::v4f32); in matchShuffleAsInsertPS()
12769 SelectionDAG &DAG) { in lowerShuffleAsInsertPS() argument
12775 if (!matchShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG)) in lowerShuffleAsInsertPS()
12779 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, in lowerShuffleAsInsertPS()
12780 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8)); in lowerShuffleAsInsertPS()
12793 SelectionDAG &DAG) { in lowerV2F64Shuffle() argument
12801 Mask, Subtarget, DAG)) in lowerV2F64Shuffle()
12811 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1, in lowerV2F64Shuffle()
12812 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8)); in lowerV2F64Shuffle()
12815 return DAG.getNode( in lowerV2F64Shuffle()
12817 Mask[0] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1, in lowerV2F64Shuffle()
12818 Mask[1] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1, in lowerV2F64Shuffle()
12819 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8)); in lowerV2F64Shuffle()
12827 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG)) in lowerV2F64Shuffle()
12833 DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV2F64Shuffle()
12840 DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG)) in lowerV2F64Shuffle()
12847 if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG)) in lowerV2F64Shuffle()
12850 return DAG.getNode( in lowerV2F64Shuffle()
12852 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S)); in lowerV2F64Shuffle()
12856 Zeroable, Subtarget, DAG)) in lowerV2F64Shuffle()
12860 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG)) in lowerV2F64Shuffle()
12864 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2, in lowerV2F64Shuffle()
12865 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8)); in lowerV2F64Shuffle()
12877 SelectionDAG &DAG) { in lowerV2I64Shuffle() argument
12885 Mask, Subtarget, DAG)) in lowerV2I64Shuffle()
12891 V1 = DAG.getBitcast(MVT::v4i32, V1); in lowerV2I64Shuffle()
12896 return DAG.getBitcast( in lowerV2I64Shuffle()
12898 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, in lowerV2I64Shuffle()
12899 getV4X86ShuffleImm8ForMask(WidenedMask, DL, DAG))); in lowerV2I64Shuffle()
12907 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG)) in lowerV2I64Shuffle()
12913 DAG, /*BitwiseOnly*/ false)) in lowerV2I64Shuffle()
12919 DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV2I64Shuffle()
12925 DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG)) in lowerV2I64Shuffle()
12933 Zeroable, Subtarget, DAG)) in lowerV2I64Shuffle()
12937 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG)) in lowerV2I64Shuffle()
12945 Zeroable, Subtarget, DAG)) in lowerV2I64Shuffle()
12949 Subtarget, DAG)) in lowerV2I64Shuffle()
12957 Subtarget, DAG); in lowerV2I64Shuffle()
12963 V1 = DAG.getBitcast(MVT::v2f64, V1); in lowerV2I64Shuffle()
12964 V2 = DAG.getBitcast(MVT::v2f64, V2); in lowerV2I64Shuffle()
12965 return DAG.getBitcast(MVT::v2i64, in lowerV2I64Shuffle()
12966 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask)); in lowerV2I64Shuffle()
12976 SDValue V2, SelectionDAG &DAG) { in lowerShuffleWithSHUFPS() argument
13000 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1, in lowerShuffleWithSHUFPS()
13001 getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG)); in lowerShuffleWithSHUFPS()
13039 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2, in lowerShuffleWithSHUFPS()
13040 getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG)); in lowerShuffleWithSHUFPS()
13055 return lowerShuffleWithSHUFPS(DL, VT, NewMask, V2, V1, DAG); in lowerShuffleWithSHUFPS()
13057 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV, in lowerShuffleWithSHUFPS()
13058 getV4X86ShuffleImm8ForMask(NewMask, DL, DAG)); in lowerShuffleWithSHUFPS()
13069 SelectionDAG &DAG) { in lowerV4F32Shuffle() argument
13076 Zeroable, Subtarget, DAG)) in lowerV4F32Shuffle()
13084 Mask, Subtarget, DAG)) in lowerV4F32Shuffle()
13090 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1); in lowerV4F32Shuffle()
13092 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1); in lowerV4F32Shuffle()
13098 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1, in lowerV4F32Shuffle()
13099 getV4X86ShuffleImm8ForMask(Mask, DL, DAG)); in lowerV4F32Shuffle()
13106 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1); in lowerV4F32Shuffle()
13108 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1); in lowerV4F32Shuffle()
13113 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1, in lowerV4F32Shuffle()
13114 getV4X86ShuffleImm8ForMask(Mask, DL, DAG)); in lowerV4F32Shuffle()
13119 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) { in lowerV4F32Shuffle()
13120 ZExt = DAG.getBitcast(MVT::v4f32, ZExt); in lowerV4F32Shuffle()
13125 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG)) in lowerV4F32Shuffle()
13135 DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV4F32Shuffle()
13140 if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG)) in lowerV4F32Shuffle()
13145 V2, Mask, DAG)) in lowerV4F32Shuffle()
13153 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2); in lowerV4F32Shuffle()
13155 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1); in lowerV4F32Shuffle()
13159 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG)) in lowerV4F32Shuffle()
13163 return lowerShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG); in lowerV4F32Shuffle()
13173 SelectionDAG &DAG) { in lowerV4I32Shuffle() argument
13182 Zeroable, Subtarget, DAG)) in lowerV4I32Shuffle()
13191 Subtarget, DAG, /*BitwiseOnly*/ true)) in lowerV4I32Shuffle()
13195 lowerShuffleAsBitRotate(DL, MVT::v4i32, V1, Mask, Subtarget, DAG)) in lowerV4I32Shuffle()
13203 Mask, Subtarget, DAG)) in lowerV4I32Shuffle()
13219 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, in lowerV4I32Shuffle()
13220 getV4X86ShuffleImm8ForMask(Mask, DL, DAG)); in lowerV4I32Shuffle()
13224 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG)) in lowerV4I32Shuffle()
13230 DAG, /*BitwiseOnly*/ false)) in lowerV4I32Shuffle()
13236 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV4I32Shuffle()
13244 Zeroable, Subtarget, DAG)) in lowerV4I32Shuffle()
13248 Zeroable, Subtarget, DAG)) in lowerV4I32Shuffle()
13252 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG)) in lowerV4I32Shuffle()
13260 Zeroable, Subtarget, DAG)) in lowerV4I32Shuffle()
13264 Subtarget, DAG)) in lowerV4I32Shuffle()
13276 Subtarget, DAG); in lowerV4I32Shuffle()
13280 Mask, Subtarget, DAG)) in lowerV4I32Shuffle()
13289 SDValue CastV1 = DAG.getBitcast(MVT::v4f32, V1); in lowerV4I32Shuffle()
13290 SDValue CastV2 = DAG.getBitcast(MVT::v4f32, V2); in lowerV4I32Shuffle()
13291 SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask); in lowerV4I32Shuffle()
13292 return DAG.getBitcast(MVT::v4i32, ShufPS); in lowerV4I32Shuffle()
13313 const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerV8I16GeneralSingleInputShuffle() argument
13324 return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13325 getV4X86ShuffleImm8ForMask(LoMask, DL, DAG)); in lowerV8I16GeneralSingleInputShuffle()
13331 return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13332 getV4X86ShuffleImm8ForMask(HiMask, DL, DAG)); in lowerV8I16GeneralSingleInputShuffle()
13357 V = DAG.getNode(ShufWOp, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13358 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG)); in lowerV8I16GeneralSingleInputShuffle()
13359 V = DAG.getBitcast(PSHUFDVT, V); in lowerV8I16GeneralSingleInputShuffle()
13360 V = DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, V, in lowerV8I16GeneralSingleInputShuffle()
13361 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)); in lowerV8I16GeneralSingleInputShuffle()
13362 return DAG.getBitcast(VT, V); in lowerV8I16GeneralSingleInputShuffle()
13491 auto FixFlippedInputs = [&V, &DL, &Mask, &DAG](int PinnedIdx, int DWord, in lowerV8I16GeneralSingleInputShuffle()
13507 V = DAG.getNode( in lowerV8I16GeneralSingleInputShuffle()
13510 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG)); in lowerV8I16GeneralSingleInputShuffle()
13533 V = DAG.getBitcast( in lowerV8I16GeneralSingleInputShuffle()
13535 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V), in lowerV8I16GeneralSingleInputShuffle()
13536 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG))); in lowerV8I16GeneralSingleInputShuffle()
13547 return lowerV8I16GeneralSingleInputShuffle(DL, VT, V, Mask, Subtarget, DAG); in lowerV8I16GeneralSingleInputShuffle()
13769 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13770 getV4X86ShuffleImm8ForMask(PSHUFLMask, DL, DAG)); in lowerV8I16GeneralSingleInputShuffle()
13772 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13773 getV4X86ShuffleImm8ForMask(PSHUFHMask, DL, DAG)); in lowerV8I16GeneralSingleInputShuffle()
13775 V = DAG.getBitcast( in lowerV8I16GeneralSingleInputShuffle()
13777 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V), in lowerV8I16GeneralSingleInputShuffle()
13778 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG))); in lowerV8I16GeneralSingleInputShuffle()
13789 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13790 getV4X86ShuffleImm8ForMask(LoMask, DL, DAG)); in lowerV8I16GeneralSingleInputShuffle()
13797 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13798 getV4X86ShuffleImm8ForMask(HiMask, DL, DAG)); in lowerV8I16GeneralSingleInputShuffle()
13807 const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) { in lowerShuffleAsBlendOfPSHUFBs() argument
13815 SmallVector<SDValue, 64> V1Mask(NumBytes, DAG.getUNDEF(MVT::i8)); in lowerShuffleAsBlendOfPSHUFBs()
13816 SmallVector<SDValue, 64> V2Mask(NumBytes, DAG.getUNDEF(MVT::i8)); in lowerShuffleAsBlendOfPSHUFBs()
13831 V1Mask[i] = DAG.getConstant(V1Idx, DL, MVT::i8); in lowerShuffleAsBlendOfPSHUFBs()
13832 V2Mask[i] = DAG.getConstant(V2Idx, DL, MVT::i8); in lowerShuffleAsBlendOfPSHUFBs()
13839 V1 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V1), in lowerShuffleAsBlendOfPSHUFBs()
13840 DAG.getBuildVector(ShufVT, DL, V1Mask)); in lowerShuffleAsBlendOfPSHUFBs()
13842 V2 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V2), in lowerShuffleAsBlendOfPSHUFBs()
13843 DAG.getBuildVector(ShufVT, DL, V2Mask)); in lowerShuffleAsBlendOfPSHUFBs()
13848 V = DAG.getNode(ISD::OR, DL, ShufVT, V1, V2); in lowerShuffleAsBlendOfPSHUFBs()
13853 return DAG.getBitcast(VT, V); in lowerShuffleAsBlendOfPSHUFBs()
13871 SelectionDAG &DAG) { in lowerV8I16Shuffle() argument
13879 Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
13884 Subtarget, DAG)) in lowerV8I16Shuffle()
13893 Subtarget, DAG, /*BitwiseOnly*/ false)) in lowerV8I16Shuffle()
13898 Mask, Subtarget, DAG)) in lowerV8I16Shuffle()
13903 Subtarget, DAG)) in lowerV8I16Shuffle()
13907 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG)) in lowerV8I16Shuffle()
13911 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG, in lowerV8I16Shuffle()
13917 Subtarget, DAG)) in lowerV8I16Shuffle()
13923 Subtarget, DAG); in lowerV8I16Shuffle()
13933 DAG, /*BitwiseOnly*/ false)) in lowerV8I16Shuffle()
13939 Zeroable, DAG)) in lowerV8I16Shuffle()
13945 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
13953 Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
13957 Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
13961 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG)) in lowerV8I16Shuffle()
13965 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG, in lowerV8I16Shuffle()
13971 Subtarget, DAG)) in lowerV8I16Shuffle()
13976 Subtarget, DAG)) in lowerV8I16Shuffle()
13980 lowerShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG)) in lowerV8I16Shuffle()
13985 Zeroable, Subtarget, DAG)) in lowerV8I16Shuffle()
13997 SDValue V1V2 = concatSubVectors(V1, V2, DAG, DL); in lowerV8I16Shuffle()
13998 V1V2 = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1V2, in lowerV8I16Shuffle()
13999 getZeroVector(MVT::v16i16, Subtarget, DAG, DL), in lowerV8I16Shuffle()
14000 DAG.getTargetConstant(0xEE, DL, MVT::i8)); in lowerV8I16Shuffle()
14001 V1V2 = DAG.getBitcast(MVT::v8i32, V1V2); in lowerV8I16Shuffle()
14002 V1 = extract128BitVector(V1V2, 0, DAG, DL); in lowerV8I16Shuffle()
14003 V2 = extract128BitVector(V1V2, 4, DAG, DL); in lowerV8I16Shuffle()
14007 DAG.getConstant(0, DL, MVT::i32)); in lowerV8I16Shuffle()
14009 DWordClearOps[i] = DAG.getConstant(0xFFFF, DL, MVT::i32); in lowerV8I16Shuffle()
14011 DAG.getBuildVector(MVT::v4i32, DL, DWordClearOps); in lowerV8I16Shuffle()
14012 V1 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V1), in lowerV8I16Shuffle()
14014 V2 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V2), in lowerV8I16Shuffle()
14018 SDValue ShAmt = DAG.getTargetConstant(16, DL, MVT::i8); in lowerV8I16Shuffle()
14019 V1 = DAG.getBitcast(MVT::v4i32, V1); in lowerV8I16Shuffle()
14020 V2 = DAG.getBitcast(MVT::v4i32, V2); in lowerV8I16Shuffle()
14021 V1 = DAG.getNode(X86ISD::VSHLI, DL, MVT::v4i32, V1, ShAmt); in lowerV8I16Shuffle()
14022 V2 = DAG.getNode(X86ISD::VSHLI, DL, MVT::v4i32, V2, ShAmt); in lowerV8I16Shuffle()
14023 V1 = DAG.getNode(X86ISD::VSRAI, DL, MVT::v4i32, V1, ShAmt); in lowerV8I16Shuffle()
14024 V2 = DAG.getNode(X86ISD::VSRAI, DL, MVT::v4i32, V2, ShAmt); in lowerV8I16Shuffle()
14029 SDValue Result = DAG.getNode(PackOpc, DL, MVT::v8i16, V1, V2); in lowerV8I16Shuffle()
14031 Result = DAG.getBitcast(MVT::v4i32, Result); in lowerV8I16Shuffle()
14032 Result = DAG.getNode(PackOpc, DL, MVT::v8i16, Result, Result); in lowerV8I16Shuffle()
14042 V1 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32, in lowerV8I16Shuffle()
14043 DAG.getBitcast(MVT::v4i32, V1), in lowerV8I16Shuffle()
14044 DAG.getTargetConstant(16, DL, MVT::i8)); in lowerV8I16Shuffle()
14045 V2 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32, in lowerV8I16Shuffle()
14046 DAG.getBitcast(MVT::v4i32, V2), in lowerV8I16Shuffle()
14047 DAG.getTargetConstant(16, DL, MVT::i8)); in lowerV8I16Shuffle()
14048 return DAG.getNode(HasSSE41 ? X86ISD::PACKUS : X86ISD::PACKSS, DL, in lowerV8I16Shuffle()
14054 Mask, Subtarget, DAG)) in lowerV8I16Shuffle()
14062 Zeroable, DAG, V1InUse, V2InUse); in lowerV8I16Shuffle()
14068 Mask, Subtarget, DAG); in lowerV8I16Shuffle()
14075 SelectionDAG &DAG) { in lowerV8F16Shuffle() argument
14085 Mask, Subtarget, DAG)) in lowerV8F16Shuffle()
14090 DL, MVT::v8f16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV8F16Shuffle()
14094 V1 = DAG.getBitcast(MVT::v8i16, V1); in lowerV8F16Shuffle()
14095 V2 = DAG.getBitcast(MVT::v8i16, V2); in lowerV8F16Shuffle()
14096 return DAG.getBitcast(MVT::v8f16, in lowerV8F16Shuffle()
14097 DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask)); in lowerV8F16Shuffle()
14106 SelectionDAG &DAG) { in lowerShuffleWithPERMV() argument
14111 V1 = widenSubVector(V1, false, Subtarget, DAG, DL, 512); in lowerShuffleWithPERMV()
14112 V2 = widenSubVector(V2, false, Subtarget, DAG, DL, 512); in lowerShuffleWithPERMV()
14122 MaskNode = getConstVector(AdjustedMask, MaskVT, DAG, DL, true); in lowerShuffleWithPERMV()
14123 MaskNode = widenSubVector(MaskNode, false, Subtarget, DAG, DL, 512); in lowerShuffleWithPERMV()
14125 MaskNode = getConstVector(Mask, MaskVT, DAG, DL, true); in lowerShuffleWithPERMV()
14130 Result = DAG.getNode(X86ISD::VPERMV, DL, ShuffleVT, MaskNode, V1); in lowerShuffleWithPERMV()
14132 Result = DAG.getNode(X86ISD::VPERMV3, DL, ShuffleVT, V1, MaskNode, V2); in lowerShuffleWithPERMV()
14135 Result = extractSubVector(Result, 0, DAG, DL, VT.getSizeInBits()); in lowerShuffleWithPERMV()
14150 SelectionDAG &DAG) { in lowerV16I8Shuffle() argument
14158 DAG, /*BitwiseOnly*/ false)) in lowerV16I8Shuffle()
14163 Subtarget, DAG)) in lowerV16I8Shuffle()
14167 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG, in lowerV16I8Shuffle()
14173 Zeroable, Subtarget, DAG)) in lowerV16I8Shuffle()
14178 Subtarget, DAG)) in lowerV16I8Shuffle()
14182 Subtarget, DAG)) in lowerV16I8Shuffle()
14188 Zeroable, DAG)) in lowerV16I8Shuffle()
14197 Mask, Subtarget, DAG)) in lowerV16I8Shuffle()
14202 Subtarget, DAG)) in lowerV16I8Shuffle()
14205 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG)) in lowerV16I8Shuffle()
14267 V1 = DAG.getBitcast( in lowerV16I8Shuffle()
14269 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1), in lowerV16I8Shuffle()
14270 DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle)); in lowerV16I8Shuffle()
14280 V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL, in lowerV16I8Shuffle()
14281 MVT::v16i8, EvenInUse ? V1 : DAG.getUNDEF(MVT::v16i8), in lowerV16I8Shuffle()
14282 OddInUse ? V1 : DAG.getUNDEF(MVT::v16i8)); in lowerV16I8Shuffle()
14295 return DAG.getBitcast( in lowerV16I8Shuffle()
14297 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1), in lowerV16I8Shuffle()
14298 DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle)); in lowerV16I8Shuffle()
14305 Zeroable, Subtarget, DAG)) in lowerV16I8Shuffle()
14309 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG)) in lowerV16I8Shuffle()
14314 Zeroable, Subtarget, DAG)) in lowerV16I8Shuffle()
14342 DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse); in lowerV16I8Shuffle()
14350 Zeroable, Subtarget, DAG)) in lowerV16I8Shuffle()
14362 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8Shuffle()
14368 DAG); in lowerV16I8Shuffle()
14372 SDValue MaskNode = getConstVector(Mask, MVT::v16i8, DAG, DL, true); in lowerV16I8Shuffle()
14373 return DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, V1, V2, MaskNode); in lowerV16I8Shuffle()
14379 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8Shuffle()
14389 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I8Shuffle()
14392 if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG)) in lowerV16I8Shuffle()
14410 SmallVector<SDValue, 8> WordClearOps(8, DAG.getConstant(0, DL, MVT::i16)); in lowerV16I8Shuffle()
14412 WordClearOps[i] = DAG.getConstant(0xFF, DL, MVT::i16); in lowerV16I8Shuffle()
14413 SDValue WordClearMask = DAG.getBuildVector(MVT::v8i16, DL, WordClearOps); in lowerV16I8Shuffle()
14414 V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V1), in lowerV16I8Shuffle()
14417 V2 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V2), in lowerV16I8Shuffle()
14421 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, in lowerV16I8Shuffle()
14424 Result = DAG.getBitcast(MVT::v8i16, Result); in lowerV16I8Shuffle()
14425 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result); in lowerV16I8Shuffle()
14432 V1 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16, in lowerV16I8Shuffle()
14433 DAG.getBitcast(MVT::v8i16, V1), in lowerV16I8Shuffle()
14434 DAG.getTargetConstant(8, DL, MVT::i8)); in lowerV16I8Shuffle()
14436 V2 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16, in lowerV16I8Shuffle()
14437 DAG.getBitcast(MVT::v8i16, V2), in lowerV16I8Shuffle()
14438 DAG.getTargetConstant(8, DL, MVT::i8)); in lowerV16I8Shuffle()
14439 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, in lowerV16I8Shuffle()
14446 Subtarget, DAG); in lowerV16I8Shuffle()
14466 VLoHalf = DAG.getBitcast(MVT::v8i16, V); in lowerV16I8Shuffle()
14467 VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf, in lowerV16I8Shuffle()
14468 DAG.getConstant(0x00FF, DL, MVT::v8i16)); in lowerV16I8Shuffle()
14471 VHiHalf = DAG.getUNDEF(MVT::v8i16); in lowerV16I8Shuffle()
14483 SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL); in lowerV16I8Shuffle()
14485 VLoHalf = DAG.getBitcast( in lowerV16I8Shuffle()
14486 MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero)); in lowerV16I8Shuffle()
14487 VHiHalf = DAG.getBitcast( in lowerV16I8Shuffle()
14488 MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero)); in lowerV16I8Shuffle()
14491 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask); in lowerV16I8Shuffle()
14492 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask); in lowerV16I8Shuffle()
14494 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV); in lowerV16I8Shuffle()
14505 SelectionDAG &DAG) { in lower128BitShuffle() argument
14507 V1 = DAG.getBitcast(MVT::v8i16, V1); in lower128BitShuffle()
14508 V2 = DAG.getBitcast(MVT::v8i16, V2); in lower128BitShuffle()
14509 return DAG.getBitcast(VT, in lower128BitShuffle()
14510 DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask)); in lower128BitShuffle()
14515 return lowerV2I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
14517 return lowerV2F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
14519 return lowerV4I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
14521 return lowerV4F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
14523 return lowerV8I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
14525 return lowerV8F16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
14527 return lowerV16I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitShuffle()
14541 SelectionDAG &DAG, bool SimpleOnly) { in splitAndLowerShuffle() argument
14559 std::tie(LoV, HiV) = splitVector(peekThroughBitcasts(V), DAG, DL); in splitAndLowerShuffle()
14560 return std::make_pair(DAG.getBitcast(SplitVT, LoV), in splitAndLowerShuffle()
14561 DAG.getBitcast(SplitVT, HiV)); in splitAndLowerShuffle()
14624 return DAG.getUNDEF(SplitVT); in splitAndLowerShuffle()
14626 return DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask); in splitAndLowerShuffle()
14628 return DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask); in splitAndLowerShuffle()
14632 V1Blend = DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask); in splitAndLowerShuffle()
14641 V2Blend = DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask); in splitAndLowerShuffle()
14649 return DAG.getVectorShuffle(SplitVT, DL, V1Blend, V2Blend, BlendMask); in splitAndLowerShuffle()
14657 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in splitAndLowerShuffle()
14671 SelectionDAG &DAG) { in lowerShuffleAsSplitOrBlend() argument
14697 DAG); in lowerShuffleAsSplitOrBlend()
14711 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, in lowerShuffleAsSplitOrBlend()
14717 DAG); in lowerShuffleAsSplitOrBlend()
14725 SelectionDAG &DAG) { in lowerShuffleAsLanePermuteAndSHUFP() argument
14744 SDValue LHS = DAG.getVectorShuffle(VT, DL, V1, V2, LHSMask); in lowerShuffleAsLanePermuteAndSHUFP()
14745 SDValue RHS = DAG.getVectorShuffle(VT, DL, V1, V2, RHSMask); in lowerShuffleAsLanePermuteAndSHUFP()
14746 return DAG.getNode(X86ISD::SHUFP, DL, VT, LHS, RHS, in lowerShuffleAsLanePermuteAndSHUFP()
14747 DAG.getTargetConstant(SHUFPMask, DL, MVT::i8)); in lowerShuffleAsLanePermuteAndSHUFP()
14760 SelectionDAG &DAG, const X86Subtarget &Subtarget) { in lowerShuffleAsLanePermuteAndPermute() argument
14835 SDValue CrossLane = DAG.getVectorShuffle(VT, DL, V1, V2, CrossLaneMask); in lowerShuffleAsLanePermuteAndPermute()
14836 return DAG.getVectorShuffle(VT, DL, CrossLane, DAG.getUNDEF(VT), in lowerShuffleAsLanePermuteAndPermute()
14883 SelectionDAG &DAG, const X86Subtarget &Subtarget) { in lowerShuffleAsLanePermuteAndShuffle() argument
14894 return lowerShuffleAsLanePermuteAndSHUFP(DL, VT, V1, V2, Mask, DAG); in lowerShuffleAsLanePermuteAndShuffle()
14927 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, in lowerShuffleAsLanePermuteAndShuffle()
14932 SDValue Flipped = DAG.getBitcast(PVT, V1); in lowerShuffleAsLanePermuteAndShuffle()
14934 DAG.getVectorShuffle(PVT, DL, Flipped, DAG.getUNDEF(PVT), {2, 3, 0, 1}); in lowerShuffleAsLanePermuteAndShuffle()
14935 Flipped = DAG.getBitcast(VT, Flipped); in lowerShuffleAsLanePermuteAndShuffle()
14936 return DAG.getVectorShuffle(VT, DL, V1, Flipped, InLaneMask); in lowerShuffleAsLanePermuteAndShuffle()
14944 SelectionDAG &DAG) { in lowerV2X128Shuffle() argument
14955 VT, MemVT, Ld, Ofs, DAG)) in lowerV2X128Shuffle()
14976 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1, in lowerV2X128Shuffle()
14977 DAG.getIntPtrConstant(0, DL)); in lowerV2X128Shuffle()
14978 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerV2X128Shuffle()
14979 getZeroVector(VT, Subtarget, DAG, DL), LoV, in lowerV2X128Shuffle()
14980 DAG.getIntPtrConstant(0, DL)); in lowerV2X128Shuffle()
14989 Subtarget, DAG)) in lowerV2X128Shuffle()
15004 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, in lowerV2X128Shuffle()
15006 DAG.getIntPtrConstant(0, DL)); in lowerV2X128Shuffle()
15007 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec, in lowerV2X128Shuffle()
15008 DAG.getIntPtrConstant(2, DL)); in lowerV2X128Shuffle()
15017 return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2, in lowerV2X128Shuffle()
15018 DAG.getTargetConstant(PermMask, DL, MVT::i8)); in lowerV2X128Shuffle()
15045 V1 = DAG.getUNDEF(VT); in lowerV2X128Shuffle()
15047 V2 = DAG.getUNDEF(VT); in lowerV2X128Shuffle()
15049 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2, in lowerV2X128Shuffle()
15050 DAG.getTargetConstant(PermMask, DL, MVT::i8)); in lowerV2X128Shuffle()
15062 const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerShuffleAsLanePermuteAndRepeatedMask() argument
15187 SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask); in lowerShuffleAsLanePermuteAndRepeatedMask()
15204 SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask); in lowerShuffleAsLanePermuteAndRepeatedMask()
15223 return DAG.getVectorShuffle(VT, DL, NewV1, NewV2, NewMask); in lowerShuffleAsLanePermuteAndRepeatedMask()
15287 SelectionDAG &DAG, bool UseConcat = false) { in getShuffleHalfVectors() argument
15297 return DAG.getUNDEF(HalfVT); in getShuffleHalfVectors()
15300 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V, in getShuffleHalfVectors()
15301 DAG.getIntPtrConstant(HalfIdx, DL)); in getShuffleHalfVectors()
15307 SDValue V = DAG.getVectorShuffle(HalfVT, DL, Half1, Half2, HalfMask); in getShuffleHalfVectors()
15310 SDValue Op1 = DAG.getUNDEF(HalfVT); in getShuffleHalfVectors()
15313 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Op0, Op1); in getShuffleHalfVectors()
15317 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, in getShuffleHalfVectors()
15318 DAG.getIntPtrConstant(Offset, DL)); in getShuffleHalfVectors()
15327 SelectionDAG &DAG) { in lowerShuffleWithUndefHalf() argument
15344 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1, in lowerShuffleWithUndefHalf()
15345 DAG.getIntPtrConstant(HalfNumElts, DL)); in lowerShuffleWithUndefHalf()
15346 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi, in lowerShuffleWithUndefHalf()
15347 DAG.getIntPtrConstant(0, DL)); in lowerShuffleWithUndefHalf()
15354 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1, in lowerShuffleWithUndefHalf()
15355 DAG.getIntPtrConstant(0, DL)); in lowerShuffleWithUndefHalf()
15356 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi, in lowerShuffleWithUndefHalf()
15357 DAG.getIntPtrConstant(HalfNumElts, DL)); in lowerShuffleWithUndefHalf()
15382 UndefLower, DAG); in lowerShuffleWithUndefHalf()
15389 !is128BitUnpackShuffleMask(HalfMask, DAG) && in lowerShuffleWithUndefHalf()
15405 UndefLower, DAG); in lowerShuffleWithUndefHalf()
15424 UndefLower, DAG); in lowerShuffleWithUndefHalf()
15437 const X86Subtarget &Subtarget, SelectionDAG &DAG) { in lowerShuffleAsRepeatedMaskAndLanePermute() argument
15474 SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask); in lowerShuffleAsRepeatedMaskAndLanePermute()
15487 return DAG.getVectorShuffle(VT, DL, RepeatShuf, DAG.getUNDEF(VT), in lowerShuffleAsRepeatedMaskAndLanePermute()
15607 DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask); in lowerShuffleAsRepeatedMaskAndLanePermute()
15609 return DAG.getVectorShuffle(VT, DL, RepeatedShuffle, DAG.getUNDEF(VT), in lowerShuffleAsRepeatedMaskAndLanePermute()
15683 SelectionDAG &DAG) { in lowerShuffleWithSHUFPD() argument
15695 V1 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleWithSHUFPD()
15697 V2 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleWithSHUFPD()
15699 return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2, in lowerShuffleWithSHUFPD()
15700 DAG.getTargetConstant(Immediate, DL, MVT::i8)); in lowerShuffleWithSHUFPD()
15710 SelectionDAG &DAG) { in lowerShuffleAsVTRUNCAndUnpack() argument
15721 V1 = DAG.getBitcast(MVT::v4i64, V1); in lowerShuffleAsVTRUNCAndUnpack()
15722 V2 = DAG.getBitcast(MVT::v4i64, V2); in lowerShuffleAsVTRUNCAndUnpack()
15724 V1 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V1); in lowerShuffleAsVTRUNCAndUnpack()
15725 V2 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V2); in lowerShuffleAsVTRUNCAndUnpack()
15729 SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2, in lowerShuffleAsVTRUNCAndUnpack()
15733 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v32i8, in lowerShuffleAsVTRUNCAndUnpack()
15734 DAG.getConstant(0, DL, MVT::v32i8), Unpack, in lowerShuffleAsVTRUNCAndUnpack()
15735 DAG.getIntPtrConstant(0, DL)); in lowerShuffleAsVTRUNCAndUnpack()
15757 SelectionDAG &DAG) { in lowerShufflePairAsUNPCKAndPermute() argument
15809 SDValue Unpckl = DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2); in lowerShufflePairAsUNPCKAndPermute()
15810 SDValue Unpckh = DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2); in lowerShufflePairAsUNPCKAndPermute()
15811 SDValue Perm1 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh, in lowerShufflePairAsUNPCKAndPermute()
15812 DAG.getTargetConstant(0x20, DL, MVT::i8)); in lowerShufflePairAsUNPCKAndPermute()
15813 SDValue Perm2 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh, in lowerShufflePairAsUNPCKAndPermute()
15814 DAG.getTargetConstant(0x31, DL, MVT::i8)); in lowerShufflePairAsUNPCKAndPermute()
15816 DAG.ReplaceAllUsesWith(SecondHalf, &Perm2); in lowerShufflePairAsUNPCKAndPermute()
15819 DAG.ReplaceAllUsesWith(FirstHalf, &Perm1); in lowerShufflePairAsUNPCKAndPermute()
15830 SelectionDAG &DAG) { in lowerV4F64Shuffle() argument
15836 Subtarget, DAG)) in lowerV4F64Shuffle()
15842 Mask, Subtarget, DAG)) in lowerV4F64Shuffle()
15847 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1); in lowerV4F64Shuffle()
15854 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1, in lowerV4F64Shuffle()
15855 DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8)); in lowerV4F64Shuffle()
15860 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1, in lowerV4F64Shuffle()
15861 getV4X86ShuffleImm8ForMask(Mask, DL, DAG)); in lowerV4F64Shuffle()
15866 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64Shuffle()
15871 Mask, DAG, Subtarget)) in lowerV4F64Shuffle()
15876 DAG, Subtarget); in lowerV4F64Shuffle()
15880 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG)) in lowerV4F64Shuffle()
15884 Zeroable, Subtarget, DAG)) in lowerV4F64Shuffle()
15889 Zeroable, Subtarget, DAG)) in lowerV4F64Shuffle()
15903 return lowerShuffleAsLanePermuteAndSHUFP(DL, MVT::v4f64, V1, V2, Mask, DAG); in lowerV4F64Shuffle()
15909 Subtarget, DAG); in lowerV4F64Shuffle()
15914 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64Shuffle()
15923 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64Shuffle()
15929 DAG, Subtarget)) in lowerV4F64Shuffle()
15936 Subtarget, DAG); in lowerV4F64Shuffle()
15940 Subtarget, DAG); in lowerV4F64Shuffle()
15950 SelectionDAG &DAG) { in lowerV4I64Shuffle() argument
15957 Subtarget, DAG)) in lowerV4I64Shuffle()
15961 Zeroable, Subtarget, DAG)) in lowerV4I64Shuffle()
15966 Subtarget, DAG)) in lowerV4I64Shuffle()
15973 Subtarget, DAG, /*BitwiseOnly*/ true)) in lowerV4I64Shuffle()
15983 return DAG.getBitcast( in lowerV4I64Shuffle()
15985 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, in lowerV4I64Shuffle()
15986 DAG.getBitcast(MVT::v8i32, V1), in lowerV4I64Shuffle()
15987 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG))); in lowerV4I64Shuffle()
15992 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1, in lowerV4I64Shuffle()
15993 getV4X86ShuffleImm8ForMask(Mask, DL, DAG)); in lowerV4I64Shuffle()
15999 DAG, /*BitwiseOnly*/ false)) in lowerV4I64Shuffle()
16005 Zeroable, Subtarget, DAG)) in lowerV4I64Shuffle()
16009 DAG, Subtarget)) in lowerV4I64Shuffle()
16015 Subtarget, DAG)) in lowerV4I64Shuffle()
16019 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG)) in lowerV4I64Shuffle()
16029 Subtarget, DAG); in lowerV4I64Shuffle()
16034 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG)) in lowerV4I64Shuffle()
16039 lowerShuffleAsBlendAndPermute(DL, MVT::v4i64, V1, V2, Mask, DAG)) in lowerV4I64Shuffle()
16048 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG)) in lowerV4I64Shuffle()
16053 Subtarget, DAG); in lowerV4I64Shuffle()
16063 SelectionDAG &DAG) { in lowerV8F32Shuffle() argument
16069 Zeroable, Subtarget, DAG)) in lowerV8F32Shuffle()
16074 Subtarget, DAG)) in lowerV8F32Shuffle()
16082 if (SDValue R = splitAndLowerShuffle(DL, MVT::v8f32, V1, V2, Mask, DAG, in lowerV8F32Shuffle()
16087 Zeroable, Subtarget, DAG)) in lowerV8F32Shuffle()
16088 return DAG.getBitcast(MVT::v8f32, ZExt); in lowerV8F32Shuffle()
16099 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1); in lowerV8F32Shuffle()
16101 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1); in lowerV8F32Shuffle()
16104 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1, in lowerV8F32Shuffle()
16105 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG)); in lowerV8F32Shuffle()
16108 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG)) in lowerV8F32Shuffle()
16113 return lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG); in lowerV8F32Shuffle()
16119 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG)) in lowerV8F32Shuffle()
16126 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true); in lowerV8F32Shuffle()
16127 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1, VPermMask); in lowerV8F32Shuffle()
16130 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true); in lowerV8F32Shuffle()
16131 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1); in lowerV8F32Shuffle()
16135 DAG, Subtarget); in lowerV8F32Shuffle()
16141 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG)) in lowerV8F32Shuffle()
16147 DAG, Subtarget)) in lowerV8F32Shuffle()
16157 Mask, DAG)) in lowerV8F32Shuffle()
16163 if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32, DAG)) in lowerV8F32Shuffle()
16165 DAG); in lowerV8F32Shuffle()
16171 Subtarget, DAG); in lowerV8F32Shuffle()
16175 Subtarget, DAG); in lowerV8F32Shuffle()
16185 SelectionDAG &DAG) { in lowerV8I32Shuffle() argument
16197 Zeroable, Subtarget, DAG)) in lowerV8I32Shuffle()
16204 Mask, DAG)) in lowerV8I32Shuffle()
16210 if (isUnpackWdShuffleMask(Mask, MVT::v8i32, DAG) && !V2.isUndef() && in lowerV8I32Shuffle()
16213 DAG); in lowerV8I32Shuffle()
16216 Zeroable, Subtarget, DAG)) in lowerV8I32Shuffle()
16221 Subtarget, DAG)) in lowerV8I32Shuffle()
16228 Subtarget, DAG, /*BitwiseOnly*/ true)) in lowerV8I32Shuffle()
16232 lowerShuffleAsBitRotate(DL, MVT::v8i32, V1, Mask, Subtarget, DAG)) in lowerV8I32Shuffle()
16245 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1, in lowerV8I32Shuffle()
16246 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG)); in lowerV8I32Shuffle()
16249 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG)) in lowerV8I32Shuffle()
16256 DAG, /*BitwiseOnly*/ false)) in lowerV8I32Shuffle()
16261 lowerShuffleAsBitRotate(DL, MVT::v8i32, V1, Mask, Subtarget, DAG)) in lowerV8I32Shuffle()
16267 Zeroable, Subtarget, DAG)) in lowerV8I32Shuffle()
16271 DAG, Subtarget)) in lowerV8I32Shuffle()
16277 Subtarget, DAG)) in lowerV8I32Shuffle()
16283 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) in lowerV8I32Shuffle()
16289 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, Mask, V1, V2, DAG)) in lowerV8I32Shuffle()
16294 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true); in lowerV8I32Shuffle()
16295 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32, VPermMask, V1); in lowerV8I32Shuffle()
16302 SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1); in lowerV8I32Shuffle()
16303 SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2); in lowerV8I32Shuffle()
16305 CastV1, CastV2, DAG); in lowerV8I32Shuffle()
16306 return DAG.getBitcast(MVT::v8i32, ShufPS); in lowerV8I32Shuffle()
16312 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) in lowerV8I32Shuffle()
16317 Subtarget, DAG); in lowerV8I32Shuffle()
16327 SelectionDAG &DAG) { in lowerV16I16Shuffle() argument
16337 DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I16Shuffle()
16342 Subtarget, DAG)) in lowerV16I16Shuffle()
16346 Zeroable, Subtarget, DAG)) in lowerV16I16Shuffle()
16350 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG)) in lowerV16I16Shuffle()
16354 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG, in lowerV16I16Shuffle()
16360 Subtarget, DAG)) in lowerV16I16Shuffle()
16366 Subtarget, DAG, /*BitwiseOnly*/ false)) in lowerV16I16Shuffle()
16371 Subtarget, DAG)) in lowerV16I16Shuffle()
16377 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) in lowerV16I16Shuffle()
16383 lowerShuffleAsBitRotate(DL, MVT::v16i16, V1, Mask, Subtarget, DAG)) in lowerV16I16Shuffle()
16388 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, Mask, V1, V2, DAG)) in lowerV16I16Shuffle()
16395 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget)) in lowerV16I16Shuffle()
16399 DAG, Subtarget); in lowerV16I16Shuffle()
16408 DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG); in lowerV16I16Shuffle()
16413 Zeroable, Subtarget, DAG)) in lowerV16I16Shuffle()
16418 return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, Subtarget, DAG); in lowerV16I16Shuffle()
16423 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) in lowerV16I16Shuffle()
16428 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget)) in lowerV16I16Shuffle()
16435 Mask, DAG)) in lowerV16I16Shuffle()
16440 Subtarget, DAG); in lowerV16I16Shuffle()
16450 SelectionDAG &DAG) { in lowerV32I8Shuffle() argument
16460 Zeroable, Subtarget, DAG)) in lowerV32I8Shuffle()
16465 Subtarget, DAG)) in lowerV32I8Shuffle()
16469 Zeroable, Subtarget, DAG)) in lowerV32I8Shuffle()
16473 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG)) in lowerV32I8Shuffle()
16477 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG, in lowerV32I8Shuffle()
16483 Subtarget, DAG)) in lowerV32I8Shuffle()
16489 DAG, /*BitwiseOnly*/ false)) in lowerV32I8Shuffle()
16494 Subtarget, DAG)) in lowerV32I8Shuffle()
16500 lowerShuffleAsBitRotate(DL, MVT::v32i8, V1, Mask, Subtarget, DAG)) in lowerV32I8Shuffle()
16506 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) in lowerV32I8Shuffle()
16514 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, Mask, V1, V2, DAG)) in lowerV32I8Shuffle()
16518 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget)) in lowerV32I8Shuffle()
16522 DAG, Subtarget); in lowerV32I8Shuffle()
16526 Zeroable, Subtarget, DAG)) in lowerV32I8Shuffle()
16531 return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, Subtarget, DAG); in lowerV32I8Shuffle()
16536 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) in lowerV32I8Shuffle()
16541 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget)) in lowerV32I8Shuffle()
16549 Mask, Zeroable, DAG)) in lowerV32I8Shuffle()
16556 Mask, DAG)) in lowerV32I8Shuffle()
16561 Subtarget, DAG); in lowerV32I8Shuffle()
16572 SelectionDAG &DAG) { in lower256BitShuffle() argument
16580 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lower256BitShuffle()
16585 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lower256BitShuffle()
16600 Subtarget, DAG)) in lower256BitShuffle()
16602 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG)) in lower256BitShuffle()
16604 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false); in lower256BitShuffle()
16609 V1 = DAG.getBitcast(FpVT, V1); in lower256BitShuffle()
16610 V2 = DAG.getBitcast(FpVT, V2); in lower256BitShuffle()
16611 return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask)); in lower256BitShuffle()
16615 V1 = DAG.getBitcast(MVT::v16i16, V1); in lower256BitShuffle()
16616 V2 = DAG.getBitcast(MVT::v16i16, V2); in lower256BitShuffle()
16617 return DAG.getBitcast(VT, in lower256BitShuffle()
16618 DAG.getVectorShuffle(MVT::v16i16, DL, V1, V2, Mask)); in lower256BitShuffle()
16623 return lowerV4F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
16625 return lowerV4I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
16627 return lowerV8F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
16629 return lowerV8I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
16631 return lowerV16I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
16633 return lowerV32I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitShuffle()
16644 SelectionDAG &DAG) { in lowerV4X128Shuffle() argument
16663 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1, in lowerV4X128Shuffle()
16664 DAG.getIntPtrConstant(0, DL)); in lowerV4X128Shuffle()
16665 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerV4X128Shuffle()
16666 getZeroVector(VT, Subtarget, DAG, DL), LoV, in lowerV4X128Shuffle()
16667 DAG.getIntPtrConstant(0, DL)); in lowerV4X128Shuffle()
16677 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, OnlyUsesV1 ? V1 : V2, in lowerV4X128Shuffle()
16678 DAG.getIntPtrConstant(0, DL)); in lowerV4X128Shuffle()
16679 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec, in lowerV4X128Shuffle()
16680 DAG.getIntPtrConstant(4, DL)); in lowerV4X128Shuffle()
16708 SDValue Subvec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2, in lowerV4X128Shuffle()
16709 DAG.getIntPtrConstant(0, DL)); in lowerV4X128Shuffle()
16710 return insert128BitVector(V1, Subvec, V2Index * 2, DAG, DL); in lowerV4X128Shuffle()
16724 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)}; in lowerV4X128Shuffle()
16742 return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1], in lowerV4X128Shuffle()
16743 getV4X86ShuffleImm8ForMask(PermMask, DL, DAG)); in lowerV4X128Shuffle()
16750 SelectionDAG &DAG) { in lowerV8F64Shuffle() argument
16758 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1); in lowerV8F64Shuffle()
16767 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1, in lowerV8F64Shuffle()
16768 DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8)); in lowerV8F64Shuffle()
16773 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1, in lowerV8F64Shuffle()
16774 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG)); in lowerV8F64Shuffle()
16778 V2, Subtarget, DAG)) in lowerV8F64Shuffle()
16781 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG)) in lowerV8F64Shuffle()
16786 Zeroable, Subtarget, DAG)) in lowerV8F64Shuffle()
16790 DAG, Subtarget)) in lowerV8F64Shuffle()
16794 Zeroable, Subtarget, DAG)) in lowerV8F64Shuffle()
16797 return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, Subtarget, DAG); in lowerV8F64Shuffle()
16804 SelectionDAG &DAG) { in lowerV16F32Shuffle() argument
16817 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1); in lowerV16F32Shuffle()
16819 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1); in lowerV16F32Shuffle()
16822 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1, in lowerV16F32Shuffle()
16823 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG)); in lowerV16F32Shuffle()
16826 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG)) in lowerV16F32Shuffle()
16830 Zeroable, Subtarget, DAG)) in lowerV16F32Shuffle()
16834 return lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG); in lowerV16F32Shuffle()
16838 Zeroable, Subtarget, DAG)) in lowerV16F32Shuffle()
16842 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16F32Shuffle()
16843 return DAG.getBitcast(MVT::v16f32, ZExt); in lowerV16F32Shuffle()
16848 DL, MVT::v16f32, V1, V2, Mask, Subtarget, DAG)) in lowerV16F32Shuffle()
16855 SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true); in lowerV16F32Shuffle()
16856 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask); in lowerV16F32Shuffle()
16861 V1, V2, DAG, Subtarget)) in lowerV16F32Shuffle()
16864 return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, Subtarget, DAG); in lowerV16F32Shuffle()
16871 SelectionDAG &DAG) { in lowerV8I64Shuffle() argument
16880 Subtarget, DAG, /*BitwiseOnly*/ true)) in lowerV8I64Shuffle()
16891 return DAG.getBitcast( in lowerV8I64Shuffle()
16893 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, in lowerV8I64Shuffle()
16894 DAG.getBitcast(MVT::v16i32, V1), in lowerV8I64Shuffle()
16895 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG))); in lowerV8I64Shuffle()
16900 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1, in lowerV8I64Shuffle()
16901 getV4X86ShuffleImm8ForMask(Repeated256Mask, DL, DAG)); in lowerV8I64Shuffle()
16905 V2, Subtarget, DAG)) in lowerV8I64Shuffle()
16911 DAG, /*BitwiseOnly*/ false)) in lowerV8I64Shuffle()
16916 Zeroable, Subtarget, DAG)) in lowerV8I64Shuffle()
16922 Subtarget, DAG)) in lowerV8I64Shuffle()
16925 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG)) in lowerV8I64Shuffle()
16930 DAG, Subtarget)) in lowerV8I64Shuffle()
16934 Zeroable, Subtarget, DAG)) in lowerV8I64Shuffle()
16937 return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, Subtarget, DAG); in lowerV8I64Shuffle()
16944 SelectionDAG &DAG) { in lowerV16I32Shuffle() argument
16955 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I32Shuffle()
16962 Subtarget, DAG, /*BitwiseOnly*/ true)) in lowerV16I32Shuffle()
16966 Subtarget, DAG)) in lowerV16I32Shuffle()
16979 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1, in lowerV16I32Shuffle()
16980 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG)); in lowerV16I32Shuffle()
16983 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG)) in lowerV16I32Shuffle()
16990 Subtarget, DAG, /*BitwiseOnly*/ false)) in lowerV16I32Shuffle()
16995 lowerShuffleAsBitRotate(DL, MVT::v16i32, V1, Mask, Subtarget, DAG)) in lowerV16I32Shuffle()
17000 Zeroable, Subtarget, DAG)) in lowerV16I32Shuffle()
17006 Subtarget, DAG)) in lowerV16I32Shuffle()
17012 SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1); in lowerV16I32Shuffle()
17013 SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2); in lowerV16I32Shuffle()
17015 CastV1, CastV2, DAG); in lowerV16I32Shuffle()
17016 return DAG.getBitcast(MVT::v16i32, ShufPS); in lowerV16I32Shuffle()
17022 DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG)) in lowerV16I32Shuffle()
17027 DAG, Subtarget)) in lowerV16I32Shuffle()
17031 Zeroable, Subtarget, DAG)) in lowerV16I32Shuffle()
17034 return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, Subtarget, DAG); in lowerV16I32Shuffle()
17041 SelectionDAG &DAG) { in lowerV32I16Shuffle() argument
17051 DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV32I16Shuffle()
17055 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG)) in lowerV32I16Shuffle()
17060 lowerShuffleWithPACK(DL, MVT::v32i16, Mask, V1, V2, DAG, Subtarget)) in lowerV32I16Shuffle()
17066 Subtarget, DAG, /*BitwiseOnly*/ false)) in lowerV32I16Shuffle()
17071 Subtarget, DAG)) in lowerV32I16Shuffle()
17077 lowerShuffleAsBitRotate(DL, MVT::v32i16, V1, Mask, Subtarget, DAG)) in lowerV32I16Shuffle()
17086 RepeatedMask, Subtarget, DAG); in lowerV32I16Shuffle()
17091 Zeroable, Subtarget, DAG)) in lowerV32I16Shuffle()
17095 Zeroable, Subtarget, DAG)) in lowerV32I16Shuffle()
17098 return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, Subtarget, DAG); in lowerV32I16Shuffle()
17105 SelectionDAG &DAG) { in lowerV64I8Shuffle() argument
17115 DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV64I8Shuffle()
17119 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG)) in lowerV64I8Shuffle()
17123 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG, in lowerV64I8Shuffle()
17130 DAG, /*BitwiseOnly*/ false)) in lowerV64I8Shuffle()
17135 Subtarget, DAG)) in lowerV64I8Shuffle()
17141 lowerShuffleAsBitRotate(DL, MVT::v64i8, V1, Mask, Subtarget, DAG)) in lowerV64I8Shuffle()
17146 Zeroable, Subtarget, DAG)) in lowerV64I8Shuffle()
17150 Zeroable, Subtarget, DAG)) in lowerV64I8Shuffle()
17156 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG)) in lowerV64I8Shuffle()
17160 DL, MVT::v64i8, V1, V2, Mask, DAG, Subtarget)) in lowerV64I8Shuffle()
17164 Zeroable, Subtarget, DAG)) in lowerV64I8Shuffle()
17171 Mask, Subtarget, DAG)) in lowerV64I8Shuffle()
17178 DAG, V1InUse, V2InUse); in lowerV64I8Shuffle()
17185 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG)) in lowerV64I8Shuffle()
17190 return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, Subtarget, DAG); in lowerV64I8Shuffle()
17192 return splitAndLowerShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG, /*SimpleOnly*/ false); in lowerV64I8Shuffle()
17204 SelectionDAG &DAG) { in lower512BitShuffle() argument
17215 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lower512BitShuffle()
17220 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lower512BitShuffle()
17225 Subtarget, DAG)) in lower512BitShuffle()
17232 Subtarget, DAG)) in lower512BitShuffle()
17234 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG)) in lower512BitShuffle()
17237 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false); in lower512BitShuffle()
17242 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, in lower512BitShuffle()
17245 V1 = DAG.getBitcast(MVT::v32i16, V1); in lower512BitShuffle()
17246 V2 = DAG.getBitcast(MVT::v32i16, V2); in lower512BitShuffle()
17247 return DAG.getBitcast(VT, in lower512BitShuffle()
17248 DAG.getVectorShuffle(MVT::v32i16, DL, V1, V2, Mask)); in lower512BitShuffle()
17257 return lowerV8F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
17259 return lowerV16F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
17261 return lowerV8I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
17263 return lowerV16I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
17265 return lowerV32I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
17267 return lowerV64I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitShuffle()
17277 SelectionDAG &DAG) { in lower1BitShuffleAsKSHIFTR() argument
17305 SDValue Res = widenMaskVector(V1, false, Subtarget, DAG, DL); in lower1BitShuffleAsKSHIFTR()
17306 Res = DAG.getNode(X86ISD::KSHIFTR, DL, Res.getValueType(), Res, in lower1BitShuffleAsKSHIFTR()
17307 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8)); in lower1BitShuffleAsKSHIFTR()
17308 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lower1BitShuffleAsKSHIFTR()
17309 DAG.getIntPtrConstant(0, DL)); in lower1BitShuffleAsKSHIFTR()
17353 SelectionDAG &DAG) { in lower1BitShuffle() argument
17385 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT, in lower1BitShuffle()
17387 DAG.getIntPtrConstant(0, DL)); in lower1BitShuffle()
17388 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lower1BitShuffle()
17389 DAG.getConstant(0, DL, VT), in lower1BitShuffle()
17390 Extract, DAG.getIntPtrConstant(0, DL)); in lower1BitShuffle()
17395 DAG)) in lower1BitShuffle()
17404 SDValue Res = widenMaskVector(V, false, Subtarget, DAG, DL); in lower1BitShuffle()
17410 Res = DAG.getNode(X86ISD::KSHIFTL, DL, WideVT, Res, in lower1BitShuffle()
17411 DAG.getTargetConstant(WideElts - NumElts, DL, MVT::i8)); in lower1BitShuffle()
17416 Res = DAG.getNode(Opcode, DL, WideVT, Res, in lower1BitShuffle()
17417 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8)); in lower1BitShuffle()
17418 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lower1BitShuffle()
17419 DAG.getIntPtrConstant(0, DL)); in lower1BitShuffle()
17433 return DAG.getSetCC( in lower1BitShuffle()
17434 DL, VT, DAG.getVectorShuffle(OpVT, DL, Op0, DAG.getUNDEF(OpVT), Mask), in lower1BitShuffle()
17435 DAG.getVectorShuffle(OpVT, DL, Op1, DAG.getUNDEF(OpVT), Mask), CC); in lower1BitShuffle()
17473 V1 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V1); in lower1BitShuffle()
17474 V2 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V2); in lower1BitShuffle()
17476 SDValue Shuffle = DAG.getVectorShuffle(ExtVT, DL, V1, V2, Mask); in lower1BitShuffle()
17481 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT), in lower1BitShuffle()
17484 return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle); in lower1BitShuffle()
17606 unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG,
17617 SelectionDAG &DAG) { in lowerVECTOR_SHUFFLE() argument
17633 return DAG.getUNDEF(VT); in lowerVECTOR_SHUFFLE()
17639 return DAG.getCommutedVectorShuffle(*SVOp); in lowerVECTOR_SHUFFLE()
17650 return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask); in lowerVECTOR_SHUFFLE()
17668 return getZeroVector(VT, Subtarget, DAG, DL); in lowerVECTOR_SHUFFLE()
17686 Subtarget, DAG)) in lowerVECTOR_SHUFFLE()
17696 if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) { in lowerVECTOR_SHUFFLE()
17711 V2 = getZeroVector(NewVT, Subtarget, DAG, DL); in lowerVECTOR_SHUFFLE()
17713 V1 = DAG.getBitcast(NewVT, V1); in lowerVECTOR_SHUFFLE()
17714 V2 = DAG.getBitcast(NewVT, V2); in lowerVECTOR_SHUFFLE()
17715 return DAG.getBitcast( in lowerVECTOR_SHUFFLE()
17716 VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask)); in lowerVECTOR_SHUFFLE()
17726 Ops, Mask, VT.getSizeInBits(), DL, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
17727 return DAG.getBitcast(VT, HOp); in lowerVECTOR_SHUFFLE()
17729 V1 = DAG.getBitcast(VT, Ops[0]); in lowerVECTOR_SHUFFLE()
17730 V2 = DAG.getBitcast(VT, Ops[1]); in lowerVECTOR_SHUFFLE()
17743 return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
17746 return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
17749 return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
17752 return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
17760 SelectionDAG &DAG) { in lowerVSELECTtoVectorShuffle() argument
17771 return DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, Mask); in lowerVSELECTtoVectorShuffle()
17777 SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const { in LowerVSELECT()
17786 return DAG.getBitcast(VT, DAG.getNode(ISD::VSELECT, dl, NVT, Cond, in LowerVSELECT()
17787 DAG.getBitcast(NVT, LHS), in LowerVSELECT()
17788 DAG.getBitcast(NVT, RHS))); in LowerVSELECT()
17800 if (SDValue BlendOp = lowerVSELECTtoVectorShuffle(Op, Subtarget, DAG)) in LowerVSELECT()
17827 SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond, in LowerVSELECT()
17828 DAG.getConstant(0, dl, CondVT), in LowerVSELECT()
17831 return DAG.getSelect(dl, VT, Mask, LHS, RHS); in LowerVSELECT()
17837 if (CondEltSize != DAG.ComputeNumSignBits(Cond)) in LowerVSELECT()
17842 Cond = DAG.getSExtOrTrunc(Cond, dl, NewCondVT); in LowerVSELECT()
17843 return DAG.getNode(ISD::VSELECT, dl, VT, Cond, LHS, RHS); in LowerVSELECT()
17853 bool FreeCond = isFreeToSplitVector(Cond.getNode(), DAG); in LowerVSELECT()
17854 bool FreeLHS = isFreeToSplitVector(LHS.getNode(), DAG) || in LowerVSELECT()
17856 bool FreeRHS = isFreeToSplitVector(RHS.getNode(), DAG) || in LowerVSELECT()
17859 return splitVectorOp(Op, DAG, dl); in LowerVSELECT()
17881 Cond = DAG.getBitcast(CastVT, Cond); in LowerVSELECT()
17882 LHS = DAG.getBitcast(CastVT, LHS); in LowerVSELECT()
17883 RHS = DAG.getBitcast(CastVT, RHS); in LowerVSELECT()
17884 SDValue Select = DAG.getNode(ISD::VSELECT, dl, CastVT, Cond, LHS, RHS); in LowerVSELECT()
17885 return DAG.getBitcast(VT, Select); in LowerVSELECT()
17890 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) { in LowerEXTRACT_VECTOR_ELT_SSE4() argument
17905 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, in LowerEXTRACT_VECTOR_ELT_SSE4()
17906 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
17907 DAG.getBitcast(MVT::v4i32, Vec), Idx)); in LowerEXTRACT_VECTOR_ELT_SSE4()
17910 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, Vec, in LowerEXTRACT_VECTOR_ELT_SSE4()
17911 DAG.getTargetConstant(IdxVal, dl, MVT::i8)); in LowerEXTRACT_VECTOR_ELT_SSE4()
17912 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract); in LowerEXTRACT_VECTOR_ELT_SSE4()
17928 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
17929 DAG.getBitcast(MVT::v4i32, Vec), Idx); in LowerEXTRACT_VECTOR_ELT_SSE4()
17930 return DAG.getBitcast(MVT::f32, Extract); in LowerEXTRACT_VECTOR_ELT_SSE4()
17941 static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG, in ExtractBitFromMaskVector() argument
17960 Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl); in ExtractBitFromMaskVector()
17962 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, DAG.getBitcast(IntVT, Vec)); in ExtractBitFromMaskVector()
17966 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec); in ExtractBitFromMaskVector()
17967 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ExtEltVT, Ext, Idx); in ExtractBitFromMaskVector()
17968 return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt); in ExtractBitFromMaskVector()
17976 Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl); in ExtractBitFromMaskVector()
17979 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, Vec.getSimpleValueType(), Vec, in ExtractBitFromMaskVector()
17980 DAG.getTargetConstant(IdxVal, dl, MVT::i8)); in ExtractBitFromMaskVector()
17982 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec, in ExtractBitFromMaskVector()
17983 DAG.getIntPtrConstant(0, dl)); in ExtractBitFromMaskVector()
18022 SelectionDAG &DAG) const { in LowerEXTRACT_VECTOR_ELT()
18030 return ExtractBitFromMaskVector(Op, DAG, Subtarget); in LowerEXTRACT_VECTOR_ELT()
18072 Vec = extract128BitVector(Vec, IdxVal, DAG, dl); in LowerEXTRACT_VECTOR_ELT()
18081 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec, in LowerEXTRACT_VECTOR_ELT()
18082 DAG.getIntPtrConstant(IdxVal, dl)); in LowerEXTRACT_VECTOR_ELT()
18097 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT()
18098 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
18099 DAG.getBitcast(MVT::v4i32, Vec), Idx)); in LowerEXTRACT_VECTOR_ELT()
18102 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec, in LowerEXTRACT_VECTOR_ELT()
18103 DAG.getTargetConstant(IdxVal, dl, MVT::i8)); in LowerEXTRACT_VECTOR_ELT()
18104 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract); in LowerEXTRACT_VECTOR_ELT()
18108 if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG)) in LowerEXTRACT_VECTOR_ELT()
18121 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
18122 DAG.getBitcast(MVT::v4i32, Vec), in LowerEXTRACT_VECTOR_ELT()
18123 DAG.getIntPtrConstant(DWordIdx, dl)); in LowerEXTRACT_VECTOR_ELT()
18126 Res = DAG.getNode(ISD::SRL, dl, MVT::i32, Res, in LowerEXTRACT_VECTOR_ELT()
18127 DAG.getConstant(ShiftVal, dl, MVT::i8)); in LowerEXTRACT_VECTOR_ELT()
18128 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerEXTRACT_VECTOR_ELT()
18133 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT()
18134 DAG.getBitcast(MVT::v8i16, Vec), in LowerEXTRACT_VECTOR_ELT()
18135 DAG.getIntPtrConstant(WordIdx, dl)); in LowerEXTRACT_VECTOR_ELT()
18138 Res = DAG.getNode(ISD::SRL, dl, MVT::i16, Res, in LowerEXTRACT_VECTOR_ELT()
18139 DAG.getConstant(ShiftVal, dl, MVT::i8)); in LowerEXTRACT_VECTOR_ELT()
18140 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerEXTRACT_VECTOR_ELT()
18151 Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask); in LowerEXTRACT_VECTOR_ELT()
18152 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
18153 DAG.getIntPtrConstant(0, dl)); in LowerEXTRACT_VECTOR_ELT()
18167 Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask); in LowerEXTRACT_VECTOR_ELT()
18168 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
18169 DAG.getIntPtrConstant(0, dl)); in LowerEXTRACT_VECTOR_ELT()
18177 static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG, in InsertBitToMaskVector() argument
18191 SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT, in InsertBitToMaskVector()
18192 DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec), in InsertBitToMaskVector()
18193 DAG.getNode(ISD::SIGN_EXTEND, dl, ExtEltVT, Elt), Idx); in InsertBitToMaskVector()
18194 return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp); in InsertBitToMaskVector()
18198 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt); in InsertBitToMaskVector()
18199 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VecVT, Vec, EltInVec, Idx); in InsertBitToMaskVector()
18203 SelectionDAG &DAG) const { in LowerINSERT_VECTOR_ELT()
18210 return InsertBitToMaskVector(Op, DAG, Subtarget); in LowerINSERT_VECTOR_ELT()
18220 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVT, in LowerINSERT_VECTOR_ELT()
18221 DAG.getBitcast(IVT, N0), in LowerINSERT_VECTOR_ELT()
18222 DAG.getBitcast(MVT::i16, N1), N2); in LowerINSERT_VECTOR_ELT()
18223 return DAG.getBitcast(VT, Res); in LowerINSERT_VECTOR_ELT()
18240 SDValue IdxExt = DAG.getZExtOrTrunc(N2, dl, IdxSVT); in LowerINSERT_VECTOR_ELT()
18241 SDValue IdxSplat = DAG.getSplatBuildVector(IdxVT, dl, IdxExt); in LowerINSERT_VECTOR_ELT()
18242 SDValue EltSplat = DAG.getSplatBuildVector(VT, dl, N1); in LowerINSERT_VECTOR_ELT()
18246 RawIndices.push_back(DAG.getConstant(I, dl, IdxSVT)); in LowerINSERT_VECTOR_ELT()
18247 SDValue Indices = DAG.getBuildVector(IdxVT, dl, RawIndices); in LowerINSERT_VECTOR_ELT()
18250 return DAG.getSelectCC(dl, IdxSplat, Indices, EltSplat, N0, in LowerINSERT_VECTOR_ELT()
18267 SDValue ZeroCst = DAG.getConstant(0, dl, VT.getScalarType()); in LowerINSERT_VECTOR_ELT()
18268 SDValue OnesCst = DAG.getAllOnesConstant(dl, VT.getScalarType()); in LowerINSERT_VECTOR_ELT()
18271 SDValue CstVector = DAG.getBuildVector(VT, dl, CstVectorElts); in LowerINSERT_VECTOR_ELT()
18272 return DAG.getNode(ISD::OR, dl, VT, N0, CstVector); in LowerINSERT_VECTOR_ELT()
18281 SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl) in LowerINSERT_VECTOR_ELT()
18282 : getOnesVector(VT, DAG, dl); in LowerINSERT_VECTOR_ELT()
18283 return DAG.getVectorShuffle(VT, dl, N0, CstVector, BlendMask); in LowerINSERT_VECTOR_ELT()
18298 SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1); in LowerINSERT_VECTOR_ELT()
18299 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec, in LowerINSERT_VECTOR_ELT()
18300 DAG.getTargetConstant(1, dl, MVT::i8)); in LowerINSERT_VECTOR_ELT()
18315 SDValue N1SplatVec = DAG.getSplatBuildVector(VT, dl, N1); in LowerINSERT_VECTOR_ELT()
18319 return DAG.getVectorShuffle(VT, dl, N0, N1SplatVec, BlendMask); in LowerINSERT_VECTOR_ELT()
18323 SDValue V = extract128BitVector(N0, IdxVal, DAG, dl); in LowerINSERT_VECTOR_ELT()
18329 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1, in LowerINSERT_VECTOR_ELT()
18330 DAG.getIntPtrConstant(IdxIn128, dl)); in LowerINSERT_VECTOR_ELT()
18333 return insert128BitVector(N0, V, IdxVal, DAG, dl); in LowerINSERT_VECTOR_ELT()
18341 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1); in LowerINSERT_VECTOR_ELT()
18342 return getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG); in LowerINSERT_VECTOR_ELT()
18348 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
18350 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, N1); in LowerINSERT_VECTOR_ELT()
18351 N1 = getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG); in LowerINSERT_VECTOR_ELT()
18352 return DAG.getBitcast(VT, N1); in LowerINSERT_VECTOR_ELT()
18370 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
18371 N2 = DAG.getTargetConstant(IdxVal, dl, MVT::i8); in LowerINSERT_VECTOR_ELT()
18372 return DAG.getNode(Opc, dl, VT, N0, N1, N2); in LowerINSERT_VECTOR_ELT()
18386 bool MinSize = DAG.getMachineFunction().getFunction().hasMinSize(); in LowerINSERT_VECTOR_ELT()
18395 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
18396 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1, in LowerINSERT_VECTOR_ELT()
18397 DAG.getTargetConstant(1, dl, MVT::i8)); in LowerINSERT_VECTOR_ELT()
18400 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
18401 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, in LowerINSERT_VECTOR_ELT()
18402 DAG.getTargetConstant(IdxVal << 4, dl, MVT::i8)); in LowerINSERT_VECTOR_ELT()
18414 SelectionDAG &DAG) { in LowerSCALAR_TO_VECTOR() argument
18421 return getZeroVector(OpVT, Subtarget, DAG, dl); in LowerSCALAR_TO_VECTOR()
18431 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
18434 return insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl); in LowerSCALAR_TO_VECTOR()
18444 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
18445 return DAG.getBitcast( in LowerSCALAR_TO_VECTOR()
18446 OpVT, DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, AnyExt)); in LowerSCALAR_TO_VECTOR()
18453 SelectionDAG &DAG) { in LowerINSERT_SUBVECTOR() argument
18456 return insert1BitVector(Op, DAG, Subtarget); in LowerINSERT_SUBVECTOR()
18460 SelectionDAG &DAG) { in LowerEXTRACT_SUBVECTOR() argument
18472 Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl); in LowerEXTRACT_SUBVECTOR()
18475 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, Vec.getSimpleValueType(), Vec, in LowerEXTRACT_SUBVECTOR()
18476 DAG.getTargetConstant(IdxVal, dl, MVT::i8)); in LowerEXTRACT_SUBVECTOR()
18478 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, Op.getValueType(), Vec, in LowerEXTRACT_SUBVECTOR()
18479 DAG.getIntPtrConstant(0, dl)); in LowerEXTRACT_SUBVECTOR()
18509 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const { in LowerConstantPool()
18516 auto PtrVT = getPointerTy(DAG.getDataLayout()); in LowerConstantPool()
18517 SDValue Result = DAG.getTargetConstantPool( in LowerConstantPool()
18521 DAG.getNode(getGlobalWrapperKind(nullptr, OpFlag), DL, PtrVT, Result); in LowerConstantPool()
18525 DAG.getNode(ISD::ADD, DL, PtrVT, in LowerConstantPool()
18526 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result); in LowerConstantPool()
18532 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { in LowerJumpTable()
18539 auto PtrVT = getPointerTy(DAG.getDataLayout()); in LowerJumpTable()
18540 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag); in LowerJumpTable()
18543 DAG.getNode(getGlobalWrapperKind(nullptr, OpFlag), DL, PtrVT, Result); in LowerJumpTable()
18548 DAG.getNode(ISD::ADD, DL, PtrVT, in LowerJumpTable()
18549 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result); in LowerJumpTable()
18555 SelectionDAG &DAG) const { in LowerExternalSymbol()
18556 return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false); in LowerExternalSymbol()
18560 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const { in LowerBlockAddress()
18567 auto PtrVT = getPointerTy(DAG.getDataLayout()); in LowerBlockAddress()
18568 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags); in LowerBlockAddress()
18570 DAG.getNode(getGlobalWrapperKind(nullptr, OpFlags), dl, PtrVT, Result); in LowerBlockAddress()
18574 Result = DAG.getNode(ISD::ADD, dl, PtrVT, in LowerBlockAddress()
18575 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result); in LowerBlockAddress()
18583 SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG, in LowerGlobalOrExternal() argument
18599 const Module &Mod = *DAG.getMachineFunction().getFunction().getParent(); in LowerGlobalOrExternal()
18608 CodeModel::Model M = DAG.getTarget().getCodeModel(); in LowerGlobalOrExternal()
18609 auto PtrVT = getPointerTy(DAG.getDataLayout()); in LowerGlobalOrExternal()
18623 Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, GlobalOffset, OpFlags); in LowerGlobalOrExternal()
18626 Result = DAG.getTargetExternalSymbol(ExternalSym, PtrVT, OpFlags); in LowerGlobalOrExternal()
18634 Result = DAG.getNode(getGlobalWrapperKind(GV, OpFlags), dl, PtrVT, Result); in LowerGlobalOrExternal()
18638 Result = DAG.getNode(ISD::ADD, dl, PtrVT, in LowerGlobalOrExternal()
18639 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result); in LowerGlobalOrExternal()
18645 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result, in LowerGlobalOrExternal()
18646 MachinePointerInfo::getGOT(DAG.getMachineFunction())); in LowerGlobalOrExternal()
18651 Result = DAG.getNode(ISD::ADD, dl, PtrVT, Result, in LowerGlobalOrExternal()
18652 DAG.getConstant(Offset, dl, PtrVT)); in LowerGlobalOrExternal()
18658 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const { in LowerGlobalAddress()
18659 return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false); in LowerGlobalAddress()
18663 GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, in GetTLSADDR() argument
18666 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); in GetTLSADDR()
18667 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in GetTLSADDR()
18670 bool UseTLSDESC = DAG.getTarget().useTLSDESC(); in GetTLSADDR()
18672 TGA = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT, OperandFlags); in GetTLSADDR()
18678 TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0), in GetTLSADDR()
18688 Chain = DAG.getNode(CallType, dl, NodeTys, Ops); in GetTLSADDR()
18691 Chain = DAG.getNode(CallType, dl, NodeTys, Ops); in GetTLSADDR()
18699 SDValue Ret = DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue); in GetTLSADDR()
18704 const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>(); in GetTLSADDR()
18707 Value *Ptr = Constant::getNullValue(PointerType::get(*DAG.getContext(), Seg)); in GetTLSADDR()
18709 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0, dl), in GetTLSADDR()
18711 return DAG.getNode(ISD::ADD, dl, PtrVT, Ret, Offset); in GetTLSADDR()
18716 LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, in LowerToTLSGeneralDynamicModel32() argument
18720 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX, in LowerToTLSGeneralDynamicModel32()
18721 DAG.getNode(X86ISD::GlobalBaseReg, in LowerToTLSGeneralDynamicModel32()
18725 return GetTLSADDR(DAG, Chain, GA, &InGlue, PtrVT, X86::EAX, X86II::MO_TLSGD); in LowerToTLSGeneralDynamicModel32()
18730 LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, in LowerToTLSGeneralDynamicModel64() argument
18732 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, in LowerToTLSGeneralDynamicModel64()
18738 LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA, SelectionDAG &DAG, in LowerToTLSGeneralDynamicModelX32() argument
18740 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, in LowerToTLSGeneralDynamicModelX32()
18745 SelectionDAG &DAG, const EVT PtrVT, in LowerToTLSLocalDynamicModel() argument
18750 X86MachineFunctionInfo *MFI = DAG.getMachineFunction() in LowerToTLSLocalDynamicModel()
18757 Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, ReturnReg, in LowerToTLSLocalDynamicModel()
18761 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX, in LowerToTLSLocalDynamicModel()
18762 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InGlue); in LowerToTLSLocalDynamicModel()
18764 Base = GetTLSADDR(DAG, Chain, GA, &InGlue, PtrVT, X86::EAX, in LowerToTLSLocalDynamicModel()
18774 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, in LowerToTLSLocalDynamicModel()
18777 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA); in LowerToTLSLocalDynamicModel()
18780 return DAG.getNode(ISD::ADD, dl, PtrVT, Offset, Base); in LowerToTLSLocalDynamicModel()
18784 static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, in LowerToTLSExecModel() argument
18791 PointerType::get(*DAG.getContext(), is64Bit ? 257 : 256)); in LowerToTLSExecModel()
18794 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0, dl), in LowerToTLSExecModel()
18818 DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0), in LowerToTLSExecModel()
18820 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA); in LowerToTLSExecModel()
18824 Offset = DAG.getNode(ISD::ADD, dl, PtrVT, in LowerToTLSExecModel()
18825 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), in LowerToTLSExecModel()
18829 Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset, in LowerToTLSExecModel()
18830 MachinePointerInfo::getGOT(DAG.getMachineFunction())); in LowerToTLSExecModel()
18835 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset); in LowerToTLSExecModel()
18839 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { in LowerGlobalTLSAddress()
18843 if (DAG.getTarget().useEmulatedTLS()) in LowerGlobalTLSAddress()
18844 return LowerToTLSEmulatedModel(GA, DAG); in LowerGlobalTLSAddress()
18847 auto PtrVT = getPointerTy(DAG.getDataLayout()); in LowerGlobalTLSAddress()
18851 TLSModel::Model model = DAG.getTarget().getTLSModel(GV); in LowerGlobalTLSAddress()
18856 return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT); in LowerGlobalTLSAddress()
18857 return LowerToTLSGeneralDynamicModelX32(GA, DAG, PtrVT); in LowerGlobalTLSAddress()
18859 return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT); in LowerGlobalTLSAddress()
18861 return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(), in LowerGlobalTLSAddress()
18865 return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(), in LowerGlobalTLSAddress()
18887 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL, in LowerGlobalTLSAddress()
18890 SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result); in LowerGlobalTLSAddress()
18894 Offset = DAG.getNode(ISD::ADD, DL, PtrVT, in LowerGlobalTLSAddress()
18895 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), in LowerGlobalTLSAddress()
18900 SDValue Chain = DAG.getEntryNode(); in LowerGlobalTLSAddress()
18901 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerGlobalTLSAddress()
18902 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL); in LowerGlobalTLSAddress()
18904 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args); in LowerGlobalTLSAddress()
18905 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, Chain.getValue(1), DL); in LowerGlobalTLSAddress()
18908 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); in LowerGlobalTLSAddress()
18914 return DAG.getCopyFromReg(Chain, DL, Reg, PtrVT, Chain.getValue(1)); in LowerGlobalTLSAddress()
18930 SDValue Chain = DAG.getEntryNode(); in LowerGlobalTLSAddress()
18936 Subtarget.is64Bit() ? PointerType::get(*DAG.getContext(), 256) in LowerGlobalTLSAddress()
18937 : PointerType::get(*DAG.getContext(), 257)); in LowerGlobalTLSAddress()
18940 ? DAG.getIntPtrConstant(0x58, dl) in LowerGlobalTLSAddress()
18942 ? DAG.getIntPtrConstant(0x2C, dl) in LowerGlobalTLSAddress()
18943 : DAG.getExternalSymbol("_tls_array", PtrVT)); in LowerGlobalTLSAddress()
18946 DAG.getLoad(PtrVT, dl, Chain, TlsArray, MachinePointerInfo(Ptr)); in LowerGlobalTLSAddress()
18953 SDValue IDX = DAG.getExternalSymbol("_tls_index", PtrVT); in LowerGlobalTLSAddress()
18955 IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, PtrVT, Chain, IDX, in LowerGlobalTLSAddress()
18958 IDX = DAG.getLoad(PtrVT, dl, Chain, IDX, MachinePointerInfo()); in LowerGlobalTLSAddress()
18960 const DataLayout &DL = DAG.getDataLayout(); in LowerGlobalTLSAddress()
18962 DAG.getConstant(Log2_64_Ceil(DL.getPointerSize()), dl, MVT::i8); in LowerGlobalTLSAddress()
18963 IDX = DAG.getNode(ISD::SHL, dl, PtrVT, IDX, Scale); in LowerGlobalTLSAddress()
18965 res = DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, IDX); in LowerGlobalTLSAddress()
18968 res = DAG.getLoad(PtrVT, dl, Chain, res, MachinePointerInfo()); in LowerGlobalTLSAddress()
18971 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, in LowerGlobalTLSAddress()
18974 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA); in LowerGlobalTLSAddress()
18978 return DAG.getNode(ISD::ADD, dl, PtrVT, res, Offset); in LowerGlobalTLSAddress()
19011 static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) { in LowerShiftParts() argument
19013 DAG.getTargetLoweringInfo().expandShiftParts(Op.getNode(), Lo, Hi, DAG); in LowerShiftParts()
19014 return DAG.getMergeValues({Lo, Hi}, SDLoc(Op)); in LowerShiftParts()
19020 SelectionDAG &DAG, in LowerI64IntToFP_AVX512DQ() argument
19044 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecInVT, Src); in LowerI64IntToFP_AVX512DQ()
19046 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {VecVT, MVT::Other}, in LowerI64IntToFP_AVX512DQ()
19049 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP_AVX512DQ()
19050 DAG.getIntPtrConstant(0, dl)); in LowerI64IntToFP_AVX512DQ()
19051 return DAG.getMergeValues({Value, Chain}, dl); in LowerI64IntToFP_AVX512DQ()
19054 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, VecVT, InVec); in LowerI64IntToFP_AVX512DQ()
19056 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP_AVX512DQ()
19057 DAG.getIntPtrConstant(0, dl)); in LowerI64IntToFP_AVX512DQ()
19061 static SDValue LowerI64IntToFP16(SDValue Op, const SDLoc &dl, SelectionDAG &DAG, in LowerI64IntToFP16() argument
19080 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src); in LowerI64IntToFP16()
19082 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {MVT::v2f16, MVT::Other}, in LowerI64IntToFP16()
19085 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP16()
19086 DAG.getIntPtrConstant(0, dl)); in LowerI64IntToFP16()
19087 return DAG.getMergeValues({Value, Chain}, dl); in LowerI64IntToFP16()
19090 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, MVT::v2f16, InVec); in LowerI64IntToFP16()
19092 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP16()
19093 DAG.getIntPtrConstant(0, dl)); in LowerI64IntToFP16()
19122 SelectionDAG &DAG, in vectorizeExtractedCast() argument
19146 VecOp = DAG.getVectorShuffle(FromVT, DL, VecOp, DAG.getUNDEF(FromVT), Mask); in vectorizeExtractedCast()
19151 VecOp = extract128BitVector(VecOp, 0, DAG, DL); in vectorizeExtractedCast()
19155 SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp); in vectorizeExtractedCast()
19156 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, DestVT, VCast, in vectorizeExtractedCast()
19157 DAG.getIntPtrConstant(0, DL)); in vectorizeExtractedCast()
19164 SelectionDAG &DAG, in lowerFPToIntToFP() argument
19203 SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL); in lowerFPToIntToFP()
19204 SDValue VecX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, X); in lowerFPToIntToFP()
19205 SDValue VCastToInt = DAG.getNode(ToIntOpcode, DL, VecIntVT, VecX); in lowerFPToIntToFP()
19206 SDValue VCastToFP = DAG.getNode(ToFPOpcode, DL, VecVT, VCastToInt); in lowerFPToIntToFP()
19207 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VCastToFP, ZeroIdx); in lowerFPToIntToFP()
19211 SelectionDAG &DAG, in lowerINT_TO_FP_vXi64() argument
19231 SDValue Tmp = IsStrict ? DAG.getConstant(0, DL, MVT::v8i64) in lowerINT_TO_FP_vXi64()
19232 : DAG.getUNDEF(MVT::v8i64); in lowerINT_TO_FP_vXi64()
19233 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i64, Tmp, Src, in lowerINT_TO_FP_vXi64()
19234 DAG.getIntPtrConstant(0, DL)); in lowerINT_TO_FP_vXi64()
19237 Res = DAG.getNode(Op.getOpcode(), DL, {WideVT, MVT::Other}, in lowerINT_TO_FP_vXi64()
19241 Res = DAG.getNode(Op.getOpcode(), DL, WideVT, Src); in lowerINT_TO_FP_vXi64()
19244 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lowerINT_TO_FP_vXi64()
19245 DAG.getIntPtrConstant(0, DL)); in lowerINT_TO_FP_vXi64()
19248 return DAG.getMergeValues({Res, Chain}, DL); in lowerINT_TO_FP_vXi64()
19257 SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64); in lowerINT_TO_FP_vXi64()
19258 SDValue One = DAG.getConstant(1, DL, MVT::v4i64); in lowerINT_TO_FP_vXi64()
19259 SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64, in lowerINT_TO_FP_vXi64()
19260 DAG.getNode(ISD::SRL, DL, MVT::v4i64, Src, One), in lowerINT_TO_FP_vXi64()
19261 DAG.getNode(ISD::AND, DL, MVT::v4i64, Src, One)); in lowerINT_TO_FP_vXi64()
19262 SDValue IsNeg = DAG.getSetCC(DL, MVT::v4i64, Src, Zero, ISD::SETLT); in lowerINT_TO_FP_vXi64()
19263 SDValue SignSrc = DAG.getSelect(DL, MVT::v4i64, IsNeg, Sign, Src); in lowerINT_TO_FP_vXi64()
19267 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, SignSrc, in lowerINT_TO_FP_vXi64()
19268 DAG.getIntPtrConstant(i, DL)); in lowerINT_TO_FP_vXi64()
19271 DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, {MVT::f32, MVT::Other}, in lowerINT_TO_FP_vXi64()
19275 SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, DL, MVT::f32, Elt); in lowerINT_TO_FP_vXi64()
19278 SDValue SignCvt = DAG.getBuildVector(VT, DL, SignCvts); in lowerINT_TO_FP_vXi64()
19282 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in lowerINT_TO_FP_vXi64()
19283 Slow = DAG.getNode(ISD::STRICT_FADD, DL, {MVT::v4f32, MVT::Other}, in lowerINT_TO_FP_vXi64()
19287 Slow = DAG.getNode(ISD::FADD, DL, MVT::v4f32, SignCvt, SignCvt); in lowerINT_TO_FP_vXi64()
19290 IsNeg = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i32, IsNeg); in lowerINT_TO_FP_vXi64()
19291 SDValue Cvt = DAG.getSelect(DL, MVT::v4f32, IsNeg, Slow, SignCvt); in lowerINT_TO_FP_vXi64()
19294 return DAG.getMergeValues({Cvt, Chain}, DL); in lowerINT_TO_FP_vXi64()
19300 SelectionDAG &DAG) { in promoteXINT_TO_FP() argument
19303 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode(); in promoteXINT_TO_FP()
19307 SDValue Rnd = DAG.getIntPtrConstant(0, dl); in promoteXINT_TO_FP()
19309 return DAG.getNode( in promoteXINT_TO_FP()
19312 DAG.getNode(Op.getOpcode(), dl, {NVT, MVT::Other}, {Chain, Src}), in promoteXINT_TO_FP()
19314 return DAG.getNode(ISD::FP_ROUND, dl, VT, in promoteXINT_TO_FP()
19315 DAG.getNode(Op.getOpcode(), dl, NVT, Src), Rnd); in promoteXINT_TO_FP()
19339 SelectionDAG &DAG) const { in LowerSINT_TO_FP()
19343 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode(); in LowerSINT_TO_FP()
19349 return promoteXINT_TO_FP(Op, dl, DAG); in LowerSINT_TO_FP()
19354 return LowerWin64_INT128_TO_FP(Op, DAG); in LowerSINT_TO_FP()
19356 if (SDValue Extract = vectorizeExtractedCast(Op, dl, DAG, Subtarget)) in LowerSINT_TO_FP()
19359 if (SDValue R = lowerFPToIntToFP(Op, dl, DAG, Subtarget)) in LowerSINT_TO_FP()
19367 return DAG.getNode( in LowerSINT_TO_FP()
19369 {Chain, DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, in LowerSINT_TO_FP()
19370 DAG.getUNDEF(SrcVT))}); in LowerSINT_TO_FP()
19371 return DAG.getNode(X86ISD::CVTSI2P, dl, VT, in LowerSINT_TO_FP()
19372 DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, in LowerSINT_TO_FP()
19373 DAG.getUNDEF(SrcVT))); in LowerSINT_TO_FP()
19376 return lowerINT_TO_FP_vXi64(Op, dl, DAG, Subtarget); in LowerSINT_TO_FP()
19393 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, dl, DAG, Subtarget)) in LowerSINT_TO_FP()
19395 if (SDValue V = LowerI64IntToFP16(Op, dl, DAG, Subtarget)) in LowerSINT_TO_FP()
19400 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, Src); in LowerSINT_TO_FP()
19402 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in LowerSINT_TO_FP()
19405 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Ext); in LowerSINT_TO_FP()
19416 ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore); in LowerSINT_TO_FP()
19420 MachineFunction &MF = DAG.getMachineFunction(); in LowerSINT_TO_FP()
19424 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI); in LowerSINT_TO_FP()
19425 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); in LowerSINT_TO_FP()
19426 Chain = DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, Alignment); in LowerSINT_TO_FP()
19428 BuildFILD(VT, SrcVT, dl, Chain, StackSlot, MPI, Alignment, DAG); in LowerSINT_TO_FP()
19431 return DAG.getMergeValues({Tmp.first, Tmp.second}, dl); in LowerSINT_TO_FP()
19438 MachinePointerInfo PtrInfo, Align Alignment, SelectionDAG &DAG) const { in BuildFILD()
19443 Tys = DAG.getVTList(MVT::f80, MVT::Other); in BuildFILD()
19445 Tys = DAG.getVTList(DstVT, MVT::Other); in BuildFILD()
19449 DAG.getMemIntrinsicNode(X86ISD::FILD, DL, Tys, FILDOps, SrcVT, PtrInfo, in BuildFILD()
19454 MachineFunction &MF = DAG.getMachineFunction(); in BuildFILD()
19459 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); in BuildFILD()
19460 Tys = DAG.getVTList(MVT::Other); in BuildFILD()
19462 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand( in BuildFILD()
19463 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI), in BuildFILD()
19467 DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys, FSTOps, DstVT, StoreMMO); in BuildFILD()
19468 Result = DAG.getLoad( in BuildFILD()
19470 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI)); in BuildFILD()
19480 static bool shouldUseHorizontalOp(bool IsSingleSource, SelectionDAG &DAG, in shouldUseHorizontalOp() argument
19482 bool IsOptimizingSize = DAG.shouldOptForSize(); in shouldUseHorizontalOp()
19489 SelectionDAG &DAG, in LowerUINT_TO_FP_i64() argument
19508 LLVMContext *Context = DAG.getContext(); in LowerUINT_TO_FP_i64()
19513 auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); in LowerUINT_TO_FP_i64()
19514 SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, Align(16)); in LowerUINT_TO_FP_i64()
19524 SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, Align(16)); in LowerUINT_TO_FP_i64()
19528 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Op.getOperand(0)); in LowerUINT_TO_FP_i64()
19529 SDValue CLod0 = DAG.getLoad( in LowerUINT_TO_FP_i64()
19530 MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0, in LowerUINT_TO_FP_i64()
19531 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16)); in LowerUINT_TO_FP_i64()
19533 getUnpackl(DAG, dl, MVT::v4i32, DAG.getBitcast(MVT::v4i32, XR1), CLod0); in LowerUINT_TO_FP_i64()
19535 SDValue CLod1 = DAG.getLoad( in LowerUINT_TO_FP_i64()
19537 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16)); in LowerUINT_TO_FP_i64()
19538 SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1); in LowerUINT_TO_FP_i64()
19540 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); in LowerUINT_TO_FP_i64()
19544 shouldUseHorizontalOp(true, DAG, Subtarget)) { in LowerUINT_TO_FP_i64()
19545 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub); in LowerUINT_TO_FP_i64()
19547 SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1}); in LowerUINT_TO_FP_i64()
19548 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub); in LowerUINT_TO_FP_i64()
19550 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result, in LowerUINT_TO_FP_i64()
19551 DAG.getIntPtrConstant(0, dl)); in LowerUINT_TO_FP_i64()
19557 SelectionDAG &DAG, in LowerUINT_TO_FP_i32() argument
19561 SDValue Bias = DAG.getConstantFP( in LowerUINT_TO_FP_i32()
19566 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Op.getOperand(OpNo)); in LowerUINT_TO_FP_i32()
19569 Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget, DAG); in LowerUINT_TO_FP_i32()
19572 SDValue Or = DAG.getNode( in LowerUINT_TO_FP_i32()
19574 DAG.getBitcast(MVT::v2i64, Load), in LowerUINT_TO_FP_i32()
19575 DAG.getBitcast(MVT::v2i64, in LowerUINT_TO_FP_i32()
19576 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias))); in LowerUINT_TO_FP_i32()
19578 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
19579 DAG.getBitcast(MVT::v2f64, Or), DAG.getIntPtrConstant(0, dl)); in LowerUINT_TO_FP_i32()
19585 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other}, in LowerUINT_TO_FP_i32()
19592 std::pair<SDValue, SDValue> ResultPair = DAG.getStrictFPExtendOrRound( in LowerUINT_TO_FP_i32()
19595 return DAG.getMergeValues({ResultPair.first, ResultPair.second}, dl); in LowerUINT_TO_FP_i32()
19600 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias); in LowerUINT_TO_FP_i32()
19603 return DAG.getFPExtendOrRound(Sub, dl, Op.getSimpleValueType()); in LowerUINT_TO_FP_i32()
19607 SelectionDAG &DAG, in lowerUINT_TO_FP_v2i32() argument
19623 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0, in lowerUINT_TO_FP_v2i32()
19624 DAG.getConstant(0, DL, MVT::v2i32)); in lowerUINT_TO_FP_v2i32()
19625 SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other}, in lowerUINT_TO_FP_v2i32()
19628 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2f64, Res, in lowerUINT_TO_FP_v2i32()
19629 DAG.getIntPtrConstant(0, DL)); in lowerUINT_TO_FP_v2i32()
19630 return DAG.getMergeValues({Res, Chain}, DL); in lowerUINT_TO_FP_v2i32()
19634 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0, in lowerUINT_TO_FP_v2i32()
19635 DAG.getUNDEF(MVT::v2i32)); in lowerUINT_TO_FP_v2i32()
19637 return DAG.getNode(X86ISD::STRICT_CVTUI2P, DL, {MVT::v2f64, MVT::Other}, in lowerUINT_TO_FP_v2i32()
19639 return DAG.getNode(X86ISD::CVTUI2P, DL, MVT::v2f64, N0); in lowerUINT_TO_FP_v2i32()
19646 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v2i64, N0); in lowerUINT_TO_FP_v2i32()
19647 SDValue VBias = DAG.getConstantFP( in lowerUINT_TO_FP_v2i32()
19649 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v2i64, ZExtIn, in lowerUINT_TO_FP_v2i32()
19650 DAG.getBitcast(MVT::v2i64, VBias)); in lowerUINT_TO_FP_v2i32()
19651 Or = DAG.getBitcast(MVT::v2f64, Or); in lowerUINT_TO_FP_v2i32()
19654 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v2f64, MVT::Other}, in lowerUINT_TO_FP_v2i32()
19656 return DAG.getNode(ISD::FSUB, DL, MVT::v2f64, Or, VBias); in lowerUINT_TO_FP_v2i32()
19660 SelectionDAG &DAG, in lowerUINT_TO_FP_vXi32() argument
19684 IsStrict ? DAG.getConstant(0, DL, WideIntVT) : DAG.getUNDEF(WideIntVT); in lowerUINT_TO_FP_vXi32()
19685 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideIntVT, Tmp, V, in lowerUINT_TO_FP_vXi32()
19686 DAG.getIntPtrConstant(0, DL)); in lowerUINT_TO_FP_vXi32()
19689 Res = DAG.getNode(ISD::STRICT_UINT_TO_FP, DL, {WideVT, MVT::Other}, in lowerUINT_TO_FP_vXi32()
19693 Res = DAG.getNode(ISD::UINT_TO_FP, DL, WideVT, V); in lowerUINT_TO_FP_vXi32()
19696 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lowerUINT_TO_FP_vXi32()
19697 DAG.getIntPtrConstant(0, DL)); in lowerUINT_TO_FP_vXi32()
19700 return DAG.getMergeValues({Res, Chain}, DL); in lowerUINT_TO_FP_vXi32()
19706 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i64, V); in lowerUINT_TO_FP_vXi32()
19708 *DAG.getContext(), in lowerUINT_TO_FP_vXi32()
19710 auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); in lowerUINT_TO_FP_vXi32()
19711 SDValue CPIdx = DAG.getConstantPool(Bias, PtrVT, Align(8)); in lowerUINT_TO_FP_vXi32()
19712 SDVTList Tys = DAG.getVTList(MVT::v4f64, MVT::Other); in lowerUINT_TO_FP_vXi32()
19713 SDValue Ops[] = {DAG.getEntryNode(), CPIdx}; in lowerUINT_TO_FP_vXi32()
19714 SDValue VBias = DAG.getMemIntrinsicNode( in lowerUINT_TO_FP_vXi32()
19716 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(8), in lowerUINT_TO_FP_vXi32()
19719 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn, in lowerUINT_TO_FP_vXi32()
19720 DAG.getBitcast(MVT::v4i64, VBias)); in lowerUINT_TO_FP_vXi32()
19721 Or = DAG.getBitcast(MVT::v4f64, Or); in lowerUINT_TO_FP_vXi32()
19724 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v4f64, MVT::Other}, in lowerUINT_TO_FP_vXi32()
19726 return DAG.getNode(ISD::FSUB, DL, MVT::v4f64, Or, VBias); in lowerUINT_TO_FP_vXi32()
19756 SDValue VecCstLow = DAG.getConstant(0x4b000000, DL, VecIntVT); in lowerUINT_TO_FP_vXi32()
19758 SDValue VecCstHigh = DAG.getConstant(0x53000000, DL, VecIntVT); in lowerUINT_TO_FP_vXi32()
19761 SDValue VecCstShift = DAG.getConstant(16, DL, VecIntVT); in lowerUINT_TO_FP_vXi32()
19762 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift); in lowerUINT_TO_FP_vXi32()
19768 SDValue VecCstLowBitcast = DAG.getBitcast(VecI16VT, VecCstLow); in lowerUINT_TO_FP_vXi32()
19769 SDValue VecBitcast = DAG.getBitcast(VecI16VT, V); in lowerUINT_TO_FP_vXi32()
19772 Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast, in lowerUINT_TO_FP_vXi32()
19773 VecCstLowBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8)); in lowerUINT_TO_FP_vXi32()
19776 SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh); in lowerUINT_TO_FP_vXi32()
19777 SDValue VecShiftBitcast = DAG.getBitcast(VecI16VT, HighShift); in lowerUINT_TO_FP_vXi32()
19780 High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast, in lowerUINT_TO_FP_vXi32()
19781 VecCstHighBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8)); in lowerUINT_TO_FP_vXi32()
19783 SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT); in lowerUINT_TO_FP_vXi32()
19785 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask); in lowerUINT_TO_FP_vXi32()
19786 Low = DAG.getNode(ISD::OR, DL, VecIntVT, LowAnd, VecCstLow); in lowerUINT_TO_FP_vXi32()
19789 High = DAG.getNode(ISD::OR, DL, VecIntVT, HighShift, VecCstHigh); in lowerUINT_TO_FP_vXi32()
19793 SDValue VecCstFSub = DAG.getConstantFP( in lowerUINT_TO_FP_vXi32()
19800 SDValue HighBitcast = DAG.getBitcast(VecFloatVT, High); in lowerUINT_TO_FP_vXi32()
19803 SDValue LowBitcast = DAG.getBitcast(VecFloatVT, Low); in lowerUINT_TO_FP_vXi32()
19806 SDValue FHigh = DAG.getNode(ISD::STRICT_FSUB, DL, {VecFloatVT, MVT::Other}, in lowerUINT_TO_FP_vXi32()
19808 return DAG.getNode(ISD::STRICT_FADD, DL, {VecFloatVT, MVT::Other}, in lowerUINT_TO_FP_vXi32()
19813 DAG.getNode(ISD::FSUB, DL, VecFloatVT, HighBitcast, VecCstFSub); in lowerUINT_TO_FP_vXi32()
19814 return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh); in lowerUINT_TO_FP_vXi32()
19817 static SDValue lowerUINT_TO_FP_vec(SDValue Op, const SDLoc &dl, SelectionDAG &DAG, in lowerUINT_TO_FP_vec() argument
19827 return lowerUINT_TO_FP_v2i32(Op, dl, DAG, Subtarget); in lowerUINT_TO_FP_vec()
19830 return lowerUINT_TO_FP_vXi32(Op, dl, DAG, Subtarget); in lowerUINT_TO_FP_vec()
19833 return lowerINT_TO_FP_vXi64(Op, dl, DAG, Subtarget); in lowerUINT_TO_FP_vec()
19838 SelectionDAG &DAG) const { in LowerUINT_TO_FP()
19843 auto PtrVT = getPointerTy(DAG.getDataLayout()); in LowerUINT_TO_FP()
19846 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode(); in LowerUINT_TO_FP()
19853 return promoteXINT_TO_FP(Op, dl, DAG); in LowerUINT_TO_FP()
19858 return lowerUINT_TO_FP_vec(Op, dl, DAG, Subtarget); in LowerUINT_TO_FP()
19861 return LowerWin64_INT128_TO_FP(Op, DAG); in LowerUINT_TO_FP()
19863 if (SDValue Extract = vectorizeExtractedCast(Op, dl, DAG, Subtarget)) in LowerUINT_TO_FP()
19875 Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Src); in LowerUINT_TO_FP()
19877 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {DstVT, MVT::Other}, in LowerUINT_TO_FP()
19879 return DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Src); in LowerUINT_TO_FP()
19882 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, dl, DAG, Subtarget)) in LowerUINT_TO_FP()
19884 if (SDValue V = LowerI64IntToFP16(Op, dl, DAG, Subtarget)) in LowerUINT_TO_FP()
19891 return LowerUINT_TO_FP_i64(Op, dl, DAG, Subtarget); in LowerUINT_TO_FP()
19896 return LowerUINT_TO_FP_i32(Op, dl, DAG, Subtarget); in LowerUINT_TO_FP()
19902 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64, 8); in LowerUINT_TO_FP()
19906 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI); in LowerUINT_TO_FP()
19909 DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl); in LowerUINT_TO_FP()
19910 SDValue Store1 = DAG.getStore(Chain, dl, Src, StackSlot, MPI, SlotAlign); in LowerUINT_TO_FP()
19911 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32), in LowerUINT_TO_FP()
19914 BuildFILD(DstVT, MVT::i64, dl, Store2, StackSlot, MPI, SlotAlign, DAG); in LowerUINT_TO_FP()
19916 return DAG.getMergeValues({Tmp.first, Tmp.second}, dl); in LowerUINT_TO_FP()
19927 ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore); in LowerUINT_TO_FP()
19930 DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, SlotAlign); in LowerUINT_TO_FP()
19934 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); in LowerUINT_TO_FP()
19937 DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, MVT::i64, MPI, in LowerUINT_TO_FP()
19942 SDValue SignSet = DAG.getSetCC( in LowerUINT_TO_FP()
19943 dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64), in LowerUINT_TO_FP()
19944 Op.getOperand(OpNo), DAG.getConstant(0, dl, MVT::i64), ISD::SETLT); in LowerUINT_TO_FP()
19949 DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF), PtrVT); in LowerUINT_TO_FP()
19953 SDValue Zero = DAG.getIntPtrConstant(0, dl); in LowerUINT_TO_FP()
19954 SDValue Four = DAG.getIntPtrConstant(4, dl); in LowerUINT_TO_FP()
19955 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, Four, Zero); in LowerUINT_TO_FP()
19956 FudgePtr = DAG.getNode(ISD::ADD, dl, PtrVT, FudgePtr, Offset); in LowerUINT_TO_FP()
19959 SDValue Fudge = DAG.getExtLoad( in LowerUINT_TO_FP()
19961 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, in LowerUINT_TO_FP()
19973 DAG.getNode(Opc, dl, {MVT::f80, MVT::Other}, {Chain, Fild, Fudge}); in LowerUINT_TO_FP()
19977 return DAG.getNode(ISD::STRICT_FP_ROUND, dl, {DstVT, MVT::Other}, in LowerUINT_TO_FP()
19978 {Add.getValue(1), Add, DAG.getIntPtrConstant(0, dl)}); in LowerUINT_TO_FP()
19985 SDValue Add = DAG.getNode(Opc, dl, MVT::f80, Fild, Fudge); in LowerUINT_TO_FP()
19986 return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add, in LowerUINT_TO_FP()
19987 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)); in LowerUINT_TO_FP()
19996 SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, in FP_TO_INTHelper() argument
20005 auto PtrVT = getPointerTy(DAG.getDataLayout()); in FP_TO_INTHelper()
20033 MachineFunction &MF = DAG.getMachineFunction(); in FP_TO_INTHelper()
20037 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); in FP_TO_INTHelper()
20039 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode(); in FP_TO_INTHelper()
20075 SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT); in FP_TO_INTHelper()
20077 EVT ResVT = getSetCCResultType(DAG.getDataLayout(), in FP_TO_INTHelper()
20078 *DAG.getContext(), TheVT); in FP_TO_INTHelper()
20081 Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE, Chain, in FP_TO_INTHelper()
20085 Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE); in FP_TO_INTHelper()
20099 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Cmp); in FP_TO_INTHelper()
20100 SDValue Const63 = DAG.getConstant(63, DL, MVT::i8); in FP_TO_INTHelper()
20101 Adjust = DAG.getNode(ISD::SHL, DL, MVT::i64, Zext, Const63); in FP_TO_INTHelper()
20103 SDValue FltOfs = DAG.getSelect(DL, TheVT, Cmp, ThreshVal, in FP_TO_INTHelper()
20104 DAG.getConstantFP(0.0, DL, TheVT)); in FP_TO_INTHelper()
20107 Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other}, in FP_TO_INTHelper()
20111 Value = DAG.getNode(ISD::FSUB, DL, TheVT, Value, FltOfs); in FP_TO_INTHelper()
20120 Chain = DAG.getStore(Chain, DL, Value, StackSlot, MPI); in FP_TO_INTHelper()
20121 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); in FP_TO_INTHelper()
20128 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, TheVT, MMO); in FP_TO_INTHelper()
20136 SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL, in FP_TO_INTHelper()
20137 DAG.getVTList(MVT::Other), in FP_TO_INTHelper()
20140 SDValue Res = DAG.getLoad(Op.getValueType(), DL, FIST, StackSlot, MPI); in FP_TO_INTHelper()
20145 Res = DAG.getNode(ISD::XOR, DL, MVT::i64, Res, Adjust); in FP_TO_INTHelper()
20150 static SDValue LowerAVXExtend(SDValue Op, const SDLoc &dl, SelectionDAG &DAG, in LowerAVXExtend() argument
20171 unsigned ExtendInVecOpc = DAG.getOpcode_EXTEND_VECTOR_INREG(Opc); in LowerAVXExtend()
20175 return splitVectorIntUnary(Op, DAG, dl); in LowerAVXExtend()
20194 SDValue OpLo = DAG.getNode(ExtendInVecOpc, dl, HalfVT, In); in LowerAVXExtend()
20200 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpLo); in LowerAVXExtend()
20202 SDValue ZeroVec = DAG.getConstant(0, dl, InVT); in LowerAVXExtend()
20203 SDValue Undef = DAG.getUNDEF(InVT); in LowerAVXExtend()
20205 SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef); in LowerAVXExtend()
20206 OpHi = DAG.getBitcast(HalfVT, OpHi); in LowerAVXExtend()
20208 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); in LowerAVXExtend()
20213 const SDLoc &dl, SelectionDAG &DAG) { in SplitAndExtendv16i1() argument
20215 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In, in SplitAndExtendv16i1()
20216 DAG.getIntPtrConstant(0, dl)); in SplitAndExtendv16i1()
20217 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In, in SplitAndExtendv16i1()
20218 DAG.getIntPtrConstant(8, dl)); in SplitAndExtendv16i1()
20219 Lo = DAG.getNode(ExtOpc, dl, MVT::v8i16, Lo); in SplitAndExtendv16i1()
20220 Hi = DAG.getNode(ExtOpc, dl, MVT::v8i16, Hi); in SplitAndExtendv16i1()
20221 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi); in SplitAndExtendv16i1()
20222 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in SplitAndExtendv16i1()
20227 SelectionDAG &DAG) { in LowerZERO_EXTEND_Mask() argument
20237 SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In); in LowerZERO_EXTEND_Mask()
20238 return DAG.getNode(ISD::SRL, DL, VT, Extend, in LowerZERO_EXTEND_Mask()
20239 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT)); in LowerZERO_EXTEND_Mask()
20247 return SplitAndExtendv16i1(ISD::ZERO_EXTEND, VT, In, DL, DAG); in LowerZERO_EXTEND_Mask()
20257 In = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT, DAG.getUNDEF(InVT), in LowerZERO_EXTEND_Mask()
20258 In, DAG.getIntPtrConstant(0, DL)); in LowerZERO_EXTEND_Mask()
20263 SDValue One = DAG.getConstant(1, DL, WideVT); in LowerZERO_EXTEND_Mask()
20264 SDValue Zero = DAG.getConstant(0, DL, WideVT); in LowerZERO_EXTEND_Mask()
20266 SDValue SelectedVal = DAG.getSelect(DL, WideVT, In, One, Zero); in LowerZERO_EXTEND_Mask()
20271 SelectedVal = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SelectedVal); in LowerZERO_EXTEND_Mask()
20276 SelectedVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SelectedVal, in LowerZERO_EXTEND_Mask()
20277 DAG.getIntPtrConstant(0, DL)); in LowerZERO_EXTEND_Mask()
20283 SelectionDAG &DAG) { in LowerZERO_EXTEND() argument
20289 return LowerZERO_EXTEND_Mask(Op, DL, Subtarget, DAG); in LowerZERO_EXTEND()
20292 return LowerAVXExtend(Op, DL, DAG, Subtarget); in LowerZERO_EXTEND()
20301 const SDLoc &DL, SelectionDAG &DAG, in truncateVectorWithPACK() argument
20326 LLVMContext &Ctx = *DAG.getContext(); in truncateVectorWithPACK()
20344 In = widenSubVector(In, false, Subtarget, DAG, DL, 128); in truncateVectorWithPACK()
20345 SDValue LHS = DAG.getBitcast(InVT, In); in truncateVectorWithPACK()
20346 SDValue RHS = Subtarget.hasAVX512() ? DAG.getUNDEF(InVT) : LHS; in truncateVectorWithPACK()
20347 SDValue Res = DAG.getNode(Opcode, DL, OutVT, LHS, RHS); in truncateVectorWithPACK()
20348 Res = extractSubVector(Res, 0, DAG, DL, SrcSizeInBits / 2); in truncateVectorWithPACK()
20349 Res = DAG.getBitcast(PackedVT, Res); in truncateVectorWithPACK()
20350 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget); in truncateVectorWithPACK()
20355 std::tie(Lo, Hi) = splitVector(In, DAG, DL); in truncateVectorWithPACK()
20361 truncateVectorWithPACK(Opcode, DstHalfVT, Lo, DL, DAG, Subtarget)) in truncateVectorWithPACK()
20362 return widenSubVector(Res, false, Subtarget, DAG, DL, DstSizeInBits); in truncateVectorWithPACK()
20371 Lo = DAG.getBitcast(InVT, Lo); in truncateVectorWithPACK()
20372 Hi = DAG.getBitcast(InVT, Hi); in truncateVectorWithPACK()
20373 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi); in truncateVectorWithPACK()
20374 return DAG.getBitcast(DstVT, Res); in truncateVectorWithPACK()
20380 Lo = DAG.getBitcast(InVT, Lo); in truncateVectorWithPACK()
20381 Hi = DAG.getBitcast(InVT, Hi); in truncateVectorWithPACK()
20382 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi); in truncateVectorWithPACK()
20390 Res = DAG.getVectorShuffle(OutVT, DL, Res, Res, Mask); in truncateVectorWithPACK()
20393 return DAG.getBitcast(DstVT, Res); in truncateVectorWithPACK()
20396 Res = DAG.getBitcast(PackedVT, Res); in truncateVectorWithPACK()
20397 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget); in truncateVectorWithPACK()
20407 truncateVectorWithPACK(Opcode, PackedVT, In, DL, DAG, Subtarget); in truncateVectorWithPACK()
20408 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget); in truncateVectorWithPACK()
20412 Lo = truncateVectorWithPACK(Opcode, HalfPackedVT, Lo, DL, DAG, Subtarget); in truncateVectorWithPACK()
20413 Hi = truncateVectorWithPACK(Opcode, HalfPackedVT, Hi, DL, DAG, Subtarget); in truncateVectorWithPACK()
20414 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi); in truncateVectorWithPACK()
20415 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget); in truncateVectorWithPACK()
20424 SelectionDAG &DAG) { in truncateVectorWithPACKUS() argument
20425 In = DAG.getZeroExtendInReg(In, DL, DstVT); in truncateVectorWithPACKUS()
20426 return truncateVectorWithPACK(X86ISD::PACKUS, DstVT, In, DL, DAG, Subtarget); in truncateVectorWithPACKUS()
20432 SelectionDAG &DAG) { in truncateVectorWithPACKSS() argument
20434 In = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SrcVT, In, in truncateVectorWithPACKSS()
20435 DAG.getValueType(DstVT)); in truncateVectorWithPACKSS()
20436 return truncateVectorWithPACK(X86ISD::PACKSS, DstVT, In, DL, DAG, Subtarget); in truncateVectorWithPACKSS()
20444 SelectionDAG &DAG, in matchTruncateWithPACK() argument
20475 !isFreeToSplitVector(In.getNode(), DAG) && in matchTruncateWithPACK()
20476 (!Subtarget.hasAVX() || DAG.ComputeNumSignBits(In) != 64)) in matchTruncateWithPACK()
20490 KnownBits Known = DAG.computeKnownBits(In); in matchTruncateWithPACK()
20499 unsigned NumSignBits = DAG.ComputeNumSignBits(In); in matchTruncateWithPACK()
20519 if (std::optional<uint64_t> ShAmt = DAG.getValidShiftAmount(In)) { in matchTruncateWithPACK()
20522 return DAG.getNode(ISD::SRA, DL, SrcVT, In->ops()); in matchTruncateWithPACK()
20535 SelectionDAG &DAG) { in LowerTruncateVecPackWithSignBits() argument
20547 if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) { in LowerTruncateVecPackWithSignBits()
20550 Subtarget, DAG)) in LowerTruncateVecPackWithSignBits()
20551 return widenSubVector(Res, false, Subtarget, DAG, DL, in LowerTruncateVecPackWithSignBits()
20558 matchTruncateWithPACK(PackOpcode, DstVT, In, DL, DAG, Subtarget)) in LowerTruncateVecPackWithSignBits()
20559 return truncateVectorWithPACK(PackOpcode, DstVT, Src, DL, DAG, Subtarget); in LowerTruncateVecPackWithSignBits()
20568 SelectionDAG &DAG) { in LowerTruncateVecPack() argument
20590 if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) { in LowerTruncateVecPack()
20592 if (SDValue Res = LowerTruncateVecPack(DstHalfVT, Lo, DL, Subtarget, DAG)) in LowerTruncateVecPack()
20593 return widenSubVector(Res, false, Subtarget, DAG, DL, in LowerTruncateVecPack()
20602 return truncateVectorWithPACKUS(DstVT, In, DL, Subtarget, DAG); in LowerTruncateVecPack()
20605 return truncateVectorWithPACKSS(DstVT, In, DL, Subtarget, DAG); in LowerTruncateVecPack()
20610 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, In); in LowerTruncateVecPack()
20611 return truncateVectorWithPACKSS(DstVT, Trunc, DL, Subtarget, DAG); in LowerTruncateVecPack()
20618 SelectionDAG &DAG, in LowerTruncateVecI1() argument
20630 if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) { in LowerTruncateVecI1()
20634 In = DAG.getNode(ISD::SHL, DL, ExtVT, in LowerTruncateVecI1()
20635 DAG.getBitcast(ExtVT, In), in LowerTruncateVecI1()
20636 DAG.getConstant(ShiftInx, DL, ExtVT)); in LowerTruncateVecI1()
20637 In = DAG.getBitcast(InVT, In); in LowerTruncateVecI1()
20639 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), in LowerTruncateVecI1()
20659 Lo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, In); in LowerTruncateVecI1()
20660 Hi = DAG.getVectorShuffle( in LowerTruncateVecI1()
20663 Hi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, Hi); in LowerTruncateVecI1()
20666 Lo = extract128BitVector(In, 0, DAG, DL); in LowerTruncateVecI1()
20667 Hi = extract128BitVector(In, 8, DAG, DL); in LowerTruncateVecI1()
20671 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Lo); in LowerTruncateVecI1()
20672 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Hi); in LowerTruncateVecI1()
20673 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in LowerTruncateVecI1()
20680 In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In); in LowerTruncateVecI1()
20685 if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) { in LowerTruncateVecI1()
20687 In = DAG.getNode(ISD::SHL, DL, InVT, In, in LowerTruncateVecI1()
20688 DAG.getConstant(ShiftInx, DL, InVT)); in LowerTruncateVecI1()
20692 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT); in LowerTruncateVecI1()
20693 return DAG.getSetCC(DL, VT, In, DAG.getConstant(0, DL, InVT), ISD::SETNE); in LowerTruncateVecI1()
20696 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { in LowerTRUNCATE()
20705 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerTRUNCATE()
20715 std::tie(Lo, Hi) = DAG.SplitVector(In, DL); in LowerTRUNCATE()
20718 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in LowerTRUNCATE()
20720 Lo = DAG.getNode(ISD::TRUNCATE, DL, LoVT, Lo); in LowerTRUNCATE()
20721 Hi = DAG.getNode(ISD::TRUNCATE, DL, HiVT, Hi); in LowerTRUNCATE()
20722 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in LowerTRUNCATE()
20729 LowerTruncateVecPackWithSignBits(VT, In, DL, Subtarget, DAG)) in LowerTRUNCATE()
20734 return LowerTruncateVecPack(VT, In, DL, Subtarget, DAG); in LowerTRUNCATE()
20741 return LowerTruncateVecI1(Op, DL, DAG, Subtarget); in LowerTRUNCATE()
20745 if (!Subtarget.hasAVX512() || isFreeToSplitVector(In.getNode(), DAG)) in LowerTRUNCATE()
20747 LowerTruncateVecPackWithSignBits(VT, In, DL, Subtarget, DAG)) in LowerTRUNCATE()
20754 return splitVectorIntUnary(Op, DAG, DL); in LowerTRUNCATE()
20773 In = DAG.getBitcast(MVT::v8i32, In); in LowerTRUNCATE()
20774 In = DAG.getVectorShuffle(MVT::v8i32, DL, In, In, ShufMask); in LowerTRUNCATE()
20775 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In, in LowerTRUNCATE()
20776 DAG.getIntPtrConstant(0, DL)); in LowerTRUNCATE()
20779 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
20780 DAG.getIntPtrConstant(0, DL)); in LowerTRUNCATE()
20781 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
20782 DAG.getIntPtrConstant(2, DL)); in LowerTRUNCATE()
20784 return DAG.getVectorShuffle(VT, DL, DAG.getBitcast(MVT::v4i32, OpLo), in LowerTRUNCATE()
20785 DAG.getBitcast(MVT::v4i32, OpHi), ShufMask); in LowerTRUNCATE()
20796 In = DAG.getBitcast(MVT::v32i8, In); in LowerTRUNCATE()
20797 In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1); in LowerTRUNCATE()
20798 In = DAG.getBitcast(MVT::v4i64, In); in LowerTRUNCATE()
20801 In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2); in LowerTRUNCATE()
20802 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
20803 DAG.getIntPtrConstant(0, DL)); in LowerTRUNCATE()
20804 return DAG.getBitcast(MVT::v8i16, In); in LowerTRUNCATE()
20808 ? truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG) in LowerTRUNCATE()
20809 : truncateVectorWithPACKSS(VT, In, DL, Subtarget, DAG); in LowerTRUNCATE()
20813 return truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG); in LowerTRUNCATE()
20821 SelectionDAG &DAG, in expandFP_TO_UINT_SSE() argument
20829 SDValue Small = DAG.getNode(X86ISD::CVTTP2SI, dl, VT, Src); in expandFP_TO_UINT_SSE()
20831 DAG.getNode(X86ISD::CVTTP2SI, dl, VT, in expandFP_TO_UINT_SSE()
20832 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, in expandFP_TO_UINT_SSE()
20833 DAG.getConstantFP(2147483648.0f, dl, SrcVT))); in expandFP_TO_UINT_SSE()
20845 SDValue Overflow = DAG.getNode(ISD::OR, dl, VT, Small, Big); in expandFP_TO_UINT_SSE()
20846 return DAG.getNode(X86ISD::BLENDV, dl, VT, Small, Overflow, Small); in expandFP_TO_UINT_SSE()
20850 DAG.getNode(X86ISD::VSRAI, dl, VT, Small, in expandFP_TO_UINT_SSE()
20851 DAG.getTargetConstant(DstBits - 1, dl, MVT::i8)); in expandFP_TO_UINT_SSE()
20852 return DAG.getNode(ISD::OR, dl, VT, Small, in expandFP_TO_UINT_SSE()
20853 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown)); in expandFP_TO_UINT_SSE()
20856 SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const { in LowerFP_TO_INT()
20870 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other}, in LowerFP_TO_INT()
20871 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl, in LowerFP_TO_INT()
20873 return DAG.getNode(Op.getOpcode(), dl, VT, in LowerFP_TO_INT()
20874 DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src)); in LowerFP_TO_INT()
20898 SDValue Tmp = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v8f64) in LowerFP_TO_INT()
20899 : DAG.getUNDEF(MVT::v8f64); in LowerFP_TO_INT()
20900 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8f64, Tmp, Src, in LowerFP_TO_INT()
20901 DAG.getIntPtrConstant(0, dl)); in LowerFP_TO_INT()
20904 Res = DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {Chain, Src}); in LowerFP_TO_INT()
20907 Res = DAG.getNode(Opc, dl, ResVT, Src); in LowerFP_TO_INT()
20910 Res = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Res); in LowerFP_TO_INT()
20911 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i1, Res, in LowerFP_TO_INT()
20912 DAG.getIntPtrConstant(0, dl)); in LowerFP_TO_INT()
20914 return DAG.getMergeValues({Res, Chain}, dl); in LowerFP_TO_INT()
20929 IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT); in LowerFP_TO_INT()
20932 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops); in LowerFP_TO_INT()
20936 Res = DAG.getNode(IsSigned ? X86ISD::STRICT_CVTTP2SI in LowerFP_TO_INT()
20941 Res = DAG.getNode(IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI, dl, in LowerFP_TO_INT()
20948 Res = DAG.getNode(ISD::TRUNCATE, dl, ResVT, Res); in LowerFP_TO_INT()
20952 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
20953 DAG.getIntPtrConstant(0, dl)); in LowerFP_TO_INT()
20956 return DAG.getMergeValues({Res, Chain}, dl); in LowerFP_TO_INT()
20967 Res = DAG.getNode(IsSigned ? ISD::STRICT_FP_TO_SINT in LowerFP_TO_INT()
20972 Res = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, in LowerFP_TO_INT()
20977 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerFP_TO_INT()
20980 return DAG.getMergeValues({Res, Chain}, dl); in LowerFP_TO_INT()
21003 IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT); in LowerFP_TO_INT()
21004 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src, in LowerFP_TO_INT()
21005 DAG.getIntPtrConstant(0, dl)); in LowerFP_TO_INT()
21008 Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, dl, {ResVT, MVT::Other}, in LowerFP_TO_INT()
21012 Res = DAG.getNode(ISD::FP_TO_UINT, dl, ResVT, Src); in LowerFP_TO_INT()
21015 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
21016 DAG.getIntPtrConstant(0, dl)); in LowerFP_TO_INT()
21019 return DAG.getMergeValues({Res, Chain}, dl); in LowerFP_TO_INT()
21033 IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT); in LowerFP_TO_INT()
21034 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src, in LowerFP_TO_INT()
21035 DAG.getIntPtrConstant(0, dl)); in LowerFP_TO_INT()
21038 Res = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other}, in LowerFP_TO_INT()
21042 Res = DAG.getNode(Op.getOpcode(), dl, MVT::v8i64, Src); in LowerFP_TO_INT()
21045 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
21046 DAG.getIntPtrConstant(0, dl)); in LowerFP_TO_INT()
21049 return DAG.getMergeValues({Res, Chain}, dl); in LowerFP_TO_INT()
21060 SDValue Zero = DAG.getConstantFP(0.0, dl, MVT::v2f32); in LowerFP_TO_INT()
21061 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f32, in LowerFP_TO_INT()
21063 Tmp = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other}, in LowerFP_TO_INT()
21066 Tmp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i64, Tmp, in LowerFP_TO_INT()
21067 DAG.getIntPtrConstant(0, dl)); in LowerFP_TO_INT()
21068 return DAG.getMergeValues({Tmp, Chain}, dl); in LowerFP_TO_INT()
21072 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, in LowerFP_TO_INT()
21073 DAG.getUNDEF(MVT::v2f32)); in LowerFP_TO_INT()
21077 return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp}); in LowerFP_TO_INT()
21080 return DAG.getNode(Opc, dl, VT, Tmp); in LowerFP_TO_INT()
21089 return expandFP_TO_UINT_SSE(VT, Src, dl, DAG, Subtarget); in LowerFP_TO_INT()
21110 SDValue FloatOffset = DAG.getNode(ISD::UINT_TO_FP, dl, SrcVT, in LowerFP_TO_INT()
21111 DAG.getConstant(UIntLimit, dl, VT)); in LowerFP_TO_INT()
21118 DAG.getNode(X86ISD::CVTTS2SI, dl, VT, in LowerFP_TO_INT()
21119 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT, Src)); in LowerFP_TO_INT()
21120 SDValue Big = DAG.getNode( in LowerFP_TO_INT()
21122 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT, in LowerFP_TO_INT()
21123 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FloatOffset))); in LowerFP_TO_INT()
21131 SDValue IsOverflown = DAG.getNode( in LowerFP_TO_INT()
21132 ISD::SRA, dl, VT, Small, DAG.getConstant(DstBits - 1, dl, MVT::i8)); in LowerFP_TO_INT()
21133 return DAG.getNode(ISD::OR, dl, VT, Small, in LowerFP_TO_INT()
21134 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown)); in LowerFP_TO_INT()
21148 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i64, MVT::Other}, in LowerFP_TO_INT()
21152 Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i64, Src); in LowerFP_TO_INT()
21154 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerFP_TO_INT()
21156 return DAG.getMergeValues({Res, Chain}, dl); in LowerFP_TO_INT()
21172 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i32, MVT::Other}, in LowerFP_TO_INT()
21176 Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src); in LowerFP_TO_INT()
21178 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerFP_TO_INT()
21180 return DAG.getMergeValues({Res, Chain}, dl); in LowerFP_TO_INT()
21198 makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain); in LowerFP_TO_INT()
21201 return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl); in LowerFP_TO_INT()
21207 if (SDValue V = FP_TO_INTHelper(Op, DAG, IsSigned, Chain)) { in LowerFP_TO_INT()
21209 return DAG.getMergeValues({V, Chain}, dl); in LowerFP_TO_INT()
21217 SelectionDAG &DAG) const { in LowerLRINT_LLRINT()
21232 return LRINT_LLRINTHelper(Op.getNode(), DAG); in LowerLRINT_LLRINT()
21236 SelectionDAG &DAG) const { in LRINT_LLRINTHelper()
21248 SDValue Chain = DAG.getEntryNode(); in LRINT_LLRINTHelper()
21255 SDValue StackPtr = DAG.CreateStackTemporary(DstVT, OtherVT); in LRINT_LLRINTHelper()
21258 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); in LRINT_LLRINTHelper()
21262 Chain = DAG.getStore(Chain, DL, Src, StackPtr, MPI); in LRINT_LLRINTHelper()
21263 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); in LRINT_LLRINTHelper()
21266 Src = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, SrcVT, MPI, in LRINT_LLRINTHelper()
21273 Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, DL, DAG.getVTList(MVT::Other), in LRINT_LLRINTHelper()
21277 return DAG.getLoad(DstVT, DL, Chain, StackPtr, MPI); in LRINT_LLRINTHelper()
21281 X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const { in LowerFP_TO_INT_SAT()
21339 APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT)); in LowerFP_TO_INT_SAT()
21340 APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT)); in LowerFP_TO_INT_SAT()
21349 SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT); in LowerFP_TO_INT_SAT()
21350 SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT); in LowerFP_TO_INT_SAT()
21357 SDValue MinClamped = DAG.getNode( in LowerFP_TO_INT_SAT()
21360 SDValue BothClamped = DAG.getNode( in LowerFP_TO_INT_SAT()
21363 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, BothClamped); in LowerFP_TO_INT_SAT()
21367 return DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt); in LowerFP_TO_INT_SAT()
21371 SDValue MinClamped = DAG.getNode( in LowerFP_TO_INT_SAT()
21374 SDValue BothClamped = DAG.getNode( in LowerFP_TO_INT_SAT()
21377 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, DstVT, BothClamped); in LowerFP_TO_INT_SAT()
21386 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT); in LowerFP_TO_INT_SAT()
21387 return DAG.getSelectCC( in LowerFP_TO_INT_SAT()
21391 SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT); in LowerFP_TO_INT_SAT()
21392 SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT); in LowerFP_TO_INT_SAT()
21395 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, Src); in LowerFP_TO_INT_SAT()
21400 FpToInt = DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt); in LowerFP_TO_INT_SAT()
21410 Select = DAG.getSelectCC( in LowerFP_TO_INT_SAT()
21415 Select = DAG.getSelectCC( in LowerFP_TO_INT_SAT()
21425 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT); in LowerFP_TO_INT_SAT()
21426 return DAG.getSelectCC( in LowerFP_TO_INT_SAT()
21430 SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const { in LowerFP_EXTEND()
21455 return DAG.getNode( in LowerFP_EXTEND()
21457 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, DL, in LowerFP_EXTEND()
21460 return DAG.getNode(ISD::FP_EXTEND, DL, VT, in LowerFP_EXTEND()
21461 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, In)); in LowerFP_EXTEND()
21471 TargetLowering::CallLoweringInfo CLI(DAG); in LowerFP_EXTEND()
21472 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode(); in LowerFP_EXTEND()
21474 In = DAG.getBitcast(MVT::i16, In); in LowerFP_EXTEND()
21478 Entry.Ty = EVT(MVT::i16).getTypeForEVT(*DAG.getContext()); in LowerFP_EXTEND()
21483 SDValue Callee = DAG.getExternalSymbol( in LowerFP_EXTEND()
21485 getPointerTy(DAG.getDataLayout())); in LowerFP_EXTEND()
21487 CallingConv::C, EVT(VT).getTypeForEVT(*DAG.getContext()), Callee, in LowerFP_EXTEND()
21493 Res = DAG.getMergeValues({Res, Chain}, DL); in LowerFP_EXTEND()
21498 In = DAG.getBitcast(MVT::i16, In); in LowerFP_EXTEND()
21499 In = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v8i16, in LowerFP_EXTEND()
21500 getZeroVector(MVT::v8i16, Subtarget, DAG, DL), In, in LowerFP_EXTEND()
21501 DAG.getIntPtrConstant(0, DL)); in LowerFP_EXTEND()
21504 Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, DL, {MVT::v4f32, MVT::Other}, in LowerFP_EXTEND()
21508 Res = DAG.getNode(X86ISD::CVTPH2PS, DL, MVT::v4f32, In, in LowerFP_EXTEND()
21509 DAG.getTargetConstant(4, DL, MVT::i32)); in LowerFP_EXTEND()
21511 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Res, in LowerFP_EXTEND()
21512 DAG.getIntPtrConstant(0, DL)); in LowerFP_EXTEND()
21514 return DAG.getMergeValues({Res, Chain}, DL); in LowerFP_EXTEND()
21526 In = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f16, In, in LowerFP_EXTEND()
21527 DAG.getUNDEF(MVT::v2f16)); in LowerFP_EXTEND()
21528 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f16, In, in LowerFP_EXTEND()
21529 DAG.getUNDEF(MVT::v4f16)); in LowerFP_EXTEND()
21531 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other}, in LowerFP_EXTEND()
21533 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res); in LowerFP_EXTEND()
21541 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, In, DAG.getUNDEF(SVT)); in LowerFP_EXTEND()
21543 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other}, in LowerFP_EXTEND()
21545 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res); in LowerFP_EXTEND()
21548 SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { in LowerFP_ROUND()
21566 TargetLowering::CallLoweringInfo CLI(DAG); in LowerFP_ROUND()
21567 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode(); in LowerFP_ROUND()
21572 Entry.Ty = EVT(SVT).getTypeForEVT(*DAG.getContext()); in LowerFP_ROUND()
21577 SDValue Callee = DAG.getExternalSymbol( in LowerFP_ROUND()
21580 getPointerTy(DAG.getDataLayout())); in LowerFP_ROUND()
21582 CallingConv::C, EVT(MVT::i16).getTypeForEVT(*DAG.getContext()), Callee, in LowerFP_ROUND()
21588 Res = DAG.getBitcast(MVT::f16, Res); in LowerFP_ROUND()
21591 Res = DAG.getMergeValues({Res, Chain}, DL); in LowerFP_ROUND()
21612 SDValue Rnd = DAG.getTargetConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, DL, in LowerFP_ROUND()
21615 Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4f32, in LowerFP_ROUND()
21616 DAG.getConstantFP(0, DL, MVT::v4f32), In, in LowerFP_ROUND()
21617 DAG.getIntPtrConstant(0, DL)); in LowerFP_ROUND()
21618 Res = DAG.getNode(X86ISD::STRICT_CVTPS2PH, DL, {MVT::v8i16, MVT::Other}, in LowerFP_ROUND()
21623 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, In); in LowerFP_ROUND()
21624 Res = DAG.getNode(X86ISD::CVTPS2PH, DL, MVT::v8i16, Res, Rnd); in LowerFP_ROUND()
21627 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, Res, in LowerFP_ROUND()
21628 DAG.getIntPtrConstant(0, DL)); in LowerFP_ROUND()
21629 Res = DAG.getBitcast(MVT::f16, Res); in LowerFP_ROUND()
21632 return DAG.getMergeValues({Res, Chain}, DL); in LowerFP_ROUND()
21640 static SDValue LowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG) { in LowerFP16_TO_FP() argument
21647 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, in LowerFP16_TO_FP()
21648 DAG.getConstant(0, dl, MVT::v8i16), Src, in LowerFP16_TO_FP()
21649 DAG.getIntPtrConstant(0, dl)); in LowerFP16_TO_FP()
21653 Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {MVT::v4f32, MVT::Other}, in LowerFP16_TO_FP()
21657 Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res); in LowerFP16_TO_FP()
21660 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res, in LowerFP16_TO_FP()
21661 DAG.getIntPtrConstant(0, dl)); in LowerFP16_TO_FP()
21664 return DAG.getMergeValues({Res, Chain}, dl); in LowerFP16_TO_FP()
21669 static SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) { in LowerFP_TO_FP16() argument
21678 Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v4f32, in LowerFP_TO_FP16()
21679 DAG.getConstantFP(0, dl, MVT::v4f32), Src, in LowerFP_TO_FP16()
21680 DAG.getIntPtrConstant(0, dl)); in LowerFP_TO_FP16()
21681 Res = DAG.getNode( in LowerFP_TO_FP16()
21683 {Op.getOperand(0), Res, DAG.getTargetConstant(4, dl, MVT::i32)}); in LowerFP_TO_FP16()
21687 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, Src); in LowerFP_TO_FP16()
21688 Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res, in LowerFP_TO_FP16()
21689 DAG.getTargetConstant(4, dl, MVT::i32)); in LowerFP_TO_FP16()
21692 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Res, in LowerFP_TO_FP16()
21693 DAG.getIntPtrConstant(0, dl)); in LowerFP_TO_FP16()
21696 return DAG.getMergeValues({Res, Chain}, dl); in LowerFP_TO_FP16()
21702 SelectionDAG &DAG) const { in LowerFP_TO_BF16()
21709 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, Op.getOperand(0)); in LowerFP_TO_BF16()
21710 Res = DAG.getNode(X86ISD::CVTNEPS2BF16, DL, MVT::v8bf16, Res); in LowerFP_TO_BF16()
21711 Res = DAG.getBitcast(MVT::v8i16, Res); in LowerFP_TO_BF16()
21712 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, Res, in LowerFP_TO_BF16()
21713 DAG.getIntPtrConstant(0, DL)); in LowerFP_TO_BF16()
21719 makeLibCall(DAG, LC, MVT::f16, Op.getOperand(0), CallOptions, DL).first; in LowerFP_TO_BF16()
21720 return DAG.getBitcast(MVT::i16, Res); in LowerFP_TO_BF16()
21726 SelectionDAG &DAG, in lowerAddSubToHorizontalOp() argument
21747 !shouldUseHorizontalOp(true, DAG, Subtarget)) in lowerAddSubToHorizontalOp()
21784 X = extract128BitVector(X, LaneIdx * NumEltsPerLane, DAG, DL); in lowerAddSubToHorizontalOp()
21792 SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X); in lowerAddSubToHorizontalOp()
21793 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getSimpleValueType(), HOp, in lowerAddSubToHorizontalOp()
21794 DAG.getIntPtrConstant(LExtIndex / 2, DL)); in lowerAddSubToHorizontalOp()
21799 SDValue X86TargetLowering::lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const { in lowerFaddFsub()
21802 return lowerAddSubToHorizontalOp(Op, SDLoc(Op), DAG, Subtarget); in lowerFaddFsub()
21809 static SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) { in LowerFROUND() argument
21821 SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT, in LowerFROUND()
21822 DAG.getConstantFP(Point5Pred, dl, VT), N0); in LowerFROUND()
21823 N0 = DAG.getNode(ISD::FADD, dl, VT, N0, Adder); in LowerFROUND()
21826 return DAG.getNode(ISD::FTRUNC, dl, VT, N0); in LowerFROUND()
21831 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) { in LowerFABSorFNEG() argument
21849 DAG.getTargetLoweringInfo().isTypeLegal(VT) && in LowerFABSorFNEG()
21872 SDValue Mask = DAG.getConstantFP(APFloat(Sem, MaskElt), dl, LogicVT); in LowerFABSorFNEG()
21882 return DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask); in LowerFABSorFNEG()
21886 Operand = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Operand); in LowerFABSorFNEG()
21887 SDValue LogicNode = DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask); in LowerFABSorFNEG()
21888 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, LogicNode, in LowerFABSorFNEG()
21889 DAG.getIntPtrConstant(0, dl)); in LowerFABSorFNEG()
21892 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) { in LowerFCOPYSIGN() argument
21900 Sign = DAG.getNode(ISD::FP_EXTEND, dl, VT, Sign); in LowerFCOPYSIGN()
21904 Sign = DAG.getNode(ISD::FP_ROUND, dl, VT, Sign, in LowerFCOPYSIGN()
21905 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)); in LowerFCOPYSIGN()
21911 DAG.getTargetLoweringInfo().isTypeLegal(VT) && in LowerFCOPYSIGN()
21930 SDValue SignMask = DAG.getConstantFP( in LowerFCOPYSIGN()
21932 SDValue MagMask = DAG.getConstantFP( in LowerFCOPYSIGN()
21937 Sign = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Sign); in LowerFCOPYSIGN()
21938 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Sign, SignMask); in LowerFCOPYSIGN()
21947 MagBits = DAG.getConstantFP(APF, dl, LogicVT); in LowerFCOPYSIGN()
21951 Mag = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Mag); in LowerFCOPYSIGN()
21952 MagBits = DAG.getNode(X86ISD::FAND, dl, LogicVT, Mag, MagMask); in LowerFCOPYSIGN()
21956 SDValue Or = DAG.getNode(X86ISD::FOR, dl, LogicVT, MagBits, SignBit); in LowerFCOPYSIGN()
21957 return !IsFakeVector ? Or : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Or, in LowerFCOPYSIGN()
21958 DAG.getIntPtrConstant(0, dl)); in LowerFCOPYSIGN()
21961 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) { in LowerFGETSIGN() argument
21972 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N0); in LowerFGETSIGN()
21973 Res = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32, Res); in LowerFGETSIGN()
21974 Res = DAG.getZExtOrTrunc(Res, dl, VT); in LowerFGETSIGN()
21975 Res = DAG.getNode(ISD::AND, dl, VT, Res, DAG.getConstant(1, dl, VT)); in LowerFGETSIGN()
21980 static SDValue getBT(SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG) { in getBT() argument
21987 Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Src); in getBT()
21990 if (!DAG.getTargetLoweringInfo().isTypeLegal(Src.getValueType())) in getBT()
21998 DAG.MaskedValueIsZero(BitNo, APInt(BitNo.getValueSizeInBits(), 32))) in getBT()
21999 Src = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Src); in getBT()
22008 BitNo = DAG.getNode(ISD::AND, DL, Src.getValueType(), in getBT()
22009 DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(), in getBT()
22011 DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(), in getBT()
22014 BitNo = DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(), BitNo); in getBT()
22017 return DAG.getNode(X86ISD::BT, DL, MVT::i32, Src, BitNo); in getBT()
22022 SelectionDAG &DAG) { in getSETCC() argument
22023 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in getSETCC()
22024 DAG.getTargetConstant(Cond, dl, MVT::i8), EFLAGS); in getSETCC()
22041 static SDValue emitOrXorXorTree(SDValue X, const SDLoc &DL, SelectionDAG &DAG, in emitOrXorXorTree() argument
22046 SDValue A = emitOrXorXorTree(Op0, DL, DAG, VecVT, CmpVT, HasPT, SToV); in emitOrXorXorTree()
22047 SDValue B = emitOrXorXorTree(Op1, DL, DAG, VecVT, CmpVT, HasPT, SToV); in emitOrXorXorTree()
22049 return DAG.getNode(ISD::OR, DL, CmpVT, A, B); in emitOrXorXorTree()
22051 return DAG.getNode(ISD::OR, DL, VecVT, A, B); in emitOrXorXorTree()
22052 return DAG.getNode(ISD::AND, DL, CmpVT, A, B); in emitOrXorXorTree()
22058 return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETNE); in emitOrXorXorTree()
22060 return DAG.getNode(ISD::XOR, DL, VecVT, A, B); in emitOrXorXorTree()
22061 return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETEQ); in emitOrXorXorTree()
22071 SelectionDAG &DAG, in combineVectorSizedSetCCEquality() argument
22104 DAG.getMachineFunction().getFunction().hasFnAttribute( in combineVectorSizedSetCCEquality()
22160 X = DAG.getBitcast(TmpCastVT, X); in combineVectorSizedSetCCEquality()
22163 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, in combineVectorSizedSetCCEquality()
22164 DAG.getConstant(0, DL, VecVT), X, in combineVectorSizedSetCCEquality()
22165 DAG.getVectorIdxConstant(0, DL)); in combineVectorSizedSetCCEquality()
22174 Cmp = emitOrXorXorTree(X, DL, DAG, VecVT, CmpVT, HasPT, ScalarToVector); in combineVectorSizedSetCCEquality()
22179 Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETNE); in combineVectorSizedSetCCEquality()
22181 Cmp = DAG.getNode(ISD::XOR, DL, VecVT, VecX, VecY); in combineVectorSizedSetCCEquality()
22183 Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETEQ); in combineVectorSizedSetCCEquality()
22191 return DAG.getSetCC(DL, VT, DAG.getBitcast(KRegVT, Cmp), in combineVectorSizedSetCCEquality()
22192 DAG.getConstant(0, DL, KRegVT), CC); in combineVectorSizedSetCCEquality()
22196 DAG.getBitcast(OpSize == 256 ? MVT::v4i64 : MVT::v2i64, Cmp); in combineVectorSizedSetCCEquality()
22197 SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp); in combineVectorSizedSetCCEquality()
22199 SDValue X86SetCC = getSETCC(X86CC, PT, DL, DAG); in combineVectorSizedSetCCEquality()
22200 return DAG.getNode(ISD::TRUNCATE, DL, VT, X86SetCC.getValue(0)); in combineVectorSizedSetCCEquality()
22207 SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp); in combineVectorSizedSetCCEquality()
22208 SDValue FFFFs = DAG.getConstant(0xFFFF, DL, MVT::i32); in combineVectorSizedSetCCEquality()
22209 return DAG.getSetCC(DL, VT, MovMsk, FFFFs, CC); in combineVectorSizedSetCCEquality()
22291 SelectionDAG &DAG, X86::CondCode &X86CC) { in LowerVectorAllEqual() argument
22316 SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT); in LowerVectorAllEqual()
22317 return DAG.getNode(ISD::AND, DL, SrcVT, Src, MaskValue); in LowerVectorAllEqual()
22322 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); in LowerVectorAllEqual()
22323 if (!DAG.getTargetLoweringInfo().isTypeLegal(IntVT)) { in LowerVectorAllEqual()
22326 auto SplitLHS = DAG.SplitScalar(DAG.getBitcast(IntVT, MaskBits(LHS)), DL, in LowerVectorAllEqual()
22328 auto SplitRHS = DAG.SplitScalar(DAG.getBitcast(IntVT, MaskBits(RHS)), DL, in LowerVectorAllEqual()
22331 DAG.getNode(ISD::XOR, DL, MVT::i32, SplitLHS.first, SplitRHS.first); in LowerVectorAllEqual()
22333 DAG.getNode(ISD::XOR, DL, MVT::i32, SplitLHS.second, SplitRHS.second); in LowerVectorAllEqual()
22334 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, in LowerVectorAllEqual()
22335 DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi), in LowerVectorAllEqual()
22336 DAG.getConstant(0, DL, MVT::i32)); in LowerVectorAllEqual()
22338 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, in LowerVectorAllEqual()
22339 DAG.getBitcast(IntVT, MaskBits(LHS)), in LowerVectorAllEqual()
22340 DAG.getBitcast(IntVT, MaskBits(RHS))); in LowerVectorAllEqual()
22358 VT = EVT::getVectorVT(*DAG.getContext(), MVT::i64, VT.getSizeInBits() / 64); in LowerVectorAllEqual()
22359 LHS = DAG.getBitcast(VT, LHS); in LowerVectorAllEqual()
22360 RHS = DAG.getBitcast(VT, RHS); in LowerVectorAllEqual()
22365 KnownBits KnownRHS = DAG.computeKnownBits(RHS); in LowerVectorAllEqual()
22369 auto Split = DAG.SplitVector(LHS, DL); in LowerVectorAllEqual()
22371 LHS = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second); in LowerVectorAllEqual()
22373 RHS = DAG.getAllOnesConstant(DL, VT); in LowerVectorAllEqual()
22379 LHS = DAG.getBitcast(VT, MaskBits(LHS)); in LowerVectorAllEqual()
22380 RHS = DAG.getBitcast(VT, MaskBits(RHS)); in LowerVectorAllEqual()
22382 SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETEQ); in LowerVectorAllEqual()
22383 V = DAG.getSExtOrTrunc(V, DL, VT); in LowerVectorAllEqual()
22385 auto Split = DAG.SplitVector(V, DL); in LowerVectorAllEqual()
22387 V = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second); in LowerVectorAllEqual()
22389 V = DAG.getNOT(DL, V, VT); in LowerVectorAllEqual()
22390 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V); in LowerVectorAllEqual()
22391 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V, in LowerVectorAllEqual()
22392 DAG.getConstant(0, DL, MVT::i32)); in LowerVectorAllEqual()
22395 SDValue V = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS); in LowerVectorAllEqual()
22397 auto Split = DAG.SplitVector(V, DL); in LowerVectorAllEqual()
22399 V = DAG.getNode(ISD::OR, DL, VT, Split.first, Split.second); in LowerVectorAllEqual()
22402 RHS = DAG.getConstant(0, DL, VT); in LowerVectorAllEqual()
22409 LHS = DAG.getBitcast(TestVT, MaskBits(LHS)); in LowerVectorAllEqual()
22410 RHS = DAG.getBitcast(TestVT, MaskBits(RHS)); in LowerVectorAllEqual()
22411 SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETNE); in LowerVectorAllEqual()
22412 return DAG.getNode(X86ISD::KORTEST, DL, MVT::i32, V, V); in LowerVectorAllEqual()
22417 LHS = DAG.getBitcast(TestVT, MaskBits(LHS)); in LowerVectorAllEqual()
22418 RHS = DAG.getBitcast(TestVT, MaskBits(RHS)); in LowerVectorAllEqual()
22419 SDValue V = DAG.getNode(ISD::XOR, DL, TestVT, LHS, RHS); in LowerVectorAllEqual()
22420 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, V, V); in LowerVectorAllEqual()
22425 LHS = DAG.getBitcast(MaskVT, MaskBits(LHS)); in LowerVectorAllEqual()
22426 RHS = DAG.getBitcast(MaskVT, MaskBits(RHS)); in LowerVectorAllEqual()
22427 SDValue V = DAG.getNode(X86ISD::PCMPEQ, DL, MaskVT, LHS, RHS); in LowerVectorAllEqual()
22428 V = DAG.getNOT(DL, V, MaskVT); in LowerVectorAllEqual()
22429 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V); in LowerVectorAllEqual()
22430 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V, in LowerVectorAllEqual()
22431 DAG.getConstant(0, DL, MVT::i32)); in LowerVectorAllEqual()
22439 SelectionDAG &DAG, in MatchVectorAllEqualTest() argument
22498 VecIns.push_back(DAG.getNode(LogicOp, DL, VT, LHS, RHS)); in MatchVectorAllEqualTest()
22502 CmpNull ? DAG.getConstant(0, DL, VT) in MatchVectorAllEqualTest()
22503 : DAG.getAllOnesConstant(DL, VT), in MatchVectorAllEqualTest()
22504 CC, Mask, Subtarget, DAG, X86CC); in MatchVectorAllEqualTest()
22512 DAG.matchBinOpReduction(Op.getNode(), BinOp, {LogicOp})) { in MatchVectorAllEqualTest()
22515 CmpNull ? DAG.getConstant(0, DL, MatchVT) in MatchVectorAllEqualTest()
22516 : DAG.getAllOnesConstant(DL, MatchVT), in MatchVectorAllEqualTest()
22517 CC, Mask, Subtarget, DAG, X86CC); in MatchVectorAllEqualTest()
22537 return LowerVectorAllEqual(DL, LHS, RHS, CC, SrcMask, Subtarget, DAG, in MatchVectorAllEqualTest()
22552 DAG.getConstant(Cmp, DL, InnerVT), CC, in MatchVectorAllEqualTest()
22553 SrcMask, Subtarget, DAG, X86CC); in MatchVectorAllEqualTest()
22597 SelectionDAG &DAG, const X86Subtarget &Subtarget) { in EmitTest() argument
22634 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
22635 DAG.getConstant(0, dl, Op.getValueType())); in EmitTest()
22683 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in EmitTest()
22684 return DAG.getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0), in EmitTest()
22693 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
22694 DAG.getConstant(0, dl, Op.getValueType())); in EmitTest()
22696 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in EmitTest()
22699 SDValue New = DAG.getNode(Opcode, dl, VTs, Ops); in EmitTest()
22700 DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), New); in EmitTest()
22707 const SDLoc &dl, SelectionDAG &DAG, in EmitCmp() argument
22710 return EmitTest(Op0, X86CC, dl, DAG, Subtarget); in EmitCmp()
22723 !DAG.getMachineFunction().getFunction().hasMinSize()) { in EmitCmp()
22735 if (DAG.ComputeMaxSignificantBits(Op0.getOperand(0)) <= 16) in EmitCmp()
22738 if (DAG.ComputeMaxSignificantBits(Op1.getOperand(0)) <= 16) in EmitCmp()
22744 Op0 = DAG.getNode(ExtendOp, dl, CmpVT, Op0); in EmitCmp()
22745 Op1 = DAG.getNode(ExtendOp, dl, CmpVT, Op1); in EmitCmp()
22753 DAG.MaskedValueIsZero(Op1, APInt::getHighBitsSet(64, 32)) && in EmitCmp()
22754 DAG.MaskedValueIsZero(Op0, APInt::getHighBitsSet(64, 32))) { in EmitCmp()
22756 Op0 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op0); in EmitCmp()
22757 Op1 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op1); in EmitCmp()
22764 SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32); in EmitCmp()
22765 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(1), Op1); in EmitCmp()
22773 SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32); in EmitCmp()
22774 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0, Op1.getOperand(1)); in EmitCmp()
22779 SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32); in EmitCmp()
22780 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1); in EmitCmp()
22808 bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const { in isFsqrtCheap()
22816 if (DAG.doesNodeExist(X86ISD::FRSQRT, DAG.getVTList(VT), Op)) in isFsqrtCheap()
22827 SelectionDAG &DAG, int Enabled, in getSqrtEstimate() argument
22853 SDValue Estimate = DAG.getNode(Opcode, DL, VT, Op); in getSqrtEstimate()
22855 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Op, Estimate); in getSqrtEstimate()
22866 SDValue Zero = DAG.getIntPtrConstant(0, DL); in getSqrtEstimate()
22867 SDValue Undef = DAG.getUNDEF(MVT::v8f16); in getSqrtEstimate()
22868 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op); in getSqrtEstimate()
22869 Op = DAG.getNode(X86ISD::RSQRT14S, DL, MVT::v8f16, Undef, Op); in getSqrtEstimate()
22870 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero); in getSqrtEstimate()
22873 return DAG.getNode(X86ISD::RSQRT14, DL, VT, Op); in getSqrtEstimate()
22880 SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG, in getRecipEstimate() argument
22908 return DAG.getNode(Opcode, DL, VT, Op); in getRecipEstimate()
22917 SDValue Zero = DAG.getIntPtrConstant(0, DL); in getRecipEstimate()
22918 SDValue Undef = DAG.getUNDEF(MVT::v8f16); in getRecipEstimate()
22919 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op); in getRecipEstimate()
22920 Op = DAG.getNode(X86ISD::RCP14S, DL, MVT::v8f16, Undef, Op); in getRecipEstimate()
22921 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero); in getRecipEstimate()
22924 return DAG.getNode(X86ISD::RCP14, DL, VT, Op); in getRecipEstimate()
22941 SelectionDAG &DAG, in BuildSDIVPow2() argument
22943 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes(); in BuildSDIVPow2()
22967 return TargetLowering::buildSDIVPow2WithCMov(N, Divisor, DAG, Created); in BuildSDIVPow2()
22973 SelectionDAG &DAG, X86::CondCode &X86CC) { in LowerAndToBT() argument
22992 KnownBits Known = DAG.computeKnownBits(Op0); in LowerAndToBT()
23010 bool OptForSize = DAG.shouldOptForSize(); in LowerAndToBT()
23014 BitNo = DAG.getConstant(Log2_64_Ceil(AndRHSVal), dl, in LowerAndToBT()
23031 if (SDValue BT = getBT(Src, BitNo, dl, DAG)) { in LowerAndToBT()
23110 ISD::CondCode Cond, SelectionDAG &DAG, in splitIntVSETCC() argument
23115 SDValue CC = DAG.getCondCode(Cond); in splitIntVSETCC()
23119 std::tie(LHS1, LHS2) = splitVector(LHS, DAG, dl); in splitIntVSETCC()
23123 std::tie(RHS1, RHS2) = splitVector(RHS, DAG, dl); in splitIntVSETCC()
23127 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in splitIntVSETCC()
23128 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, in splitIntVSETCC()
23129 DAG.getNode(ISD::SETCC, dl, LoVT, LHS1, RHS1, CC), in splitIntVSETCC()
23130 DAG.getNode(ISD::SETCC, dl, HiVT, LHS2, RHS2, CC)); in splitIntVSETCC()
23134 SelectionDAG &DAG) { in LowerIntVSETCC_AVX512() argument
23150 return DAG.getSetCC(dl, VT, Op0, Op1, SetCCOpcode); in LowerIntVSETCC_AVX512()
23157 static SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc, in incDecVectorConstant() argument
23181 NewVecC.push_back(DAG.getConstant(EltC + (IsInc ? 1 : -1), DL, EltVT)); in incDecVectorConstant()
23184 return DAG.getBuildVector(VT, DL, NewVecC); in incDecVectorConstant()
23194 SelectionDAG &DAG) { in LowerVSETCCWithSUBUS() argument
23214 incDecVectorConstant(Op1, DAG, /*IsInc*/ false, /*NSW*/ false); in LowerVSETCCWithSUBUS()
23226 incDecVectorConstant(Op1, DAG, /*IsInc*/ true, /*NSW*/ false); in LowerVSETCCWithSUBUS()
23241 SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1); in LowerVSETCCWithSUBUS()
23242 return DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result, in LowerVSETCCWithSUBUS()
23243 DAG.getConstant(0, dl, VT)); in LowerVSETCCWithSUBUS()
23247 SelectionDAG &DAG) { in LowerVSETCC() argument
23304 SDValue SignalCmp = DAG.getNode( in LowerVSETCC()
23306 {Chain, Op0, Op1, DAG.getTargetConstant(1, dl, MVT::i8)}); // LT_OS in LowerVSETCC()
23334 Cmp0 = DAG.getNode( in LowerVSETCC()
23336 {Chain, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8)}); in LowerVSETCC()
23337 Cmp1 = DAG.getNode( in LowerVSETCC()
23339 {Chain, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)}); in LowerVSETCC()
23340 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Cmp0.getValue(1), in LowerVSETCC()
23343 Cmp0 = DAG.getNode( in LowerVSETCC()
23344 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8)); in LowerVSETCC()
23345 Cmp1 = DAG.getNode( in LowerVSETCC()
23346 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)); in LowerVSETCC()
23348 Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1); in LowerVSETCC()
23351 Cmp = DAG.getNode( in LowerVSETCC()
23353 {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)}); in LowerVSETCC()
23356 Cmp = DAG.getNode( in LowerVSETCC()
23357 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)); in LowerVSETCC()
23364 Cmp = DAG.getNode( in LowerVSETCC()
23366 {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)}); in LowerVSETCC()
23369 Cmp = DAG.getNode( in LowerVSETCC()
23370 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)); in LowerVSETCC()
23378 Cmp = DAG.getBitcast(CastVT, Cmp); in LowerVSETCC()
23379 Cmp = DAG.getSetCC(dl, Op.getSimpleValueType(), Cmp, in LowerVSETCC()
23380 DAG.getConstant(0, dl, CastVT), ISD::SETNE); in LowerVSETCC()
23385 Cmp = DAG.getBitcast(Op.getSimpleValueType(), Cmp); in LowerVSETCC()
23389 return DAG.getMergeValues({Cmp, Chain}, dl); in LowerVSETCC()
23413 return LowerIntVSETCC_AVX512(Op, dl, DAG); in LowerVSETCC()
23440 return DAG.getNode(Opc, dl, VT, Op0, Op1, in LowerVSETCC()
23441 DAG.getTargetConstant(CmpMode, dl, MVT::i8)); in LowerVSETCC()
23456 Op1 = DAG.getBitcast(VT, BC0.getOperand(1)); in LowerVSETCC()
23471 Result = DAG.getNode(ISD::SHL, dl, VT, Result, in LowerVSETCC()
23472 DAG.getConstant(ShiftAmt, dl, VT)); in LowerVSETCC()
23473 Result = DAG.getNode(ISD::SRA, dl, VT, Result, in LowerVSETCC()
23474 DAG.getConstant(BitWidth - 1, dl, VT)); in LowerVSETCC()
23481 return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl); in LowerVSETCC()
23486 return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl); in LowerVSETCC()
23500 else if (ConstValue.isZero() && DAG.SignBitIsZero(Op0)) in LowerVSETCC()
23509 !(DAG.SignBitIsZero(Op0) && DAG.SignBitIsZero(Op1)); in LowerVSETCC()
23512 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerVSETCC()
23521 incDecVectorConstant(Op1, DAG, /*IsInc*/ true, /*NSW*/ false)) { in LowerVSETCC()
23529 incDecVectorConstant(Op1, DAG, /*IsInc*/ false, /*NSW*/ false)) { in LowerVSETCC()
23546 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); in LowerVSETCC()
23547 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result); in LowerVSETCC()
23551 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
23559 LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG)) in LowerVSETCC()
23584 Op0 = DAG.getConstant(0, dl, MVT::v4i32); in LowerVSETCC()
23585 Op1 = DAG.getBitcast(MVT::v4i32, Op1); in LowerVSETCC()
23587 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
23589 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi); in LowerVSETCC()
23591 return DAG.getBitcast(VT, Result); in LowerVSETCC()
23595 Op0 = DAG.getBitcast(MVT::v4i32, Op0); in LowerVSETCC()
23596 Op1 = DAG.getConstant(-1, dl, MVT::v4i32); in LowerVSETCC()
23598 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
23600 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi); in LowerVSETCC()
23602 return DAG.getBitcast(VT, Result); in LowerVSETCC()
23607 if (!FlipSigns && !Invert && DAG.ComputeNumSignBits(Op0) > 32 && in LowerVSETCC()
23608 DAG.ComputeNumSignBits(Op1) > 32) { in LowerVSETCC()
23609 Op0 = DAG.getBitcast(MVT::v4i32, Op0); in LowerVSETCC()
23610 Op1 = DAG.getBitcast(MVT::v4i32, Op1); in LowerVSETCC()
23612 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
23614 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo); in LowerVSETCC()
23616 return DAG.getBitcast(VT, Result); in LowerVSETCC()
23622 SDValue SB = DAG.getConstant(FlipSigns ? 0x8000000080000000ULL in LowerVSETCC()
23626 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op0, SB); in LowerVSETCC()
23627 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op1, SB); in LowerVSETCC()
23630 Op0 = DAG.getBitcast(MVT::v4i32, Op0); in LowerVSETCC()
23631 Op1 = DAG.getBitcast(MVT::v4i32, Op1); in LowerVSETCC()
23634 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
23635 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
23640 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi); in LowerVSETCC()
23641 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo); in LowerVSETCC()
23642 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi); in LowerVSETCC()
23644 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo); in LowerVSETCC()
23645 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi); in LowerVSETCC()
23648 Result = DAG.getNOT(dl, Result, MVT::v4i32); in LowerVSETCC()
23650 return DAG.getBitcast(VT, Result); in LowerVSETCC()
23659 Op0 = DAG.getBitcast(MVT::v4i32, Op0); in LowerVSETCC()
23660 Op1 = DAG.getBitcast(MVT::v4i32, Op1); in LowerVSETCC()
23663 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
23667 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask); in LowerVSETCC()
23668 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf); in LowerVSETCC()
23671 Result = DAG.getNOT(dl, Result, MVT::v4i32); in LowerVSETCC()
23673 return DAG.getBitcast(VT, Result); in LowerVSETCC()
23681 SDValue SM = DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl, in LowerVSETCC()
23683 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM); in LowerVSETCC()
23684 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM); in LowerVSETCC()
23687 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); in LowerVSETCC()
23691 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
23698 const SDLoc &dl, SelectionDAG &DAG, in EmitAVX512Test() argument
23734 X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8); in EmitAVX512Test()
23735 return DAG.getNode(X86ISD::KTEST, dl, MVT::i32, LHS, RHS); in EmitAVX512Test()
23746 X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8); in EmitAVX512Test()
23747 return DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS); in EmitAVX512Test()
23754 SelectionDAG &DAG, in emitFlagsForSetcc() argument
23765 if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CondCode)) { in emitFlagsForSetcc()
23766 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8); in emitFlagsForSetcc()
23772 if (SDValue CmpZ = MatchVectorAllEqualTest(Op0, Op1, CC, dl, Subtarget, DAG, in emitFlagsForSetcc()
23774 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8); in emitFlagsForSetcc()
23779 if (SDValue Test = EmitAVX512Test(Op0, Op1, CC, dl, DAG, Subtarget, X86CC)) in emitFlagsForSetcc()
23794 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8); in emitFlagsForSetcc()
23806 SDVTList CmpVTs = DAG.getVTList(VT, MVT::i32); in emitFlagsForSetcc()
23808 X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8); in emitFlagsForSetcc()
23809 SDValue Neg = DAG.getNode(X86ISD::SUB, dl, CmpVTs, in emitFlagsForSetcc()
23810 DAG.getConstant(0, dl, VT), Op0); in emitFlagsForSetcc()
23820 SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32); in emitFlagsForSetcc()
23822 SDValue New = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(0), in emitFlagsForSetcc()
23824 DAG.ReplaceAllUsesOfValueWith(SDValue(Op0.getNode(), 0), New); in emitFlagsForSetcc()
23826 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8); in emitFlagsForSetcc()
23833 TranslateX86CC(CC, dl, /*IsFP*/ false, Op0, Op1, DAG); in emitFlagsForSetcc()
23836 SDValue EFLAGS = EmitCmp(Op0, Op1, CondCode, dl, DAG, Subtarget); in emitFlagsForSetcc()
23837 X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8); in emitFlagsForSetcc()
23841 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { in LowerSETCC()
23847 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG); in LowerSETCC()
23863 softenSetCCOperands(DAG, MVT::f128, Op0, Op1, CC, dl, Op0, Op1, Chain, in LowerSETCC()
23871 return DAG.getMergeValues({Op0, Chain}, dl); in LowerSETCC()
23896 Op1 = DAG.getConstant(Op1ValPlusOne, dl, Op0.getValueType()); in LowerSETCC()
23905 SDValue EFLAGS = emitFlagsForSetcc(Op0, Op1, CC, dl, DAG, X86CC); in LowerSETCC()
23906 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS); in LowerSETCC()
23907 return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res; in LowerSETCC()
23911 X86::CondCode CondCode = TranslateX86CC(CC, dl, /*IsFP*/ true, Op0, Op1, DAG); in LowerSETCC()
23919 DAG.getNode(IsSignaling ? X86ISD::STRICT_FCMPS : X86ISD::STRICT_FCMP, in LowerSETCC()
23923 EFLAGS = DAG.getNode(X86ISD::FCMP, dl, MVT::i32, Op0, Op1); in LowerSETCC()
23926 SDValue X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8); in LowerSETCC()
23927 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS); in LowerSETCC()
23928 return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res; in LowerSETCC()
23931 SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const { in LowerSETCCCARRY()
23943 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32), in LowerSETCCCARRY()
23944 Carry, DAG.getAllOnesConstant(DL, CarryVT)); in LowerSETCCCARRY()
23946 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); in LowerSETCCCARRY()
23947 SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1)); in LowerSETCCCARRY()
23948 return getSETCC(CC, Cmp.getValue(1), DL, DAG); in LowerSETCCCARRY()
23956 getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) { in getX86XALUOOp() argument
23993 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in getX86XALUOOp()
23994 Value = DAG.getNode(BaseOp, DL, VTs, LHS, RHS); in getX86XALUOOp()
24001 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) { in LowerXALUO() argument
24009 std::tie(Value, Overflow) = getX86XALUOOp(Cond, Op, DAG); in LowerXALUO()
24011 SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG); in LowerXALUO()
24013 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC); in LowerXALUO()
24031 static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) { in isTruncWithZeroHighBitsInput() argument
24038 return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits)); in isTruncWithZeroHighBitsInput()
24041 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const { in LowerSELECT()
24052 return DAG.getBitcast(VT, DAG.getNode(ISD::SELECT, DL, NVT, Cond, in LowerSELECT()
24053 DAG.getBitcast(NVT, Op1), in LowerSELECT()
24054 DAG.getBitcast(NVT, Op2))); in LowerSELECT()
24070 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0, CondOp1, in LowerSELECT()
24071 DAG.getTargetConstant(SSECC, DL, MVT::i8)); in LowerSELECT()
24073 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2); in LowerSELECT()
24077 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1, in LowerSELECT()
24078 DAG.getTargetConstant(SSECC, DL, MVT::i8)); in LowerSELECT()
24097 SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1); in LowerSELECT()
24098 SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2); in LowerSELECT()
24099 SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp); in LowerSELECT()
24102 VCmp = DAG.getBitcast(VCmpVT, VCmp); in LowerSELECT()
24104 SDValue VSel = DAG.getSelect(DL, VecVT, VCmp, VOp1, VOp2); in LowerSELECT()
24106 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in LowerSELECT()
24107 VSel, DAG.getIntPtrConstant(0, DL)); in LowerSELECT()
24109 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2); in LowerSELECT()
24110 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1); in LowerSELECT()
24111 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And); in LowerSELECT()
24117 SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond); in LowerSELECT()
24118 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2); in LowerSELECT()
24123 if (SDValue NewCond = LowerSETCC(Cond, DAG)) { in LowerSELECT()
24164 SDVTList CmpVTs = DAG.getVTList(CmpOp0.getValueType(), MVT::i32); in LowerSELECT()
24175 SDValue Zero = DAG.getConstant(0, DL, CmpOp0.getValueType()); in LowerSELECT()
24176 Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, Zero, CmpOp0); in LowerSELECT()
24178 SDValue One = DAG.getConstant(1, DL, CmpOp0.getValueType()); in LowerSELECT()
24179 Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, CmpOp0, One); in LowerSELECT()
24181 SDValue SBB = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in LowerSELECT()
24182 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), in LowerSELECT()
24184 return DAG.getNode(ISD::OR, DL, VT, SBB, Y); in LowerSELECT()
24209 Neg = DAG.getNode(ISD::TRUNCATE, DL, VT, CmpOp0); in LowerSELECT()
24211 Neg = DAG.getNode(ISD::AND, DL, VT, in LowerSELECT()
24212 DAG.getNode(ISD::ANY_EXTEND, DL, VT, CmpOp0.getOperand(0)), in LowerSELECT()
24213 DAG.getConstant(1, DL, VT)); in LowerSELECT()
24216 SDValue Mask = DAG.getNegative(Neg, DL, VT); // -(and (x, 0x1)) in LowerSELECT()
24217 SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z in LowerSELECT()
24218 return DAG.getNode(Op2.getOpcode(), DL, VT, And, Src2); // And Op y in LowerSELECT()
24231 SDValue ShiftAmt = DAG.getConstant(ShCt, DL, VT); in LowerSELECT()
24232 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, Op1, ShiftAmt); in LowerSELECT()
24234 Shift = DAG.getNOT(DL, Shift, VT); in LowerSELECT()
24235 return DAG.getNode(ISD::AND, DL, VT, Shift, Op1); in LowerSELECT()
24268 std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG); in LowerSELECT()
24270 CC = DAG.getTargetConstant(X86Cond, DL, MVT::i8); in LowerSELECT()
24276 if (isTruncWithZeroHighBitsInput(Cond, DAG)) in LowerSELECT()
24283 if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, X86CondCode)) { in LowerSELECT()
24284 CC = DAG.getTargetConstant(X86CondCode, DL, MVT::i8); in LowerSELECT()
24292 CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8); in LowerSELECT()
24293 Cond = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget); in LowerSELECT()
24307 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(), in LowerSELECT()
24308 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cond); in LowerSELECT()
24310 return DAG.getNOT(DL, Res, Res.getValueType()); in LowerSELECT()
24324 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1, in LowerSELECT()
24326 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov); in LowerSELECT()
24339 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1); in LowerSELECT()
24340 Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2); in LowerSELECT()
24342 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops); in LowerSELECT()
24343 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov); in LowerSELECT()
24349 return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops, Op->getFlags()); in LowerSELECT()
24354 SelectionDAG &DAG) { in LowerSIGN_EXTEND_Mask() argument
24367 return SplitAndExtendv16i1(Op.getOpcode(), VT, In, dl, DAG); in LowerSIGN_EXTEND_Mask()
24377 In = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, InVT, DAG.getUNDEF(InVT), in LowerSIGN_EXTEND_Mask()
24378 In, DAG.getIntPtrConstant(0, dl)); in LowerSIGN_EXTEND_Mask()
24386 V = DAG.getNode(Op.getOpcode(), dl, WideVT, In); in LowerSIGN_EXTEND_Mask()
24388 SDValue NegOne = DAG.getConstant(-1, dl, WideVT); in LowerSIGN_EXTEND_Mask()
24389 SDValue Zero = DAG.getConstant(0, dl, WideVT); in LowerSIGN_EXTEND_Mask()
24390 V = DAG.getSelect(dl, WideVT, In, NegOne, Zero); in LowerSIGN_EXTEND_Mask()
24396 V = DAG.getNode(ISD::TRUNCATE, dl, WideVT, V); in LowerSIGN_EXTEND_Mask()
24401 V = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, V, in LowerSIGN_EXTEND_Mask()
24402 DAG.getIntPtrConstant(0, dl)); in LowerSIGN_EXTEND_Mask()
24408 SelectionDAG &DAG) { in LowerANY_EXTEND() argument
24414 return LowerSIGN_EXTEND_Mask(Op, DL, Subtarget, DAG); in LowerANY_EXTEND()
24417 return LowerAVXExtend(Op, DL, DAG, Subtarget); in LowerANY_EXTEND()
24426 SelectionDAG &DAG) { in LowerEXTEND_VECTOR_INREG() argument
24454 In = extractSubVector(In, 0, DAG, dl, std::max(InSize, 128)); in LowerEXTEND_VECTOR_INREG()
24465 return DAG.getNode(Op.getOpcode(), dl, VT, In); in LowerEXTEND_VECTOR_INREG()
24472 return DAG.getNode(ExtOpc, dl, VT, In); in LowerEXTEND_VECTOR_INREG()
24486 SDValue Lo = DAG.getNode(Opc, dl, HalfVT, In); in LowerEXTEND_VECTOR_INREG()
24487 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, DAG.getUNDEF(InVT), HiMask); in LowerEXTEND_VECTOR_INREG()
24488 Hi = DAG.getNode(Opc, dl, HalfVT, Hi); in LowerEXTEND_VECTOR_INREG()
24489 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerEXTEND_VECTOR_INREG()
24500 if (DAG.ComputeNumSignBits(In, DemandedElts) == InVT.getScalarSizeInBits()) { in LowerEXTEND_VECTOR_INREG()
24505 return DAG.getBitcast(VT, in LowerEXTEND_VECTOR_INREG()
24506 DAG.getVectorShuffle(InVT, dl, In, In, ShuffleMask)); in LowerEXTEND_VECTOR_INREG()
24528 Curr = DAG.getVectorShuffle(InVT, dl, In, In, Mask); in LowerEXTEND_VECTOR_INREG()
24529 Curr = DAG.getBitcast(DestVT, Curr); in LowerEXTEND_VECTOR_INREG()
24532 SignExt = DAG.getNode(X86ISD::VSRAI, dl, DestVT, Curr, in LowerEXTEND_VECTOR_INREG()
24533 DAG.getTargetConstant(SignExtShift, dl, MVT::i8)); in LowerEXTEND_VECTOR_INREG()
24538 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32); in LowerEXTEND_VECTOR_INREG()
24539 SDValue Sign = DAG.getSetCC(dl, MVT::v4i32, Zero, Curr, ISD::SETGT); in LowerEXTEND_VECTOR_INREG()
24540 SignExt = DAG.getVectorShuffle(MVT::v4i32, dl, SignExt, Sign, {0, 4, 1, 5}); in LowerEXTEND_VECTOR_INREG()
24541 SignExt = DAG.getBitcast(VT, SignExt); in LowerEXTEND_VECTOR_INREG()
24548 SelectionDAG &DAG) { in LowerSIGN_EXTEND() argument
24555 return LowerSIGN_EXTEND_Mask(Op, dl, Subtarget, DAG); in LowerSIGN_EXTEND()
24571 return splitVectorIntUnary(Op, DAG, dl); in LowerSIGN_EXTEND()
24586 SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In); in LowerSIGN_EXTEND()
24593 SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask); in LowerSIGN_EXTEND()
24594 OpHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, OpHi); in LowerSIGN_EXTEND()
24596 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); in LowerSIGN_EXTEND()
24600 static SDValue splitVectorStore(StoreSDNode *Store, SelectionDAG &DAG) { in splitVectorStore() argument
24616 std::tie(Value0, Value1) = splitVector(StoredVal, DAG, DL); in splitVectorStore()
24620 DAG.getMemBasePlusOffset(Ptr0, TypeSize::getFixed(HalfOffset), DL); in splitVectorStore()
24622 DAG.getStore(Store->getChain(), DL, Value0, Ptr0, Store->getPointerInfo(), in splitVectorStore()
24625 SDValue Ch1 = DAG.getStore(Store->getChain(), DL, Value1, Ptr1, in splitVectorStore()
24629 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Ch0, Ch1); in splitVectorStore()
24635 SelectionDAG &DAG) { in scalarizeVectorStore() argument
24639 StoredVal = DAG.getBitcast(StoreVT, StoredVal); in scalarizeVectorStore()
24655 SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(), in scalarizeVectorStore()
24657 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal, in scalarizeVectorStore()
24658 DAG.getIntPtrConstant(i, DL)); in scalarizeVectorStore()
24659 SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr, in scalarizeVectorStore()
24665 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores); in scalarizeVectorStore()
24669 SelectionDAG &DAG) { in LowerStore() argument
24684 StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1, in LowerStore()
24685 DAG.getUNDEF(MVT::v16i1), StoredVal, in LowerStore()
24686 DAG.getIntPtrConstant(0, dl)); in LowerStore()
24687 StoredVal = DAG.getBitcast(MVT::i16, StoredVal); in LowerStore()
24688 StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal); in LowerStore()
24691 StoredVal = DAG.getZeroExtendInReg( in LowerStore()
24692 StoredVal, dl, EVT::getIntegerVT(*DAG.getContext(), NumElts)); in LowerStore()
24694 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), in LowerStore()
24710 if (StoredVal.hasOneUse() && isFreeToSplitVector(StoredVal.getNode(), DAG)) in LowerStore()
24711 return splitVectorStore(St, DAG); in LowerStore()
24718 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerStore()
24720 assert(TLI.getTypeAction(*DAG.getContext(), StoreVT) == in LowerStore()
24724 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StoreVT); in LowerStore()
24725 StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, StoredVal, in LowerStore()
24726 DAG.getUNDEF(StoreVT)); in LowerStore()
24733 StoredVal = DAG.getBitcast(CastVT, StoredVal); in LowerStore()
24734 StoredVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, StVT, StoredVal, in LowerStore()
24735 DAG.getIntPtrConstant(0, dl)); in LowerStore()
24737 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), in LowerStore()
24742 SDVTList Tys = DAG.getVTList(MVT::Other); in LowerStore()
24744 return DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops, MVT::i64, in LowerStore()
24756 SelectionDAG &DAG) { in LowerLoad() argument
24772 SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(), in LowerLoad()
24779 SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd); in LowerLoad()
24780 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RegVT, in LowerLoad()
24781 DAG.getBitcast(MVT::v16i1, Val), in LowerLoad()
24782 DAG.getIntPtrConstant(0, dl)); in LowerLoad()
24783 return DAG.getMergeValues({Val, NewLd.getValue(1)}, dl); in LowerLoad()
24801 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { in LowerBRCOND()
24823 std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, LHS.getValue(0), DAG); in LowerBRCOND()
24828 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8); in LowerBRCOND()
24829 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
24835 SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, CC, SDLoc(Cond), DAG, CCVal); in LowerBRCOND()
24836 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
24854 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest); in LowerBRCOND()
24860 DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS); in LowerBRCOND()
24861 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8); in LowerBRCOND()
24862 Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, in LowerBRCOND()
24864 CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8); in LowerBRCOND()
24865 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
24873 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS); in LowerBRCOND()
24874 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8); in LowerBRCOND()
24876 DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, Cmp); in LowerBRCOND()
24877 CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8); in LowerBRCOND()
24878 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
24882 TranslateX86CC(CC, dl, /*IsFP*/ true, LHS, RHS, DAG); in LowerBRCOND()
24883 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS); in LowerBRCOND()
24884 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8); in LowerBRCOND()
24885 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
24893 std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG); in LowerBRCOND()
24895 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8); in LowerBRCOND()
24896 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
24901 if (isTruncWithZeroHighBitsInput(Cond, DAG)) in LowerBRCOND()
24909 DAG.getNode(ISD::AND, dl, CondVT, Cond, DAG.getConstant(1, dl, CondVT)); in LowerBRCOND()
24912 SDValue RHS = DAG.getConstant(0, dl, CondVT); in LowerBRCOND()
24915 SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, ISD::SETNE, dl, DAG, CCVal); in LowerBRCOND()
24916 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBRCOND()
24927 SelectionDAG &DAG) const { in LowerDYNAMIC_STACKALLOC()
24928 MachineFunction &MF = DAG.getMachineFunction(); in LowerDYNAMIC_STACKALLOC()
24944 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl); in LowerDYNAMIC_STACKALLOC()
24947 MVT SPTy = getPointerTy(DAG.getDataLayout()); in LowerDYNAMIC_STACKALLOC()
24951 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerDYNAMIC_STACKALLOC()
24963 Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size); in LowerDYNAMIC_STACKALLOC()
24964 Result = DAG.getNode(X86ISD::PROBED_ALLOCA, dl, SPTy, Chain, in LowerDYNAMIC_STACKALLOC()
24965 DAG.getRegister(Vreg, SPTy)); in LowerDYNAMIC_STACKALLOC()
24967 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); in LowerDYNAMIC_STACKALLOC()
24969 Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value in LowerDYNAMIC_STACKALLOC()
24973 DAG.getNode(ISD::AND, dl, VT, Result, in LowerDYNAMIC_STACKALLOC()
24974 DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT)); in LowerDYNAMIC_STACKALLOC()
24975 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Result); // Output chain in LowerDYNAMIC_STACKALLOC()
24992 Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size); in LowerDYNAMIC_STACKALLOC()
24993 Result = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain, in LowerDYNAMIC_STACKALLOC()
24994 DAG.getRegister(Vreg, SPTy)); in LowerDYNAMIC_STACKALLOC()
24996 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerDYNAMIC_STACKALLOC()
24997 Chain = DAG.getNode(X86ISD::DYN_ALLOCA, dl, NodeTys, Chain, Size); in LowerDYNAMIC_STACKALLOC()
25002 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy); in LowerDYNAMIC_STACKALLOC()
25006 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
25007 DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT)); in LowerDYNAMIC_STACKALLOC()
25008 Chain = DAG.getCopyToReg(Chain, dl, SPReg, SP); in LowerDYNAMIC_STACKALLOC()
25014 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl); in LowerDYNAMIC_STACKALLOC()
25017 return DAG.getMergeValues(Ops, dl); in LowerDYNAMIC_STACKALLOC()
25020 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { in LowerVASTART()
25021 MachineFunction &MF = DAG.getMachineFunction(); in LowerVASTART()
25032 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); in LowerVASTART()
25033 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1), in LowerVASTART()
25045 SDValue Store = DAG.getStore( in LowerVASTART()
25047 DAG.getConstant(FuncInfo->getVarArgsGPOffset(), DL, MVT::i32), FIN, in LowerVASTART()
25052 FIN = DAG.getMemBasePlusOffset(FIN, TypeSize::getFixed(4), DL); in LowerVASTART()
25053 Store = DAG.getStore( in LowerVASTART()
25055 DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32), FIN, in LowerVASTART()
25060 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4, DL)); in LowerVASTART()
25061 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); in LowerVASTART()
25063 DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN, MachinePointerInfo(SV, 8)); in LowerVASTART()
25067 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant( in LowerVASTART()
25069 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT); in LowerVASTART()
25070 Store = DAG.getStore( in LowerVASTART()
25074 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerVASTART()
25077 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { in LowerVAARG()
25082 MachineFunction &MF = DAG.getMachineFunction(); in LowerVAARG()
25085 return DAG.expandVAArg(Op.getNode()); in LowerVAARG()
25094 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); in LowerVAARG()
25095 uint32_t ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy); in LowerVAARG()
25120 DAG.getTargetConstant(ArgSize, dl, MVT::i32), in LowerVAARG()
25121 DAG.getTargetConstant(ArgMode, dl, MVT::i8), in LowerVAARG()
25122 DAG.getTargetConstant(Align, dl, MVT::i32)}; in LowerVAARG()
25123 SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other); in LowerVAARG()
25124 SDValue VAARG = DAG.getMemIntrinsicNode( in LowerVAARG()
25132 return DAG.getLoad(ArgVT, dl, Chain, VAARG, MachinePointerInfo()); in LowerVAARG()
25136 SelectionDAG &DAG) { in LowerVACOPY() argument
25141 DAG.getMachineFunction().getFunction().getCallingConv())) in LowerVACOPY()
25143 return DAG.expandVACopy(Op.getNode()); in LowerVACOPY()
25152 return DAG.getMemcpy( in LowerVACOPY()
25154 DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 24 : 16, DL), in LowerVACOPY()
25183 SelectionDAG &DAG) { in getTargetVShiftByConstNode() argument
25189 SrcOp = DAG.getBitcast(VT, SrcOp); in getTargetVShiftByConstNode()
25200 return DAG.getConstant(0, dl, VT); in getTargetVShiftByConstNode()
25223 SDValue Amt = DAG.getConstant(ShiftAmt, dl, VT); in getTargetVShiftByConstNode()
25224 if (SDValue C = DAG.FoldConstantArithmetic(ShiftOpc, dl, VT, {SrcOp, Amt})) in getTargetVShiftByConstNode()
25228 return DAG.getNode(Opc, dl, VT, SrcOp, in getTargetVShiftByConstNode()
25229 DAG.getTargetConstant(ShiftAmt, dl, MVT::i8)); in getTargetVShiftByConstNode()
25236 SelectionDAG &DAG) { in getTargetVShiftNode() argument
25246 ShAmt = DAG.getVectorShuffle(AmtVT, dl, ShAmt, DAG.getUNDEF(AmtVT), Mask); in getTargetVShiftNode()
25268 ShAmt = DAG.getZExtOrTrunc(ShAmt.getOperand(0), dl, MVT::i32); in getTargetVShiftNode()
25269 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt); in getTargetVShiftNode()
25270 ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, ShAmt); in getTargetVShiftNode()
25279 DAG.getConstant(0, dl, AmtVT.getScalarType())); in getTargetVShiftNode()
25280 MaskElts[0] = DAG.getAllOnesConstant(dl, AmtVT.getScalarType()); in getTargetVShiftNode()
25281 SDValue Mask = DAG.getBuildVector(AmtVT, dl, MaskElts); in getTargetVShiftNode()
25282 if ((Mask = DAG.FoldConstantArithmetic(ISD::AND, dl, AmtVT, in getTargetVShiftNode()
25284 ShAmt = DAG.getNode(ISD::AND, dl, AmtVT, ShAmt.getOperand(0), Mask); in getTargetVShiftNode()
25292 ShAmt = extract128BitVector(ShAmt, 0, DAG, dl); in getTargetVShiftNode()
25301 ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, SDLoc(ShAmt), MVT::v4i32, ShAmt); in getTargetVShiftNode()
25303 ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt), in getTargetVShiftNode()
25306 SDValue ByteShift = DAG.getTargetConstant( in getTargetVShiftNode()
25308 ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt); in getTargetVShiftNode()
25309 ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt, in getTargetVShiftNode()
25311 ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt, in getTargetVShiftNode()
25324 ShAmt = DAG.getBitcast(ShVT, ShAmt); in getTargetVShiftNode()
25325 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt); in getTargetVShiftNode()
25331 const X86Subtarget &Subtarget, SelectionDAG &DAG, in getMaskNode() argument
25335 return DAG.getConstant(1, dl, MaskVT); in getMaskNode()
25337 return DAG.getConstant(0, dl, MaskVT); in getMaskNode()
25346 std::tie(Lo, Hi) = DAG.SplitScalar(Mask, dl, MVT::i32, MVT::i32); in getMaskNode()
25347 Lo = DAG.getBitcast(MVT::v32i1, Lo); in getMaskNode()
25348 Hi = DAG.getBitcast(MVT::v32i1, Hi); in getMaskNode()
25349 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi); in getMaskNode()
25355 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT, in getMaskNode()
25356 DAG.getBitcast(BitcastVT, Mask), in getMaskNode()
25357 DAG.getIntPtrConstant(0, dl)); in getMaskNode()
25367 SelectionDAG &DAG) { in getVectorMaskingNode() argument
25376 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getVectorMaskingNode()
25379 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl); in getVectorMaskingNode()
25380 return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc); in getVectorMaskingNode()
25393 SelectionDAG &DAG) { in getScalarMaskingNode() argument
25403 SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1, in getScalarMaskingNode()
25404 DAG.getBitcast(MVT::v8i1, Mask), in getScalarMaskingNode()
25405 DAG.getIntPtrConstant(0, dl)); in getScalarMaskingNode()
25409 return DAG.getNode(ISD::AND, dl, VT, Op, IMask); in getScalarMaskingNode()
25412 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl); in getScalarMaskingNode()
25413 return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc); in getScalarMaskingNode()
25439 static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn, in recoverFramePointer() argument
25441 MachineFunction &MF = DAG.getMachineFunction(); in recoverFramePointer()
25444 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in recoverFramePointer()
25445 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); in recoverFramePointer()
25457 SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT); in recoverFramePointer()
25459 DAG.getNode(ISD::LOCAL_RECOVER, dl, PtrVT, OffsetSymVal); in recoverFramePointer()
25463 const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>(); in recoverFramePointer()
25465 return DAG.getNode(ISD::ADD, dl, PtrVT, EntryEBP, ParentFrameOffset); in recoverFramePointer()
25470 SDValue RegNodeBase = DAG.getNode(ISD::SUB, dl, PtrVT, EntryEBP, in recoverFramePointer()
25471 DAG.getConstant(RegNodeSize, dl, PtrVT)); in recoverFramePointer()
25472 return DAG.getNode(ISD::SUB, dl, PtrVT, RegNodeBase, ParentFrameOffset); in recoverFramePointer()
25476 SelectionDAG &DAG) const { in LowerINTRINSIC_WO_CHAIN()
25520 SelectionDAG::FlagInserter FlagsInserter(DAG, Op->getFlags()); in LowerINTRINSIC_WO_CHAIN()
25533 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25535 DAG.getTargetConstant(RC, dl, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
25539 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25553 return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
25566 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25568 DAG.getTargetConstant(RC, dl, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
25573 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25587 return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
25598 Src3 = DAG.getTargetConstant(Src3->getAsZExtVal() & 0xff, dl, MVT::i8); in LowerINTRINSIC_WO_CHAIN()
25609 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25611 DAG.getTargetConstant(RC, dl, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
25616 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25623 Src4 = DAG.getTargetConstant(Src4->getAsZExtVal() & 0xff, dl, MVT::i8); in LowerINTRINSIC_WO_CHAIN()
25626 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25643 DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25644 Src, DAG.getTargetConstant(RC, dl, MVT::i32)), in LowerINTRINSIC_WO_CHAIN()
25645 Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25650 DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru, in LowerINTRINSIC_WO_CHAIN()
25651 Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25667 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src), Mask, PassThru, in LowerINTRINSIC_WO_CHAIN()
25668 Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25686 DAG.getNode(IntrWithRoundingModeOpcode, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
25687 DAG.getTargetConstant(RC, dl, MVT::i32)), in LowerINTRINSIC_WO_CHAIN()
25688 Mask, passThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25692 return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1, in LowerINTRINSIC_WO_CHAIN()
25694 Mask, passThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25708 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, in LowerINTRINSIC_WO_CHAIN()
25710 Mask, passThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25722 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2); in LowerINTRINSIC_WO_CHAIN()
25724 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
25725 DAG.getTargetConstant(RC, dl, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
25729 return getScalarMaskingNode(NewOp, Mask, passThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25745 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), in LowerINTRINSIC_WO_CHAIN()
25746 Mask, passThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25758 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
25759 DAG.getTargetConstant(RC, dl, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
25764 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2); in LowerINTRINSIC_WO_CHAIN()
25765 return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25782 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), in LowerINTRINSIC_WO_CHAIN()
25783 Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25800 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3), in LowerINTRINSIC_WO_CHAIN()
25801 Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25818 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3), in LowerINTRINSIC_WO_CHAIN()
25819 Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25827 Src3 = DAG.getBitcast(MaskVT, Src3); in LowerINTRINSIC_WO_CHAIN()
25830 return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1); in LowerINTRINSIC_WO_CHAIN()
25837 return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1); in LowerINTRINSIC_WO_CHAIN()
25849 PassThru = getZeroVector(VT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
25859 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, Src3, in LowerINTRINSIC_WO_CHAIN()
25860 DAG.getTargetConstant(RC, dl, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
25865 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2, Src3); in LowerINTRINSIC_WO_CHAIN()
25866 return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25871 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25877 SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm); in LowerINTRINSIC_WO_CHAIN()
25879 Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25882 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1, in LowerINTRINSIC_WO_CHAIN()
25883 DAG.getConstant(0, dl, MVT::v8i1), in LowerINTRINSIC_WO_CHAIN()
25884 FPclassMask, DAG.getIntPtrConstant(0, dl)); in LowerINTRINSIC_WO_CHAIN()
25885 return DAG.getBitcast(MVT::i8, Ins); in LowerINTRINSIC_WO_CHAIN()
25898 return DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
25904 return DAG.getNode(IntrData->Opc0, dl, MaskVT, in LowerINTRINSIC_WO_CHAIN()
25917 Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::v1i1, Src1, Src2, CC, Sae); in LowerINTRINSIC_WO_CHAIN()
25923 Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC); in LowerINTRINSIC_WO_CHAIN()
25926 Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
25929 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1, in LowerINTRINSIC_WO_CHAIN()
25930 DAG.getConstant(0, dl, MVT::v8i1), in LowerINTRINSIC_WO_CHAIN()
25931 CmpMask, DAG.getIntPtrConstant(0, dl)); in LowerINTRINSIC_WO_CHAIN()
25932 return DAG.getBitcast(MVT::i8, Ins); in LowerINTRINSIC_WO_CHAIN()
25942 SDValue Comi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
25946 SetCC = getSETCC(X86::COND_E, Comi, dl, DAG); in LowerINTRINSIC_WO_CHAIN()
25947 SDValue SetNP = getSETCC(X86::COND_NP, Comi, dl, DAG); in LowerINTRINSIC_WO_CHAIN()
25948 SetCC = DAG.getNode(ISD::AND, dl, MVT::i8, SetCC, SetNP); in LowerINTRINSIC_WO_CHAIN()
25952 SetCC = getSETCC(X86::COND_NE, Comi, dl, DAG); in LowerINTRINSIC_WO_CHAIN()
25953 SDValue SetP = getSETCC(X86::COND_P, Comi, dl, DAG); in LowerINTRINSIC_WO_CHAIN()
25954 SetCC = DAG.getNode(ISD::OR, dl, MVT::i8, SetCC, SetP); in LowerINTRINSIC_WO_CHAIN()
25959 SetCC = getSETCC(X86::COND_A, Comi, dl, DAG); in LowerINTRINSIC_WO_CHAIN()
25964 SetCC = getSETCC(X86::COND_AE, Comi, dl, DAG); in LowerINTRINSIC_WO_CHAIN()
25969 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
25979 FCmp = DAG.getNode(X86ISD::FSETCCM, dl, MVT::v1i1, LHS, RHS, in LowerINTRINSIC_WO_CHAIN()
25980 DAG.getTargetConstant(CondVal, dl, MVT::i8)); in LowerINTRINSIC_WO_CHAIN()
25982 FCmp = DAG.getNode(X86ISD::FSETCCM_SAE, dl, MVT::v1i1, LHS, RHS, in LowerINTRINSIC_WO_CHAIN()
25983 DAG.getTargetConstant(CondVal, dl, MVT::i8), Sae); in LowerINTRINSIC_WO_CHAIN()
25988 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1, in LowerINTRINSIC_WO_CHAIN()
25989 DAG.getConstant(0, dl, MVT::v16i1), in LowerINTRINSIC_WO_CHAIN()
25990 FCmp, DAG.getIntPtrConstant(0, dl)); in LowerINTRINSIC_WO_CHAIN()
25991 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
25992 DAG.getBitcast(MVT::i16, Ins)); in LowerINTRINSIC_WO_CHAIN()
26004 CShAmt->getZExtValue(), DAG); in LowerINTRINSIC_WO_CHAIN()
26006 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt); in LowerINTRINSIC_WO_CHAIN()
26008 SrcOp, ShAmt, 0, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26019 PassThru = getZeroVector(VT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
26021 return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru, in LowerINTRINSIC_WO_CHAIN()
26033 : getZeroVector(VT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
26044 SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm); in LowerINTRINSIC_WO_CHAIN()
26047 return getVectorMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26049 return getScalarMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
26056 SDValue RoundingMode = DAG.getTargetConstant(Round & 0xf, dl, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
26057 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26065 SDValue RoundingMode = DAG.getTargetConstant(Round & 0xf, dl, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
26066 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26073 SDValue Control = DAG.getTargetConstant(Imm & 0xffff, dl, in LowerINTRINSIC_WO_CHAIN()
26075 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26080 SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
26081 SDVTList VTs = DAG.getVTList(Op.getOperand(2).getValueType(), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
26087 Res = DAG.getNode(IntrData->Opc1, dl, VTs, Op.getOperand(2), in LowerINTRINSIC_WO_CHAIN()
26090 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
26091 DAG.getConstant(-1, dl, MVT::i8)); in LowerINTRINSIC_WO_CHAIN()
26092 Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(2), in LowerINTRINSIC_WO_CHAIN()
26095 SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG); in LowerINTRINSIC_WO_CHAIN()
26097 return DAG.getMergeValues(Results, dl); in LowerINTRINSIC_WO_CHAIN()
26108 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src); in LowerINTRINSIC_WO_CHAIN()
26112 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
26113 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26128 Rnd = DAG.getTargetConstant(RC, dl, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
26132 return DAG.getNode(Opc, dl, Op.getValueType(), Src, Rnd); in LowerINTRINSIC_WO_CHAIN()
26140 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
26141 return DAG.getNode(Opc, dl, Op.getValueType(), Src, Rnd, PassThru, Mask); in LowerINTRINSIC_WO_CHAIN()
26149 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src); in LowerINTRINSIC_WO_CHAIN()
26153 PassThru = DAG.getConstant(0, dl, PassThru.getValueType()); in LowerINTRINSIC_WO_CHAIN()
26155 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, PassThru, in LowerINTRINSIC_WO_CHAIN()
26251 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
26252 SDValue SetCC = getSETCC(X86CC, Test, dl, DAG); in LowerINTRINSIC_WO_CHAIN()
26253 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
26312 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
26313 SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2); in LowerINTRINSIC_WO_CHAIN()
26314 SDValue SetCC = getSETCC(X86CC, PCMP, dl, DAG); in LowerINTRINSIC_WO_CHAIN()
26315 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
26327 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
26328 return DAG.getNode(Opcode, dl, VTs, NewOps); in LowerINTRINSIC_WO_CHAIN()
26340 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
26341 return DAG.getNode(Opcode, dl, VTs, NewOps).getValue(1); in LowerINTRINSIC_WO_CHAIN()
26345 MachineFunction &MF = DAG.getMachineFunction(); in LowerINTRINSIC_WO_CHAIN()
26346 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerINTRINSIC_WO_CHAIN()
26347 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); in LowerINTRINSIC_WO_CHAIN()
26351 return DAG.getNode(getGlobalWrapperKind(nullptr, /*OpFlags=*/0), dl, VT, in LowerINTRINSIC_WO_CHAIN()
26352 DAG.getMCSymbol(S, PtrVT)); in LowerINTRINSIC_WO_CHAIN()
26357 MachineFunction &MF = DAG.getMachineFunction(); in LowerINTRINSIC_WO_CHAIN()
26365 SDValue Result = DAG.getMCSymbol(LSDASym, VT); in LowerINTRINSIC_WO_CHAIN()
26366 return DAG.getNode(X86ISD::Wrapper, dl, VT, Result); in LowerINTRINSIC_WO_CHAIN()
26377 return recoverFramePointer(DAG, Fn, IncomingFPOp); in LowerINTRINSIC_WO_CHAIN()
26383 MachineFunction &MF = DAG.getMachineFunction(); in LowerINTRINSIC_WO_CHAIN()
26395 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT); in LowerINTRINSIC_WO_CHAIN()
26405 SDVTList VTs = DAG.getVTList(MVT::Untyped, MVT::Other); in LowerINTRINSIC_WO_CHAIN()
26409 DAG.getNode(X86ISD::VP2INTERSECT, DL, VTs, in LowerINTRINSIC_WO_CHAIN()
26412 SDValue Result0 = DAG.getTargetExtractSubreg(X86::sub_mask_0, DL, in LowerINTRINSIC_WO_CHAIN()
26414 SDValue Result1 = DAG.getTargetExtractSubreg(X86::sub_mask_1, DL, in LowerINTRINSIC_WO_CHAIN()
26416 return DAG.getMergeValues({Result0, Result1}, DL); in LowerINTRINSIC_WO_CHAIN()
26436 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26438 DAG.getTargetConstant(ShiftAmount, DL, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
26473 ShAmt = DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, ShAmt); in LowerINTRINSIC_WO_CHAIN()
26474 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26475 DAG.getTargetConstant(NewIntrinsic, DL, in LowerINTRINSIC_WO_CHAIN()
26476 getPointerTy(DAG.getDataLayout())), in LowerINTRINSIC_WO_CHAIN()
26482 EVT PtrVT = getPointerTy(DAG.getDataLayout()); in LowerINTRINSIC_WO_CHAIN()
26485 *DAG.getContext(), Subtarget.is64Bit() ? X86AS::FS : X86AS::GS)); in LowerINTRINSIC_WO_CHAIN()
26486 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), in LowerINTRINSIC_WO_CHAIN()
26487 DAG.getIntPtrConstant(0, dl), MachinePointerInfo(Ptr)); in LowerINTRINSIC_WO_CHAIN()
26495 static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, in getAVX2GatherNode() argument
26504 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getAVX2GatherNode()
26505 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, in getAVX2GatherNode()
26506 TLI.getPointerTy(DAG.getDataLayout())); in getAVX2GatherNode()
26508 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other); in getAVX2GatherNode()
26513 Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl); in getAVX2GatherNode()
26516 Mask = DAG.getBitcast(MaskVT, Mask); in getAVX2GatherNode()
26522 DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops, in getAVX2GatherNode()
26524 return DAG.getMergeValues({Res, Res.getValue(1)}, dl); in getAVX2GatherNode()
26527 static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG, in getGatherNode() argument
26537 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getGatherNode()
26538 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, in getGatherNode()
26539 TLI.getPointerTy(DAG.getDataLayout())); in getGatherNode()
26547 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getGatherNode()
26549 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other); in getGatherNode()
26554 Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl); in getGatherNode()
26560 DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops, in getGatherNode()
26562 return DAG.getMergeValues({Res, Res.getValue(1)}, dl); in getGatherNode()
26565 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, in getScatterNode() argument
26574 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getScatterNode()
26575 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, in getScatterNode()
26576 TLI.getPointerTy(DAG.getDataLayout())); in getScatterNode()
26584 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getScatterNode()
26588 SDVTList VTs = DAG.getVTList(MVT::Other); in getScatterNode()
26591 DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops, in getScatterNode()
26596 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, in getPrefetchNode() argument
26605 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in getPrefetchNode()
26606 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, in getPrefetchNode()
26607 TLI.getPointerTy(DAG.getDataLayout())); in getPrefetchNode()
26608 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32); in getPrefetchNode()
26609 SDValue Segment = DAG.getRegister(0, MVT::i32); in getPrefetchNode()
26612 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getPrefetchNode()
26614 SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops); in getPrefetchNode()
26627 SelectionDAG &DAG, in expandIntrinsicWChainHelper() argument
26637 Chain = DAG.getCopyToReg(Chain, DL, SrcReg, N->getOperand(2), Glue); in expandIntrinsicWChainHelper()
26641 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in expandIntrinsicWChainHelper()
26643 SDNode *N1 = DAG.getMachineNode( in expandIntrinsicWChainHelper()
26650 LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1)); in expandIntrinsicWChainHelper()
26651 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64, in expandIntrinsicWChainHelper()
26654 LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1)); in expandIntrinsicWChainHelper()
26655 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32, in expandIntrinsicWChainHelper()
26663 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in expandIntrinsicWChainHelper()
26664 DAG.getConstant(32, DL, MVT::i8)); in expandIntrinsicWChainHelper()
26665 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in expandIntrinsicWChainHelper()
26672 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in expandIntrinsicWChainHelper()
26682 SelectionDAG &DAG, in getReadTimeStampCounter() argument
26688 SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode, in getReadTimeStampCounter()
26697 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue); in getReadTimeStampCounter()
26703 SelectionDAG &DAG) { in LowerREADCYCLECOUNTER() argument
26706 getReadTimeStampCounter(Op.getNode(), DL, X86::RDTSC, DAG, Subtarget, in LowerREADCYCLECOUNTER()
26708 return DAG.getMergeValues(Results, DL); in LowerREADCYCLECOUNTER()
26711 static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG) { in MarkEHRegistrationNode() argument
26712 MachineFunction &MF = DAG.getMachineFunction(); in MarkEHRegistrationNode()
26729 static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) { in MarkEHGuard() argument
26730 MachineFunction &MF = DAG.getMachineFunction(); in MarkEHGuard()
26751 SelectionDAG &DAG) { in EmitTruncSStore() argument
26752 SDVTList VTs = DAG.getVTList(MVT::Other); in EmitTruncSStore()
26753 SDValue Undef = DAG.getUNDEF(Ptr.getValueType()); in EmitTruncSStore()
26756 return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO); in EmitTruncSStore()
26763 MachineMemOperand *MMO, SelectionDAG &DAG) { in EmitMaskedTruncSStore() argument
26764 SDVTList VTs = DAG.getVTList(MVT::Other); in EmitMaskedTruncSStore()
26767 return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO); in EmitMaskedTruncSStore()
26780 SelectionDAG &DAG) { in LowerINTRINSIC_W_CHAIN() argument
26788 auto &MF = DAG.getMachineFunction(); in LowerINTRINSIC_W_CHAIN()
26794 SDValue CopyRBP = DAG.getCopyFromReg(Chain, dl, X86::RBP, MVT::i64); in LowerINTRINSIC_W_CHAIN()
26796 SDValue(DAG.getMachineNode(X86::SUB64ri32, dl, MVT::i64, CopyRBP, in LowerINTRINSIC_W_CHAIN()
26797 DAG.getTargetConstant(8, dl, MVT::i32)), in LowerINTRINSIC_W_CHAIN()
26800 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, in LowerINTRINSIC_W_CHAIN()
26810 DAG.getFrameIndex(*X86FI->getSwiftAsyncContextFrameIdx(), in LowerINTRINSIC_W_CHAIN()
26813 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, in LowerINTRINSIC_W_CHAIN()
26819 return MarkEHRegistrationNode(Op, DAG); in LowerINTRINSIC_W_CHAIN()
26821 return MarkEHGuard(Op, DAG); in LowerINTRINSIC_W_CHAIN()
26824 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other); in LowerINTRINSIC_W_CHAIN()
26826 return DAG.getNode(X86ISD::RDPKRU, dl, VTs, Op.getOperand(0), in LowerINTRINSIC_W_CHAIN()
26827 DAG.getConstant(0, dl, MVT::i32)); in LowerINTRINSIC_W_CHAIN()
26833 return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other, in LowerINTRINSIC_W_CHAIN()
26835 DAG.getConstant(0, dl, MVT::i32), in LowerINTRINSIC_W_CHAIN()
26836 DAG.getConstant(0, dl, MVT::i32)); in LowerINTRINSIC_W_CHAIN()
26840 DAG.getMachineFunction().getFrameInfo().setAdjustsStack(true); in LowerINTRINSIC_W_CHAIN()
26850 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); in LowerINTRINSIC_W_CHAIN()
26862 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other); in LowerINTRINSIC_W_CHAIN()
26880 DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2), in LowerINTRINSIC_W_CHAIN()
26882 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG); in LowerINTRINSIC_W_CHAIN()
26883 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, in LowerINTRINSIC_W_CHAIN()
26890 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other); in LowerINTRINSIC_W_CHAIN()
26901 SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op.getOperand(2), in LowerINTRINSIC_W_CHAIN()
26903 SDValue SetCC = getSETCC(X86::COND_E, Operation.getValue(0), dl, DAG); in LowerINTRINSIC_W_CHAIN()
26904 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, in LowerINTRINSIC_W_CHAIN()
26912 SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::i32, MVT::Other); in LowerINTRINSIC_W_CHAIN()
26935 SDValue Operation = DAG.getMemIntrinsicNode( in LowerINTRINSIC_W_CHAIN()
26938 SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(1), DL, DAG); in LowerINTRINSIC_W_CHAIN()
26940 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
26948 SDVTList VTs = DAG.getVTList( in LowerINTRINSIC_W_CHAIN()
26973 SDValue Operation = DAG.getMemIntrinsicNode( in LowerINTRINSIC_W_CHAIN()
26979 SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(0), DL, DAG); in LowerINTRINSIC_W_CHAIN()
26981 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
26991 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other); in LowerINTRINSIC_W_CHAIN()
26992 SDValue Operation = DAG.getNode(X86ISD::TESTUI, dl, VTs, Chain); in LowerINTRINSIC_W_CHAIN()
26993 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG); in LowerINTRINSIC_W_CHAIN()
26994 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, in LowerINTRINSIC_W_CHAIN()
27010 DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other), in LowerINTRINSIC_W_CHAIN()
27013 Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT); in LowerINTRINSIC_W_CHAIN()
27014 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain); in LowerINTRINSIC_W_CHAIN()
27027 SDValue Size = DAG.getConstant(VT.getScalarSizeInBits(), DL, MVT::i32); in LowerINTRINSIC_W_CHAIN()
27030 DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other), in LowerINTRINSIC_W_CHAIN()
27033 Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT); in LowerINTRINSIC_W_CHAIN()
27036 Res = DAG.getNode(ISD::SHL, DL, VT, Res, in LowerINTRINSIC_W_CHAIN()
27037 DAG.getShiftAmountConstant(Imm, VT, DL)); in LowerINTRINSIC_W_CHAIN()
27038 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain); in LowerINTRINSIC_W_CHAIN()
27049 SDValue Operation = DAG.getMemIntrinsicNode( in LowerINTRINSIC_W_CHAIN()
27089 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
27125 DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other), in LowerINTRINSIC_W_CHAIN()
27128 return DAG.getMergeValues({getSETCC(CC, LockArith, DL, DAG), Chain}, DL); in LowerINTRINSIC_W_CHAIN()
27140 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32, MVT::Other); in LowerINTRINSIC_W_CHAIN()
27141 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0)); in LowerINTRINSIC_W_CHAIN()
27145 SDValue Ops[] = {DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)), in LowerINTRINSIC_W_CHAIN()
27146 DAG.getConstant(1, dl, Op->getValueType(1)), in LowerINTRINSIC_W_CHAIN()
27147 DAG.getTargetConstant(X86::COND_B, dl, MVT::i8), in LowerINTRINSIC_W_CHAIN()
27149 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops); in LowerINTRINSIC_W_CHAIN()
27152 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid, in LowerINTRINSIC_W_CHAIN()
27162 return getAVX2GatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, in LowerINTRINSIC_W_CHAIN()
27173 return getGatherNode(Op, DAG, Src, Mask, Base, Index, Scale, in LowerINTRINSIC_W_CHAIN()
27184 return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, in LowerINTRINSIC_W_CHAIN()
27197 return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain, in LowerINTRINSIC_W_CHAIN()
27203 getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget, in LowerINTRINSIC_W_CHAIN()
27205 return DAG.getMergeValues(Results, dl); in LowerINTRINSIC_W_CHAIN()
27219 expandIntrinsicWChainHelper(Op.getNode(), dl, DAG, IntrData->Opc0, X86::ECX, in LowerINTRINSIC_W_CHAIN()
27221 return DAG.getMergeValues(Results, dl); in LowerINTRINSIC_W_CHAIN()
27225 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other); in LowerINTRINSIC_W_CHAIN()
27226 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0)); in LowerINTRINSIC_W_CHAIN()
27228 SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG); in LowerINTRINSIC_W_CHAIN()
27229 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC); in LowerINTRINSIC_W_CHAIN()
27230 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
27250 return DAG.getTruncStore(Chain, dl, DataToTruncate, Addr, MemVT, in LowerINTRINSIC_W_CHAIN()
27254 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_W_CHAIN()
27255 SDValue Offset = DAG.getUNDEF(VMask.getValueType()); in LowerINTRINSIC_W_CHAIN()
27257 return DAG.getMaskedStore(Chain, dl, DataToTruncate, Addr, Offset, VMask, in LowerINTRINSIC_W_CHAIN()
27266 MemIntr->getMemOperand(), DAG); in LowerINTRINSIC_W_CHAIN()
27269 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_W_CHAIN()
27272 VMask, MemVT, MemIntr->getMemOperand(), DAG); in LowerINTRINSIC_W_CHAIN()
27282 SelectionDAG &DAG) const { in LowerRETURNADDR()
27283 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); in LowerRETURNADDR()
27286 if (verifyReturnAddressArgumentIsConstant(Op, DAG)) in LowerRETURNADDR()
27291 EVT PtrVT = getPointerTy(DAG.getDataLayout()); in LowerRETURNADDR()
27294 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); in LowerRETURNADDR()
27296 SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), dl, PtrVT); in LowerRETURNADDR()
27297 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), in LowerRETURNADDR()
27298 DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset), in LowerRETURNADDR()
27303 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG); in LowerRETURNADDR()
27304 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI, in LowerRETURNADDR()
27309 SelectionDAG &DAG) const { in LowerADDROFRETURNADDR()
27310 DAG.getMachineFunction().getFrameInfo().setReturnAddressIsTaken(true); in LowerADDROFRETURNADDR()
27311 return getReturnAddressFrameIndex(DAG); in LowerADDROFRETURNADDR()
27314 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { in LowerFRAMEADDR()
27315 MachineFunction &MF = DAG.getMachineFunction(); in LowerFRAMEADDR()
27335 return DAG.getFrameIndex(FrameAddrIndex, VT); in LowerFRAMEADDR()
27339 RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction()); in LowerFRAMEADDR()
27345 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT); in LowerFRAMEADDR()
27347 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, in LowerFRAMEADDR()
27388 SelectionDAG &DAG) const { in LowerFRAME_TO_ARGS_OFFSET()
27390 return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize(), SDLoc(Op)); in LowerFRAME_TO_ARGS_OFFSET()
27413 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const { in LowerEH_RETURN()
27419 EVT PtrVT = getPointerTy(DAG.getDataLayout()); in LowerEH_RETURN()
27421 Register FrameReg = RegInfo->getFrameRegister(DAG.getMachineFunction()); in LowerEH_RETURN()
27425 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT); in LowerEH_RETURN()
27428 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame, in LowerEH_RETURN()
27429 DAG.getIntPtrConstant(RegInfo->getSlotSize(), in LowerEH_RETURN()
27431 StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset); in LowerEH_RETURN()
27432 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo()); in LowerEH_RETURN()
27433 Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr); in LowerEH_RETURN()
27435 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain, in LowerEH_RETURN()
27436 DAG.getRegister(StoreAddrReg, PtrVT)); in LowerEH_RETURN()
27440 SelectionDAG &DAG) const { in lowerEH_SJLJ_SETJMP()
27450 (void)TII->getGlobalBaseReg(&DAG.getMachineFunction()); in lowerEH_SJLJ_SETJMP()
27452 return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL, in lowerEH_SJLJ_SETJMP()
27453 DAG.getVTList(MVT::i32, MVT::Other), in lowerEH_SJLJ_SETJMP()
27458 SelectionDAG &DAG) const { in lowerEH_SJLJ_LONGJMP()
27460 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other, in lowerEH_SJLJ_LONGJMP()
27465 SelectionDAG &DAG) const { in lowerEH_SJLJ_SETUP_DISPATCH()
27467 return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other, in lowerEH_SJLJ_SETUP_DISPATCH()
27471 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) { in LowerADJUST_TRAMPOLINE() argument
27476 SelectionDAG &DAG) const { in LowerINIT_TRAMPOLINE()
27501 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16), in LowerINIT_TRAMPOLINE()
27504 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
27505 DAG.getConstant(2, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
27506 OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr, in LowerINIT_TRAMPOLINE()
27512 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
27513 DAG.getConstant(10, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
27514 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16), in LowerINIT_TRAMPOLINE()
27517 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
27518 DAG.getConstant(12, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
27519 OutChains[3] = DAG.getStore(Root, dl, Nest, Addr, in LowerINIT_TRAMPOLINE()
27524 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
27525 DAG.getConstant(20, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
27526 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16), in LowerINIT_TRAMPOLINE()
27530 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
27531 DAG.getConstant(22, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
27532 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, dl, MVT::i8), in LowerINIT_TRAMPOLINE()
27535 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
27562 const DataLayout &DL = DAG.getDataLayout(); in LowerINIT_TRAMPOLINE()
27588 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
27589 DAG.getConstant(10, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
27590 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr); in LowerINIT_TRAMPOLINE()
27596 DAG.getStore(Root, dl, DAG.getConstant(MOV32ri | N86Reg, dl, MVT::i8), in LowerINIT_TRAMPOLINE()
27599 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
27600 DAG.getConstant(1, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
27601 OutChains[1] = DAG.getStore(Root, dl, Nest, Addr, in LowerINIT_TRAMPOLINE()
27605 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
27606 DAG.getConstant(5, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
27608 DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8), Addr, in LowerINIT_TRAMPOLINE()
27611 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
27612 DAG.getConstant(6, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
27613 OutChains[3] = DAG.getStore(Root, dl, Disp, Addr, in LowerINIT_TRAMPOLINE()
27616 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
27621 SelectionDAG &DAG) const { in LowerGET_ROUNDING()
27644 MachineFunction &MF = DAG.getMachineFunction(); in LowerGET_ROUNDING()
27651 DAG.getFrameIndex(SSFI, getPointerTy(DAG.getDataLayout())); in LowerGET_ROUNDING()
27657 Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL, in LowerGET_ROUNDING()
27658 DAG.getVTList(MVT::Other), Ops, MVT::i16, MPI, in LowerGET_ROUNDING()
27662 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI, Align(2)); in LowerGET_ROUNDING()
27667 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerGET_ROUNDING()
27668 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerGET_ROUNDING()
27669 CWD, DAG.getConstant(0xc00, DL, MVT::i16)), in LowerGET_ROUNDING()
27670 DAG.getConstant(9, DL, MVT::i8)); in LowerGET_ROUNDING()
27671 Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Shift); in LowerGET_ROUNDING()
27673 SDValue LUT = DAG.getConstant(0x2d, DL, MVT::i32); in LowerGET_ROUNDING()
27675 DAG.getNode(ISD::AND, DL, MVT::i32, in LowerGET_ROUNDING()
27676 DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift), in LowerGET_ROUNDING()
27677 DAG.getConstant(3, DL, MVT::i32)); in LowerGET_ROUNDING()
27679 RetVal = DAG.getZExtOrTrunc(RetVal, DL, VT); in LowerGET_ROUNDING()
27681 return DAG.getMergeValues({RetVal, Chain}, DL); in LowerGET_ROUNDING()
27685 SelectionDAG &DAG) const { in LowerSET_ROUNDING()
27686 MachineFunction &MF = DAG.getMachineFunction(); in LowerSET_ROUNDING()
27694 DAG.getFrameIndex(OldCWFrameIdx, getPointerTy(DAG.getDataLayout())); in LowerSET_ROUNDING()
27701 Chain = DAG.getMemIntrinsicNode( in LowerSET_ROUNDING()
27702 X86ISD::FNSTCW16m, DL, DAG.getVTList(MVT::Other), Ops, MVT::i16, MMO); in LowerSET_ROUNDING()
27705 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI); in LowerSET_ROUNDING()
27707 CWD = DAG.getNode(ISD::AND, DL, MVT::i16, CWD.getValue(0), in LowerSET_ROUNDING()
27708 DAG.getConstant(0xf3ff, DL, MVT::i16)); in LowerSET_ROUNDING()
27726 RMBits = DAG.getConstant(FieldVal, DL, MVT::i16); in LowerSET_ROUNDING()
27741 DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, in LowerSET_ROUNDING()
27742 DAG.getNode(ISD::ADD, DL, MVT::i32, in LowerSET_ROUNDING()
27743 DAG.getNode(ISD::SHL, DL, MVT::i32, NewRM, in LowerSET_ROUNDING()
27744 DAG.getConstant(1, DL, MVT::i8)), in LowerSET_ROUNDING()
27745 DAG.getConstant(4, DL, MVT::i32))); in LowerSET_ROUNDING()
27747 DAG.getNode(ISD::SHL, DL, MVT::i16, DAG.getConstant(0xc9, DL, MVT::i16), in LowerSET_ROUNDING()
27749 RMBits = DAG.getNode(ISD::AND, DL, MVT::i16, Shifted, in LowerSET_ROUNDING()
27750 DAG.getConstant(0xc00, DL, MVT::i16)); in LowerSET_ROUNDING()
27754 CWD = DAG.getNode(ISD::OR, DL, MVT::i16, CWD, RMBits); in LowerSET_ROUNDING()
27755 Chain = DAG.getStore(Chain, DL, CWD, StackSlot, MPI, Align(2)); in LowerSET_ROUNDING()
27761 Chain = DAG.getMemIntrinsicNode( in LowerSET_ROUNDING()
27762 X86ISD::FLDCW16m, DL, DAG.getVTList(MVT::Other), OpsLD, MVT::i16, MMOL); in LowerSET_ROUNDING()
27768 Chain = DAG.getNode( in LowerSET_ROUNDING()
27769 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain, in LowerSET_ROUNDING()
27770 DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32), in LowerSET_ROUNDING()
27774 SDValue CWD = DAG.getLoad(MVT::i32, DL, Chain, StackSlot, MPI); in LowerSET_ROUNDING()
27776 CWD = DAG.getNode(ISD::AND, DL, MVT::i32, CWD.getValue(0), in LowerSET_ROUNDING()
27777 DAG.getConstant(0xffff9fff, DL, MVT::i32)); in LowerSET_ROUNDING()
27780 RMBits = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, RMBits); in LowerSET_ROUNDING()
27781 RMBits = DAG.getNode(ISD::SHL, DL, MVT::i32, RMBits, in LowerSET_ROUNDING()
27782 DAG.getConstant(3, DL, MVT::i8)); in LowerSET_ROUNDING()
27785 CWD = DAG.getNode(ISD::OR, DL, MVT::i32, CWD, RMBits); in LowerSET_ROUNDING()
27786 Chain = DAG.getStore(Chain, DL, CWD, StackSlot, MPI, Align(4)); in LowerSET_ROUNDING()
27789 Chain = DAG.getNode( in LowerSET_ROUNDING()
27790 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain, in LowerSET_ROUNDING()
27791 DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32), in LowerSET_ROUNDING()
27803 SelectionDAG &DAG) const { in LowerGET_FPENV_MEM()
27804 MachineFunction &MF = DAG.getMachineFunction(); in LowerGET_FPENV_MEM()
27816 DAG.getMemIntrinsicNode(X86ISD::FNSTENVm, DL, DAG.getVTList(MVT::Other), in LowerGET_FPENV_MEM()
27825 DAG.getMemIntrinsicNode(X86ISD::FLDENVm, DL, DAG.getVTList(MVT::Other), in LowerGET_FPENV_MEM()
27832 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); in LowerGET_FPENV_MEM()
27833 SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, in LowerGET_FPENV_MEM()
27834 DAG.getConstant(X87StateSize, DL, PtrVT)); in LowerGET_FPENV_MEM()
27836 Chain = DAG.getNode( in LowerGET_FPENV_MEM()
27837 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain, in LowerGET_FPENV_MEM()
27838 DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32), in LowerGET_FPENV_MEM()
27847 SelectionDAG &DAG, in createSetFPEnvNodes() argument
27852 DAG.getMemIntrinsicNode(X86ISD::FLDENVm, DL, DAG.getVTList(MVT::Other), in createSetFPEnvNodes()
27857 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); in createSetFPEnvNodes()
27858 SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, in createSetFPEnvNodes()
27859 DAG.getConstant(X87StateSize, DL, PtrVT)); in createSetFPEnvNodes()
27861 Chain = DAG.getNode( in createSetFPEnvNodes()
27862 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain, in createSetFPEnvNodes()
27863 DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32), in createSetFPEnvNodes()
27870 SelectionDAG &DAG) const { in LowerSET_FPENV_MEM()
27878 return createSetFPEnvNodes(Ptr, Chain, DL, MemVT, MMO, DAG, Subtarget); in LowerSET_FPENV_MEM()
27882 SelectionDAG &DAG) const { in LowerRESET_FPENV()
27883 MachineFunction &MF = DAG.getMachineFunction(); in LowerRESET_FPENV()
27887 IntegerType *ItemTy = Type::getInt32Ty(*DAG.getContext()); in LowerRESET_FPENV()
27904 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); in LowerRESET_FPENV()
27905 SDValue Env = DAG.getConstantPool(FPEnvBits, PtrVT); in LowerRESET_FPENV()
27907 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()); in LowerRESET_FPENV()
27911 return createSetFPEnvNodes(Env, Chain, DL, MVT::i32, MMO, DAG, Subtarget); in LowerRESET_FPENV()
27920 static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG, in LowerVectorCTLZ_AVX512CDI() argument
27934 return splitVectorIntUnary(Op, DAG, dl); in LowerVectorCTLZ_AVX512CDI()
27941 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, NewVT, Op.getOperand(0)); in LowerVectorCTLZ_AVX512CDI()
27942 SDValue CtlzNode = DAG.getNode(ISD::CTLZ, dl, NewVT, Op); in LowerVectorCTLZ_AVX512CDI()
27943 SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode); in LowerVectorCTLZ_AVX512CDI()
27944 SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT); in LowerVectorCTLZ_AVX512CDI()
27946 return DAG.getNode(ISD::SUB, dl, VT, TruncNode, Delta); in LowerVectorCTLZ_AVX512CDI()
27952 SelectionDAG &DAG) { in LowerVectorCTLZInRegLUT() argument
27966 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8)); in LowerVectorCTLZInRegLUT()
27967 SDValue InRegLUT = DAG.getBuildVector(CurrVT, DL, LUTVec); in LowerVectorCTLZInRegLUT()
27974 SDValue Op0 = DAG.getBitcast(CurrVT, Op.getOperand(0)); in LowerVectorCTLZInRegLUT()
27975 SDValue Zero = DAG.getConstant(0, DL, CurrVT); in LowerVectorCTLZInRegLUT()
27977 SDValue NibbleShift = DAG.getConstant(0x4, DL, CurrVT); in LowerVectorCTLZInRegLUT()
27979 SDValue Hi = DAG.getNode(ISD::SRL, DL, CurrVT, Op0, NibbleShift); in LowerVectorCTLZInRegLUT()
27983 HiZ = DAG.getSetCC(DL, MaskVT, Hi, Zero, ISD::SETEQ); in LowerVectorCTLZInRegLUT()
27984 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ); in LowerVectorCTLZInRegLUT()
27986 HiZ = DAG.getSetCC(DL, CurrVT, Hi, Zero, ISD::SETEQ); in LowerVectorCTLZInRegLUT()
27989 Lo = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Lo); in LowerVectorCTLZInRegLUT()
27990 Hi = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Hi); in LowerVectorCTLZInRegLUT()
27991 Lo = DAG.getNode(ISD::AND, DL, CurrVT, Lo, HiZ); in LowerVectorCTLZInRegLUT()
27992 SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi); in LowerVectorCTLZInRegLUT()
28004 SDValue Shift = DAG.getConstant(CurrScalarSizeInBits, DL, NextVT); in LowerVectorCTLZInRegLUT()
28009 HiZ = DAG.getSetCC(DL, MaskVT, DAG.getBitcast(CurrVT, Op0), in LowerVectorCTLZInRegLUT()
28010 DAG.getBitcast(CurrVT, Zero), ISD::SETEQ); in LowerVectorCTLZInRegLUT()
28011 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ); in LowerVectorCTLZInRegLUT()
28013 HiZ = DAG.getSetCC(DL, CurrVT, DAG.getBitcast(CurrVT, Op0), in LowerVectorCTLZInRegLUT()
28014 DAG.getBitcast(CurrVT, Zero), ISD::SETEQ); in LowerVectorCTLZInRegLUT()
28016 HiZ = DAG.getBitcast(NextVT, HiZ); in LowerVectorCTLZInRegLUT()
28021 SDValue ResNext = Res = DAG.getBitcast(NextVT, Res); in LowerVectorCTLZInRegLUT()
28022 SDValue R0 = DAG.getNode(ISD::SRL, DL, NextVT, ResNext, Shift); in LowerVectorCTLZInRegLUT()
28023 SDValue R1 = DAG.getNode(ISD::SRL, DL, NextVT, HiZ, Shift); in LowerVectorCTLZInRegLUT()
28024 R1 = DAG.getNode(ISD::AND, DL, NextVT, ResNext, R1); in LowerVectorCTLZInRegLUT()
28025 Res = DAG.getNode(ISD::ADD, DL, NextVT, R0, R1); in LowerVectorCTLZInRegLUT()
28034 SelectionDAG &DAG) { in LowerVectorCTLZ() argument
28040 return LowerVectorCTLZ_AVX512CDI(Op, DAG, Subtarget); in LowerVectorCTLZ()
28044 return splitVectorIntUnary(Op, DAG, DL); in LowerVectorCTLZ()
28048 return splitVectorIntUnary(Op, DAG, DL); in LowerVectorCTLZ()
28051 return LowerVectorCTLZInRegLUT(Op, DL, Subtarget, DAG); in LowerVectorCTLZ()
28055 SelectionDAG &DAG) { in LowerCTLZ() argument
28063 return LowerVectorCTLZ(Op, dl, Subtarget, DAG); in LowerCTLZ()
28069 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); in LowerCTLZ()
28073 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); in LowerCTLZ()
28074 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op); in LowerCTLZ()
28078 SDValue Ops[] = {Op, DAG.getConstant(NumBits + NumBits - 1, dl, OpVT), in LowerCTLZ()
28079 DAG.getTargetConstant(X86::COND_E, dl, MVT::i8), in LowerCTLZ()
28081 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops); in LowerCTLZ()
28085 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, in LowerCTLZ()
28086 DAG.getConstant(NumBits - 1, dl, OpVT)); in LowerCTLZ()
28089 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ()
28094 SelectionDAG &DAG) { in LowerCTTZ() argument
28104 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerCTTZ()
28105 Op = DAG.getNode(X86ISD::BSF, dl, VTs, N0); in LowerCTTZ()
28108 if (DAG.isKnownNeverZero(N0)) in LowerCTTZ()
28112 SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT), in LowerCTTZ()
28113 DAG.getTargetConstant(X86::COND_E, dl, MVT::i8), in LowerCTTZ()
28115 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops); in LowerCTTZ()
28118 static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG, in lowerAddSub() argument
28124 return lowerAddSubToHorizontalOp(Op, DL, DAG, Subtarget); in lowerAddSub()
28127 return splitVectorIntBinary(Op, DAG, DL); in lowerAddSub()
28132 return splitVectorIntBinary(Op, DAG, DL); in lowerAddSub()
28135 static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG, in LowerADDSAT_SUBSAT() argument
28146 return splitVectorIntBinary(Op, DAG, DL); in LowerADDSAT_SUBSAT()
28150 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerADDSAT_SUBSAT()
28152 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerADDSAT_SUBSAT()
28164 SDValue SignMask = DAG.getConstant(C->getAPIntValue(), DL, VT); in LowerADDSAT_SUBSAT()
28165 SDValue ShiftAmt = DAG.getConstant(BitWidth - 1, DL, VT); in LowerADDSAT_SUBSAT()
28166 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, SignMask); in LowerADDSAT_SUBSAT()
28167 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShiftAmt); in LowerADDSAT_SUBSAT()
28168 return DAG.getNode(ISD::AND, DL, VT, Xor, Sra); in LowerADDSAT_SUBSAT()
28173 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y); in LowerADDSAT_SUBSAT()
28174 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT); in LowerADDSAT_SUBSAT()
28177 DAG.ComputeNumSignBits(Cmp) == VT.getScalarSizeInBits()) in LowerADDSAT_SUBSAT()
28178 return DAG.getNode(ISD::AND, DL, VT, Cmp, Sub); in LowerADDSAT_SUBSAT()
28179 return DAG.getSelect(DL, VT, Cmp, Sub, DAG.getConstant(0, DL, VT)); in LowerADDSAT_SUBSAT()
28187 SDValue Zero = DAG.getConstant(0, DL, VT); in LowerADDSAT_SUBSAT()
28189 DAG.getNode(Opcode == ISD::SADDSAT ? ISD::SADDO : ISD::SSUBO, DL, in LowerADDSAT_SUBSAT()
28190 DAG.getVTList(VT, SetCCResultType), X, Y); in LowerADDSAT_SUBSAT()
28193 SDValue SatMin = DAG.getConstant(MinVal, DL, VT); in LowerADDSAT_SUBSAT()
28194 SDValue SatMax = DAG.getConstant(MaxVal, DL, VT); in LowerADDSAT_SUBSAT()
28196 DAG.getSetCC(DL, SetCCResultType, SumDiff, Zero, ISD::SETLT); in LowerADDSAT_SUBSAT()
28197 Result = DAG.getSelect(DL, VT, SumNeg, SatMax, SatMin); in LowerADDSAT_SUBSAT()
28198 return DAG.getSelect(DL, VT, Overflow, Result, SumDiff); in LowerADDSAT_SUBSAT()
28206 SelectionDAG &DAG) { in LowerABS() argument
28214 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32), in LowerABS()
28215 DAG.getConstant(0, DL, VT), N0); in LowerABS()
28216 SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_NS, DL, MVT::i8), in LowerABS()
28218 return DAG.getNode(X86ISD::CMOV, DL, VT, Ops); in LowerABS()
28224 SDValue Neg = DAG.getNegative(Src, DL, VT); in LowerABS()
28225 return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Neg, Src); in LowerABS()
28231 return splitVectorIntUnary(Op, DAG, DL); in LowerABS()
28235 return splitVectorIntUnary(Op, DAG, DL); in LowerABS()
28242 SelectionDAG &DAG) { in LowerAVG() argument
28248 return splitVectorIntBinary(Op, DAG, DL); in LowerAVG()
28251 return splitVectorIntBinary(Op, DAG, DL); in LowerAVG()
28258 SelectionDAG &DAG) { in LowerMINMAX() argument
28264 return splitVectorIntBinary(Op, DAG, DL); in LowerMINMAX()
28267 return splitVectorIntBinary(Op, DAG, DL); in LowerMINMAX()
28274 SelectionDAG &DAG) { in LowerFMINIMUM_FMAXIMUM() argument
28277 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerFMINIMUM_FMAXIMUM()
28295 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerFMINIMUM_FMAXIMUM()
28338 bool IsXNeverNaN = DAG.isKnownNeverNaN(X); in LowerFMINIMUM_FMAXIMUM()
28339 bool IsYNeverNaN = DAG.isKnownNeverNaN(Y); in LowerFMINIMUM_FMAXIMUM()
28340 bool IgnoreSignedZero = DAG.getTarget().Options.NoSignedZerosFPMath || in LowerFMINIMUM_FMAXIMUM()
28342 DAG.isKnownNeverZeroFloat(X) || in LowerFMINIMUM_FMAXIMUM()
28343 DAG.isKnownNeverZeroFloat(Y); in LowerFMINIMUM_FMAXIMUM()
28360 SDValue VX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VectorType, X); in LowerFMINIMUM_FMAXIMUM()
28364 SDValue Imm = DAG.getTargetConstant(MinMaxOp == X86ISD::FMAX ? 0b11 : 0b101, in LowerFMINIMUM_FMAXIMUM()
28366 SDValue IsNanZero = DAG.getNode(X86ISD::VFPCLASSS, DL, MVT::v1i1, VX, Imm); in LowerFMINIMUM_FMAXIMUM()
28367 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i1, in LowerFMINIMUM_FMAXIMUM()
28368 DAG.getConstant(0, DL, MVT::v8i1), IsNanZero, in LowerFMINIMUM_FMAXIMUM()
28369 DAG.getIntPtrConstant(0, DL)); in LowerFMINIMUM_FMAXIMUM()
28370 SDValue NeedSwap = DAG.getBitcast(MVT::i8, Ins); in LowerFMINIMUM_FMAXIMUM()
28371 NewX = DAG.getSelect(DL, VT, NeedSwap, Y, X); in LowerFMINIMUM_FMAXIMUM()
28372 NewY = DAG.getSelect(DL, VT, NeedSwap, X, Y); in LowerFMINIMUM_FMAXIMUM()
28373 return DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags()); in LowerFMINIMUM_FMAXIMUM()
28377 SDValue XInt = DAG.getNode(ISD::BITCAST, DL, IVT, X); in LowerFMINIMUM_FMAXIMUM()
28378 SDValue ZeroCst = DAG.getConstant(0, DL, IVT); in LowerFMINIMUM_FMAXIMUM()
28379 IsXSigned = DAG.getSetCC(DL, SetCCType, XInt, ZeroCst, ISD::SETLT); in LowerFMINIMUM_FMAXIMUM()
28382 SDValue Ins = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v2f64, in LowerFMINIMUM_FMAXIMUM()
28383 DAG.getConstantFP(0, DL, MVT::v2f64), X, in LowerFMINIMUM_FMAXIMUM()
28384 DAG.getIntPtrConstant(0, DL)); in LowerFMINIMUM_FMAXIMUM()
28385 SDValue VX = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, Ins); in LowerFMINIMUM_FMAXIMUM()
28386 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VX, in LowerFMINIMUM_FMAXIMUM()
28387 DAG.getIntPtrConstant(1, DL)); in LowerFMINIMUM_FMAXIMUM()
28388 Hi = DAG.getBitcast(MVT::i32, Hi); in LowerFMINIMUM_FMAXIMUM()
28389 SDValue ZeroCst = DAG.getConstant(0, DL, MVT::i32); in LowerFMINIMUM_FMAXIMUM()
28390 EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(), in LowerFMINIMUM_FMAXIMUM()
28391 *DAG.getContext(), MVT::i32); in LowerFMINIMUM_FMAXIMUM()
28392 IsXSigned = DAG.getSetCC(DL, SetCCType, Hi, ZeroCst, ISD::SETLT); in LowerFMINIMUM_FMAXIMUM()
28395 NewX = DAG.getSelect(DL, VT, IsXSigned, X, Y); in LowerFMINIMUM_FMAXIMUM()
28396 NewY = DAG.getSelect(DL, VT, IsXSigned, Y, X); in LowerFMINIMUM_FMAXIMUM()
28398 NewX = DAG.getSelect(DL, VT, IsXSigned, Y, X); in LowerFMINIMUM_FMAXIMUM()
28399 NewY = DAG.getSelect(DL, VT, IsXSigned, X, Y); in LowerFMINIMUM_FMAXIMUM()
28403 bool IgnoreNaN = DAG.getTarget().Options.NoNaNsFPMath || in LowerFMINIMUM_FMAXIMUM()
28409 if (IgnoreSignedZero && !IgnoreNaN && DAG.isKnownNeverNaN(NewY)) in LowerFMINIMUM_FMAXIMUM()
28412 SDValue MinMax = DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags()); in LowerFMINIMUM_FMAXIMUM()
28414 if (IgnoreNaN || DAG.isKnownNeverNaN(NewX)) in LowerFMINIMUM_FMAXIMUM()
28417 SDValue IsNaN = DAG.getSetCC(DL, SetCCType, NewX, NewX, ISD::SETUO); in LowerFMINIMUM_FMAXIMUM()
28418 return DAG.getSelect(DL, VT, IsNaN, NewX, MinMax); in LowerFMINIMUM_FMAXIMUM()
28422 SelectionDAG &DAG) { in LowerABD() argument
28428 return splitVectorIntBinary(Op, DAG, dl); in LowerABD()
28431 return splitVectorIntBinary(Op, DAG, dl); in LowerABD()
28434 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerABD()
28444 SDValue LHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(0)); in LowerABD()
28445 SDValue RHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(1)); in LowerABD()
28446 SDValue Diff = DAG.getNode(ISD::SUB, dl, WideVT, LHS, RHS); in LowerABD()
28447 SDValue AbsDiff = DAG.getNode(ISD::ABS, dl, WideVT, Diff); in LowerABD()
28448 return DAG.getNode(ISD::TRUNCATE, dl, VT, AbsDiff); in LowerABD()
28457 SelectionDAG &DAG) { in LowerMUL() argument
28463 return splitVectorIntBinary(Op, DAG, dl); in LowerMUL()
28466 return splitVectorIntBinary(Op, DAG, dl); in LowerMUL()
28481 return DAG.getNode( in LowerMUL()
28483 DAG.getNode(ISD::MUL, dl, ExVT, in LowerMUL()
28484 DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, A), in LowerMUL()
28485 DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, B))); in LowerMUL()
28505 SDValue Mask = DAG.getBitcast(VT, DAG.getConstant(0x00FF, dl, ExVT)); in LowerMUL()
28506 SDValue BLo = DAG.getNode(ISD::AND, dl, VT, Mask, B); in LowerMUL()
28507 SDValue BHi = DAG.getNode(X86ISD::ANDNP, dl, VT, Mask, B); in LowerMUL()
28508 SDValue RLo = DAG.getNode(X86ISD::VPMADDUBSW, dl, ExVT, A, BLo); in LowerMUL()
28509 SDValue RHi = DAG.getNode(X86ISD::VPMADDUBSW, dl, ExVT, A, BHi); in LowerMUL()
28510 RLo = DAG.getNode(ISD::AND, dl, VT, DAG.getBitcast(VT, RLo), Mask); in LowerMUL()
28511 RHi = DAG.getNode(X86ISD::VSHLI, dl, ExVT, RHi, in LowerMUL()
28512 DAG.getTargetConstant(8, dl, MVT::i8)); in LowerMUL()
28513 return DAG.getNode(ISD::OR, dl, VT, RLo, DAG.getBitcast(VT, RHi)); in LowerMUL()
28521 SDValue Undef = DAG.getUNDEF(VT); in LowerMUL()
28522 SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef)); in LowerMUL()
28523 SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef)); in LowerMUL()
28531 LoOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j), dl, in LowerMUL()
28533 HiOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j + 8), dl, in LowerMUL()
28538 BLo = DAG.getBuildVector(ExVT, dl, LoOps); in LowerMUL()
28539 BHi = DAG.getBuildVector(ExVT, dl, HiOps); in LowerMUL()
28541 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Undef)); in LowerMUL()
28542 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Undef)); in LowerMUL()
28546 SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo); in LowerMUL()
28547 SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi); in LowerMUL()
28548 return getPack(DAG, Subtarget, dl, VT, RLo, RHi); in LowerMUL()
28558 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask); in LowerMUL()
28559 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask); in LowerMUL()
28562 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, in LowerMUL()
28563 DAG.getBitcast(MVT::v2i64, A), in LowerMUL()
28564 DAG.getBitcast(MVT::v2i64, B)); in LowerMUL()
28566 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, in LowerMUL()
28567 DAG.getBitcast(MVT::v2i64, Aodds), in LowerMUL()
28568 DAG.getBitcast(MVT::v2i64, Bodds)); in LowerMUL()
28570 Evens = DAG.getBitcast(VT, Evens); in LowerMUL()
28571 Odds = DAG.getBitcast(VT, Odds); in LowerMUL()
28576 return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask); in LowerMUL()
28592 KnownBits AKnown = DAG.computeKnownBits(A); in LowerMUL()
28593 KnownBits BKnown = DAG.computeKnownBits(B); in LowerMUL()
28603 SDValue Zero = DAG.getConstant(0, dl, VT); in LowerMUL()
28608 AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B); in LowerMUL()
28612 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG); in LowerMUL()
28613 AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi); in LowerMUL()
28618 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG); in LowerMUL()
28619 AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B); in LowerMUL()
28622 SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo); in LowerMUL()
28623 Hi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Hi, 32, DAG); in LowerMUL()
28625 return DAG.getNode(ISD::ADD, dl, VT, AloBlo, Hi); in LowerMUL()
28631 SelectionDAG &DAG, in LowervXi8MulWithUNPCK() argument
28648 SDValue Zero = DAG.getConstant(0, dl, VT); in LowervXi8MulWithUNPCK()
28652 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, A)); in LowervXi8MulWithUNPCK()
28653 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, A)); in LowervXi8MulWithUNPCK()
28655 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Zero)); in LowervXi8MulWithUNPCK()
28656 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Zero)); in LowervXi8MulWithUNPCK()
28669 LoOp = DAG.getAnyExtOrTrunc(LoOp, dl, MVT::i16); in LowervXi8MulWithUNPCK()
28670 HiOp = DAG.getAnyExtOrTrunc(HiOp, dl, MVT::i16); in LowervXi8MulWithUNPCK()
28671 LoOp = DAG.getNode(ISD::SHL, dl, MVT::i16, LoOp, in LowervXi8MulWithUNPCK()
28672 DAG.getConstant(8, dl, MVT::i16)); in LowervXi8MulWithUNPCK()
28673 HiOp = DAG.getNode(ISD::SHL, dl, MVT::i16, HiOp, in LowervXi8MulWithUNPCK()
28674 DAG.getConstant(8, dl, MVT::i16)); in LowervXi8MulWithUNPCK()
28676 LoOp = DAG.getZExtOrTrunc(LoOp, dl, MVT::i16); in LowervXi8MulWithUNPCK()
28677 HiOp = DAG.getZExtOrTrunc(HiOp, dl, MVT::i16); in LowervXi8MulWithUNPCK()
28685 BLo = DAG.getBuildVector(ExVT, dl, LoOps); in LowervXi8MulWithUNPCK()
28686 BHi = DAG.getBuildVector(ExVT, dl, HiOps); in LowervXi8MulWithUNPCK()
28688 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B)); in LowervXi8MulWithUNPCK()
28689 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B)); in LowervXi8MulWithUNPCK()
28691 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero)); in LowervXi8MulWithUNPCK()
28692 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Zero)); in LowervXi8MulWithUNPCK()
28698 SDValue RLo = DAG.getNode(MulOpc, dl, ExVT, ALo, BLo); in LowervXi8MulWithUNPCK()
28699 SDValue RHi = DAG.getNode(MulOpc, dl, ExVT, AHi, BHi); in LowervXi8MulWithUNPCK()
28702 *Low = getPack(DAG, Subtarget, dl, VT, RLo, RHi); in LowervXi8MulWithUNPCK()
28704 return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf*/ true); in LowervXi8MulWithUNPCK()
28708 SelectionDAG &DAG) { in LowerMULH() argument
28718 return splitVectorIntBinary(Op, DAG, dl); in LowerMULH()
28721 return splitVectorIntBinary(Op, DAG, dl); in LowerMULH()
28744 DAG.getVectorShuffle(VT, dl, A, A, ArrayRef(&Mask[0], NumElts)); in LowerMULH()
28747 DAG.getVectorShuffle(VT, dl, B, B, ArrayRef(&Mask[0], NumElts)); in LowerMULH()
28756 SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT, in LowerMULH()
28757 DAG.getBitcast(MulVT, A), in LowerMULH()
28758 DAG.getBitcast(MulVT, B))); in LowerMULH()
28761 SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT, in LowerMULH()
28762 DAG.getBitcast(MulVT, Odd0), in LowerMULH()
28763 DAG.getBitcast(MulVT, Odd1))); in LowerMULH()
28770 SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask); in LowerMULH()
28775 SDValue Zero = DAG.getConstant(0, dl, VT); in LowerMULH()
28776 SDValue T1 = DAG.getNode(ISD::AND, dl, VT, in LowerMULH()
28777 DAG.getSetCC(dl, VT, Zero, A, ISD::SETGT), B); in LowerMULH()
28778 SDValue T2 = DAG.getNode(ISD::AND, dl, VT, in LowerMULH()
28779 DAG.getSetCC(dl, VT, Zero, B, ISD::SETGT), A); in LowerMULH()
28781 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2); in LowerMULH()
28782 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Fixup); in LowerMULH()
28803 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A); in LowerMULH()
28804 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B); in LowerMULH()
28805 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB); in LowerMULH()
28806 Mul = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG); in LowerMULH()
28807 return DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in LowerMULH()
28810 return LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG); in LowerMULH()
28815 SelectionDAG &DAG) { in LowerMULO() argument
28820 return LowerXALUO(Op, DAG); in LowerMULO()
28832 std::tie(LHSLo, LHSHi) = splitVector(A, DAG, dl); in LowerMULO()
28836 std::tie(RHSLo, RHSHi) = splitVector(B, DAG, dl); in LowerMULO()
28839 std::tie(LoOvfVT, HiOvfVT) = DAG.GetSplitDestVTs(OvfVT); in LowerMULO()
28840 SDVTList LoVTs = DAG.getVTList(LHSLo.getValueType(), LoOvfVT); in LowerMULO()
28841 SDVTList HiVTs = DAG.getVTList(LHSHi.getValueType(), HiOvfVT); in LowerMULO()
28844 SDValue Lo = DAG.getNode(Op.getOpcode(), dl, LoVTs, LHSLo, RHSLo); in LowerMULO()
28845 SDValue Hi = DAG.getNode(Op.getOpcode(), dl, HiVTs, LHSHi, RHSHi); in LowerMULO()
28848 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerMULO()
28849 SDValue Ovf = DAG.getNode(ISD::CONCAT_VECTORS, dl, OvfVT, Lo.getValue(1), in LowerMULO()
28852 return DAG.getMergeValues({Res, Ovf}, dl); in LowerMULO()
28855 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerMULO()
28857 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerMULO()
28864 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A); in LowerMULO()
28865 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B); in LowerMULO()
28866 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB); in LowerMULO()
28868 SDValue Low = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in LowerMULO()
28877 High = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Mul, 8, DAG); in LowerMULO()
28880 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExVT, Mul, 8, DAG); in LowerMULO()
28882 15, DAG); in LowerMULO()
28886 High = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, High); in LowerMULO()
28887 LowSign = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, LowSign); in LowerMULO()
28891 High = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG); in LowerMULO()
28892 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High); in LowerMULO()
28894 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT)); in LowerMULO()
28897 Ovf = DAG.getSetCC(dl, SetccVT, LowSign, High, ISD::SETNE); in LowerMULO()
28900 getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG); in LowerMULO()
28907 High = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v16i32, High); in LowerMULO()
28911 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High); in LowerMULO()
28915 DAG.getSetCC(dl, SetccVT, High, in LowerMULO()
28916 DAG.getConstant(0, dl, High.getValueType()), ISD::SETNE); in LowerMULO()
28919 Ovf = DAG.getSExtOrTrunc(Ovf, dl, OvfVT); in LowerMULO()
28921 return DAG.getMergeValues({Low, Ovf}, dl); in LowerMULO()
28926 LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG, &Low); in LowerMULO()
28932 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT)); in LowerMULO()
28933 Ovf = DAG.getSetCC(dl, SetccVT, LowSign, High, ISD::SETNE); in LowerMULO()
28937 DAG.getSetCC(dl, SetccVT, High, DAG.getConstant(0, dl, VT), ISD::SETNE); in LowerMULO()
28940 Ovf = DAG.getSExtOrTrunc(Ovf, dl, OvfVT); in LowerMULO()
28942 return DAG.getMergeValues({Low, Ovf}, dl); in LowerMULO()
28945 SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const { in LowerWin64_i128OP()
28953 if (expandDIVREMByConstant(Op.getNode(), Result, MVT::i64, DAG)) in LowerWin64_i128OP()
28954 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(Op), VT, Result[0], Result[1]); in LowerWin64_i128OP()
28970 SDValue InChain = DAG.getEntryNode(); in LowerWin64_i128OP()
28978 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16); in LowerWin64_i128OP()
28981 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); in LowerWin64_i128OP()
28984 DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MPI, Align(16)); in LowerWin64_i128OP()
28985 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); in LowerWin64_i128OP()
28992 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC), in LowerWin64_i128OP()
28993 getPointerTy(DAG.getDataLayout())); in LowerWin64_i128OP()
28995 TargetLowering::CallLoweringInfo CLI(DAG); in LowerWin64_i128OP()
29000 static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()), Callee, in LowerWin64_i128OP()
29007 return DAG.getBitcast(VT, CallInfo.first); in LowerWin64_i128OP()
29011 SelectionDAG &DAG, in LowerWin64_FP_TO_INT128() argument
29033 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode(); in LowerWin64_FP_TO_INT128()
29039 makeLibCall(DAG, LC, MVT::v2i64, Arg, CallOptions, dl, Chain); in LowerWin64_FP_TO_INT128()
29040 Result = DAG.getBitcast(VT, Result); in LowerWin64_FP_TO_INT128()
29045 SelectionDAG &DAG) const { in LowerWin64_INT128_TO_FP()
29066 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode(); in LowerWin64_INT128_TO_FP()
29069 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16); in LowerWin64_INT128_TO_FP()
29072 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); in LowerWin64_INT128_TO_FP()
29073 Chain = DAG.getStore(Chain, dl, Arg, StackPtr, MPI, Align(16)); in LowerWin64_INT128_TO_FP()
29077 makeLibCall(DAG, LC, VT, StackPtr, CallOptions, dl, Chain); in LowerWin64_INT128_TO_FP()
29078 return IsStrict ? DAG.getMergeValues({Result, Chain}, dl) : Result; in LowerWin64_INT128_TO_FP()
29105 SDValue getGFNICtrlMask(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, MVT VT, in getGFNICtrlMask() argument
29113 MaskBits.push_back(DAG.getConstant(Bits, DL, MVT::i8)); in getGFNICtrlMask()
29115 return DAG.getBuildVector(VT, DL, MaskBits); in getGFNICtrlMask()
29183 static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG, in LowerShiftByScalarImmediate() argument
29195 SDValue Ex = DAG.getBitcast(ExVT, R); in LowerShiftByScalarImmediate()
29201 return DAG.getNode(X86ISD::PCMPGT, dl, VT, DAG.getConstant(0, dl, VT), R); in LowerShiftByScalarImmediate()
29207 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex, 31, DAG); in LowerShiftByScalarImmediate()
29209 ShiftAmt - 32, DAG); in LowerShiftByScalarImmediate()
29211 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {5, 1, 7, 3}); in LowerShiftByScalarImmediate()
29213 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, in LowerShiftByScalarImmediate()
29218 ShiftAmt, DAG); in LowerShiftByScalarImmediate()
29220 getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG); in LowerShiftByScalarImmediate()
29221 Lower = DAG.getBitcast(ExVT, Lower); in LowerShiftByScalarImmediate()
29223 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {4, 1, 6, 3}); in LowerShiftByScalarImmediate()
29225 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, in LowerShiftByScalarImmediate()
29228 return DAG.getBitcast(VT, Ex); in LowerShiftByScalarImmediate()
29238 return DAG.getUNDEF(VT); in LowerShiftByScalarImmediate()
29252 R = DAG.getFreeze(R); in LowerShiftByScalarImmediate()
29253 return DAG.getNode(ISD::ADD, dl, VT, R, R); in LowerShiftByScalarImmediate()
29256 return getTargetVShiftByConstNode(X86Opc, dl, VT, R, ShiftAmt, DAG); in LowerShiftByScalarImmediate()
29268 DAG.ComputeNumSignBits(R) == EltSizeInBits) { in LowerShiftByScalarImmediate()
29269 SDValue Mask = DAG.getAllOnesConstant(dl, VT); in LowerShiftByScalarImmediate()
29270 Mask = DAG.getNode(Op.getOpcode(), dl, VT, Mask, Amt); in LowerShiftByScalarImmediate()
29271 return DAG.getNode(ISD::AND, dl, VT, R, Mask); in LowerShiftByScalarImmediate()
29286 R = DAG.getFreeze(R); in LowerShiftByScalarImmediate()
29287 return DAG.getNode(ISD::ADD, dl, VT, R, R); in LowerShiftByScalarImmediate()
29292 SDValue Zeros = DAG.getConstant(0, dl, VT); in LowerShiftByScalarImmediate()
29295 SDValue CMP = DAG.getSetCC(dl, MVT::v64i1, Zeros, R, ISD::SETGT); in LowerShiftByScalarImmediate()
29296 return DAG.getNode(ISD::SIGN_EXTEND, dl, VT, CMP); in LowerShiftByScalarImmediate()
29298 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R); in LowerShiftByScalarImmediate()
29306 SDValue Mask = getGFNICtrlMask(Op.getOpcode(), DAG, dl, VT, ShiftAmt); in LowerShiftByScalarImmediate()
29307 return DAG.getNode(X86ISD::GF2P8AFFINEQB, dl, VT, R, Mask, in LowerShiftByScalarImmediate()
29308 DAG.getTargetConstant(0, dl, MVT::i8)); in LowerShiftByScalarImmediate()
29314 ShiftAmt, DAG); in LowerShiftByScalarImmediate()
29315 SHL = DAG.getBitcast(VT, SHL); in LowerShiftByScalarImmediate()
29318 return DAG.getNode(ISD::AND, dl, VT, SHL, DAG.getConstant(Mask, dl, VT)); in LowerShiftByScalarImmediate()
29323 ShiftAmt, DAG); in LowerShiftByScalarImmediate()
29324 SRL = DAG.getBitcast(VT, SRL); in LowerShiftByScalarImmediate()
29327 return DAG.getNode(ISD::AND, dl, VT, SRL, DAG.getConstant(Mask, dl, VT)); in LowerShiftByScalarImmediate()
29331 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt); in LowerShiftByScalarImmediate()
29333 SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT); in LowerShiftByScalarImmediate()
29334 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask); in LowerShiftByScalarImmediate()
29335 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask); in LowerShiftByScalarImmediate()
29344 static SDValue LowerShiftByScalarVariable(SDValue Op, SelectionDAG &DAG, in LowerShiftByScalarVariable() argument
29354 if (SDValue BaseShAmt = DAG.getSplatSourceVector(Amt, BaseShAmtIdx)) { in LowerShiftByScalarVariable()
29357 Subtarget, DAG); in LowerShiftByScalarVariable()
29372 SDValue BitMask = DAG.getConstant(-1, dl, ExtVT); in LowerShiftByScalarVariable()
29374 BaseShAmt, BaseShAmtIdx, Subtarget, DAG); in LowerShiftByScalarVariable()
29377 8, DAG); in LowerShiftByScalarVariable()
29378 BitMask = DAG.getBitcast(VT, BitMask); in LowerShiftByScalarVariable()
29379 BitMask = DAG.getVectorShuffle(VT, dl, BitMask, BitMask, in LowerShiftByScalarVariable()
29383 DAG.getBitcast(ExtVT, R), BaseShAmt, in LowerShiftByScalarVariable()
29384 BaseShAmtIdx, Subtarget, DAG); in LowerShiftByScalarVariable()
29385 Res = DAG.getBitcast(VT, Res); in LowerShiftByScalarVariable()
29386 Res = DAG.getNode(ISD::AND, dl, VT, Res, BitMask); in LowerShiftByScalarVariable()
29391 SDValue SignMask = DAG.getConstant(0x8080, dl, ExtVT); in LowerShiftByScalarVariable()
29394 BaseShAmtIdx, Subtarget, DAG); in LowerShiftByScalarVariable()
29395 SignMask = DAG.getBitcast(VT, SignMask); in LowerShiftByScalarVariable()
29396 Res = DAG.getNode(ISD::XOR, dl, VT, Res, SignMask); in LowerShiftByScalarVariable()
29397 Res = DAG.getNode(ISD::SUB, dl, VT, Res, SignMask); in LowerShiftByScalarVariable()
29410 SelectionDAG &DAG) { in convertShiftLeftToScale() argument
29428 SmallVector<SDValue> Elts(NumElems, DAG.getUNDEF(SVT)); in convertShiftLeftToScale()
29433 Elts[I] = DAG.getConstant(One.shl(ShAmt), dl, SVT); in convertShiftLeftToScale()
29435 return DAG.getBuildVector(VT, dl, Elts); in convertShiftLeftToScale()
29441 Amt = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT)); in convertShiftLeftToScale()
29442 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, in convertShiftLeftToScale()
29443 DAG.getConstant(0x3f800000U, dl, VT)); in convertShiftLeftToScale()
29444 Amt = DAG.getBitcast(MVT::v4f32, Amt); in convertShiftLeftToScale()
29445 return DAG.getNode(ISD::FP_TO_SINT, dl, VT, Amt); in convertShiftLeftToScale()
29450 SDValue Z = DAG.getConstant(0, dl, VT); in convertShiftLeftToScale()
29451 SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z)); in convertShiftLeftToScale()
29452 SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z)); in convertShiftLeftToScale()
29453 Lo = convertShiftLeftToScale(Lo, dl, Subtarget, DAG); in convertShiftLeftToScale()
29454 Hi = convertShiftLeftToScale(Hi, dl, Subtarget, DAG); in convertShiftLeftToScale()
29456 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi); in convertShiftLeftToScale()
29457 return getPack(DAG, Subtarget, dl, VT, Lo, Hi); in convertShiftLeftToScale()
29464 SelectionDAG &DAG) { in LowerShift() argument
29479 if (SDValue V = LowerShiftByScalarImmediate(Op, DAG, Subtarget)) in LowerShift()
29482 if (SDValue V = LowerShiftByScalarVariable(Op, DAG, Subtarget)) in LowerShift()
29494 SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT); in LowerShift()
29495 SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt); in LowerShift()
29496 R = DAG.getNode(ISD::SRL, dl, VT, R, Amt); in LowerShift()
29497 R = DAG.getNode(ISD::XOR, dl, VT, R, M); in LowerShift()
29498 R = DAG.getNode(ISD::SUB, dl, VT, R, M); in LowerShift()
29507 Amt = DAG.getNegative(Amt, dl, VT); in LowerShift()
29509 return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt); in LowerShift()
29511 return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt); in LowerShift()
29518 SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0}); in LowerShift()
29519 SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1}); in LowerShift()
29520 SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0); in LowerShift()
29521 SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1); in LowerShift()
29522 return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3}); in LowerShift()
29570 Cst1->getZExtValue(), DAG); in LowerShift()
29572 Cst2->getZExtValue(), DAG); in LowerShift()
29573 return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask); in LowerShift()
29583 if (SDValue Scale = convertShiftLeftToScale(Amt, dl, Subtarget, DAG)) in LowerShift()
29584 return DAG.getNode(ISD::MUL, dl, VT, R, Scale); in LowerShift()
29590 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT); in LowerShift()
29591 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt); in LowerShift()
29592 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) { in LowerShift()
29593 SDValue Zero = DAG.getConstant(0, dl, VT); in LowerShift()
29594 SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ); in LowerShift()
29595 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale); in LowerShift()
29596 return DAG.getSelect(dl, VT, ZAmt, R, Res); in LowerShift()
29608 DAG.isKnownNeverZero(Amt))) { in LowerShift()
29609 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT); in LowerShift()
29610 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt); in LowerShift()
29611 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) { in LowerShift()
29613 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(0, dl, VT), ISD::SETEQ); in LowerShift()
29615 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(1, dl, VT), ISD::SETEQ); in LowerShift()
29617 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, 1, DAG); in LowerShift()
29618 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale); in LowerShift()
29619 Res = DAG.getSelect(dl, VT, Amt0, R, Res); in LowerShift()
29620 return DAG.getSelect(dl, VT, Amt1, Sra1, Res); in LowerShift()
29632 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0}); in LowerShift()
29633 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1}); in LowerShift()
29634 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2}); in LowerShift()
29635 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3}); in LowerShift()
29642 SDValue Z = DAG.getConstant(0, dl, VT); in LowerShift()
29643 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1}); in LowerShift()
29644 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1}); in LowerShift()
29645 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1}); in LowerShift()
29646 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1}); in LowerShift()
29648 SDValue Amt01 = DAG.getBitcast(MVT::v8i16, Amt); in LowerShift()
29649 SDValue Amt23 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01, in LowerShift()
29651 SDValue Msk02 = getV4X86ShuffleImm8ForMask({0, 1, 1, 1}, dl, DAG); in LowerShift()
29652 SDValue Msk13 = getV4X86ShuffleImm8ForMask({2, 3, 3, 3}, dl, DAG); in LowerShift()
29653 Amt0 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk02); in LowerShift()
29654 Amt1 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk13); in LowerShift()
29655 Amt2 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk02); in LowerShift()
29656 Amt3 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk13); in LowerShift()
29661 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0)); in LowerShift()
29662 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1)); in LowerShift()
29663 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2)); in LowerShift()
29664 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3)); in LowerShift()
29669 SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1}); in LowerShift()
29670 SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7}); in LowerShift()
29671 return DAG.getVectorShuffle(VT, dl, R02, R13, {0, 5, 2, 7}); in LowerShift()
29673 SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5}); in LowerShift()
29674 SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7}); in LowerShift()
29675 return DAG.getVectorShuffle(VT, dl, R01, R23, {0, 3, 4, 7}); in LowerShift()
29692 R = DAG.getNode(ExtOpc, dl, ExtVT, R); in LowerShift()
29693 Amt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Amt); in LowerShift()
29694 return DAG.getNode(ISD::TRUNCATE, dl, VT, in LowerShift()
29695 DAG.getNode(Opc, dl, ExtVT, R, Amt)); in LowerShift()
29705 SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8); in LowerShift()
29710 Amt = DAG.getZExtOrTrunc(Amt, dl, ExVT); in LowerShift()
29711 Amt = DAG.getNode(ISD::SUB, dl, ExVT, DAG.getConstant(8, dl, ExVT), Amt); in LowerShift()
29712 Amt = DAG.getNode(ISD::SHL, dl, ExVT, DAG.getConstant(1, dl, ExVT), Amt); in LowerShift()
29718 R = DAG.getExtOrTrunc(IsSigned, R, dl, ExVT); in LowerShift()
29719 R = DAG.getNode(ISD::MUL, dl, ExVT, R, Amt); in LowerShift()
29720 R = DAG.getNode(X86ISD::VSRLI, dl, ExVT, R, Cst8); in LowerShift()
29721 return DAG.getZExtOrTrunc(R, dl, VT); in LowerShift()
29733 SDValue LoA = DAG.getBuildVector(VT16, dl, LoAmt); in LowerShift()
29734 SDValue HiA = DAG.getBuildVector(VT16, dl, HiAmt); in LowerShift()
29736 SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R)); in LowerShift()
29737 SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R)); in LowerShift()
29738 LoR = DAG.getNode(X86OpcI, dl, VT16, LoR, Cst8); in LowerShift()
29739 HiR = DAG.getNode(X86OpcI, dl, VT16, HiR, Cst8); in LowerShift()
29740 LoR = DAG.getNode(ISD::MUL, dl, VT16, LoR, LoA); in LowerShift()
29741 HiR = DAG.getNode(ISD::MUL, dl, VT16, HiR, HiA); in LowerShift()
29742 LoR = DAG.getNode(X86ISD::VSRLI, dl, VT16, LoR, Cst8); in LowerShift()
29743 HiR = DAG.getNode(X86ISD::VSRLI, dl, VT16, HiR, Cst8); in LowerShift()
29744 return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR); in LowerShift()
29758 V0 = DAG.getBitcast(VT, V0); in LowerShift()
29759 V1 = DAG.getBitcast(VT, V1); in LowerShift()
29760 Sel = DAG.getBitcast(VT, Sel); in LowerShift()
29761 Sel = DAG.getSetCC(dl, MaskVT, DAG.getConstant(0, dl, VT), Sel, in LowerShift()
29763 return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1)); in LowerShift()
29767 V0 = DAG.getBitcast(VT, V0); in LowerShift()
29768 V1 = DAG.getBitcast(VT, V1); in LowerShift()
29769 Sel = DAG.getBitcast(VT, Sel); in LowerShift()
29770 return DAG.getBitcast(SelVT, in LowerShift()
29771 DAG.getNode(X86ISD::BLENDV, dl, VT, Sel, V0, V1)); in LowerShift()
29776 SDValue Z = DAG.getConstant(0, dl, SelVT); in LowerShift()
29777 SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel); in LowerShift()
29778 return DAG.getSelect(dl, SelVT, C, V0, V1); in LowerShift()
29784 Amt = DAG.getBitcast(ExtVT, Amt); in LowerShift()
29785 Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExtVT, Amt, 5, DAG); in LowerShift()
29786 Amt = DAG.getBitcast(VT, Amt); in LowerShift()
29790 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT)); in LowerShift()
29794 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
29797 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT)); in LowerShift()
29801 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
29804 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT)); in LowerShift()
29813 SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt); in LowerShift()
29814 SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt); in LowerShift()
29815 SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R); in LowerShift()
29816 SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R); in LowerShift()
29817 ALo = DAG.getBitcast(ExtVT, ALo); in LowerShift()
29818 AHi = DAG.getBitcast(ExtVT, AHi); in LowerShift()
29819 RLo = DAG.getBitcast(ExtVT, RLo); in LowerShift()
29820 RHi = DAG.getBitcast(ExtVT, RHi); in LowerShift()
29823 SDValue MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 4, DAG); in LowerShift()
29824 SDValue MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 4, DAG); in LowerShift()
29829 ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo); in LowerShift()
29830 AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi); in LowerShift()
29833 MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 2, DAG); in LowerShift()
29834 MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 2, DAG); in LowerShift()
29839 ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo); in LowerShift()
29840 AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi); in LowerShift()
29843 MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 1, DAG); in LowerShift()
29844 MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 1, DAG); in LowerShift()
29850 RLo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RLo, 8, DAG); in LowerShift()
29851 RHi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RHi, 8, DAG); in LowerShift()
29852 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi); in LowerShift()
29858 SDValue Z = DAG.getConstant(0, dl, VT); in LowerShift()
29859 SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z); in LowerShift()
29860 SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z); in LowerShift()
29861 SDValue RLo = getUnpackl(DAG, dl, VT, Z, R); in LowerShift()
29862 SDValue RHi = getUnpackh(DAG, dl, VT, Z, R); in LowerShift()
29863 ALo = DAG.getBitcast(ExtVT, ALo); in LowerShift()
29864 AHi = DAG.getBitcast(ExtVT, AHi); in LowerShift()
29865 RLo = DAG.getBitcast(ExtVT, RLo); in LowerShift()
29866 RHi = DAG.getBitcast(ExtVT, RHi); in LowerShift()
29867 SDValue Lo = DAG.getNode(Opc, dl, ExtVT, RLo, ALo); in LowerShift()
29868 SDValue Hi = DAG.getNode(Opc, dl, ExtVT, RHi, AHi); in LowerShift()
29869 Lo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Lo, 16, DAG); in LowerShift()
29870 Hi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Hi, 16, DAG); in LowerShift()
29871 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi); in LowerShift()
29885 V0 = DAG.getBitcast(ExtVT, V0); in LowerShift()
29886 V1 = DAG.getBitcast(ExtVT, V1); in LowerShift()
29887 Sel = DAG.getBitcast(ExtVT, Sel); in LowerShift()
29888 return DAG.getBitcast( in LowerShift()
29889 VT, DAG.getNode(X86ISD::BLENDV, dl, ExtVT, Sel, V0, V1)); in LowerShift()
29895 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Sel, 15, DAG); in LowerShift()
29896 return DAG.getSelect(dl, VT, C, V0, V1); in LowerShift()
29903 Amt = DAG.getNode( in LowerShift()
29905 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 4, DAG), in LowerShift()
29906 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG)); in LowerShift()
29908 Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG); in LowerShift()
29912 SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG); in LowerShift()
29916 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
29919 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 4, DAG); in LowerShift()
29923 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
29926 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 2, DAG); in LowerShift()
29930 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
29933 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 1, DAG); in LowerShift()
29940 return splitVectorIntBinary(Op, DAG, dl); in LowerShift()
29943 return splitVectorIntBinary(Op, DAG, dl); in LowerShift()
29949 SelectionDAG &DAG) { in LowerFunnelShift() argument
29972 SDValue Imm = DAG.getTargetConstant(ShiftAmt, DL, MVT::i8); in LowerFunnelShift()
29974 {Op0, Op1, Imm}, DAG, Subtarget); in LowerFunnelShift()
29977 {Op0, Op1, Amt}, DAG, Subtarget); in LowerFunnelShift()
30006 DAG.getNode(ISD::SHL, DL, WideVT, DAG.getBitcast(WideVT, Op0), in LowerFunnelShift()
30007 DAG.getShiftAmountConstant(ShXAmt, WideVT, DL)); in LowerFunnelShift()
30009 DAG.getNode(ISD::SRL, DL, WideVT, DAG.getBitcast(WideVT, Op1), in LowerFunnelShift()
30010 DAG.getShiftAmountConstant(ShYAmt, WideVT, DL)); in LowerFunnelShift()
30011 ShX = DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, ShX), in LowerFunnelShift()
30012 DAG.getConstant(MaskX, DL, VT)); in LowerFunnelShift()
30013 ShY = DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, ShY), in LowerFunnelShift()
30014 DAG.getConstant(MaskY, DL, VT)); in LowerFunnelShift()
30015 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY); in LowerFunnelShift()
30018 SDValue ShX = DAG.getNode(ISD::SHL, DL, VT, Op0, in LowerFunnelShift()
30019 DAG.getShiftAmountConstant(ShXAmt, VT, DL)); in LowerFunnelShift()
30020 SDValue ShY = DAG.getNode(ISD::SRL, DL, VT, Op1, in LowerFunnelShift()
30021 DAG.getShiftAmountConstant(ShYAmt, VT, DL)); in LowerFunnelShift()
30022 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY); in LowerFunnelShift()
30025 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT); in LowerFunnelShift()
30026 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerFunnelShift()
30044 Op = DAG.getNode(Op.getOpcode(), DL, VT, Op0, Op1, AmtMod); in LowerFunnelShift()
30045 return splitVectorOp(Op, DAG, DL); in LowerFunnelShift()
30051 if (SDValue ScalarAmt = DAG.getSplatSourceVector(AmtMod, ScalarAmtIdx)) { in LowerFunnelShift()
30056 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30057 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30059 ScalarAmtIdx, Subtarget, DAG); in LowerFunnelShift()
30061 ScalarAmtIdx, Subtarget, DAG); in LowerFunnelShift()
30062 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR); in LowerFunnelShift()
30079 Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Op0); in LowerFunnelShift()
30080 Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op1); in LowerFunnelShift()
30081 AmtMod = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod); in LowerFunnelShift()
30083 EltSizeInBits, DAG); in LowerFunnelShift()
30084 SDValue Res = DAG.getNode(ISD::OR, DL, WideVT, Op0, Op1); in LowerFunnelShift()
30085 Res = DAG.getNode(ShiftOpc, DL, WideVT, Res, AmtMod); in LowerFunnelShift()
30088 EltSizeInBits, DAG); in LowerFunnelShift()
30089 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in LowerFunnelShift()
30095 SDValue Z = DAG.getConstant(0, DL, VT); in LowerFunnelShift()
30096 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30097 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30098 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z)); in LowerFunnelShift()
30099 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z)); in LowerFunnelShift()
30100 SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo); in LowerFunnelShift()
30101 SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi); in LowerFunnelShift()
30102 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR); in LowerFunnelShift()
30113 bool OptForSize = DAG.shouldOptForSize(); in LowerFunnelShift()
30120 SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, Amt.getValueType()); in LowerFunnelShift()
30121 SDValue HiShift = DAG.getConstant(EltSizeInBits, DL, Amt.getValueType()); in LowerFunnelShift()
30122 Op0 = DAG.getAnyExtOrTrunc(Op0, DL, MVT::i32); in LowerFunnelShift()
30123 Op1 = DAG.getZExtOrTrunc(Op1, DL, MVT::i32); in LowerFunnelShift()
30124 Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt, Mask); in LowerFunnelShift()
30125 SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift); in LowerFunnelShift()
30126 Res = DAG.getNode(ISD::OR, DL, MVT::i32, Res, Op1); in LowerFunnelShift()
30128 Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, Amt); in LowerFunnelShift()
30130 Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Res, Amt); in LowerFunnelShift()
30131 Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, HiShift); in LowerFunnelShift()
30133 return DAG.getZExtOrTrunc(Res, DL, VT); in LowerFunnelShift()
30141 Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt, in LowerFunnelShift()
30142 DAG.getConstant(15, DL, Amt.getValueType())); in LowerFunnelShift()
30144 return DAG.getNode(FSHOp, DL, VT, Op0, Op1, Amt); in LowerFunnelShift()
30151 SelectionDAG &DAG) { in LowerRotate() argument
30179 return DAG.getNode(RotOpc, DL, VT, R, in LowerRotate()
30180 DAG.getTargetConstant(RotAmt, DL, MVT::i8)); in LowerRotate()
30190 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt); in LowerRotate()
30193 SDValue Z = DAG.getConstant(0, DL, VT); in LowerRotate()
30198 if (SDValue NegAmt = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {Z, Amt})) in LowerRotate()
30199 return DAG.getNode(ISD::ROTL, DL, VT, R, NegAmt); in LowerRotate()
30203 return DAG.getNode(ISD::ROTL, DL, VT, R, in LowerRotate()
30204 DAG.getNode(ISD::SUB, DL, VT, Z, Amt)); in LowerRotate()
30209 DAG.getTargetLoweringInfo().isTypeLegal(VT)) { in LowerRotate()
30211 SDValue Mask = getGFNICtrlMask(Opcode, DAG, DL, VT, RotAmt); in LowerRotate()
30212 return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, R, Mask, in LowerRotate()
30213 DAG.getTargetConstant(0, DL, MVT::i8)); in LowerRotate()
30218 return splitVectorIntBinary(Op, DAG, DL); in LowerRotate()
30230 return DAG.getNode(X86ISD::VROTLI, DL, VT, R, in LowerRotate()
30231 DAG.getTargetConstant(RotAmt, DL, MVT::i8)); in LowerRotate()
30245 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, R, in LowerRotate()
30246 DAG.getShiftAmountConstant(ShlAmt, VT, DL)); in LowerRotate()
30247 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, R, in LowerRotate()
30248 DAG.getShiftAmountConstant(SrlAmt, VT, DL)); in LowerRotate()
30249 return DAG.getNode(ISD::OR, DL, VT, Shl, Srl); in LowerRotate()
30254 return splitVectorIntBinary(Op, DAG, DL); in LowerRotate()
30266 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT); in LowerRotate()
30267 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30274 if (SDValue BaseRotAmt = DAG.getSplatSourceVector(AmtMod, BaseRotAmtIdx)) { in LowerRotate()
30277 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt); in LowerRotate()
30280 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R)); in LowerRotate()
30281 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R)); in LowerRotate()
30283 BaseRotAmtIdx, Subtarget, DAG); in LowerRotate()
30285 BaseRotAmtIdx, Subtarget, DAG); in LowerRotate()
30286 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL); in LowerRotate()
30300 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R)); in LowerRotate()
30301 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R)); in LowerRotate()
30302 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z)); in LowerRotate()
30303 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z)); in LowerRotate()
30304 SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo); in LowerRotate()
30305 SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi); in LowerRotate()
30306 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL); in LowerRotate()
30325 R = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, R); in LowerRotate()
30326 R = DAG.getNode( in LowerRotate()
30328 getTargetVShiftByConstNode(X86ISD::VSHLI, DL, WideVT, R, 8, DAG)); in LowerRotate()
30329 Amt = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod); in LowerRotate()
30330 R = DAG.getNode(ShiftOpc, DL, WideVT, R, Amt); in LowerRotate()
30332 R = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, WideVT, R, 8, DAG); in LowerRotate()
30333 return DAG.getNode(ISD::TRUNCATE, DL, VT, R); in LowerRotate()
30341 V0 = DAG.getBitcast(VT, V0); in LowerRotate()
30342 V1 = DAG.getBitcast(VT, V1); in LowerRotate()
30343 Sel = DAG.getBitcast(VT, Sel); in LowerRotate()
30344 return DAG.getBitcast(SelVT, in LowerRotate()
30345 DAG.getNode(X86ISD::BLENDV, DL, VT, Sel, V0, V1)); in LowerRotate()
30350 SDValue Z = DAG.getConstant(0, DL, SelVT); in LowerRotate()
30351 SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel); in LowerRotate()
30352 return DAG.getSelect(DL, SelVT, C, V0, V1); in LowerRotate()
30357 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt); in LowerRotate()
30367 Amt = DAG.getBitcast(ExtVT, Amt); in LowerRotate()
30368 Amt = DAG.getNode(ISD::SHL, DL, ExtVT, Amt, DAG.getConstant(5, DL, ExtVT)); in LowerRotate()
30369 Amt = DAG.getBitcast(VT, Amt); in LowerRotate()
30373 M = DAG.getNode( in LowerRotate()
30375 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(4, DL, VT)), in LowerRotate()
30376 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(4, DL, VT))); in LowerRotate()
30380 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt); in LowerRotate()
30383 M = DAG.getNode( in LowerRotate()
30385 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(2, DL, VT)), in LowerRotate()
30386 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(6, DL, VT))); in LowerRotate()
30390 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt); in LowerRotate()
30393 M = DAG.getNode( in LowerRotate()
30395 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(1, DL, VT)), in LowerRotate()
30396 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(7, DL, VT))); in LowerRotate()
30400 bool IsSplatAmt = DAG.isSplatValue(Amt); in LowerRotate()
30407 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30408 SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT); in LowerRotate()
30409 AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt); in LowerRotate()
30410 SDValue SHL = DAG.getNode(IsROTL ? ISD::SHL : ISD::SRL, DL, VT, R, Amt); in LowerRotate()
30411 SDValue SRL = DAG.getNode(IsROTL ? ISD::SRL : ISD::SHL, DL, VT, R, AmtR); in LowerRotate()
30412 return DAG.getNode(ISD::OR, DL, VT, SHL, SRL); in LowerRotate()
30417 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt); in LowerRotate()
30422 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30428 SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG); in LowerRotate()
30434 SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale); in LowerRotate()
30435 SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale); in LowerRotate()
30436 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi); in LowerRotate()
30444 SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask); in LowerRotate()
30445 SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask); in LowerRotate()
30447 SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64, in LowerRotate()
30448 DAG.getBitcast(MVT::v2i64, R), in LowerRotate()
30449 DAG.getBitcast(MVT::v2i64, Scale)); in LowerRotate()
30450 SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64, in LowerRotate()
30451 DAG.getBitcast(MVT::v2i64, R13), in LowerRotate()
30452 DAG.getBitcast(MVT::v2i64, Scale13)); in LowerRotate()
30453 Res02 = DAG.getBitcast(VT, Res02); in LowerRotate()
30454 Res13 = DAG.getBitcast(VT, Res13); in LowerRotate()
30456 return DAG.getNode(ISD::OR, DL, VT, in LowerRotate()
30457 DAG.getVectorShuffle(VT, DL, Res02, Res13, {0, 4, 2, 6}), in LowerRotate()
30458 DAG.getVectorShuffle(VT, DL, Res02, Res13, {1, 5, 3, 7})); in LowerRotate()
30989 static SDValue emitLockedStackOp(SelectionDAG &DAG, in emitLockedStackOp() argument
31016 auto &MF = DAG.getMachineFunction(); in emitLockedStackOp()
31021 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32); in emitLockedStackOp()
31023 DAG.getRegister(X86::RSP, MVT::i64), // Base in emitLockedStackOp()
31024 DAG.getTargetConstant(1, DL, MVT::i8), // Scale in emitLockedStackOp()
31025 DAG.getRegister(0, MVT::i64), // Index in emitLockedStackOp()
31026 DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp in emitLockedStackOp()
31027 DAG.getRegister(0, MVT::i16), // Segment. in emitLockedStackOp()
31030 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32, in emitLockedStackOp()
31035 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32); in emitLockedStackOp()
31037 DAG.getRegister(X86::ESP, MVT::i32), // Base in emitLockedStackOp()
31038 DAG.getTargetConstant(1, DL, MVT::i8), // Scale in emitLockedStackOp()
31039 DAG.getRegister(0, MVT::i32), // Index in emitLockedStackOp()
31040 DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp in emitLockedStackOp()
31041 DAG.getRegister(0, MVT::i16), // Segment. in emitLockedStackOp()
31045 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32, in emitLockedStackOp()
31051 SelectionDAG &DAG) { in LowerATOMIC_FENCE() argument
31063 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
31066 return emitLockedStackOp(DAG, Subtarget, Chain, dl); in LowerATOMIC_FENCE()
31070 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
31074 SelectionDAG &DAG) { in LowerCMP_SWAP() argument
31089 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg, in LowerCMP_SWAP()
31094 DAG.getTargetConstant(size, DL, MVT::i8), in LowerCMP_SWAP()
31096 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCMP_SWAP()
31098 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys, in LowerCMP_SWAP()
31102 DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1)); in LowerCMP_SWAP()
31103 SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS, in LowerCMP_SWAP()
31105 SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG); in LowerCMP_SWAP()
31107 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), in LowerCMP_SWAP()
31112 static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG, in getPMOVMSKB() argument
31118 std::tie(Lo, Hi) = DAG.SplitVector(V, DL); in getPMOVMSKB()
31119 Lo = getPMOVMSKB(DL, Lo, DAG, Subtarget); in getPMOVMSKB()
31120 Hi = getPMOVMSKB(DL, Hi, DAG, Subtarget); in getPMOVMSKB()
31121 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Lo); in getPMOVMSKB()
31122 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Hi); in getPMOVMSKB()
31123 Hi = DAG.getNode(ISD::SHL, DL, MVT::i64, Hi, in getPMOVMSKB()
31124 DAG.getConstant(32, DL, MVT::i8)); in getPMOVMSKB()
31125 return DAG.getNode(ISD::OR, DL, MVT::i64, Lo, Hi); in getPMOVMSKB()
31129 std::tie(Lo, Hi) = DAG.SplitVector(V, DL); in getPMOVMSKB()
31130 Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo); in getPMOVMSKB()
31131 Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi); in getPMOVMSKB()
31132 Hi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi, in getPMOVMSKB()
31133 DAG.getConstant(16, DL, MVT::i8)); in getPMOVMSKB()
31134 return DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi); in getPMOVMSKB()
31137 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V); in getPMOVMSKB()
31141 SelectionDAG &DAG) { in LowerBITCAST() argument
31153 std::tie(Lo, Hi) = DAG.SplitScalar(Src, dl, MVT::i32, MVT::i32); in LowerBITCAST()
31154 Lo = DAG.getBitcast(MVT::v32i1, Lo); in LowerBITCAST()
31155 Hi = DAG.getBitcast(MVT::v32i1, Hi); in LowerBITCAST()
31156 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi); in LowerBITCAST()
31164 SDValue V = DAG.getSExtOrTrunc(Src, DL, SExtVT); in LowerBITCAST()
31165 V = getPMOVMSKB(DL, V, DAG, Subtarget); in LowerBITCAST()
31166 return DAG.getZExtOrTrunc(V, DL, DstVT); in LowerBITCAST()
31184 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT, Src, in LowerBITCAST()
31185 DAG.getUNDEF(SrcVT)); in LowerBITCAST()
31189 Src = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src); in LowerBITCAST()
31193 Src = DAG.getNode(ISD::BITCAST, dl, V2X64VT, Src); in LowerBITCAST()
31196 return DAG.getNode(X86ISD::MOVDQ2Q, dl, DstVT, Src); in LowerBITCAST()
31198 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, DstVT, Src, in LowerBITCAST()
31199 DAG.getIntPtrConstant(0, dl)); in LowerBITCAST()
31210 SelectionDAG &DAG) { in LowerHorizontalByteSum() argument
31224 SDValue Zeros = DAG.getConstant(0, DL, ByteVecVT); in LowerHorizontalByteSum()
31226 V = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, V, Zeros); in LowerHorizontalByteSum()
31227 return DAG.getBitcast(VT, V); in LowerHorizontalByteSum()
31236 SDValue Zeros = DAG.getConstant(0, DL, VT); in LowerHorizontalByteSum()
31237 SDValue V32 = DAG.getBitcast(VT, V); in LowerHorizontalByteSum()
31238 SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros); in LowerHorizontalByteSum()
31239 SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros); in LowerHorizontalByteSum()
31242 Zeros = DAG.getConstant(0, DL, ByteVecVT); in LowerHorizontalByteSum()
31244 Low = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, in LowerHorizontalByteSum()
31245 DAG.getBitcast(ByteVecVT, Low), Zeros); in LowerHorizontalByteSum()
31246 High = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, in LowerHorizontalByteSum()
31247 DAG.getBitcast(ByteVecVT, High), Zeros); in LowerHorizontalByteSum()
31251 V = DAG.getNode(X86ISD::PACKUS, DL, ByteVecVT, in LowerHorizontalByteSum()
31252 DAG.getBitcast(ShortVecVT, Low), in LowerHorizontalByteSum()
31253 DAG.getBitcast(ShortVecVT, High)); in LowerHorizontalByteSum()
31255 return DAG.getBitcast(VT, V); in LowerHorizontalByteSum()
31265 SDValue ShifterV = DAG.getConstant(8, DL, VT); in LowerHorizontalByteSum()
31266 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV); in LowerHorizontalByteSum()
31267 V = DAG.getNode(ISD::ADD, DL, ByteVecVT, DAG.getBitcast(ByteVecVT, Shl), in LowerHorizontalByteSum()
31268 DAG.getBitcast(ByteVecVT, V)); in LowerHorizontalByteSum()
31269 return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), ShifterV); in LowerHorizontalByteSum()
31274 SelectionDAG &DAG) { in LowerVectorCTPOPInRegLUT() argument
31298 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8)); in LowerVectorCTPOPInRegLUT()
31299 SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec); in LowerVectorCTPOPInRegLUT()
31300 SDValue M0F = DAG.getConstant(0x0F, DL, VT); in LowerVectorCTPOPInRegLUT()
31303 SDValue FourV = DAG.getConstant(4, DL, VT); in LowerVectorCTPOPInRegLUT()
31304 SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV); in LowerVectorCTPOPInRegLUT()
31307 SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F); in LowerVectorCTPOPInRegLUT()
31312 SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles); in LowerVectorCTPOPInRegLUT()
31313 SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles); in LowerVectorCTPOPInRegLUT()
31314 return DAG.getNode(ISD::ADD, DL, VT, HiPopCnt, LoPopCnt); in LowerVectorCTPOPInRegLUT()
31321 SelectionDAG &DAG) { in LowerVectorCTPOP() argument
31334 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, Op0); in LowerVectorCTPOP()
31335 Op = DAG.getNode(ISD::CTPOP, DL, NewVT, Op); in LowerVectorCTPOP()
31336 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in LowerVectorCTPOP()
31342 return splitVectorIntUnary(Op, DAG, DL); in LowerVectorCTPOP()
31346 return splitVectorIntUnary(Op, DAG, DL); in LowerVectorCTPOP()
31351 SDValue ByteOp = DAG.getBitcast(ByteVT, Op0); in LowerVectorCTPOP()
31352 SDValue PopCnt8 = DAG.getNode(ISD::CTPOP, DL, ByteVT, ByteOp); in LowerVectorCTPOP()
31353 return LowerHorizontalByteSum(PopCnt8, VT, Subtarget, DAG); in LowerVectorCTPOP()
31360 return LowerVectorCTPOPInRegLUT(Op0, DL, Subtarget, DAG); in LowerVectorCTPOP()
31364 SelectionDAG &DAG) { in LowerCTPOP() argument
31373 KnownBits Known = DAG.computeKnownBits(Op); in LowerCTPOP()
31383 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in LowerCTPOP()
31384 DAG.getShiftAmountConstant(TZ, VT, DL)); in LowerCTPOP()
31385 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32); in LowerCTPOP()
31386 Op = DAG.getNode(ISD::SUB, DL, MVT::i32, Op, in LowerCTPOP()
31387 DAG.getNode(ISD::SRL, DL, MVT::i32, Op, in LowerCTPOP()
31388 DAG.getShiftAmountConstant(1, VT, DL))); in LowerCTPOP()
31389 return DAG.getZExtOrTrunc(Op, DL, VT); in LowerCTPOP()
31395 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in LowerCTPOP()
31396 DAG.getShiftAmountConstant(TZ, VT, DL)); in LowerCTPOP()
31397 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32); in LowerCTPOP()
31398 Op = DAG.getNode(ISD::SHL, DL, MVT::i32, Op, in LowerCTPOP()
31399 DAG.getShiftAmountConstant(1, VT, DL)); in LowerCTPOP()
31400 Op = DAG.getNode(ISD::SRL, DL, MVT::i32, in LowerCTPOP()
31401 DAG.getConstant(0b1110100110010100U, DL, MVT::i32), Op); in LowerCTPOP()
31402 Op = DAG.getNode(ISD::AND, DL, MVT::i32, Op, in LowerCTPOP()
31403 DAG.getConstant(0x3, DL, MVT::i32)); in LowerCTPOP()
31404 return DAG.getZExtOrTrunc(Op, DL, VT); in LowerCTPOP()
31409 DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64)) { in LowerCTPOP()
31410 SDValue LUT = DAG.getConstant(0x4332322132212110ULL, DL, MVT::i64); in LowerCTPOP()
31412 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in LowerCTPOP()
31413 DAG.getShiftAmountConstant(TZ, VT, DL)); in LowerCTPOP()
31414 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32); in LowerCTPOP()
31415 Op = DAG.getNode(ISD::MUL, DL, MVT::i32, Op, in LowerCTPOP()
31416 DAG.getConstant(4, DL, MVT::i32)); in LowerCTPOP()
31417 Op = DAG.getNode(ISD::SRL, DL, MVT::i64, LUT, in LowerCTPOP()
31418 DAG.getShiftAmountOperand(MVT::i64, Op)); in LowerCTPOP()
31419 Op = DAG.getNode(ISD::AND, DL, MVT::i64, Op, in LowerCTPOP()
31420 DAG.getConstant(0x7, DL, MVT::i64)); in LowerCTPOP()
31421 return DAG.getZExtOrTrunc(Op, DL, VT); in LowerCTPOP()
31426 SDValue Mask11 = DAG.getConstant(0x11111111U, DL, MVT::i32); in LowerCTPOP()
31428 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in LowerCTPOP()
31429 DAG.getShiftAmountConstant(TZ, VT, DL)); in LowerCTPOP()
31430 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32); in LowerCTPOP()
31431 Op = DAG.getNode(ISD::MUL, DL, MVT::i32, Op, in LowerCTPOP()
31432 DAG.getConstant(0x08040201U, DL, MVT::i32)); in LowerCTPOP()
31433 Op = DAG.getNode(ISD::SRL, DL, MVT::i32, Op, in LowerCTPOP()
31434 DAG.getShiftAmountConstant(3, MVT::i32, DL)); in LowerCTPOP()
31435 Op = DAG.getNode(ISD::AND, DL, MVT::i32, Op, Mask11); in LowerCTPOP()
31436 Op = DAG.getNode(ISD::MUL, DL, MVT::i32, Op, Mask11); in LowerCTPOP()
31437 Op = DAG.getNode(ISD::SRL, DL, MVT::i32, Op, in LowerCTPOP()
31438 DAG.getShiftAmountConstant(28, MVT::i32, DL)); in LowerCTPOP()
31439 return DAG.getZExtOrTrunc(Op, DL, VT); in LowerCTPOP()
31447 return LowerVectorCTPOP(N, DL, Subtarget, DAG); in LowerCTPOP()
31450 static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG) { in LowerBITREVERSE_XOP() argument
31459 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In); in LowerBITREVERSE_XOP()
31460 Res = DAG.getNode(ISD::BITREVERSE, DL, VecVT, Res); in LowerBITREVERSE_XOP()
31461 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Res, in LowerBITREVERSE_XOP()
31462 DAG.getIntPtrConstant(0, DL)); in LowerBITREVERSE_XOP()
31470 return splitVectorIntUnary(Op, DAG, DL); in LowerBITREVERSE_XOP()
31484 MaskElts.push_back(DAG.getConstant(PermuteByte, DL, MVT::i8)); in LowerBITREVERSE_XOP()
31488 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts); in LowerBITREVERSE_XOP()
31489 SDValue Res = DAG.getBitcast(MVT::v16i8, In); in LowerBITREVERSE_XOP()
31490 Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8), in LowerBITREVERSE_XOP()
31492 return DAG.getBitcast(VT, Res); in LowerBITREVERSE_XOP()
31496 SelectionDAG &DAG) { in LowerBITREVERSE() argument
31500 return LowerBITREVERSE_XOP(Op, DAG); in LowerBITREVERSE()
31509 return splitVectorIntUnary(Op, DAG, DL); in LowerBITREVERSE()
31513 return splitVectorIntUnary(Op, DAG, DL); in LowerBITREVERSE()
31521 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In); in LowerBITREVERSE()
31522 Res = DAG.getNode(ISD::BITREVERSE, DL, MVT::v16i8, in LowerBITREVERSE()
31523 DAG.getBitcast(MVT::v16i8, Res)); in LowerBITREVERSE()
31524 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in LowerBITREVERSE()
31525 DAG.getBitcast(VecVT, Res), DAG.getIntPtrConstant(0, DL)); in LowerBITREVERSE()
31526 return (VT == MVT::i8) ? Res : DAG.getNode(ISD::BSWAP, DL, VT, Res); in LowerBITREVERSE()
31534 SDValue Res = DAG.getNode(ISD::BSWAP, DL, VT, In); in LowerBITREVERSE()
31535 Res = DAG.getBitcast(ByteVT, Res); in LowerBITREVERSE()
31536 Res = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Res); in LowerBITREVERSE()
31537 return DAG.getBitcast(VT, Res); in LowerBITREVERSE()
31546 SDValue Matrix = getGFNICtrlMask(ISD::BITREVERSE, DAG, DL, VT); in LowerBITREVERSE()
31547 return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, In, Matrix, in LowerBITREVERSE()
31548 DAG.getTargetConstant(0, DL, MVT::i8)); in LowerBITREVERSE()
31554 SDValue NibbleMask = DAG.getConstant(0xF, DL, VT); in LowerBITREVERSE()
31555 SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask); in LowerBITREVERSE()
31556 SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT)); in LowerBITREVERSE()
31571 LoMaskElts.push_back(DAG.getConstant(LoLUT[i % 16], DL, MVT::i8)); in LowerBITREVERSE()
31572 HiMaskElts.push_back(DAG.getConstant(HiLUT[i % 16], DL, MVT::i8)); in LowerBITREVERSE()
31575 SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts); in LowerBITREVERSE()
31576 SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts); in LowerBITREVERSE()
31577 Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo); in LowerBITREVERSE()
31578 Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi); in LowerBITREVERSE()
31579 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi); in LowerBITREVERSE()
31583 SelectionDAG &DAG) { in LowerPARITY() argument
31590 DAG.MaskedValueIsZero(X, APInt::getBitsSetFrom(VT.getSizeInBits(), 8))) { in LowerPARITY()
31591 X = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X); in LowerPARITY()
31592 SDValue Flags = DAG.getNode(X86ISD::CMP, DL, MVT::i32, X, in LowerPARITY()
31593 DAG.getConstant(0, DL, MVT::i8)); in LowerPARITY()
31595 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG); in LowerPARITY()
31597 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp); in LowerPARITY()
31606 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, in LowerPARITY()
31607 DAG.getNode(ISD::SRL, DL, MVT::i64, X, in LowerPARITY()
31608 DAG.getConstant(32, DL, MVT::i8))); in LowerPARITY()
31609 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X); in LowerPARITY()
31610 X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi); in LowerPARITY()
31615 SDValue Hi16 = DAG.getNode(ISD::SRL, DL, MVT::i32, X, in LowerPARITY()
31616 DAG.getConstant(16, DL, MVT::i8)); in LowerPARITY()
31617 X = DAG.getNode(ISD::XOR, DL, MVT::i32, X, Hi16); in LowerPARITY()
31620 X = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, X); in LowerPARITY()
31625 SDValue Hi = DAG.getNode( in LowerPARITY()
31627 DAG.getNode(ISD::SRL, DL, MVT::i32, X, DAG.getConstant(8, DL, MVT::i8))); in LowerPARITY()
31628 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X); in LowerPARITY()
31629 SDVTList VTs = DAG.getVTList(MVT::i8, MVT::i32); in LowerPARITY()
31630 SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1); in LowerPARITY()
31633 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG); in LowerPARITY()
31635 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp); in LowerPARITY()
31638 static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG, in lowerAtomicArithWithLOCK() argument
31663 return DAG.getMemIntrinsicNode( in lowerAtomicArithWithLOCK()
31664 NewOpc, SDLoc(N), DAG.getVTList(MVT::i32, MVT::Other), in lowerAtomicArithWithLOCK()
31670 static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG, in lowerAtomicArith() argument
31690 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS, in lowerAtomicArith()
31691 DAG.getNegative(RHS, DL, VT), AN->getMemOperand()); in lowerAtomicArith()
31716 SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL); in lowerAtomicArith()
31719 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), in lowerAtomicArith()
31720 DAG.getUNDEF(VT), NewChain); in lowerAtomicArith()
31723 SDValue NewChain = DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Chain); in lowerAtomicArith()
31726 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), in lowerAtomicArith()
31727 DAG.getUNDEF(VT), NewChain); in lowerAtomicArith()
31730 SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget); in lowerAtomicArith()
31734 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), in lowerAtomicArith()
31735 DAG.getUNDEF(VT), LockOp.getValue(1)); in lowerAtomicArith()
31738 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG, in LowerATOMIC_STORE() argument
31746 bool IsTypeLegal = DAG.getTargetLoweringInfo().isTypeLegal(VT); in LowerATOMIC_STORE()
31754 !DAG.getMachineFunction().getFunction().hasFnAttribute( in LowerATOMIC_STORE()
31760 SDValue VecVal = DAG.getBitcast(MVT::v2i64, Node->getVal()); in LowerATOMIC_STORE()
31761 Chain = DAG.getStore(Node->getChain(), dl, VecVal, Node->getBasePtr(), in LowerATOMIC_STORE()
31770 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Node->getVal()); in LowerATOMIC_STORE()
31772 SclToVec = DAG.getBitcast(StVT, SclToVec); in LowerATOMIC_STORE()
31773 SDVTList Tys = DAG.getVTList(MVT::Other); in LowerATOMIC_STORE()
31775 Chain = DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops, in LowerATOMIC_STORE()
31780 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64); in LowerATOMIC_STORE()
31783 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); in LowerATOMIC_STORE()
31784 Chain = DAG.getStore(Node->getChain(), dl, Node->getVal(), StackPtr, in LowerATOMIC_STORE()
31786 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); in LowerATOMIC_STORE()
31788 SDValue Value = DAG.getMemIntrinsicNode( in LowerATOMIC_STORE()
31796 DAG.getMemIntrinsicNode(X86ISD::FIST, dl, DAG.getVTList(MVT::Other), in LowerATOMIC_STORE()
31805 Chain = emitLockedStackOp(DAG, Subtarget, Chain, dl); in LowerATOMIC_STORE()
31814 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, Node->getMemoryVT(), in LowerATOMIC_STORE()
31820 static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) { in LowerADDSUBO_CARRY() argument
31826 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in LowerADDSUBO_CARRY()
31829 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDSUBO_CARRY()
31835 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32), in LowerADDSUBO_CARRY()
31836 Carry, DAG.getAllOnesConstant(DL, CarryVT)); in LowerADDSUBO_CARRY()
31839 SDValue Sum = DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs, in LowerADDSUBO_CARRY()
31845 Sum.getValue(1), DL, DAG); in LowerADDSUBO_CARRY()
31847 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC); in LowerADDSUBO_CARRY()
31849 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC); in LowerADDSUBO_CARRY()
31853 SelectionDAG &DAG) { in LowerFSINCOS() argument
31862 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); in LowerFSINCOS()
31877 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerFSINCOS()
31881 DAG.getExternalSymbol(LibcallName, TLI.getPointerTy(DAG.getDataLayout())); in LowerFSINCOS()
31886 TargetLowering::CallLoweringInfo CLI(DAG); in LowerFSINCOS()
31888 .setChain(DAG.getEntryNode()) in LowerFSINCOS()
31898 SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT, in LowerFSINCOS()
31899 CallResult.first, DAG.getIntPtrConstant(0, dl)); in LowerFSINCOS()
31900 SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT, in LowerFSINCOS()
31901 CallResult.first, DAG.getIntPtrConstant(1, dl)); in LowerFSINCOS()
31902 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT); in LowerFSINCOS()
31903 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal); in LowerFSINCOS()
31908 static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG, in ExtendToType() argument
31916 return DAG.getUNDEF(NVT); in ExtendToType()
31945 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) : in ExtendToType()
31946 DAG.getUNDEF(EltVT); in ExtendToType()
31949 return DAG.getBuildVector(NVT, dl, Ops); in ExtendToType()
31951 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, NVT) : in ExtendToType()
31952 DAG.getUNDEF(NVT); in ExtendToType()
31953 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NVT, FillVal, in ExtendToType()
31954 InOp, DAG.getIntPtrConstant(0, dl)); in ExtendToType()
31958 SelectionDAG &DAG) { in LowerMSCATTER() argument
31978 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in LowerMSCATTER()
31979 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); in LowerMSCATTER()
31980 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Src, DAG.getUNDEF(VT)); in LowerMSCATTER()
31981 SDVTList VTs = DAG.getVTList(MVT::Other); in LowerMSCATTER()
31983 return DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops, in LowerMSCATTER()
32009 Src = ExtendToType(Src, VT, DAG); in LowerMSCATTER()
32010 Index = ExtendToType(Index, IndexVT, DAG); in LowerMSCATTER()
32011 Mask = ExtendToType(Mask, MaskVT, DAG, true); in LowerMSCATTER()
32014 SDVTList VTs = DAG.getVTList(MVT::Other); in LowerMSCATTER()
32016 return DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops, in LowerMSCATTER()
32021 SelectionDAG &DAG) { in LowerMLOAD() argument
32037 SDValue NewLoad = DAG.getMaskedLoad( in LowerMLOAD()
32039 getZeroVector(VT, Subtarget, DAG, dl), N->getMemoryVT(), in LowerMLOAD()
32043 SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru); in LowerMLOAD()
32044 return DAG.getMergeValues({ Select, NewLoad.getValue(1) }, dl); in LowerMLOAD()
32065 PassThru = ExtendToType(PassThru, WideDataVT, DAG); in LowerMLOAD()
32073 Mask = ExtendToType(Mask, WideMaskVT, DAG, true); in LowerMLOAD()
32074 SDValue NewLoad = DAG.getMaskedLoad( in LowerMLOAD()
32080 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, NewLoad.getValue(0), in LowerMLOAD()
32081 DAG.getIntPtrConstant(0, dl)); in LowerMLOAD()
32083 return DAG.getMergeValues(RetOps, dl); in LowerMLOAD()
32087 SelectionDAG &DAG) { in LowerMSTORE() argument
32120 DataToStore = ExtendToType(DataToStore, WideDataVT, DAG); in LowerMSTORE()
32121 Mask = ExtendToType(Mask, WideMaskVT, DAG, true); in LowerMSTORE()
32122 return DAG.getMaskedStore(N->getChain(), dl, DataToStore, N->getBasePtr(), in LowerMSTORE()
32129 SelectionDAG &DAG) { in LowerMGATHER() argument
32162 PassThru = ExtendToType(PassThru, VT, DAG); in LowerMGATHER()
32163 Index = ExtendToType(Index, IndexVT, DAG); in LowerMGATHER()
32164 Mask = ExtendToType(Mask, MaskVT, DAG, true); in LowerMGATHER()
32169 PassThru = getZeroVector(VT, Subtarget, DAG, dl); in LowerMGATHER()
32173 SDValue NewGather = DAG.getMemIntrinsicNode( in LowerMGATHER()
32174 X86ISD::MGATHER, dl, DAG.getVTList(VT, MVT::Other), Ops, N->getMemoryVT(), in LowerMGATHER()
32176 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OrigVT, in LowerMGATHER()
32177 NewGather, DAG.getIntPtrConstant(0, dl)); in LowerMGATHER()
32178 return DAG.getMergeValues({Extract, NewGather.getValue(1)}, dl); in LowerMGATHER()
32181 static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) { in LowerADDRSPACECAST() argument
32193 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Src); in LowerADDRSPACECAST()
32195 Op = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Src); in LowerADDRSPACECAST()
32197 Op = DAG.getNode(ISD::TRUNCATE, dl, DstVT, Src); in LowerADDRSPACECAST()
32205 SelectionDAG &DAG) const { in LowerGC_TRANSITION()
32217 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); in LowerGC_TRANSITION()
32218 return SDValue(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0); in LowerGC_TRANSITION()
32222 static SDValue LowerCVTPS2PH(SDValue Op, SelectionDAG &DAG) { in LowerCVTPS2PH() argument
32226 std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0); in LowerCVTPS2PH()
32228 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in LowerCVTPS2PH()
32230 Lo = DAG.getNode(X86ISD::CVTPS2PH, dl, LoVT, Lo, RC); in LowerCVTPS2PH()
32231 Hi = DAG.getNode(X86ISD::CVTPS2PH, dl, HiVT, Hi, RC); in LowerCVTPS2PH()
32232 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerCVTPS2PH()
32236 SelectionDAG &DAG) { in LowerPREFETCH() argument
32301 static SDValue getFlagsOfCmpZeroFori1(SelectionDAG &DAG, const SDLoc &DL, in getFlagsOfCmpZeroFori1() argument
32304 auto V = DAG.getBitcast(MVT::i1, Mask); in getFlagsOfCmpZeroFori1()
32305 auto VE = DAG.getZExtOrTrunc(V, DL, Ty); in getFlagsOfCmpZeroFori1()
32306 auto Zero = DAG.getConstant(0, DL, Ty); in getFlagsOfCmpZeroFori1()
32307 SDVTList X86SubVTs = DAG.getVTList(Ty, MVT::i32); in getFlagsOfCmpZeroFori1()
32308 auto CmpZero = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, VE); in getFlagsOfCmpZeroFori1()
32313 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, in visitMaskedLoad() argument
32322 SDVTList Tys = DAG.getVTList(Ty, MVT::Other); in visitMaskedLoad()
32323 auto ScalarPassThru = PassThru.isUndef() ? DAG.getConstant(0, DL, Ty) in visitMaskedLoad()
32324 : DAG.getBitcast(Ty, PassThru); in visitMaskedLoad()
32325 auto Flags = getFlagsOfCmpZeroFori1(DAG, DL, Mask); in visitMaskedLoad()
32326 auto COND_NE = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8); in visitMaskedLoad()
32328 NewLoad = DAG.getMemIntrinsicNode(X86ISD::CLOAD, DL, Tys, Ops, Ty, MMO); in visitMaskedLoad()
32329 return DAG.getBitcast(VTy, NewLoad); in visitMaskedLoad()
32332 SDValue X86TargetLowering::visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, in visitMaskedStore() argument
32341 SDVTList Tys = DAG.getVTList(MVT::Other); in visitMaskedStore()
32342 auto ScalarVal = DAG.getBitcast(Ty, Val); in visitMaskedStore()
32343 auto Flags = getFlagsOfCmpZeroFori1(DAG, DL, Mask); in visitMaskedStore()
32344 auto COND_NE = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8); in visitMaskedStore()
32346 return DAG.getMemIntrinsicNode(X86ISD::CSTORE, DL, Tys, Ops, Ty, MMO); in visitMaskedStore()
32350 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { in LowerOperation()
32354 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG); in LowerOperation()
32356 return LowerCMP_SWAP(Op, Subtarget, DAG); in LowerOperation()
32357 case ISD::CTPOP: return LowerCTPOP(Op, Subtarget, DAG); in LowerOperation()
32362 case ISD::ATOMIC_LOAD_AND: return lowerAtomicArith(Op, DAG, Subtarget); in LowerOperation()
32363 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op, DAG, Subtarget); in LowerOperation()
32364 case ISD::BITREVERSE: return LowerBITREVERSE(Op, Subtarget, DAG); in LowerOperation()
32365 case ISD::PARITY: return LowerPARITY(Op, Subtarget, DAG); in LowerOperation()
32366 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); in LowerOperation()
32367 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, Subtarget, DAG); in LowerOperation()
32368 case ISD::VECTOR_SHUFFLE: return lowerVECTOR_SHUFFLE(Op, Subtarget, DAG); in LowerOperation()
32369 case ISD::VSELECT: return LowerVSELECT(Op, DAG); in LowerOperation()
32370 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); in LowerOperation()
32371 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); in LowerOperation()
32372 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG); in LowerOperation()
32373 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG); in LowerOperation()
32374 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, Subtarget,DAG); in LowerOperation()
32375 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); in LowerOperation()
32376 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); in LowerOperation()
32377 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); in LowerOperation()
32378 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); in LowerOperation()
32379 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); in LowerOperation()
32382 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG); in LowerOperation()
32384 case ISD::FSHR: return LowerFunnelShift(Op, Subtarget, DAG); in LowerOperation()
32386 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); in LowerOperation()
32388 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); in LowerOperation()
32389 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG); in LowerOperation()
32390 case ISD::ZERO_EXTEND: return LowerZERO_EXTEND(Op, Subtarget, DAG); in LowerOperation()
32391 case ISD::SIGN_EXTEND: return LowerSIGN_EXTEND(Op, Subtarget, DAG); in LowerOperation()
32392 case ISD::ANY_EXTEND: return LowerANY_EXTEND(Op, Subtarget, DAG); in LowerOperation()
32395 return LowerEXTEND_VECTOR_INREG(Op, Subtarget, DAG); in LowerOperation()
32399 case ISD::STRICT_FP_TO_UINT: return LowerFP_TO_INT(Op, DAG); in LowerOperation()
32401 case ISD::FP_TO_UINT_SAT: return LowerFP_TO_INT_SAT(Op, DAG); in LowerOperation()
32403 case ISD::STRICT_FP_EXTEND: return LowerFP_EXTEND(Op, DAG); in LowerOperation()
32405 case ISD::STRICT_FP_ROUND: return LowerFP_ROUND(Op, DAG); in LowerOperation()
32407 case ISD::STRICT_FP16_TO_FP: return LowerFP16_TO_FP(Op, DAG); in LowerOperation()
32409 case ISD::STRICT_FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG); in LowerOperation()
32410 case ISD::FP_TO_BF16: return LowerFP_TO_BF16(Op, DAG); in LowerOperation()
32411 case ISD::LOAD: return LowerLoad(Op, Subtarget, DAG); in LowerOperation()
32412 case ISD::STORE: return LowerStore(Op, Subtarget, DAG); in LowerOperation()
32414 case ISD::FSUB: return lowerFaddFsub(Op, DAG); in LowerOperation()
32415 case ISD::FROUND: return LowerFROUND(Op, DAG); in LowerOperation()
32417 case ISD::FNEG: return LowerFABSorFNEG(Op, DAG); in LowerOperation()
32418 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); in LowerOperation()
32419 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG); in LowerOperation()
32421 case ISD::LLRINT: return LowerLRINT_LLRINT(Op, DAG); in LowerOperation()
32424 case ISD::STRICT_FSETCCS: return LowerSETCC(Op, DAG); in LowerOperation()
32425 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG); in LowerOperation()
32426 case ISD::SELECT: return LowerSELECT(Op, DAG); in LowerOperation()
32427 case ISD::BRCOND: return LowerBRCOND(Op, DAG); in LowerOperation()
32428 case ISD::JumpTable: return LowerJumpTable(Op, DAG); in LowerOperation()
32429 case ISD::VASTART: return LowerVASTART(Op, DAG); in LowerOperation()
32430 case ISD::VAARG: return LowerVAARG(Op, DAG); in LowerOperation()
32431 case ISD::VACOPY: return LowerVACOPY(Op, Subtarget, DAG); in LowerOperation()
32432 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); in LowerOperation()
32434 case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, Subtarget, DAG); in LowerOperation()
32435 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); in LowerOperation()
32436 case ISD::ADDROFRETURNADDR: return LowerADDROFRETURNADDR(Op, DAG); in LowerOperation()
32437 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); in LowerOperation()
32439 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG); in LowerOperation()
32440 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); in LowerOperation()
32441 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); in LowerOperation()
32442 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG); in LowerOperation()
32443 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG); in LowerOperation()
32445 return lowerEH_SJLJ_SETUP_DISPATCH(Op, DAG); in LowerOperation()
32446 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG); in LowerOperation()
32447 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG); in LowerOperation()
32448 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG); in LowerOperation()
32449 case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG); in LowerOperation()
32450 case ISD::GET_FPENV_MEM: return LowerGET_FPENV_MEM(Op, DAG); in LowerOperation()
32451 case ISD::SET_FPENV_MEM: return LowerSET_FPENV_MEM(Op, DAG); in LowerOperation()
32452 case ISD::RESET_FPENV: return LowerRESET_FPENV(Op, DAG); in LowerOperation()
32454 case ISD::CTLZ_ZERO_UNDEF: return LowerCTLZ(Op, Subtarget, DAG); in LowerOperation()
32456 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op, Subtarget, DAG); in LowerOperation()
32457 case ISD::MUL: return LowerMUL(Op, Subtarget, DAG); in LowerOperation()
32459 case ISD::MULHU: return LowerMULH(Op, Subtarget, DAG); in LowerOperation()
32461 case ISD::ROTR: return LowerRotate(Op, Subtarget, DAG); in LowerOperation()
32464 case ISD::SHL: return LowerShift(Op, Subtarget, DAG); in LowerOperation()
32468 case ISD::USUBO: return LowerXALUO(Op, DAG); in LowerOperation()
32470 case ISD::UMULO: return LowerMULO(Op, Subtarget, DAG); in LowerOperation()
32471 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, Subtarget,DAG); in LowerOperation()
32472 case ISD::BITCAST: return LowerBITCAST(Op, Subtarget, DAG); in LowerOperation()
32476 case ISD::USUBO_CARRY: return LowerADDSUBO_CARRY(Op, DAG); in LowerOperation()
32478 case ISD::SUB: return lowerAddSub(Op, DAG, Subtarget); in LowerOperation()
32482 case ISD::SSUBSAT: return LowerADDSAT_SUBSAT(Op, DAG, Subtarget); in LowerOperation()
32486 case ISD::UMIN: return LowerMINMAX(Op, Subtarget, DAG); in LowerOperation()
32489 return LowerFMINIMUM_FMAXIMUM(Op, Subtarget, DAG); in LowerOperation()
32490 case ISD::ABS: return LowerABS(Op, Subtarget, DAG); in LowerOperation()
32492 case ISD::ABDU: return LowerABD(Op, Subtarget, DAG); in LowerOperation()
32493 case ISD::AVGCEILU: return LowerAVG(Op, Subtarget, DAG); in LowerOperation()
32494 case ISD::FSINCOS: return LowerFSINCOS(Op, Subtarget, DAG); in LowerOperation()
32495 case ISD::MLOAD: return LowerMLOAD(Op, Subtarget, DAG); in LowerOperation()
32496 case ISD::MSTORE: return LowerMSTORE(Op, Subtarget, DAG); in LowerOperation()
32497 case ISD::MGATHER: return LowerMGATHER(Op, Subtarget, DAG); in LowerOperation()
32498 case ISD::MSCATTER: return LowerMSCATTER(Op, Subtarget, DAG); in LowerOperation()
32500 case ISD::GC_TRANSITION_END: return LowerGC_TRANSITION(Op, DAG); in LowerOperation()
32501 case ISD::ADDRSPACECAST: return LowerADDRSPACECAST(Op, DAG); in LowerOperation()
32502 case X86ISD::CVTPS2PH: return LowerCVTPS2PH(Op, DAG); in LowerOperation()
32503 case ISD::PREFETCH: return LowerPREFETCH(Op, Subtarget, DAG); in LowerOperation()
32512 SelectionDAG &DAG) const { in ReplaceNodeResults()
32518 N->dump(&DAG); in ReplaceNodeResults()
32524 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); in ReplaceNodeResults()
32526 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in ReplaceNodeResults()
32527 Lo = DAG.getNode(X86ISD::CVTPH2PS, dl, LoVT, Lo); in ReplaceNodeResults()
32528 Hi = DAG.getNode(X86ISD::CVTPH2PS, dl, HiVT, Hi); in ReplaceNodeResults()
32529 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32536 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 1); in ReplaceNodeResults()
32538 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in ReplaceNodeResults()
32539 Lo = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {LoVT, MVT::Other}, in ReplaceNodeResults()
32541 Hi = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {HiVT, MVT::Other}, in ReplaceNodeResults()
32543 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in ReplaceNodeResults()
32545 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32551 Results.push_back(LowerCVTPS2PH(SDValue(N, 0), DAG)); in ReplaceNodeResults()
32557 KnownBits Known = DAG.computeKnownBits(N->getOperand(0)); in ReplaceNodeResults()
32561 SDValue Op = DAG.getNode(ISD::SRL, dl, MVT::i64, N->getOperand(0), in ReplaceNodeResults()
32562 DAG.getShiftAmountConstant(TZ, MVT::i64, dl)); in ReplaceNodeResults()
32563 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Op); in ReplaceNodeResults()
32564 Op = DAG.getNode(ISD::CTPOP, dl, MVT::i32, Op); in ReplaceNodeResults()
32565 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Op); in ReplaceNodeResults()
32571 DAG.getMachineFunction().getFunction().hasFnAttribute( in ReplaceNodeResults()
32575 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, N->getOperand(0)); in ReplaceNodeResults()
32576 Wide = DAG.getNode(ISD::CTPOP, dl, MVT::v2i64, Wide); in ReplaceNodeResults()
32579 Wide = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Wide); in ReplaceNodeResults()
32580 Wide = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Wide, in ReplaceNodeResults()
32581 DAG.getIntPtrConstant(0, dl)); in ReplaceNodeResults()
32582 Wide = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Wide); in ReplaceNodeResults()
32589 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32594 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0)); in ReplaceNodeResults()
32595 SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1)); in ReplaceNodeResults()
32596 SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1); in ReplaceNodeResults()
32597 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
32599 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT)); in ReplaceNodeResults()
32601 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, ConcatOps); in ReplaceNodeResults()
32608 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32612 SDValue Op0 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(0)); in ReplaceNodeResults()
32613 SDValue Op1 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(1)); in ReplaceNodeResults()
32614 SDValue Res = DAG.getNode(ISD::MUL, dl, MVT::v2i64, Op0, Op1); in ReplaceNodeResults()
32617 SDValue Hi = DAG.getBitcast(MVT::v4i32, Res); in ReplaceNodeResults()
32618 Hi = DAG.getVectorShuffle(MVT::v4i32, dl, Hi, Hi, {1, 3, -1, -1}); in ReplaceNodeResults()
32619 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Hi, in ReplaceNodeResults()
32620 DAG.getIntPtrConstant(0, dl)); in ReplaceNodeResults()
32623 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
32628 HiCmp = DAG.getNode(ISD::SRA, dl, VT, Res, DAG.getConstant(31, dl, VT)); in ReplaceNodeResults()
32631 HiCmp = DAG.getConstant(0, dl, VT); in ReplaceNodeResults()
32633 SDValue Ovf = DAG.getSetCC(dl, N->getValueType(1), Hi, HiCmp, ISD::SETNE); in ReplaceNodeResults()
32636 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res, in ReplaceNodeResults()
32637 DAG.getUNDEF(VT)); in ReplaceNodeResults()
32650 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32654 EVT InWideVT = EVT::getVectorVT(*DAG.getContext(), in ReplaceNodeResults()
32657 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), in ReplaceNodeResults()
32661 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT)); in ReplaceNodeResults()
32663 SDValue InVec0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops); in ReplaceNodeResults()
32665 SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops); in ReplaceNodeResults()
32667 SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1); in ReplaceNodeResults()
32678 SDValue UNDEF = DAG.getUNDEF(VT); in ReplaceNodeResults()
32679 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
32681 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
32683 Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS)); in ReplaceNodeResults()
32692 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32700 SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT)); in ReplaceNodeResults()
32702 EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT); in ReplaceNodeResults()
32703 SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0); in ReplaceNodeResults()
32704 SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT); in ReplaceNodeResults()
32705 SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1); in ReplaceNodeResults()
32711 SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG); in ReplaceNodeResults()
32717 if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector) in ReplaceNodeResults()
32723 MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT(); in ReplaceNodeResults()
32735 matchTruncateWithPACK(PackOpcode, VT, In, dl, DAG, Subtarget)) { in ReplaceNodeResults()
32737 dl, DAG, Subtarget)) { in ReplaceNodeResults()
32738 Res = widenSubVector(WidenVT, Res, false, Subtarget, DAG, dl); in ReplaceNodeResults()
32752 SDValue WidenIn = widenSubVector(In, false, Subtarget, DAG, dl, 128); in ReplaceNodeResults()
32755 WidenIn = DAG.getBitcast(WidenVT, WidenIn); in ReplaceNodeResults()
32757 DAG.getVectorShuffle(WidenVT, dl, WidenIn, WidenIn, TruncMask)); in ReplaceNodeResults()
32768 Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In)); in ReplaceNodeResults()
32773 In = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i64, In, in ReplaceNodeResults()
32774 DAG.getUNDEF(MVT::v4i64)); in ReplaceNodeResults()
32775 Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In)); in ReplaceNodeResults()
32780 getTypeAction(*DAG.getContext(), InVT) == TypeSplitVector && in ReplaceNodeResults()
32785 std::tie(Lo, Hi) = DAG.SplitVector(In, dl); in ReplaceNodeResults()
32787 Lo = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Lo); in ReplaceNodeResults()
32788 Hi = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Hi); in ReplaceNodeResults()
32789 SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi, in ReplaceNodeResults()
32803 SDValue WidenIn = widenSubVector(In, false, Subtarget, DAG, dl, in ReplaceNodeResults()
32805 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, WidenVT, WidenIn)); in ReplaceNodeResults()
32824 assert(getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector && in ReplaceNodeResults()
32831 In = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, In); in ReplaceNodeResults()
32834 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32); in ReplaceNodeResults()
32835 SDValue SignBits = DAG.getSetCC(dl, MVT::v4i32, Zero, In, ISD::SETGT); in ReplaceNodeResults()
32839 SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits, in ReplaceNodeResults()
32841 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Lo); in ReplaceNodeResults()
32842 SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits, in ReplaceNodeResults()
32844 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Hi); in ReplaceNodeResults()
32846 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32855 if (getTypeAction(*DAG.getContext(), InVT) != TypePromoteInteger) in ReplaceNodeResults()
32857 InVT = getTypeToTransformTo(*DAG.getContext(), InVT); in ReplaceNodeResults()
32863 In = DAG.getNode(N->getOpcode(), dl, InVT, In); in ReplaceNodeResults()
32869 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); in ReplaceNodeResults()
32872 SDValue Lo = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, LoVT, In, DAG); in ReplaceNodeResults()
32881 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask); in ReplaceNodeResults()
32882 Hi = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, HiVT, Hi, DAG); in ReplaceNodeResults()
32884 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32906 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other}, in ReplaceNodeResults()
32907 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl, in ReplaceNodeResults()
32911 Res = DAG.getNode(N->getOpcode(), dl, VT, in ReplaceNodeResults()
32912 DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src)); in ReplaceNodeResults()
32928 IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT); in ReplaceNodeResults()
32931 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops); in ReplaceNodeResults()
32938 DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src}); in ReplaceNodeResults()
32942 Res = DAG.getNode(Opc, dl, ResVT, Src); in ReplaceNodeResults()
32948 Res = DAG.getNode(ISD::TRUNCATE, dl, TmpVT, Res); in ReplaceNodeResults()
32953 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(TmpVT)); in ReplaceNodeResults()
32955 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps); in ReplaceNodeResults()
32966 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32976 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {PromoteVT, MVT::Other}, in ReplaceNodeResults()
32980 Res = DAG.getNode(ISD::FP_TO_SINT, dl, PromoteVT, Src); in ReplaceNodeResults()
32985 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res, in ReplaceNodeResults()
32986 DAG.getUNDEF(MVT::v2i32)); in ReplaceNodeResults()
32988 Res = DAG.getNode(!IsSigned ? ISD::AssertZext : ISD::AssertSext, dl, in ReplaceNodeResults()
32990 DAG.getValueType(VT.getVectorElementType())); in ReplaceNodeResults()
32993 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res, in ReplaceNodeResults()
32994 DAG.getIntPtrConstant(0, dl)); in ReplaceNodeResults()
32997 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
33003 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT)); in ReplaceNodeResults()
33005 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps); in ReplaceNodeResults()
33017 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
33022 expandFP_TO_UINT_SSE(MVT::v4i32, Src, dl, DAG, Subtarget); in ReplaceNodeResults()
33042 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f64, Src, in ReplaceNodeResults()
33043 DAG.getConstantFP(0.0, dl, MVT::v2f64)); in ReplaceNodeResults()
33049 Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other}, in ReplaceNodeResults()
33053 Res = DAG.getNode(Opc, dl, MVT::v4i32, Src); in ReplaceNodeResults()
33064 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, in ReplaceNodeResults()
33065 DAG.getConstantFP(0.0, dl, MVT::v2f32)); in ReplaceNodeResults()
33066 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other}, in ReplaceNodeResults()
33098 SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl); in ReplaceNodeResults()
33099 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT, in ReplaceNodeResults()
33100 DAG.getConstantFP(0.0, dl, VecInVT), Src, in ReplaceNodeResults()
33104 SDVTList Tys = DAG.getVTList(VecVT, MVT::Other); in ReplaceNodeResults()
33105 Res = DAG.getNode(Opc, SDLoc(N), Tys, N->getOperand(0), Res); in ReplaceNodeResults()
33108 Res = DAG.getNode(Opc, SDLoc(N), VecVT, Res); in ReplaceNodeResults()
33109 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, ZeroIdx); in ReplaceNodeResults()
33118 SDValue V = LowerWin64_FP_TO_INT128(SDValue(N, 0), DAG, Chain); in ReplaceNodeResults()
33125 if (SDValue V = FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, Chain)) { in ReplaceNodeResults()
33134 if (SDValue V = LRINT_LLRINTHelper(N, DAG)) in ReplaceNodeResults()
33154 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, in ReplaceNodeResults()
33155 IsStrict ? DAG.getConstant(0, dl, MVT::v2i32) in ReplaceNodeResults()
33156 : DAG.getUNDEF(MVT::v2i32)); in ReplaceNodeResults()
33160 SDValue Res = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other}, in ReplaceNodeResults()
33166 Results.push_back(DAG.getNode(Opc, dl, MVT::v8f16, Src)); in ReplaceNodeResults()
33177 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other}, in ReplaceNodeResults()
33183 Results.push_back(DAG.getNode(Opc, dl, MVT::v4f32, Src)); in ReplaceNodeResults()
33189 SDValue Zero = DAG.getConstant(0, dl, SrcVT); in ReplaceNodeResults()
33190 SDValue One = DAG.getConstant(1, dl, SrcVT); in ReplaceNodeResults()
33191 SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT, in ReplaceNodeResults()
33192 DAG.getNode(ISD::SRL, dl, SrcVT, Src, One), in ReplaceNodeResults()
33193 DAG.getNode(ISD::AND, dl, SrcVT, Src, One)); in ReplaceNodeResults()
33194 SDValue IsNeg = DAG.getSetCC(dl, MVT::v2i64, Src, Zero, ISD::SETLT); in ReplaceNodeResults()
33195 SDValue SignSrc = DAG.getSelect(dl, SrcVT, IsNeg, Sign, Src); in ReplaceNodeResults()
33196 SmallVector<SDValue, 4> SignCvts(4, DAG.getConstantFP(0.0, dl, MVT::f32)); in ReplaceNodeResults()
33198 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, in ReplaceNodeResults()
33199 SignSrc, DAG.getIntPtrConstant(i, dl)); in ReplaceNodeResults()
33202 DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {MVT::f32, MVT::Other}, in ReplaceNodeResults()
33205 SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Elt); in ReplaceNodeResults()
33207 SDValue SignCvt = DAG.getBuildVector(MVT::v4f32, dl, SignCvts); in ReplaceNodeResults()
33210 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in ReplaceNodeResults()
33212 Slow = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::v4f32, MVT::Other}, in ReplaceNodeResults()
33216 Slow = DAG.getNode(ISD::FADD, dl, MVT::v4f32, SignCvt, SignCvt); in ReplaceNodeResults()
33218 IsNeg = DAG.getBitcast(MVT::v4i32, IsNeg); in ReplaceNodeResults()
33220 DAG.getVectorShuffle(MVT::v4i32, dl, IsNeg, IsNeg, {1, 3, -1, -1}); in ReplaceNodeResults()
33221 SDValue Cvt = DAG.getSelect(dl, MVT::v4f32, IsNeg, Slow, SignCvt); in ReplaceNodeResults()
33237 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, in ReplaceNodeResults()
33238 DAG.getConstant(0, dl, MVT::v2i32)); in ReplaceNodeResults()
33239 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other}, in ReplaceNodeResults()
33247 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src); in ReplaceNodeResults()
33248 SDValue VBias = DAG.getConstantFP( in ReplaceNodeResults()
33250 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn, in ReplaceNodeResults()
33251 DAG.getBitcast(MVT::v2i64, VBias)); in ReplaceNodeResults()
33252 Or = DAG.getBitcast(MVT::v2f64, Or); in ReplaceNodeResults()
33254 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other}, in ReplaceNodeResults()
33256 SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl, in ReplaceNodeResults()
33263 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias); in ReplaceNodeResults()
33264 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub)); in ReplaceNodeResults()
33278 SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f32) in ReplaceNodeResults()
33279 : DAG.getUNDEF(MVT::v2f32); in ReplaceNodeResults()
33280 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, Ext); in ReplaceNodeResults()
33288 V = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other}, in ReplaceNodeResults()
33291 V = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Src, Rnd); in ReplaceNodeResults()
33293 Results.push_back(DAG.getBitcast(MVT::v8f16, V)); in ReplaceNodeResults()
33302 V = DAG.getNode(X86ISD::STRICT_VFPROUND, dl, {NewVT, MVT::Other}, in ReplaceNodeResults()
33305 V = DAG.getNode(X86ISD::VFPROUND, dl, NewVT, Src); in ReplaceNodeResults()
33323 SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f16) in ReplaceNodeResults()
33324 : DAG.getUNDEF(MVT::v2f16); in ReplaceNodeResults()
33325 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f16, Src, Ext); in ReplaceNodeResults()
33327 V = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::v4f32, MVT::Other}, in ReplaceNodeResults()
33330 V = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, V); in ReplaceNodeResults()
33342 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget, in ReplaceNodeResults()
33345 return getReadTimeStampCounter(N, dl, X86::RDTSCP, DAG, Subtarget, in ReplaceNodeResults()
33348 expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPMC, X86::ECX, Subtarget, in ReplaceNodeResults()
33352 expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPRU, X86::ECX, Subtarget, in ReplaceNodeResults()
33356 expandIntrinsicWChainHelper(N, dl, DAG, X86::XGETBV, X86::ECX, Subtarget, in ReplaceNodeResults()
33362 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget, Results); in ReplaceNodeResults()
33373 DAG.SplitScalar(N->getOperand(2), dl, HalfT, HalfT); in ReplaceNodeResults()
33374 cpInL = DAG.getCopyToReg(N->getOperand(0), dl, in ReplaceNodeResults()
33377 DAG.getCopyToReg(cpInL.getValue(0), dl, Regs64bit ? X86::RDX : X86::EDX, in ReplaceNodeResults()
33381 DAG.SplitScalar(N->getOperand(3), dl, HalfT, HalfT); in ReplaceNodeResults()
33383 DAG.getCopyToReg(cpInH.getValue(0), dl, Regs64bit ? X86::RCX : X86::ECX, in ReplaceNodeResults()
33393 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in ReplaceNodeResults()
33399 DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG16_DAG, dl, Tys, Ops, T, MMO); in ReplaceNodeResults()
33401 swapInL = DAG.getCopyToReg(swapInH.getValue(0), dl, X86::EBX, swapInL, in ReplaceNodeResults()
33406 DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG8_DAG, dl, Tys, Ops, T, MMO); in ReplaceNodeResults()
33409 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, in ReplaceNodeResults()
33412 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl, in ReplaceNodeResults()
33417 SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS, in ReplaceNodeResults()
33419 SDValue Success = getSETCC(X86::COND_E, EFLAGS, dl, DAG); in ReplaceNodeResults()
33420 Success = DAG.getZExtOrTrunc(Success, dl, N->getValueType(1)); in ReplaceNodeResults()
33422 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF)); in ReplaceNodeResults()
33432 DAG.getMachineFunction().getFunction().hasFnAttribute( in ReplaceNodeResults()
33439 SDValue Ld = DAG.getLoad(MVT::v2i64, dl, Node->getChain(), in ReplaceNodeResults()
33441 SDValue ResL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld, in ReplaceNodeResults()
33442 DAG.getIntPtrConstant(0, dl)); in ReplaceNodeResults()
33443 SDValue ResH = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld, in ReplaceNodeResults()
33444 DAG.getIntPtrConstant(1, dl)); in ReplaceNodeResults()
33445 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, N->getValueType(0), in ReplaceNodeResults()
33456 SDVTList Tys = DAG.getVTList(LdVT, MVT::Other); in ReplaceNodeResults()
33458 SDValue Ld = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, in ReplaceNodeResults()
33461 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld, in ReplaceNodeResults()
33462 DAG.getIntPtrConstant(0, dl)); in ReplaceNodeResults()
33470 SDValue Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Ld, in ReplaceNodeResults()
33471 DAG.getIntPtrConstant(0, dl)); in ReplaceNodeResults()
33472 Res = DAG.getBitcast(MVT::i64, Res); in ReplaceNodeResults()
33480 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); in ReplaceNodeResults()
33482 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::FILD, in ReplaceNodeResults()
33491 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64); in ReplaceNodeResults()
33494 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); in ReplaceNodeResults()
33496 Chain = DAG.getMemIntrinsicNode( in ReplaceNodeResults()
33497 X86ISD::FIST, dl, DAG.getVTList(MVT::Other), StoreOps, MVT::i64, in ReplaceNodeResults()
33503 Result = DAG.getLoad(MVT::i64, dl, Chain, StackPtr, MPI); in ReplaceNodeResults()
33539 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); in ReplaceNodeResults()
33540 Lo = DAG.getBitcast(MVT::i32, Lo); in ReplaceNodeResults()
33541 Hi = DAG.getBitcast(MVT::i32, Hi); in ReplaceNodeResults()
33542 SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); in ReplaceNodeResults()
33550 assert(getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector && in ReplaceNodeResults()
33552 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), DstVT); in ReplaceNodeResults()
33553 SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, in ReplaceNodeResults()
33555 Res = DAG.getBitcast(WideVT, Res); in ReplaceNodeResults()
33570 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
33572 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT); in ReplaceNodeResults()
33575 SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, in ReplaceNodeResults()
33577 DAG.getUNDEF(VT)); in ReplaceNodeResults()
33581 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask, in ReplaceNodeResults()
33582 DAG.getUNDEF(MVT::v2i1)); in ReplaceNodeResults()
33583 Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask); in ReplaceNodeResults()
33587 SDValue Res = DAG.getMemIntrinsicNode( in ReplaceNodeResults()
33588 X86ISD::MGATHER, dl, DAG.getVTList(WideVT, MVT::Other), Ops, in ReplaceNodeResults()
33602 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
33609 SDValue Res = DAG.getLoad(LdVT, dl, Ld->getChain(), Ld->getBasePtr(), in ReplaceNodeResults()
33614 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Res); in ReplaceNodeResults()
33615 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT); in ReplaceNodeResults()
33616 Res = DAG.getBitcast(WideVT, Res); in ReplaceNodeResults()
33622 SDVTList Tys = DAG.getVTList(MVT::v4f32, MVT::Other); in ReplaceNodeResults()
33624 SDValue Res = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, in ReplaceNodeResults()
33631 SDValue V = LowerADDRSPACECAST(SDValue(N,0), DAG); in ReplaceNodeResults()
33640 Results.push_back(LowerBITREVERSE(SDValue(N, 0), Subtarget, DAG)); in ReplaceNodeResults()
33650 SDValue Split = DAG.getBitcast(ExtVT, N->getOperand(0)); in ReplaceNodeResults()
33651 Split = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Split, in ReplaceNodeResults()
33653 Split = DAG.getBitcast(MVT::f16, Split); in ReplaceNodeResults()
37045 EVT ExtSVT = EVT::getIntegerVT(*TLO.DAG.getContext(), ActiveBits); in targetShrinkDemandedConstant()
37046 EVT ExtVT = EVT::getVectorVT(*TLO.DAG.getContext(), ExtSVT, in targetShrinkDemandedConstant()
37049 TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(Op), VT, in targetShrinkDemandedConstant()
37050 Op.getOperand(1), TLO.DAG.getValueType(ExtVT)); in targetShrinkDemandedConstant()
37052 TLO.DAG.getNode(Opcode, SDLoc(Op), VT, Op.getOperand(0), NewC); in targetShrinkDemandedConstant()
37100 SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT); in targetShrinkDemandedConstant()
37101 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC); in targetShrinkDemandedConstant()
37108 const SelectionDAG &DAG, unsigned Depth) { in computeKnownBitsForPSADBW() argument
37112 Known = DAG.computeKnownBits(RHS, DemandedSrcElts, Depth + 1); in computeKnownBitsForPSADBW()
37113 Known2 = DAG.computeKnownBits(LHS, DemandedSrcElts, Depth + 1); in computeKnownBitsForPSADBW()
37128 const SelectionDAG &DAG, in computeKnownBitsForPMADDWD() argument
37138 KnownBits LHSLo = DAG.computeKnownBits(LHS, DemandedLoElts, Depth + 1); in computeKnownBitsForPMADDWD()
37139 KnownBits LHSHi = DAG.computeKnownBits(LHS, DemandedHiElts, Depth + 1); in computeKnownBitsForPMADDWD()
37140 KnownBits RHSLo = DAG.computeKnownBits(RHS, DemandedLoElts, Depth + 1); in computeKnownBitsForPMADDWD()
37141 KnownBits RHSHi = DAG.computeKnownBits(RHS, DemandedHiElts, Depth + 1); in computeKnownBitsForPMADDWD()
37151 const SelectionDAG &DAG, in computeKnownBitsForPMADDUBSW() argument
37162 KnownBits LHSLo = DAG.computeKnownBits(LHS, DemandedLoElts, Depth + 1); in computeKnownBitsForPMADDUBSW()
37163 KnownBits LHSHi = DAG.computeKnownBits(LHS, DemandedHiElts, Depth + 1); in computeKnownBitsForPMADDUBSW()
37164 KnownBits RHSLo = DAG.computeKnownBits(RHS, DemandedLoElts, Depth + 1); in computeKnownBitsForPMADDUBSW()
37165 KnownBits RHSHi = DAG.computeKnownBits(RHS, DemandedHiElts, Depth + 1); in computeKnownBitsForPMADDUBSW()
37173 const SelectionDAG &DAG, in computeKnownBitsForHorizontalOperation() argument
37182 [&DAG, Depth, KnownBitsFunc](SDValue Op, APInt &DemandedEltsOp) { in computeKnownBitsForHorizontalOperation()
37184 DAG.computeKnownBits(Op, DemandedEltsOp, Depth + 1), in computeKnownBitsForHorizontalOperation()
37185 DAG.computeKnownBits(Op, DemandedEltsOp << 1, Depth + 1)); in computeKnownBitsForHorizontalOperation()
37200 const SelectionDAG &DAG, in computeKnownBitsForTargetNode() argument
37218 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37219 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37237 Known = DAG.computeKnownBits(Src, DemandedElt, Depth + 1); in computeKnownBitsForTargetNode()
37257 Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37284 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedLHS, Depth + 1); in computeKnownBitsForTargetNode()
37288 Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedRHS, Depth + 1); in computeKnownBitsForTargetNode()
37305 KnownBits KnownIdx = DAG.computeKnownBits(Idx, DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37307 Known = DAG.computeKnownBits(Src, Depth + 1); in computeKnownBitsForTargetNode()
37313 Known = DAG.computeKnownBits(Src, Depth + 1); in computeKnownBitsForTargetNode()
37321 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37322 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37329 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37330 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37339 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37340 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37352 computeKnownBitsForPSADBW(LHS, RHS, Known, DemandedElts, DAG, Depth); in computeKnownBitsForTargetNode()
37358 DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37360 DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37379 computeKnownBitsForPMADDWD(LHS, RHS, Known, DemandedElts, DAG, Depth); in computeKnownBitsForTargetNode()
37389 computeKnownBitsForPMADDUBSW(LHS, RHS, Known, DemandedElts, DAG, Depth); in computeKnownBitsForTargetNode()
37394 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37395 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37403 Known = DAG.computeKnownBits(Op.getOperand(1), Depth + 1); in computeKnownBitsForTargetNode()
37407 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1); in computeKnownBitsForTargetNode()
37429 Known = DAG.computeKnownBits(Op0, Depth + 1); in computeKnownBitsForTargetNode()
37438 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37439 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37448 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); in computeKnownBitsForTargetNode()
37529 Op, DemandedElts, Depth, DAG, in computeKnownBitsForTargetNode()
37548 computeKnownBitsForPMADDWD(LHS, RHS, Known, DemandedElts, DAG, Depth); in computeKnownBitsForTargetNode()
37560 computeKnownBitsForPMADDUBSW(LHS, RHS, Known, DemandedElts, DAG, Depth); in computeKnownBitsForTargetNode()
37572 computeKnownBitsForPSADBW(LHS, RHS, Known, DemandedElts, DAG, Depth); in computeKnownBitsForTargetNode()
37622 DAG.computeKnownBits(Ops[i], DemandedOps[i], Depth + 1); in computeKnownBitsForTargetNode()
37631 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, in ComputeNumSignBitsForTargetNode() argument
37647 unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedSrc, Depth + 1); in ComputeNumSignBitsForTargetNode()
37670 DAG.ComputeNumSignBits(BC0, Depth + 1) == 64 && in ComputeNumSignBitsForTargetNode()
37671 DAG.ComputeNumSignBits(BC1, Depth + 1) == 64) in ComputeNumSignBitsForTargetNode()
37674 return DAG.ComputeNumSignBits(V, Elts, Depth + 1); in ComputeNumSignBitsForTargetNode()
37692 return DAG.ComputeNumSignBits(Src, Depth + 1); in ComputeNumSignBitsForTargetNode()
37701 unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1); in ComputeNumSignBitsForTargetNode()
37712 unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1); in ComputeNumSignBitsForTargetNode()
37734 DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); in ComputeNumSignBitsForTargetNode()
37737 DAG.ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); in ComputeNumSignBitsForTargetNode()
37742 unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth+1); in ComputeNumSignBitsForTargetNode()
37744 unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth+1); in ComputeNumSignBitsForTargetNode()
37787 DAG.ComputeNumSignBits(Ops[i], DemandedOps[i], Depth + 1); in ComputeNumSignBitsForTargetNode()
37808 SelectionDAG &DAG) { in narrowLoadToVZLoad() argument
37813 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in narrowLoadToVZLoad()
37815 return DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, SDLoc(LN), Tys, Ops, MemVT, in narrowLoadToVZLoad()
37825 SDValue V1, const SelectionDAG &DAG, in matchUnaryShuffle() argument
37852 DAG.ComputeNumSignBits(V1) == MaskEltSize; in matchUnaryShuffle()
37882 Shuffle = DAG.getOpcode_EXTEND_VECTOR_INREG(Shuffle); in matchUnaryShuffle()
37908 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}, DAG, V1)) { in matchUnaryShuffle()
37913 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, DAG, V1)) { in matchUnaryShuffle()
37918 if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3}, DAG, V1)) { in matchUnaryShuffle()
37927 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, DAG, V1)) { in matchUnaryShuffle()
37932 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, DAG, in matchUnaryShuffle()
37938 if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3, 5, 5, 7, 7}, DAG, in matchUnaryShuffle()
37949 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, DAG, in matchUnaryShuffle()
37957 {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14}, DAG, V1)) { in matchUnaryShuffle()
37964 {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15}, DAG, V1)) { in matchUnaryShuffle()
37980 const SelectionDAG &DAG, in matchUnaryPermuteShuffle() argument
38129 SelectionDAG &DAG, const X86Subtarget &Subtarget, in matchBinaryShuffle() argument
38137 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}, DAG) && in matchBinaryShuffle()
38140 V1 = (SM_SentinelUndef == Mask[0] ? DAG.getUNDEF(MVT::v4f32) : V1); in matchBinaryShuffle()
38145 if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1}, DAG) && in matchBinaryShuffle()
38152 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 3}, DAG) && in matchBinaryShuffle()
38159 if (isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3}, DAG) && in matchBinaryShuffle()
38166 DAG) && in matchBinaryShuffle()
38178 if (matchShuffleWithPACK(MaskVT, SrcVT, V1, V2, Shuffle, Mask, DAG, in matchBinaryShuffle()
38186 isTargetShuffleEquivalent(MaskVT, Mask, {0, 2, 4, 6}, DAG) && in matchBinaryShuffle()
38190 unsigned MinLZV1 = DAG.computeKnownBits(V1).countMinLeadingZeros(); in matchBinaryShuffle()
38191 unsigned MinLZV2 = DAG.computeKnownBits(V2).countMinLeadingZeros(); in matchBinaryShuffle()
38206 if (DAG.ComputeNumSignBits(V1) > 48 && DAG.ComputeNumSignBits(V2) > 48) { in matchBinaryShuffle()
38221 if (matchShuffleWithUNPCK(MaskVT, V1, V2, Shuffle, IsUnary, Mask, DL, DAG, in matchBinaryShuffle()
38265 if (DAG.MaskedVectorIsZero(V1, DemandedZeroV1) && in matchBinaryShuffle()
38266 DAG.MaskedVectorIsZero(V2, DemandedZeroV2)) { in matchBinaryShuffle()
38275 auto computeKnownBitsElementWise = [&DAG](SDValue V) { in matchBinaryShuffle()
38280 KnownBits PeepholeKnown = DAG.computeKnownBits(V, Mask); in matchBinaryShuffle()
38325 const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, in matchBinaryPermuteShuffle() argument
38382 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1; in matchBinaryPermuteShuffle()
38383 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2; in matchBinaryPermuteShuffle()
38389 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1; in matchBinaryPermuteShuffle()
38390 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2; in matchBinaryPermuteShuffle()
38403 matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) { in matchBinaryPermuteShuffle()
38417 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1; in matchBinaryPermuteShuffle()
38418 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2; in matchBinaryPermuteShuffle()
38439 return DAG.getUNDEF(MaskVT); in matchBinaryPermuteShuffle()
38443 return getZeroVector(MaskVT, Subtarget, DAG, DL); in matchBinaryPermuteShuffle()
38475 matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) { in matchBinaryPermuteShuffle()
38487 bool AllowVariablePerLaneMask, SelectionDAG &DAG,
38504 SelectionDAG &DAG, in combineX86ShuffleChain() argument
38518 Op = widenSubVector(Op, false, Subtarget, DAG, DL, VT.getSizeInBits()); in combineX86ShuffleChain()
38520 Op = extractSubVector(Op, 0, DAG, DL, VT.getSizeInBits()); in combineX86ShuffleChain()
38521 return DAG.getBitcast(VT, Op); in combineX86ShuffleChain()
38528 SDValue V2 = (UnaryShuffle ? DAG.getUNDEF(V1.getValueType()) in combineX86ShuffleChain()
38544 bool OptForSize = DAG.shouldOptForSize(); in combineX86ShuffleChain()
38567 DAG.isSplatValue(V1, /*AllowUndefs*/ false)) { in combineX86ShuffleChain()
38582 if (isTargetShuffleEquivalent(RootVT, ScaledMask, IdentityMask, DAG, V1, in combineX86ShuffleChain()
38602 Res = extractSubVector(Res, SubIdx, DAG, DL, BaseMaskEltSizeInBits); in combineX86ShuffleChain()
38603 return widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits); in combineX86ShuffleChain()
38614 SelectionDAG &DAG) { in combineX86ShuffleChain() argument
38617 SDValue Ops[2] = {DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT)}; in combineX86ShuffleChain()
38633 return DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT, in combineX86ShuffleChain()
38636 getV4X86ShuffleImm8ForMask(PermMask, DL, DAG)); in combineX86ShuffleChain()
38655 if (SDValue V = MatchSHUF128(ShuffleVT, DL, ScaledMask, V1, V2, DAG)) in combineX86ShuffleChain()
38656 return DAG.getBitcast(RootVT, V); in combineX86ShuffleChain()
38670 Res = extract128BitVector(Res, Mask[0] * (NumRootElts / 2), DAG, DL); in combineX86ShuffleChain()
38671 return widenSubVector(Res, Mask[1] == SM_SentinelZero, Subtarget, DAG, DL, in combineX86ShuffleChain()
38684 Hi = extractSubVector(Hi, 0, DAG, DL, 128); in combineX86ShuffleChain()
38685 return insertSubVector(Lo, Hi, NumRootElts / 2, DAG, DL, 128); in combineX86ShuffleChain()
38700 return DAG.getNode( in combineX86ShuffleChain()
38702 DAG.getUNDEF(RootVT), DAG.getTargetConstant(PermMask, DL, MVT::i8)); in combineX86ShuffleChain()
38719 return DAG.getNode(X86ISD::VPERM2X128, DL, RootVT, in combineX86ShuffleChain()
38722 DAG.getTargetConstant(PermMask, DL, MVT::i8)); in combineX86ShuffleChain()
38750 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineX86ShuffleChain()
38791 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res); in combineX86ShuffleChain()
38792 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38798 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res); in combineX86ShuffleChain()
38799 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38805 DAG, Subtarget, Shuffle, ShuffleSrcVT, ShuffleVT) && in combineX86ShuffleChain()
38811 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res); in combineX86ShuffleChain()
38812 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38816 AllowIntDomain, DAG, Subtarget, Shuffle, ShuffleVT, in combineX86ShuffleChain()
38823 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res, in combineX86ShuffleChain()
38824 DAG.getTargetConstant(PermuteImm, DL, MVT::i8)); in combineX86ShuffleChain()
38825 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38834 !isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3}, DAG)) { in combineX86ShuffleChain()
38838 DAG) && in combineX86ShuffleChain()
38842 Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, in combineX86ShuffleChain()
38845 DAG.getTargetConstant(PermuteImm, DL, MVT::i8)); in combineX86ShuffleChain()
38846 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38850 isTargetShuffleEquivalent(MaskVT, Mask, {0, 2}, DAG) && in combineX86ShuffleChain()
38856 Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, in combineX86ShuffleChain()
38859 DAG.getTargetConstant(PermuteImm, DL, MVT::i8)); in combineX86ShuffleChain()
38860 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38867 NewV2, DL, DAG, Subtarget, Shuffle, ShuffleSrcVT, in combineX86ShuffleChain()
38874 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2); in combineX86ShuffleChain()
38875 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38881 AllowIntDomain, NewV1, NewV2, DL, DAG, in combineX86ShuffleChain()
38888 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2, in combineX86ShuffleChain()
38889 DAG.getTargetConstant(PermuteImm, DL, MVT::i8)); in combineX86ShuffleChain()
38890 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38905 Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1, in combineX86ShuffleChain()
38906 DAG.getTargetConstant(BitLen, DL, MVT::i8), in combineX86ShuffleChain()
38907 DAG.getTargetConstant(BitIdx, DL, MVT::i8)); in combineX86ShuffleChain()
38908 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38916 Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2, in combineX86ShuffleChain()
38917 DAG.getTargetConstant(BitLen, DL, MVT::i8), in combineX86ShuffleChain()
38918 DAG.getTargetConstant(BitIdx, DL, MVT::i8)); in combineX86ShuffleChain()
38919 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38935 Res = DAG.getNode(Opc, DL, ShuffleVT, V1); in combineX86ShuffleChain()
38937 Res = widenSubVector(Res, true, Subtarget, DAG, DL, RootSizeInBits); in combineX86ShuffleChain()
38938 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38959 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShuffleSrcVT, V1, V2); in combineX86ShuffleChain()
38960 Res = DAG.getNode(ISD::TRUNCATE, DL, IntMaskVT, Res); in combineX86ShuffleChain()
38961 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
38991 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true); in combineX86ShuffleChain()
38993 Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res); in combineX86ShuffleChain()
38994 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
39005 V2 = DAG.getUNDEF(MaskVT); in combineX86ShuffleChain()
39006 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); in combineX86ShuffleChain()
39007 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
39028 V2 = getZeroVector(MaskVT, Subtarget, DAG, DL); in combineX86ShuffleChain()
39029 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); in combineX86ShuffleChain()
39030 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
39037 AllowVariableCrossLaneMask, AllowVariablePerLaneMask, DAG, in combineX86ShuffleChain()
39055 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); in combineX86ShuffleChain()
39056 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
39080 SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL); in combineX86ShuffleChain()
39084 Res = DAG.getNode(AndOpcode, DL, MaskVT, Res, BitMask); in combineX86ShuffleChain()
39085 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
39097 M < 0 ? DAG.getUNDEF(MVT::i32) : DAG.getConstant(M % 4, DL, MVT::i32); in combineX86ShuffleChain()
39100 SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx); in combineX86ShuffleChain()
39102 Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask); in combineX86ShuffleChain()
39103 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
39135 SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true); in combineX86ShuffleChain()
39136 Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp, in combineX86ShuffleChain()
39137 DAG.getTargetConstant(M2ZImm, DL, MVT::i8)); in combineX86ShuffleChain()
39138 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
39156 PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8)); in combineX86ShuffleChain()
39160 PSHUFBMask.push_back(DAG.getConstant(0x80, DL, MVT::i8)); in combineX86ShuffleChain()
39165 PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8)); in combineX86ShuffleChain()
39169 SDValue PSHUFBMaskOp = DAG.getBuildVector(ByteVT, DL, PSHUFBMask); in combineX86ShuffleChain()
39170 Res = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Res, PSHUFBMaskOp); in combineX86ShuffleChain()
39171 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
39188 VPPERMMask.push_back(DAG.getUNDEF(MVT::i8)); in combineX86ShuffleChain()
39192 VPPERMMask.push_back(DAG.getConstant(0x80, DL, MVT::i8)); in combineX86ShuffleChain()
39196 VPPERMMask.push_back(DAG.getConstant(M, DL, MVT::i8)); in combineX86ShuffleChain()
39201 SDValue VPPERMMaskOp = DAG.getBuildVector(ByteVT, DL, VPPERMMask); in combineX86ShuffleChain()
39202 Res = DAG.getNode(X86ISD::VPPERM, DL, ByteVT, V1, V2, VPPERMMaskOp); in combineX86ShuffleChain()
39203 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
39210 AllowVariableCrossLaneMask, AllowVariablePerLaneMask, DAG, Subtarget)) in combineX86ShuffleChain()
39230 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); in combineX86ShuffleChain()
39231 return DAG.getBitcast(RootVT, Res); in combineX86ShuffleChain()
39249 bool AllowVariablePerLaneMask, SelectionDAG &DAG, in combineX86ShuffleChainWithExtract() argument
39268 if (DAG.getTargetLoweringInfo().isTypeLegal(Input.getValueType()) && in combineX86ShuffleChainWithExtract()
39357 AllowVariablePerLaneMask, DAG, Subtarget)) { in combineX86ShuffleChainWithExtract()
39359 extractSubVector(WideShuffle, 0, DAG, SDLoc(Root), RootSizeInBits); in combineX86ShuffleChainWithExtract()
39360 return DAG.getBitcast(RootVT, WideShuffle); in combineX86ShuffleChainWithExtract()
39370 unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG, in canonicalizeShuffleMaskWithHorizOp() argument
39408 (isPack || shouldUseHorizontalOp(Ops.size() == 1, DAG, Subtarget))) { in canonicalizeShuffleMaskWithHorizOp()
39420 return DAG.getUNDEF(VT0); in canonicalizeShuffleMaskWithHorizOp()
39422 return getZeroVector(VT0.getSimpleVT(), Subtarget, DAG, DL); in canonicalizeShuffleMaskWithHorizOp()
39434 SDValue LHS = DAG.getNode(Opcode0, DL, SrcVT, M0, M1); in canonicalizeShuffleMaskWithHorizOp()
39435 SDValue RHS = DAG.getNode(Opcode0, DL, SrcVT, M2, M3); in canonicalizeShuffleMaskWithHorizOp()
39436 return DAG.getNode(Opcode0, DL, VT0, LHS, RHS); in canonicalizeShuffleMaskWithHorizOp()
39464 LHS = DAG.getBitcast(SrcVT, LHS); in canonicalizeShuffleMaskWithHorizOp()
39465 RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS); in canonicalizeShuffleMaskWithHorizOp()
39466 SDValue Res = DAG.getNode(Opcode0, DL, VT0, LHS, RHS); in canonicalizeShuffleMaskWithHorizOp()
39470 Res = DAG.getBitcast(ShuffleVT, Res); in canonicalizeShuffleMaskWithHorizOp()
39471 return DAG.getNode(X86ISD::SHUFP, DL, ShuffleVT, Res, Res, in canonicalizeShuffleMaskWithHorizOp()
39472 getV4X86ShuffleImm8ForMask(PostMask, DL, DAG)); in canonicalizeShuffleMaskWithHorizOp()
39535 shouldUseHorizontalOp(SingleOp, DAG, Subtarget)) { in canonicalizeShuffleMaskWithHorizOp()
39541 SDValue Undef = DAG.getUNDEF(SrcVT); in canonicalizeShuffleMaskWithHorizOp()
39542 SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL); in canonicalizeShuffleMaskWithHorizOp()
39548 return DAG.getNode(Opcode0, DL, VT0, Lo, Hi); in canonicalizeShuffleMaskWithHorizOp()
39564 SDValue V0 = extract128BitVector(BC[0].getOperand(M0 & 1), Idx0, DAG, DL); in canonicalizeShuffleMaskWithHorizOp()
39565 SDValue V1 = extract128BitVector(BC[0].getOperand(M1 & 1), Idx1, DAG, DL); in canonicalizeShuffleMaskWithHorizOp()
39566 SDValue Res = DAG.getNode(Opcode0, DL, HalfVT, V0, V1); in canonicalizeShuffleMaskWithHorizOp()
39567 return widenSubVector(Res, false, Subtarget, DAG, DL, 256); in canonicalizeShuffleMaskWithHorizOp()
39580 SelectionDAG &DAG, const SDLoc &DL, in combineX86ShufflesConstants() argument
39600 bool IsOptimizingSize = DAG.shouldOptForSize(); in combineX86ShufflesConstants()
39645 return getZeroVector(VT, Subtarget, DAG, DL); in combineX86ShufflesConstants()
39655 if (!DAG.getTargetLoweringInfo().isTypeLegal(MaskVT)) in combineX86ShufflesConstants()
39658 SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL); in combineX86ShufflesConstants()
39659 return DAG.getBitcast(VT, CstOp); in combineX86ShufflesConstants()
39703 bool AllowVariablePerLaneMask, SelectionDAG &DAG, in combineX86ShufflesRecursively() argument
39760 OpZero, DAG, Depth, false)) { in combineX86ShufflesRecursively()
39965 return DAG.getUNDEF(RootVT); in combineX86ShufflesRecursively()
39967 return getZeroVector(RootVT, Subtarget, DAG, DL); in combineX86ShufflesRecursively()
39970 return getOnesVector(RootVT, DAG, DL); in combineX86ShufflesRecursively()
40003 HasVariableMask, AllowCrossLaneVar, AllowPerLaneVar, DAG, in combineX86ShufflesRecursively()
40011 RootVT, Ops, Mask, HasVariableMask, DAG, DL, Subtarget)) in combineX86ShufflesRecursively()
40031 Ops, Mask, RootSizeInBits, DL, DAG, Subtarget)) in combineX86ShufflesRecursively()
40032 return DAG.getBitcast(RootVT, HOp); in combineX86ShufflesRecursively()
40070 DAG.getTargetLoweringInfo().SimplifyMultipleUseDemandedVectorElts( in combineX86ShufflesRecursively()
40071 Op, OpScaledDemandedElts, DAG)) in combineX86ShufflesRecursively()
40084 Op = widenSubVector(Op, false, Subtarget, DAG, SDLoc(Op), in combineX86ShufflesRecursively()
40114 AllowVariablePerLaneMask, DAG, Subtarget)) in combineX86ShufflesRecursively()
40133 AllowVariablePerLaneMask, DAG, Subtarget); in combineX86ShufflesRecursively()
40137 static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG, in combineX86ShufflesRecursively() argument
40142 /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, DAG, in combineX86ShufflesRecursively()
40195 SelectionDAG &DAG) { in combineRedundantDWordShuffle() argument
40287 V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0), in combineRedundantDWordShuffle()
40288 getV4X86ShuffleImm8ForMask(Mask, DL, DAG)); in combineRedundantDWordShuffle()
40295 V = DAG.getBitcast(W.getOperand(0).getValueType(), V); in combineRedundantDWordShuffle()
40303 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, V); in combineRedundantDWordShuffle()
40309 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, W.getOperand(1)); in combineRedundantDWordShuffle()
40314 V = DAG.getBitcast(N.getValueType(), V); in combineRedundantDWordShuffle()
40323 SelectionDAG &DAG) { in combineCommutableSHUFP() argument
40329 auto commuteSHUFP = [&VT, &DL, &DAG](SDValue Parent, SDValue V) { in combineCommutableSHUFP()
40335 const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>(); in combineCommutableSHUFP()
40340 return DAG.getNode(X86ISD::SHUFP, DL, VT, N1, N0, in combineCommutableSHUFP()
40341 DAG.getTargetConstant(Imm, DL, MVT::i8)); in combineCommutableSHUFP()
40348 return DAG.getNode(X86ISD::VPERMILPI, DL, VT, NewSHUFP, in combineCommutableSHUFP()
40349 DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8)); in combineCommutableSHUFP()
40358 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, NewSHUFP, in combineCommutableSHUFP()
40359 DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8)); in combineCommutableSHUFP()
40361 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, N1, in combineCommutableSHUFP()
40362 DAG.getTargetConstant(Imm ^ 0x0A, DL, MVT::i8)); in combineCommutableSHUFP()
40364 return DAG.getNode(X86ISD::SHUFP, DL, VT, N0, NewSHUFP, in combineCommutableSHUFP()
40365 DAG.getTargetConstant(Imm ^ 0xA0, DL, MVT::i8)); in combineCommutableSHUFP()
40378 const APInt &DemandedElts, SelectionDAG &DAG, in combineBlendOfPermutes() argument
40463 DAG.getVectorShuffle(VT, DL, DAG.getBitcast(VT, Ops0[0]), in combineBlendOfPermutes()
40464 DAG.getBitcast(VT, Ops1[0]), NewBlendMask); in combineBlendOfPermutes()
40465 return DAG.getVectorShuffle(VT, DL, NewBlend, DAG.getUNDEF(VT), in combineBlendOfPermutes()
40482 static SDValue canonicalizeShuffleWithOp(SDValue N, SelectionDAG &DAG, in canonicalizeShuffleWithOp() argument
40484 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in canonicalizeShuffleWithOp()
40488 auto IsMergeableWithShuffle = [Opc, &DAG](SDValue Op, bool FoldShuf = true, in canonicalizeShuffleWithOp()
40503 DAG.isSplatValue(Op, /*AllowUndefs*/ false); in canonicalizeShuffleWithOp()
40541 Op00 = DAG.getBitcast(ShuffleVT, Op00); in canonicalizeShuffleWithOp()
40542 Op01 = DAG.getBitcast(ShuffleVT, Op01); in canonicalizeShuffleWithOp()
40544 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, N.getOperand(1)); in canonicalizeShuffleWithOp()
40545 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, N.getOperand(1)); in canonicalizeShuffleWithOp()
40547 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00); in canonicalizeShuffleWithOp()
40548 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01); in canonicalizeShuffleWithOp()
40551 return DAG.getBitcast(ShuffleVT, in canonicalizeShuffleWithOp()
40552 DAG.getNode(SrcOpcode, DL, OpVT, in canonicalizeShuffleWithOp()
40553 DAG.getBitcast(OpVT, LHS), in canonicalizeShuffleWithOp()
40554 DAG.getBitcast(OpVT, RHS))); in canonicalizeShuffleWithOp()
40595 Op00 = DAG.getBitcast(ShuffleVT, Op00); in canonicalizeShuffleWithOp()
40596 Op10 = DAG.getBitcast(ShuffleVT, Op10); in canonicalizeShuffleWithOp()
40597 Op01 = DAG.getBitcast(ShuffleVT, Op01); in canonicalizeShuffleWithOp()
40598 Op11 = DAG.getBitcast(ShuffleVT, Op11); in canonicalizeShuffleWithOp()
40600 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10, N.getOperand(2)); in canonicalizeShuffleWithOp()
40601 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11, N.getOperand(2)); in canonicalizeShuffleWithOp()
40603 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10); in canonicalizeShuffleWithOp()
40604 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11); in canonicalizeShuffleWithOp()
40607 return DAG.getBitcast(ShuffleVT, in canonicalizeShuffleWithOp()
40608 DAG.getNode(SrcOpcode, DL, OpVT, in canonicalizeShuffleWithOp()
40609 DAG.getBitcast(OpVT, LHS), in canonicalizeShuffleWithOp()
40610 DAG.getBitcast(OpVT, RHS))); in canonicalizeShuffleWithOp()
40620 Op00 = DAG.getBitcast(ShuffleVT, Op00); in canonicalizeShuffleWithOp()
40621 Op10 = DAG.getBitcast(ShuffleVT, Op10); in canonicalizeShuffleWithOp()
40623 Res = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10, N.getOperand(2)); in canonicalizeShuffleWithOp()
40625 Res = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10); in canonicalizeShuffleWithOp()
40628 return DAG.getBitcast( in canonicalizeShuffleWithOp()
40630 DAG.getNode(SrcOpcode, DL, OpVT, DAG.getBitcast(OpVT, Res))); in canonicalizeShuffleWithOp()
40641 SelectionDAG &DAG, in canonicalizeLaneShuffleWithRepeatedOps() argument
40661 DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS, V.getOperand(2)); in canonicalizeLaneShuffleWithRepeatedOps()
40662 Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res); in canonicalizeLaneShuffleWithRepeatedOps()
40663 return DAG.getBitcast(VT, Res); in canonicalizeLaneShuffleWithRepeatedOps()
40680 SDValue Res = DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS, in canonicalizeLaneShuffleWithRepeatedOps()
40682 Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res, Src0.getOperand(1)); in canonicalizeLaneShuffleWithRepeatedOps()
40683 return DAG.getBitcast(VT, Res); in canonicalizeLaneShuffleWithRepeatedOps()
40693 SelectionDAG &DAG, in combineTargetShuffle() argument
40699 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineTargetShuffle()
40701 if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG)) in combineTargetShuffle()
40712 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::f64, MVT::v2f64, DAG)) { in combineTargetShuffle()
40713 SDValue Movddup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, VZLoad); in combineTargetShuffle()
40715 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1)); in combineTargetShuffle()
40742 /*AllowPerLaneVarMask*/ true, DAG, Subtarget)) in combineTargetShuffle()
40743 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineTargetShuffle()
40744 DAG.getBitcast(SrcVT, Res)); in combineTargetShuffle()
40753 BCVT.getScalarType().getTypeForEVT(*DAG.getContext()))) { in combineTargetShuffle()
40754 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), BCVT.getScalarType(), in combineTargetShuffle()
40756 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC)); in combineTargetShuffle()
40771 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC)); in combineTargetShuffle()
40776 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineTargetShuffle()
40777 extract128BitVector(Src, 0, DAG, DL)); in combineTargetShuffle()
40782 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0)); in combineTargetShuffle()
40790 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0)); in combineTargetShuffle()
40799 return extractSubVector(SDValue(User, 0), 0, DAG, DL, in combineTargetShuffle()
40808 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40811 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, in combineTargetShuffle()
40817 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1)); in combineTargetShuffle()
40820 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd, in combineTargetShuffle()
40821 DAG.getIntPtrConstant(0, DL)); in combineTargetShuffle()
40840 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40842 SDValue BcastLd = DAG.getMemIntrinsicNode( in combineTargetShuffle()
40847 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1)); in combineTargetShuffle()
40858 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40861 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, in combineTargetShuffle()
40864 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1)); in combineTargetShuffle()
40883 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40884 SDValue Ptr = DAG.getMemBasePlusOffset( in combineTargetShuffle()
40887 SDValue BcastLd = DAG.getMemIntrinsicNode( in combineTargetShuffle()
40893 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1)); in combineTargetShuffle()
40904 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40907 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, in combineTargetShuffle()
40910 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1)); in combineTargetShuffle()
40923 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40925 SDValue BcastLd = DAG.getMemIntrinsicNode( in combineTargetShuffle()
40930 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1)); in combineTargetShuffle()
40946 narrowLoadToVZLoad(LN, VT.getVectorElementType(), VT, DAG)) { in combineTargetShuffle()
40948 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1)); in combineTargetShuffle()
40960 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40963 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, in combineTargetShuffle()
40966 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1)); in combineTargetShuffle()
40980 if (DAG.MaskedValueIsZero(In, Mask)) { in combineTargetShuffle()
40981 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, In); in combineTargetShuffle()
40983 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Trunc); in combineTargetShuffle()
40984 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, VecVT, SclVec); in combineTargetShuffle()
40985 return DAG.getBitcast(VT, Movl); in combineTargetShuffle()
40996 Type *ScalarTy = ScalarVT.getTypeForEVT(*DAG.getContext()); in combineTargetShuffle()
41003 MVT PVT = TLI.getPointerTy(DAG.getDataLayout()); in combineTargetShuffle()
41004 SDValue CP = DAG.getConstantPool(ConstantVector::get(ConstantVec), PVT); in combineTargetShuffle()
41006 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()); in combineTargetShuffle()
41008 return DAG.getLoad(VT, DL, DAG.getEntryNode(), CP, MPI, Alignment, in combineTargetShuffle()
41026 In = DAG.getBitcast(SubVT, In); in combineTargetShuffle()
41027 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, SubVT, In); in combineTargetShuffle()
41028 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in combineTargetShuffle()
41029 getZeroVector(VT, Subtarget, DAG, DL), Movl, in combineTargetShuffle()
41052 return DAG.getBitcast( in combineTargetShuffle()
41053 VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0), in combineTargetShuffle()
41055 DAG.getTargetConstant(NewBlendMask.getZExtValue(), in combineTargetShuffle()
41082 /*HasVariableMask=*/true, DAG, DL, Subtarget)) { in combineTargetShuffle()
41083 SDValue NewLHS = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, in combineTargetShuffle()
41085 SDValue NewRHS = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, in combineTargetShuffle()
41087 return DAG.getNode(X86ISD::BLENDI, DL, VT, in combineTargetShuffle()
41088 DAG.getBitcast(VT, NewLHS), in combineTargetShuffle()
41089 DAG.getBitcast(VT, NewRHS), N.getOperand(2)); in combineTargetShuffle()
41110 if (getTargetShuffleInputs(Sub, SubOps, SubMask, DAG, 0, false) && in combineTargetShuffle()
41116 Ops[i] = DAG.getBitcast(VT, SubOps[0]); in combineTargetShuffle()
41124 Ops.push_back(getV4X86ShuffleImm8ForMask(Mask, DL, DAG)); in combineTargetShuffle()
41125 return DAG.getNode(X86ISD::SHUFP, DL, VT, Ops); in combineTargetShuffle()
41140 SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1); in combineTargetShuffle()
41141 return DAG.getBitcast(VT, Res); in combineTargetShuffle()
41157 collectConcatOps(LHS.getNode(), LHSOps, DAG) && LHSOps.size() == 2) { in combineTargetShuffle()
41158 NewLHS = widenSubVector(LHSOps[1], false, Subtarget, DAG, DL, 512); in combineTargetShuffle()
41162 collectConcatOps(RHS.getNode(), RHSOps, DAG) && RHSOps.size() == 2) { in combineTargetShuffle()
41163 NewRHS = widenSubVector(RHSOps[1], false, Subtarget, DAG, DL, 512); in combineTargetShuffle()
41167 return DAG.getNode(X86ISD::SHUF128, DL, VT, NewLHS ? NewLHS : LHS, in combineTargetShuffle()
41169 DAG.getTargetConstant(Mask, DL, MVT::i8)); in combineTargetShuffle()
41181 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT, in combineTargetShuffle()
41182 DAG.getBitcast(SrcVT, LHS), in combineTargetShuffle()
41183 DAG.getBitcast(SrcVT, RHS), in combineTargetShuffle()
41189 if (SDValue Res = canonicalizeLaneShuffleWithRepeatedOps(N, DAG, DL)) in combineTargetShuffle()
41199 if (collectConcatOps(Src.getNode(), SubOps, DAG) && SubOps.size() == 2) in combineTargetShuffle()
41213 SubLo = DAG.getBitcast(SubVT, SubLo); in combineTargetShuffle()
41214 SubHi = DAG.getBitcast(SubVT, SubHi); in combineTargetShuffle()
41215 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubLo, SubHi); in combineTargetShuffle()
41238 SDValue Res = DAG.getNode(Opcode, DL, VT, in combineTargetShuffle()
41239 DAG.getBitcast(VT, V.getOperand(0)), N1); in combineTargetShuffle()
41240 Res = DAG.getBitcast(InnerVT, Res); in combineTargetShuffle()
41241 Res = DAG.getNode(V.getOpcode(), DL, InnerVT, Res, V.getOperand(1)); in combineTargetShuffle()
41242 return DAG.getBitcast(VT, Res); in combineTargetShuffle()
41272 SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL); in combineTargetShuffle()
41273 N10 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N10, ZeroIdx); in combineTargetShuffle()
41274 N11 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N11, ZeroIdx); in combineTargetShuffle()
41275 SDValue Scl = DAG.getNode(Opcode1, DL, SVT, N10, N11); in combineTargetShuffle()
41276 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl); in combineTargetShuffle()
41277 return DAG.getNode(Opcode, DL, VT, N0, SclVec); in combineTargetShuffle()
41294 return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1, in combineTargetShuffle()
41295 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8)); in combineTargetShuffle()
41299 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT), in combineTargetShuffle()
41300 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8)); in combineTargetShuffle()
41311 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT), in combineTargetShuffle()
41312 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8)); in combineTargetShuffle()
41319 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1, in combineTargetShuffle()
41320 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8)); in combineTargetShuffle()
41366 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1, in combineTargetShuffle()
41367 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8)); in combineTargetShuffle()
41376 SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(), in combineTargetShuffle()
41379 SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, in combineTargetShuffle()
41380 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, in combineTargetShuffle()
41382 DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8)); in combineTargetShuffle()
41383 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1)); in combineTargetShuffle()
41406 SDValue Mask = widenSubVector(N.getOperand(1), false, Subtarget, DAG, in combineTargetShuffle()
41408 SDValue Perm = DAG.getNode(X86ISD::VPERMV, DL, WideVT, Mask, in combineTargetShuffle()
41409 DAG.getBitcast(WideVT, V1.getOperand(0))); in combineTargetShuffle()
41410 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Perm, in combineTargetShuffle()
41411 DAG.getIntPtrConstant(0, DL)); in combineTargetShuffle()
41442 V = DAG.getBitcast(DVT, V); in combineTargetShuffle()
41443 V = DAG.getNode(X86ISD::PSHUFD, DL, DVT, V, in combineTargetShuffle()
41444 getV4X86ShuffleImm8ForMask(DMask, DL, DAG)); in combineTargetShuffle()
41445 return DAG.getBitcast(VT, V); in combineTargetShuffle()
41474 V = DAG.getBitcast(VT, D.getOperand(0)); in combineTargetShuffle()
41475 return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL in combineTargetShuffle()
41485 if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DL, DAG)) in combineTargetShuffle()
41535 SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, in isAddSubOrSubAdd() argument
41539 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in isAddSubOrSubAdd()
41596 SelectionDAG &DAG) { in combineShuffleToFMAddSub() argument
41604 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineShuffleToFMAddSub()
41630 return DAG.getNode(Opcode, DL, VT, FMAdd.getOperand(0), FMAdd.getOperand(1), in combineShuffleToFMAddSub()
41638 SelectionDAG &DAG) { in combineShuffleToAddSubOrFMAddSub() argument
41639 if (SDValue V = combineShuffleToFMAddSub(N, DL, Subtarget, DAG)) in combineShuffleToAddSubOrFMAddSub()
41644 if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd)) in combineShuffleToAddSubOrFMAddSub()
41651 if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, 2)) { in combineShuffleToAddSubOrFMAddSub()
41653 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2); in combineShuffleToAddSubOrFMAddSub()
41671 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1); in combineShuffleToAddSubOrFMAddSub()
41678 SelectionDAG &DAG, in combineShuffleOfConcatUndef() argument
41714 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0), in combineShuffleOfConcatUndef()
41716 return DAG.getVectorShuffle(VT, DL, Concat, DAG.getUNDEF(VT), Mask); in combineShuffleOfConcatUndef()
41722 static SDValue narrowShuffle(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG) { in narrowShuffle() argument
41724 if (!DAG.getTargetLoweringInfo().isTypeLegal(Shuf->getValueType(0))) in narrowShuffle()
41749 HalfIdx2, false, DAG, /*UseConcat*/ true); in narrowShuffle()
41752 static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG, in combineShuffle() argument
41756 if (SDValue V = narrowShuffle(Shuf, DAG)) in combineShuffle()
41763 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineShuffle()
41766 combineShuffleToAddSubOrFMAddSub(N, dl, Subtarget, DAG)) in combineShuffle()
41771 VT, SDValue(N, 0), dl, DAG, Subtarget, /*IsAfterLegalize*/ true)) in combineShuffle()
41780 if (SDValue ShufConcat = combineShuffleOfConcatUndef(N, dl, DAG, Subtarget)) in combineShuffle()
41785 if (SDValue Shuffle = combineTargetShuffle(Op, dl, DAG, DCI, Subtarget)) in combineShuffle()
41793 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineShuffle()
41806 if (SDValue BinOp = canonicalizeShuffleWithOp(Op, DAG, dl)) in combineShuffle()
41873 SDValue CV = TLO.DAG.getConstantPool(ConstantVector::get(ConstVecOps), BCVT); in SimplifyDemandedVectorEltsForTargetShuffle()
41874 SDValue LegalCV = LowerConstantPool(CV, TLO.DAG); in SimplifyDemandedVectorEltsForTargetShuffle()
41875 SDValue NewMask = TLO.DAG.getLoad( in SimplifyDemandedVectorEltsForTargetShuffle()
41876 BCVT, DL, TLO.DAG.getEntryNode(), LegalCV, in SimplifyDemandedVectorEltsForTargetShuffle()
41877 MachinePointerInfo::getConstantPool(TLO.DAG.getMachineFunction()), in SimplifyDemandedVectorEltsForTargetShuffle()
41879 return TLO.CombineTo(Mask, TLO.DAG.getBitcast(Mask.getValueType(), NewMask)); in SimplifyDemandedVectorEltsForTargetShuffle()
41948 LHS, DemandedSrcElts, TLO.DAG, Depth + 1); in SimplifyDemandedVectorEltsForTargetNode()
41950 RHS, DemandedSrcElts, TLO.DAG, Depth + 1); in SimplifyDemandedVectorEltsForTargetNode()
41955 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS)); in SimplifyDemandedVectorEltsForTargetNode()
41997 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
42002 Src, DemandedElts, TLO.DAG, Depth + 1)) in SimplifyDemandedVectorEltsForTargetNode()
42004 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1))); in SimplifyDemandedVectorEltsForTargetNode()
42023 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
42069 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8); in SimplifyDemandedVectorEltsForTargetNode()
42071 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA)); in SimplifyDemandedVectorEltsForTargetNode()
42108 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8); in SimplifyDemandedVectorEltsForTargetNode()
42110 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA)); in SimplifyDemandedVectorEltsForTargetNode()
42173 TLO.DAG, Depth + 1); in SimplifyDemandedVectorEltsForTargetNode()
42175 TLO.DAG, Depth + 1); in SimplifyDemandedVectorEltsForTargetNode()
42180 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS)); in SimplifyDemandedVectorEltsForTargetNode()
42221 TLO.DAG, Depth + 1); in SimplifyDemandedVectorEltsForTargetNode()
42223 TLO.DAG, Depth + 1); in SimplifyDemandedVectorEltsForTargetNode()
42228 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1)); in SimplifyDemandedVectorEltsForTargetNode()
42258 TLO.DAG, Depth + 1); in SimplifyDemandedVectorEltsForTargetNode()
42260 TLO.DAG, Depth + 1); in SimplifyDemandedVectorEltsForTargetNode()
42265 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1)); in SimplifyDemandedVectorEltsForTargetNode()
42289 DemandedElts, TLO.DAG, Subtarget, SDLoc(Op))) in SimplifyDemandedVectorEltsForTargetNode()
42319 if (TLO.DAG.MaskedVectorIsZero(Src, DemandedUpperElts, Depth + 1)) in SimplifyDemandedVectorEltsForTargetNode()
42330 SDValue Elt = TLO.DAG.getLoad(SVT, DL, Mem->getChain(), Mem->getBasePtr(), in SimplifyDemandedVectorEltsForTargetNode()
42332 SDValue Vec = TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Elt); in SimplifyDemandedVectorEltsForTargetNode()
42333 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Vec)); in SimplifyDemandedVectorEltsForTargetNode()
42345 Src = widenSubVector(VT.getSimpleVT(), Src, false, Subtarget, TLO.DAG, in SimplifyDemandedVectorEltsForTargetNode()
42357 Src, SrcElts, TLO.DAG, Depth + 1)) in SimplifyDemandedVectorEltsForTargetNode()
42358 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedVectorEltsForTargetNode()
42399 Src = extractSubVector(Src, 0, TLO.DAG, DL, ExtSizeInBits); in SimplifyDemandedVectorEltsForTargetNode()
42400 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), in SimplifyDemandedVectorEltsForTargetNode()
42402 SDValue Bcst = TLO.DAG.getNode(X86ISD::VBROADCAST, DL, BcstVT, Src); in SimplifyDemandedVectorEltsForTargetNode()
42403 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0, in SimplifyDemandedVectorEltsForTargetNode()
42404 TLO.DAG, DL, ExtSizeInBits)); in SimplifyDemandedVectorEltsForTargetNode()
42409 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), in SimplifyDemandedVectorEltsForTargetNode()
42411 SDVTList Tys = TLO.DAG.getVTList(BcstVT, MVT::Other); in SimplifyDemandedVectorEltsForTargetNode()
42413 SDValue Bcst = TLO.DAG.getMemIntrinsicNode( in SimplifyDemandedVectorEltsForTargetNode()
42416 TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1), in SimplifyDemandedVectorEltsForTargetNode()
42418 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0, in SimplifyDemandedVectorEltsForTargetNode()
42419 TLO.DAG, DL, ExtSizeInBits)); in SimplifyDemandedVectorEltsForTargetNode()
42428 TLO.DAG.getLoad(MemVT, DL, MemIntr->getChain(), in SimplifyDemandedVectorEltsForTargetNode()
42430 TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1), in SimplifyDemandedVectorEltsForTargetNode()
42432 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Ld, 0, in SimplifyDemandedVectorEltsForTargetNode()
42433 TLO.DAG, DL, ExtSizeInBits)); in SimplifyDemandedVectorEltsForTargetNode()
42436 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), in SimplifyDemandedVectorEltsForTargetNode()
42439 getBROADCAST_LOAD(Opc, DL, BcstVT, MemVT, MemIntr, 0, TLO.DAG)) in SimplifyDemandedVectorEltsForTargetNode()
42441 insertSubVector(TLO.DAG.getUNDEF(VT), BcstLd, 0, in SimplifyDemandedVectorEltsForTargetNode()
42442 TLO.DAG, DL, ExtSizeInBits)); in SimplifyDemandedVectorEltsForTargetNode()
42459 extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits); in SimplifyDemandedVectorEltsForTargetNode()
42461 TLO.DAG.getNode(Opc, DL, Ext0.getValueType(), Ext0, Op.getOperand(1)); in SimplifyDemandedVectorEltsForTargetNode()
42462 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
42464 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits); in SimplifyDemandedVectorEltsForTargetNode()
42475 SDValue Ext = extractSubVector(Op.getOperand(0), 2, TLO.DAG, DL, 128); in SimplifyDemandedVectorEltsForTargetNode()
42476 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
42477 SDValue Insert = insertSubVector(UndefVec, Ext, 0, TLO.DAG, DL, 128); in SimplifyDemandedVectorEltsForTargetNode()
42489 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, DL)); in SimplifyDemandedVectorEltsForTargetNode()
42493 extractSubVector(Op.getOperand(SrcIdx), EltIdx, TLO.DAG, DL, 128); in SimplifyDemandedVectorEltsForTargetNode()
42494 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
42496 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits); in SimplifyDemandedVectorEltsForTargetNode()
42540 Ops.push_back(SrcVT.isVector() ? extractSubVector(SrcOp, 0, TLO.DAG, DL, in SimplifyDemandedVectorEltsForTargetNode()
42547 SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ops); in SimplifyDemandedVectorEltsForTargetNode()
42548 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
42550 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits); in SimplifyDemandedVectorEltsForTargetNode()
42559 if (!DemandedElts.isOne() && TLO.DAG.isSplatValue(Op, /*AllowUndefs*/false)) in SimplifyDemandedVectorEltsForTargetNode()
42567 OpZero, TLO.DAG, Depth, false)) in SimplifyDemandedVectorEltsForTargetNode()
42589 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorEltsForTargetNode()
42594 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
42598 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, OpInputs[Src])); in SimplifyDemandedVectorEltsForTargetNode()
42640 /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, TLO.DAG, in SimplifyDemandedVectorEltsForTargetNode()
42683 if (!Is32BitAVX512 || !TLO.DAG.isSplatValue(LHS)) in SimplifyDemandedBitsForTargetNode()
42685 if (!Is32BitAVX512 || !TLO.DAG.isSplatValue(RHS)) in SimplifyDemandedBitsForTargetNode()
42700 SDValue Mask = TLO.DAG.getConstant(DemandedMask, DL, VT); in SimplifyDemandedBitsForTargetNode()
42701 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, DL, VT, LHS, Mask)); in SimplifyDemandedBitsForTargetNode()
42706 LHS, DemandedMaskLHS, OriginalDemandedElts, TLO.DAG, Depth + 1); in SimplifyDemandedBitsForTargetNode()
42708 RHS, DemandedMaskRHS, OriginalDemandedElts, TLO.DAG, Depth + 1); in SimplifyDemandedBitsForTargetNode()
42713 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, DemandedLHS, DemandedRHS)); in SimplifyDemandedBitsForTargetNode()
42761 SDValue NewShift = TLO.DAG.getNode( in SimplifyDemandedBitsForTargetNode()
42763 TLO.DAG.getTargetConstant(std::abs(Diff), SDLoc(Op), MVT::i8)); in SimplifyDemandedBitsForTargetNode()
42770 TLO.DAG.ComputeNumSignBits(Op0, OriginalDemandedElts, Depth + 1); in SimplifyDemandedBitsForTargetNode()
42823 TLO.DAG.ComputeNumSignBits(Op00, OriginalDemandedElts); in SimplifyDemandedBitsForTargetNode()
42845 Op, TLO.DAG.getNode(X86ISD::VSRLI, SDLoc(Op), VT, Op0, Op1)); in SimplifyDemandedBitsForTargetNode()
42859 Sel, SignMask, OriginalDemandedElts, TLO.DAG, Depth + 1); in SimplifyDemandedBitsForTargetNode()
42861 LHS, OriginalDemandedBits, OriginalDemandedElts, TLO.DAG, Depth + 1); in SimplifyDemandedBitsForTargetNode()
42863 RHS, OriginalDemandedBits, OriginalDemandedElts, TLO.DAG, Depth + 1); in SimplifyDemandedBitsForTargetNode()
42869 return TLO.CombineTo(Op, TLO.DAG.getNode(X86ISD::BLENDV, SDLoc(Op), VT, in SimplifyDemandedBitsForTargetNode()
42889 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedBitsForTargetNode()
42903 Vec, DemandedVecBits, DemandedVecElts, TLO.DAG, Depth + 1)) in SimplifyDemandedBitsForTargetNode()
42905 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, V, Op.getOperand(1))); in SimplifyDemandedBitsForTargetNode()
42962 Op.getOperand(0), SignMask, DemandedLHS, TLO.DAG, Depth + 1); in SimplifyDemandedBitsForTargetNode()
42964 Op.getOperand(1), SignMask, DemandedRHS, TLO.DAG, Depth + 1); in SimplifyDemandedBitsForTargetNode()
42968 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, Op0, Op1)); in SimplifyDemandedBitsForTargetNode()
42989 TLO.DAG.getNode(ISD::TRUNCATE, SDLoc(Src), NewSrcVT, Src); in SimplifyDemandedBitsForTargetNode()
42992 TLO.DAG.getNode(X86ISD::VBROADCAST, SDLoc(Op), NewVT, NewSrc); in SimplifyDemandedBitsForTargetNode()
42993 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, NewBcst)); in SimplifyDemandedBitsForTargetNode()
43012 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedBitsForTargetNode()
43017 SDValue NewSrc = extract128BitVector(Src, 0, TLO.DAG, SDLoc(Src)); in SimplifyDemandedBitsForTargetNode()
43018 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedBitsForTargetNode()
43045 Src, DemandedSrcBits, DemandedElts, TLO.DAG, Depth + 1)) in SimplifyDemandedBitsForTargetNode()
43046 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedBitsForTargetNode()
43092 Op, TLO.DAG.getNode(X86ISD::BEXTR, DL, VT, Op0, in SimplifyDemandedBitsForTargetNode()
43093 TLO.DAG.getConstant(MaskedVal1, DL, VT))); in SimplifyDemandedBitsForTargetNode()
43124 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedBitsForTargetNode()
43165 SelectionDAG &DAG, unsigned Depth) const { in SimplifyMultipleUseDemandedBitsForTargetNode() argument
43188 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1); in SimplifyMultipleUseDemandedBitsForTargetNode()
43213 KnownBits CondKnown = DAG.computeKnownBits(Cond, DemandedElts, Depth + 1); in SimplifyMultipleUseDemandedBitsForTargetNode()
43225 KnownBits LHSKnown = DAG.computeKnownBits(LHS, DemandedElts, Depth + 1); in SimplifyMultipleUseDemandedBitsForTargetNode()
43226 KnownBits RHSKnown = DAG.computeKnownBits(RHS, DemandedElts, Depth + 1); in SimplifyMultipleUseDemandedBitsForTargetNode()
43241 ShuffleUndef, ShuffleZero, DAG, Depth, false)) { in SimplifyMultipleUseDemandedBitsForTargetNode()
43251 return DAG.getUNDEF(VT); in SimplifyMultipleUseDemandedBitsForTargetNode()
43253 return getZeroVector(VT.getSimpleVT(), Subtarget, DAG, SDLoc(Op)); in SimplifyMultipleUseDemandedBitsForTargetNode()
43275 return DAG.getBitcast(VT, ShuffleOps[IdentityOp.countr_zero()]); in SimplifyMultipleUseDemandedBitsForTargetNode()
43280 Op, DemandedBits, DemandedElts, DAG, Depth); in SimplifyMultipleUseDemandedBitsForTargetNode()
43284 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, in isGuaranteedNotToBeUndefOrPoisonForTargetNode() argument
43307 !DAG.isGuaranteedNotToBeUndefOrPoison( in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
43316 Op, DemandedElts, DAG, PoisonOnly, Depth); in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
43320 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, in canCreateUndefOrPoisonForTargetNode() argument
43339 Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth); in canCreateUndefOrPoisonForTargetNode()
43345 const SelectionDAG &DAG, in isSplatValueForTargetNode() argument
43358 DAG, Depth); in isSplatValueForTargetNode()
43405 static SDValue adjustBitcastSrcVectorSSE1(SelectionDAG &DAG, SDValue Src, in adjustBitcastSrcVectorSSE1() argument
43418 return DAG.getBitcast(MVT::v4f32, Op0); in adjustBitcastSrcVectorSSE1()
43427 SDValue Op0 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(0), DL); in adjustBitcastSrcVectorSSE1()
43428 SDValue Op1 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(1), DL); in adjustBitcastSrcVectorSSE1()
43430 return DAG.getNode(getAltBitOpcode(Src.getOpcode()), DL, MVT::v4f32, Op0, in adjustBitcastSrcVectorSSE1()
43439 static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT, in signExtendBitcastSrcVector() argument
43446 return DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src); in signExtendBitcastSrcVector()
43450 return DAG.getNode( in signExtendBitcastSrcVector()
43452 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(0), DL), in signExtendBitcastSrcVector()
43453 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL)); in signExtendBitcastSrcVector()
43456 return DAG.getSelect( in signExtendBitcastSrcVector()
43458 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL), in signExtendBitcastSrcVector()
43459 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(2), DL)); in signExtendBitcastSrcVector()
43470 static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src, in combineBitcastvxi1() argument
43480 if (SDValue V = adjustBitcastSrcVectorSSE1(DAG, Src, DL)) { in combineBitcastvxi1()
43481 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, in combineBitcastvxi1()
43482 DAG.getBitcast(MVT::v4f32, V)); in combineBitcastvxi1()
43483 return DAG.getZExtOrTrunc(V, DL, VT); in combineBitcastvxi1()
43516 if (collectConcatOps(Src.getNode(), SubSrcOps, DAG) && in combineBitcastvxi1()
43523 *DAG.getContext(), LowerOp.getValueType().getVectorMinNumElements()); in combineBitcastvxi1()
43524 if (SDValue V = combineBitcastvxi1(DAG, SubVT, LowerOp, DL, Subtarget)) { in combineBitcastvxi1()
43525 EVT IntVT = VT.getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); in combineBitcastvxi1()
43526 return DAG.getBitcast(VT, DAG.getNode(ISD::ANY_EXTEND, DL, IntVT, V)); in combineBitcastvxi1()
43599 SDValue V = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL) in combineBitcastvxi1()
43600 : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src); in combineBitcastvxi1()
43603 V = getPMOVMSKB(DL, V, DAG, Subtarget); in combineBitcastvxi1()
43606 V = widenSubVector(V, false, Subtarget, DAG, DL, 256); in combineBitcastvxi1()
43607 V = DAG.getNode(ISD::TRUNCATE, DL, MVT::v16i8, V); in combineBitcastvxi1()
43609 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V); in combineBitcastvxi1()
43613 EVT::getIntegerVT(*DAG.getContext(), SrcVT.getVectorNumElements()); in combineBitcastvxi1()
43614 V = DAG.getZExtOrTrunc(V, DL, IntVT); in combineBitcastvxi1()
43615 return DAG.getBitcast(VT, V); in combineBitcastvxi1()
43619 static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) { in combinevXi1ConstantToInteger() argument
43632 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), Imm.getBitWidth()); in combinevXi1ConstantToInteger()
43633 return DAG.getConstant(Imm, SDLoc(Op), IntVT); in combinevXi1ConstantToInteger()
43636 static SDValue combineCastedMaskArithmetic(SDNode *N, SelectionDAG &DAG, in combineCastedMaskArithmetic() argument
43673 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT, LHS.getOperand(0), in combineCastedMaskArithmetic()
43674 DAG.getBitcast(DstVT, RHS)); in combineCastedMaskArithmetic()
43678 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT, in combineCastedMaskArithmetic()
43679 DAG.getBitcast(DstVT, LHS), RHS.getOperand(0)); in combineCastedMaskArithmetic()
43684 RHS = combinevXi1ConstantToInteger(RHS, DAG); in combineCastedMaskArithmetic()
43685 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT, in combineCastedMaskArithmetic()
43686 DAG.getBitcast(DstVT, LHS), RHS); in combineCastedMaskArithmetic()
43692 static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG, in createMMXBuildVector() argument
43701 return DAG.getUNDEF(MVT::x86mmx); in createMMXBuildVector()
43704 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, V); in createMMXBuildVector()
43705 V = DAG.getBitcast(MVT::v2i64, V); in createMMXBuildVector()
43706 return DAG.getNode(X86ISD::MOVDQ2Q, DL, MVT::x86mmx, V); in createMMXBuildVector()
43708 V = DAG.getBitcast(MVT::i32, V); in createMMXBuildVector()
43710 V = DAG.getAnyExtOrTrunc(V, DL, MVT::i32); in createMMXBuildVector()
43712 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, V); in createMMXBuildVector()
43718 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in createMMXBuildVector()
43723 return DAG.getUNDEF(MVT::x86mmx); in createMMXBuildVector()
43730 Splat = DAG.getNode( in createMMXBuildVector()
43732 DAG.getTargetConstant(Intrinsic::x86_mmx_punpcklbw, DL, in createMMXBuildVector()
43733 TLI.getPointerTy(DAG.getDataLayout())), in createMMXBuildVector()
43738 return DAG.getNode( in createMMXBuildVector()
43740 DAG.getTargetConstant(Intrinsic::x86_sse_pshuf_w, DL, in createMMXBuildVector()
43741 TLI.getPointerTy(DAG.getDataLayout())), in createMMXBuildVector()
43742 Splat, DAG.getTargetConstant(ShufMask, DL, MVT::i8)); in createMMXBuildVector()
43757 SDValue Intrin = DAG.getTargetConstant( in createMMXBuildVector()
43758 IntrinOp, DL, TLI.getPointerTy(DAG.getDataLayout())); in createMMXBuildVector()
43760 Ops[i / 2] = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, Intrin, in createMMXBuildVector()
43773 SelectionDAG &DAG, in combineBitcastToBoolVector() argument
43779 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineBitcastToBoolVector()
43787 return DAG.getBitcast(VT, Src); in combineBitcastToBoolVector()
43793 return DAG.getConstant(0, DL, VT); in combineBitcastToBoolVector()
43795 return DAG.getAllOnesConstant(DL, VT); in combineBitcastToBoolVector()
43802 EVT::getVectorVT(*DAG.getContext(), MVT::i1, Src.getValueSizeInBits()); in combineBitcastToBoolVector()
43804 if (SDValue N0 = combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG, in combineBitcastToBoolVector()
43806 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N0, in combineBitcastToBoolVector()
43807 DAG.getIntPtrConstant(0, DL)); in combineBitcastToBoolVector()
43814 EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in combineBitcastToBoolVector()
43817 if (SDValue N0 = combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG, in combineBitcastToBoolVector()
43819 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in combineBitcastToBoolVector()
43820 Opc == ISD::ANY_EXTEND ? DAG.getUNDEF(VT) in combineBitcastToBoolVector()
43821 : DAG.getConstant(0, DL, VT), in combineBitcastToBoolVector()
43822 N0, DAG.getIntPtrConstant(0, DL)); in combineBitcastToBoolVector()
43829 if (SDValue N0 = combineBitcastToBoolVector(VT, V.getOperand(0), DL, DAG, in combineBitcastToBoolVector()
43831 if (SDValue N1 = combineBitcastToBoolVector(VT, V.getOperand(1), DL, DAG, in combineBitcastToBoolVector()
43833 return DAG.getNode(Opc, DL, VT, N0, N1); in combineBitcastToBoolVector()
43844 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget, in combineBitcastToBoolVector()
43846 return DAG.getNode( in combineBitcastToBoolVector()
43848 DAG.getTargetConstant(Amt->getZExtValue(), DL, MVT::i8)); in combineBitcastToBoolVector()
43855 if (SDNode *Alt = DAG.getNodeIfExists(ISD::BITCAST, DAG.getVTList(VT), {V})) in combineBitcastToBoolVector()
43861 static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG, in combineBitcast() argument
43867 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineBitcast()
43877 if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget)) in combineBitcast()
43884 N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i8, N0); in combineBitcast()
43885 N0 = DAG.getBitcast(MVT::v8i1, N0); in combineBitcast()
43886 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, N0, in combineBitcast()
43887 DAG.getIntPtrConstant(0, dl)); in combineBitcast()
43907 Ops.resize(NumConcats, DAG.getConstant(0, dl, SrcVT)); in combineBitcast()
43908 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops); in combineBitcast()
43909 N0 = DAG.getBitcast(MVT::i8, N0); in combineBitcast()
43910 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in combineBitcast()
43915 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(SrcVT)); in combineBitcast()
43917 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops); in combineBitcast()
43918 N0 = DAG.getBitcast(MVT::i8, N0); in combineBitcast()
43919 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in combineBitcast()
43927 combineBitcastToBoolVector(VT, N0, SDLoc(N), DAG, Subtarget)) in combineBitcast()
43941 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, in combineBitcast()
43942 DAG.getBitcast(MVT::i16, N0.getOperand(0))); in combineBitcast()
43960 SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other); in combineBitcast()
43963 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops, in combineBitcast()
43965 DAG.ReplaceAllUsesOfValueWith(SDValue(BCast, 1), ResNode.getValue(1)); in combineBitcast()
43966 return DAG.getBitcast(VT, ResNode); in combineBitcast()
43983 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT, in combineBitcast()
43984 DAG.getConstant(EltBits[0].trunc(32), DL, MVT::i32)); in combineBitcast()
43988 return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64)); in combineBitcast()
44004 N00 = LowUndef ? DAG.getAnyExtOrTrunc(N00, dl, MVT::i32) in combineBitcast()
44005 : DAG.getZExtOrTrunc(N00, dl, MVT::i32); in combineBitcast()
44006 return DAG.getNode(X86ISD::MMX_MOVW2D, dl, VT, N00); in combineBitcast()
44016 return createMMXBuildVector(cast<BuildVectorSDNode>(N0), DAG, Subtarget); in combineBitcast()
44024 return DAG.getNode(X86ISD::MOVDQ2Q, SDLoc(N00), VT, in combineBitcast()
44025 DAG.getBitcast(MVT::v2i64, N00)); in combineBitcast()
44031 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0, in combineBitcast()
44032 DAG.getUNDEF(MVT::v2i32)); in combineBitcast()
44033 return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT, in combineBitcast()
44034 DAG.getBitcast(MVT::v2i64, Res)); in combineBitcast()
44043 return combinevXi1ConstantToInteger(N0, DAG); in combineBitcast()
44050 return DAG.getConstant(1, SDLoc(N0), VT); in combineBitcast()
44052 return DAG.getConstant(0, SDLoc(N0), VT); in combineBitcast()
44079 MovmskIn = DAG.getBitcast(IntVT, MovmskIn); in combineBitcast()
44082 SDValue Cmp = DAG.getSetCC(dl, CmpVT, MovmskIn, in combineBitcast()
44083 DAG.getConstant(0, dl, IntVT), ISD::SETLT); in combineBitcast()
44090 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, CmpVT)); in combineBitcast()
44092 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Ops); in combineBitcast()
44099 if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget)) in combineBitcast()
44134 SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1); in combineBitcast()
44136 return DAG.getNode(Opcode, DL0, VT, LogicOp0.getOperand(0), CastedOp1); in combineBitcast()
44143 SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0); in combineBitcast()
44145 return DAG.getNode(Opcode, DL0, VT, LogicOp1.getOperand(0), CastedOp0); in combineBitcast()
44152 static bool detectExtMul(SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0, in detectExtMul() argument
44175 DAG.computeKnownBits(Op0).countMaxActiveBits() <= 8) && in detectExtMul()
44176 (IsFreeTruncation(Op1) && DAG.ComputeMaxSignificantBits(Op1) <= 8)) in detectExtMul()
44203 static SDValue createVPDPBUSD(SelectionDAG &DAG, SDValue LHS, SDValue RHS, in createVPDPBUSD() argument
44209 LHS = DAG.getZExtOrTrunc(LHS, DL, Vi8VT); in createVPDPBUSD()
44210 RHS = DAG.getSExtOrTrunc(RHS, DL, Vi8VT); in createVPDPBUSD()
44226 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, Vi8VT)); in createVPDPBUSD()
44229 SDValue DpOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops); in createVPDPBUSD()
44231 SDValue DpOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops); in createVPDPBUSD()
44235 auto DpBuilder = [](SelectionDAG &DAG, const SDLoc &DL, in createVPDPBUSD()
44238 return DAG.getNode(X86ISD::VPDPBUSD, DL, VT, Ops); in createVPDPBUSD()
44241 SDValue Zero = DAG.getConstant(0, DL, DpVT); in createVPDPBUSD()
44243 return SplitOpsAndApply(DAG, Subtarget, DL, DpVT, {Zero, DpOp0, DpOp1}, in createVPDPBUSD()
44249 static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0, in createPSADBW() argument
44259 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, InVT)); in createPSADBW()
44262 SDValue SadOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops); in createPSADBW()
44264 SDValue SadOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops); in createPSADBW()
44267 auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL, in createPSADBW()
44270 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops); in createPSADBW()
44273 return SplitOpsAndApply(DAG, Subtarget, DL, SadVT, { SadOp0, SadOp1 }, in createPSADBW()
44279 static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG, in combineMinMaxReduction() argument
44291 SDValue Src = DAG.matchBinOpReduction( in combineMinMaxReduction()
44307 std::tie(Lo, Hi) = splitVector(MinPos, DAG, DL); in combineMinMaxReduction()
44309 MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi); in combineMinMaxReduction()
44320 Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT); in combineMinMaxReduction()
44322 Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT); in combineMinMaxReduction()
44324 Mask = DAG.getAllOnesConstant(DL, SrcVT); in combineMinMaxReduction()
44327 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos); in combineMinMaxReduction()
44334 SDValue Upper = DAG.getVectorShuffle( in combineMinMaxReduction()
44335 SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8), in combineMinMaxReduction()
44337 MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper); in combineMinMaxReduction()
44341 MinPos = DAG.getBitcast(MVT::v8i16, MinPos); in combineMinMaxReduction()
44342 MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos); in combineMinMaxReduction()
44343 MinPos = DAG.getBitcast(SrcVT, MinPos); in combineMinMaxReduction()
44346 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos); in combineMinMaxReduction()
44348 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos, in combineMinMaxReduction()
44349 DAG.getIntPtrConstant(0, DL)); in combineMinMaxReduction()
44353 static SDValue combinePredicateReduction(SDNode *Extract, SelectionDAG &DAG, in combinePredicateReduction() argument
44367 SDValue Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::OR, ISD::AND}); in combinePredicateReduction()
44369 Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::XOR}); in combinePredicateReduction()
44383 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combinePredicateReduction()
44384 LLVMContext &Ctx = *DAG.getContext(); in combinePredicateReduction()
44397 SDValue LHS = DAG.getFreeze(Match.getOperand(0)); in combinePredicateReduction()
44398 SDValue RHS = DAG.getFreeze(Match.getOperand(1)); in combinePredicateReduction()
44401 DAG, X86CC)) in combinePredicateReduction()
44402 return DAG.getNode(ISD::TRUNCATE, DL, ExtractVT, in combinePredicateReduction()
44403 getSETCC(X86CC, V, DL, DAG)); in combinePredicateReduction()
44409 Movmsk = DAG.getBitcast(MovmskVT, Match); in combinePredicateReduction()
44414 std::tie(Lo, Hi) = DAG.SplitVector(Match, DL); in combinePredicateReduction()
44415 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi); in combinePredicateReduction()
44419 Movmsk = combineBitcastvxi1(DAG, MovmskVT, Match, DL, Subtarget); in combinePredicateReduction()
44423 Movmsk = DAG.getZExtOrTrunc(Movmsk, DL, NumElts > 32 ? MVT::i64 : MVT::i32); in combinePredicateReduction()
44439 if (DAG.ComputeNumSignBits(Match) != BitWidth) in combinePredicateReduction()
44444 std::tie(Lo, Hi) = DAG.SplitVector(Match, DL); in combinePredicateReduction()
44445 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi); in combinePredicateReduction()
44457 SDValue BitcastLogicOp = DAG.getBitcast(MaskSrcVT, Match); in combinePredicateReduction()
44458 Movmsk = getPMOVMSKB(DL, BitcastLogicOp, DAG, Subtarget); in combinePredicateReduction()
44467 SDValue Result = DAG.getNode(ISD::PARITY, DL, CmpVT, Movmsk); in combinePredicateReduction()
44468 return DAG.getZExtOrTrunc(Result, DL, ExtractVT); in combinePredicateReduction()
44475 CmpC = DAG.getConstant(0, DL, CmpVT); in combinePredicateReduction()
44479 CmpC = DAG.getConstant(APInt::getLowBitsSet(CmpVT.getSizeInBits(), NumElts), in combinePredicateReduction()
44486 EVT SetccVT = TLI.getSetCCResultType(DAG.getDataLayout(), Ctx, CmpVT); in combinePredicateReduction()
44487 SDValue Setcc = DAG.getSetCC(DL, SetccVT, Movmsk, CmpC, CondCode); in combinePredicateReduction()
44488 SDValue Zext = DAG.getZExtOrTrunc(Setcc, DL, ExtractVT); in combinePredicateReduction()
44489 return DAG.getNegative(Zext, DL, ExtractVT); in combinePredicateReduction()
44492 static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG, in combineVPDPBUSDPattern() argument
44509 SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD}); in combineVPDPBUSDPattern()
44527 if (!detectExtMul(DAG, Root, LHS, RHS)) in combineVPDPBUSDPattern()
44533 SDValue DP = createVPDPBUSD(DAG, LHS, RHS, StageBias, DL, Subtarget); in combineVPDPBUSDPattern()
44549 DAG.getVectorShuffle(DpVT, DL, DP, DAG.getUNDEF(DpVT), Mask); in combineVPDPBUSDPattern()
44550 DP = DAG.getNode(ISD::ADD, DL, DpVT, DP, Shuffle); in combineVPDPBUSDPattern()
44556 EVT::getVectorVT(*DAG.getContext(), ExtractVT, in combineVPDPBUSDPattern()
44558 DP = DAG.getBitcast(ResVT, DP); in combineVPDPBUSDPattern()
44559 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, DP, in combineVPDPBUSDPattern()
44563 static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG, in combineBasicSADPattern() argument
44581 SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD}); in combineBasicSADPattern()
44608 SDValue SAD = createPSADBW(DAG, Zext0, Zext1, DL, Subtarget); in combineBasicSADPattern()
44623 DAG.getVectorShuffle(SadVT, DL, SAD, DAG.getUNDEF(SadVT), Mask); in combineBasicSADPattern()
44624 SAD = DAG.getNode(ISD::ADD, DL, SadVT, SAD, Shuffle); in combineBasicSADPattern()
44630 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), ExtractVT, in combineBasicSADPattern()
44632 SAD = DAG.getBitcast(ResVT, SAD); in combineBasicSADPattern()
44633 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, SAD, in combineBasicSADPattern()
44647 const SDLoc &dl, SelectionDAG &DAG, in combineExtractFromVectorLoad() argument
44652 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineExtractFromVectorLoad()
44667 DAG, LoadVec->getBasePtr(), VecVT, DAG.getVectorIdxConstant(Idx, dl)); in combineExtractFromVectorLoad()
44672 DAG.getLoad(VT, dl, LoadVec->getChain(), NewPtr, MPI, Alignment, in combineExtractFromVectorLoad()
44674 DAG.makeEquivalentMemoryOrdering(LoadVec, Load); in combineExtractFromVectorLoad()
44683 static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG, in combineExtractWithShuffle() argument
44719 SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, SrcVT.getScalarType()); in combineExtractWithShuffle()
44720 SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, VT); in combineExtractWithShuffle()
44733 SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(), in combineExtractWithShuffle()
44738 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1)); in combineExtractWithShuffle()
44756 Scl = DAG.getNode(ISD::SRL, dl, SclVT, Scl, in combineExtractWithShuffle()
44757 DAG.getShiftAmountConstant(Offset, SclVT, dl)); in combineExtractWithShuffle()
44759 Scl = DAG.getZExtOrTrunc(Scl, dl, SrcVT.getScalarType()); in combineExtractWithShuffle()
44760 Scl = DAG.getZExtOrTrunc(Scl, dl, VT); in combineExtractWithShuffle()
44770 Src = extract128BitVector(Src.getOperand(0), 0, DAG, dl); in combineExtractWithShuffle()
44772 return DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(ExtractVT, Src), in combineExtractWithShuffle()
44779 auto GetLegalExtract = [&Subtarget, &DAG, &dl](SDValue Vec, EVT VecVT, in combineExtractWithShuffle()
44789 VecVT = EVT::getVectorVT(*DAG.getContext(), VecSVT, NumEltsPerLane); in combineExtractWithShuffle()
44790 Vec = extract128BitVector(Vec, LaneIdx, DAG, dl); in combineExtractWithShuffle()
44795 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VecVT.getScalarType(), in combineExtractWithShuffle()
44796 DAG.getBitcast(VecVT, Vec), in combineExtractWithShuffle()
44797 DAG.getIntPtrConstant(Idx, dl)); in combineExtractWithShuffle()
44802 return DAG.getNode(OpCode, dl, MVT::i32, DAG.getBitcast(VecVT, Vec), in combineExtractWithShuffle()
44803 DAG.getTargetConstant(Idx, dl, MVT::i8)); in combineExtractWithShuffle()
44811 if (!getTargetShuffleInputs(SrcBC, Ops, Mask, DAG)) in combineExtractWithShuffle()
44858 EVT ExtractSVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltBits / Scale); in combineExtractWithShuffle()
44859 ExtractVT = EVT::getVectorVT(*DAG.getContext(), ExtractSVT, Mask.size()); in combineExtractWithShuffle()
44866 return DAG.getUNDEF(VT); in combineExtractWithShuffle()
44869 return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, dl, VT) in combineExtractWithShuffle()
44870 : DAG.getConstant(0, dl, VT); in combineExtractWithShuffle()
44875 return DAG.getZExtOrTrunc(V, dl, VT); in combineExtractWithShuffle()
44879 N, SrcVT, peekThroughBitcasts(SrcOp), ExtractIdx, dl, DAG, DCI)) in combineExtractWithShuffle()
44887 static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG, in scalarizeExtEltFP() argument
44909 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, in scalarizeExtEltFP()
44911 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, in scalarizeExtEltFP()
44913 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1, Vec.getOperand(2)); in scalarizeExtEltFP()
44932 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, in scalarizeExtEltFP()
44935 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in scalarizeExtEltFP()
44937 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in scalarizeExtEltFP()
44939 return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2); in scalarizeExtEltFP()
44977 ExtOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, Index)); in scalarizeExtEltFP()
44978 return DAG.getNode(Vec.getOpcode(), DL, VT, ExtOps); in scalarizeExtEltFP()
44988 static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG, in combineArithReduction() argument
44997 SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc, in combineArithReduction()
45019 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32, in combineArithReduction()
45020 DAG.getConstant(0, DL, MVT::v4i32), in combineArithReduction()
45021 DAG.getBitcast(MVT::i32, V), in combineArithReduction()
45022 DAG.getIntPtrConstant(0, DL)); in combineArithReduction()
45023 return DAG.getBitcast(MVT::v16i8, V); in combineArithReduction()
45025 V = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, V, in combineArithReduction()
45026 ZeroExtend ? DAG.getConstant(0, DL, MVT::v4i8) in combineArithReduction()
45027 : DAG.getUNDEF(MVT::v4i8)); in combineArithReduction()
45029 return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V, in combineArithReduction()
45030 DAG.getUNDEF(MVT::v8i8)); in combineArithReduction()
45038 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2); in combineArithReduction()
45039 SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT)); in combineArithReduction()
45040 SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT)); in combineArithReduction()
45041 Lo = DAG.getBitcast(WideVT, Lo); in combineArithReduction()
45042 Hi = DAG.getBitcast(WideVT, Hi); in combineArithReduction()
45043 Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi); in combineArithReduction()
45045 std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL); in combineArithReduction()
45046 Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi); in combineArithReduction()
45050 Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8)); in combineArithReduction()
45051 Rdx = DAG.getBitcast(MVT::v8i16, Rdx); in combineArithReduction()
45054 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx, in combineArithReduction()
45055 DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx, in combineArithReduction()
45057 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx, in combineArithReduction()
45058 DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx, in combineArithReduction()
45060 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx, in combineArithReduction()
45061 DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx, in combineArithReduction()
45063 Rdx = DAG.getBitcast(MVT::v16i8, Rdx); in combineArithReduction()
45064 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
45070 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx, in combineArithReduction()
45071 DAG.getConstant(0, DL, MVT::v16i8)); in combineArithReduction()
45072 Rdx = DAG.getBitcast(MVT::v16i8, Rdx); in combineArithReduction()
45073 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
45084 std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL); in combineArithReduction()
45086 Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi); in combineArithReduction()
45090 SDValue Hi = DAG.getVectorShuffle( in combineArithReduction()
45093 Rdx = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Rdx, Hi); in combineArithReduction()
45094 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx, in combineArithReduction()
45095 getZeroVector(MVT::v16i8, Subtarget, DAG, DL)); in combineArithReduction()
45096 Rdx = DAG.getBitcast(MVT::v16i8, Rdx); in combineArithReduction()
45097 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
45105 DAG.computeKnownBits(Rdx).getMaxValue().ule(255) && in combineArithReduction()
45109 Rdx = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Rdx, in combineArithReduction()
45110 DAG.getUNDEF(MVT::v8i16)); in combineArithReduction()
45113 Rdx = DAG.getNode(ISD::TRUNCATE, DL, ByteVT, Rdx); in combineArithReduction()
45119 auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL, in combineArithReduction()
45122 SDValue Zero = DAG.getConstant(0, DL, Ops[0].getValueType()); in combineArithReduction()
45123 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops[0], Zero); in combineArithReduction()
45126 Rdx = SplitOpsAndApply(DAG, Subtarget, DL, SadVT, {Rdx}, PSADBWBuilder); in combineArithReduction()
45131 std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL); in combineArithReduction()
45133 Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi); in combineArithReduction()
45138 SDValue RdxHi = DAG.getVectorShuffle(MVT::v2i64, DL, Rdx, Rdx, {1, -1}); in combineArithReduction()
45139 Rdx = DAG.getNode(ISD::ADD, DL, MVT::v2i64, Rdx, RdxHi); in combineArithReduction()
45143 Rdx = DAG.getBitcast(VecVT, Rdx); in combineArithReduction()
45144 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
45148 if (!shouldUseHorizontalOp(true, DAG, Subtarget)) in combineArithReduction()
45160 SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL); in combineArithReduction()
45161 SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL); in combineArithReduction()
45162 Rdx = DAG.getNode(HorizOpcode, DL, Lo.getValueType(), Hi, Lo); in combineArithReduction()
45172 Rdx = DAG.getNode(HorizOpcode, DL, VecVT, Rdx, Rdx); in combineArithReduction()
45174 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
45181 static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG, in combineExtractVectorElt() argument
45184 if (SDValue NewOp = combineExtractWithShuffle(N, DAG, DCI, Subtarget)) in combineExtractVectorElt()
45197 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineExtractVectorElt()
45200 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT); in combineExtractVectorElt()
45212 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT); in combineExtractVectorElt()
45213 return DAG.getConstant(EltBits[Idx].zext(NumEltBits), dl, VT); in combineExtractVectorElt()
45223 SDValue Sub = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Src, in combineExtractVectorElt()
45224 DAG.getIntPtrConstant(CIdx->getZExtValue() * NumEltBits, dl)); in combineExtractVectorElt()
45225 return DAG.getBitcast(VT, Sub); in combineExtractVectorElt()
45242 Scl = DAG.getNode(ISD::TRUNCATE, dl, SrcVT.getScalarType(), Scl); in combineExtractVectorElt()
45243 return DAG.getZExtOrTrunc(Scl, dl, VT); in combineExtractVectorElt()
45257 return DAG.getBitcast(VT, InputVector); in combineExtractVectorElt()
45264 return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32, in combineExtractVectorElt()
45270 if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget)) in combineExtractVectorElt()
45273 if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget)) in combineExtractVectorElt()
45277 if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget)) in combineExtractVectorElt()
45281 if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget)) in combineExtractVectorElt()
45285 if (SDValue V = combineArithReduction(N, DAG, Subtarget)) in combineExtractVectorElt()
45288 if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget)) in combineExtractVectorElt()
45294 dl, DAG, DCI)) in combineExtractVectorElt()
45324 EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcElts); in combineExtractVectorElt()
45326 combineBitcastvxi1(DAG, BCVT, InputVector, dl, Subtarget)) { in combineExtractVectorElt()
45330 SDValue MaskIdx = DAG.getZExtOrTrunc(Use->getOperand(1), dl, MVT::i8); in combineExtractVectorElt()
45331 SDValue MaskBit = DAG.getConstant(1, dl, BCVT); in combineExtractVectorElt()
45332 SDValue Mask = DAG.getNode(ISD::SHL, dl, BCVT, MaskBit, MaskIdx); in combineExtractVectorElt()
45333 SDValue Res = DAG.getNode(ISD::AND, dl, BCVT, BC, Mask); in combineExtractVectorElt()
45334 Res = DAG.getSetCC(dl, MVT::i1, Res, Mask, ISD::SETEQ); in combineExtractVectorElt()
45348 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TruncSVT, TruncSrc, EltIdx); in combineExtractVectorElt()
45349 return DAG.getAnyExtOrTrunc(NewExt, dl, VT); in combineExtractVectorElt()
45359 unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG, in combineToExtendBoolVectorInReg() argument
45400 EVT BroadcastVT = EVT::getVectorVT(*DAG.getContext(), SclVT, EltSizeInBits); in combineToExtendBoolVectorInReg()
45401 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00); in combineToExtendBoolVectorInReg()
45402 Vec = DAG.getBitcast(VT, Vec); in combineToExtendBoolVectorInReg()
45406 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask); in combineToExtendBoolVectorInReg()
45416 EVT::getVectorVT(*DAG.getContext(), SclVT, NumElts * Scale); in combineToExtendBoolVectorInReg()
45417 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00); in combineToExtendBoolVectorInReg()
45419 Vec = DAG.getVectorShuffle(BroadcastVT, DL, Vec, Vec, ShuffleMask); in combineToExtendBoolVectorInReg()
45420 Vec = DAG.getBitcast(VT, Vec); in combineToExtendBoolVectorInReg()
45425 SDValue Scl = DAG.getAnyExtOrTrunc(N00, DL, SVT); in combineToExtendBoolVectorInReg()
45426 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl); in combineToExtendBoolVectorInReg()
45428 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask); in combineToExtendBoolVectorInReg()
45436 Bits.push_back(DAG.getConstant(Bit, DL, SVT)); in combineToExtendBoolVectorInReg()
45438 SDValue BitMask = DAG.getBuildVector(VT, DL, Bits); in combineToExtendBoolVectorInReg()
45439 Vec = DAG.getNode(ISD::AND, DL, VT, Vec, BitMask); in combineToExtendBoolVectorInReg()
45442 EVT CCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts); in combineToExtendBoolVectorInReg()
45443 Vec = DAG.getSetCC(DL, CCVT, Vec, BitMask, ISD::SETEQ); in combineToExtendBoolVectorInReg()
45444 Vec = DAG.getSExtOrTrunc(Vec, DL, VT); in combineToExtendBoolVectorInReg()
45450 return DAG.getNode(ISD::SRL, DL, VT, Vec, in combineToExtendBoolVectorInReg()
45451 DAG.getConstant(EltSizeInBits - 1, DL, VT)); in combineToExtendBoolVectorInReg()
45458 combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, in combineVSelectWithAllOnesOrZeros() argument
45466 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineVSelectWithAllOnesOrZeros()
45483 return DAG.getConstantFP(0.0, DL, VT); in combineVSelectWithAllOnesOrZeros()
45484 return DAG.getConstant(0, DL, VT); in combineVSelectWithAllOnesOrZeros()
45502 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) == in combineVSelectWithAllOnesOrZeros()
45510 Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1), in combineVSelectWithAllOnesOrZeros()
45519 if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits()) in combineVSelectWithAllOnesOrZeros()
45524 return DAG.getBitcast(VT, Cond); in combineVSelectWithAllOnesOrZeros()
45531 SDValue CastRHS = DAG.getBitcast(CondVT, RHS); in combineVSelectWithAllOnesOrZeros()
45532 SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS); in combineVSelectWithAllOnesOrZeros()
45533 return DAG.getBitcast(VT, Or); in combineVSelectWithAllOnesOrZeros()
45538 SDValue CastLHS = DAG.getBitcast(CondVT, LHS); in combineVSelectWithAllOnesOrZeros()
45539 SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS); in combineVSelectWithAllOnesOrZeros()
45540 return DAG.getBitcast(VT, And); in combineVSelectWithAllOnesOrZeros()
45545 SDValue CastRHS = DAG.getBitcast(CondVT, RHS); in combineVSelectWithAllOnesOrZeros()
45549 AndN = DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT), in combineVSelectWithAllOnesOrZeros()
45552 AndN = DAG.getNode(X86ISD::ANDNP, DL, CondVT, Cond, CastRHS); in combineVSelectWithAllOnesOrZeros()
45553 return DAG.getBitcast(VT, AndN); in combineVSelectWithAllOnesOrZeros()
45563 static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, in narrowVectorSelect() argument
45579 !isFreeToSplitVector(TVal.getNode(), DAG) || in narrowVectorSelect()
45580 !isFreeToSplitVector(FVal.getNode(), DAG)) in narrowVectorSelect()
45583 auto makeBlend = [Opcode](SelectionDAG &DAG, const SDLoc &DL, in narrowVectorSelect()
45585 return DAG.getNode(Opcode, DL, Ops[1].getValueType(), Ops); in narrowVectorSelect()
45587 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {Cond, TVal, FVal}, makeBlend, in narrowVectorSelect()
45591 static SDValue combineSelectOfTwoConstants(SDNode *N, SelectionDAG &DAG, in combineSelectOfTwoConstants() argument
45604 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in combineSelectOfTwoConstants()
45642 Cond = DAG.getNOT(DL, Cond, MVT::i1); in combineSelectOfTwoConstants()
45647 SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in combineSelectOfTwoConstants()
45651 R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT)); in combineSelectOfTwoConstants()
45655 R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0)); in combineSelectOfTwoConstants()
45668 static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG, in combineVSelectToBLENDV() argument
45678 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineVSelectToBLENDV()
45729 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), in combineVSelectToBLENDV()
45743 SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0), in combineVSelectToBLENDV()
45745 DAG.ReplaceAllUsesOfValueWith(SDValue(U, 0), SB); in combineVSelectToBLENDV()
45753 if (SDValue V = TLI.SimplifyMultipleUseDemandedBits(Cond, DemandedBits, DAG)) in combineVSelectToBLENDV()
45754 return DAG.getNode(X86ISD::BLENDV, DL, N->getValueType(0), V, in combineVSelectToBLENDV()
45778 SelectionDAG &DAG, const X86Subtarget &Subtarget) { in combineLogicBlendIntoConditionalNegate() argument
45781 DAG.ComputeNumSignBits(Mask) == MaskVT.getScalarSizeInBits() && in combineLogicBlendIntoConditionalNegate()
45786 if (!DAG.getTargetLoweringInfo().isOperationLegal(ISD::SUB, MaskVT)) in combineLogicBlendIntoConditionalNegate()
45802 SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask); in combineLogicBlendIntoConditionalNegate()
45818 SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2); in combineLogicBlendIntoConditionalNegate()
45819 return DAG.getBitcast(VT, Res); in combineLogicBlendIntoConditionalNegate()
45822 static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, in commuteSelect() argument
45847 Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), Cond.getOperand(0), in commuteSelect()
45849 return DAG.getSelect(DL, LHS.getValueType(), Cond, RHS, LHS); in commuteSelect()
45853 static SDValue combineSelect(SDNode *N, SelectionDAG &DAG, in combineSelect() argument
45863 if (SDValue V = DAG.simplifySelect(Cond, LHS, RHS)) in combineSelect()
45870 if (SDValue V = commuteSelect(N, DAG, DL, Subtarget)) in combineSelect()
45875 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineSelect()
45884 DAG.ComputeNumSignBits(Cond) == CondVT.getScalarSizeInBits()) in combineSelect()
45886 DL, DAG, Subtarget)) in combineSelect()
45895 return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask); in combineSelect()
45922 LHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, LHS.getOperand(0), in combineSelect()
45923 getConstVector(LHSMask, SimpleVT, DAG, DL, true)); in combineSelect()
45924 RHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, RHS.getOperand(0), in combineSelect()
45925 getConstVector(RHSMask, SimpleVT, DAG, DL, true)); in combineSelect()
45926 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in combineSelect()
45944 if (DAG.isEqualTo(LHS, Cond.getOperand(0)) && in combineSelect()
45945 DAG.isEqualTo(RHS, Cond.getOperand(1))) { in combineSelect()
45952 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) { in combineSelect()
45953 if (!DAG.getTarget().Options.NoSignedZerosFPMath && in combineSelect()
45954 !(DAG.isKnownNeverZeroFloat(LHS) || in combineSelect()
45955 DAG.isKnownNeverZeroFloat(RHS))) in combineSelect()
45964 if (!DAG.getTarget().Options.NoSignedZerosFPMath && in combineSelect()
45965 !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS)) in combineSelect()
45983 if (!DAG.getTarget().Options.NoSignedZerosFPMath && in combineSelect()
45984 !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS)) in combineSelect()
45992 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) { in combineSelect()
45993 if (!DAG.getTarget().Options.NoSignedZerosFPMath && in combineSelect()
45994 !(DAG.isKnownNeverZeroFloat(LHS) || in combineSelect()
45995 DAG.isKnownNeverZeroFloat(RHS))) in combineSelect()
46013 } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) && in combineSelect()
46014 DAG.isEqualTo(RHS, Cond.getOperand(0))) { in combineSelect()
46021 if (!DAG.getTarget().Options.NoSignedZerosFPMath && in combineSelect()
46022 !(DAG.isKnownNeverZeroFloat(LHS) || in combineSelect()
46023 DAG.isKnownNeverZeroFloat(RHS))) { in combineSelect()
46024 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) in combineSelect()
46032 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) in combineSelect()
46049 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) in combineSelect()
46057 if (!DAG.getTarget().Options.NoSignedZerosFPMath && in combineSelect()
46058 !DAG.isKnownNeverZeroFloat(LHS) && in combineSelect()
46059 !DAG.isKnownNeverZeroFloat(RHS)) { in combineSelect()
46060 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) in combineSelect()
46080 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS); in combineSelect()
46097 AndNode = DAG.getZExtOrTrunc(AndNode, DL, MVT::i8); in combineSelect()
46098 return DAG.getNode(ISD::SELECT, DL, VT, AndNode, RHS, LHS); in combineSelect()
46113 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in combineSelect()
46114 return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS); in combineSelect()
46140 LHS = insertSubVector(DAG.getUNDEF(SrcVT), LHS, 0, DAG, DL, in combineSelect()
46142 RHS = insertSubVector(DAG.getUNDEF(SrcVT), RHS, 0, DAG, DL, in combineSelect()
46144 Cond = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcCondVT, in combineSelect()
46145 DAG.getUNDEF(SrcCondVT), Cond, in combineSelect()
46146 DAG.getIntPtrConstant(0, DL)); in combineSelect()
46147 SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS); in combineSelect()
46148 return extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits()); in combineSelect()
46152 if (SDValue V = combineSelectOfTwoConstants(N, DAG, DL)) in combineSelect()
46185 Cond = DAG.getSetCC(SDLoc(Cond), CondVT, Cond0, Cond1, NewCC); in combineSelect()
46186 return DAG.getSelect(DL, VT, Cond, LHS, RHS); in combineSelect()
46190 Cond = DAG.getSetCC(SDLoc(Cond), CondVT, Cond0, Cond1, NewCC); in combineSelect()
46191 return DAG.getSelect(DL, VT, Cond, LHS, RHS); in combineSelect()
46221 Cond = DAG.getSetCC(DL, CondVT, Cond0, Cond1, NewCC); in combineSelect()
46222 return DAG.getSelect(DL, VT, Cond, LHS, RHS.getOperand(2)); in combineSelect()
46236 SDValue CondNew = DAG.getNOT(DL, Cond, CondVT); in combineSelect()
46238 return DAG.getSelect(DL, VT, CondNew, RHS, LHS); in combineSelect()
46248 ISD::SIGN_EXTEND, DL, ExtCondVT, Cond, DAG, DCI, Subtarget)) { in combineSelect()
46249 ExtCond = DAG.getNode(ISD::TRUNCATE, DL, CondVT, ExtCond); in combineSelect()
46250 return DAG.getSelect(DL, VT, ExtCond, LHS, RHS); in combineSelect()
46272 return DAG.getNode(LHS.getOpcode() == ISD::SRL ? X86ISD::VSRLV in combineSelect()
46284 return DAG.getNode(RHS.getOpcode() == ISD::SRL ? X86ISD::VSRLV in combineSelect()
46294 if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DL, DCI, Subtarget)) in combineSelect()
46297 if (SDValue V = combineVSelectToBLENDV(N, DAG, DL, DCI, Subtarget)) in combineSelect()
46300 if (SDValue V = narrowVectorSelect(N, DAG, DL, Subtarget)) in combineSelect()
46305 if (SDValue CondNot = IsNOT(Cond, DAG)) in combineSelect()
46306 return DAG.getNode(N->getOpcode(), DL, VT, in combineSelect()
46307 DAG.getBitcast(CondVT, CondNot), RHS, LHS); in combineSelect()
46314 Cond = DAG.getNode(X86ISD::PCMPGT, DL, CondVT, in combineSelect()
46315 DAG.getConstant(0, DL, CondVT), Cond.getOperand(0)); in combineSelect()
46316 return DAG.getNode(N->getOpcode(), DL, VT, Cond, RHS, LHS); in combineSelect()
46328 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements()); in combineSelect()
46338 LHS = combinevXi1ConstantToInteger(LHS, DAG); in combineSelect()
46343 RHS = combinevXi1ConstantToInteger(RHS, DAG); in combineSelect()
46347 SDValue Select = DAG.getSelect(DL, IntVT, Cond, LHS, RHS); in combineSelect()
46348 return DAG.getBitcast(VT, Select); in combineSelect()
46369 DAG.getSetCC(DL, CondVT, And, Cond.getOperand(1), ISD::SETNE); in combineSelect()
46370 return DAG.getSelect(DL, VT, NotCond, RHS, LHS); in combineSelect()
46394 SDValue ShlAmt = getConstVector(ShlVals, VT.getSimpleVT(), DAG, DL); in combineSelect()
46395 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt); in combineSelect()
46397 DAG.getSetCC(DL, CondVT, Shl, Cond.getOperand(1), ISD::SETLT); in combineSelect()
46398 return DAG.getSelect(DL, VT, NewCond, RHS, LHS); in combineSelect()
46412 SelectionDAG &DAG, in combineSetCCAtomicArith() argument
46495 auto AtomicSub = DAG.getAtomic( in combineSetCCAtomicArith()
46498 /*RHS*/ DAG.getConstant(NegAddend, SDLoc(CmpRHS), CmpVT), in combineSetCCAtomicArith()
46500 auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget); in combineSetCCAtomicArith()
46501 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT)); in combineSetCCAtomicArith()
46502 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1)); in combineSetCCAtomicArith()
46522 SDValue LockOp = lowerAtomicArithWithLOCK(CmpLHS, DAG, Subtarget); in combineSetCCAtomicArith()
46523 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT)); in combineSetCCAtomicArith()
46524 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1)); in combineSetCCAtomicArith()
46531 SelectionDAG &DAG) { in checkSignTestSetCCCombine() argument
46553 if (DAG.SignBitIsZero(Cmp.getOperand(0))) in checkSignTestSetCCCombine()
46555 else if (DAG.SignBitIsZero(Cmp.getOperand(1))) in checkSignTestSetCCCombine()
46571 if (std::optional<uint64_t> ShiftAmt = DAG.getValidShiftAmount(Src)) { in checkSignTestSetCCCombine()
46577 SDValue Mask = DAG.getNode(ISD::AND, DL, SrcVT, Src, in checkSignTestSetCCCombine()
46578 DAG.getConstant(BitMask, DL, SrcVT)); in checkSignTestSetCCCombine()
46580 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Mask, in checkSignTestSetCCCombine()
46581 DAG.getConstant(0, DL, SrcVT)); in checkSignTestSetCCCombine()
46758 static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) { in combineCarryThroughADD() argument
46789 DAG.getNode(X86ISD::SUB, SDLoc(CarryOp1), CarryOp1->getVTList(), in combineCarryThroughADD()
46802 SDValue BitNo = DAG.getConstant(0, DL, Carry.getValueType()); in combineCarryThroughADD()
46807 return getBT(Carry, BitNo, DL, DAG); in combineCarryThroughADD()
46818 SelectionDAG &DAG, in combinePTESTCC() argument
46833 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combinePTESTCC()
46836 if (SDValue NotOp0 = IsNOT(Op0, DAG)) { in combinePTESTCC()
46867 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
46868 DAG.getBitcast(OpVT, NotOp0), Op1); in combinePTESTCC()
46874 if (SDValue NotOp1 = IsNOT(Op1, DAG)) { in combinePTESTCC()
46877 return DAG.getNode( in combinePTESTCC()
46878 EFLAGS.getOpcode(), DL, VT, DAG.getBitcast(OpVT, NotOp1), in combinePTESTCC()
46879 DAG.getBitcast(OpVT, in combinePTESTCC()
46880 DAG.getAllOnesConstant(DL, NotOp1.getValueType()))); in combinePTESTCC()
46887 if (SDValue NotOp1 = IsNOT(Op1, DAG)) { in combinePTESTCC()
46889 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
46890 DAG.getBitcast(OpVT, NotOp1), Op0); in combinePTESTCC()
46899 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
46900 DAG.getBitcast(OpVT, BC.getOperand(0)), in combinePTESTCC()
46901 DAG.getBitcast(OpVT, BC.getOperand(1))); in combinePTESTCC()
46907 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
46908 DAG.getBitcast(OpVT, BC.getOperand(0)), in combinePTESTCC()
46909 DAG.getBitcast(OpVT, BC.getOperand(1))); in combinePTESTCC()
46919 if (DAG.ComputeNumSignBits(BC) == EltBits) { in combinePTESTCC()
46923 TLI.SimplifyMultipleUseDemandedBits(BC, SignMask, DAG)) { in combinePTESTCC()
46931 Res = DAG.getBitcast(FloatVT, Res); in combinePTESTCC()
46932 return DAG.getNode(X86ISD::TESTP, SDLoc(EFLAGS), VT, Res, Res); in combinePTESTCC()
46935 Res = DAG.getBitcast(MovmskVT, Res); in combinePTESTCC()
46936 Res = getPMOVMSKB(DL, Res, DAG, Subtarget); in combinePTESTCC()
46937 Res = DAG.getNode(ISD::AND, DL, MVT::i32, Res, in combinePTESTCC()
46938 DAG.getConstant(0xAAAAAAAA, DL, MVT::i32)); in combinePTESTCC()
46940 Res = getPMOVMSKB(DL, Res, DAG, Subtarget); in combinePTESTCC()
46942 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Res, in combinePTESTCC()
46943 DAG.getConstant(0, DL, MVT::i32)); in combinePTESTCC()
46951 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op1, Op1); in combinePTESTCC()
46955 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op0, Op0); in combinePTESTCC()
46969 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
46970 DAG.getBitcast(OpVT2, Src0), in combinePTESTCC()
46971 DAG.getBitcast(OpVT2, Src1)); in combinePTESTCC()
46982 SelectionDAG &DAG, in combineSetCCMOVMSK() argument
47044 DAG.ComputeNumSignBits(BC) > (BCNumEltBits - NumEltBits)) { in combineSetCCMOVMSK()
47047 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, in combineSetCCMOVMSK()
47048 DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, BC), in combineSetCCMOVMSK()
47049 DAG.getConstant(CmpMask, DL, MVT::i32)); in combineSetCCMOVMSK()
47059 if (collectConcatOps(peekThroughBitcasts(Vec).getNode(), Ops, DAG) && in combineSetCCMOVMSK()
47064 SDValue V = DAG.getNode(IsAnyOf ? ISD::OR : ISD::AND, DL, SubVT, in combineSetCCMOVMSK()
47065 DAG.getBitcast(SubVT, Ops[0]), in combineSetCCMOVMSK()
47066 DAG.getBitcast(SubVT, Ops[1])); in combineSetCCMOVMSK()
47067 V = DAG.getBitcast(VecVT.getHalfNumVectorElementsVT(), V); in combineSetCCMOVMSK()
47068 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, in combineSetCCMOVMSK()
47069 DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V), in combineSetCCMOVMSK()
47070 DAG.getConstant(CmpMask, DL, MVT::i32)); in combineSetCCMOVMSK()
47084 SDValue V = DAG.getNode(ISD::XOR, SDLoc(BC), BC.getValueType(), in combineSetCCMOVMSK()
47086 V = DAG.getBitcast(TestVT, V); in combineSetCCMOVMSK()
47087 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V); in combineSetCCMOVMSK()
47095 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), LHS.getValueType(), in combineSetCCMOVMSK()
47097 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), RHS.getValueType(), in combineSetCCMOVMSK()
47099 LHS = DAG.getBitcast(TestVT, LHS); in combineSetCCMOVMSK()
47100 RHS = DAG.getBitcast(TestVT, RHS); in combineSetCCMOVMSK()
47101 SDValue V = DAG.getNode(ISD::OR, SDLoc(EFLAGS), TestVT, LHS, RHS); in combineSetCCMOVMSK()
47102 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V); in combineSetCCMOVMSK()
47115 bool SignExt0 = DAG.ComputeNumSignBits(VecOp0) > 8; in combineSetCCMOVMSK()
47116 bool SignExt1 = DAG.ComputeNumSignBits(VecOp1) > 8; in combineSetCCMOVMSK()
47120 SDValue Result = DAG.getBitcast(MVT::v16i8, VecOp0); in combineSetCCMOVMSK()
47121 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result); in combineSetCCMOVMSK()
47122 Result = DAG.getZExtOrTrunc(Result, DL, MVT::i16); in combineSetCCMOVMSK()
47124 Result = DAG.getNode(ISD::AND, DL, MVT::i16, Result, in combineSetCCMOVMSK()
47125 DAG.getConstant(0xAAAA, DL, MVT::i16)); in combineSetCCMOVMSK()
47127 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result, in combineSetCCMOVMSK()
47128 DAG.getConstant(0, DL, MVT::i16)); in combineSetCCMOVMSK()
47139 SDValue V = DAG.getNode(ISD::XOR, DL, Result.getValueType(), in combineSetCCMOVMSK()
47141 V = DAG.getBitcast(MVT::v4i64, V); in combineSetCCMOVMSK()
47142 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V); in combineSetCCMOVMSK()
47144 Result = DAG.getBitcast(MVT::v32i8, Result); in combineSetCCMOVMSK()
47145 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result); in combineSetCCMOVMSK()
47150 Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result, in combineSetCCMOVMSK()
47151 DAG.getConstant(0xAAAAAAAA, DL, MVT::i32)); in combineSetCCMOVMSK()
47153 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result, in combineSetCCMOVMSK()
47154 DAG.getConstant(CmpMask, DL, MVT::i32)); in combineSetCCMOVMSK()
47178 ShuffleMask, DAG) && in combineSetCCMOVMSK()
47183 SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]); in combineSetCCMOVMSK()
47184 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result); in combineSetCCMOVMSK()
47186 DAG.getZExtOrTrunc(Result, DL, EFLAGS.getOperand(0).getValueType()); in combineSetCCMOVMSK()
47187 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result, EFLAGS.getOperand(1)); in combineSetCCMOVMSK()
47203 SDValue RHS = IsAnyOf ? Vec : DAG.getAllOnesConstant(DL, IntVT); in combineSetCCMOVMSK()
47205 return DAG.getNode(X86ISD::TESTP, DL, MVT::i32, in combineSetCCMOVMSK()
47206 DAG.getBitcast(FloatVT, LHS), in combineSetCCMOVMSK()
47207 DAG.getBitcast(FloatVT, RHS)); in combineSetCCMOVMSK()
47217 SelectionDAG &DAG, in combineSetCCEFLAGS() argument
47220 if (SDValue Flags = combineCarryThroughADD(EFLAGS, DAG)) in combineSetCCEFLAGS()
47223 if (SDValue R = checkSignTestSetCCCombine(EFLAGS, CC, DAG)) in combineSetCCEFLAGS()
47229 if (SDValue R = combinePTESTCC(EFLAGS, CC, DAG, Subtarget)) in combineSetCCEFLAGS()
47232 if (SDValue R = combineSetCCMOVMSK(EFLAGS, CC, DAG, Subtarget)) in combineSetCCEFLAGS()
47235 return combineSetCCAtomicArith(EFLAGS, CC, DAG, Subtarget); in combineSetCCEFLAGS()
47239 static SDValue combineCMov(SDNode *N, SelectionDAG &DAG, in combineCMov() argument
47255 if (SDValue Flags = combineSetCCEFLAGS(Cond, CC, DAG, Subtarget)) { in combineCMov()
47260 SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8), in combineCMov()
47262 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops); in combineCMov()
47283 Cond = getSETCC(CC, Cond, DL, DAG); in combineCMov()
47286 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond); in combineCMov()
47289 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond, in combineCMov()
47290 DAG.getConstant(ShAmt, DL, MVT::i8)); in combineCMov()
47297 Cond = getSETCC(CC, Cond, DL, DAG); in combineCMov()
47300 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, in combineCMov()
47302 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in combineCMov()
47331 Cond = getSETCC(CC, Cond, DL ,DAG); in combineCMov()
47333 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), in combineCMov()
47337 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, in combineCMov()
47338 DAG.getConstant(Diff, DL, Cond.getValueType())); in combineCMov()
47342 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in combineCMov()
47382 DAG.getTargetConstant(CC, DL, MVT::i8), Cond}; in combineCMov()
47383 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops); in combineCMov()
47406 DAG.getNode(X86ISD::SUB, DL, Cond->getVTList(), Cond.getOperand(0), in combineCMov()
47407 DAG.getConstant(1, DL, CondVT)); in combineCMov()
47409 return DAG.getNode(X86ISD::ADC, DL, DAG.getVTList(OuterVT, MVT::i32), in combineCMov()
47410 TrueOp, DAG.getConstant(0, DL, OuterVT), EFLAGS); in combineCMov()
47443 DAG.getTargetConstant(CC0, DL, MVT::i8), Flags}; in combineCMov()
47444 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps); in combineCMov()
47445 SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8), in combineCMov()
47447 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops); in combineCMov()
47477 SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1)); in combineCMov()
47479 DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0), in combineCMov()
47480 DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond); in combineCMov()
47481 return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1)); in combineCMov()
47491 static bool canReduceVMulWidth(SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode) { in canReduceVMulWidth() argument
47502 SignBits[i] = DAG.ComputeNumSignBits(Opd); in canReduceVMulWidth()
47503 IsPositive[i] = DAG.SignBitIsZero(Opd); in canReduceVMulWidth()
47553 static SDValue reduceVMULWidth(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, in reduceVMULWidth() argument
47564 bool OptForMinSize = DAG.getMachineFunction().getFunction().hasMinSize(); in reduceVMULWidth()
47569 if (!canReduceVMulWidth(N, DAG, Mode)) in reduceVMULWidth()
47579 EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts); in reduceVMULWidth()
47582 SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0); in reduceVMULWidth()
47583 SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1); in reduceVMULWidth()
47587 SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1); in reduceVMULWidth()
47589 return DAG.getNode((Mode == ShrinkMode::MULU8) ? ISD::ZERO_EXTEND in reduceVMULWidth()
47593 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts / 2); in reduceVMULWidth()
47597 DAG.getNode(Mode == ShrinkMode::MULS16 ? ISD::MULHS : ISD::MULHU, DL, in reduceVMULWidth()
47609 DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask); in reduceVMULWidth()
47610 ResLo = DAG.getBitcast(ResVT, ResLo); in reduceVMULWidth()
47617 DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask); in reduceVMULWidth()
47618 ResHi = DAG.getBitcast(ResVT, ResHi); in reduceVMULWidth()
47619 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi); in reduceVMULWidth()
47622 static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG, in combineMulSpecial() argument
47626 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMulSpecial()
47627 DAG.getConstant(Mult, DL, VT)); in combineMulSpecial()
47628 Result = DAG.getNode(ISD::SHL, DL, VT, Result, in combineMulSpecial()
47629 DAG.getConstant(Shift, DL, MVT::i8)); in combineMulSpecial()
47630 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result, in combineMulSpecial()
47636 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMulSpecial()
47637 DAG.getConstant(Mul1, DL, VT)); in combineMulSpecial()
47638 Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result, in combineMulSpecial()
47639 DAG.getConstant(Mul2, DL, VT)); in combineMulSpecial()
47640 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result, in combineMulSpecial()
47659 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), in combineMulSpecial()
47684 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), in combineMulSpecial()
47697 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMulSpecial()
47698 DAG.getConstant(ShiftAmt, DL, MVT::i8)); in combineMulSpecial()
47699 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMulSpecial()
47700 DAG.getConstant(ScaleShift, DL, MVT::i8)); in combineMulSpecial()
47701 return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2); in combineMulSpecial()
47712 SelectionDAG &DAG, in combineMulToPMADDWD() argument
47762 if (DAG.ComputeMaxSignificantBits(N1) > 16 || in combineMulToPMADDWD()
47763 DAG.ComputeMaxSignificantBits(N0) > 16) in combineMulToPMADDWD()
47770 if (DAG.MaskedValueIsZero(Op, Mask17)) in combineMulToPMADDWD()
47774 return DAG.getNode(ISD::AND, DL, VT, Op, DAG.getConstant(0xFFFF, DL, VT)); in combineMulToPMADDWD()
47779 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Src); in combineMulToPMADDWD()
47784 Src = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, Src); in combineMulToPMADDWD()
47785 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Src); in combineMulToPMADDWD()
47793 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Src); in combineMulToPMADDWD()
47798 return DAG.getNode(X86ISD::VSRLI, DL, VT, Op.getOperand(0), in combineMulToPMADDWD()
47811 auto PMADDWDBuilder = [](SelectionDAG &DAG, const SDLoc &DL, in combineMulToPMADDWD()
47815 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, in combineMulToPMADDWD()
47816 DAG.getBitcast(OpVT, Ops[0]), in combineMulToPMADDWD()
47817 DAG.getBitcast(OpVT, Ops[1])); in combineMulToPMADDWD()
47819 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMADDWDBuilder); in combineMulToPMADDWD()
47822 static SDValue combineMulToPMULDQ(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, in combineMulToPMULDQ() argument
47840 if (Subtarget.hasSSE41() && DAG.ComputeNumSignBits(N0) > 32 && in combineMulToPMULDQ()
47841 DAG.ComputeNumSignBits(N1) > 32) { in combineMulToPMULDQ()
47842 auto PMULDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL, in combineMulToPMULDQ()
47844 return DAG.getNode(X86ISD::PMULDQ, DL, Ops[0].getValueType(), Ops); in combineMulToPMULDQ()
47846 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMULDQBuilder, in combineMulToPMULDQ()
47852 if (DAG.MaskedValueIsZero(N0, Mask) && DAG.MaskedValueIsZero(N1, Mask)) { in combineMulToPMULDQ()
47853 auto PMULUDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL, in combineMulToPMULDQ() argument
47855 return DAG.getNode(X86ISD::PMULUDQ, DL, Ops[0].getValueType(), Ops); in combineMulToPMULDQ()
47857 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMULUDQBuilder, in combineMulToPMULDQ()
47864 static SDValue combineMul(SDNode *N, SelectionDAG &DAG, in combineMul() argument
47870 if (SDValue V = combineMulToPMADDWD(N, DL, DAG, Subtarget)) in combineMul()
47873 if (SDValue V = combineMulToPMULDQ(N, DL, DAG, Subtarget)) in combineMul()
47877 return reduceVMULWidth(N, DL, DAG, Subtarget); in combineMul()
47885 if (DAG.getMachineFunction().getFunction().hasMinSize()) in combineMul()
47921 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMul()
47922 DAG.getConstant(AbsMulAmt, DL, VT)); in combineMul()
47924 NewMul = DAG.getNegative(NewMul, DL, VT); in combineMul()
47956 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
47957 DAG.getConstant(Log2_64(MulAmt1), DL, MVT::i8)); in combineMul()
47959 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMul()
47960 DAG.getConstant(MulAmt1, DL, VT)); in combineMul()
47963 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul, in combineMul()
47964 DAG.getConstant(Log2_64(MulAmt2), DL, MVT::i8)); in combineMul()
47966 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul, in combineMul()
47967 DAG.getConstant(MulAmt2, DL, VT)); in combineMul()
47971 NewMul = DAG.getNegative(NewMul, DL, VT); in combineMul()
47973 NewMul = combineMulSpecial(C->getZExtValue(), N, DAG, VT, DL); in combineMul()
47983 NewMul = DAG.getNode( in combineMul()
47985 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
47986 DAG.getConstant(Log2_64(AbsMulAmt - 1), DL, ShiftVT))); in combineMul()
47988 NewMul = DAG.getNegative(NewMul, DL, VT); in combineMul()
47992 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
47993 DAG.getConstant(Log2_64(AbsMulAmt + 1), DL, ShiftVT)); in combineMul()
47996 NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul); in combineMul()
47998 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0)); in combineMul()
48003 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48004 DAG.getConstant(Log2_64(AbsMulAmt - 2), DL, ShiftVT)); in combineMul()
48005 NewMul = DAG.getNode( in combineMul()
48007 DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0))); in combineMul()
48012 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48013 DAG.getConstant(Log2_64(AbsMulAmt + 2), DL, ShiftVT)); in combineMul()
48014 NewMul = DAG.getNode( in combineMul()
48016 DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0))); in combineMul()
48032 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48033 DAG.getConstant(Log2_64(ShiftAmt1), DL, ShiftVT)); in combineMul()
48035 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48036 DAG.getConstant(Log2_64(AbsMulAmtLowBit), DL, ShiftVT)); in combineMul()
48037 NewMul = DAG.getNode(*Opc, DL, VT, Shift1, Shift2); in combineMul()
48052 static SDValue combineShiftToPMULH(SDNode *N, SelectionDAG &DAG, in combineShiftToPMULH() argument
48095 SDValue Mulh = DAG.getNode(Opc, DL, MulVT, LHS, RHS); in combineShiftToPMULH()
48098 return DAG.getNode(ExtOpc, DL, VT, Mulh); in combineShiftToPMULH()
48101 static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG, in combineShiftLeft() argument
48122 return DAG.getNode(X86ISD::VSHLV, DL, VT, N00, N1); in combineShiftLeft()
48128 return DAG.getNode(X86ISD::VSHLV, DL, VT, N01, N1); in combineShiftLeft()
48163 return DAG.getNode(ISD::AND, DL, VT, N00, DAG.getConstant(Mask, DL, VT)); in combineShiftLeft()
48169 static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG, in combineShiftRightArithmetic() argument
48178 if (SDValue V = combineShiftToPMULH(N, DAG, DL, Subtarget)) in combineShiftRightArithmetic()
48186 return DAG.getNode(X86ISD::VSRAV, DL, VT, N0, ShrAmtVal); in combineShiftRightArithmetic()
48225 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT)); in combineShiftRightArithmetic()
48229 return DAG.getNode(ISD::SHL, DL, VT, NN, in combineShiftRightArithmetic()
48230 DAG.getConstant(ShlConst - SraConst, DL, CVT)); in combineShiftRightArithmetic()
48231 return DAG.getNode(ISD::SRA, DL, VT, NN, in combineShiftRightArithmetic()
48232 DAG.getConstant(SraConst - ShlConst, DL, CVT)); in combineShiftRightArithmetic()
48237 static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG, in combineShiftRightLogical() argument
48247 if (SDValue V = combineShiftToPMULH(N, DAG, DL, Subtarget)) in combineShiftRightLogical()
48261 return DAG.getNode(X86ISD::VSRLV, DL, VT, N00, N1); in combineShiftRightLogical()
48267 return DAG.getNode(X86ISD::VSRLV, DL, VT, N01, N1); in combineShiftRightLogical()
48306 SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT); in combineShiftRightLogical()
48307 SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1); in combineShiftRightLogical()
48308 return DAG.getNode(ISD::AND, DL, VT, NewShift, NewMask); in combineShiftRightLogical()
48313 static SDValue combineHorizOpWithShuffle(SDNode *N, SelectionDAG &DAG, in combineHorizOpWithShuffle() argument
48339 if (getTargetShuffleInputs(Vec, ShuffleOps, ShuffleMask, DAG)) { in combineHorizOpWithShuffle()
48348 std::tie(Lo, Hi) = DAG.SplitVector(ShuffleOps[0], DL); in combineHorizOpWithShuffle()
48349 Lo = DAG.getBitcast(SrcVT, Lo); in combineHorizOpWithShuffle()
48350 Hi = DAG.getBitcast(SrcVT, Hi); in combineHorizOpWithShuffle()
48351 SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi); in combineHorizOpWithShuffle()
48352 Res = DAG.getBitcast(ShufVT, Res); in combineHorizOpWithShuffle()
48353 Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ScaledMask); in combineHorizOpWithShuffle()
48354 return DAG.getBitcast(VT, Res); in combineHorizOpWithShuffle()
48367 getTargetShuffleInputs(BC0, Ops0, Mask0, DAG) && !isAnyZero(Mask0) && in combineHorizOpWithShuffle()
48371 getTargetShuffleInputs(BC1, Ops1, Mask1, DAG) && !isAnyZero(Mask1) && in combineHorizOpWithShuffle()
48406 LHS = DAG.getBitcast(SrcVT, LHS); in combineHorizOpWithShuffle()
48407 RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS); in combineHorizOpWithShuffle()
48409 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineHorizOpWithShuffle()
48410 Res = DAG.getBitcast(ShufVT, Res); in combineHorizOpWithShuffle()
48411 Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, PostShuffle); in combineHorizOpWithShuffle()
48412 return DAG.getBitcast(VT, Res); in combineHorizOpWithShuffle()
48422 if (getTargetShuffleInputs(BC0, Ops0, Mask0, DAG) && !isAnyZero(Mask0) && in combineHorizOpWithShuffle()
48423 getTargetShuffleInputs(BC1, Ops1, Mask1, DAG) && !isAnyZero(Mask1) && in combineHorizOpWithShuffle()
48445 SDValue Res = DAG.getNode(Opcode, DL, VT, DAG.getBitcast(SrcVT, Op00), in combineHorizOpWithShuffle()
48446 DAG.getBitcast(SrcVT, Op01)); in combineHorizOpWithShuffle()
48447 Res = DAG.getBitcast(ShufVT, Res); in combineHorizOpWithShuffle()
48448 Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ShuffleMask); in combineHorizOpWithShuffle()
48449 return DAG.getBitcast(VT, Res); in combineHorizOpWithShuffle()
48457 static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG, in combineVectorPack() argument
48527 return getConstVector(Bits, Undefs, VT.getSimpleVT(), DAG, SDLoc(N)); in combineVectorPack()
48531 if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget)) in combineVectorPack()
48537 (N0.isUndef() || DAG.ComputeNumSignBits(N0) == SrcBitsPerElt) && in combineVectorPack()
48538 (N1.isUndef() || DAG.ComputeNumSignBits(N1) == SrcBitsPerElt)) { in combineVectorPack()
48539 SDValue Not0 = N0.isUndef() ? N0 : IsNOT(N0, DAG); in combineVectorPack()
48540 SDValue Not1 = N1.isUndef() ? N1 : IsNOT(N1, DAG); in combineVectorPack()
48545 DAG.getNode(X86ISD::PACKSS, DL, VT, DAG.getBitcast(SrcVT, Not0), in combineVectorPack()
48546 DAG.getBitcast(SrcVT, Not1)); in combineVectorPack()
48547 return DAG.getNOT(DL, Pack, VT); in combineVectorPack()
48556 if ((IsSigned && DAG.ComputeNumSignBits(N0) > 8) || in combineVectorPack()
48558 DAG.MaskedValueIsZero(N0, APInt::getHighBitsSet(16, 8)))) { in combineVectorPack()
48560 return DAG.getNode(X86ISD::VTRUNC, SDLoc(N), VT, N0.getOperand(0)); in combineVectorPack()
48564 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32, in combineVectorPack()
48565 N0.getOperand(0), DAG.getUNDEF(MVT::v8i32)); in combineVectorPack()
48566 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Concat); in combineVectorPack()
48586 Src0 = Src0 ? Src0 : DAG.getUNDEF(Src1.getValueType()); in combineVectorPack()
48587 Src1 = Src1 ? Src1 : DAG.getUNDEF(Src0.getValueType()); in combineVectorPack()
48588 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Src0, Src1); in combineVectorPack()
48597 DAG); in combineVectorPack()
48602 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineVectorPack()
48608 static SDValue combineVectorHADDSUB(SDNode *N, SelectionDAG &DAG, in combineVectorHADDSUB() argument
48615 if (!shouldUseHorizontalOp(true, DAG, Subtarget)) { in combineVectorHADDSUB()
48632 SDValue Res = DAG.getNode(LHS.getOpcode(), DL, LHS.getValueType(), in combineVectorHADDSUB()
48636 Res = DAG.getBitcast(ShufVT, Res); in combineVectorHADDSUB()
48638 DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res, in combineVectorHADDSUB()
48639 getV4X86ShuffleImm8ForMask({0, 1, 0, 1}, DL, DAG)); in combineVectorHADDSUB()
48641 DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res, in combineVectorHADDSUB()
48642 getV4X86ShuffleImm8ForMask({2, 3, 2, 3}, DL, DAG)); in combineVectorHADDSUB()
48643 return DAG.getNode(N->getOpcode(), DL, VT, DAG.getBitcast(VT, NewLHS), in combineVectorHADDSUB()
48644 DAG.getBitcast(VT, NewRHS)); in combineVectorHADDSUB()
48650 if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget)) in combineVectorHADDSUB()
48656 static SDValue combineVectorShiftVar(SDNode *N, SelectionDAG &DAG, in combineVectorShiftVar() argument
48668 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftVar()
48678 EltBits[0].getZExtValue(), DAG); in combineVectorShiftVar()
48681 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineVectorShiftVar()
48689 static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG, in combineVectorShiftImm() argument
48707 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
48714 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
48726 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
48732 return DAG.getConstant(-1, SDLoc(N), VT); in combineVectorShiftImm()
48740 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
48743 return DAG.getNode(Opcode, SDLoc(N), VT, N0.getOperand(0), in combineVectorShiftImm()
48744 DAG.getTargetConstant(NewShiftVal, SDLoc(N), MVT::i8)); in combineVectorShiftImm()
48759 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineVectorShiftImm()
48777 Src = DAG.getBitcast(VT, Src); in combineVectorShiftImm()
48778 Src = DAG.getNode(X86ISD::PSHUFD, DL, VT, Src, in combineVectorShiftImm()
48779 getV4X86ShuffleImm8ForMask({0, 0, 2, 2}, DL, DAG)); in combineVectorShiftImm()
48780 Src = DAG.getNode(X86ISD::VSHLI, DL, VT, Src, N1); in combineVectorShiftImm()
48781 Src = DAG.getNode(X86ISD::VSRAI, DL, VT, Src, N1); in combineVectorShiftImm()
48811 return getConstVector(EltBits, UndefElts, VT.getSimpleVT(), DAG, SDLoc(N)); in combineVectorShiftImm()
48827 SDValue LHS = DAG.getNode(Opcode, DL, VT, in combineVectorShiftImm()
48828 DAG.getBitcast(VT, BC.getOperand(0)), N1); in combineVectorShiftImm()
48829 return DAG.getNode(BC.getOpcode(), DL, VT, LHS, RHS); in combineVectorShiftImm()
48834 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineVectorShiftImm()
48842 static SDValue combineVectorInsert(SDNode *N, SelectionDAG &DAG, in combineVectorInsert() argument
48858 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Scl); in combineVectorInsert()
48862 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineVectorInsert()
48871 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineVectorInsert()
48881 static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG, in combineCompareEqual() argument
48943 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01, in combineCompareEqual()
48944 DAG.getTargetConstant(x86cc, DL, MVT::i8)); in combineCompareEqual()
48947 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1, in combineCompareEqual()
48948 DAG.getConstant(0, DL, MVT::v16i1), in combineCompareEqual()
48949 FSetCC, DAG.getIntPtrConstant(0, DL)); in combineCompareEqual()
48950 return DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Ins), DL, in combineCompareEqual()
48954 DAG.getNode(X86ISD::FSETCC, DL, CMP00.getValueType(), CMP00, in combineCompareEqual()
48955 CMP01, DAG.getTargetConstant(x86cc, DL, MVT::i8)); in combineCompareEqual()
48966 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, in combineCompareEqual()
48968 SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64); in combineCompareEqual()
48969 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, in combineCompareEqual()
48970 Vector32, DAG.getIntPtrConstant(0, DL)); in combineCompareEqual()
48974 SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF); in combineCompareEqual()
48975 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI, in combineCompareEqual()
48976 DAG.getConstant(1, DL, IntVT)); in combineCompareEqual()
48977 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, in combineCompareEqual()
48988 static SDValue combineAndNotIntoANDNP(SDNode *N, SelectionDAG &DAG) { in combineAndNotIntoANDNP() argument
48999 if (SDValue Not = IsNOT(N0, DAG)) { in combineAndNotIntoANDNP()
49002 } else if (SDValue Not = IsNOT(N1, DAG)) { in combineAndNotIntoANDNP()
49008 X = DAG.getBitcast(VT, X); in combineAndNotIntoANDNP()
49009 Y = DAG.getBitcast(VT, Y); in combineAndNotIntoANDNP()
49010 return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y); in combineAndNotIntoANDNP()
49019 static SDValue combineAndShuffleNot(SDNode *N, SelectionDAG &DAG, in combineAndShuffleNot() argument
49030 auto GetNot = [&DAG](SDValue V) { in combineAndShuffleNot()
49047 if (SDValue Not = IsNOT(Src, DAG)) { in combineAndShuffleNot()
49048 SDValue NotSrc = DAG.getBitcast(Src.getValueType(), Not); in combineAndShuffleNot()
49050 DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(IVEN), IVEN.getValueType(), in combineAndShuffleNot()
49052 return DAG.getVectorShuffle(SVN->getValueType(0), SDLoc(SVN), NotIVEN, in combineAndShuffleNot()
49061 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineAndShuffleNot()
49072 X = DAG.getBitcast(VT, X); in combineAndShuffleNot()
49073 Y = DAG.getBitcast(VT, Y); in combineAndShuffleNot()
49079 TLI.isTypeLegal(VT.getHalfNumVectorElementsVT(*DAG.getContext()))) { in combineAndShuffleNot()
49081 std::tie(LoX, HiX) = splitVector(X, DAG, DL); in combineAndShuffleNot()
49083 std::tie(LoY, HiY) = splitVector(Y, DAG, DL); in combineAndShuffleNot()
49085 SDValue LoV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {LoX, LoY}); in combineAndShuffleNot()
49086 SDValue HiV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {HiX, HiY}); in combineAndShuffleNot()
49087 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, {LoV, HiV}); in combineAndShuffleNot()
49091 return DAG.getNode(X86ISD::ANDNP, DL, VT, {X, Y}); in combineAndShuffleNot()
49106 SelectionDAG &DAG, unsigned Depth) { in PromoteMaskArithmetic() argument
49117 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in PromoteMaskArithmetic()
49121 if (SDValue NN0 = PromoteMaskArithmetic(N0, DL, VT, DAG, Depth + 1)) in PromoteMaskArithmetic()
49135 if (SDValue NN1 = PromoteMaskArithmetic(N1, DL, VT, DAG, Depth + 1)) in PromoteMaskArithmetic()
49144 DAG.FoldConstantArithmetic(ISD::ZERO_EXTEND, DL, VT, {N1})) in PromoteMaskArithmetic()
49150 return DAG.getNode(N.getOpcode(), DL, VT, N0, N1); in PromoteMaskArithmetic()
49160 SelectionDAG &DAG, in PromoteMaskArithmetic() argument
49172 SDValue Op = PromoteMaskArithmetic(Narrow, DL, VT, DAG, 0); in PromoteMaskArithmetic()
49180 return DAG.getZeroExtendInReg(Op, DL, NarrowVT); in PromoteMaskArithmetic()
49182 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, in PromoteMaskArithmetic()
49183 Op, DAG.getValueType(NarrowVT)); in PromoteMaskArithmetic()
49203 static SDValue convertIntLogicToFPLogic(SDNode *N, SelectionDAG &DAG, in convertIntLogicToFPLogic() argument
49228 SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10); in convertIntLogicToFPLogic()
49229 return DAG.getBitcast(VT, FPLogic); in convertIntLogicToFPLogic()
49250 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), N00Type, NumElts); in convertIntLogicToFPLogic()
49251 EVT BoolVecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts); in convertIntLogicToFPLogic()
49252 SDValue ZeroIndex = DAG.getVectorIdxConstant(0, DL); in convertIntLogicToFPLogic()
49255 SDValue Vec00 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N00); in convertIntLogicToFPLogic()
49256 SDValue Vec01 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N01); in convertIntLogicToFPLogic()
49257 SDValue Vec10 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N10); in convertIntLogicToFPLogic()
49258 SDValue Vec11 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N11); in convertIntLogicToFPLogic()
49259 SDValue Setcc0 = DAG.getSetCC(DL, BoolVecVT, Vec00, Vec01, CC0); in convertIntLogicToFPLogic()
49260 SDValue Setcc1 = DAG.getSetCC(DL, BoolVecVT, Vec10, Vec11, CC1); in convertIntLogicToFPLogic()
49261 SDValue Logic = DAG.getNode(N->getOpcode(), DL, BoolVecVT, Setcc0, Setcc1); in convertIntLogicToFPLogic()
49262 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Logic, ZeroIndex); in convertIntLogicToFPLogic()
49267 static SDValue combineBitOpWithMOVMSK(SDNode *N, SelectionDAG &DAG) { in combineBitOpWithMOVMSK() argument
49295 DAG.getNode(VecOpc, DL, VecVT0, Vec0, DAG.getBitcast(VecVT0, Vec1)); in combineBitOpWithMOVMSK()
49296 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result); in combineBitOpWithMOVMSK()
49302 static SDValue combineBitOpWithShift(SDNode *N, SelectionDAG &DAG) { in combineBitOpWithShift() argument
49333 DAG.getNode(Opc, DL, BCVT, BC0.getOperand(0), BC1.getOperand(0)); in combineBitOpWithShift()
49334 SDValue Shift = DAG.getNode(BCOpc, DL, BCVT, BitOp, BC0.getOperand(1)); in combineBitOpWithShift()
49335 return DAG.getBitcast(VT, Shift); in combineBitOpWithShift()
49345 static SDValue combineBitOpWithPACK(SDNode *N, SelectionDAG &DAG) { in combineBitOpWithPACK() argument
49373 if (DAG.ComputeNumSignBits(N0.getOperand(0)) != NumSrcBits || in combineBitOpWithPACK()
49374 DAG.ComputeNumSignBits(N0.getOperand(1)) != NumSrcBits || in combineBitOpWithPACK()
49375 DAG.ComputeNumSignBits(N1.getOperand(0)) != NumSrcBits || in combineBitOpWithPACK()
49376 DAG.ComputeNumSignBits(N1.getOperand(1)) != NumSrcBits) in combineBitOpWithPACK()
49380 SDValue LHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(0), N1.getOperand(0)); in combineBitOpWithPACK()
49381 SDValue RHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(1), N1.getOperand(1)); in combineBitOpWithPACK()
49382 return DAG.getBitcast(VT, DAG.getNode(X86ISD::PACKSS, DL, DstVT, LHS, RHS)); in combineBitOpWithPACK()
49388 static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG, in combineAndMaskToShift() argument
49421 VT.getScalarSizeInBits() - 1, DAG); in combineAndMaskToShift()
49422 return DAG.getNode(X86ISD::ANDNP, DL, VT, Sra, Y); in combineAndMaskToShift()
49438 if (EltBitWidth != DAG.ComputeNumSignBits(Op0)) in combineAndMaskToShift()
49443 SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8); in combineAndMaskToShift()
49444 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT, Op0, ShAmt); in combineAndMaskToShift()
49445 return DAG.getBitcast(N->getValueType(0), Shift); in combineAndMaskToShift()
49483 static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG, in combineAndLoadToBZHI() argument
49538 SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32); in combineAndLoadToBZHI()
49544 Index = DAG.getZExtOrTrunc(Index, dl, MVT::i32); in combineAndLoadToBZHI()
49546 SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index); in combineAndLoadToBZHI()
49547 Sub = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Sub); in combineAndLoadToBZHI()
49549 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT); in combineAndLoadToBZHI()
49550 SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub); in combineAndLoadToBZHI()
49552 return DAG.getNode(ISD::AND, dl, VT, Inp, LShr); in combineAndLoadToBZHI()
49565 static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG, in combineScalarAndWithMaskSetcc() argument
49597 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineScalarAndWithMaskSetcc()
49637 DAG.getConstant(0, dl, SubVecVT)); in combineScalarAndWithMaskSetcc()
49639 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT, in combineScalarAndWithMaskSetcc()
49641 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcVT.getSizeInBits()); in combineScalarAndWithMaskSetcc()
49642 return DAG.getZExtOrTrunc(DAG.getBitcast(IntVT, Concat), dl, VT); in combineScalarAndWithMaskSetcc()
49645 static SDValue getBMIMatchingOp(unsigned Opc, SelectionDAG &DAG, in getBMIMatchingOp() argument
49664 getBMIMatchingOp(Opc, DAG, OpMustEq, Op.getOperand(OpIdx), Depth)) in getBMIMatchingOp()
49665 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), R, in getBMIMatchingOp()
49672 return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op); in getBMIMatchingOp()
49678 return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op); in getBMIMatchingOp()
49685 return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op); in getBMIMatchingOp()
49690 static SDValue combineBMILogicOp(SDNode *N, SelectionDAG &DAG, in combineBMILogicOp() argument
49703 getBMIMatchingOp(N->getOpcode(), DAG, N->getOperand(OpIdx), in combineBMILogicOp()
49710 SelectionDAG &DAG, in combineX86SubCmpForFlags() argument
49745 X = DAG.getMergeValues({N->getOperand(0), X}, SDLoc(N)); in combineX86SubCmpForFlags()
49759 Ops[CondNo] = DAG.getTargetConstant(OppositeCC, SDLoc(BrCond), MVT::i8); in combineX86SubCmpForFlags()
49764 DAG.getNode(X86ISD::BRCOND, SDLoc(BrCond), BrCond->getValueType(0), Ops); in combineX86SubCmpForFlags()
49771 static SDValue combineAndOrForCcmpCtest(SDNode *N, SelectionDAG &DAG, in combineAndOrForCcmpCtest() argument
49825 IsOR ? DAG.getTargetConstant(X86::GetOppositeBranchCondition(CC0), in combineAndOrForCcmpCtest()
49834 SDValue CFlags = DAG.getTargetConstant( in combineAndOrForCcmpCtest()
49841 ? DAG.getNode(X86ISD::CCMP, DL, MVT::i32, in combineAndOrForCcmpCtest()
49844 : DAG.getNode(X86ISD::CTEST, DL, MVT::i32, in combineAndOrForCcmpCtest()
49848 return DAG.getNode(X86ISD::SETCC, DL, MVT::i8, {CC1N, CCMP}); in combineAndOrForCcmpCtest()
49851 static SDValue combineAnd(SDNode *N, SelectionDAG &DAG, in combineAnd() argument
49858 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineAnd()
49862 return DAG.getBitcast(MVT::v4i32, in combineAnd()
49863 DAG.getNode(X86ISD::FAND, dl, MVT::v4f32, in combineAnd()
49864 DAG.getBitcast(MVT::v4f32, N0), in combineAnd()
49865 DAG.getBitcast(MVT::v4f32, N1))); in combineAnd()
49871 if (DAG.MaskedValueIsZero(N1, HiMask) || in combineAnd()
49872 DAG.MaskedValueIsZero(N0, HiMask)) { in combineAnd()
49873 SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N0); in combineAnd()
49874 SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N1); in combineAnd()
49875 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, in combineAnd()
49876 DAG.getNode(ISD::AND, dl, MVT::i32, LHS, RHS)); in combineAnd()
49888 EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts); in combineAnd()
49889 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget); in combineAnd()
49891 Mask = DAG.getBitcast(MaskVT, SrcOps[0]); in combineAnd()
49895 SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT); in combineAnd()
49896 Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits); in combineAnd()
49897 return DAG.getSetCC(dl, MVT::i1, Mask, PartialBits, ISD::SETEQ); in combineAnd()
49923 SDValue Neg = DAG.getNegative(N0.getOperand(0), dl, VT); in combineAnd()
49927 SDValue Shift = DAG.getNode(ISD::SHL, dl, VT, Neg, in combineAnd()
49928 DAG.getConstant(MulCLowBitLog, dl, VT)); in combineAnd()
49929 return DAG.getNode(ISD::AND, dl, VT, Shift, N1); in combineAnd()
49934 if (SDValue SetCC = combineAndOrForCcmpCtest(N, DAG, DCI, Subtarget)) in combineAnd()
49937 if (SDValue V = combineScalarAndWithMaskSetcc(N, DAG, Subtarget)) in combineAnd()
49940 if (SDValue R = combineBitOpWithMOVMSK(N, DAG)) in combineAnd()
49943 if (SDValue R = combineBitOpWithShift(N, DAG)) in combineAnd()
49946 if (SDValue R = combineBitOpWithPACK(N, DAG)) in combineAnd()
49949 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget)) in combineAnd()
49952 if (SDValue R = combineAndShuffleNot(N, DAG, Subtarget)) in combineAnd()
49958 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget)) in combineAnd()
49961 if (SDValue R = combineAndNotIntoANDNP(N, DAG)) in combineAnd()
49964 if (SDValue ShiftRight = combineAndMaskToShift(N, DAG, Subtarget)) in combineAnd()
49967 if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget)) in combineAnd()
49977 DAG.ComputeNumSignBits(N1) == VT.getScalarSizeInBits() && in combineAnd()
49979 SDValue MaskMul = DAG.getNode(ISD::AND, dl, VT, N0.getOperand(1), N1); in combineAnd()
49980 return DAG.getNode(Opc0, dl, VT, N0.getOperand(0), MaskMul); in combineAnd()
50012 if (SDValue BT = getBT(Src, BitNo, dl, DAG)) in combineAnd()
50013 return DAG.getZExtOrTrunc(getSETCC(X86CC, BT, dl, DAG), dl, VT); in combineAnd()
50020 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineAnd()
50064 SDValue NewN0 = TLI.SimplifyMultipleUseDemandedBits(N0, Bits0, Elts0, DAG); in combineAnd()
50065 SDValue NewN1 = TLI.SimplifyMultipleUseDemandedBits(N1, Bits1, Elts1, DAG); in combineAnd()
50067 return DAG.getNode(ISD::AND, dl, VT, NewN0 ? NewN0 : N0, in combineAnd()
50104 /*AllowVarPerLaneMask*/ true, DAG, Subtarget)) in combineAnd()
50105 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle, in combineAnd()
50110 if (SDValue R = combineBMILogicOp(N, DAG, Subtarget)) in combineAnd()
50117 static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG, in canonicalizeBitSelect() argument
50165 SDValue A = DAG.getBitcast(OpVT, N0.getOperand(1)); in canonicalizeBitSelect()
50166 SDValue B = DAG.getBitcast(OpVT, N0.getOperand(0)); in canonicalizeBitSelect()
50167 SDValue C = DAG.getBitcast(OpVT, N1.getOperand(0)); in canonicalizeBitSelect()
50168 SDValue Imm = DAG.getTargetConstant(0xCA, DL, MVT::i8); in canonicalizeBitSelect()
50170 DAG, Subtarget); in canonicalizeBitSelect()
50171 return DAG.getBitcast(VT, Res); in canonicalizeBitSelect()
50176 DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)), in canonicalizeBitSelect()
50177 DAG.getBitcast(VT, N1.getOperand(0))); in canonicalizeBitSelect()
50178 return DAG.getNode(ISD::OR, DL, VT, X, Y); in canonicalizeBitSelect()
50221 static SDValue combineLogicBlendIntoPBLENDV(SDNode *N, SelectionDAG &DAG, in combineLogicBlendIntoPBLENDV() argument
50243 if (!MaskVT.isInteger() || DAG.ComputeNumSignBits(Mask) != EltBits) in combineLogicBlendIntoPBLENDV()
50250 DAG, Subtarget)) in combineLogicBlendIntoPBLENDV()
50263 X = DAG.getBitcast(BlendVT, X); in combineLogicBlendIntoPBLENDV()
50264 Y = DAG.getBitcast(BlendVT, Y); in combineLogicBlendIntoPBLENDV()
50265 Mask = DAG.getBitcast(BlendVT, Mask); in combineLogicBlendIntoPBLENDV()
50266 Mask = DAG.getSelect(DL, BlendVT, Mask, Y, X); in combineLogicBlendIntoPBLENDV()
50267 return DAG.getBitcast(VT, Mask); in combineLogicBlendIntoPBLENDV()
50276 static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) { in lowerX86CmpEqZeroToCtlzSrl() argument
50281 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0)); in lowerX86CmpEqZeroToCtlzSrl()
50284 SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32); in lowerX86CmpEqZeroToCtlzSrl()
50285 SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc, in lowerX86CmpEqZeroToCtlzSrl()
50286 DAG.getConstant(Log2b, dl, MVT::i8)); in lowerX86CmpEqZeroToCtlzSrl()
50297 static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG, in combineOrCmpEqZeroToCtlzSrl() argument
50347 SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, DAG); in combineOrCmpEqZeroToCtlzSrl()
50349 if (NewLHS && (NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG))) in combineOrCmpEqZeroToCtlzSrl()
50350 Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, NewLHS, NewRHS); in combineOrCmpEqZeroToCtlzSrl()
50363 NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG); in combineOrCmpEqZeroToCtlzSrl()
50366 Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, Ret, NewRHS); in combineOrCmpEqZeroToCtlzSrl()
50369 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), Ret); in combineOrCmpEqZeroToCtlzSrl()
50374 const SDLoc &DL, SelectionDAG &DAG) { in foldMaskedMergeImpl() argument
50386 SDValue Freeze_And0_R = DAG.getNode(ISD::FREEZE, SDLoc(), VT, And0_R); in foldMaskedMergeImpl()
50387 SDValue Xor0 = DAG.getNode(ISD::XOR, DL, VT, And1_R, Freeze_And0_R); in foldMaskedMergeImpl()
50388 SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor0, NotOp); in foldMaskedMergeImpl()
50389 SDValue Xor1 = DAG.getNode(ISD::XOR, DL, VT, And, Freeze_And0_R); in foldMaskedMergeImpl()
50398 static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG) { in foldMaskedMerge() argument
50414 if (SDValue Result = foldMaskedMergeImpl(N00, N01, N10, N11, DL, DAG)) in foldMaskedMerge()
50416 if (SDValue Result = foldMaskedMergeImpl(N01, N00, N10, N11, DL, DAG)) in foldMaskedMerge()
50418 if (SDValue Result = foldMaskedMergeImpl(N10, N11, N00, N01, DL, DAG)) in foldMaskedMerge()
50420 if (SDValue Result = foldMaskedMergeImpl(N11, N10, N00, N01, DL, DAG)) in foldMaskedMerge()
50431 SelectionDAG &DAG, in combineAddOrSubToADCOrSBB() argument
50433 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in combineAddOrSubToADCOrSBB()
50447 EFLAGS = LowerAndToBT(Y, ISD::SETNE, DL, DAG, CC); in combineAddOrSubToADCOrSBB()
50462 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
50463 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), in combineAddOrSubToADCOrSBB()
50475 SDValue NewSub = DAG.getNode( in combineAddOrSubToADCOrSBB()
50479 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
50480 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), in combineAddOrSubToADCOrSBB()
50489 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, in combineAddOrSubToADCOrSBB()
50490 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
50491 DAG.getConstant(0, DL, VT), EFLAGS); in combineAddOrSubToADCOrSBB()
50508 DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(), in combineAddOrSubToADCOrSBB()
50511 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, in combineAddOrSubToADCOrSBB()
50512 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
50513 DAG.getConstant(0, DL, VT), NewEFLAGS); in combineAddOrSubToADCOrSBB()
50520 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, in combineAddOrSubToADCOrSBB()
50521 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
50522 DAG.getConstant(-1, DL, VT), EFLAGS); in combineAddOrSubToADCOrSBB()
50538 DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(), in combineAddOrSubToADCOrSBB()
50541 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, in combineAddOrSubToADCOrSBB()
50542 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
50543 DAG.getConstant(-1, DL, VT), NewEFLAGS); in combineAddOrSubToADCOrSBB()
50567 SDValue Zero = DAG.getConstant(0, DL, ZVT); in combineAddOrSubToADCOrSBB()
50568 SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32); in combineAddOrSubToADCOrSBB()
50569 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z); in combineAddOrSubToADCOrSBB()
50570 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
50571 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), in combineAddOrSubToADCOrSBB()
50581 SDValue One = DAG.getConstant(1, DL, ZVT); in combineAddOrSubToADCOrSBB()
50582 SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32); in combineAddOrSubToADCOrSBB()
50583 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One); in combineAddOrSubToADCOrSBB()
50584 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
50585 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), in combineAddOrSubToADCOrSBB()
50591 SDValue One = DAG.getConstant(1, DL, ZVT); in combineAddOrSubToADCOrSBB()
50592 SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32); in combineAddOrSubToADCOrSBB()
50593 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One); in combineAddOrSubToADCOrSBB()
50596 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineAddOrSubToADCOrSBB()
50601 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, VTs, X, in combineAddOrSubToADCOrSBB()
50602 DAG.getConstant(-1ULL, DL, VT), Cmp1.getValue(1)); in combineAddOrSubToADCOrSBB()
50606 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, VTs, X, in combineAddOrSubToADCOrSBB()
50607 DAG.getConstant(0, DL, VT), Cmp1.getValue(1)); in combineAddOrSubToADCOrSBB()
50614 SelectionDAG &DAG) { in combineAddOrSubToADCOrSBB() argument
50620 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, X, Y, DAG)) in combineAddOrSubToADCOrSBB()
50624 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, Y, X, DAG)) { in combineAddOrSubToADCOrSBB()
50626 ADCOrSBB = DAG.getNegative(ADCOrSBB, DL, VT); in combineAddOrSubToADCOrSBB()
50634 SelectionDAG &DAG) { in combineOrXorWithSETCC() argument
50652 if (SDValue R = combineAddOrSubToADCOrSBB(IsSub, DL, VT, N1, N0, DAG)) in combineOrXorWithSETCC()
50674 return DAG.getNode(X86ISD::PCMPEQ, SDLoc(N), VT, N0.getOperand(0), in combineOrXorWithSETCC()
50682 static SDValue combineOr(SDNode *N, SelectionDAG &DAG, in combineOr() argument
50689 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineOr()
50693 return DAG.getBitcast(MVT::v4i32, in combineOr()
50694 DAG.getNode(X86ISD::FOR, dl, MVT::v4f32, in combineOr()
50695 DAG.getBitcast(MVT::v4f32, N0), in combineOr()
50696 DAG.getBitcast(MVT::v4f32, N1))); in combineOr()
50707 EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts); in combineOr()
50708 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget); in combineOr()
50710 Mask = DAG.getBitcast(MaskVT, SrcOps[0]); in combineOr()
50714 SDValue ZeroBits = DAG.getConstant(0, dl, MaskVT); in combineOr()
50715 SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT); in combineOr()
50716 Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits); in combineOr()
50717 return DAG.getSetCC(dl, MVT::i1, Mask, ZeroBits, ISD::SETNE); in combineOr()
50722 if (SDValue SetCC = combineAndOrForCcmpCtest(N, DAG, DCI, Subtarget)) in combineOr()
50725 if (SDValue R = combineBitOpWithMOVMSK(N, DAG)) in combineOr()
50728 if (SDValue R = combineBitOpWithShift(N, DAG)) in combineOr()
50731 if (SDValue R = combineBitOpWithPACK(N, DAG)) in combineOr()
50734 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget)) in combineOr()
50740 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget)) in combineOr()
50743 if (SDValue R = canonicalizeBitSelect(N, DAG, Subtarget)) in combineOr()
50746 if (SDValue R = combineLogicBlendIntoPBLENDV(N, DAG, Subtarget)) in combineOr()
50763 SDValue NotCond = getSETCC(CCode, Cond.getOperand(1), SDLoc(Cond), DAG); in combineOr()
50765 SDValue R = DAG.getZExtOrTrunc(NotCond, dl, VT); in combineOr()
50766 R = DAG.getNode(ISD::MUL, dl, VT, R, DAG.getConstant(Val + 1, dl, VT)); in combineOr()
50767 R = DAG.getNode(ISD::SUB, dl, VT, R, DAG.getConstant(1, dl, VT)); in combineOr()
50784 DAG.MaskedVectorIsZero(N0, UpperElts)) { in combineOr()
50785 return DAG.getNode( in combineOr()
50787 extractSubVector(N0, 0, DAG, dl, HalfElts), in combineOr()
50788 extractSubVector(N1.getOperand(0), 0, DAG, dl, HalfElts)); in combineOr()
50792 DAG.MaskedVectorIsZero(N1, UpperElts)) { in combineOr()
50793 return DAG.getNode( in combineOr()
50795 extractSubVector(N1, 0, DAG, dl, HalfElts), in combineOr()
50796 extractSubVector(N0.getOperand(0), 0, DAG, dl, HalfElts)); in combineOr()
50803 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineOr()
50832 if (SDValue R = foldMaskedMerge(N, DAG)) in combineOr()
50835 if (SDValue R = combineOrXorWithSETCC(N, N0, N1, DAG)) in combineOr()
50845 static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) { in foldXorTruncShiftIntoCmp() argument
50883 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in foldXorTruncShiftIntoCmp()
50884 EVT SetCCResultType = TLI.getSetCCResultType(DAG.getDataLayout(), in foldXorTruncShiftIntoCmp()
50885 *DAG.getContext(), ResultType); in foldXorTruncShiftIntoCmp()
50886 SDValue Cond = DAG.getSetCC(DL, SetCCResultType, ShiftOp, in foldXorTruncShiftIntoCmp()
50887 DAG.getConstant(-1, DL, ShiftOpTy), ISD::SETGT); in foldXorTruncShiftIntoCmp()
50889 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, ResultType, Cond); in foldXorTruncShiftIntoCmp()
50900 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG, in foldVectorXorShiftIntoCmp() argument
50937 return DAG.getSetCC(SDLoc(N), VT, Shift.getOperand(0), Ones, ISD::SETGT); in foldVectorXorShiftIntoCmp()
50956 static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG, in detectUSatPattern() argument
50988 return DAG.getNode(ISD::SMAX, DL, InVT, SMin, In.getOperand(1)); in detectUSatPattern()
51038 SelectionDAG &DAG, in combineTruncateWithSat() argument
51056 DL, DAG, Subtarget); in combineTruncateWithSat()
51058 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, Mid); in combineTruncateWithSat()
51083 DAG, Subtarget); in combineTruncateWithSat()
51085 SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG, in combineTruncateWithSat()
51090 return truncateVectorWithPACK(X86ISD::PACKUS, VT, USatVal, DL, DAG, in combineTruncateWithSat()
51094 return truncateVectorWithPACK(X86ISD::PACKSS, VT, SSatVal, DL, DAG, in combineTruncateWithSat()
51098 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineTruncateWithSat()
51107 } else if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL)) { in combineTruncateWithSat()
51118 SmallVector<SDValue, 4> ConcatOps(NumConcats, DAG.getUNDEF(InVT)); in combineTruncateWithSat()
51120 InVT = EVT::getVectorVT(*DAG.getContext(), InSVT, in combineTruncateWithSat()
51122 SatVal = DAG.getNode(ISD::CONCAT_VECTORS, DL, InVT, ConcatOps); in combineTruncateWithSat()
51127 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), SVT, ResElts); in combineTruncateWithSat()
51128 SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal); in combineTruncateWithSat()
51129 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in combineTruncateWithSat()
51130 DAG.getIntPtrConstant(0, DL)); in combineTruncateWithSat()
51138 SelectionDAG &DAG, in combineConstantPoolLoads() argument
51199 SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits()); in combineConstantPoolLoads()
51200 Extract = DAG.getBitcast(RegVT, Extract); in combineConstantPoolLoads()
51212 static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, in combineLoad() argument
51219 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineLoad()
51230 (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), RegVT, in combineLoad()
51240 DAG.getMemBasePlusOffset(Ptr1, TypeSize::getFixed(HalfOffset), dl); in combineLoad()
51241 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), in combineLoad()
51244 DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr1, Ld->getPointerInfo(), in combineLoad()
51247 SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr2, in combineLoad()
51251 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in combineLoad()
51254 SDValue NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, RegVT, Load1, Load2); in combineLoad()
51263 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumElts); in combineLoad()
51265 SDValue IntLoad = DAG.getLoad(IntVT, dl, Ld->getChain(), Ld->getBasePtr(), in combineLoad()
51269 SDValue BoolVec = DAG.getBitcast(RegVT, IntLoad); in combineLoad()
51289 SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, dl, in combineLoad()
51291 Extract = DAG.getBitcast(RegVT, Extract); in combineLoad()
51297 if (SDValue V = combineConstantPoolLoads(Ld, dl, DAG, DCI, Subtarget)) in combineLoad()
51304 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); in combineLoad()
51307 DAG.getAddrSpaceCast(dl, PtrVT, Ld->getBasePtr(), AddrSpace, 0); in combineLoad()
51308 return DAG.getExtLoad(Ext, dl, RegVT, Ld->getChain(), Cast, in combineLoad()
51356 SelectionDAG &DAG, SDValue &Addr, in getParamsForOneTrueMaskedElt() argument
51370 Addr = DAG.getMemBasePlusOffset(Addr, TypeSize::getFixed(Offset), in getParamsForOneTrueMaskedElt()
51374 Index = DAG.getIntPtrConstant(TrueMaskElt, SDLoc(MaskedOp)); in getParamsForOneTrueMaskedElt()
51385 reduceMaskedLoadToScalarLoad(MaskedLoadSDNode *ML, SelectionDAG &DAG, in reduceMaskedLoadToScalarLoad() argument
51396 if (!getParamsForOneTrueMaskedElt(ML, DAG, Addr, VecIndex, Alignment, Offset)) in reduceMaskedLoadToScalarLoad()
51412 DAG.getLoad(EltVT, DL, ML->getChain(), Addr, in reduceMaskedLoadToScalarLoad()
51416 SDValue PassThru = DAG.getBitcast(CastVT, ML->getPassThru()); in reduceMaskedLoadToScalarLoad()
51420 DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, CastVT, PassThru, Load, VecIndex); in reduceMaskedLoadToScalarLoad()
51421 Insert = DAG.getBitcast(VT, Insert); in reduceMaskedLoadToScalarLoad()
51426 combineMaskedLoadConstantMask(MaskedLoadSDNode *ML, SelectionDAG &DAG, in combineMaskedLoadConstantMask() argument
51443 SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(), in combineMaskedLoadConstantMask()
51445 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd, in combineMaskedLoadConstantMask()
51464 SDValue NewML = DAG.getMaskedLoad( in combineMaskedLoadConstantMask()
51466 DAG.getUNDEF(VT), ML->getMemoryVT(), ML->getMemOperand(), in combineMaskedLoadConstantMask()
51468 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML, in combineMaskedLoadConstantMask()
51474 static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG, in combineMaskedLoad() argument
51485 reduceMaskedLoadToScalarLoad(Mld, DAG, DCI, Subtarget)) in combineMaskedLoad()
51490 if (SDValue Blend = combineMaskedLoadConstantMask(Mld, DAG, DCI)) in combineMaskedLoad()
51499 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineMaskedLoad()
51507 TLI.SimplifyMultipleUseDemandedBits(Mask, DemandedBits, DAG)) in combineMaskedLoad()
51508 return DAG.getMaskedLoad( in combineMaskedLoad()
51522 SelectionDAG &DAG, in reduceMaskedStoreToScalarStore() argument
51531 if (!getParamsForOneTrueMaskedElt(MS, DAG, Addr, VecIndex, Alignment, Offset)) in reduceMaskedStoreToScalarStore()
51542 Value = DAG.getBitcast(CastVT, Value); in reduceMaskedStoreToScalarStore()
51545 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Value, VecIndex); in reduceMaskedStoreToScalarStore()
51548 return DAG.getStore(MS->getChain(), DL, Extract, Addr, in reduceMaskedStoreToScalarStore()
51553 static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG, in combineMaskedStore() argument
51562 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineMaskedStore()
51567 if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG, Subtarget)) in combineMaskedStore()
51581 TLI.SimplifyMultipleUseDemandedBits(Mask, DemandedBits, DAG)) in combineMaskedStore()
51582 return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Mst->getValue(), in combineMaskedStore()
51592 return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Value.getOperand(0), in combineMaskedStore()
51601 static SDValue combineStore(SDNode *N, SelectionDAG &DAG, in combineStore() argument
51609 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineStore()
51615 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements()); in combineStore()
51616 StoredVal = DAG.getBitcast(NewVT, StoredVal); in combineStore()
51618 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), in combineStore()
51630 Val = DAG.getZeroExtendInReg(Val, dl, MVT::i1); in combineStore()
51631 return DAG.getStore(St->getChain(), dl, Val, in combineStore()
51642 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, VT)); in combineStore()
51644 StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops); in combineStore()
51645 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), in combineStore()
51656 SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl, in combineStore()
51658 Lo = combinevXi1ConstantToInteger(Lo, DAG); in combineStore()
51659 SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl, in combineStore()
51661 Hi = combinevXi1ConstantToInteger(Hi, DAG); in combineStore()
51664 SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, TypeSize::getFixed(4), dl); in combineStore()
51667 DAG.getStore(St->getChain(), dl, Lo, Ptr0, St->getPointerInfo(), in combineStore()
51671 DAG.getStore(St->getChain(), dl, Hi, Ptr1, in combineStore()
51675 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1); in combineStore()
51678 StoredVal = combinevXi1ConstantToInteger(StoredVal, DAG); in combineStore()
51679 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), in combineStore()
51688 TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in combineStore()
51695 return splitVectorStore(St, DAG); in combineStore()
51707 return splitVectorStore(St, DAG); in combineStore()
51716 return scalarizeVectorStore(St, NTVT, DAG); in combineStore()
51727 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v16i32, in combineStore()
51729 return DAG.getTruncStore(St->getChain(), dl, Ext, St->getBasePtr(), in combineStore()
51742 VT, St->getMemOperand(), DAG); in combineStore()
51768 return DAG.getTruncStore(St->getChain(), dl, Src, St->getBasePtr(), in combineStore()
51783 St->getMemoryVT(), St->getMemOperand(), DAG); in combineStore()
51785 DAG, dl)) in combineStore()
51788 St->getMemoryVT(), St->getMemOperand(), DAG); in combineStore()
51798 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); in combineStore()
51801 DAG.getAddrSpaceCast(dl, PtrVT, St->getBasePtr(), AddrSpace, 0); in combineStore()
51802 return DAG.getTruncStore( in combineStore()
51818 const Function &F = DAG.getMachineFunction().getFunction(); in combineStore()
51841 SDValue NewLd = DAG.getLoad(MVT::f64, LdDL, Ld->getChain(), in combineStore()
51845 DAG.makeEquivalentMemoryOrdering(Ld, NewLd); in combineStore()
51846 return DAG.getStore(St->getChain(), StDL, NewLd, St->getBasePtr(), in combineStore()
51861 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, VecSize / 64); in combineStore()
51862 SDValue BitCast = DAG.getBitcast(VecVT, ExtOp0); in combineStore()
51863 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in combineStore()
51865 return DAG.getStore(St->getChain(), dl, NewExtract, St->getBasePtr(), in combineStore()
51873 static SDValue combineVEXTRACT_STORE(SDNode *N, SelectionDAG &DAG, in combineVEXTRACT_STORE() argument
51886 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineVEXTRACT_STORE()
51910 SelectionDAG &DAG, const X86Subtarget &Subtarget, in isHorizontalBinOp() argument
51944 if (getTargetShuffleInputs(BC, SrcOps, SrcMask, DAG) && in isHorizontalBinOp()
51957 std::tie(N0, N1) = DAG.SplitVector(SrcOps[0], SDLoc(Op)); in isHorizontalBinOp()
52088 DAG, Subtarget)) in isHorizontalBinOp()
52091 LHS = DAG.getBitcast(VT, NewLHS); in isHorizontalBinOp()
52092 RHS = DAG.getBitcast(VT, NewRHS); in isHorizontalBinOp()
52097 static SDValue combineToHorizontalAddSub(SDNode *N, SelectionDAG &DAG, in combineToHorizontalAddSub() argument
52119 if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd, in combineToHorizontalAddSub()
52121 SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS); in combineToHorizontalAddSub()
52123 HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp, in combineToHorizontalAddSub()
52124 DAG.getUNDEF(VT), PostShuffleMask); in combineToHorizontalAddSub()
52136 if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd, in combineToHorizontalAddSub()
52138 auto HOpBuilder = [HorizOpcode](SelectionDAG &DAG, const SDLoc &DL, in combineToHorizontalAddSub()
52140 return DAG.getNode(HorizOpcode, DL, Ops[0].getValueType(), Ops); in combineToHorizontalAddSub()
52142 SDValue HorizBinOp = SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, in combineToHorizontalAddSub()
52145 HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp, in combineToHorizontalAddSub()
52146 DAG.getUNDEF(VT), PostShuffleMask); in combineToHorizontalAddSub()
52169 static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG, in combineFMulcFCMulc() argument
52180 KnownBits XORRHS = DAG.computeKnownBits(XOR.getOperand(1)); in combineFMulcFCMulc()
52188 SelectionDAG::FlagInserter FlagsInserter(DAG, N); in combineFMulcFCMulc()
52189 SDValue I2F = DAG.getBitcast(VT, LHS.getOperand(0).getOperand(0)); in combineFMulcFCMulc()
52190 SDValue FCMulC = DAG.getNode(CombineOpcode, SDLoc(N), VT, RHS, I2F); in combineFMulcFCMulc()
52191 r = DAG.getBitcast(VT, FCMulC); in combineFMulcFCMulc()
52210 static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG, in combineFaddCFmul() argument
52212 auto AllowContract = [&DAG](const SDNodeFlags &Flags) { in combineFaddCFmul()
52213 return DAG.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast || in combineFaddCFmul()
52217 auto HasNoSignedZero = [&DAG](const SDNodeFlags &Flags) { in combineFaddCFmul()
52218 return DAG.getTarget().Options.NoSignedZerosFPMath || in combineFaddCFmul()
52221 auto IsVectorAllNegativeZero = [&DAG](SDValue Op) { in combineFaddCFmul()
52223 KnownBits Bits = DAG.computeKnownBits(Op); in combineFaddCFmul()
52275 FAddOp1 = DAG.getBitcast(CVT, FAddOp1); in combineFaddCFmul()
52280 DAG.getNode(NewOp, SDLoc(N), CVT, MulOp0, MulOp1, FAddOp1, N->getFlags()); in combineFaddCFmul()
52281 return DAG.getBitcast(VT, CFmul); in combineFaddCFmul()
52285 static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG, in combineFaddFsub() argument
52287 if (SDValue HOp = combineToHorizontalAddSub(N, DAG, Subtarget)) in combineFaddFsub()
52290 if (SDValue COp = combineFaddCFmul(N, DAG, Subtarget)) in combineFaddFsub()
52296 static SDValue combineLRINT_LLRINT(SDNode *N, SelectionDAG &DAG, in combineLRINT_LLRINT() argument
52307 return DAG.getNode(X86ISD::CVTP2SI, DL, VT, in combineLRINT_LLRINT()
52308 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, Src, in combineLRINT_LLRINT()
52309 DAG.getUNDEF(SrcVT))); in combineLRINT_LLRINT()
52317 static SDValue combineTruncatedArithmetic(SDNode *N, SelectionDAG &DAG, in combineTruncatedArithmetic() argument
52323 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineTruncatedArithmetic()
52348 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0); in combineTruncatedArithmetic()
52349 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in combineTruncatedArithmetic()
52350 return DAG.getNode(SrcOpcode, DL, VT, Trunc0, Trunc1); in combineTruncatedArithmetic()
52399 SelectionDAG &DAG, const X86Subtarget &Subtarget) { in combinePMULH() argument
52431 auto IsSext = [&DAG](SDValue V) { in combinePMULH()
52432 return DAG.ComputeMaxSignificantBits(V) <= 16; in combinePMULH()
52434 auto IsZext = [&DAG](SDValue V) { in combinePMULH()
52435 return DAG.computeKnownBits(V).countMaxActiveBits() <= 16; in combinePMULH()
52459 EVT BCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, in combinePMULH()
52461 SDValue Res = DAG.getNode(ISD::MULHU, DL, BCVT, DAG.getBitcast(BCVT, LHS), in combinePMULH()
52462 DAG.getBitcast(BCVT, RHS)); in combinePMULH()
52463 return DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getBitcast(InVT, Res)); in combinePMULH()
52467 LHS = DAG.getNode(ISD::TRUNCATE, DL, VT, LHS); in combinePMULH()
52468 RHS = DAG.getNode(ISD::TRUNCATE, DL, VT, RHS); in combinePMULH()
52471 return DAG.getNode(Opc, DL, VT, LHS, RHS); in combinePMULH()
52482 static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG, in detectPMADDUBSW() argument
52597 auto ExtractVec = [&DAG, &DL, NumElems](SDValue &Ext) { in detectPMADDUBSW()
52601 Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, Ext, in detectPMADDUBSW()
52602 DAG.getIntPtrConstant(0, DL)); in detectPMADDUBSW()
52608 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL, in detectPMADDUBSW()
52616 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, in detectPMADDUBSW()
52618 return DAG.getNode(X86ISD::VPMADDUBSW, DL, ResVT, Ops[0], Ops[1]); in detectPMADDUBSW()
52620 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn }, in detectPMADDUBSW()
52624 static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG, in combineTruncate() argument
52631 if (SDValue V = combineTruncatedArithmetic(N, DAG, Subtarget, DL)) in combineTruncate()
52635 if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL)) in combineTruncate()
52639 if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget)) in combineTruncate()
52643 if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget)) in combineTruncate()
52651 return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc); in combineTruncate()
52657 return DAG.getNode(ISD::LRINT, DL, VT, Src.getOperand(0)); in combineTruncate()
52662 static SDValue combineVTRUNC(SDNode *N, SelectionDAG &DAG, in combineVTRUNC() argument
52669 return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal); in combineVTRUNC()
52670 if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL)) in combineVTRUNC()
52671 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal); in combineVTRUNC()
52673 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineVTRUNC()
52690 static SDValue isFNEG(SelectionDAG &DAG, SDNode *N, unsigned Depth = 0) { in isFNEG() argument
52714 if (SDValue NegOp0 = isFNEG(DAG, Op.getOperand(0).getNode(), Depth + 1)) in isFNEG()
52716 return DAG.getVectorShuffle(VT, SDLoc(Op), NegOp0, DAG.getUNDEF(VT), in isFNEG()
52727 if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode(), Depth + 1)) in isFNEG()
52729 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector, in isFNEG()
52836 static SDValue combineFneg(SDNode *N, SelectionDAG &DAG, in combineFneg() argument
52840 SDValue Arg = isFNEG(DAG, N); in combineFneg()
52844 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineFneg()
52858 SDValue Zero = DAG.getConstantFP(0.0, DL, VT); in combineFneg()
52859 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0), in combineFneg()
52861 return DAG.getBitcast(OrigVT, NewNode); in combineFneg()
52864 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize(); in combineFneg()
52867 TLI.getNegatedExpression(Arg, DAG, LegalOperations, CodeSize)) in combineFneg()
52868 return DAG.getBitcast(OrigVT, NegArg); in combineFneg()
52873 SDValue X86TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, in getNegatedExpression() argument
52879 if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth)) { in getNegatedExpression()
52881 return DAG.getBitcast(Op.getValueType(), Arg); in getNegatedExpression()
52912 Op.getOperand(i), DAG, LegalOperations, ForCodeSize, Depth + 1); in getNegatedExpression()
52926 return DAG.getNode(NewOpc, SDLoc(Op), VT, NewOps); in getNegatedExpression()
52930 getNegatedExpression(Op.getOperand(0), DAG, LegalOperations, in getNegatedExpression()
52932 return DAG.getNode(Opc, SDLoc(Op), VT, NegOp0); in getNegatedExpression()
52936 return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations, in getNegatedExpression()
52940 static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG, in lowerX86FPLogicOp() argument
52953 SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0)); in lowerX86FPLogicOp()
52954 SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1)); in lowerX86FPLogicOp()
52965 SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1); in lowerX86FPLogicOp()
52966 return DAG.getBitcast(VT, IntOp); in lowerX86FPLogicOp()
52971 static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) { in foldXor1SetCC() argument
52982 return getSETCC(NewCC, LHS->getOperand(1), DL, DAG); in foldXor1SetCC()
52985 static SDValue combineXorSubCTLZ(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, in combineXorSubCTLZ() argument
53030 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, OpVT, Op); in combineXorSubCTLZ()
53033 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); in combineXorSubCTLZ()
53034 Op = DAG.getNode(X86ISD::BSR, DL, VTs, Op); in combineXorSubCTLZ()
53036 Op = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Op); in combineXorSubCTLZ()
53041 static SDValue combineXor(SDNode *N, SelectionDAG &DAG, in combineXor() argument
53051 return DAG.getBitcast(MVT::v4i32, in combineXor()
53052 DAG.getNode(X86ISD::FXOR, DL, MVT::v4f32, in combineXor()
53053 DAG.getBitcast(MVT::v4f32, N0), in combineXor()
53054 DAG.getBitcast(MVT::v4f32, N1))); in combineXor()
53057 if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget)) in combineXor()
53060 if (SDValue R = combineBitOpWithMOVMSK(N, DAG)) in combineXor()
53063 if (SDValue R = combineBitOpWithShift(N, DAG)) in combineXor()
53066 if (SDValue R = combineBitOpWithPACK(N, DAG)) in combineXor()
53069 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget)) in combineXor()
53072 if (SDValue R = combineXorSubCTLZ(N, DL, DAG, Subtarget)) in combineXor()
53078 if (SDValue SetCC = foldXor1SetCC(N, DAG)) in combineXor()
53081 if (SDValue R = combineOrXorWithSETCC(N, N0, N1, DAG)) in combineXor()
53084 if (SDValue RV = foldXorTruncShiftIntoCmp(N, DAG)) in combineXor()
53088 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineXor()
53093 return DAG.getBitcast( in combineXor()
53094 VT, DAG.getNOT(DL, N0.getOperand(0), N0.getOperand(0).getValueType())); in combineXor()
53103 return DAG.getNode( in combineXor()
53105 DAG.getNOT(DL, N0.getOperand(1), N0.getOperand(1).getValueType()), in combineXor()
53118 SDValue LHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(0), DL, VT); in combineXor()
53119 SDValue RHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(1), DL, VT); in combineXor()
53120 return DAG.getNode(ISD::XOR, DL, VT, LHS, in combineXor()
53121 DAG.getNode(ISD::XOR, DL, VT, RHS, N1)); in combineXor()
53125 if (SDValue R = combineBMILogicOp(N, DAG, Subtarget)) in combineXor()
53128 return combineFneg(N, DAG, DCI, Subtarget); in combineXor()
53131 static SDValue combineBITREVERSE(SDNode *N, SelectionDAG &DAG, in combineBITREVERSE() argument
53143 DAG.getTargetLoweringInfo().isTypeLegal(SrcVT)) && in combineBITREVERSE()
53150 DAG.getVectorShuffle(SrcVT, SDLoc(N), Src, Src, ReverseMask); in combineBITREVERSE()
53151 return DAG.getBitcast(VT, Rev); in combineBITREVERSE()
53159 static SDValue combineAVG(SDNode *N, SelectionDAG &DAG, in combineAVG() argument
53173 SDValue SignMask = DAG.getConstant(SignBit, DL, VT); in combineAVG()
53174 N0 = DAG.getNode(ISD::XOR, DL, VT, N0, SignMask); in combineAVG()
53175 N1 = DAG.getNode(ISD::XOR, DL, VT, N1, SignMask); in combineAVG()
53176 return DAG.getNode(ISD::XOR, DL, VT, in combineAVG()
53177 DAG.getNode(ISD::AVGCEILU, DL, VT, N0, N1), SignMask); in combineAVG()
53183 static SDValue combineBEXTR(SDNode *N, SelectionDAG &DAG, in combineBEXTR() argument
53192 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineBEXTR()
53210 static SDValue getNullFPConstForNullVal(SDValue V, SelectionDAG &DAG, in getNullFPConstForNullVal() argument
53216 return getZeroVector(V.getSimpleValueType(), Subtarget, DAG, SDLoc(V)); in getNullFPConstForNullVal()
53221 static SDValue combineFAndFNotToFAndn(SDNode *N, SelectionDAG &DAG, in combineFAndFNotToFAndn() argument
53243 return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1); in combineFAndFNotToFAndn()
53247 return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0); in combineFAndFNotToFAndn()
53253 static SDValue combineFAnd(SDNode *N, SelectionDAG &DAG, in combineFAnd() argument
53256 if (SDValue V = getNullFPConstForNullVal(N->getOperand(0), DAG, Subtarget)) in combineFAnd()
53260 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget)) in combineFAnd()
53263 if (SDValue V = combineFAndFNotToFAndn(N, DAG, Subtarget)) in combineFAnd()
53266 return lowerX86FPLogicOp(N, DAG, Subtarget); in combineFAnd()
53270 static SDValue combineFAndn(SDNode *N, SelectionDAG &DAG, in combineFAndn() argument
53277 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget)) in combineFAndn()
53280 return lowerX86FPLogicOp(N, DAG, Subtarget); in combineFAndn()
53284 static SDValue combineFOr(SDNode *N, SelectionDAG &DAG, in combineFOr() argument
53297 if (SDValue NewVal = combineFneg(N, DAG, DCI, Subtarget)) in combineFOr()
53300 return lowerX86FPLogicOp(N, DAG, Subtarget); in combineFOr()
53304 static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG) { in combineFMinFMax() argument
53308 if (!DAG.getTarget().Options.NoNaNsFPMath || in combineFMinFMax()
53309 !DAG.getTarget().Options.NoSignedZerosFPMath) in combineFMinFMax()
53321 return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0), in combineFMinFMax()
53325 static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG, in combineFMinNumFMaxNum() argument
53331 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineFMinNumFMaxNum()
53346 if (DAG.getTarget().Options.NoNaNsFPMath || N->getFlags().hasNoNaNs()) in combineFMinNumFMaxNum()
53347 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags()); in combineFMinNumFMaxNum()
53351 if (DAG.isKnownNeverNaN(Op1)) in combineFMinNumFMaxNum()
53352 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags()); in combineFMinNumFMaxNum()
53353 if (DAG.isKnownNeverNaN(Op0)) in combineFMinNumFMaxNum()
53354 return DAG.getNode(MinMaxOp, DL, VT, Op1, Op0, N->getFlags()); in combineFMinNumFMaxNum()
53358 if (!VT.isVector() && DAG.getMachineFunction().getFunction().hasMinSize()) in combineFMinNumFMaxNum()
53361 EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), in combineFMinNumFMaxNum()
53383 SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0); in combineFMinNumFMaxNum()
53384 SDValue IsOp0Nan = DAG.getSetCC(DL, SetCCType, Op0, Op0, ISD::SETUO); in combineFMinNumFMaxNum()
53388 return DAG.getSelect(DL, VT, IsOp0Nan, Op1, MinOrMax); in combineFMinNumFMaxNum()
53391 static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG, in combineX86INT_TO_FP() argument
53394 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineX86INT_TO_FP()
53410 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) { in combineX86INT_TO_FP()
53412 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT, in combineX86INT_TO_FP()
53413 DAG.getBitcast(InVT, VZLoad)); in combineX86INT_TO_FP()
53415 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1)); in combineX86INT_TO_FP()
53424 static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG, in combineCVTP2I_CVTTP2I() argument
53439 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) { in combineCVTP2I_CVTTP2I()
53443 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other}, in combineCVTP2I_CVTTP2I()
53444 {N->getOperand(0), DAG.getBitcast(InVT, VZLoad)}); in combineCVTP2I_CVTTP2I()
53448 DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(InVT, VZLoad)); in combineCVTP2I_CVTTP2I()
53451 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1)); in combineCVTP2I_CVTTP2I()
53461 static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG, in combineAndnp() argument
53474 return DAG.getConstant(0, DL, VT); in combineAndnp()
53482 return DAG.getConstant(0, DL, VT); in combineAndnp()
53486 return DAG.getNOT(DL, N0, VT); in combineAndnp()
53489 if (SDValue Not = IsNOT(N0, DAG)) in combineAndnp()
53490 return DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, Not), N1); in combineAndnp()
53495 if (SDValue Not = IsNOT(N1, DAG)) in combineAndnp()
53496 return DAG.getNOT( in combineAndnp()
53497 DL, DAG.getNode(ISD::OR, DL, VT, N0, DAG.getBitcast(VT, Not)), VT); in combineAndnp()
53511 return getConstVector(ResultBits, VT, DAG, DL); in combineAndnp()
53522 SDValue Not = getConstVector(EltBits0, VT, DAG, DL); in combineAndnp()
53523 return DAG.getNode(ISD::AND, DL, VT, Not, N1); in combineAndnp()
53531 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineAndnp()
53565 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineAndnp()
53579 static SDValue combineBT(SDNode *N, SelectionDAG &DAG, in combineBT() argument
53586 if (DAG.getTargetLoweringInfo().SimplifyDemandedBits(N1, DemandedMask, DCI)) { in combineBT()
53595 static SDValue combineCVTPH2PS(SDNode *N, SelectionDAG &DAG, in combineCVTPH2PS() argument
53601 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineCVTPH2PS()
53612 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::i64, MVT::v2i64, DAG)) { in combineCVTPH2PS()
53615 SDValue Convert = DAG.getNode( in combineCVTPH2PS()
53617 {N->getOperand(0), DAG.getBitcast(MVT::v8i16, VZLoad)}); in combineCVTPH2PS()
53620 SDValue Convert = DAG.getNode(N->getOpcode(), dl, MVT::v4f32, in combineCVTPH2PS()
53621 DAG.getBitcast(MVT::v8i16, VZLoad)); in combineCVTPH2PS()
53625 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1)); in combineCVTPH2PS()
53636 static SDValue combineSextInRegCmov(SDNode *N, SelectionDAG &DAG) { in combineSextInRegCmov() argument
53673 CMovOp0 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp0); in combineSextInRegCmov()
53674 CMovOp1 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp1); in combineSextInRegCmov()
53677 CMovOp0 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp0, N1); in combineSextInRegCmov()
53678 CMovOp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp1, N1); in combineSextInRegCmov()
53684 CMovOp0 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp0); in combineSextInRegCmov()
53685 CMovOp1 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp1); in combineSextInRegCmov()
53688 SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, CMovVT, CMovOp0, CMovOp1, in combineSextInRegCmov()
53692 CMov = DAG.getNode(ISD::TRUNCATE, DL, DstVT, CMov); in combineSextInRegCmov()
53697 static SDValue combineSignExtendInReg(SDNode *N, SelectionDAG &DAG, in combineSignExtendInReg() argument
53701 if (SDValue V = combineSextInRegCmov(N, DAG)) in combineSignExtendInReg()
53727 if (SDValue Promote = PromoteMaskArithmetic(N0, dl, DAG, Subtarget)) in combineSignExtendInReg()
53728 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Promote, N1); in combineSignExtendInReg()
53732 DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, N00, N1); in combineSignExtendInReg()
53733 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp); in combineSignExtendInReg()
53744 static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG, in promoteExtBeforeAdd() argument
53764 NSW = NSW || (Sext && DAG.willNotOverflowAdd(true, AddOp0, AddOp1)); in promoteExtBeforeAdd()
53765 NUW = NUW || (!Sext && DAG.willNotOverflowAdd(false, AddOp0, AddOp1)); in promoteExtBeforeAdd()
53796 SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0); in promoteExtBeforeAdd()
53797 SDValue NewConstant = DAG.getConstant(AddC, SDLoc(Add), VT); in promoteExtBeforeAdd()
53804 return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, Flags); in promoteExtBeforeAdd()
53819 static SDValue combineToExtendCMOV(SDNode *Extend, SelectionDAG &DAG) { in combineToExtendCMOV() argument
53851 CMovOp0 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp0); in combineToExtendCMOV()
53852 CMovOp1 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp1); in combineToExtendCMOV()
53854 SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1, in combineToExtendCMOV()
53859 Res = DAG.getNode(ExtendOpcode, DL, TargetVT, Res); in combineToExtendCMOV()
53866 static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG, in combineExtSetcc() argument
53902 SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC); in combineExtSetcc()
53905 Res = DAG.getZeroExtendInReg(Res, dl, N0.getValueType()); in combineExtSetcc()
53910 static SDValue combineSext(SDNode *N, SelectionDAG &DAG, in combineSext() argument
53920 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0), in combineSext()
53926 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), in combineSext()
53934 if (SDValue NewCMov = combineToExtendCMOV(N, DAG)) in combineSext()
53940 if (SDValue V = combineExtSetcc(N, DAG, Subtarget)) in combineSext()
53944 DAG, DCI, Subtarget)) in combineSext()
53948 if (SDValue R = PromoteMaskArithmetic(SDValue(N, 0), DL, DAG, Subtarget)) in combineSext()
53952 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0)); in combineSext()
53955 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget)) in combineSext()
53967 static SDValue getInvertedVectorForFMA(SDValue V, SelectionDAG &DAG) { in getInvertedVectorForFMA() argument
53984 Ops.push_back(DAG.getConstantFP(-Cst->getValueAPF(), SDLoc(Op), EltVT)); in getInvertedVectorForFMA()
53987 Ops.push_back(DAG.getUNDEF(EltVT)); in getInvertedVectorForFMA()
53991 SDNode *NV = DAG.getNodeIfExists(ISD::BUILD_VECTOR, DAG.getVTList(VT), Ops); in getInvertedVectorForFMA()
54014 static SDValue combineFMA(SDNode *N, SelectionDAG &DAG, in combineFMA() argument
54022 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineFMA()
54035 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, VT, A, B, Flags); in combineFMA()
54036 return DAG.getNode(ISD::FADD, dl, VT, Fmul, C, Flags); in combineFMA()
54045 auto invertIfNegative = [&DAG, &TLI, &DCI](SDValue &V) { in combineFMA()
54046 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize(); in combineFMA()
54048 if (SDValue NegV = TLI.getCheaperNegatedExpression(V, DAG, LegalOperations, in combineFMA()
54059 Vec, DAG, LegalOperations, CodeSize)) { in combineFMA()
54060 V = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), V.getValueType(), in combineFMA()
54067 if (SDValue NegV = getInvertedVectorForFMA(V, DAG)) { in combineFMA()
54088 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); in combineFMA()
54091 return DAG.getNode(NewOpcode, dl, {VT, MVT::Other}, in combineFMA()
54095 return DAG.getNode(NewOpcode, dl, VT, A, B, C, N->getOperand(3)); in combineFMA()
54096 return DAG.getNode(NewOpcode, dl, VT, A, B, C); in combineFMA()
54102 static SDValue combineFMADDSUB(SDNode *N, SelectionDAG &DAG, in combineFMADDSUB() argument
54106 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineFMADDSUB()
54107 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize(); in combineFMADDSUB()
54113 TLI.getCheaperNegatedExpression(N2, DAG, LegalOperations, CodeSize); in combineFMADDSUB()
54119 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1), in combineFMADDSUB()
54121 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1), in combineFMADDSUB()
54125 static SDValue combineZext(SDNode *N, SelectionDAG &DAG, in combineZext() argument
54136 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0), in combineZext()
54142 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), in combineZext()
54150 if (SDValue NewCMov = combineToExtendCMOV(N, DAG)) in combineZext()
54154 if (SDValue V = combineExtSetcc(N, DAG, Subtarget)) in combineZext()
54158 DAG, DCI, Subtarget)) in combineZext()
54162 if (SDValue R = PromoteMaskArithmetic(SDValue(N, 0), dl, DAG, Subtarget)) in combineZext()
54165 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget)) in combineZext()
54168 if (SDValue R = combineOrCmpEqZeroToCtlzSrl(N, DAG, DCI, Subtarget)) in combineZext()
54178 if ((N00.isUndef() || DAG.MaskedValueIsZero(N00, ZeroMask)) && in combineZext()
54179 (N01.isUndef() || DAG.MaskedValueIsZero(N01, ZeroMask))) { in combineZext()
54180 return concatSubVectors(N00, N01, DAG, dl); in combineZext()
54192 const SDLoc &DL, SelectionDAG &DAG, in truncateAVX512SetCCNoBWI() argument
54198 SDValue Setcc = DAG.getSetCC(DL, OpVT, LHS, RHS, CC); in truncateAVX512SetCCNoBWI()
54199 return DAG.getNode(ISD::TRUNCATE, DL, VT, Setcc); in truncateAVX512SetCCNoBWI()
54204 static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG, in combineSetCC() argument
54215 if (SDValue V = combineVectorSizedSetCCEquality(VT, LHS, RHS, CC, DL, DAG, in combineSetCC()
54222 MatchVectorAllEqualTest(LHS, RHS, CC, DL, Subtarget, DAG, X86CC)) in combineSetCC()
54223 return DAG.getNode(ISD::TRUNCATE, DL, VT, getSETCC(X86CC, V, DL, DAG)); in combineSetCC()
54232 return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT), in combineSetCC()
54235 return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT), in combineSetCC()
54241 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
54243 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
54250 return DAG.getNode(ISD::AND, DL, OpVT, N1, in combineSetCC()
54251 DAG.getNOT(DL, N0.getOperand(1), OpVT)); in combineSetCC()
54253 return DAG.getNode(ISD::AND, DL, OpVT, N1, in combineSetCC()
54254 DAG.getNOT(DL, N0.getOperand(0), OpVT)); in combineSetCC()
54259 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
54261 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
54272 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineSetCC()
54273 if (DAG.MaskedValueIsZero(LHS.getOperand(0), UpperBits) && in combineSetCC()
54275 return DAG.getSetCC(DL, VT, LHS.getOperand(0), in combineSetCC()
54276 DAG.getZExtOrTrunc(RHS, DL, SrcVT), CC); in combineSetCC()
54293 SDValue SETCC0 = DAG.getSetCC(DL, VT, BaseOp, RHS, CC); in combineSetCC()
54294 SDValue SETCC1 = DAG.getSetCC( in combineSetCC()
54295 DL, VT, BaseOp, DAG.getConstant(-CInt, DL, OpVT), CC); in combineSetCC()
54296 return DAG.getNode(CC == ISD::SETEQ ? ISD::OR : ISD::AND, DL, VT, in combineSetCC()
54326 return DAG.getConstant(0, DL, VT); in combineSetCC()
54328 return DAG.getConstant(1, DL, VT); in combineSetCC()
54330 return DAG.getNOT(DL, Op0.getOperand(0), VT); in combineSetCC()
54346 DAG.computeKnownBits(LHS).intersectWith(DAG.computeKnownBits(RHS)); in combineSetCC()
54365 if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ true, in combineSetCC()
54369 RHS, DAG, /*IsInc*/ false, /*NSW*/ true)) in combineSetCC()
54381 if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ false, in combineSetCC()
54384 else if (SDValue NewRHS = incDecVectorConstant(RHS, DAG, /*IsInc*/ true, in combineSetCC()
54400 NewCC, DL, DAG, Subtarget)) in combineSetCC()
54402 return DAG.getSetCC(DL, VT, LHSOut, RHSOut, NewCC); in combineSetCC()
54408 truncateAVX512SetCCNoBWI(VT, OpVT, LHS, RHS, CC, DL, DAG, Subtarget)) in combineSetCC()
54435 DAG.isConstantIntBuildVectorOrConstantInt(AddC)) { in combineSetCC()
54445 DAG.getTargetLoweringInfo().isOperationLegal(ISD::UMIN, OpVT)) { in combineSetCC()
54452 C0 = DAG.getNegative(AddC, DL, OpVT); in combineSetCC()
54453 C1 = DAG.getNode(ISD::SUB, DL, OpVT, C0, in combineSetCC()
54454 DAG.getAllOnesConstant(DL, OpVT)); in combineSetCC()
54460 C0 = DAG.getNOT(DL, AddC, OpVT); in combineSetCC()
54461 C1 = DAG.getNode(ISD::ADD, DL, OpVT, C0, in combineSetCC()
54462 DAG.getAllOnesConstant(DL, OpVT)); in combineSetCC()
54466 DAG.getSetCC(DL, VT, LHS.getOperand(0), C0, ISD::SETEQ); in combineSetCC()
54468 DAG.getSetCC(DL, VT, LHS.getOperand(0), C1, ISD::SETEQ); in combineSetCC()
54469 return DAG.getNode(ISD::OR, DL, VT, NewLHS, NewRHS); in combineSetCC()
54478 return LowerVSETCC(SDValue(N, 0), Subtarget, DAG); in combineSetCC()
54485 SDVTList FNegVT = DAG.getVTList(OpVT); in combineSetCC()
54486 if (SDNode *FNeg = DAG.getNodeIfExists(ISD::FNEG, FNegVT, {LHS})) in combineSetCC()
54487 return DAG.getSetCC(DL, VT, LHS, SDValue(FNeg, 0), CC); in combineSetCC()
54493 static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG, in combineMOVMSK() argument
54515 return DAG.getConstant(Imm, SDLoc(N), VT); in combineMOVMSK()
54522 return DAG.getNode(X86ISD::MOVMSK, SDLoc(N), VT, Src.getOperand(0)); in combineMOVMSK()
54526 if (SDValue NotSrc = IsNOT(Src, DAG)) { in combineMOVMSK()
54529 NotSrc = DAG.getBitcast(SrcVT, NotSrc); in combineMOVMSK()
54530 return DAG.getNode(ISD::XOR, DL, VT, in combineMOVMSK()
54531 DAG.getNode(X86ISD::MOVMSK, DL, VT, NotSrc), in combineMOVMSK()
54532 DAG.getConstant(NotMask, DL, VT)); in combineMOVMSK()
54541 return DAG.getNode(ISD::XOR, DL, VT, in combineMOVMSK()
54542 DAG.getNode(X86ISD::MOVMSK, DL, VT, Src.getOperand(0)), in combineMOVMSK()
54543 DAG.getConstant(NotMask, DL, VT)); in combineMOVMSK()
54552 KnownBits KnownLHS = DAG.computeKnownBits(Src.getOperand(0)); in combineMOVMSK()
54553 KnownBits KnownRHS = DAG.computeKnownBits(Src.getOperand(1)); in combineMOVMSK()
54565 ShiftLHS = DAG.getBitcast(ShiftVT, ShiftLHS); in combineMOVMSK()
54566 ShiftRHS = DAG.getBitcast(ShiftVT, ShiftRHS); in combineMOVMSK()
54569 ShiftLHS, ShiftAmt, DAG); in combineMOVMSK()
54571 ShiftRHS, ShiftAmt, DAG); in combineMOVMSK()
54572 ShiftLHS = DAG.getBitcast(SrcVT, ShiftLHS); in combineMOVMSK()
54573 ShiftRHS = DAG.getBitcast(SrcVT, ShiftRHS); in combineMOVMSK()
54574 SDValue Res = DAG.getNode(ISD::XOR, DL, SrcVT, ShiftLHS, ShiftRHS); in combineMOVMSK()
54575 return DAG.getNode(X86ISD::MOVMSK, DL, VT, DAG.getNOT(DL, Res, SrcVT)); in combineMOVMSK()
54593 SDValue NewSrc = DAG.getBitcast(SrcVT, SrcBC.getOperand(0)); in combineMOVMSK()
54594 SDValue NewMovMsk = DAG.getNode(X86ISD::MOVMSK, DL, VT, NewSrc); in combineMOVMSK()
54595 return DAG.getNode(SrcBC.getOpcode(), DL, VT, NewMovMsk, in combineMOVMSK()
54596 DAG.getConstant(Mask, DL, VT)); in combineMOVMSK()
54602 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineMOVMSK()
54610 static SDValue combineTESTP(SDNode *N, SelectionDAG &DAG, in combineTESTP() argument
54617 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineTESTP()
54625 static SDValue combineX86GatherScatter(SDNode *N, SelectionDAG &DAG, in combineX86GatherScatter() argument
54632 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineX86GatherScatter()
54646 SelectionDAG &DAG) { in rebuildGatherScatter() argument
54652 return DAG.getMaskedGather(Gather->getVTList(), in rebuildGatherScatter()
54661 return DAG.getMaskedScatter(Scatter->getVTList(), in rebuildGatherScatter()
54668 static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG, in combineGatherScatter() argument
54675 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineGatherScatter()
54689 DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) { in combineGatherScatter()
54691 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index); in combineGatherScatter()
54692 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG); in combineGatherScatter()
54703 DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) { in combineGatherScatter()
54705 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index); in combineGatherScatter()
54706 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG); in combineGatherScatter()
54710 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); in combineGatherScatter()
54727 Base = DAG.getNode(ISD::ADD, DL, PtrVT, Base, in combineGatherScatter()
54728 DAG.getConstant(Adder, DL, PtrVT)); in combineGatherScatter()
54730 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG); in combineGatherScatter()
54738 SDValue Splat = DAG.getSplatBuildVector(Index.getValueType(), DL, Base); in combineGatherScatter()
54740 Splat = DAG.getNode(ISD::ADD, DL, Index.getValueType(), in combineGatherScatter()
54743 Index = DAG.getNode(ISD::ADD, DL, Index.getValueType(), in combineGatherScatter()
54745 Base = DAG.getConstant(0, DL, Base.getValueType()); in combineGatherScatter()
54746 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG); in combineGatherScatter()
54758 Index = DAG.getSExtOrTrunc(Index, DL, IndexVT); in combineGatherScatter()
54759 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG); in combineGatherScatter()
54778 static SDValue combineX86SetCC(SDNode *N, SelectionDAG &DAG, in combineX86SetCC() argument
54785 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) in combineX86SetCC()
54786 return getSETCC(CC, Flags, DL, DAG); in combineX86SetCC()
54792 static SDValue combineBrCond(SDNode *N, SelectionDAG &DAG, in combineBrCond() argument
54801 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) { in combineBrCond()
54802 SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8); in combineBrCond()
54803 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0), in combineBrCond()
54812 SelectionDAG &DAG) { in combineVectorCompareAndMaskUnaryOp() argument
54829 DAG.ComputeNumSignBits(Op0.getOperand(0)) != NumEltBits || in combineVectorCompareAndMaskUnaryOp()
54849 SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other}, in combineVectorCompareAndMaskUnaryOp()
54852 SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0)); in combineVectorCompareAndMaskUnaryOp()
54854 SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst); in combineVectorCompareAndMaskUnaryOp()
54855 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0), in combineVectorCompareAndMaskUnaryOp()
54857 SDValue Res = DAG.getBitcast(VT, NewAnd); in combineVectorCompareAndMaskUnaryOp()
54859 return DAG.getMergeValues({Res, SourceConst.getValue(1)}, DL); in combineVectorCompareAndMaskUnaryOp()
54869 static SDValue combineToFPTruncExtElt(SDNode *N, SelectionDAG &DAG) { in combineToFPTruncExtElt() argument
54893 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), TruncVT, NumElts); in combineToFPTruncExtElt()
54894 SDValue BitcastVec = DAG.getBitcast(BitcastVT, ExtElt.getOperand(0)); in combineToFPTruncExtElt()
54896 SDValue NewExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TruncVT, in combineToFPTruncExtElt()
54898 return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), NewExtElt); in combineToFPTruncExtElt()
54901 static SDValue combineUIntToFP(SDNode *N, SelectionDAG &DAG, in combineUIntToFP() argument
54923 EVT::getVectorVT(*DAG.getContext(), in combineUIntToFP()
54928 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0); in combineUIntToFP()
54930 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineUIntToFP()
54932 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineUIntToFP()
54942 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0); in combineUIntToFP()
54946 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineUIntToFP()
54948 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineUIntToFP()
54955 if (Flags.hasNonNeg() || DAG.SignBitIsZero(Op0)) { in combineUIntToFP()
54957 return DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(N), {VT, MVT::Other}, in combineUIntToFP()
54959 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0); in combineUIntToFP()
54965 static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG, in combineSIntToFP() argument
54971 if (SDValue Res = combineVectorCompareAndMaskUnaryOp(N, DAG)) in combineSIntToFP()
54994 EVT::getVectorVT(*DAG.getContext(), in combineSIntToFP()
54999 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0); in combineSIntToFP()
55001 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
55003 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineSIntToFP()
55013 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0); in combineSIntToFP()
55015 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
55017 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineSIntToFP()
55025 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0); in combineSIntToFP()
55032 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Op0); in combineSIntToFP()
55034 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
55036 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Trunc); in combineSIntToFP()
55041 SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0); in combineSIntToFP()
55042 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast, in combineSIntToFP()
55045 return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other}, in combineSIntToFP()
55047 return DAG.getNode(X86ISD::CVTSI2P, dl, VT, Shuf); in combineSIntToFP()
55071 Ld->getPointerInfo(), Ld->getOriginalAlign(), DAG); in combineSIntToFP()
55072 DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), Tmp.second); in combineSIntToFP()
55080 if (SDValue V = combineToFPTruncExtElt(N, DAG)) in combineSIntToFP()
55148 static SDValue combineCMP(SDNode *N, SelectionDAG &DAG, in combineCMP() argument
55162 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineCMP()
55165 combineX86SubCmpForFlags(N, SDValue(N, 0), DAG, DCI, Subtarget)) in combineCMP()
55182 Op = DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), in combineCMP()
55183 DAG.getConstant(Mask, dl, VT)); in combineCMP()
55184 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in combineCMP()
55185 DAG.getConstant(0, dl, VT)); in combineCMP()
55205 BoolVec = widenMaskVector(BoolVec, false, Subtarget, DAG, dl); in combineCMP()
55208 EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), BitWidth); in combineCMP()
55211 Op = DAG.getBitcast(BCVT, BoolVec); in combineCMP()
55212 Op = DAG.getNode(ISD::AND, dl, BCVT, Op, in combineCMP()
55213 DAG.getConstant(Mask, dl, BCVT)); in combineCMP()
55214 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in combineCMP()
55215 DAG.getConstant(0, dl, BCVT)); in combineCMP()
55225 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Src, in combineCMP()
55226 DAG.getConstant(0, dl, SrcVT)); in combineCMP()
55242 if (OpVT == MVT::i32 && DAG.MaskedValueIsZero(Op, UpperBits) && in combineCMP()
55244 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in combineCMP()
55245 DAG.getConstant(0, dl, OpVT)); in combineCMP()
55279 SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0)); in combineCMP()
55280 SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1)); in combineCMP()
55283 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineCMP()
55284 Op = DAG.getNode(NewOpc, dl, VTs, Op0, Op1); in combineCMP()
55288 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in combineCMP()
55289 DAG.getConstant(0, dl, VT)); in combineCMP()
55295 static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG, in combineX86AddSub() argument
55309 if (SDValue CMP = combineX86SubCmpForFlags(N, SDValue(N, 1), DAG, DCI, ST)) in combineX86AddSub()
55314 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS); in combineX86AddSub()
55315 return DAG.getMergeValues({Res, DAG.getConstant(0, DL, MVT::i32)}, DL); in combineX86AddSub()
55321 SDVTList VTs = DAG.getVTList(N->getValueType(0)); in combineX86AddSub()
55322 if (SDNode *GenericAddSub = DAG.getNodeIfExists(GenericOpc, VTs, Ops)) { in combineX86AddSub()
55325 Op = DAG.getNegative(Op, DL, VT); in combineX86AddSub()
55334 return combineAddOrSubToADCOrSBB(IsSub, DL, VT, LHS, RHS, DAG, in combineX86AddSub()
55338 static SDValue combineSBB(SDNode *N, SelectionDAG &DAG) { in combineSBB() argument
55343 if (SDValue Flags = combineCarryThroughADD(BorrowIn, DAG)) { in combineSBB()
55345 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineSBB()
55346 return DAG.getNode(X86ISD::SBB, SDLoc(N), VTs, LHS, RHS, Flags); in combineSBB()
55353 return DAG.getNode(X86ISD::SBB, SDLoc(N), N->getVTList(), LHS.getOperand(0), in combineSBB()
55360 static SDValue combineADC(SDNode *N, SelectionDAG &DAG, in combineADC() argument
55370 return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), RHS, LHS, in combineADC()
55382 SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1)); in combineADC()
55383 SDValue Res1 = DAG.getNode( in combineADC()
55385 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineADC()
55386 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), CarryIn), in combineADC()
55387 DAG.getConstant(1, DL, VT)); in combineADC()
55397 return DAG.getNode(X86ISD::ADC, DL, N->getVTList(), in combineADC()
55398 DAG.getConstant(0, DL, LHS.getValueType()), in combineADC()
55399 DAG.getConstant(Sum, DL, LHS.getValueType()), CarryIn); in combineADC()
55402 if (SDValue Flags = combineCarryThroughADD(CarryIn, DAG)) { in combineADC()
55404 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineADC()
55405 return DAG.getNode(X86ISD::ADC, SDLoc(N), VTs, LHS, RHS, Flags); in combineADC()
55412 return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), LHS.getOperand(0), in combineADC()
55418 static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1, in matchPMADDWD() argument
55503 if (!canReduceVMulWidth(Mul.getNode(), DAG, Mode) || in matchPMADDWD()
55507 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, in matchPMADDWD()
55509 SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(0)); in matchPMADDWD()
55510 SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(1)); in matchPMADDWD()
55512 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL, in matchPMADDWD()
55516 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in matchPMADDWD()
55518 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]); in matchPMADDWD()
55520 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { N0, N1 }, PMADDBuilder); in matchPMADDWD()
55526 static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1, in matchPMADDWD_2() argument
55635 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL, in matchPMADDWD_2()
55641 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in matchPMADDWD_2()
55643 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]); in matchPMADDWD_2()
55648 EVT OutVT16 = EVT::getVectorVT(*DAG.getContext(), MVT::i16, in matchPMADDWD_2()
55651 In0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In0, in matchPMADDWD_2()
55652 DAG.getIntPtrConstant(0, DL)); in matchPMADDWD_2()
55655 In1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In1, in matchPMADDWD_2()
55656 DAG.getIntPtrConstant(0, DL)); in matchPMADDWD_2()
55658 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 }, in matchPMADDWD_2()
55666 static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1, in combineAddOfPMADDWD() argument
55681 DAG.MaskedValueIsZero(N0.getOperand(0), DemandedBits, DemandedHiElts) || in combineAddOfPMADDWD()
55682 DAG.MaskedValueIsZero(N0.getOperand(1), DemandedBits, DemandedHiElts); in combineAddOfPMADDWD()
55684 DAG.MaskedValueIsZero(N1.getOperand(0), DemandedBits, DemandedHiElts) || in combineAddOfPMADDWD()
55685 DAG.MaskedValueIsZero(N1.getOperand(1), DemandedBits, DemandedHiElts); in combineAddOfPMADDWD()
55700 DAG.getVectorShuffle(OpVT, DL, N0.getOperand(0), N1.getOperand(0), Mask); in combineAddOfPMADDWD()
55702 DAG.getVectorShuffle(OpVT, DL, N0.getOperand(1), N1.getOperand(1), Mask); in combineAddOfPMADDWD()
55703 return DAG.getNode(X86ISD::VPMADDWD, DL, VT, LHS, RHS); in combineAddOfPMADDWD()
55711 SelectionDAG &DAG, in pushAddIntoCmovOfConsts() argument
55761 FalseOp = DAG.getNode(ISD::ADD, DL, VT, X, FalseOp); in pushAddIntoCmovOfConsts()
55762 TrueOp = DAG.getNode(ISD::ADD, DL, VT, X, TrueOp); in pushAddIntoCmovOfConsts()
55763 Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, in pushAddIntoCmovOfConsts()
55765 return DAG.getNode(ISD::ADD, DL, VT, Cmov, Y); in pushAddIntoCmovOfConsts()
55769 FalseOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, FalseOp); in pushAddIntoCmovOfConsts()
55770 TrueOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, TrueOp); in pushAddIntoCmovOfConsts()
55771 return DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, Cmov.getOperand(2), in pushAddIntoCmovOfConsts()
55775 static SDValue combineAdd(SDNode *N, SelectionDAG &DAG, in combineAdd() argument
55783 if (SDValue Select = pushAddIntoCmovOfConsts(N, DL, DAG, Subtarget)) in combineAdd()
55786 if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, DL, VT, Subtarget)) in combineAdd()
55788 if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, DL, VT, Subtarget)) in combineAdd()
55790 if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT)) in combineAdd()
55794 if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget)) in combineAdd()
55802 if (DAG.willNotOverflowAdd(false, Op0.getOperand(0), Op1.getOperand(0))) { in combineAdd()
55805 DAG.getNode(ISD::ADD, DL, OpVT, Op0.getOperand(0), Op1.getOperand(0)); in combineAdd()
55806 return DAG.getNode(X86ISD::PSADBW, DL, VT, Sum, in combineAdd()
55807 getZeroVector(OpVT, Subtarget, DAG, DL)); in combineAdd()
55817 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineAdd()
55821 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0)); in combineAdd()
55822 return DAG.getNode(ISD::SUB, DL, VT, Op1, SExt); in combineAdd()
55828 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0)); in combineAdd()
55829 return DAG.getNode(ISD::SUB, DL, VT, Op0, SExt); in combineAdd()
55837 return DAG.getNode(X86ISD::ADC, SDLoc(Op0), Op0->getVTList(), Op1, in combineAdd()
55841 return combineAddOrSubToADCOrSBB(N, DL, DAG); in combineAdd()
55848 static SDValue combineSubABS(SDNode *N, SelectionDAG &DAG) { in combineSubABS() argument
55879 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, TrueOp, FalseOp, in combineSubABS()
55882 return DAG.getNode(ISD::ADD, DL, VT, N0, Cmov); in combineSubABS()
55885 static SDValue combineSubSetcc(SDNode *N, SelectionDAG &DAG) { in combineSubSetcc() argument
55903 SDValue NewSetCC = getSETCC(NewCC, SetCC.getOperand(1), DL, DAG); in combineSubSetcc()
55904 NewSetCC = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NewSetCC); in combineSubSetcc()
55905 return DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(VT, VT), NewSetCC, in combineSubSetcc()
55906 DAG.getConstant(NewImm, DL, VT)); in combineSubSetcc()
55912 static SDValue combineX86CloadCstore(SDNode *N, SelectionDAG &DAG) { in combineX86CloadCstore() argument
55933 return DAG.getMemIntrinsicNode(N->getOpcode(), SDLoc(N), N->getVTList(), Ops, in combineX86CloadCstore()
55938 static SDValue combineSub(SDNode *N, SelectionDAG &DAG, in combineSub() argument
55947 if (SDNode *C = DAG.isConstantIntBuildVectorOrConstantInt(Op)) { in combineSub()
55964 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT, Op1.getOperand(0), in combineSub()
55965 DAG.getNOT(SDLoc(Op1), Op1.getOperand(1), VT)); in combineSub()
55967 DAG.getNode(ISD::ADD, DL, VT, Op0, DAG.getConstant(1, DL, VT)); in combineSub()
55968 return DAG.getNode(ISD::ADD, DL, VT, NewXor, NewAdd); in combineSub()
55971 if (SDValue V = combineSubABS(N, DAG)) in combineSub()
55975 if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget)) in combineSub()
55982 return DAG.getNode(X86ISD::SBB, SDLoc(Op1), Op1->getVTList(), Op0, in combineSub()
55991 SDValue ADC = DAG.getNode(X86ISD::ADC, SDLoc(Op1), Op1->getVTList(), Op0, in combineSub()
55993 return DAG.getNode(ISD::SUB, DL, Op0.getValueType(), ADC.getValue(0), in combineSub()
55997 if (SDValue V = combineXorSubCTLZ(N, DL, DAG, Subtarget)) in combineSub()
56000 if (SDValue V = combineAddOrSubToADCOrSBB(N, DL, DAG)) in combineSub()
56003 return combineSubSetcc(N, DAG); in combineSub()
56006 static SDValue combineVectorCompare(SDNode *N, SelectionDAG &DAG, in combineVectorCompare() argument
56020 return (Opcode == X86ISD::PCMPEQ) ? DAG.getAllOnesConstant(DL, VT) in combineVectorCompare()
56021 : DAG.getConstant(0, DL, VT); in combineVectorCompare()
56043 return getConstVector(Results, LHSUndefs | RHSUndefs, VT, DAG, DL); in combineVectorCompare()
56044 return getConstVector(Results, VT, DAG, DL); in combineVectorCompare()
56076 ArrayRef<SDValue> Ops, SelectionDAG &DAG, in combineConcatVectorOps() argument
56083 return DAG.getUNDEF(VT); in combineConcatVectorOps()
56088 return getZeroVector(VT, Subtarget, DAG, DL); in combineConcatVectorOps()
56093 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineConcatVectorOps()
56094 LLVMContext &Ctx = *DAG.getContext(); in combineConcatVectorOps()
56101 return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0)); in combineConcatVectorOps()
56108 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineConcatVectorOps()
56109 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f64, in combineConcatVectorOps()
56111 DAG.getIntPtrConstant(0, DL))); in combineConcatVectorOps()
56119 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Op0.getOperand(0)); in combineConcatVectorOps()
56139 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56140 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, in combineConcatVectorOps()
56160 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, in combineConcatVectorOps()
56161 DAG.getBitcast(VT, Src0.getOperand(0)), in combineConcatVectorOps()
56162 DAG.getBitcast(VT, Src1.getOperand(0)), in combineConcatVectorOps()
56163 DAG.getTargetConstant(0x31, DL, MVT::i8)); in combineConcatVectorOps()
56182 EVT::getVectorVT(*DAG.getContext(), SubVT.getScalarType(), in combineConcatVectorOps()
56185 Sub = DAG.getBitcast(SubVT, Sub); in combineConcatVectorOps()
56186 return DAG.getBitcast( in combineConcatVectorOps()
56187 VT, DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Subs)); in combineConcatVectorOps()
56189 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs); in combineConcatVectorOps()
56213 return DAG.getNode(X86ISD::UNPCKL, DL, VT, in combineConcatVectorOps()
56218 return DAG.getNode(VT == MVT::v8f32 ? X86ISD::VPERMILPI in combineConcatVectorOps()
56221 getV4X86ShuffleImm8ForMask({0, 0, 0, 0}, DL, DAG)); in combineConcatVectorOps()
56229 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56239 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56257 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56268 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56280 SDValue Res = DAG.getBitcast(FloatVT, ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
56282 DAG.getNode(X86ISD::VPERMILPI, DL, FloatVT, Res, Op0.getOperand(1)); in combineConcatVectorOps()
56283 return DAG.getBitcast(VT, Res); in combineConcatVectorOps()
56289 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56291 DAG.getTargetConstant(Idx, DL, MVT::i8)); in combineConcatVectorOps()
56303 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56327 Ops[1].getOperand(1), DAG, DL); in combineConcatVectorOps()
56330 SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true); in combineConcatVectorOps()
56331 return DAG.getNode(X86ISD::VPERMV, DL, VT, Mask, Src); in combineConcatVectorOps()
56355 Ops[1].getOperand(0), DAG, DL); in combineConcatVectorOps()
56357 Ops[1].getOperand(2), DAG, DL); in combineConcatVectorOps()
56360 SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true); in combineConcatVectorOps()
56361 return DAG.getNode(X86ISD::VPERMV3, DL, VT, Src0, Mask, Src1); in combineConcatVectorOps()
56376 Ops[0].getOperand(1), DAG, DL); in combineConcatVectorOps()
56378 Ops[1].getOperand(1), DAG, DL); in combineConcatVectorOps()
56379 SDValue Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT, in combineConcatVectorOps()
56380 DAG.getBitcast(ShuffleVT, LHS), in combineConcatVectorOps()
56381 DAG.getBitcast(ShuffleVT, RHS), in combineConcatVectorOps()
56382 getV4X86ShuffleImm8ForMask(Mask, DL, DAG)); in combineConcatVectorOps()
56383 return DAG.getBitcast(VT, Res); in combineConcatVectorOps()
56395 Ops[0].getOperand(1), DAG, DL); in combineConcatVectorOps()
56397 Ops[1].getOperand(1), DAG, DL); in combineConcatVectorOps()
56398 return DAG.getNode(X86ISD::SHUF128, DL, VT, LHS, RHS, in combineConcatVectorOps()
56399 DAG.getTargetConstant(Imm, DL, MVT::i8)); in combineConcatVectorOps()
56412 return DAG.getNode(ISD::TRUNCATE, DL, VT, in combineConcatVectorOps()
56425 SDValue Res = DAG.getBitcast(MVT::v8i32, ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
56426 SDValue Zero = getZeroVector(MVT::v8i32, Subtarget, DAG, DL); in combineConcatVectorOps()
56428 Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero, in combineConcatVectorOps()
56431 Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero, in combineConcatVectorOps()
56434 return DAG.getBitcast(VT, Res); in combineConcatVectorOps()
56447 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56458 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56468 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56479 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56489 DAG.ComputeMaxSignificantBits(Ops[I].getOperand(0))); in combineConcatVectorOps()
56492 DAG.ComputeMaxSignificantBits(Ops[I].getOperand(1))); in combineConcatVectorOps()
56509 LHS = DAG.getNode(*CastOpc, DL, FpVT, LHS); in combineConcatVectorOps()
56510 RHS = DAG.getNode(*CastOpc, DL, FpVT, RHS); in combineConcatVectorOps()
56515 return DAG.getBitcast( in combineConcatVectorOps()
56516 VT, DAG.getNode(X86ISD::CMPP, DL, FpVT, LHS, RHS, in combineConcatVectorOps()
56517 DAG.getTargetConstant(FSETCC, DL, MVT::i8))); in combineConcatVectorOps()
56528 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56539 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56550 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56564 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56572 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56583 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56595 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56607 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56625 DAG.getBitcast(MaskVT, DAG.getConstant(Mask, DL, MaskSVT)); in combineConcatVectorOps()
56626 return DAG.getSelect(DL, VT, Sel, ConcatSubOperand(VT, Ops, 1), in combineConcatVectorOps()
56640 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56654 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56668 if (TLI->allowsMemoryAccess(Ctx, DAG.getDataLayout(), VT, in combineConcatVectorOps()
56672 EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false)) in combineConcatVectorOps()
56693 MVT PVT = TLI.getPointerTy(DAG.getDataLayout()); in combineConcatVectorOps()
56694 SDValue CV = DAG.getConstantPool(C, PVT); in combineConcatVectorOps()
56695 MachineFunction &MF = DAG.getMachineFunction(); in combineConcatVectorOps()
56697 SDValue Ld = DAG.getLoad(VT, DL, DAG.getEntryNode(), CV, MPI); in combineConcatVectorOps()
56698 SDValue Sub = extractSubVector(Ld, 0, DAG, DL, Op0.getValueSizeInBits()); in combineConcatVectorOps()
56699 DAG.ReplaceAllUsesOfValueWith(Op0, Sub); in combineConcatVectorOps()
56717 getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) { in combineConcatVectorOps()
56719 extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits()); in combineConcatVectorOps()
56720 DAG.ReplaceAllUsesOfValueWith(Op0, BcastSrc); in combineConcatVectorOps()
56730 SDValue Res = widenSubVector(Op0, false, Subtarget, DAG, DL, 512); in combineConcatVectorOps()
56731 Res = DAG.getBitcast(ShuffleVT, Res); in combineConcatVectorOps()
56732 Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT, Res, Res, in combineConcatVectorOps()
56733 getV4X86ShuffleImm8ForMask({0, 0, 0, 0}, DL, DAG)); in combineConcatVectorOps()
56734 return DAG.getBitcast(VT, Res); in combineConcatVectorOps()
56740 static SDValue combineCONCAT_VECTORS(SDNode *N, SelectionDAG &DAG, in combineCONCAT_VECTORS() argument
56745 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineCONCAT_VECTORS()
56757 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); in combineCONCAT_VECTORS()
56759 return DAG.getBitcast(VT, DAG.getConstant(Constant, SDLoc(N), IntVT)); in combineCONCAT_VECTORS()
56768 if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG, in combineCONCAT_VECTORS()
56776 static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG, in combineINSERT_SUBVECTOR() argument
56794 return DAG.getUNDEF(OpVT); in combineINSERT_SUBVECTOR()
56799 return getZeroVector(OpVT, Subtarget, DAG, dl); in combineINSERT_SUBVECTOR()
56807 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, in combineINSERT_SUBVECTOR()
56808 getZeroVector(OpVT, Subtarget, DAG, dl), in combineINSERT_SUBVECTOR()
56810 DAG.getIntPtrConstant(IdxVal + Idx2Val, dl)); in combineINSERT_SUBVECTOR()
56825 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, in combineINSERT_SUBVECTOR()
56826 getZeroVector(OpVT, Subtarget, DAG, dl), in combineINSERT_SUBVECTOR()
56842 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, Vec, in combineINSERT_SUBVECTOR()
56863 return DAG.getVectorShuffle(OpVT, dl, Vec, SubVec.getOperand(0), Mask); in combineINSERT_SUBVECTOR()
56869 if (collectConcatOps(N, SubVectorOps, DAG)) { in combineINSERT_SUBVECTOR()
56871 combineConcatVectorOps(dl, OpVT, SubVectorOps, DAG, DCI, Subtarget)) in combineINSERT_SUBVECTOR()
56881 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, in combineINSERT_SUBVECTOR()
56882 getZeroVector(OpVT, Subtarget, DAG, dl), in combineINSERT_SUBVECTOR()
56883 SubVectorOps[0], DAG.getIntPtrConstant(0, dl)); in combineINSERT_SUBVECTOR()
56890 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineINSERT_SUBVECTOR()
56897 return DAG.getNode(X86ISD::VBROADCAST, dl, OpVT, SubVec.getOperand(0)); in combineINSERT_SUBVECTOR()
56904 SDVTList Tys = DAG.getVTList(OpVT, MVT::Other); in combineINSERT_SUBVECTOR()
56907 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, in combineINSERT_SUBVECTOR()
56910 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1)); in combineINSERT_SUBVECTOR()
56921 DAG.areNonVolatileConsecutiveLoads(SubLd, VecLd, in combineINSERT_SUBVECTOR()
56924 SubLd, 0, DAG); in combineINSERT_SUBVECTOR()
56937 SelectionDAG &DAG) { in narrowExtractedVectorSelect() argument
56940 !isFreeToSplitVector(Sel.getOperand(0).getNode(), DAG)) in narrowExtractedVectorSelect()
56980 SDValue ExtCond = extract128BitVector(Sel.getOperand(0), ExtIdx, DAG, DL); in narrowExtractedVectorSelect()
56981 SDValue ExtT = extract128BitVector(Sel.getOperand(1), ExtIdx, DAG, DL); in narrowExtractedVectorSelect()
56982 SDValue ExtF = extract128BitVector(Sel.getOperand(2), ExtIdx, DAG, DL); in narrowExtractedVectorSelect()
56983 SDValue NarrowSel = DAG.getSelect(DL, NarrowSelVT, ExtCond, ExtT, ExtF); in narrowExtractedVectorSelect()
56984 return DAG.getBitcast(VT, NarrowSel); in narrowExtractedVectorSelect()
56987 static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG, in combineEXTRACT_SUBVECTOR() argument
57011 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineEXTRACT_SUBVECTOR()
57027 SDValue Concat = splitVectorIntBinary(InVecBC, DAG, SDLoc(InVecBC)); in combineEXTRACT_SUBVECTOR()
57028 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, in combineEXTRACT_SUBVECTOR()
57029 DAG.getBitcast(InVecVT, Concat), N->getOperand(1)); in combineEXTRACT_SUBVECTOR()
57036 if (SDValue V = narrowExtractedVectorSelect(N, DL, DAG)) in combineEXTRACT_SUBVECTOR()
57040 return getZeroVector(VT, Subtarget, DAG, DL); in combineEXTRACT_SUBVECTOR()
57044 return DAG.getConstant(1, DL, VT); in combineEXTRACT_SUBVECTOR()
57045 return getOnesVector(VT, DAG, DL); in combineEXTRACT_SUBVECTOR()
57049 return DAG.getBuildVector(VT, DL, InVec->ops().slice(IdxVal, NumSubElts)); in combineEXTRACT_SUBVECTOR()
57059 SDValue NewExt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, in combineEXTRACT_SUBVECTOR()
57062 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, NewExt, in combineEXTRACT_SUBVECTOR()
57064 DAG.getVectorIdxConstant(NewIdxVal, DL)); in combineEXTRACT_SUBVECTOR()
57072 DAG.isSplatValue(InVec, /*AllowUndefs*/ false))) in combineEXTRACT_SUBVECTOR()
57073 return extractSubVector(InVec, 0, DAG, DL, SizeInBits); in combineEXTRACT_SUBVECTOR()
57078 return extractSubVector(InVec, 0, DAG, DL, SizeInBits); in combineEXTRACT_SUBVECTOR()
57087 if (getTargetShuffleInputs(InVecBC, ShuffleInputs, ShuffleMask, DAG) && in combineEXTRACT_SUBVECTOR()
57091 return DAG.getUNDEF(VT); in combineEXTRACT_SUBVECTOR()
57093 return getZeroVector(VT, Subtarget, DAG, DL); in combineEXTRACT_SUBVECTOR()
57098 return extractSubVector(DAG.getBitcast(InVecVT, Src), SrcEltIdx, DAG, in combineEXTRACT_SUBVECTOR()
57121 return DAG.getNode(X86ISD::CVTSI2P, DL, VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
57126 return DAG.getNode(X86ISD::CVTUI2P, DL, VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
57131 return DAG.getNode(X86ISD::VFPEXT, DL, VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
57138 return DAG.getNode(InOpcode, DL, VT, in combineEXTRACT_SUBVECTOR()
57139 extractSubVector(Src, IdxVal, DAG, DL, SizeInBits)); in combineEXTRACT_SUBVECTOR()
57147 Ext = extractSubVector(Ext, 0, DAG, DL, SizeInBits); in combineEXTRACT_SUBVECTOR()
57148 unsigned ExtOp = DAG.getOpcode_EXTEND_VECTOR_INREG(InOpcode); in combineEXTRACT_SUBVECTOR()
57149 return DAG.getNode(ExtOp, DL, VT, Ext); in combineEXTRACT_SUBVECTOR()
57155 SDValue Ext0 = extractSubVector(InVec.getOperand(0), 0, DAG, DL, 128); in combineEXTRACT_SUBVECTOR()
57156 SDValue Ext1 = extractSubVector(InVec.getOperand(1), 0, DAG, DL, 128); in combineEXTRACT_SUBVECTOR()
57157 SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128); in combineEXTRACT_SUBVECTOR()
57158 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2); in combineEXTRACT_SUBVECTOR()
57164 SDValue Ext = extractSubVector(InVecSrc, 0, DAG, DL, Scale * SizeInBits); in combineEXTRACT_SUBVECTOR()
57165 return DAG.getNode(InOpcode, DL, VT, Ext); in combineEXTRACT_SUBVECTOR()
57173 extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits); in combineEXTRACT_SUBVECTOR()
57175 extractSubVector(InVec.getOperand(1), IdxVal, DAG, DL, SizeInBits); in combineEXTRACT_SUBVECTOR()
57177 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, InVec.getOperand(2)); in combineEXTRACT_SUBVECTOR()
57178 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1); in combineEXTRACT_SUBVECTOR()
57183 extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits); in combineEXTRACT_SUBVECTOR()
57184 return DAG.getNode(InOpcode, DL, VT, Ext0); in combineEXTRACT_SUBVECTOR()
57194 extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits); in combineEXTRACT_SUBVECTOR()
57195 return DAG.getNode(InOpcode, DL, VT, Ext, InVec.getOperand(1)); in combineEXTRACT_SUBVECTOR()
57201 static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) { in combineScalarToVector() argument
57212 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Src.getOperand(0)); in combineScalarToVector()
57219 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src.getOperand(0), in combineScalarToVector()
57225 auto IsExt64 = [&DAG](SDValue Op, bool IsZeroExt) { in combineScalarToVector()
57238 KnownBits Known = DAG.computeKnownBits(Op); in combineScalarToVector()
57246 return DAG.getBitcast( in combineScalarToVector()
57247 VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, in combineScalarToVector()
57248 DAG.getAnyExtOrTrunc(AnyExt, DL, MVT::i32))); in combineScalarToVector()
57251 return DAG.getBitcast( in combineScalarToVector()
57253 DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v4i32, in combineScalarToVector()
57254 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, in combineScalarToVector()
57255 DAG.getZExtOrTrunc(ZeroExt, DL, MVT::i32)))); in combineScalarToVector()
57261 return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, Src.getOperand(0)); in combineScalarToVector()
57275 return extractSubVector(SDValue(User, 0), 0, DAG, DL, SizeInBits); in combineScalarToVector()
57284 static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG, in combinePMULDQ() argument
57291 if (DAG.isConstantIntBuildVectorOrConstantInt(LHS) && in combinePMULDQ()
57292 !DAG.isConstantIntBuildVectorOrConstantInt(RHS)) in combinePMULDQ()
57293 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), RHS, LHS); in combinePMULDQ()
57298 return DAG.getConstant(0, SDLoc(N), N->getValueType(0)); in combinePMULDQ()
57301 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combinePMULDQ()
57317 LHS = DAG.getVectorShuffle(MVT::v4i32, dl, LHS.getOperand(0), in combinePMULDQ()
57319 LHS = DAG.getBitcast(MVT::v2i64, LHS); in combinePMULDQ()
57320 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS); in combinePMULDQ()
57327 RHS = DAG.getVectorShuffle(MVT::v4i32, dl, RHS.getOperand(0), in combinePMULDQ()
57329 RHS = DAG.getBitcast(MVT::v2i64, RHS); in combinePMULDQ()
57330 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS); in combinePMULDQ()
57337 static SDValue combineVPMADD(SDNode *N, SelectionDAG &DAG, in combineVPMADD() argument
57351 return DAG.getConstant(0, SDLoc(N), VT); in combineVPMADD()
57371 return getConstVector(Result, VT, DAG, SDLoc(N)); in combineVPMADD()
57374 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineVPMADD()
57382 static SDValue combineEXTEND_VECTOR_INREG(SDNode *N, SelectionDAG &DAG, in combineEXTEND_VECTOR_INREG() argument
57389 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineEXTEND_VECTOR_INREG()
57403 SDValue Load = DAG.getExtLoad( in combineEXTEND_VECTOR_INREG()
57406 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1)); in combineEXTEND_VECTOR_INREG()
57414 return DAG.getNode(Opcode, DL, VT, In.getOperand(0)); in combineEXTEND_VECTOR_INREG()
57420 In.getOperand(0).getOpcode() == DAG.getOpcode_EXTEND(Opcode) && in combineEXTEND_VECTOR_INREG()
57423 return DAG.getNode(Opcode, DL, VT, In.getOperand(0).getOperand(0)); in combineEXTEND_VECTOR_INREG()
57433 SmallVector<SDValue> Elts(Scale * NumElts, DAG.getConstant(0, DL, EltVT)); in combineEXTEND_VECTOR_INREG()
57436 return DAG.getBitcast(VT, DAG.getBuildVector(In.getValueType(), DL, Elts)); in combineEXTEND_VECTOR_INREG()
57443 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) in combineEXTEND_VECTOR_INREG()
57450 static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG, in combineKSHIFT() argument
57455 return DAG.getConstant(0, SDLoc(N), VT); in combineKSHIFT()
57457 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combineKSHIFT()
57468 static SDValue combineFP16_TO_FP(SDNode *N, SelectionDAG &DAG, in combineFP16_TO_FP() argument
57481 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, in combineFP16_TO_FP()
57483 Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res, in combineFP16_TO_FP()
57484 DAG.getTargetConstant(4, dl, MVT::i32)); in combineFP16_TO_FP()
57485 Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res); in combineFP16_TO_FP()
57486 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res, in combineFP16_TO_FP()
57487 DAG.getIntPtrConstant(0, dl)); in combineFP16_TO_FP()
57490 static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG, in combineFP_EXTEND() argument
57510 return DAG.getNode(ISD::FP_EXTEND, dl, VT, in combineFP_EXTEND()
57511 DAG.getNode(ISD::FP_EXTEND, dl, TmpVT, Src)); in combineFP_EXTEND()
57515 Src = DAG.getBitcast(SrcVT.changeTypeToInteger(), Src); in combineFP_EXTEND()
57516 Src = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Src); in combineFP_EXTEND()
57517 Src = DAG.getNode(ISD::SHL, dl, NVT, Src, DAG.getConstant(16, dl, NVT)); in combineFP_EXTEND()
57518 return DAG.getBitcast(VT, Src); in combineFP_EXTEND()
57540 Src = DAG.getBitcast(IntVT, Src); in combineFP_EXTEND()
57545 SDValue Fill = NumElts == 4 ? DAG.getUNDEF(IntVT) in combineFP_EXTEND()
57546 : DAG.getConstant(0, dl, IntVT); in combineFP_EXTEND()
57549 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, Ops); in combineFP_EXTEND()
57553 EVT CvtVT = EVT::getVectorVT(*DAG.getContext(), MVT::f32, in combineFP_EXTEND()
57557 Cvt = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {CvtVT, MVT::Other}, in combineFP_EXTEND()
57561 Cvt = DAG.getNode(X86ISD::CVTPH2PS, dl, CvtVT, Src); in combineFP_EXTEND()
57566 Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Cvt, in combineFP_EXTEND()
57567 DAG.getIntPtrConstant(0, dl)); in combineFP_EXTEND()
57573 Cvt = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {VT, MVT::Other}, in combineFP_EXTEND()
57577 return DAG.getMergeValues({Cvt, Chain}, dl); in combineFP_EXTEND()
57581 return DAG.getNode(ISD::FP_EXTEND, dl, VT, Cvt); in combineFP_EXTEND()
57587 static SDValue combineBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG, in combineBROADCAST_LOAD() argument
57614 SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N), in combineBROADCAST_LOAD()
57616 Extract = DAG.getBitcast(VT, Extract); in combineBROADCAST_LOAD()
57623 static SDValue combineFP_ROUND(SDNode *N, SelectionDAG &DAG, in combineFP_ROUND() argument
57663 Cvt0 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other}, in combineFP_ROUND()
57665 Cvt1 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other}, in combineFP_ROUND()
57667 Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask); in combineFP_ROUND()
57668 return DAG.getMergeValues({Cvt, Cvt0.getValue(1)}, dl); in combineFP_ROUND()
57672 Cvt0 = DAG.getNode(Opc, dl, MVT::v8f16, Op0.getOperand(0)); in combineFP_ROUND()
57673 Cvt1 = DAG.getNode(Opc, dl, MVT::v8f16, Op1.getOperand(0)); in combineFP_ROUND()
57674 return Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask); in combineFP_ROUND()
57684 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, in combineFP_ROUND()
57685 DAG.getConstantFP(0.0, dl, SrcVT)); in combineFP_ROUND()
57689 EVT::getVectorVT(*DAG.getContext(), MVT::i16, std::max(8U, NumElts)); in combineFP_ROUND()
57690 SDValue Rnd = DAG.getTargetConstant(4, dl, MVT::i32); in combineFP_ROUND()
57692 Cvt = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {CvtVT, MVT::Other}, in combineFP_ROUND()
57696 Cvt = DAG.getNode(X86ISD::CVTPS2PH, dl, CvtVT, Src, Rnd); in combineFP_ROUND()
57702 Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, IntVT, Cvt, in combineFP_ROUND()
57703 DAG.getIntPtrConstant(0, dl)); in combineFP_ROUND()
57706 Cvt = DAG.getBitcast(VT, Cvt); in combineFP_ROUND()
57709 return DAG.getMergeValues({Cvt, Chain}, dl); in combineFP_ROUND()
57714 static SDValue combineMOVDQ2Q(SDNode *N, SelectionDAG &DAG) { in combineMOVDQ2Q() argument
57722 SDValue NewLd = DAG.getLoad(MVT::x86mmx, SDLoc(N), LN->getChain(), in combineMOVDQ2Q()
57727 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), NewLd.getValue(1)); in combineMOVDQ2Q()
57735 static SDValue combinePDEP(SDNode *N, SelectionDAG &DAG, in combinePDEP() argument
57738 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); in combinePDEP()
57747 SelectionDAG &DAG = DCI.DAG; in PerformDAGCombine() local
57752 return combineScalarToVector(N, DAG); in PerformDAGCombine()
57756 return combineExtractVectorElt(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57758 return combineCONCAT_VECTORS(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57760 return combineINSERT_SUBVECTOR(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57762 return combineEXTRACT_SUBVECTOR(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57765 case X86ISD::BLENDV: return combineSelect(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57766 case ISD::BITCAST: return combineBitcast(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57767 case X86ISD::CMOV: return combineCMov(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57768 case X86ISD::CMP: return combineCMP(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57769 case ISD::ADD: return combineAdd(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57770 case ISD::SUB: return combineSub(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57772 case X86ISD::SUB: return combineX86AddSub(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57774 case X86ISD::CSTORE: return combineX86CloadCstore(N, DAG); in PerformDAGCombine()
57775 case X86ISD::SBB: return combineSBB(N, DAG); in PerformDAGCombine()
57776 case X86ISD::ADC: return combineADC(N, DAG, DCI); in PerformDAGCombine()
57777 case ISD::MUL: return combineMul(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57778 case ISD::SHL: return combineShiftLeft(N, DAG, Subtarget); in PerformDAGCombine()
57779 case ISD::SRA: return combineShiftRightArithmetic(N, DAG, Subtarget); in PerformDAGCombine()
57780 case ISD::SRL: return combineShiftRightLogical(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57781 case ISD::AND: return combineAnd(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57782 case ISD::OR: return combineOr(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57783 case ISD::XOR: return combineXor(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57784 case ISD::BITREVERSE: return combineBITREVERSE(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57788 case ISD::AVGFLOORU: return combineAVG(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57790 case X86ISD::BEXTRI: return combineBEXTR(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57791 case ISD::LOAD: return combineLoad(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57792 case ISD::MLOAD: return combineMaskedLoad(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57793 case ISD::STORE: return combineStore(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57794 case ISD::MSTORE: return combineMaskedStore(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57796 return combineVEXTRACT_STORE(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57799 return combineSIntToFP(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57802 return combineUIntToFP(N, DAG, Subtarget); in PerformDAGCombine()
57804 case ISD::LLRINT: return combineLRINT_LLRINT(N, DAG, Subtarget); in PerformDAGCombine()
57806 case ISD::FSUB: return combineFaddFsub(N, DAG, Subtarget); in PerformDAGCombine()
57808 case X86ISD::VFMULC: return combineFMulcFCMulc(N, DAG, Subtarget); in PerformDAGCombine()
57809 case ISD::FNEG: return combineFneg(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57810 case ISD::TRUNCATE: return combineTruncate(N, DAG, Subtarget); in PerformDAGCombine()
57811 case X86ISD::VTRUNC: return combineVTRUNC(N, DAG, DCI); in PerformDAGCombine()
57812 case X86ISD::ANDNP: return combineAndnp(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57813 case X86ISD::FAND: return combineFAnd(N, DAG, Subtarget); in PerformDAGCombine()
57814 case X86ISD::FANDN: return combineFAndn(N, DAG, Subtarget); in PerformDAGCombine()
57816 case X86ISD::FOR: return combineFOr(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57818 case X86ISD::FMAX: return combineFMinFMax(N, DAG); in PerformDAGCombine()
57820 case ISD::FMAXNUM: return combineFMinNumFMaxNum(N, DAG, Subtarget); in PerformDAGCombine()
57822 case X86ISD::CVTUI2P: return combineX86INT_TO_FP(N, DAG, DCI); in PerformDAGCombine()
57829 return combineCVTP2I_CVTTP2I(N, DAG, DCI); in PerformDAGCombine()
57831 case X86ISD::CVTPH2PS: return combineCVTPH2PS(N, DAG, DCI); in PerformDAGCombine()
57832 case X86ISD::BT: return combineBT(N, DAG, DCI); in PerformDAGCombine()
57834 case ISD::ZERO_EXTEND: return combineZext(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57835 case ISD::SIGN_EXTEND: return combineSext(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57836 case ISD::SIGN_EXTEND_INREG: return combineSignExtendInReg(N, DAG, Subtarget); in PerformDAGCombine()
57840 return combineEXTEND_VECTOR_INREG(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57841 case ISD::SETCC: return combineSetCC(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57842 case X86ISD::SETCC: return combineX86SetCC(N, DAG, Subtarget); in PerformDAGCombine()
57843 case X86ISD::BRCOND: return combineBrCond(N, DAG, Subtarget); in PerformDAGCombine()
57845 case X86ISD::PACKUS: return combineVectorPack(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57849 case X86ISD::FHSUB: return combineVectorHADDSUB(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57853 return combineVectorShiftVar(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57857 return combineVectorShiftImm(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57860 case X86ISD::PINSRW: return combineVectorInsert(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57895 case ISD::VECTOR_SHUFFLE: return combineShuffle(N, DAG, DCI,Subtarget); in PerformDAGCombine()
57907 case ISD::STRICT_FMA: return combineFMA(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57911 case X86ISD::FMSUBADD: return combineFMADDSUB(N, DAG, DCI); in PerformDAGCombine()
57912 case X86ISD::MOVMSK: return combineMOVMSK(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57913 case X86ISD::TESTP: return combineTESTP(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57915 case X86ISD::MSCATTER: return combineX86GatherScatter(N, DAG, DCI); in PerformDAGCombine()
57917 case ISD::MSCATTER: return combineGatherScatter(N, DAG, DCI); in PerformDAGCombine()
57919 case X86ISD::PCMPGT: return combineVectorCompare(N, DAG, Subtarget); in PerformDAGCombine()
57921 case X86ISD::PMULUDQ: return combinePMULDQ(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57923 case X86ISD::VPMADDWD: return combineVPMADD(N, DAG, DCI); in PerformDAGCombine()
57925 case X86ISD::KSHIFTR: return combineKSHIFT(N, DAG, DCI); in PerformDAGCombine()
57926 case ISD::FP16_TO_FP: return combineFP16_TO_FP(N, DAG, Subtarget); in PerformDAGCombine()
57928 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DAG, DCI, Subtarget); in PerformDAGCombine()
57930 case ISD::FP_ROUND: return combineFP_ROUND(N, DAG, Subtarget); in PerformDAGCombine()
57932 case X86ISD::SUBV_BROADCAST_LOAD: return combineBROADCAST_LOAD(N, DAG, DCI); in PerformDAGCombine()
57933 case X86ISD::MOVDQ2Q: return combineMOVDQ2Q(N, DAG); in PerformDAGCombine()
57934 case X86ISD::PDEP: return combinePDEP(N, DAG, DCI); in PerformDAGCombine()
58004 SelectionDAG &DAG) const { in expandIndirectJTBranch()
58005 const Module *M = DAG.getMachineFunction().getFunction().getParent(); in expandIndirectJTBranch()
58012 SDValue JTInfo = DAG.getJumpTableDebugInfo(JTI, Value, dl); in expandIndirectJTBranch()
58013 return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, JTInfo, Addr); in expandIndirectJTBranch()
58016 return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, JTI, DAG); in expandIndirectJTBranch()
58517 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const { in LowerAsmOutputForConstraint()
58528 Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32, Glue); in LowerAsmOutputForConstraint()
58531 Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32); in LowerAsmOutputForConstraint()
58533 SDValue CC = getSETCC(Cond, Glue, DL, DAG); in LowerAsmOutputForConstraint()
58535 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC); in LowerAsmOutputForConstraint()
58545 SelectionDAG &DAG) const { in LowerAsmOperandForConstraint()
58553 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58562 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58571 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58581 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58590 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58599 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58608 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58617 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()), in LowerAsmOperandForConstraint()
58620 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64); in LowerAsmOperandForConstraint()
58633 Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(), in LowerAsmOperandForConstraint()
58643 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op), in LowerAsmOperandForConstraint()
58651 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()), in LowerAsmOperandForConstraint()
58653 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58671 Result = DAG.getTargetConstant(ExtVal, SDLoc(Op), MVT::i64); in LowerAsmOperandForConstraint()
58698 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); in LowerAsmOperandForConstraint()