Lines Matching +full:flip +full:- +full:horizontal
1 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
12 //===----------------------------------------------------------------------===//
71 #define DEBUG_TYPE "x86-isel"
74 "x86-experimental-pref-innermost-loop-alignment", cl::init(4),
78 "alignment set by x86-experimental-pref-loop-alignment."),
82 "x86-br-merging-base-cost", cl::init(2),
88 "will be merged, and above which conditionals will be split. Set to -1 "
93 "x86-br-merging-ccmp-bias", cl::init(6),
94 cl::desc("Increases 'x86-br-merging-base-cost' in cases that the target "
99 "x86-br-merging-likely-bias", cl::init(0),
100 cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely "
105 "the instruction cost threshold. Set to -1 to never merge likely "
110 "x86-br-merging-unlikely-bias", cl::init(-1),
112 "Decreases 'x86-br-merging-base-cost' in cases that it is unlikely "
117 "the instruction cost threshold. Set to -1 to never merge unlikely "
122 "mul-constant-optimization", cl::init(true),
137 // X86-SSE is even stranger. It uses -1 or 0 for vector masks. in X86TargetLowering()
141 // default expansion to a no-op. in X86TargetLowering()
144 // For 64-bit, since we have so many registers, use the ILP scheduler. in X86TargetLowering()
145 // For 32-bit, use the register pressure specific scheduling. in X86TargetLowering()
154 setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister()); in X86TargetLowering()
264 // We have an algorithm for SSE2, and we turn this into a 64-bit in X86TargetLowering()
268 // We have an algorithm for SSE2->double, and we turn this into a in X86TargetLowering()
269 // 64-bit FILD followed by conditional FADD for other targets. in X86TargetLowering()
284 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 in X86TargetLowering()
298 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64 in X86TargetLowering()
350 // Without SSE, i64->f64 goes through memory. in X86TargetLowering()
358 // the two-result form to trivial CSE, which is able to combine x/y and x%y in X86TargetLowering()
361 // Scalar integer multiply-high is also lowered to use two-result in X86TargetLowering()
363 // (low) operations are left as Legal, as there are single-result in X86TargetLowering()
364 // instructions for this in x86. Using the two-result multiply instructions in X86TargetLowering()
436 // Special handling for half-precision floating point conversions. in X86TargetLowering()
505 // LLVM/Clang supports zero-cost DWARF and SEH exception handling. in X86TargetLowering()
524 // 64-bit shl, sra, srl (iff 32-bit x86) in X86TargetLowering()
553 // All CPUs supporting AVX will atomically load/store aligned 128-bit in X86TargetLowering()
562 // FIXME - use subtarget debug flags in X86TargetLowering()
646 // Disable f32->f64 extload as we can only generate this in one instruction in X86TargetLowering()
649 // non-optsize case. in X86TargetLowering()
662 // These might be better off as horizontal vector ops. in X86TargetLowering()
774 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS in X86TargetLowering()
775 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS in X86TargetLowering()
784 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS in X86TargetLowering()
785 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS in X86TargetLowering()
793 // Handle constrained floating-point operations of scalar. in X86TargetLowering()
832 // clang-format off in X86TargetLowering()
843 // clang-format on in X86TargetLowering()
857 // Handle constrained floating-point operations of scalar. in X86TargetLowering()
869 // FIXME: When the target is 64-bit, STRICT_FP_ROUND will be overwritten in X86TargetLowering()
896 // clang-format off in X86TargetLowering()
904 // clang-format on in X86TargetLowering()
956 // clang-format off in X86TargetLowering()
970 // clang-format on in X86TargetLowering()
1021 // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like in X86TargetLowering()
1037 // with -msoft-float, disable use of MMX as well. in X86TargetLowering()
1073 // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM in X86TargetLowering()
1243 // 64-bit targets and two 32-bit loads on a 32-bit target. Similar for in X86TargetLowering()
1252 // Add 32-bit vector stores to help vectorization opportunities. in X86TargetLowering()
1322 // These might be better off as horizontal vector ops. in X86TargetLowering()
1360 // FIXME: Do we need to handle scalar-to-vector here? in X86TargetLowering()
1386 // We need to scalarize v4i64->v432 uint_to_fp using cvtsi2ss, but we can in X86TargetLowering()
1505 // These types need custom splitting if their input is a 128-bit vector. in X86TargetLowering()
1613 // when we have a 256bit-wide blend with immediate. in X86TargetLowering()
1635 // (result) is 128-bit but the source is 256-bit wide. in X86TargetLowering()
1641 // Custom lower several nodes for 256-bit types. in X86TargetLowering()
1693 // available with AVX512. 512-bit vectors are in a separate block controlled in X86TargetLowering()
1719 // There is no byte sized k-register load or store without AVX512DQ. in X86TargetLowering()
1732 // Extends of v16i1/v8i1/v4i1/v2i1 to 128-bit vectors. in X86TargetLowering()
1765 // This block controls legalization for 512-bit operations with 8/16/32/64 bit in X86TargetLowering()
1766 // elements. 512-bits can be disabled based on prefer-vector-width and in X86TargetLowering()
1767 // required-vector-width function attributes. in X86TargetLowering()
1846 // With 512-bit vectors and no VLX, we prefer to widen MLOAD/MSTORE in X86TargetLowering()
1847 // to 512-bit rather than use the AVX2 instructions so that we can use in X86TargetLowering()
1848 // k-masks. in X86TargetLowering()
1871 // Extends from v64i1 masks to 512-bit vectors. in X86TargetLowering()
1984 // NonVLX sub-targets extend 128/256 vectors to use the 512 version. in X86TargetLowering()
1996 // (result) is 256-bit but the source is 512-bit wide. in X86TargetLowering()
1997 // 128-bit was made Legal under AVX1. in X86TargetLowering()
2062 // pre-AVX512 equivalents. Without VLX we use 512-bit operations for in X86TargetLowering()
2065 // These operations are handled on non-VLX by artificially widening in in X86TargetLowering()
2151 // Extends from v32i1 masks to 256-bit vectors. in X86TargetLowering()
2161 // These operations are handled on non-VLX by artificially widening in in X86TargetLowering()
2163 // TODO: Custom widen in lowering on non-VLX and drop the isel patterns? in X86TargetLowering()
2380 // vcvttph2[u]dq v4f16 -> v4i32/64, v2f16 -> v2i32/64 in X86TargetLowering()
2389 // vcvt[u]dq2ph v4i32/64 -> v4f16, v2i32/64 -> v2f16 in X86TargetLowering()
2398 // vcvtps2phx v4f32 -> v4f16, v2f32 -> v2f16 in X86TargetLowering()
2403 // vcvtph2psx v4f16 -> v4f32, v2f16 -> v2f32 in X86TargetLowering()
2423 // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't in X86TargetLowering()
2427 // subtraction on x86-32 once PR3203 is fixed. We really can't do much better in X86TargetLowering()
2428 // than generic legalization for 64-bit multiplication-with-overflow, though. in X86TargetLowering()
2470 // On 32 bit MSVC, `fmodf(f32)` is not defined - only `fmod(f64)` in X86TargetLowering()
2471 // is. We should promote the value to 64-bits to solve this. in X86TargetLowering()
2472 // This is what the CRT headers do - `fmodf` is an inline header in X86TargetLowering()
2476 // clang-format off in X86TargetLowering()
2496 // clang-format on in X86TargetLowering()
2498 // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined. MinGW has in X86TargetLowering()
2506 // We have target-specific dag combine patterns for the following nodes: in X86TargetLowering()
2567 MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores in X86TargetLowering()
2569 MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores in X86TargetLowering()
2571 MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores in X86TargetLowering()
2580 // Default loop alignment, which can be overridden by -align-loops. in X86TargetLowering()
2583 // An out-of-order CPU can speculatively execute past a predictable branch, in X86TargetLowering()
2591 // Default to having -disable-strictnode-mutation on in X86TargetLowering()
2595 // This has so far only been implemented for 64-bit MachO.
2636 //===----------------------------------------------------------------------===//
2638 //===----------------------------------------------------------------------===//
2650 Ld->getValueSizeInBits(0) == 128 && Ld->getAlign() < Align(16)) in mayFoldLoad()
2653 // TODO: If this is a non-temporal load and the target has an instruction in mayFoldLoad()
2666 // We can not replace a wide volatile load with a broadcast-from-memory, in mayFoldLoadIntoBroadcastFromMem()
2669 return !Ld->isVolatile() || in mayFoldLoadIntoBroadcastFromMem()
2670 Ld->getValueSizeInBits(0) == EltVT.getScalarSizeInBits(); in mayFoldLoadIntoBroadcastFromMem()
2674 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin()); in mayFoldIntoStore()
2679 unsigned Opcode = Op.getNode()->use_begin()->getOpcode(); in mayFoldIntoZeroExtend()
2754 int ReturnAddrIndex = FuncInfo->getRAIndex(); in getReturnAddressFrameIndex()
2758 unsigned SlotSize = RegInfo->getSlotSize(); in getReturnAddressFrameIndex()
2760 -(int64_t)SlotSize, in getReturnAddressFrameIndex()
2762 FuncInfo->setRAIndex(ReturnAddrIndex); in getReturnAddressFrameIndex()
2774 // If we don't have a symbolic displacement - we don't have any extra in isOffsetSuitableForCodeModel()
2780 // 64-bit offsets. in isOffsetSuitableForCodeModel()
2790 // For other non-large code models we assume that latest small object is 16MB in isOffsetSuitableForCodeModel()
2819 // clang-format off in TranslateIntegerX86CC()
2831 // clang-format on in TranslateIntegerX86CC()
2835 /// Do a one-to-one translation of a ISD::CondCode to the X86-specific
2843 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnes()) { in TranslateX86CC()
2844 // X > -1 -> X == 0, jump !sign. in TranslateX86CC()
2848 if (SetCCOpcode == ISD::SETLT && RHSC->isZero()) { in TranslateX86CC()
2849 // X < 0 -> X == 0, jump on sign. in TranslateX86CC()
2852 if (SetCCOpcode == ISD::SETGE && RHSC->isZero()) { in TranslateX86CC()
2853 // X >= 0 -> X == 0, jump on !sign. in TranslateX86CC()
2856 if (SetCCOpcode == ISD::SETLT && RHSC->isOne()) { in TranslateX86CC()
2857 // X < 1 -> X <= 0 in TranslateX86CC()
2866 // First determine if it is required or is profitable to flip the operands. in TranslateX86CC()
2868 // If LHS is a foldable load, but RHS is not, flip the condition. in TranslateX86CC()
2892 // clang-format off in TranslateX86CC()
2893 default: llvm_unreachable("Condcode should be pre-legalized away"); in TranslateX86CC()
2914 // clang-format on in TranslateX86CC()
2956 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48); in getTgtMemIntrinsic()
2964 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64); in getTgtMemIntrinsic()
2972 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48); in getTgtMemIntrinsic()
2980 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64); in getTgtMemIntrinsic()
2991 unsigned Size = I.getType()->getScalarSizeInBits(); in getTgtMemIntrinsic()
2992 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size); in getTgtMemIntrinsic()
3003 unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits(); in getTgtMemIntrinsic()
3004 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size); in getTgtMemIntrinsic()
3025 unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits(); in getTgtMemIntrinsic()
3026 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size); in getTgtMemIntrinsic()
3036 switch (IntrData->Type) { in getTgtMemIntrinsic()
3042 MVT VT = MVT::getVT(I.getArgOperand(1)->getType()); in getTgtMemIntrinsic()
3044 if (IntrData->Type == TRUNCATE_TO_MEM_VI8) in getTgtMemIntrinsic()
3046 else if (IntrData->Type == TRUNCATE_TO_MEM_VI16) in getTgtMemIntrinsic()
3048 else if (IntrData->Type == TRUNCATE_TO_MEM_VI32) in getTgtMemIntrinsic()
3061 MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType()); in getTgtMemIntrinsic()
3072 MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType()); in getTgtMemIntrinsic()
3073 MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType()); in getTgtMemIntrinsic()
3102 assert(cast<LoadSDNode>(Load)->isSimple() && "illegal to narrow"); in shouldReduceLoadWidth()
3104 // "ELF Handling for Thread-Local Storage" specifies that R_X86_64_GOTTPOFF in shouldReduceLoadWidth()
3106 SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr(); in shouldReduceLoadWidth()
3109 return GA->getTargetFlags() != X86II::MO_GOTTPOFF; in shouldReduceLoadWidth()
3113 // can be store-folded. Therefore, it's probably not worth splitting the load. in shouldReduceLoadWidth()
3114 EVT VT = Load->getValueType(0); in shouldReduceLoadWidth()
3115 if ((VT.is256BitVector() || VT.is512BitVector()) && !Load->hasOneUse()) { in shouldReduceLoadWidth()
3116 for (auto UI = Load->use_begin(), UE = Load->use_end(); UI != UE; ++UI) { in shouldReduceLoadWidth()
3122 if (UI->getOpcode() != ISD::EXTRACT_SUBVECTOR || !UI->hasOneUse() || in shouldReduceLoadWidth()
3123 UI->use_begin()->getOpcode() != ISD::STORE) in shouldReduceLoadWidth()
3126 // All non-chain uses are extract + store. in shouldReduceLoadWidth()
3137 assert(Ty->isIntegerTy()); in shouldConvertConstantLoadToIntImm()
3139 unsigned BitSize = Ty->getPrimitiveSizeInBits(); in shouldConvertConstantLoadToIntImm()
3147 // a floating-point compare and we have blendv or conditional move, then it is in reduceSelectOfFPConstantLoads()
3148 // cheaper to select instead of doing a cross-register move and creating a in reduceSelectOfFPConstantLoads()
3175 // through type legalization on 32-bit targets so we would need to special in decomposeMulByConstant()
3182 // most implementations, sub-vXi32 vector multiplies are always fast, in decomposeMulByConstant()
3191 return (MulC + 1).isPowerOf2() || (MulC - 1).isPowerOf2() || in decomposeMulByConstant()
3192 (1 - MulC).isPowerOf2() || (-(MulC + 1)).isPowerOf2(); in decomposeMulByConstant()
3213 // TODO - do we have any exceptions? in shouldScalarizeBinop()
3239 (!Ty->isVectorTy() && Ty->getScalarSizeInBits() < 32); in isCheapToSpeculateCttz()
3313 // There are only 32-bit and 64-bit forms for 'andn'. in hasAndNotCompare()
3317 return !isa<ConstantSDNode>(Y) || cast<ConstantSDNode>(Y)->isOpaque(); in hasAndNotCompare()
3360 // Pre-AVX2 vector codegen for this pattern is best for variant with 'shl'. in shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd()
3382 VT.getScalarSizeInBits() - ShiftOrRotateAmt.getZExtValue(); in preferedOpcodeForCmpEqPiecesOfOperand()
3402 // at least imm32 mask (or be zext i32 -> i64). in preferedOpcodeForCmpEqPiecesOfOperand()
3404 return AndMask->getSignificantBits() > 32 ? (unsigned)ISD::SRL in preferedOpcodeForCmpEqPiecesOfOperand()
3407 // We can only benefit if req at least 7-bit for the mask. We in preferedOpcodeForCmpEqPiecesOfOperand()
3414 // Keep exactly 32-bit imm64, this is zext i32 -> i64 which is in preferedOpcodeForCmpEqPiecesOfOperand()
3416 return AndMask->getSignificantBits() > 33 ? (unsigned)ISD::SHL : ShiftOpc; in preferedOpcodeForCmpEqPiecesOfOperand()
3427 // Non-vector type and we have a zext mask with SRL. in preferedOpcodeForCmpEqPiecesOfOperand()
3453 return N->getOpcode() != ISD::FP_EXTEND; in preferScalarizeSplat()
3458 assert(((N->getOpcode() == ISD::SHL && in shouldFoldConstantShiftPairToMask()
3459 N->getOperand(0).getOpcode() == ISD::SRL) || in shouldFoldConstantShiftPairToMask()
3460 (N->getOpcode() == ISD::SRL && in shouldFoldConstantShiftPairToMask()
3461 N->getOperand(0).getOpcode() == ISD::SHL)) && in shouldFoldConstantShiftPairToMask()
3462 "Expected shift-shift mask"); in shouldFoldConstantShiftPairToMask()
3464 EVT VT = N->getValueType(0); in shouldFoldConstantShiftPairToMask()
3467 // Only fold if the shift values are equal - so it folds to AND. in shouldFoldConstantShiftPairToMask()
3468 // TODO - we should fold if either is a non-uniform vector but we don't do in shouldFoldConstantShiftPairToMask()
3469 // the fold for non-splats yet. in shouldFoldConstantShiftPairToMask()
3470 return N->getOperand(1) == N->getOperand(0).getOperand(1); in shouldFoldConstantShiftPairToMask()
3482 // 64-bit shifts on 32-bit targets produce really bad bloated code. in shouldFoldMaskToVariableShiftPair()
3518 // TODO: Allow 64-bit type for 32-bit target. in hasFastEqualityCompare()
3519 // TODO: 512-bit types should be allowed, but make sure that those in hasFastEqualityCompare()
3612 /// Return true if every element in Mask, is an in-place blend/select mask or is
3624 /// sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.
3670 /// shuffle masks. The latter have the special property of a '-2' representing
3671 /// a zero-ed lane of a vector.
3686 // a pair of values. If we find such a case, use the non-undef mask's value. in canWidenShuffleElements()
3730 assert(!Zeroable.isZero() && "V2's non-undef elements are used?!"); in canWidenShuffleElements()
3784 // Use an UNDEF node if MaskElt == -1.
3785 // Split 64-bit constants in the 32-bit mode.
3872 // available, use a floating-point +0.0 instead. in getZeroVector()
3926 // This is the index of the first element of the vectorWidth-bit chunk in extractSubVector()
3928 IdxVal &= ~(ElemsPerChunk - 1); in extractSubVector()
3933 Vec->ops().slice(IdxVal, ElemsPerChunk)); in extractSubVector()
3945 /// Generate a DAG to grab 128-bits from a vector > 128 bits. This
3947 /// or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4
3949 /// 128 bits we want. It need not be aligned to a 128-bit boundary. That makes
3958 /// Generate a DAG to grab 256-bits from a 512-bit vector.
3981 // This is the index of the first element of the vectorWidth-bit chunk in insertSubVector()
3983 IdxVal &= ~(ElemsPerChunk - 1); in insertSubVector()
3989 /// Generate a DAG to put 128-bits into a vector > 128 bits. This
3991 /// AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a
3993 /// we want. It need not be aligned to a 128-bit boundary. That makes
4055 if (N->getOpcode() == ISD::CONCAT_VECTORS) { in collectConcatOps()
4056 Ops.append(N->op_begin(), N->op_end()); in collectConcatOps()
4060 if (N->getOpcode() == ISD::INSERT_SUBVECTOR) { in collectConcatOps()
4061 SDValue Src = N->getOperand(0); in collectConcatOps()
4062 SDValue Sub = N->getOperand(1); in collectConcatOps()
4063 const APInt &Idx = N->getConstantOperandAPInt(2); in collectConcatOps()
4149 // If this is a splat value (with no-undefs) then use the lower subvector, in splitVector()
4187 // Make sure we only try to split 256/512-bit types to avoid creating in splitVectorIntUnary()
4213 // Useful for operations that are available on SSE2 in 128-bit, on AVX2 in
4214 // 256-bit and on AVX512BW in 512-bit. The argument VT is the type used for
4259 // Helper function that extends a non-512-bit vector op to 512-bits on non-VLX
4271 // AVX512 broadcasts 32/64-bit operands. in getAVX512Node()
4272 // TODO: Support float once getAVX512Node is used by fp-ops. in getAVX512Node()
4283 if (BV->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, in getAVX512Node()
4318 // Perform the 512-bit op then extract the bottom subvector. in getAVX512Node()
4324 /// Insert i1-subvector to i1-vector.
4394 if (llvm::all_of(Vec->ops().slice(IdxVal + SubVecNumElems), in insert1BitVector()
4400 unsigned ShiftLeft = NumElems - SubVecNumElems; in insert1BitVector()
4401 unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal; in insert1BitVector()
4427 SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8); in insert1BitVector()
4442 unsigned ShiftLeft = NumElems - SubVecNumElems; in insert1BitVector()
4443 unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal; in insert1BitVector()
4469 unsigned LowShift = NumElems - IdxVal; in insert1BitVector()
4507 "Expected a 128/256/512-bit vector type"); in getOnesVector()
4521 // For 256-bit vectors, we only need the lower (128-bit) input half. in getEXTEND_VECTOR_INREG()
4522 // For 512-bit vectors, we only need the lower input half or quarter. in getEXTEND_VECTOR_INREG()
4562 /// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
4564 /// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
4565 /// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
4630 // Rely on vector shuffles for vXi64 -> vXi32 packing. in getPack()
4664 SDValue Mask = DAG.getConstant((1ULL << EltSizeInBits) - 1, dl, OpVT); in getPack()
4706 // TODO: Add support for non-zero offsets.
4709 if (!CNode || CNode->isMachineConstantPoolEntry() || CNode->getOffset() != 0) in getTargetConstantFromBasePtr()
4711 return CNode->getConstVal(); in getTargetConstantFromBasePtr()
4717 return getTargetConstantFromBasePtr(Load->getBasePtr()); in getTargetConstantFromNode()
4812 Mask = CInt->getValue(); in getTargetConstantBitsFromNode()
4816 Mask = CFP->getValueAPF().bitcastToAPInt(); in getTargetConstantBitsFromNode()
4820 Type *Ty = CDS->getType(); in getTargetConstantBitsFromNode()
4821 Mask = APInt::getZero(Ty->getPrimitiveSizeInBits()); in getTargetConstantBitsFromNode()
4822 Type *EltTy = CDS->getElementType(); in getTargetConstantBitsFromNode()
4823 bool IsInteger = EltTy->isIntegerTy(); in getTargetConstantBitsFromNode()
4825 EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy(); in getTargetConstantBitsFromNode()
4828 unsigned EltBits = EltTy->getPrimitiveSizeInBits(); in getTargetConstantBitsFromNode()
4829 for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) in getTargetConstantBitsFromNode()
4831 Mask.insertBits(CDS->getElementAsAPInt(I), I * EltBits); in getTargetConstantBitsFromNode()
4833 Mask.insertBits(CDS->getElementAsAPFloat(I).bitcastToAPInt(), in getTargetConstantBitsFromNode()
4850 SmallVector<APInt, 64> SrcEltBits(1, Cst->getAPIntValue()); in getTargetConstantBitsFromNode()
4855 APInt RawBits = Cst->getValueAPF().bitcastToAPInt(); in getTargetConstantBitsFromNode()
4865 if (BV->getConstantRawBits(true, SrcEltSizeInBits, SrcEltBits, Undefs)) { in getTargetConstantBitsFromNode()
4876 Type *CstTy = Cst->getType(); in getTargetConstantBitsFromNode()
4877 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits(); in getTargetConstantBitsFromNode()
4878 if (!CstTy->isVectorTy() || (CstSizeInBits % SizeInBits) != 0) in getTargetConstantBitsFromNode()
4881 unsigned SrcEltSizeInBits = CstTy->getScalarSizeInBits(); in getTargetConstantBitsFromNode()
4889 if (!CollectConstantBits(Cst->getAggregateElement(i), SrcEltBits[i], in getTargetConstantBitsFromNode()
4900 if (MemIntr->getMemoryVT().getStoreSizeInBits() != VT.getScalarSizeInBits()) in getTargetConstantBitsFromNode()
4903 SDValue Ptr = MemIntr->getBasePtr(); in getTargetConstantBitsFromNode()
4915 SrcEltBits.append(NumSrcElts - 1, SrcEltBits[0]); in getTargetConstantBitsFromNode()
4924 SDValue Ptr = MemIntr->getBasePtr(); in getTargetConstantBitsFromNode()
4928 Type *CstTy = Cst->getType(); in getTargetConstantBitsFromNode()
4929 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits(); in getTargetConstantBitsFromNode()
4930 unsigned SubVecSizeInBits = MemIntr->getMemoryVT().getStoreSizeInBits(); in getTargetConstantBitsFromNode()
4931 if (!CstTy->isVectorTy() || (CstSizeInBits % SubVecSizeInBits) != 0 || in getTargetConstantBitsFromNode()
4934 unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits(); in getTargetConstantBitsFromNode()
4941 if (!CollectConstantBits(Cst->getAggregateElement(i), SubEltBits[i], in getTargetConstantBitsFromNode()
4964 SrcEltBits.append(NumSrcElts - 1, APInt(SrcEltSizeInBits, 0)); in getTargetConstantBitsFromNode()
4970 // If bitcasts to larger elements we might lose track of undefs - don't in getTargetConstantBitsFromNode()
4995 // TODO - support extract_subvector through bitcasts. in getTargetConstantBitsFromNode()
5017 // TODO - support shuffle through bitcasts. in getTargetConstantBitsFromNode()
5021 ArrayRef<int> Mask = SVN->getMask(); in getTargetConstantBitsFromNode()
5050 if (UndefElts1[M - NumElts]) in getTargetConstantBitsFromNode()
5052 EltBits.push_back(EltBits1[M - NumElts]); in getTargetConstantBitsFromNode()
5069 int SplatIndex = -1; in isConstantSplat()
5074 SplatIndex = -1; in isConstantSplat()
5108 // Match not(xor X, -1) -> X.
5109 // Match not(pcmpgt(C, X)) -> pcmpgt(X, C - 1).
5110 // Match not(extract_subvector(xor X, -1)) -> extract_subvector(X).
5111 // Match not(concat_vectors(xor X, -1, xor Y, -1)) -> concat_vectors(X, Y).
5135 // Don't fold min_signed_value -> (min_signed_value - 1) in IsNOT()
5139 Elt -= 1; in IsNOT()
5162 /// A multi-stage pack shuffle mask is created by specifying NumStages > 1.
5171 unsigned Repetitions = 1u << (NumStages - 1); in createPackShuffleMask()
5219 /// Calculates the shuffle mask corresponding to the target-specific opcode.
5225 /// It is an error to call this with non-empty Mask/Ops vectors.
5248 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5255 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5262 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5313 "Only 32-bit and 64-bit elements are supported!"); in getTargetShuffleMask()
5316 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5326 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5335 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5342 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5349 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5355 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5361 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5371 // We only decode broadcasts of same-sized vectors, peeking through to in getTargetShuffleMask()
5405 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5419 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5426 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1); in getTargetShuffleMask()
5452 unsigned CtrlImm = CtrlOp->getZExtValue(); in getTargetShuffleMask()
5514 // inputs that are actually the same node. Re-map the mask to always point in getTargetShuffleMask()
5519 M -= Mask.size(); in getTargetShuffleMask()
5521 // If we didn't already add operands in the opcode-specific code, default to in getTargetShuffleMask()
5587 int Scale = Size / V->getNumOperands(); in computeZeroableShuffleElements()
5594 APInt Val = Cst->getAPIntValue(); in computeZeroableShuffleElements()
5599 APInt Val = Cst->getValueAPF().bitcastToAPInt(); in computeZeroableShuffleElements()
5610 int Scale = V->getNumOperands() / Size; in computeZeroableShuffleElements()
5689 // SCALAR_TO_VECTOR - only the first element is defined, and the rest UNDEF. in getTargetShuffleAndZeroables()
5690 // TODO: We currently only set UNDEF for integer types - floats use the same in getTargetShuffleAndZeroables()
5704 // INSERT_SUBVECTOR - to widen vectors we often insert them into UNDEF in getTargetShuffleAndZeroables()
5827 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(N)->getMask(); in getFauxShuffleMask()
5838 // Attempt to decode as a per-byte mask. in getFauxShuffleMask()
5849 // We can't assume an undef src element gives an undef dst - the other src in getFauxShuffleMask()
5906 if (!N->isOnlyUserOf(Sub.getNode())) in getFauxShuffleMask()
5933 // Limit this to vXi64 512-bit vector cases to make the most of AVX512 in getFauxShuffleMask()
6011 // Check we have an in-range constant insertion index. in getFauxShuffleMask()
6104 // PACKSS then it was likely being used for sign-extension for a in getFauxShuffleMask()
6106 if (N0.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N0.getNode()) && in getFauxShuffleMask()
6111 if (N1.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N1.getNode()) && in getFauxShuffleMask()
6164 Mask.append(NumElts - NumSrcElts, SM_SentinelZero); in getFauxShuffleMask()
6190 Mask[i + j] = i + j - ByteShift; in getFauxShuffleMask()
6194 Mask[i + j - ByteShift] = i + j; in getFauxShuffleMask()
6206 Offset = (X86ISD::VROTLI == Opcode ? NumBytesPerElt - Offset : Offset); in getFauxShuffleMask()
6239 // We can only handle all-signbits extensions. in getFauxShuffleMask()
6295 M -= MaskWidth; in resolveTargetShuffleInputsAndMask()
6306 M = (M < hi) ? ((M - lo) + (j * MaskWidth)) : (M - MaskWidth); in resolveTargetShuffleInputsAndMask()
6379 // Ensure this is a simple (non-atomic, non-voltile), temporal read memop. in getBROADCAST_LOAD()
6380 if (!Mem || !Mem->readMem() || !Mem->isSimple() || Mem->isNonTemporal()) in getBROADCAST_LOAD()
6383 SDValue Ptr = DAG.getMemBasePlusOffset(Mem->getBasePtr(), in getBROADCAST_LOAD()
6386 SDValue Ops[] = {Mem->getChain(), Ptr}; in getBROADCAST_LOAD()
6390 Mem->getMemOperand(), Offset, MemVT.getStoreSize())); in getBROADCAST_LOAD()
6408 int Elt = SV->getMaskElt(Index); in getShuffleScalarElt()
6413 SDValue Src = (Elt < (int)NumElems) ? SV->getOperand(0) : SV->getOperand(1); in getShuffleScalarElt()
6447 return getShuffleScalarElt(Sub, Index - SubIdx, DAG, Depth + 1); in getShuffleScalarElt()
6478 // For insert_vector_elt - either return the index matching scalar or recurse in getShuffleScalarElt()
6525 assert(0 == i && "Expected insertion into zero-index"); in LowerBuildVectorAsInsert()
6548 // SSE4.1 - use PINSRB to insert each byte directly. in LowerBuildVectorv16i8()
6555 // Pre-SSE4.1 - merge byte pairs and insert with PINSRW. in LowerBuildVectorv16i8()
6556 // If both the lowest 16-bits are non-zero, then convert to MOVD. in LowerBuildVectorv16i8()
6665 assert(Zeroable.size() - Zeroable.count() > 1 && in LowerBuildVectorv4x32()
6666 "We expect at least two non-zero elements!"); in LowerBuildVectorv4x32()
6679 // Make sure that this node is extracting from a 128-bit vector. in LowerBuildVectorv4x32()
6704 Elt = Op->getOperand(EltIdx); in LowerBuildVectorv4x32()
6735 SDValue Current = Op->getOperand(i); in LowerBuildVectorv4x32()
6736 SDValue SrcVector = Current->getOperand(0); in LowerBuildVectorv4x32()
6745 assert(V1.getNode() && "Expected at least two non-zero elements!"); in LowerBuildVectorv4x32()
6781 SDValue Ptr = LD->getBasePtr(); in LowerAsSplatVectorLoad()
6782 if (!ISD::isNormalLoad(LD) || !LD->isSimple()) in LowerAsSplatVectorLoad()
6784 EVT PVT = LD->getValueType(0); in LowerAsSplatVectorLoad()
6788 int FI = -1; in LowerAsSplatVectorLoad()
6791 FI = FINode->getIndex(); in LowerAsSplatVectorLoad()
6795 FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); in LowerAsSplatVectorLoad()
6802 // FIXME: 256-bit vector instructions don't require a strict alignment, in LowerAsSplatVectorLoad()
6805 SDValue Chain = LD->getChain(); in LowerAsSplatVectorLoad()
6826 int64_t StartOffset = Offset & ~int64_t(RequiredAlign.value() - 1); in LowerAsSplatVectorLoad()
6833 int EltNo = (Offset - StartOffset) >> 2; in LowerAsSplatVectorLoad()
6838 LD->getPointerInfo().getWithOffset(StartOffset)); in LowerAsSplatVectorLoad()
6852 if (!BaseLd->isSimple()) in findEltLoadSrc()
6866 uint64_t Amt = AmtC->getZExtValue(); in findEltLoadSrc()
6880 uint64_t Idx = IdxC->getZExtValue(); in findEltLoadSrc()
6895 /// Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a
6905 int LastLoadedElt = -1; in EltsFromConsecutiveLoads()
6936 unsigned LoadSizeInBits = Loads[i]->getValueSizeInBits(0); in EltsFromConsecutiveLoads()
6947 // Handle Special Cases - all undef or undef/zero. in EltsFromConsecutiveLoads()
6964 int NumLoadedElts = (1 + LastLoadedElt - FirstLoadedElt); in EltsFromConsecutiveLoads()
6966 assert((BaseSizeInBits % 8) == 0 && "Sub-byte element loads detected"); in EltsFromConsecutiveLoads()
6978 int64_t BaseIdx = EltIdx - (ByteOffset / BaseSizeInBytes); in EltsFromConsecutiveLoads()
6983 EltIdx - FirstLoadedElt); in EltsFromConsecutiveLoads()
7004 auto MMOFlags = LDBase->getMemOperand()->getFlags(); in EltsFromConsecutiveLoads()
7005 assert(LDBase->isSimple() && in EltsFromConsecutiveLoads()
7008 DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), in EltsFromConsecutiveLoads()
7009 LDBase->getPointerInfo(), LDBase->getOriginalAlign(), in EltsFromConsecutiveLoads()
7018 bool IsDereferenceable = LDBase->getPointerInfo().isDereferenceable( in EltsFromConsecutiveLoads()
7021 // LOAD - all consecutive load/undefs (must start/end with a load or be in EltsFromConsecutiveLoads()
7032 // Don't create 256-bit non-temporal aligned loads without AVX2 as these in EltsFromConsecutiveLoads()
7034 if (LDBase->isNonTemporal() && LDBase->getAlign() >= Align(32) && in EltsFromConsecutiveLoads()
7044 // IsConsecutiveLoadWithZeros - we need to create a shuffle of the loaded in EltsFromConsecutiveLoads()
7050 SmallVector<int, 4> ClearMask(NumMaskElts, -1); in EltsFromConsecutiveLoads()
7081 // VZEXT_LOAD - consecutive 32/64-bit load/undefs followed by zeros/undefs. in EltsFromConsecutiveLoads()
7095 SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() }; in EltsFromConsecutiveLoads()
7097 X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT, LDBase->getPointerInfo(), in EltsFromConsecutiveLoads()
7098 LDBase->getOriginalAlign(), MachineMemOperand::MOLoad); in EltsFromConsecutiveLoads()
7106 // BROADCAST - match the smallest possible repetition pattern, load that in EltsFromConsecutiveLoads()
7116 // Don't attempt a 1:N subvector broadcast - it should be caught by in EltsFromConsecutiveLoads()
7177 // are consecutive, non-overlapping, and in the right order.
7200 auto getConstantScalar = [&](const APInt &Val) -> Constant * { in getConstantVector()
7224 auto getConstantScalar = [&](const APInt &Val) -> Constant * { in getConstantVector()
7249 for (auto *U : N->uses()) { in isFoldableUseOfShuffle()
7250 unsigned Opc = U->getOpcode(); in isFoldableUseOfShuffle()
7252 if (Opc == X86ISD::VPERMV && U->getOperand(0).getNode() == N) in isFoldableUseOfShuffle()
7254 if (Opc == X86ISD::VPERMV3 && U->getOperand(1).getNode() == N) in isFoldableUseOfShuffle()
7260 if (N->hasOneUse()) { in isFoldableUseOfShuffle()
7263 if (Opc == X86ISD::VPDPBUSD && U->getOperand(2).getNode() != N) in isFoldableUseOfShuffle()
7283 // TODO: Splats could be generated for non-AVX CPUs using SSE in lowerBuildVectorAsBroadcast()
7284 // instructions, but there's less potential gain for only 128-bit vectors. in lowerBuildVectorAsBroadcast()
7288 MVT VT = BVOp->getSimpleValueType(0); in lowerBuildVectorAsBroadcast()
7298 if (BVOp->getRepeatedSequence(Sequence, &UndefElements)) { in lowerBuildVectorAsBroadcast()
7342 if (!Ld || (NumElts - NumUndefElts) <= 1) { in lowerBuildVectorAsBroadcast()
7347 if (BVOp->isConstantSplat(SplatValue, Undef, SplatBitSize, HasUndef) && in lowerBuildVectorAsBroadcast()
7365 Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign(); in lowerBuildVectorAsBroadcast()
7381 Align Alignment = cast<ConstantPoolSDNode>(VCP)->getAlign(); in lowerBuildVectorAsBroadcast()
7397 if (!Ld || NumElts - NumUndefElts != 1) in lowerBuildVectorAsBroadcast()
7408 // TODO: Handle broadcasts of non-constant sequences. in lowerBuildVectorAsBroadcast()
7410 // Make sure that all of the users of a non-constant load are from the in lowerBuildVectorAsBroadcast()
7412 // FIXME: Is the use count needed for non-constant, non-load case? in lowerBuildVectorAsBroadcast()
7413 if (!ConstSplatVal && !IsLoad && !BVOp->isOnlyUserOf(Ld.getNode())) in lowerBuildVectorAsBroadcast()
7431 // TODO: Check if splatting is recommended for other AVX-capable CPUs. in lowerBuildVectorAsBroadcast()
7446 C = CI->getConstantIntValue(); in lowerBuildVectorAsBroadcast()
7448 C = CF->getConstantFPValue(); in lowerBuildVectorAsBroadcast()
7454 Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign(); in lowerBuildVectorAsBroadcast()
7465 // Handle AVX2 in-register broadcasts. in lowerBuildVectorAsBroadcast()
7474 // Make sure the non-chain result is only used by this build vector. in lowerBuildVectorAsBroadcast()
7475 if (!Ld->hasNUsesOfValue(NumElts - NumUndefElts, 0)) in lowerBuildVectorAsBroadcast()
7482 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()}; in lowerBuildVectorAsBroadcast()
7485 LN->getMemoryVT(), LN->getMemOperand()); in lowerBuildVectorAsBroadcast()
7490 // The integer check is needed for the 64-bit into 128-bit so it doesn't match in lowerBuildVectorAsBroadcast()
7496 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()}; in lowerBuildVectorAsBroadcast()
7499 LN->getMemoryVT(), LN->getMemOperand()); in lowerBuildVectorAsBroadcast()
7518 int Idx = ExtIdx->getAsZExtVal(); in getUnderlyingExtractedFromVec()
7522 // For 256-bit vectors, LowerEXTRACT_VECTOR_ELT_SSE4 may have already in getUnderlyingExtractedFromVec()
7533 SDValue ShuffleVec = SVOp->getOperand(0); in getUnderlyingExtractedFromVec()
7538 int ShuffleIdx = SVOp->getMaskElt(Idx); in getUnderlyingExtractedFromVec()
7559 SmallVector<int, 8> Mask(NumElems, -1); in buildFromShuffleMostly()
7579 // Quit if non-constant index. in buildFromShuffleMostly()
7647 int SplatIdx = -1; in LowerBUILD_VECTORvXi1()
7653 Immediate |= (InC->getZExtValue() & 0x1) << idx; in LowerBUILD_VECTORvXi1()
7664 // for splat use " (select i1 splat_elt, all-ones, all-zeroes)" in LowerBUILD_VECTORvXi1()
7738 /// 128-bit partial horizontal operation on a 256-bit vector, but that operation
7739 /// may not match the layout of an x86 256-bit horizontal instruction.
7741 /// be required to produce a valid horizontal instruction.
7743 /// Parameter \p Opcode defines the kind of horizontal operation to match.
7745 /// checks if \p N implements a horizontal arithmetic add; if instead \p Opcode
7746 /// is equal to ISD::SUB, then this function checks if this is a horizontal
7753 /// horizontal operations, but the index-matching logic is incorrect for that.
7755 /// code because it is only used for partial h-op matching now?
7760 EVT VT = N->getValueType(0); in isHorizontalBinOpPart()
7761 assert(VT.is256BitVector() && "Only use for matching partial 256-bit h-ops"); in isHorizontalBinOpPart()
7769 unsigned NumElts = LastIdx - BaseIdx; in isHorizontalBinOpPart()
7773 // Check if N implements a horizontal binop. in isHorizontalBinOpPart()
7775 SDValue Op = N->getOperand(i + BaseIdx); in isHorizontalBinOpPart()
7778 if (Op->isUndef()) { in isHorizontalBinOpPart()
7786 CanFold = Op->getOpcode() == Opcode && Op->hasOneUse(); in isHorizontalBinOpPart()
7839 /// Emit a sequence of two 128-bit horizontal add/sub followed by
7843 /// This function expects two 256-bit vectors called V0 and V1.
7844 /// At first, each vector is split into two separate 128-bit vectors.
7845 /// Then, the resulting 128-bit vectors are used to implement two
7846 /// horizontal binary operations.
7848 /// The kind of horizontal binary operation is defined by \p X86Opcode.
7850 /// \p Mode specifies how the 128-bit parts of V0 and V1 are passed in input to
7851 /// the two new horizontal binop.
7852 /// When Mode is set, the first horizontal binop dag node would take as input
7853 /// the lower 128-bit of V0 and the upper 128-bit of V0. The second
7854 /// horizontal binop dag node would take as input the lower 128-bit of V1
7855 /// and the upper 128-bit of V1.
7860 /// Otherwise, the first horizontal binop dag node takes as input the lower
7861 /// 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop
7862 /// dag node takes the upper 128-bit of V0 and the upper 128-bit of V1.
7868 /// 128-bits of the result. If \p isUndefHI is set, then UNDEF is propagated to
7869 /// the upper 128-bits of the result.
7889 // Don't emit a horizontal binop if the result is expected to be UNDEF. in ExpandHorizontalBinOp()
7890 if (!isUndefLO && !V0->isUndef()) in ExpandHorizontalBinOp()
7892 if (!isUndefHI && !V1->isUndef()) in ExpandHorizontalBinOp()
7895 // Don't emit a horizontal binop if the result is expected to be UNDEF. in ExpandHorizontalBinOp()
7896 if (!isUndefLO && (!V0_LO->isUndef() || !V1_LO->isUndef())) in ExpandHorizontalBinOp()
7899 if (!isUndefHI && (!V0_HI->isUndef() || !V1_HI->isUndef())) in ExpandHorizontalBinOp()
7908 /// If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1
7909 /// (SUBADD = Opnd0 -+ Opnd1) operation are written to the parameters
7917 MVT VT = BV->getSimpleValueType(0); in isAddSubOrSubAdd()
7927 // Odd-numbered elements in the input build vector are obtained from in isAddSubOrSubAdd()
7929 // Even-numbered elements in the input build vector are obtained from in isAddSubOrSubAdd()
7933 SDValue Op = BV->getOperand(i); in isAddSubOrSubAdd()
8036 /// is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit
8043 !Opnd0->hasNUsesOfValue(ExpectedUses, 0) || !Subtarget.hasAnyFMA()) in isFMAddSubOrFMSubAdd()
8077 MVT VT = BV->getSimpleValueType(0); in lowerToAddSubOrFMAddSub()
8090 // There are no known X86 targets with 512-bit ADDSUB instructions! in lowerToAddSubOrFMAddSub()
8109 MVT VT = BV->getSimpleValueType(0); in isHopBuildVector()
8114 // x86 256-bit horizontal ops are defined in a non-obvious way. Each 128-bit in isHopBuildVector()
8115 // half of the result is calculated independently from the 128-bit halves of in isHopBuildVector()
8116 // the inputs, so that makes the index-checking logic below more complicated. in isHopBuildVector()
8125 SDValue Op = BV->getOperand(i * NumEltsIn128Bits + j); in isHopBuildVector()
8133 // Initialize horizontal opcode. in isHopBuildVector()
8137 // clang-format off in isHopBuildVector()
8143 // clang-format on in isHopBuildVector()
8156 // The source vector is chosen based on which 64-bit half of the in isHopBuildVector()
8187 // Extract indexes do not match horizontal requirement. in isHopBuildVector()
8200 // This is free (examples: zmm --> xmm, xmm --> ymm). in getHopForBuildVector()
8201 MVT VT = BV->getSimpleValueType(0); in getHopForBuildVector()
8216 if (BV->getOperand(i).isUndef()) in getHopForBuildVector()
8232 /// Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
8236 // We need at least 2 non-undef elements to make this worthwhile by default. in LowerToHorizontalOp()
8238 count_if(BV->op_values(), [](SDValue V) { return !V.isUndef(); }); in LowerToHorizontalOp()
8242 // There are 4 sets of horizontal math operations distinguished by type: in LowerToHorizontalOp()
8243 // int/FP at 128-bit/256-bit. Each type was introduced with a different in LowerToHorizontalOp()
8245 MVT VT = BV->getSimpleValueType(0); in LowerToHorizontalOp()
8256 // Try harder to match 256-bit ops by using extract/concat. in LowerToHorizontalOp()
8266 if (BV->getOperand(i)->isUndef()) in LowerToHorizontalOp()
8270 if (BV->getOperand(i)->isUndef()) in LowerToHorizontalOp()
8296 // Do not try to expand this build_vector into a pair of horizontal in LowerToHorizontalOp()
8301 // Convert this build_vector into a pair of horizontal binops followed by in LowerToHorizontalOp()
8302 // a concat vector. We must adjust the outputs from the partial horizontal in LowerToHorizontalOp()
8306 assert((!V0.isUndef() || !V1.isUndef()) && "Horizontal-op of undefs?"); in LowerToHorizontalOp()
8332 // Don't try to expand this build_vector into a pair of horizontal add/sub in LowerToHorizontalOp()
8337 // Convert this build_vector into two horizontal add/sub followed by in LowerToHorizontalOp()
8360 MVT VT = Op->getSimpleValueType(0); in lowerBuildVectorToBitOp()
8366 unsigned Opcode = Op->getOperand(0).getOpcode(); in lowerBuildVectorToBitOp()
8368 if (Opcode != Op->getOperand(i).getOpcode()) in lowerBuildVectorToBitOp()
8384 // Don't do this if the buildvector is a splat - we'd replace one in lowerBuildVectorToBitOp()
8386 if (Op->getSplatValue()) in lowerBuildVectorToBitOp()
8394 for (SDValue Elt : Op->ops()) { in lowerBuildVectorToBitOp()
8415 // TODO: Permit non-uniform XOP/AVX2/MULLO cases? in lowerBuildVectorToBitOp()
8443 // Vectors containing all ones can be matched by pcmpeqd on 128-bit width in materializeVectorConstant()
8444 // vectors or broken into v4i32 operations on 256-bit vectors. AVX2 can use in materializeVectorConstant()
8445 // vpcmpeqd on 256-bit vectors. in materializeVectorConstant()
8478 // Zero-extend the index elements within the vector. in createVariablePermute()
8515 // e.g. v4i32 -> v16i8 (Scale = 4) in createVariablePermute()
8564 // SSE41 can compare v2i64 - select between indices 0 and 1. in createVariablePermute()
8716 // Tries to lower a BUILD_VECTOR composed of extract-extract chains that can be
8717 // reasoned to be a permutation of a vector by indices in a non-constant vector.
8721 // ->
8726 // construction of vectors with constant-0 elements.
8733 // This is done by checking that the i-th build_vector operand is of the form: in LowerBUILD_VECTORAsVariablePermute()
8747 SDValue ExtractedIndex = Op->getOperand(1); in LowerBUILD_VECTORAsVariablePermute()
8764 if (!PermIdx || PermIdx->getAPIntValue() != Idx) in LowerBUILD_VECTORAsVariablePermute()
8808 OneUseFrozenUndefs = OneUseFrozenUndefs && Elt->hasOneUse(); in LowerBUILD_VECTOR()
8815 NumConstants--; in LowerBUILD_VECTOR()
8836 // If we have multiple FREEZE-UNDEF operands, we are likely going to end up in LowerBUILD_VECTOR()
8838 // our source BUILD_VECTOR, create another FREEZE-UNDEF splat BUILD_VECTOR, in LowerBUILD_VECTOR()
8839 // and blend the FREEZE-UNDEF operands back in. in LowerBUILD_VECTOR()
8840 // FIXME: is this worthwhile even for a single FREEZE-UNDEF operand? in LowerBUILD_VECTOR()
8843 SmallVector<int, 16> BlendMask(NumElems, -1); in LowerBUILD_VECTOR()
8847 BlendMask[i] = -1; in LowerBUILD_VECTOR()
8874 NumUpperUndefsOrZeros >= (NumElems - (NumElems / 4))) in LowerBUILD_VECTOR()
8875 UpperElems = NumElems - (NumElems / 4); in LowerBUILD_VECTOR()
8878 MVT LowerVT = MVT::getVectorVT(EltVT, NumElems - UpperElems); in LowerBUILD_VECTOR()
8880 DAG.getBuildVector(LowerVT, dl, Op->ops().drop_back(UpperElems)); in LowerBUILD_VECTOR()
8897 // If we are inserting one variable into a vector of non-zero constants, try in LowerBUILD_VECTOR()
8901 // constants. Insertion into a zero vector is handled as a special-case in LowerBUILD_VECTOR()
8903 if (NumConstants == NumElems - 1 && NumNonZero != 1 && in LowerBUILD_VECTOR()
8907 // Create an all-constant vector. The variable element in the old in LowerBUILD_VECTOR()
8918 ConstVecOps[i] = ConstantInt::get(Context, C->getAPIntValue()); in LowerBUILD_VECTOR()
8920 ConstVecOps[i] = ConstantFP::get(Context, C->getValueAPF()); in LowerBUILD_VECTOR()
8941 unsigned InsertC = InsIndex->getAsZExtVal(); in LowerBUILD_VECTOR()
8946 // There's no good way to insert into the high elements of a >128-bit in LowerBUILD_VECTOR()
8949 assert(Subtarget.hasAVX() && "Must have AVX with >16-byte vector"); in LowerBUILD_VECTOR()
8958 // Special case for single non-zero, non-undef, element. in LowerBUILD_VECTOR()
8963 // If we have a constant or non-constant insertion into the low element of in LowerBUILD_VECTOR()
9009 // is a non-constant being inserted into an element other than the low one, in LowerBUILD_VECTOR()
9028 if (Op.getNode()->isOnlyUserOf(Item.getNode())) in LowerBUILD_VECTOR()
9035 // handled, so this is best done with a single constant-pool load. in LowerBUILD_VECTOR()
9044 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems); in LowerBUILD_VECTOR()
9050 // If this is a splat of pairs of 32-bit elements, we can use a narrower in LowerBUILD_VECTOR()
9076 // For AVX-length vectors, build the individual 128-bit pieces and use in LowerBUILD_VECTOR()
9083 DAG.getBuildVector(HVT, dl, Op->ops().slice(0, NumElems / 2)); in LowerBUILD_VECTOR()
9085 HVT, dl, Op->ops().slice(NumElems / 2, NumElems /2)); in LowerBUILD_VECTOR()
9091 // Let legalizer expand 2-wide build_vectors. in LowerBUILD_VECTOR()
9159 assert(Values.size() > 1 && "Expected non-undef and non-splat vector"); in LowerBUILD_VECTOR()
9182 // our (non-undef) elements to the full vector width with the element in the in LowerBUILD_VECTOR()
9203 Mask.append(NumElems - Mask.size(), SM_SentinelUndef); in LowerBUILD_VECTOR()
9211 // 256-bit AVX can use the vinsertf128 instruction
9212 // to create 256-bit vectors from two other 128-bit ones.
9220 ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide"); in LowerAVXCONCAT_VECTORS()
9247 // If we have more than 2 non-zeros, build each half separately. in LowerAVXCONCAT_VECTORS()
9250 ArrayRef<SDUse> Ops = Op->ops(); in LowerAVXCONCAT_VECTORS()
9279 // k-register.
9306 // If we are inserting non-zero vector and there are zeros in LSBs and undef in LowerCONCAT_VECTORSvXi1()
9310 Log2_64(NonZeros) != NumOperands - 1) { in LowerCONCAT_VECTORSvXi1()
9322 // If there are zero or one non-zeros we can handle this very simply. in LowerCONCAT_VECTORSvXi1()
9336 ArrayRef<SDUse> Ops = Op->ops(); in LowerCONCAT_VECTORSvXi1()
9367 // AVX can use the vinsertf128 instruction to create 256-bit vectors in LowerCONCAT_VECTORS()
9368 // from two other 128-bit ones. in LowerCONCAT_VECTORS()
9370 // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors in LowerCONCAT_VECTORS()
9374 //===----------------------------------------------------------------------===//
9383 //===----------------------------------------------------------------------===//
9385 /// Tiny helper function to identify a no-op mask.
9388 /// array input, which is assumed to be a single-input shuffle mask of the kind
9391 /// in-place shuffle are 'no-op's.
9394 assert(Mask[i] >= -1 && "Out of bound mask element!"); in isNoopShuffleMask()
9404 /// X86 divides up its shuffles into in-lane and cross-lane shuffle operations
9420 /// Test whether there are elements crossing 128-bit lanes in this
9427 /// from multiple lanes - this is different to isLaneCrossingShuffleMask to
9440 int SrcLane = -1; in isMultiLaneShuffleMask()
9455 /// Test whether a shuffle mask is equivalent within each sub-lane.
9458 /// lane-relative shuffle in each sub-lane. This trivially implies
9459 /// that it is also not lane-crossing. It may however involve a blend from the
9463 /// non-trivial to compute in the face of undef lanes. The representation is
9464 /// suitable for use with existing 128-bit shuffles as entries from the second
9470 RepeatedMask.assign(LaneSize, -1); in isRepeatedShuffleMask()
9480 // Ok, handle the in-lane shuffles by detecting if and when they repeat. in isRepeatedShuffleMask()
9485 // This is the first non-undef entry in this slot of a 128-bit lane. in isRepeatedShuffleMask()
9494 /// Test whether a shuffle mask is equivalent within each 128-bit lane.
9507 /// Test whether a shuffle mask is equivalent within each 256-bit lane.
9514 /// Test whether a target shuffle mask is equivalent within each sub-lane.
9537 // Handle the in-lane shuffles by detecting if and when they repeat. Adjust in isRepeatedTargetShuffleMask()
9542 // This is the first non-undef entry in this slot of a 128-bit lane. in isRepeatedTargetShuffleMask()
9551 /// Test whether a target shuffle mask is equivalent within each sub-lane.
9620 /// each element of the mask is either -1 (signifying undef) or the value given
9630 assert(Mask[i] >= -1 && "Out of bound mask element!"); in isShuffleEquivalent()
9636 MaskIdx = MaskIdx < Size ? MaskIdx : (MaskIdx - Size); in isShuffleEquivalent()
9637 ExpectedIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size); in isShuffleEquivalent()
9649 /// If an element in Mask matches SM_SentinelUndef (-1) then the corresponding
9666 // Check for out-of-range target shuffle mask indices. in isTargetShuffleEquivalent()
9693 int BitIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size); in isTargetShuffleEquivalent()
9702 MaskIdx = MaskIdx < Size ? MaskIdx : (MaskIdx - Size); in isTargetShuffleEquivalent()
9703 ExpectedIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size); in isTargetShuffleEquivalent()
9733 // Create 128-bit vector type based on mask size. in is128BitUnpackShuffleMask()
9766 /// Get a 4-lane 8-bit shuffle immediate for a mask.
9768 /// This helper function produces an 8-bit shuffle immediate corresponding to
9775 assert(Mask.size() == 4 && "Only 4-lane shuffle masks"); in getV4X86ShuffleImm()
9776 assert(Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!"); in getV4X86ShuffleImm()
9777 assert(Mask[1] >= -1 && Mask[1] < 4 && "Out of bound mask element!"); in getV4X86ShuffleImm()
9778 assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!"); in getV4X86ShuffleImm()
9779 assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!"); in getV4X86ShuffleImm()
9781 // If the mask only uses one non-undef element, then fully 'splat' it to in getV4X86ShuffleImm()
9783 int FirstIndex = find_if(Mask, [](int M) { return M >= 0; }) - Mask.begin(); in getV4X86ShuffleImm()
9808 // The function looks for a sub-mask that the nonzero elements are in
9809 // increasing order. If such sub-mask exist. The function returns true.
9813 int NextElement = -1; in isNonZeroElementsInOrder()
9817 assert(Mask[i] >= -1 && "Out of bound mask element!"); in isNonZeroElementsInOrder()
10031 /// Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit)
10032 /// followed by unpack 256-bit.
10048 // This is a "natural" unpack operation (rather than the 128-bit sectored in lowerShuffleWithUNPCK256()
10049 // operation implemented by AVX). We need to rearrange 64-bit chunks of the in lowerShuffleWithUNPCK256()
10076 unsigned UpperElts = NumElts - NumSrcElts; in matchShuffleAsVTRUNC()
10126 // Non-VLX targets must truncate from a 512-bit type, so we need to in getAVX512TruncNode()
10173 unsigned UpperElts = NumElts - NumSrcElts; in lowerShuffleWithVPMOV()
10222 // TODO: Support non-BWI VPMOVWB truncations? in lowerShuffleAsVTRUNC()
10237 unsigned UpperElts = NumElts - NumSrcElts; in lowerShuffleAsVTRUNC()
10264 // and truncate from the double-sized src. in lowerShuffleAsVTRUNC()
10307 /// FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here
10318 "We should only be called with masks with a power-of-2 size!"); in canLowerByDroppingElements()
10320 uint64_t ModMask = (uint64_t)ShuffleModulus - 1; in canLowerByDroppingElements()
10323 // We track whether the input is viable for all power-of-2 strides 2^1, 2^2, in canLowerByDroppingElements()
10340 if ((uint64_t)(Mask[i] - Offset) == (((uint64_t)i << N) & ModMask)) in canLowerByDroppingElements()
10374 unsigned NumPackedBits = NumSrcBits - BitSize; in matchShuffleWithPACK()
10448 // Don't lower multi-stage packs on AVX512, truncation is better. in lowerShuffleWithPACK()
10453 // vXi64/vXi32 -> PACK*SDW and vXi16 -> PACK*SWB. in lowerShuffleWithPACK()
10524 return SDValue(); // No non-zeroable elements! in lowerShuffleAsBitMask()
10579 // For 32/64-bit elements, if we only reference one input (plus any undefs), in matchShuffleAsBlend()
10604 IsElementEquivalent(NumElts, V2, V2, M - NumElts, Elt))) { in matchShuffleAsBlend()
10632 LaneBlendMask = (1ull << NumEltsPerLane) - 1; in matchShuffleAsBlend()
10657 // Create a REAL zero vector - ISD::isBuildVectorAllZeros allows UNDEFs. in lowerShuffleAsBlend()
10668 assert(Subtarget.hasAVX2() && "256-bit integer blends require AVX2!"); in lowerShuffleAsBlend()
10672 assert(Subtarget.hasAVX() && "256-bit float blends require AVX!"); in lowerShuffleAsBlend()
10679 assert(Subtarget.hasSSE41() && "128-bit blends require SSE41!"); in lowerShuffleAsBlend()
10686 // We can lower these with PBLENDW which is mirrored across 128-bit lanes. in lowerShuffleAsBlend()
10696 // TODO - we should allow 2 PBLENDW here and leave shuffle combine to in lowerShuffleAsBlend()
10712 assert(Subtarget.hasAVX2() && "256-bit byte-blends require AVX2!"); in lowerShuffleAsBlend()
10715 assert(Subtarget.hasSSE41() && "128-bit byte-blends require SSE41!"); in lowerShuffleAsBlend()
10743 // If V2 can be load-folded and V1 cannot be load-folded, then commute to in lowerShuffleAsBlend()
10744 // allow that load-folding possibility. in lowerShuffleAsBlend()
10753 // are -1 for true and 0 for false. We then use the LLVM semantics of 'true' in lowerShuffleAsBlend()
10755 // reality in x86 is that vector masks (pre-AVX-512) use only the high bit in lowerShuffleAsBlend()
10766 : DAG.getConstant(Mask[i] < Size ? -1 : 0, DL, in lowerShuffleAsBlend()
10790 // Otherwise load an immediate into a GPR, cast to k-register, and use a in lowerShuffleAsBlend()
10802 /// a single-input permutation.
10805 /// then reduce the shuffle to a single-input permutation.
10813 SmallVector<int, 32> BlendMask(Mask.size(), -1); in lowerShuffleAsBlendAndPermute()
10814 SmallVector<int, 32> PermuteMask(Mask.size(), -1); in lowerShuffleAsBlendAndPermute()
10841 /// a single-input permutation.
10844 /// then reduce the shuffle to a single-input (wider) permutation.
10870 NormM -= NumElts; in lowerShuffleAsUNPCKAndPermute()
10895 SmallVector<int, 32> PermuteMask(NumElts, -1); in lowerShuffleAsUNPCKAndPermute()
10902 NormM -= NumElts; in lowerShuffleAsUNPCKAndPermute()
10910 assert(PermuteMask[Elt] != -1 && in lowerShuffleAsUNPCKAndPermute()
10936 // This routine only supports 128-bit integer dual input vectors. in lowerShuffleAsPermuteAndUnpack()
10948 SmallVector<int, 16> V1Mask((unsigned)Size, -1); in lowerShuffleAsPermuteAndUnpack()
10949 SmallVector<int, 16> V2Mask((unsigned)Size, -1); in lowerShuffleAsPermuteAndUnpack()
11005 // If none of the unpack-rooted lowerings worked (or were profitable) try an in lowerShuffleAsPermuteAndUnpack()
11014 // half-crossings are created. in lowerShuffleAsPermuteAndUnpack()
11017 SmallVector<int, 32> PermMask((unsigned)Size, -1); in lowerShuffleAsPermuteAndUnpack()
11025 2 * ((Mask[i] % Size) - HalfOffset) + (Mask[i] < Size ? 0 : 1); in lowerShuffleAsPermuteAndUnpack()
11037 /// Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then
11073 M -= NumElts; in lowerShuffleAsByteRotateAndPermute()
11084 // TODO - it might be worth doing this for unary shuffles if the permute in lowerShuffleAsByteRotateAndPermute()
11107 PermMask[Lane + Elt] = Lane + ((M + Ofs - RotAmt) % NumEltsPerLane); in lowerShuffleAsByteRotateAndPermute()
11109 PermMask[Lane + Elt] = Lane + ((M - Ofs - RotAmt) % NumEltsPerLane); in lowerShuffleAsByteRotateAndPermute()
11166 SmallVector<int, 32> V1Mask(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
11167 SmallVector<int, 32> V2Mask(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
11168 SmallVector<int, 32> FinalMask(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
11176 V2Mask[i] = M - NumElts; in lowerShuffleAsDecomposedShuffleMerge()
11184 // and change \p InputMask to be a no-op (identity) mask. in lowerShuffleAsDecomposedShuffleMerge()
11205 // It is possible that the shuffle for one of the inputs is already a no-op. in lowerShuffleAsDecomposedShuffleMerge()
11206 // See if we can simplify non-no-op shuffles into broadcasts, in lowerShuffleAsDecomposedShuffleMerge()
11215 // one of the input shuffles would be a no-op. We prefer to shuffle inputs as in lowerShuffleAsDecomposedShuffleMerge()
11217 // we'll have to do 2x as many shuffles in order to achieve this, a 2-input in lowerShuffleAsDecomposedShuffleMerge()
11218 // pre-shuffle first is a better strategy. in lowerShuffleAsDecomposedShuffleMerge()
11238 // Unpack/rotate failed - try again with variable blends. in lowerShuffleAsDecomposedShuffleMerge()
11250 // TODO: It doesn't have to be alternating - but each lane mustn't have more in lowerShuffleAsDecomposedShuffleMerge()
11253 V1Mask.assign(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
11254 V2Mask.assign(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
11255 FinalMask.assign(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
11263 V2Mask[i + (j / 2)] = M - NumElts; in lowerShuffleAsDecomposedShuffleMerge()
11277 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!"); in matchShuffleAsBitRotate()
11278 assert(EltSizeInBits < 64 && "Can't rotate 64-bit integers"); in matchShuffleAsBitRotate()
11286 return -1; in matchShuffleAsBitRotate()
11311 // For pre-SSSE3 targets, if we are shuffling vXi8 elts then ISD::ROTL, in lowerShuffleAsBitRotate()
11319 unsigned SrlAmt = RotateVT.getScalarSizeInBits() - RotateAmt; in lowerShuffleAsBitRotate()
11344 // [-1, 12, 13, 14, -1, -1, 1, -1] in matchShuffleAsElementRotate()
11345 // [-1, -1, -1, -1, -1, -1, 1, 2] in matchShuffleAsElementRotate()
11347 // [-1, 4, 5, 6, -1, -1, 9, -1] in matchShuffleAsElementRotate()
11348 // [-1, 4, 5, 6, -1, -1, -1, -1] in matchShuffleAsElementRotate()
11359 int StartIdx = i - (M % NumElts); in matchShuffleAsElementRotate()
11362 return -1; in matchShuffleAsElementRotate()
11367 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx; in matchShuffleAsElementRotate()
11373 return -1; in matchShuffleAsElementRotate()
11390 return -1; in matchShuffleAsElementRotate()
11410 /// byte-rotation of the concatenation of two vectors; pre-SSSE3 can use
11421 /// specified as a *right shift* because x86 is little-endian, it is a *left
11427 return -1; in matchShuffleAsByteRotate()
11429 // PALIGNR works on 128-bit lanes. in matchShuffleAsByteRotate()
11432 return -1; in matchShuffleAsByteRotate()
11436 return -1; in matchShuffleAsByteRotate()
11449 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!"); in lowerShuffleAsByteRotate()
11465 "512-bit PALIGNR requires BWI instructions"); in lowerShuffleAsByteRotate()
11472 "Rotate-based lowering only supports 128-bit lowering!"); in lowerShuffleAsByteRotate()
11474 "Can shuffle at most 16 bytes in a 128-bit vector!"); in lowerShuffleAsByteRotate()
11479 int LoByteShift = 16 - ByteRotation; in lowerShuffleAsByteRotate()
11500 /// specified as a *right shift* because x86 is little-endian, it is a *left
11508 "Only 32-bit and 64-bit elements are supported!"); in lowerShuffleAsVALIGN()
11510 // 128/256-bit vectors are only supported with VLX. in lowerShuffleAsVALIGN()
11512 && "VLX required for 128/256-bit vectors"); in lowerShuffleAsVALIGN()
11520 // See if we can use VALIGN as a cross-lane version of VSHLDQ/VSRLDQ. in lowerShuffleAsVALIGN()
11522 // TODO: We can probably make this more aggressive and use shift-pairs like in lowerShuffleAsVALIGN()
11534 if (isSequentialOrUndefInRange(Mask, ZeroLo, NumElts - ZeroLo, Low)) in lowerShuffleAsVALIGN()
11537 DAG.getTargetConstant(NumElts - ZeroLo, DL, MVT::i8)); in lowerShuffleAsVALIGN()
11543 if (isSequentialOrUndefInRange(Mask, 0, NumElts - ZeroHi, Low + ZeroHi)) in lowerShuffleAsVALIGN()
11558 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!"); in lowerShuffleAsByteShiftMask()
11559 assert(VT.is128BitVector() && "Only 128-bit vectors supported"); in lowerShuffleAsByteShiftMask()
11569 unsigned Len = NumElts - (ZeroLo + ZeroHi); in lowerShuffleAsByteShiftMask()
11584 // 01234567 --> zzzzzz01 --> 1zzzzzzz in lowerShuffleAsByteShiftMask()
11585 // 01234567 --> 4567zzzz --> zzzzz456 in lowerShuffleAsByteShiftMask()
11586 // 01234567 --> z0123456 --> 3456zzzz --> zz3456zz in lowerShuffleAsByteShiftMask()
11588 unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts); in lowerShuffleAsByteShiftMask()
11603 unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts); in lowerShuffleAsByteShiftMask()
11620 /// PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function
11623 /// bit-wise element shifts and the byte shift across an entire 128-bit double
11626 /// PSHL : (little-endian) left bit shift.
11628 /// [ -1, 4, zz, -1 ]
11629 /// PSRL : (little-endian) right bit shift.
11631 /// [ -1, -1, 7, zz]
11632 /// PSLLDQ : (little-endian) left byte shift
11634 /// [ zz, zz, -1, -1, 2, 3, 4, -1]
11635 /// [ zz, zz, zz, zz, zz, zz, -1, 1]
11636 /// PSRLDQ : (little-endian) right byte shift
11638 /// [ -1, 5, 6, 7, zz, zz, zz, zz]
11639 /// [ 1, 2, -1, -1, -1, -1, zz, zz]
11650 if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))]) in matchShuffleAsShift()
11660 unsigned Len = Scale - Shift; in matchShuffleAsShift()
11662 return -1; in matchShuffleAsShift()
11682 // SSE/AVX supports logical shifts up to 64-bit integers - so we can just in matchShuffleAsShift()
11699 return -1; in matchShuffleAsShift()
11756 for (; Len > 0; --Len) in matchShuffleAsEXTRQ()
11757 if (!Zeroable[Len - 1]) in matchShuffleAsEXTRQ()
11763 int Idx = -1; in matchShuffleAsEXTRQ()
11776 if (Idx < 0 || (Src == V && Idx == (M - i))) { in matchShuffleAsEXTRQ()
11778 Idx = M - i; in matchShuffleAsEXTRQ()
11796 // { A[0], .., A[Idx-1], B[0], .., B[Len-1], A[Idx+Len], .., UNDEF, ... }
11826 int Len = Hi - Idx; in matchShuffleAsINSERTQ()
11838 if (isUndefInRange(Mask, Hi, HalfSize - Hi)) { in matchShuffleAsINSERTQ()
11841 isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi, Hi)) { in matchShuffleAsINSERTQ()
11844 isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi, in matchShuffleAsINSERTQ()
11915 SmallVector<int, 8> ShMask((unsigned)NumElements, -1); in lowerShuffleAsSpecificZeroOrAnyExtend()
11918 ShMask[i] = SafeOffset(SrcIdx) ? SrcIdx : -1; in lowerShuffleAsSpecificZeroOrAnyExtend()
11926 // Not worth offsetting 128-bit vectors if scale == 2, a pattern using in lowerShuffleAsSpecificZeroOrAnyExtend()
11939 assert(VT.is128BitVector() && "Only 128-bit vectors can be extended."); in lowerShuffleAsSpecificZeroOrAnyExtend()
11945 int PSHUFDMask[4] = {Offset, -1, SafeOffset(Offset + 1) ? Offset + 1 : -1, in lowerShuffleAsSpecificZeroOrAnyExtend()
11946 -1}; in lowerShuffleAsSpecificZeroOrAnyExtend()
11953 int PSHUFDMask[4] = {Offset / 2, -1, in lowerShuffleAsSpecificZeroOrAnyExtend()
11954 SafeOffset(Offset + 1) ? (Offset + 1) / 2 : -1, -1}; in lowerShuffleAsSpecificZeroOrAnyExtend()
11958 int PSHUFWMask[4] = {1, -1, -1, -1}; in lowerShuffleAsSpecificZeroOrAnyExtend()
11967 // to 64-bits. in lowerShuffleAsSpecificZeroOrAnyExtend()
12015 SmallVector<int, 8> ShMask((unsigned)NumElements, -1); in lowerShuffleAsSpecificZeroOrAnyExtend()
12017 ShMask[i - AlignToUnpack] = i; in lowerShuffleAsSpecificZeroOrAnyExtend()
12019 Offset -= AlignToUnpack; in lowerShuffleAsSpecificZeroOrAnyExtend()
12027 Offset -= (NumElements / 2); in lowerShuffleAsSpecificZeroOrAnyExtend()
12047 /// match this pattern. It will use all of the micro-architectural details it
12048 /// can to emit an efficient lowering. It handles both blends with all-zero
12049 /// inputs to explicitly zero-extend and undef-lanes (sometimes undef due to
12052 /// The reason we have dedicated lowering for zext-style shuffles is that they
12063 "Exceeds 32-bit integer zero extension limit"); in lowerShuffleAsZeroOrAnyExtend()
12066 // Define a helper function to check a particular ext-scale and lower to it if in lowerShuffleAsZeroOrAnyExtend()
12068 auto Lower = [&](int Scale) -> SDValue { in lowerShuffleAsZeroOrAnyExtend()
12093 Offset = M - (i / Scale); in lowerShuffleAsZeroOrAnyExtend()
12095 return SDValue(); // Flip-flopping inputs. in lowerShuffleAsZeroOrAnyExtend()
12097 // Offset must start in the lowest 128-bit lane or at the start of an in lowerShuffleAsZeroOrAnyExtend()
12110 return SDValue(); // Non-consecutive strided elements. in lowerShuffleAsZeroOrAnyExtend()
12114 // If we fail to find an input, we have a zero-shuffle which should always in lowerShuffleAsZeroOrAnyExtend()
12129 // The widest scale possible for extending is to a 64-bit integer. in lowerShuffleAsZeroOrAnyExtend()
12143 // General extends failed, but 128-bit vectors may be able to use MOVQ. in lowerShuffleAsZeroOrAnyExtend()
12148 // MOVQ, copying the lower 64-bits and zero-extending to the upper 64-bits. in lowerShuffleAsZeroOrAnyExtend()
12202 return V->hasOneUse() && in isShuffleFoldableLoad()
12229 find_if(Mask, [&Mask](int M) { return M >= (int)Mask.size(); }) - in lowerShuffleAsElementInsertion()
12239 // Bail if a non-zero V1 isn't used in place. in lowerShuffleAsElementInsertion()
12242 V1Mask[V2Index] = -1; in lowerShuffleAsElementInsertion()
12252 SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(), in lowerShuffleAsElementInsertion()
12258 // Using zext to expand a narrow element won't work for non-zero in lowerShuffleAsElementInsertion()
12264 // Zero-extend directly to i32. in lowerShuffleAsElementInsertion()
12269 // and OR with the zero-extended scalar. in lowerShuffleAsElementInsertion()
12290 // this. We can't support integer vectors or non-zero targets cheaply. in lowerShuffleAsElementInsertion()
12291 assert(VT == ExtVT && "Cannot change extended type when non-zeroable!"); in lowerShuffleAsElementInsertion()
12338 /// Try to lower broadcast of a single - truncated - integer element,
12352 assert(VT.isInteger() && "Unexpected non-integer trunc broadcast!"); in lowerShuffleAsTruncBroadcast()
12353 assert(V0VT.isVector() && "Unexpected non-vector vector-sized value!"); in lowerShuffleAsTruncBroadcast()
12379 // If we're extracting non-least-significant bits, shift so we can truncate. in lowerShuffleAsTruncBroadcast()
12396 // This routine only handles 128-bit shufps. in isSingleSHUFPSMask()
12398 assert(Mask[0] >= -1 && Mask[0] < 8 && "Out of bound mask element!"); in isSingleSHUFPSMask()
12399 assert(Mask[1] >= -1 && Mask[1] < 8 && "Out of bound mask element!"); in isSingleSHUFPSMask()
12400 assert(Mask[2] >= -1 && Mask[2] < 8 && "Out of bound mask element!"); in isSingleSHUFPSMask()
12401 assert(Mask[3] >= -1 && Mask[3] < 8 && "Out of bound mask element!"); in isSingleSHUFPSMask()
12413 /// Test whether the specified input (0 or 1) is in-place blended by the
12428 /// If we are extracting two 128-bit halves of a vector and shuffling the
12429 /// result, match that to a 256-bit AVX2 vperm* instruction to avoid a
12430 /// multi-shuffle lowering.
12437 "VPERM* family of shuffles requires 32-bit or 64-bit elements"); in lowerShuffleOfExtractsAsVperm()
12470 NewMask.append(NumElts, -1); in lowerShuffleOfExtractsAsVperm()
12472 // shuf (extract X, 0), (extract X, 4), M --> extract (shuf X, undef, M'), 0 in lowerShuffleOfExtractsAsVperm()
12475 // This is free: ymm -> xmm. in lowerShuffleOfExtractsAsVperm()
12483 /// filtering. While a little annoying to re-dispatch on type here, there isn't
12547 BitOffset -= BeginOffset; in lowerShuffleAsBroadcast()
12557 assert((BitOffset % NumEltBits) == 0 && "Illegal bit-offset"); in lowerShuffleAsBroadcast()
12583 cast<LoadSDNode>(V)->isSimple()) { in lowerShuffleAsBroadcast()
12584 // We do not check for one-use of the vector load because a broadcast load in lowerShuffleAsBroadcast()
12590 SDValue BaseAddr = Ld->getOperand(1); in lowerShuffleAsBroadcast()
12593 assert((int)(Offset * 8) == BitOffset && "Unexpected bit-offset"); in lowerShuffleAsBroadcast()
12599 // FIXME: Should we add VBROADCAST_LOAD isel patterns for pre-AVX? in lowerShuffleAsBroadcast()
12602 SDValue Ops[] = {Ld->getChain(), NewAddr}; in lowerShuffleAsBroadcast()
12606 Ld->getMemOperand(), Offset, SVT.getStoreSize())); in lowerShuffleAsBroadcast()
12611 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr, in lowerShuffleAsBroadcast()
12613 Ld->getMemOperand(), Offset, SVT.getStoreSize())); in lowerShuffleAsBroadcast()
12619 // We can only broadcast from the zero-element of a vector register, in lowerShuffleAsBroadcast()
12620 // but it can be advantageous to broadcast from the zero-element of a in lowerShuffleAsBroadcast()
12625 // VPERMQ/VPERMPD can perform the cross-lane shuffle directly. in lowerShuffleAsBroadcast()
12629 // Only broadcast the zero-element of a 128-bit subvector. in lowerShuffleAsBroadcast()
12634 "Unexpected bit-offset"); in lowerShuffleAsBroadcast()
12660 // We only support broadcasting from 128-bit vectors to minimize the in lowerShuffleAsBroadcast()
12662 // 128-bits, removing as many bitcasts as possible. in lowerShuffleAsBroadcast()
12693 int VADstIndex = -1; in matchShuffleAsInsertPS()
12694 int VBDstIndex = -1; in matchShuffleAsInsertPS()
12710 // We can only insert a single non-zeroable element. in matchShuffleAsInsertPS()
12723 // Don't bother if we have no (non-zeroable) element for insertion. in matchShuffleAsInsertPS()
12737 VBSrcIndex = CandidateMask[VBDstIndex] - 4; in matchShuffleAsInsertPS()
12741 // the zero mask and the V2 insertion - so remove V1 dependency. in matchShuffleAsInsertPS()
12783 /// Handle lowering of 2-lane 64-bit floating point shuffles.
12785 /// This is the basis function for the 2-lane 64-bit shuffles as we have full
12821 assert(Mask[0] >= 0 && "No undef lanes in multi-input v2 shuffles!"); in lowerV2F64Shuffle()
12822 assert(Mask[1] >= 0 && "No undef lanes in multi-input v2 shuffles!"); in lowerV2F64Shuffle()
12837 int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2), in lowerV2F64Shuffle()
12838 Mask[1] < 0 ? -1 : (Mask[1] ^ 2)}; in lowerV2F64Shuffle()
12844 // blend patterns if a zero-blend above didn't work. in lowerV2F64Shuffle()
12863 unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1); in lowerV2F64Shuffle()
12868 /// Handle lowering of 2-lane 64-bit integer shuffles.
12870 /// Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by
12892 int WidenedMask[4] = {Mask[0] < 0 ? -1 : (Mask[0] * 2), in lowerV2I64Shuffle()
12893 Mask[0] < 0 ? -1 : ((Mask[0] * 2) + 1), in lowerV2I64Shuffle()
12894 Mask[1] < 0 ? -1 : (Mask[1] * 2), in lowerV2I64Shuffle()
12895 Mask[1] < 0 ? -1 : ((Mask[1] * 2) + 1)}; in lowerV2I64Shuffle()
12901 assert(Mask[0] != -1 && "No undef lanes in multi-input v2 shuffles!"); in lowerV2I64Shuffle()
12902 assert(Mask[1] != -1 && "No undef lanes in multi-input v2 shuffles!"); in lowerV2I64Shuffle()
12941 // Its more profitable for pre-SSSE3 to use shuffles/unpacks. in lowerV2I64Shuffle()
12982 int V2Index = find_if(Mask, [](int M) { return M >= 4; }) - Mask.begin(); in lowerShuffleWithSHUFPS()
12994 NewMask[V2Index] -= 4; in lowerShuffleWithSHUFPS()
12999 int BlendMask[4] = {Mask[V2Index] - 4, 0, Mask[V1Index], 0}; in lowerShuffleWithSHUFPS()
13018 NewMask[2] -= 4; in lowerShuffleWithSHUFPS()
13019 NewMask[3] -= 4; in lowerShuffleWithSHUFPS()
13024 NewMask[0] -= 4; in lowerShuffleWithSHUFPS()
13025 NewMask[1] -= 4; in lowerShuffleWithSHUFPS()
13037 (Mask[0] >= 4 ? Mask[0] : Mask[1]) - 4, in lowerShuffleWithSHUFPS()
13038 (Mask[2] >= 4 ? Mask[2] : Mask[3]) - 4}; in lowerShuffleWithSHUFPS()
13061 /// Lower 4-lane 32-bit floating point shuffles.
13128 // There are special ways we can lower some single-element blends. However, we in lowerV4F32Shuffle()
13129 // have custom ways we can lower more complex single-element blends below that in lowerV4F32Shuffle()
13131 // when the V2 input is targeting element 0 of the mask -- that is the fast in lowerV4F32Shuffle()
13166 /// Lower 4-lane i32 vector shuffles.
13168 /// We try to handle these with integer-domain shuffles where we can, but for
13200 // Try to use broadcast unless the mask only has one non-undef element. in lowerV4I32Shuffle()
13233 // There are special ways we can lower some single-element blends. in lowerV4I32Shuffle()
13256 // Its more profitable for pre-SSSE3 to use shuffles/unpacks. in lowerV4I32Shuffle()
13295 /// Lowering of single-input v8i16 shuffles is the cornerstone of SSE2
13307 /// This code also handles repeated 128-bit lanes of v8i16 shuffles, but each
13309 /// this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16
13310 /// vector, form the analogous 128-bit 8-element Mask.
13330 HiMask[i] = (HiMask[i] < 0 ? HiMask[i] : (HiMask[i] - 4)); in lowerV8I16GeneralSingleInputShuffle()
13343 int NumLToL = llvm::lower_bound(LoInputs, 4) - LoInputs.begin(); in lowerV8I16GeneralSingleInputShuffle()
13344 int NumHToL = LoInputs.size() - NumLToL; in lowerV8I16GeneralSingleInputShuffle()
13345 int NumLToH = llvm::lower_bound(HiInputs, 4) - HiInputs.begin(); in lowerV8I16GeneralSingleInputShuffle()
13346 int NumHToH = HiInputs.size() - NumLToH; in lowerV8I16GeneralSingleInputShuffle()
13352 // If we are shuffling values from one half - check how many different DWORD in lowerV8I16GeneralSingleInputShuffle()
13366 int PSHUFDMask[4] = { -1, -1, -1, -1 }; in lowerV8I16GeneralSingleInputShuffle()
13398 DWordPairs.resize(2, std::make_pair(-1, -1)); in lowerV8I16GeneralSingleInputShuffle()
13408 // Simplify the 1-into-3 and 3-into-1 cases with a single pshufd. For all in lowerV8I16GeneralSingleInputShuffle()
13413 // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h] in lowerV8I16GeneralSingleInputShuffle()
13414 // Mask: [0, 1, 2, 7, 4, 5, 6, 3] -----------------> [0, 1, 4, 7, 2, 3, 6, 5] in lowerV8I16GeneralSingleInputShuffle()
13416 // However in some very rare cases we have a 1-into-3 or 3-into-1 on one half in lowerV8I16GeneralSingleInputShuffle()
13417 // and an existing 2-into-2 on the other half. In this case we may have to in lowerV8I16GeneralSingleInputShuffle()
13418 // pre-shuffle the 2-into-2 half to avoid turning it into a 3-into-1 or in lowerV8I16GeneralSingleInputShuffle()
13419 // 1-into-3 which could cause us to cycle endlessly fixing each side in turn. in lowerV8I16GeneralSingleInputShuffle()
13420 // Fortunately, we don't have to handle anything but a 2-into-2 pattern in lowerV8I16GeneralSingleInputShuffle()
13421 // because any other situation (including a 3-into-1 or 1-into-3 in the other in lowerV8I16GeneralSingleInputShuffle()
13422 // half than the one we target for fixing) will be fixed when we re-enter this in lowerV8I16GeneralSingleInputShuffle()
13426 // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h] in lowerV8I16GeneralSingleInputShuffle()
13427 // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -THIS-IS-BAD!!!!-> [5, 7, 1, 0, 4, 7, 5, 3] in lowerV8I16GeneralSingleInputShuffle()
13429 // This now has a 1-into-3 in the high half! Instead, we do two shuffles: in lowerV8I16GeneralSingleInputShuffle()
13431 // Input: [a, b, c, d, e, f, g, h] PSHUFHW[0,2,1,3]-> [a, b, c, d, e, g, f, h] in lowerV8I16GeneralSingleInputShuffle()
13432 // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -----------------> [3, 7, 1, 0, 2, 7, 3, 6] in lowerV8I16GeneralSingleInputShuffle()
13434 // Input: [a, b, c, d, e, g, f, h] -PSHUFD[0,2,1,3]-> [a, b, e, g, c, d, f, h] in lowerV8I16GeneralSingleInputShuffle()
13435 // Mask: [3, 7, 1, 0, 2, 7, 3, 6] -----------------> [5, 7, 1, 0, 4, 7, 5, 6] in lowerV8I16GeneralSingleInputShuffle()
13462 TripleInputSum - std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0); in lowerV8I16GeneralSingleInputShuffle()
13469 // Check for one tricky case: We're fixing a 3<-1 or a 1<-3 shuffle for AToA in lowerV8I16GeneralSingleInputShuffle()
13471 // inputs, we don't try to fix it necessarily -- we'll recurse and see it in in lowerV8I16GeneralSingleInputShuffle()
13472 // the next pass. However, if we have a 2<-2 in the BToB and AToB inputs, it in lowerV8I16GeneralSingleInputShuffle()
13473 // is essential that we don't *create* a 3<-1 as then we might oscillate. in lowerV8I16GeneralSingleInputShuffle()
13477 // to balance this to ensure we don't form a 3-1 shuffle in the other in lowerV8I16GeneralSingleInputShuffle()
13545 // Recurse back into this routine to re-compute state now that this isn't in lowerV8I16GeneralSingleInputShuffle()
13559 int PSHUFLMask[4] = {-1, -1, -1, -1}; in lowerV8I16GeneralSingleInputShuffle()
13560 int PSHUFHMask[4] = {-1, -1, -1, -1}; in lowerV8I16GeneralSingleInputShuffle()
13561 int PSHUFDMask[4] = {-1, -1, -1, -1}; in lowerV8I16GeneralSingleInputShuffle()
13564 // original halves. This will then dictate the targets of the cross-half in lowerV8I16GeneralSingleInputShuffle()
13573 SourceHalfMask[InPlaceInputs[0] - HalfOffset] = in lowerV8I16GeneralSingleInputShuffle()
13574 InPlaceInputs[0] - HalfOffset; in lowerV8I16GeneralSingleInputShuffle()
13581 SourceHalfMask[Input - HalfOffset] = Input - HalfOffset; in lowerV8I16GeneralSingleInputShuffle()
13588 SourceHalfMask[InPlaceInputs[0] - HalfOffset] = in lowerV8I16GeneralSingleInputShuffle()
13589 InPlaceInputs[0] - HalfOffset; in lowerV8I16GeneralSingleInputShuffle()
13593 SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset; in lowerV8I16GeneralSingleInputShuffle()
13600 // Now gather the cross-half inputs and place them into a free dword of in lowerV8I16GeneralSingleInputShuffle()
13603 // look more like the 3-1 fixing operation. in lowerV8I16GeneralSingleInputShuffle()
13628 if (isWordClobbered(SourceHalfMask, Input - SourceOffset)) { in lowerV8I16GeneralSingleInputShuffle()
13629 if (SourceHalfMask[SourceHalfMask[Input - SourceOffset]] < 0) { in lowerV8I16GeneralSingleInputShuffle()
13630 SourceHalfMask[SourceHalfMask[Input - SourceOffset]] = in lowerV8I16GeneralSingleInputShuffle()
13631 Input - SourceOffset; in lowerV8I16GeneralSingleInputShuffle()
13634 if (M == SourceHalfMask[Input - SourceOffset] + SourceOffset) in lowerV8I16GeneralSingleInputShuffle()
13637 M = SourceHalfMask[Input - SourceOffset] + SourceOffset; in lowerV8I16GeneralSingleInputShuffle()
13639 assert(SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == in lowerV8I16GeneralSingleInputShuffle()
13640 Input - SourceOffset && in lowerV8I16GeneralSingleInputShuffle()
13643 // Note that this correctly re-maps both when we do a swap and when in lowerV8I16GeneralSingleInputShuffle()
13646 Input = SourceHalfMask[Input - SourceOffset] + SourceOffset; in lowerV8I16GeneralSingleInputShuffle()
13650 if (PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] < 0) in lowerV8I16GeneralSingleInputShuffle()
13651 PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] = Input / 2; in lowerV8I16GeneralSingleInputShuffle()
13653 assert(PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == in lowerV8I16GeneralSingleInputShuffle()
13658 // And just directly shift any other-half mask elements to be same-half in lowerV8I16GeneralSingleInputShuffle()
13663 M = M - SourceOffset + DestOffset; in lowerV8I16GeneralSingleInputShuffle()
13673 if (isWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) { in lowerV8I16GeneralSingleInputShuffle()
13674 int InputFixed = find(SourceHalfMask, -1) - std::begin(SourceHalfMask) + in lowerV8I16GeneralSingleInputShuffle()
13676 SourceHalfMask[InputFixed - SourceOffset] = in lowerV8I16GeneralSingleInputShuffle()
13677 IncomingInputs[0] - SourceOffset; in lowerV8I16GeneralSingleInputShuffle()
13684 isDWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) { in lowerV8I16GeneralSingleInputShuffle()
13685 // We have two non-adjacent or clobbered inputs we need to extract from in lowerV8I16GeneralSingleInputShuffle()
13688 int InputsFixed[2] = {IncomingInputs[0] - SourceOffset, in lowerV8I16GeneralSingleInputShuffle()
13689 IncomingInputs[1] - SourceOffset}; in lowerV8I16GeneralSingleInputShuffle()
13715 // (because there are no off-half inputs to this half) and there is no in lowerV8I16GeneralSingleInputShuffle()
13717 // swap an input with a non-input. in lowerV8I16GeneralSingleInputShuffle()
13795 M -= 4; in lowerV8I16GeneralSingleInputShuffle()
13803 /// Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the
13827 int V2Idx = M < Size ? ZeroMask : (M - Size) * Scale + i % Scale; in lowerShuffleAsBlendOfPSHUFBs()
13856 /// Generic lowering of 8-lane i16 shuffles.
13858 /// This handles both single-input shuffles and combined shuffle/blends with
13863 /// enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle
13927 "All single-input shuffles should be canonicalized to be V1-input " in lowerV8I16Shuffle()
13942 // There are special ways we can lower some single-element blends. in lowerV8I16Shuffle()
13992 // Check if this is part of a 256-bit vector truncation. in lowerV8I16Shuffle()
14008 for (unsigned i = 0; i != 4; i += 1 << (NumEvenDrops - 1)) in lowerV8I16Shuffle()
14038 // When compacting odd (upper) elements, use PACKSS pre-SSE41. in lowerV8I16Shuffle()
14065 // We can always bit-blend if we have to so the fallback strategy is to in lowerV8I16Shuffle()
14066 // decompose into single-input permutes and blends/unpacks. in lowerV8I16Shuffle()
14071 /// Lower 8-lane 16-bit floating point shuffles.
14100 // Lowers unary/binary shuffle as VPERMV/VPERMV3, for non-VLX targets,
14101 // sub-512-bit shuffles are padded to 512-bits for the shuffle and then
14121 M += (Scale - 1) * NumElts; in lowerShuffleWithPERMV()
14144 /// UNPCK to spread the i8 elements across two i16-element vectors, and uses
14145 /// the existing lowering for v8i16 blends on each half, finally PACK-ing them
14193 // For single-input shuffles, there are some nicer lowering tricks we can use. in lowerV16I8Shuffle()
14209 // Notably, this handles splat and partial-splat shuffles more efficiently. in lowerV16I8Shuffle()
14210 // However, it only makes sense if the pre-duplication shuffle simplifies in lowerV16I8Shuffle()
14212 // express the pre-duplication shuffle as an i16 shuffle. in lowerV16I8Shuffle()
14223 auto tryToWidenViaDuplication = [&]() -> SDValue { in lowerV16I8Shuffle()
14240 int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1}; in lowerV16I8Shuffle()
14284 int PostDupI16Shuffle[8] = {-1, -1, -1, -1, -1, -1, -1, -1}; in lowerV16I8Shuffle()
14287 int MappedMask = LaneMap[Mask[i]] - (TargetLo ? 0 : 8); in lowerV16I8Shuffle()
14323 // blends but after all of the single-input lowerings. If the single input in lowerV16I8Shuffle()
14332 // as a PACKUS(AND(),AND()) - which is quicker than UNPACK(PSHUFB(),PSHUFB()). in lowerV16I8Shuffle()
14345 // do so. This avoids using them to handle blends-with-zero which is in lowerV16I8Shuffle()
14358 // FIXME: It might be worth trying to detect if the unpack-feeding in lowerV16I8Shuffle()
14365 // AVX512VBMI can lower to VPERMB (non-VLX will pad to v64i8). in lowerV16I8Shuffle()
14376 // Use PALIGNR+Permute if possible - permute might become PSHUFB but the in lowerV16I8Shuffle()
14386 // There are special ways we can lower some single-element blends. in lowerV16I8Shuffle()
14411 for (unsigned i = 0; i != 8; i += 1 << (NumEvenDrops - 1)) in lowerV16I8Shuffle()
14443 // Handle multi-input cases by blending/unpacking single-input shuffles. in lowerV16I8Shuffle()
14448 // The fallback path for single-input shuffles widens this into two v8i16 in lowerV16I8Shuffle()
14453 std::array<int, 8> LoBlendMask = {{-1, -1, -1, -1, -1, -1, -1, -1}}; in lowerV16I8Shuffle()
14454 std::array<int, 8> HiBlendMask = {{-1, -1, -1, -1, -1, -1, -1, -1}}; in lowerV16I8Shuffle()
14497 /// Dispatching routine to lower various 128-bit x86 vector shuffles.
14499 /// This routine breaks down the specific type of 128-bit shuffle and
14534 /// Generic routine to split vector shuffle into half-sized shuffles.
14543 "Only for 256-bit or wider vector shuffles!"); in splitAndLowerShuffle()
14555 // Use splitVector/extractSubVector so that split build-vectors just build two in splitAndLowerShuffle()
14568 // Now create two 4-way blends of these half-width vectors. in splitAndLowerShuffle()
14589 auto CheckHalfBlendUsable = [&](const ArrayRef<int> &HalfMask) -> bool { in splitAndLowerShuffle()
14600 SmallVector<int, 32> V1BlendMask((unsigned)SplitNumElements, -1); in splitAndLowerShuffle()
14601 SmallVector<int, 32> V2BlendMask((unsigned)SplitNumElements, -1); in splitAndLowerShuffle()
14602 SmallVector<int, 32> BlendMask((unsigned)SplitNumElements, -1); in splitAndLowerShuffle()
14606 V2BlendMask[i] = M - NumElements; in splitAndLowerShuffle()
14619 // a minimal number of high-level vector shuffle nodes. in splitAndLowerShuffle()
14638 BlendMask[i] = V1BlendMask[i] - (UseLoV1 ? 0 : SplitNumElements); in splitAndLowerShuffle()
14663 /// This is provided as a good fallback for many lowerings of non-single-input
14664 /// shuffles with more than one 128-bit lane. In those cases, we want to select
14665 /// between splitting the shuffle into 128-bit components and stitching those
14666 /// back together vs. extracting the single-input shuffles and blending those
14672 assert(!V2.isUndef() && "This routine must not be used to lower single-input " in lowerShuffleAsSplitOrBlend()
14680 int V1BroadcastIdx = -1, V2BroadcastIdx = -1; in lowerShuffleAsSplitOrBlend()
14684 V2BroadcastIdx = M - Size; in lowerShuffleAsSplitOrBlend()
14685 else if (M - Size != V2BroadcastIdx) in lowerShuffleAsSplitOrBlend()
14699 // If the inputs all stem from a single 128-bit lane of each input, then we in lowerShuffleAsSplitOrBlend()
14715 // requires that the decomposed single-input shuffles don't end up here. in lowerShuffleAsSplitOrBlend()
14721 // TODO: Extend to support v8f32 (+ 512-bit shuffles).
14728 int LHSMask[4] = {-1, -1, -1, -1}; in lowerShuffleAsLanePermuteAndSHUFP()
14729 int RHSMask[4] = {-1, -1, -1, -1}; in lowerShuffleAsLanePermuteAndSHUFP()
14750 /// Lower a vector shuffle crossing multiple 128-bit lanes as
14751 /// a lane permutation followed by a per-lane permutation.
14753 /// This is mainly for cases where we can have non-repeating permutes
14771 auto getSublanePermute = [&](int NumSublanes) -> SDValue { in lowerShuffleAsLanePermuteAndPermute()
14813 // TODO - isShuffleMaskInputInPlace could be extended to something like in lowerShuffleAsLanePermuteAndPermute()
14825 if (OnlyShuffleLowestLane && NumIdentityLanes == (NumLanes - 1)) in lowerShuffleAsLanePermuteAndPermute()
14848 // Then attempt a solution with 64-bit sublanes (vpermq). in lowerShuffleAsLanePermuteAndPermute()
14852 // If that doesn't work and we have fast variable cross-lane shuffle, in lowerShuffleAsLanePermuteAndPermute()
14853 // attempt 32-bit sublanes (vpermd). in lowerShuffleAsLanePermuteAndPermute()
14874 /// Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one
14878 /// single-input cross lane shuffle which is lower than any other fully general
14879 /// cross-lane shuffle strategy I'm aware of. Special cases for each particular
14884 // FIXME: This should probably be generalized for 512-bit vectors as well. in lowerShuffleAsLanePermuteAndShuffle()
14885 assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!"); in lowerShuffleAsLanePermuteAndShuffle()
14896 // If there are only inputs from one 128-bit lane, splitting will in fact be in lowerShuffleAsLanePermuteAndShuffle()
14914 // TODO - we could support shuffling V2 in the Flipped input. in lowerShuffleAsLanePermuteAndShuffle()
14922 "In-lane shuffle mask expected"); in lowerShuffleAsLanePermuteAndShuffle()
14930 // Flip the lanes, and shuffle the results which should now be in-lane. in lowerShuffleAsLanePermuteAndShuffle()
14939 /// Handle lowering 2-lane 128-bit shuffles.
14987 // Blends are faster and handle all the non-lane-crossing cases. in lowerV2X128Shuffle()
14995 // Check for patterns which can be matched with a single insert of a 128-bit in lowerV2X128Shuffle()
15001 // this will likely become vinsertf128 which can't fold a 256-bit memop. in lowerV2X128Shuffle()
15023 // Otherwise form a 128-bit permutation. After accounting for undefs, in lowerV2X128Shuffle()
15024 // convert the 64-bit shuffle mask selection values into 128-bit in lowerV2X128Shuffle()
15029 // [1:0] - select 128 bits from sources for low half of destination in lowerV2X128Shuffle()
15030 // [2] - ignore in lowerV2X128Shuffle()
15031 // [3] - zero low half of destination in lowerV2X128Shuffle()
15032 // [5:4] - select 128 bits from sources for high half of destination in lowerV2X128Shuffle()
15033 // [6] - ignore in lowerV2X128Shuffle()
15034 // [7] - zero high half of destination in lowerV2X128Shuffle()
15053 /// Lower a vector shuffle by first fixing the 128-bit lanes and then
15071 SmallVector<int, 16> RepeatMask(NumLaneElts, -1); in lowerShuffleAsLanePermuteAndRepeatedMask()
15072 SmallVector<std::array<int, 2>, 2> LaneSrcs(NumLanes, {{-1, -1}}); in lowerShuffleAsLanePermuteAndRepeatedMask()
15077 int Srcs[2] = {-1, -1}; in lowerShuffleAsLanePermuteAndRepeatedMask()
15078 SmallVector<int, 16> InLaneMask(NumLaneElts, -1); in lowerShuffleAsLanePermuteAndRepeatedMask()
15177 SmallVector<int, 16> NewMask(NumElts, -1); in lowerShuffleAsLanePermuteAndRepeatedMask()
15181 int M = -1; in lowerShuffleAsLanePermuteAndRepeatedMask()
15192 cast<ShuffleVectorSDNode>(NewV1)->getMask() == Mask) in lowerShuffleAsLanePermuteAndRepeatedMask()
15198 int M = -1; in lowerShuffleAsLanePermuteAndRepeatedMask()
15209 cast<ShuffleVectorSDNode>(NewV2)->getMask() == Mask) in lowerShuffleAsLanePermuteAndRepeatedMask()
15214 NewMask[i] = -1; in lowerShuffleAsLanePermuteAndRepeatedMask()
15246 HalfIdx1 = -1; in getHalfShuffleMask()
15247 HalfIdx2 = -1; in getHalfShuffleMask()
15321 /// Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
15329 "Expected 256-bit or 512-bit vector"); in lowerShuffleWithUndefHalf()
15379 // Always extract lowers when setting lower - these are all free subreg ops. in lowerShuffleWithUndefHalf()
15385 // AVX2 has efficient 32/64-bit element cross-lane shuffles. in lowerShuffleWithUndefHalf()
15400 // AVX512 has efficient cross-lane shuffles for all legal 512-bit types. in lowerShuffleWithUndefHalf()
15413 // UndefLower - uuuuXXXX: an insert to high half is required if we split this. in lowerShuffleWithUndefHalf()
15415 // AVX2 has efficient 64-bit element cross-lane shuffles. in lowerShuffleWithUndefHalf()
15419 // AVX512 has efficient cross-lane shuffles for all legal 512-bit types. in lowerShuffleWithUndefHalf()
15431 /// Handle case where shuffle sources are coming from the same 128-bit lane and
15432 /// every lane can be represented as the same repeating mask - allowing us to
15451 // accounting for UNDEFs but only references the lowest 128-bit in lowerShuffleAsRepeatedMaskAndLanePermute()
15469 SmallVector<int, 8> RepeatMask((unsigned)NumElts, -1); in lowerShuffleAsRepeatedMaskAndLanePermute()
15477 SmallVector<int, 8> BroadcastMask((unsigned)NumElts, -1); in lowerShuffleAsRepeatedMaskAndLanePermute()
15492 // Bail if the shuffle mask doesn't cross 128-bit lanes. in lowerShuffleAsRepeatedMaskAndLanePermute()
15507 // can form a repeating shuffle mask (local to each sub-lane). At the same in lowerShuffleAsRepeatedMaskAndLanePermute()
15508 // time, determine the source sub-lane for each destination sub-lane. in lowerShuffleAsRepeatedMaskAndLanePermute()
15509 int TopSrcSubLane = -1; in lowerShuffleAsRepeatedMaskAndLanePermute()
15510 SmallVector<int, 8> Dst2SrcSubLanes((unsigned)NumSubLanes, -1); in lowerShuffleAsRepeatedMaskAndLanePermute()
15516 // Extract the sub-lane mask, check that it all comes from the same lane in lowerShuffleAsRepeatedMaskAndLanePermute()
15518 int SrcLane = -1; in lowerShuffleAsRepeatedMaskAndLanePermute()
15519 SmallVector<int, 8> SubLaneMask((unsigned)NumSubLaneElts, -1); in lowerShuffleAsRepeatedMaskAndLanePermute()
15532 // Whole sub-lane is UNDEF. in lowerShuffleAsRepeatedMaskAndLanePermute()
15536 // Attempt to match against the candidate repeated sub-lane masks. in lowerShuffleAsRepeatedMaskAndLanePermute()
15552 // Merge the sub-lane mask into the matching repeated sub-lane mask. in lowerShuffleAsRepeatedMaskAndLanePermute()
15562 // Track the top most source sub-lane - by setting the remaining to in lowerShuffleAsRepeatedMaskAndLanePermute()
15570 // Bail if we failed to find a matching repeated sub-lane mask. in lowerShuffleAsRepeatedMaskAndLanePermute()
15578 SmallVector<int, 8> RepeatedMask((unsigned)NumElts, -1); in lowerShuffleAsRepeatedMaskAndLanePermute()
15591 // Shuffle each source sub-lane to its destination. in lowerShuffleAsRepeatedMaskAndLanePermute()
15592 SmallVector<int, 8> SubLaneMask((unsigned)NumElts, -1); in lowerShuffleAsRepeatedMaskAndLanePermute()
15613 // On AVX2 targets we can permute 256-bit vectors as 64-bit sub-lanes in lowerShuffleAsRepeatedMaskAndLanePermute()
15614 // (with PERMQ/PERMPD). On AVX2/AVX512BW targets, permuting 32-bit sub-lanes, in lowerShuffleAsRepeatedMaskAndLanePermute()
15616 // Otherwise we can only permute whole 128-bit lanes. in lowerShuffleAsRepeatedMaskAndLanePermute()
15693 // Create a REAL zero vector - ISD::isBuildVectorAllZeros allows UNDEFs. in lowerShuffleWithSHUFPD()
15718 if (Zeroable.countl_one() < (Mask.size() - 8)) in lowerShuffleAsVTRUNCAndUnpack()
15746 // Pattern-match interleave(256b v1, 256b v2) -> 512b v3 and lower it into unpck
15748 // 256-bit vectors in earlier isel stages. Therefore, this function matches a
15749 // pair of 256-bit shuffles and makes sure the masks are consecutive.
15784 for (SDNode *User : V1->uses()) in lowerShufflePairAsUNPCKAndPermute()
15785 if (User->getOpcode() == ISD::VECTOR_SHUFFLE && User->getOperand(0) == V1 && in lowerShufflePairAsUNPCKAndPermute()
15786 User->getOperand(1) == V2) in lowerShufflePairAsUNPCKAndPermute()
15791 // Find out which half of the 512-bit shuffles is each smaller shuffle in lowerShufflePairAsUNPCKAndPermute()
15796 if (IsInterleavingPattern(SVN1->getMask(), 0, NumElts) && in lowerShufflePairAsUNPCKAndPermute()
15797 IsInterleavingPattern(SVN2->getMask(), FirstQtr, ThirdQtr)) { in lowerShufflePairAsUNPCKAndPermute()
15800 } else if (IsInterleavingPattern(SVN1->getMask(), FirstQtr, ThirdQtr) && in lowerShufflePairAsUNPCKAndPermute()
15801 IsInterleavingPattern(SVN2->getMask(), 0, NumElts)) { in lowerShufflePairAsUNPCKAndPermute()
15823 /// Handle lowering of 4-lane 64-bit floating point shuffles.
15825 /// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2
15850 // Non-half-crossing single input shuffles can be lowered with an in lowerV4F64Shuffle()
15863 // Try to create an in-lane repeating shuffle mask and then shuffle the in lowerV4F64Shuffle()
15869 // Try to permute the lanes and then use a per-lane permute. in lowerV4F64Shuffle()
15911 // Try to create an in-lane repeating shuffle mask and then shuffle the in lowerV4F64Shuffle()
15917 // Try to simplify this by merging 128-bit lanes to enable a lane-based in lowerV4F64Shuffle()
15943 /// Handle lowering of 4-lane 64-bit integer shuffles.
15977 // When the shuffle is mirrored between the 128-bit lanes of the unit, we in lowerV4I64Shuffle()
16031 // Try to create an in-lane repeating shuffle mask and then shuffle the in lowerV4I64Shuffle()
16042 // Try to simplify this by merging 128-bit lanes to enable a lane-based in lowerV4I64Shuffle()
16056 /// Handle lowering of 8-lane 32-bit floating point shuffles.
16058 /// Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2
16090 // If the shuffle mask is repeated in each 128-bit lane, we have many more in lowerV8F32Shuffle()
16116 // Try to create an in-lane repeating shuffle mask and then shuffle the in lowerV8F32Shuffle()
16123 // two 128-bit lanes use the variable mask to VPERMILPS. in lowerV8F32Shuffle()
16138 // Try to simplify this by merging 128-bit lanes to enable a lane-based in lowerV8F32Shuffle()
16160 // For non-AVX512 if the Mask is of 16bit elements in lane then try to split in lowerV8F32Shuffle()
16178 /// Handle lowering of 8-lane 32-bit integer shuffles.
16207 // For non-AVX512 if the Mask is of 16bit elements in lane then try to split in lowerV8I32Shuffle()
16236 // If the shuffle mask is repeated in each 128-bit lane we can use more in lowerV8I32Shuffle()
16237 // efficient instructions that mirror the shuffles across the two 128-bit in lowerV8I32Shuffle()
16280 // Try to create an in-lane repeating shuffle mask and then shuffle the in lowerV8I32Shuffle()
16287 // Try to produce a fixed cross-128-bit lane permute followed by unpack in lowerV8I32Shuffle()
16293 // generate a cross-lane VPERMD instruction. in lowerV8I32Shuffle()
16309 // Try to simplify this by merging 128-bit lanes to enable a lane-based in lowerV8I32Shuffle()
16320 /// Handle lowering of 16-lane 16-bit integer shuffles.
16374 // Try to create an in-lane repeating shuffle mask and then shuffle the in lowerV16I16Shuffle()
16386 // Try to produce a fixed cross-128-bit lane permute followed by unpack in lowerV16I16Shuffle()
16391 // There are no generalized cross-lane shuffle operations available on i16 in lowerV16I16Shuffle()
16404 // As this is a single-input shuffle, the repeated mask should be in lowerV16I16Shuffle()
16416 // AVX512BW can lower to VPERMW (non-VLX will pad to v32i16). in lowerV16I16Shuffle()
16420 // Try to simplify this by merging 128-bit lanes to enable a lane-based in lowerV16I16Shuffle()
16426 // Try to permute the lanes and then use a per-lane permute. in lowerV16I16Shuffle()
16443 /// Handle lowering of 32-lane 8-bit integer shuffles.
16503 // Try to create an in-lane repeating shuffle mask and then shuffle the in lowerV32I8Shuffle()
16509 // There are no generalized cross-lane shuffle operations available on i8 in lowerV32I8Shuffle()
16512 // Try to produce a fixed cross-128-bit lane permute followed by unpack in lowerV32I8Shuffle()
16529 // AVX512VBMI can lower to VPERMB (non-VLX will pad to v64i8). in lowerV32I8Shuffle()
16533 // Try to simplify this by merging 128-bit lanes to enable a lane-based in lowerV32I8Shuffle()
16539 // Try to permute the lanes and then use a per-lane permute. in lowerV32I8Shuffle()
16564 /// High-level routine to lower various 256-bit x86 vector shuffles.
16566 /// This routine either breaks down the specific type of a 256-bit x86 vector
16567 /// shuffle or splits it into two 128-bit shuffles and fuses the results back
16588 // There is a really nice hard cut-over between AVX1 and AVX2 that means we in lower256BitShuffle()
16590 // querying in the per-vector-type lowering routines. With AVX1 we have in lower256BitShuffle()
16591 // essentially *zero* ability to manipulate a 256-bit vector with integer in lower256BitShuffle()
16598 // for masking/blending then decompose into 128-bit vectors. in lower256BitShuffle()
16636 llvm_unreachable("Not a valid 256-bit x86 vector type!"); in lower256BitShuffle()
16640 /// Try to lower a vector shuffle as a 128-bit shuffles.
16652 // TODO - use Zeroable like we do for lowerV2X128VectorShuffle? in lowerV4X128Shuffle()
16670 // Check for patterns which can be matched with a single insert of a 256-bit in lowerV4X128Shuffle()
16683 // See if this is an insertion of the lower 128-bits of V2 into V1. in lowerV4X128Shuffle()
16685 int V2Index = -1; in lowerV4X128Shuffle()
16687 assert(Widened128Mask[i] >= -1 && "Illegal shuffle sentinel value"); in lowerV4X128Shuffle()
16698 // Make sure we only have a single V2 index and its the lowest 128-bits. in lowerV4X128Shuffle()
16713 // See if we can widen to a 256-bit lane shuffle, we're going to lose 128-lane in lowerV4X128Shuffle()
16725 int PermMask[4] = {-1, -1, -1, -1}; in lowerV4X128Shuffle()
16728 assert(Widened128Mask[i] >= -1 && "Illegal shuffle sentinel value"); in lowerV4X128Shuffle()
16746 /// Handle lowering of 8-lane 64-bit floating point shuffles.
16761 // Non-half-crossing single input shuffles can be lowered with an in lowerV8F64Shuffle()
16800 /// Handle lowering of 16-lane 32-bit floating point shuffles.
16809 // If the shuffle mask is repeated in each 128-bit lane, we have many more in lowerV16F32Shuffle()
16845 // Try to create an in-lane repeating shuffle mask and then shuffle the in lowerV16F32Shuffle()
16852 // 128-bit lanes and don't lane cross, use variable mask VPERMILPS. in lowerV16F32Shuffle()
16867 /// Handle lowering of 8-lane 64-bit integer shuffles.
16884 // When the shuffle is mirrored between the 128-bit lanes of the unit, we in lowerV8I64Shuffle()
16886 // 128-bit lanes. in lowerV8I64Shuffle()
16940 /// Handle lowering of 16-lane 32-bit integer shuffles.
16970 // If the shuffle mask is repeated in each 128-bit lane we can use more in lowerV16I32Shuffle()
16971 // efficient instructions that mirror the shuffles across the four 128-bit in lowerV16I32Shuffle()
17019 // Try to create an in-lane repeating shuffle mask and then shuffle the in lowerV16I32Shuffle()
17037 /// Handle lowering of 32-lane 16-bit integer shuffles.
17045 assert(Subtarget.hasBWI() && "We can only lower v32i16 with AVX-512-BWI!"); in lowerV32I16Shuffle()
17082 // As this is a single-input shuffle, the repeated mask should be in lowerV32I16Shuffle()
17101 /// Handle lowering of 64-lane 8-bit integer shuffles.
17109 assert(Subtarget.hasBWI() && "We can only lower v64i8 with AVX-512-BWI!"); in lowerV64I8Shuffle()
17153 // Try to create an in-lane repeating shuffle mask and then shuffle the in lowerV64I8Shuffle()
17168 // Use PALIGNR+Permute if possible - permute might become PSHUFB but the in lowerV64I8Shuffle()
17181 // Try to simplify this by merging 128-bit lanes to enable a lane-based in lowerV64I8Shuffle()
17195 /// High-level routine to lower various 512-bit x86 vector shuffles.
17197 /// This routine either breaks down the specific type of a 512-bit x86 vector
17198 /// shuffle or splits it into two 256-bit shuffles and fuses the results back
17206 "Cannot lower 512-bit vectors w/ basic ISA!"); in lower512BitShuffle()
17270 llvm_unreachable("Not a valid 512-bit x86 vector type!"); in lower512BitShuffle()
17282 int ShiftAmt = -1; in lower1BitShuffleAsKSHIFTR()
17291 // The first non-undef element determines our shift amount. in lower1BitShuffleAsKSHIFTR()
17293 ShiftAmt = M - i; in lower1BitShuffleAsKSHIFTR()
17298 // All non-undef elements must shift by the same amount. in lower1BitShuffleAsKSHIFTR()
17299 if (ShiftAmt != M - i) in lower1BitShuffleAsKSHIFTR()
17313 // Returns the shift amount if possible or -1 if not. This is a simplified
17321 if (!Zeroable[j + (Left ? 0 : (Size - Shift))]) in match1BitShuffleAsKSHIFT()
17330 unsigned Len = Size - Shift; in match1BitShuffleAsKSHIFT()
17341 return -1; in match1BitShuffleAsKSHIFT()
17346 // There is no a dedicated instruction on AVX-512 that shuffles the masks.
17347 // The only way to shuffle bits is to sign-extend the mask vector to SIMD
17355 "Cannot lower 512-bit vectors w/o basic ISA!"); in lower1BitShuffle()
17362 int Src = -1; in lower1BitShuffle()
17382 if ((int)Zeroable.countl_one() >= (NumElts - SubvecElts)) { in lower1BitShuffle()
17411 DAG.getTargetConstant(WideElts - NumElts, DL, MVT::i8)); in lower1BitShuffle()
17413 ShiftAmt += WideElts - NumElts; in lower1BitShuffle()
17427 if (NumV2Elements == 0 && V1.getOpcode() == ISD::SETCC && V1->hasOneUse()) { in lower1BitShuffle()
17430 ISD::CondCode CC = cast<CondCodeSDNode>(V1.getOperand(2))->get(); in lower1BitShuffle()
17449 // Take 512-bit type, more shuffles on KNL. If we have VLX use a 256-bit in lower1BitShuffle()
17454 // Take 512-bit type, unless we are avoiding 512-bit types and have the in lower1BitShuffle()
17455 // 256-bit operation available. in lower1BitShuffle()
17459 // Take 512-bit type, unless we are avoiding 512-bit types and have the in lower1BitShuffle()
17460 // 256-bit operation available. in lower1BitShuffle()
17564 // are preferable to blendw/blendvb/masked-mov. in canCombineAsMaskOperation()
17572 switch (V->getOpcode()) { in canCombineAsMaskOperation()
17591 if (!V->hasOneUse()) in canCombineAsMaskOperation()
17609 /// Top-level lowering for x86 vector shuffles.
17619 ArrayRef<int> OrigMask = SVOp->getMask(); in lowerVECTOR_SHUFFLE()
17641 // Check for non-undef masks pointing at an undef vector and make the masks in lowerVECTOR_SHUFFLE()
17649 M = -1; in lowerVECTOR_SHUFFLE()
17657 [&](int M) { return -1 <= M && M < MaskUpperLimit; }) && in lowerVECTOR_SHUFFLE()
17660 // We actually see shuffles that are entirely re-arrangements of a set of in lowerVECTOR_SHUFFLE()
17675 // integers to handle flipping the low and high halves of AVX 256-bit vectors. in lowerVECTOR_SHUFFLE()
17683 // TODO: Avoid lowering directly from this top-level function: make this in lowerVECTOR_SHUFFLE()
17684 // a query (canLowerAsBroadcast) and defer lowering to the type-based calls. in lowerVECTOR_SHUFFLE()
17698 // Modify the new Mask to take all zeros from the all-zero vector. in lowerVECTOR_SHUFFLE()
17699 // Choose indices that are blend-friendly. in lowerVECTOR_SHUFFLE()
17702 "V2's non-undef elements are used?!"); in lowerVECTOR_SHUFFLE()
17708 // Ensure all elements of V2 are zero - isBuildVectorAllZeros permits in lowerVECTOR_SHUFFLE()
17723 // Canonicalize the shuffle with any horizontal ops inputs. in lowerVECTOR_SHUFFLE()
17766 // Only non-legal VSELECTs reach this lowering, convert those into generic in lowerVSELECTtoVectorShuffle()
17767 // shuffles and re-use the shuffle lowering path for blends. in lowerVSELECTtoVectorShuffle()
17798 // Try to lower this to a blend-style vector shuffle. This can handle all in LowerVSELECT()
17804 // with patterns on the mask registers on AVX-512. in LowerVSELECT()
17821 // If the VSELECT is on a 512-bit type, we have to convert a non-i1 condition in LowerVSELECT()
17822 // into an i1 condition so that we can use the mask-based 512-bit blend in LowerVSELECT()
17863 // VSELECT-matching blend, return Op, and but if we need to expand, return in LowerVSELECT()
17909 unsigned IdxVal = Idx->getAsZExtVal(); in LowerEXTRACT_VECTOR_ELT_SSE4()
17923 SDNode *User = *Op.getNode()->use_begin(); in LowerEXTRACT_VECTOR_ELT_SSE4()
17924 if ((User->getOpcode() != ISD::STORE || isNullConstant(Idx)) && in LowerEXTRACT_VECTOR_ELT_SSE4()
17925 (User->getOpcode() != ISD::BITCAST || in LowerEXTRACT_VECTOR_ELT_SSE4()
17926 User->getValueType(0) != MVT::i32)) in LowerEXTRACT_VECTOR_ELT_SSE4()
17940 /// AVX-512 feature.
17957 // Extending v8i1/v16i1 to 512-bit get better performance on KNL in ExtractBitFromMaskVector()
17971 unsigned IdxVal = IdxC->getZExtValue(); in ExtractBitFromMaskVector()
17988 MVT VT = N->getSimpleValueType(0); in getExtractedDemandedElts()
17991 for (SDNode *User : N->uses()) { in getExtractedDemandedElts()
17992 switch (User->getOpcode()) { in getExtractedDemandedElts()
17996 if (!isa<ConstantSDNode>(User->getOperand(1))) { in getExtractedDemandedElts()
18000 DemandedElts.setBit(User->getConstantOperandVal(1)); in getExtractedDemandedElts()
18003 if (!User->getValueType(0).isSimple() || in getExtractedDemandedElts()
18004 !User->getValueType(0).isVector()) { in getExtractedDemandedElts()
18036 // (https://software.intel.com/en-us/articles/intel-architecture-code-analyzer) in LowerEXTRACT_VECTOR_ELT()
18044 // | Uops | 0 - DV | 5 | 6 | 7 | | in LowerEXTRACT_VECTOR_ELT()
18045 // --------------------------------------------- in LowerEXTRACT_VECTOR_ELT()
18056 // |Uops| 1 | 2 - D |3 - D | 4 | 5 | | in LowerEXTRACT_VECTOR_ELT()
18057 // --------------------------------------------------------- in LowerEXTRACT_VECTOR_ELT()
18058 // |2^ | | 0.5 | 0.5 |1.0| |CP| vmovaps xmmword ptr [rsp-0x18], xmm0 in LowerEXTRACT_VECTOR_ELT()
18059 // |1 |0.5| | | |0.5| | lea rax, ptr [rsp-0x18] in LowerEXTRACT_VECTOR_ELT()
18066 unsigned IdxVal = IdxC->getZExtValue(); in LowerEXTRACT_VECTOR_ELT()
18068 // If this is a 256-bit vector result, first extract the 128-bit vector and in LowerEXTRACT_VECTOR_ELT()
18069 // then extract the element from the 128-bit vector. in LowerEXTRACT_VECTOR_ELT()
18071 // Get the 128-bit vector. in LowerEXTRACT_VECTOR_ELT()
18080 IdxVal &= ElemsPerChunk - 1; in LowerEXTRACT_VECTOR_ELT()
18111 // Only extract a single element from a v16i8 source - determine the common in LowerEXTRACT_VECTOR_ELT()
18112 // DWORD/WORD that all extractions share, and extract the sub-byte. in LowerEXTRACT_VECTOR_ELT()
18118 // Extract either the lowest i32 or any i16, and extract the sub-byte. in LowerEXTRACT_VECTOR_ELT()
18149 SmallVector<int, 8> Mask(VecVT.getVectorNumElements(), -1); in LowerEXTRACT_VECTOR_ELT()
18166 int Mask[2] = { 1, -1 }; in LowerEXTRACT_VECTOR_ELT()
18176 /// AVX-512 feature.
18197 // Copy into a k-register, extract to v1i1 and insert_subvector. in InsertBitToMaskVector()
18229 // possible vector indices, and FP insertion has less gpr->simd traffic. in LowerINSERT_VECTOR_ELT()
18249 // inselt N0, N1, N2 --> select (SplatN2 == {0,1,2...}) ? SplatN1 : N0. in LowerINSERT_VECTOR_ELT()
18254 if (N2C->getAPIntValue().uge(NumElts)) in LowerINSERT_VECTOR_ELT()
18256 uint64_t IdxVal = N2C->getZExtValue(); in LowerINSERT_VECTOR_ELT()
18262 // Lower insertion of v16i8/v32i8/v64i16 -1 elts as an 'OR' blend. in LowerINSERT_VECTOR_ELT()
18287 // If the vector is wider than 128 bits, extract the 128-bit subvector, insert in LowerINSERT_VECTOR_ELT()
18290 // With a 256-bit vector, we can insert into the zero element efficiently in LowerINSERT_VECTOR_ELT()
18295 // doing anyway after extracting to a 128-bit vector. in LowerINSERT_VECTOR_ELT()
18306 "Vectors will always have power-of-two number of elements."); in LowerINSERT_VECTOR_ELT()
18308 // If we are not inserting into the low 128-bit vector chunk, in LowerINSERT_VECTOR_ELT()
18322 // Get the desired 128-bit vector chunk. in LowerINSERT_VECTOR_ELT()
18327 unsigned IdxIn128 = IdxVal & (NumEltsIn128 - 1); in LowerINSERT_VECTOR_ELT()
18335 assert(VT.is128BitVector() && "Only 128-bit vector types should be left!"); in LowerINSERT_VECTOR_ELT()
18388 // If this is an insertion of 32-bits into the low 32-bits of in LowerINSERT_VECTOR_ELT()
18393 // generate insertps because blendps does not have a 32-bit memory in LowerINSERT_VECTOR_ELT()
18423 // If this is a 256-bit vector result, first insert into a 128-bit in LowerSCALAR_TO_VECTOR()
18424 // vector and then insert into the 256-bit vector. in LowerSCALAR_TO_VECTOR()
18426 // Insert into a 128-bit vector. in LowerSCALAR_TO_VECTOR()
18433 // Insert the 128-bit vector. in LowerSCALAR_TO_VECTOR()
18485 // References to absolute symbols are never PC-relative. in getGlobalWrapperKind()
18486 if (GV && GV->isAbsoluteSymbolRef()) in getGlobalWrapperKind()
18489 // The following OpFlags under RIP-rel PIC use RIP. in getGlobalWrapperKind()
18518 CP->getConstVal(), PtrVT, CP->getAlign(), CP->getOffset(), OpFlag); in LowerConstantPool()
18540 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag); in LowerJumpTable()
18564 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); in LowerBlockAddress()
18565 int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset(); in LowerBlockAddress()
18591 GV = G->getGlobal(); in LowerGlobalOrExternal()
18592 Offset = G->getOffset(); in LowerGlobalOrExternal()
18595 ExternalSym = ES->getSymbol(); in LowerGlobalOrExternal()
18615 // Suppress the folding if Offset is negative: movl foo-1, %eax is not in LowerGlobalOrExternal()
18648 // If there was a non-zero offset that we didn't fold, create an explicit in LowerGlobalOrExternal()
18673 auto UI = TGA->use_begin(); in GetTLSADDR()
18675 if (UI != TGA->use_end()) in GetTLSADDR()
18676 return SDValue(*UI->use_begin()->use_begin(), 0); in GetTLSADDR()
18678 TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0), in GetTLSADDR()
18679 GA->getOffset(), OperandFlags); in GetTLSADDR()
18752 MFI->incNumLocalDynamicTLSAccesses(); in LowerToTLSLocalDynamicModel()
18774 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, in LowerToTLSLocalDynamicModel()
18775 GA->getValueType(0), in LowerToTLSLocalDynamicModel()
18776 GA->getOffset(), OperandFlags); in LowerToTLSLocalDynamicModel()
18789 // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit). in LowerToTLSExecModel()
18798 // Most TLS accesses are not RIP relative, even on x86-64. One exception is in LowerToTLSExecModel()
18816 // or "addl x@gotntpoff(%ebx) ,%eax" (initial exec, 32-bit pic) in LowerToTLSExecModel()
18818 DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0), in LowerToTLSExecModel()
18819 GA->getOffset(), OperandFlags); in LowerToTLSExecModel()
18846 const GlobalValue *GV = GA->getGlobal(); in LowerGlobalTLSAddress()
18887 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL, in LowerGlobalTLSAddress()
18888 GA->getValueType(0), in LowerGlobalTLSAddress()
18889 GA->getOffset(), OpFlag); in LowerGlobalTLSAddress()
18932 // Get the Thread Pointer, which is %fs:__tls_array (32-bit) or in LowerGlobalTLSAddress()
18933 // %gs:0x58 (64-bit). On MinGW, __tls_array is not available, so directly in LowerGlobalTLSAddress()
18949 if (GV->getThreadLocalMode() == GlobalVariable::LocalExecTLSModel) { in LowerGlobalTLSAddress()
18971 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, in LowerGlobalTLSAddress()
18972 GA->getValueType(0), in LowerGlobalTLSAddress()
18973 GA->getOffset(), X86II::MO_SECREL); in LowerGlobalTLSAddress()
19000 // offset and returning `true` for TLS-desc currently duplicates both in addressingModeSupportsTLS()
19001 // which is detrimental :-/ in addressingModeSupportsTLS()
19017 // Try to use a packed vector operation to handle i64 on 32-bit targets when
19027 bool IsStrict = Op->isStrictFPOpcode(); in LowerI64IntToFP_AVX512DQ()
19039 // Using 256-bit to ensure result is 128-bits for f32 case. in LowerI64IntToFP_AVX512DQ()
19060 // Try to use a packed vector operation to handle i64 on 32-bit targets.
19068 bool IsStrict = Op->isStrictFPOpcode(); in LowerI64IntToFP16()
19120 /// round-trip between XMM and GPR.
19132 // See if we have a 128-bit vector cast op for this type of cast. in vectorizeExtractedCast()
19141 // If we are extracting from a non-zero element, first shuffle the source in vectorizeExtractedCast()
19144 SmallVector<int, 16> Mask(FromVT.getVectorNumElements(), -1); in vectorizeExtractedCast()
19148 // If the source vector is wider than 128-bits, extract the low part. Do not in vectorizeExtractedCast()
19153 // cast (extelt V, 0) --> extelt (cast (extract_subv V)), 0 in vectorizeExtractedCast()
19154 // cast (extelt V, C) --> extelt (cast (extract_subv (shuffle V, [C...]))), 0 in vectorizeExtractedCast()
19161 /// try to vectorize the cast ops. This will avoid an expensive round-trip
19178 // See if we have 128-bit vector cast instructions for this type of cast. in lowerFPToIntToFP()
19191 // We need target-specific opcodes if this is v2f64 -> v4i32 -> v2f64. in lowerFPToIntToFP()
19197 // sint_to_fp (fp_to_sint X) --> extelt (sint_to_fp (fp_to_sint (s2v X))), 0 in lowerFPToIntToFP()
19213 bool IsStrict = Op->isStrictFPOpcode(); in lowerINT_TO_FP_vXi64()
19214 MVT VT = Op->getSimpleValueType(0); in lowerINT_TO_FP_vXi64()
19215 SDValue Src = Op->getOperand(IsStrict ? 1 : 0); in lowerINT_TO_FP_vXi64()
19224 // With AVX512DQ, but not VLX we need to widen to get a 512-bit result type. in lowerINT_TO_FP_vXi64()
19238 {Op->getOperand(0), Src}); in lowerINT_TO_FP_vXi64()
19252 bool IsSigned = Op->getOpcode() == ISD::SINT_TO_FP || in lowerINT_TO_FP_vXi64()
19253 Op->getOpcode() == ISD::STRICT_SINT_TO_FP; in lowerINT_TO_FP_vXi64()
19301 bool IsStrict = Op->isStrictFPOpcode(); in promoteXINT_TO_FP()
19303 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode(); in promoteXINT_TO_FP()
19340 bool IsStrict = Op->isStrictFPOpcode(); in LowerSINT_TO_FP()
19343 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode(); in LowerSINT_TO_FP()
19413 // Bitcasting to f64 here allows us to do a single 64-bit store from in LowerSINT_TO_FP()
19415 // with two 32-bit stores. in LowerSINT_TO_FP()
19477 /// Horizontal vector math instructions may be slower than normal math with
19478 /// shuffles. Limit horizontal op codegen based on size/speed trade-offs, uarch
19487 /// 64-bit unsigned integer to double expansion.
19491 // We can't use this algorithm for strict fp. It produces -0.0 instead of +0.0 in LowerUINT_TO_FP_i64()
19493 // fall back to Expand for when i64 or is legal or use FILD in 32-bit mode. in LowerUINT_TO_FP_i64()
19494 assert(!Op->isStrictFPOpcode() && "Expected non-strict uint_to_fp!"); in LowerUINT_TO_FP_i64()
19526 // Load the 64-bit value into an XMM register. in LowerUINT_TO_FP_i64()
19539 // TODO: Are there any fast-math-flags to propagate here? in LowerUINT_TO_FP_i64()
19547 SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1}); in LowerUINT_TO_FP_i64()
19555 /// 32-bit unsigned integer to float expansion.
19559 unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0; in LowerUINT_TO_FP_i32()
19564 // Load the 32-bit value into an XMM register. in LowerUINT_TO_FP_i32()
19581 if (Op.getNode()->isStrictFPOpcode()) { in LowerUINT_TO_FP_i32()
19583 // TODO: Are there any fast-math-flags to propagate here? in LowerUINT_TO_FP_i32()
19599 // TODO: Are there any fast-math-flags to propagate here? in LowerUINT_TO_FP_i32()
19612 bool IsStrict = Op->isStrictFPOpcode(); in lowerUINT_TO_FP_v2i32()
19625 SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other}, in lowerUINT_TO_FP_v2i32()
19644 // since double has 52-bits of mantissa. Then subtract 2^52 in floating in lowerUINT_TO_FP_v2i32()
19662 bool IsStrict = Op->isStrictFPOpcode(); in lowerUINT_TO_FP_vXi32()
19663 SDValue V = Op->getOperand(IsStrict ? 1 : 0); in lowerUINT_TO_FP_vXi32()
19669 // With AVX512, but not VLX we need to widen to get a 512-bit result type. in lowerUINT_TO_FP_vXi32()
19671 MVT VT = Op->getSimpleValueType(0); in lowerUINT_TO_FP_vXi32()
19673 // v8i32->v8f64 is legal with AVX512 so just return it. in lowerUINT_TO_FP_vXi32()
19690 {Op->getOperand(0), V}); in lowerUINT_TO_FP_vXi32()
19705 Op->getSimpleValueType(0) == MVT::v4f64) { in lowerUINT_TO_FP_vXi32()
19738 // float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f); in lowerUINT_TO_FP_vXi32()
19745 if (VecFloatVT != Op->getSimpleValueType(0)) in lowerUINT_TO_FP_vXi32()
19749 // - The vector of constants: in lowerUINT_TO_FP_vXi32()
19750 // -- 0x4b000000 in lowerUINT_TO_FP_vXi32()
19751 // -- 0x53000000 in lowerUINT_TO_FP_vXi32()
19752 // - A shift: in lowerUINT_TO_FP_vXi32()
19753 // -- v >> 16 in lowerUINT_TO_FP_vXi32()
19796 // float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f); in lowerUINT_TO_FP_vXi32()
19798 // constant, we avoid reassociation in MachineCombiner when unsafe-fp-math is in lowerUINT_TO_FP_vXi32()
19801 // TODO: Are there any fast-math-flags to propagate here? in lowerUINT_TO_FP_vXi32()
19819 unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0; in lowerUINT_TO_FP_vec()
19839 bool IsStrict = Op->isStrictFPOpcode(); in LowerUINT_TO_FP()
19845 MVT DstVT = Op->getSimpleValueType(0); in LowerUINT_TO_FP()
19869 // using VCVTUSI2SS/SD. Same for i64 in 64-bit mode. in LowerUINT_TO_FP()
19873 // Promote i32 to i64 and use a signed conversion on 64-bit targets. in LowerUINT_TO_FP()
19887 // The transform for i64->f64 isn't correct for 0 when rounding to negative in LowerUINT_TO_FP()
19888 // infinity. It produces -0.0, so disable under strictfp. in LowerUINT_TO_FP()
19892 // The transform for i32->f64/f32 isn't correct for 0 when rounding to in LowerUINT_TO_FP()
19901 // Make a 64-bit buffer, and use it to build an FILD. in LowerUINT_TO_FP()
19903 int SSFI = cast<FrameIndexSDNode>(StackSlot)->getIndex(); in LowerUINT_TO_FP()
19924 // Bitcasting to f64 here allows us to do a single 64-bit store from in LowerUINT_TO_FP()
19926 // with two 32-bit stores. in LowerUINT_TO_FP()
19950 Align CPAlignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlign(); in LowerUINT_TO_FP()
19965 // TODO: Are there any fast-math-flags to propagate here? in LowerUINT_TO_FP()
19999 bool IsStrict = Op->isStrictFPOpcode(); in FP_TO_INTHelper()
20015 // used for the 32-bit subtarget, but also for f80 on a 64-bit target. in FP_TO_INTHelper()
20021 // Replace the fp-to-uint32 operation with an fp-to-sint64 FIST. in FP_TO_INTHelper()
20031 // We lower FP->int64 into FISTP64 followed by a load from a temporary in FP_TO_INTHelper()
20052 // FistSrc = (Value - FltOfs); in FP_TO_INTHelper()
20053 // Fist-to-mem64 FistSrc in FP_TO_INTHelper()
20054 // Add 0 or 0x800...0ULL to the 64-bit result, which is equivalent in FP_TO_INTHelper()
20116 // FIXME This causes a redundant load/store if the SSE-class value is already in FP_TO_INTHelper()
20183 // v8i16 -> v8i32 in LowerAVXExtend()
20184 // Use vpmovzwd for 4 lower elements v8i16 -> v4i32. in LowerAVXExtend()
20185 // Use vpunpckhwd for 4 upper elements v8i16 -> v4i32. in LowerAVXExtend()
20188 // v4i32 -> v4i64 in LowerAVXExtend()
20189 // Use vpmovzdq for 4 lower elements v4i32 -> v2i64. in LowerAVXExtend()
20190 // Use vpunpckhdq for 4 upper elements v4i32 -> v2i64. in LowerAVXExtend()
20196 // Short-circuit if we can determine that each 128-bit half is the same value. in LowerAVXExtend()
20199 if (hasIdenticalHalvesShuffleMask(Shuf->getMask())) in LowerAVXExtend()
20228 MVT VT = Op->getSimpleValueType(0); in LowerZERO_EXTEND_Mask()
20229 SDValue In = Op->getOperand(0); in LowerZERO_EXTEND_Mask()
20239 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT)); in LowerZERO_EXTEND_Mask()
20252 // Widen to 512-bits if VLX is not supported. in LowerZERO_EXTEND_Mask()
20274 // Extract back to 128/256-bit if we widened. in LowerZERO_EXTEND_Mask()
20298 /// AVX2 (Int256) sub-targets require extra shuffling as the PACK*S operates
20299 /// within each 128-bit lane.
20331 // vXi64/vXi32 -> PACK*SDW and vXi16 -> PACK*SWB. in truncateVectorWithPACK()
20339 // Sub-128-bit truncation - widen to 128-bit src and pack in the lower half. in truncateVectorWithPACK()
20340 // On pre-AVX512, pack the src in both halves to help value tracking. in truncateVectorWithPACK()
20369 // 256bit -> 128bit truncate - PACK lower/upper 128-bit subvectors. in truncateVectorWithPACK()
20377 // AVX2: 512bit -> 256bit truncate - PACK lower/upper 256-bit subvectors. in truncateVectorWithPACK()
20378 // AVX2: 512bit -> 128bit truncate - PACK(PACK, PACK). in truncateVectorWithPACK()
20384 // 256-bit PACK(ARG0, ARG1) leaves us with ((LO0,LO1),(HI0,HI1)), in truncateVectorWithPACK()
20395 // If 512bit -> 128bit truncate another stage. in truncateVectorWithPACK()
20401 assert(SrcSizeInBits >= 256 && "Expected 256-bit vector or greater"); in truncateVectorWithPACK()
20404 // Avoid CONCAT_VECTORS on sub-128bit nodes as these can fail after in truncateVectorWithPACK()
20419 /// e.g. trunc <8 x i32> X to <8 x i16> -->
20464 // Truncation from 128-bit to vXi32 can be better handled with PSHUFD. in matchTruncateWithPACK()
20465 // Truncation to sub-64-bit vXi16 can be better handled with PSHUFD/PSHUFLW. in matchTruncateWithPACK()
20472 // Prefer to lower v4i64 -> v4i32 as a shuffle unless we can cheaply in matchTruncateWithPACK()
20489 // Pre-SSE41 we can only use PACKUSWB. in matchTruncateWithPACK()
20491 if ((NumSrcEltBits - NumPackedZeroBits) <= Known.countMinLeadingZeros()) { in matchTruncateWithPACK()
20496 // Truncate with PACKSS if we are truncating a vector with sign-bits in matchTruncateWithPACK()
20501 // Don't use PACKSS for vXi64 -> vXi32 truncations unless we're dealing with in matchTruncateWithPACK()
20509 unsigned MinSignBits = NumSrcEltBits - NumPackedSignBits; in matchTruncateWithPACK()
20518 if (In.getOpcode() == ISD::SRL && In->hasOneUse()) in matchTruncateWithPACK()
20522 return DAG.getNode(ISD::SRA, DL, SrcVT, In->ops()); in matchTruncateWithPACK()
20529 /// This function lowers a vector truncation of 'extended sign-bits' or
20530 /// 'extended zero-bits' values.
20598 // SSE2 provides PACKUS for only 2 x v8i16 -> v16i8 and SSE4.1 provides PACKUS in LowerTruncateVecPack()
20599 // for 2 x v4i32 -> v8i16. For SSSE3 and below, we need to use PACKSS to in LowerTruncateVecPack()
20607 // Special case vXi64 -> vXi16, shuffle to vXi32 and then use PACKSS. in LowerTruncateVecPack()
20626 unsigned ShiftInx = InVT.getScalarSizeInBits() - 1; in LowerTruncateVecI1()
20650 // trying to avoid 512-bit vectors. If we are avoiding 512-bit vectors in LowerTruncateVecI1()
20662 {8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1}); in LowerTruncateVecI1()
20675 // We either have 8 elements or we're allowed to use 512-bit vectors. in LowerTruncateVecI1()
20682 ShiftInx = InVT.getScalarSizeInBits() - 1; in LowerTruncateVecI1()
20712 // truncate the remainder. We'd rather produce two 64-bit results and in LowerTRUNCATE()
20725 // Pre-AVX512 (or prefer-256bit) see if we can make use of PACKSS/PACKUS. in LowerTRUNCATE()
20732 // Pre-AVX512 see if we can make use of PACKSS/PACKUS. in LowerTRUNCATE()
20759 // asked to avoid 512-bit vectors. The actual promotion to v16i32 will be in LowerTRUNCATE()
20770 // On AVX2, v4i64 -> v4i32 becomes VPERMD. in LowerTRUNCATE()
20772 static const int ShufMask[] = {0, 2, 4, 6, -1, -1, -1, -1}; in LowerTRUNCATE()
20789 // On AVX2, v8i32 -> v8i16 becomes PSHUFB. in LowerTRUNCATE()
20793 -1, -1, -1, -1, -1, -1, -1, -1, in LowerTRUNCATE()
20795 -1, -1, -1, -1, -1, -1, -1, -1 }; in LowerTRUNCATE()
20800 static const int ShufMask2[] = {0, 2, -1, -1}; in LowerTRUNCATE()
20815 llvm_unreachable("All 256->128 cases should have been handled above!"); in LowerTRUNCATE()
20825 assert(DstBits == 32 && "expandFP_TO_UINT_SSE - only vXi32 supported"); in expandFP_TO_UINT_SSE()
20828 // 2^31-1 ("Small") and from 2^31 to 2^32-1 ("Big"). in expandFP_TO_UINT_SSE()
20842 // AVX1 can't use the signsplat masking for 256-bit vectors - we have to in expandFP_TO_UINT_SSE()
20851 DAG.getTargetConstant(DstBits - 1, dl, MVT::i8)); in expandFP_TO_UINT_SSE()
20857 bool IsStrict = Op->isStrictFPOpcode(); in LowerFP_TO_INT()
20860 MVT VT = Op->getSimpleValueType(0); in LowerFP_TO_INT()
20862 SDValue Chain = IsStrict ? Op->getOperand(0) : SDValue(); in LowerFP_TO_INT()
20891 // Widen to 512-bits. in LowerFP_TO_INT()
20897 // TODO: Should we just do this for non-strict as well? in LowerFP_TO_INT()
20960 // v8f32/v16f32/v8f64->v8i16/v16i16 need to widen first. in LowerFP_TO_INT()
20984 // v8f64->v8i32 is legal, but we need v8i32 to be custom for v8f32. in LowerFP_TO_INT()
20991 // Widen vXi32 fp_to_uint with avx512f to 512-bit source. in LowerFP_TO_INT()
21001 // TODO: Should we just do this for non-strict as well? in LowerFP_TO_INT()
21023 // Widen vXi64 fp_to_uint/fp_to_sint with avx512dq to 512-bit source. in LowerFP_TO_INT()
21031 // TODO: Should we just do this for non-strict as well? in LowerFP_TO_INT()
21055 // Non-strict nodes without VLX can we widened to v4f32->v4i64 by type in LowerFP_TO_INT()
21077 return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp}); in LowerFP_TO_INT()
21115 // (i32) 0 to 2^31-1 ("Small") and from 2^31 to 2^32-1 ("Big"). in LowerFP_TO_INT()
21116 // (i64) 0 to 2^63-1 ("Small") and from 2^63 to 2^64-1 ("Big"). in LowerFP_TO_INT()
21132 ISD::SRA, dl, VT, Small, DAG.getConstant(DstBits - 1, dl, MVT::i8)); in LowerFP_TO_INT()
21143 // Promote i32 to i64 and use a signed operation on 64-bit targets. in LowerFP_TO_INT()
21237 EVT DstVT = N->getValueType(0); in LRINT_LLRINTHelper()
21238 SDValue Src = N->getOperand(0); in LRINT_LLRINTHelper()
21256 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in LRINT_LLRINTHelper()
21285 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT; in LowerFP_TO_INT_SAT()
21288 SDValue Src = Node->getOperand(0); in LowerFP_TO_INT_SAT()
21295 EVT DstVT = Node->getValueType(0); in LowerFP_TO_INT_SAT()
21303 EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); in LowerFP_TO_INT_SAT()
21316 // Promote conversions to unsigned 32-bit to 64-bit, because it will allow in LowerFP_TO_INT_SAT()
21329 // floating-point values. in LowerFP_TO_INT_SAT()
21431 bool IsStrict = Op->isStrictFPOpcode(); in LowerFP_EXTEND()
21439 // Let f16->f80 get lowered to a libcall, except for darwin, where we should in LowerFP_EXTEND()
21470 // Need a libcall, but ABI for f16 is soft-float on MacOS. in LowerFP_EXTEND()
21532 {Op->getOperand(0), Res}); in LowerFP_EXTEND()
21544 {Op->getOperand(0), Res}); in LowerFP_EXTEND()
21549 bool IsStrict = Op->isStrictFPOpcode(); in LowerFP_ROUND()
21622 // FIXME: Should we use zeros for upper elements for non-strict? in LowerFP_ROUND()
21641 bool IsStrict = Op->isStrictFPOpcode(); in LowerFP16_TO_FP()
21670 bool IsStrict = Op->isStrictFPOpcode(); in LowerFP_TO_FP16()
21686 // FIXME: Should we use zeros for upper elements for non-strict? in LowerFP_TO_FP16()
21734 // FP horizontal add/sub were added with SSE3. Integer with SSSE3. in lowerAddSubToHorizontalOp()
21754 // clang-format off in lowerAddSubToHorizontalOp()
21760 llvm_unreachable("Trying to lower unsupported opcode to horizontal op"); in lowerAddSubToHorizontalOp()
21761 // clang-format on in lowerAddSubToHorizontalOp()
21780 // Creating a 256-bit horizontal op would be wasteful, and there is no 512-bit in lowerAddSubToHorizontalOp()
21781 // equivalent, so extract the 256/512-bit source op to 128-bit if we can. in lowerAddSubToHorizontalOp()
21788 // add (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hadd X, X), 0 in lowerAddSubToHorizontalOp()
21789 // add (extractelt (X, 1), extractelt (X, 0)) --> extractelt (hadd X, X), 0 in lowerAddSubToHorizontalOp()
21790 // add (extractelt (X, 2), extractelt (X, 3)) --> extractelt (hadd X, X), 1 in lowerAddSubToHorizontalOp()
21791 // sub (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hsub X, X), 0 in lowerAddSubToHorizontalOp()
21840 for (SDNode *User : Op->uses()) in LowerFABSorFNEG()
21841 if (User->getOpcode() == ISD::FNEG) in LowerFABSorFNEG()
21853 // decide if we should generate a 16-byte constant mask when we only need 4 or in LowerFABSorFNEG()
21857 // generate a 16-byte vector constant and logic op even for the scalar case. in LowerFABSorFNEG()
21858 // Using a 16-byte mask allows folding the load of the mask with in LowerFABSorFNEG()
21884 // For the scalar case extend to a 128-bit vector, perform the logic op, in LowerFABSorFNEG()
21916 // Perform all scalar logic operations as 16-byte vectors because there are no in LowerFCOPYSIGN()
21945 APFloat APF = Op0CN->getValueAPF(); in LowerFCOPYSIGN()
21982 // instruction. Since the shift amount is in-range-or-undefined, we know in getBT()
21993 // See if we can use the 32-bit instruction instead of the 64-bit one for a in getBT()
22007 if (BitNo.getOpcode() == ISD::AND && BitNo->hasOneUse()) in getBT()
22066 /// Try to map a 128-bit or larger integer comparison to vector instructions
22083 // logically-combined vector-sized operands compared to zero. This pattern may in combineVectorSizedSetCCEquality()
22100 // Use XOR (plus OR) and PTEST after SSE4.1 for 128/256-bit operands. in combineVectorSizedSetCCEquality()
22101 // Use PCMPNEQ (plus OR) and KORTEST for 512-bit operands. in combineVectorSizedSetCCEquality()
22142 auto ScalarToVector = [&](SDValue X) -> SDValue { in combineVectorSizedSetCCEquality()
22170 // This is a bitwise-combined equality comparison of 2 pairs of vectors: in combineVectorSizedSetCCEquality()
22203 // setcc i128 X, Y, eq --> setcc (pmovmskb (pcmpeqb X, Y)), 0xFFFF, eq in combineVectorSizedSetCCEquality()
22204 // setcc i128 X, Y, ne --> setcc (pmovmskb (pcmpeqb X, Y)), 0xFFFF, ne in combineVectorSizedSetCCEquality()
22206 "Non 128-bit vector on pre-SSE41 target"); in combineVectorSizedSetCCEquality()
22217 /// are supported when the pointer SrcMask is non-null.
22218 /// TODO - move this to SelectionDAG?
22236 if (I->getOpcode() == unsigned(BinOp)) { in matchScalarReduction()
22237 Opnds.push_back(I->getOperand(0)); in matchScalarReduction()
22238 Opnds.push_back(I->getOperand(1)); in matchScalarReduction()
22239 // Re-evaluate the number of nodes to be traversed. in matchScalarReduction()
22244 // Quit if a non-EXTRACT_VECTOR_ELT in matchScalarReduction()
22245 if (I->getOpcode() != ISD::EXTRACT_VECTOR_ELT) in matchScalarReduction()
22249 auto *Idx = dyn_cast<ConstantSDNode>(I->getOperand(1)); in matchScalarReduction()
22253 SDValue Src = I->getOperand(0); in matchScalarReduction()
22258 if (!SrcOpMap.empty() && VT != SrcOpMap.begin()->first.getValueType()) in matchScalarReduction()
22267 unsigned CIdx = Idx->getZExtValue(); in matchScalarReduction()
22268 if (M->second[CIdx]) in matchScalarReduction()
22270 M->second.setBit(CIdx); in matchScalarReduction()
22276 SrcMask->push_back(SrcOpMap[SrcOp]); in matchScalarReduction()
22299 // Quit if not convertable to legal scalar or 128/256-bit vector. in LowerVectorAllEqual()
22303 // FCMP may use ISD::SETNE when nnan - early out if we manage to get here. in LowerVectorAllEqual()
22320 // For sub-128-bit vector, cast to (legal) integer and compare with zero. in LowerVectorAllEqual()
22343 // Without PTEST, a masked v2i64 or-reduction is not faster than in LowerVectorAllEqual()
22350 // Split down to 128/256/512-bit vector. in LowerVectorAllEqual()
22367 // If ICMP(AND(LHS,MASK),MASK) - reduce using AND splits. in LowerVectorAllEqual()
22376 // ALLOF(CMPEQ(X,Y)) -> AND(CMPEQ(X[0],Y[0]),CMPEQ(X[1],Y[1]),....) in LowerVectorAllEqual()
22423 assert(VT.getSizeInBits() == 128 && "Failure to split to 128-bits"); in LowerVectorAllEqual()
22434 // Check whether an AND/OR'd reduction tree is PTEST-able, or if we can fallback
22449 if (!Subtarget.hasSSE2() || !Op->hasOneUse()) in MatchVectorAllEqualTest()
22452 // Check whether we're masking/truncating an OR-reduction result, in which in MatchVectorAllEqualTest()
22467 Mask = Cst->getAPIntValue(); in MatchVectorAllEqualTest()
22478 // Match icmp(and(extract(X,0),extract(X,1)),-1) allof reduction patterns. in MatchVectorAllEqualTest()
22486 // Quit if not splittable to scalar/128/256/512-bit vector. in MatchVectorAllEqualTest()
22492 for (unsigned Slot = 0, e = VecIns.size(); e - Slot > 1; in MatchVectorAllEqualTest()
22508 // Match icmp(reduce_and(X),-1) allof reduction patterns. in MatchVectorAllEqualTest()
22528 // Match icmp(bitcast(icmp_eq(X,Y)),-1) reduction patterns. in MatchVectorAllEqualTest()
22533 ISD::CondCode SrcCC = cast<CondCodeSDNode>(Src.getOperand(2))->get(); in MatchVectorAllEqualTest()
22542 // Match icmp(bitcast(vXi1 trunc(Y)),-1) reduction patterns. in MatchVectorAllEqualTest()
22564 for (SDNode::use_iterator UI = Op->use_begin(), UE = Op->use_end(); UI != UE; in hasNonFlagsUse()
22568 if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) { in hasNonFlagsUse()
22570 UOpNo = User->use_begin().getOperandNo(); in hasNonFlagsUse()
22571 User = *User->use_begin(); in hasNonFlagsUse()
22574 if (User->getOpcode() != ISD::BRCOND && User->getOpcode() != ISD::SETCC && in hasNonFlagsUse()
22575 !(User->getOpcode() == ISD::SELECT && UOpNo == 0)) in hasNonFlagsUse()
22581 // Transform to an x86-specific ALU node with flags if there is a chance of
22585 for (SDNode *U : Op->uses()) in isProfitableToUseFlagOp()
22586 if (U->getOpcode() != ISD::CopyToReg && in isProfitableToUseFlagOp()
22587 U->getOpcode() != ISD::SETCC && in isProfitableToUseFlagOp()
22588 U->getOpcode() != ISD::STORE) in isProfitableToUseFlagOp()
22614 switch (Op->getOpcode()) { in EmitTest()
22619 if (Op.getNode()->getFlags().hasNoSignedWrap()) in EmitTest()
22644 // non-casted variable when we check for possible users. in EmitTest()
22660 // Otherwise use a regular EFLAGS-setting instruction. in EmitTest()
22662 // clang-format off in EmitTest()
22669 // clang-format on in EmitTest()
22684 return DAG.getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0), in EmitTest()
22685 Op->getOperand(1)).getValue(1); in EmitTest()
22697 SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands); in EmitTest()
22726 // Don't do this if the immediate can fit in 8-bits. in EmitCmp()
22727 if ((COp0 && !COp0->getAPIntValue().isSignedIntN(8)) || in EmitCmp()
22728 (COp1 && !COp1->getAPIntValue().isSignedIntN(8))) { in EmitCmp()
22750 // TODO: Add sign-bits equivalent for isX86CCSigned(X86CC)? in EmitCmp()
22760 // 0-x == y --> x+y == 0 in EmitCmp()
22761 // 0-x != y --> x+y != 0 in EmitCmp()
22769 // x == 0-y --> x+y == 0 in EmitCmp()
22770 // x != 0-y --> x+y != 0 in EmitCmp()
22791 if (N->getOpcode() == ISD::FDIV) in optimizeFMulOrFDivAsShiftAddBitcast()
22794 EVT FPVT = N->getValueType(0); in optimizeFMulOrFDivAsShiftAddBitcast()
22797 // This indicates a non-free bitcast. in optimizeFMulOrFDivAsShiftAddBitcast()
22799 // integer vector anyways for the int->fp cast. in optimizeFMulOrFDivAsShiftAddBitcast()
22824 /// The minimum architected relative accuracy is 2^-12. We need one
22825 /// Newton-Raphson step to have a good float result (24 bits of precision).
22834 // SSE1 has rsqrtss and rsqrtps. AVX adds a 256-bit variant for rsqrtps. in getSqrtEstimate()
22835 // It is likely not profitable to do this for f64 because a double-precision in getSqrtEstimate()
22851 // There is no FSQRT for 512-bits, but there is RSQRT14. in getSqrtEstimate()
22878 /// The minimum architected relative accuracy is 2^-12. We need one
22879 /// Newton-Raphson step to have a good float result (24 bits of precision).
22886 // SSE1 has rcpss and rcpps. AVX adds a 256-bit variant for rcpps. in getRecipEstimate()
22887 // It is likely not profitable to do this for f64 because a double-precision in getRecipEstimate()
22899 // real-world code. These defaults are intended to match GCC behavior. in getRecipEstimate()
22906 // There is no FSQRT for 512-bits, but there is RCP14. in getRecipEstimate()
22944 if (isIntDivCheap(N->getValueType(0), Attr)) in BuildSDIVPow2()
22956 EVT VT = N->getValueType(0); in BuildSDIVPow2()
22962 // If the divisor is 2 or -2, the default expansion is better. in BuildSDIVPow2()
22964 Divisor == APInt(Divisor.getBitWidth(), -2, /*isSigned*/ true)) in BuildSDIVPow2()
22993 if (Known.countMinLeadingZeros() < BitWidth - AndBitWidth) in LowerAndToBT()
23001 uint64_t AndRHSVal = AndRHS->getZExtValue(); in LowerAndToBT()
23024 // Remove any bit flip. in LowerAndToBT()
23039 // Check if pre-AVX condcode can be performed by a single FCMP op.
23044 /// Turns an ISD::CondCode into a value suitable for SSE floating-point mask
23052 // 0 - EQ in translateX86FSETCC()
23053 // 1 - LT in translateX86FSETCC()
23054 // 2 - LE in translateX86FSETCC()
23055 // 3 - UNORD in translateX86FSETCC()
23056 // 4 - NEQ in translateX86FSETCC()
23057 // 5 - NLT in translateX86FSETCC()
23058 // 6 - NLE in translateX86FSETCC()
23059 // 7 - ORD in translateX86FSETCC()
23061 // clang-format off in translateX86FSETCC()
23083 // clang-format on in translateX86FSETCC()
23107 /// Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then
23142 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); in LowerIntVSETCC_AVX512()
23169 auto *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i)); in incDecVectorConstant()
23170 if (!Elt || Elt->isOpaque() || Elt->getSimpleValueType(0) != EltVT) in incDecVectorConstant()
23174 const APInt &EltC = Elt->getAPIntValue(); in incDecVectorConstant()
23181 NewVecC.push_back(DAG.getConstant(EltC + (IsInc ? 1 : -1), DL, EltVT)); in incDecVectorConstant()
23210 // Only do this pre-AVX since vpcmp* is no longer destructive. in LowerVSETCCWithSUBUS()
23224 // cmpgt (xor X, SignMaskC) CmpC --> cmpeq (usubsat (CmpC+1), X), 0 in LowerVSETCCWithSUBUS()
23233 // Psubus is better than flip-sign because it requires no inversion. in LowerVSETCCWithSUBUS()
23253 MVT VT = Op->getSimpleValueType(0); in LowerVSETCC()
23254 ISD::CondCode Cond = cast<CondCodeSDNode>(CC)->get(); in LowerVSETCC()
23269 // compare like we do for non-strict, we might trigger spurious exceptions in LowerVSETCC()
23283 // floating-point vector result that matches the operand type. This allows in LowerVSETCC()
23311 SignalCmp->setFlags(Op->getFlags()); in LowerVSETCC()
23362 // Make a flip on already signaling CCs before setting bit 4 of AVX CC. in LowerVSETCC()
23402 // The non-AVX512 code below works under the assumption that source and in LowerVSETCC()
23409 // In AVX-512 architecture setcc returns mask with i1 elements, in LowerVSETCC()
23421 // clang-format off in LowerVSETCC()
23433 // clang-format on in LowerVSETCC()
23444 // (X & Y) != 0 --> (X & Y) == Y iff Y is power-of-2. in LowerVSETCC()
23462 // ICMP_EQ(AND(X,C),C) -> SRA(SHL(X,LOG2(C)),BW-1) iff C is power-of-2. in LowerVSETCC()
23466 if (C1 && C1->getAPIntValue().isPowerOf2()) { in LowerVSETCC()
23468 unsigned ShiftAmt = BitWidth - C1->getAPIntValue().logBase2() - 1; in LowerVSETCC()
23474 DAG.getConstant(BitWidth - 1, dl, VT)); in LowerVSETCC()
23479 // Break 256-bit integer vector compare into smaller ones. in LowerVSETCC()
23483 // Break 512-bit integer vector compare into smaller ones. in LowerVSETCC()
23489 // not-of-PCMPEQ: in LowerVSETCC()
23490 // X != INT_MIN --> X >s INT_MIN in LowerVSETCC()
23491 // X != INT_MAX --> X <s INT_MAX --> INT_MAX >s X in LowerVSETCC()
23492 // +X != 0 --> +X >s 0 in LowerVSETCC()
23504 // If both operands are known non-negative, then an unsigned compare is the in LowerVSETCC()
23505 // same as a signed compare and there's no need to flip signbits. in LowerVSETCC()
23519 // X > C --> X >= (C+1) --> X == umax(X, C+1) in LowerVSETCC()
23527 // X < C --> X <= (C-1) --> X == umin(X, C-1) in LowerVSETCC()
23537 // clang-format off in LowerVSETCC()
23543 // clang-format on in LowerVSETCC()
23549 // If the logical-not of the result is required, perform that now. in LowerVSETCC()
23596 Op1 = DAG.getConstant(-1, dl, MVT::v4i32); in LowerVSETCC()
23605 // If the i64 elements are sign-extended enough to be representable as i32 in LowerVSETCC()
23619 // Since SSE has no unsigned integer comparisons, we need to flip the sign in LowerVSETCC()
23665 // Make sure the lower and upper halves are both all-ones. in LowerVSETCC()
23677 // Since SSE has no unsigned integer comparisons, we need to flip the sign in LowerVSETCC()
23689 // If the logical-not of the result is required, perform that now. in LowerVSETCC()
23771 // Try to use PTEST/PMOVMSKB for a tree AND/ORs equality compared with -1/0. in emitFlagsForSetcc()
23805 if (VT == MVT::i32 || VT == MVT::i64 || Op0->hasOneUse()) { in emitFlagsForSetcc()
23816 // (seteq (add X, -1), -1). Similar for setne. in emitFlagsForSetcc()
23845 MVT VT = Op->getSimpleValueType(0); in LowerSETCC()
23849 assert(VT == MVT::i8 && "SetCC type must be 8-bit integer"); in LowerSETCC()
23855 cast<CondCodeSDNode>(Op.getOperand(IsStrict ? 3 : 2))->get(); in LowerSETCC()
23877 // Attempt to canonicalize SGT/UGT -> SGE/UGE compares with constant which in LowerSETCC()
23880 // equivalent for SLE/ULE -> SLT/ULT isn't likely to happen as we already in LowerSETCC()
23883 // encoding size - so it must either already be a i8 or i32 immediate, or it in LowerSETCC()
23888 const APInt &Op1Val = Op1C->getAPIntValue(); in LowerSETCC()
23939 X86::CondCode CC = TranslateIntegerX86CC(cast<CondCodeSDNode>(Cond)->get()); in LowerSETCCCARRY()
24012 assert(Op->getValueType(1) == MVT::i8 && "Unexpected VT!"); in LowerXALUO()
24013 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC); in LowerXALUO()
24038 return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits)); in isTruncWithZeroHighBitsInput()
24061 VT == Cond.getOperand(0).getSimpleValueType() && Cond->hasOneUse()) { in LowerSELECT()
24065 translateX86FSETCC(cast<CondCodeSDNode>(Cond.getOperand(2))->get(), in LowerSELECT()
24133 // (select (x == 0), -1, y) -> (sign_bit (x - 1)) | y in LowerSELECT()
24134 // (select (x == 0), y, -1) -> ~(sign_bit (x - 1)) | y in LowerSELECT()
24135 // (select (x != 0), y, -1) -> (sign_bit (x - 1)) | y in LowerSELECT()
24136 // (select (x != 0), -1, y) -> ~(sign_bit (x - 1)) | y in LowerSELECT()
24137 // (select (and (x , 0x1) == 0), y, (z ^ y) ) -> (-(and (x , 0x1)) & z ) ^ y in LowerSELECT()
24138 // (select (and (x , 0x1) == 0), y, (z | y) ) -> (-(and (x , 0x1)) & z ) | y in LowerSELECT()
24139 // (select (x > 0), x, 0) -> (~(x >> (size_in_bits(x)-1))) & x in LowerSELECT()
24140 // (select (x < 0), x, 0) -> ((x >> (size_in_bits(x)-1))) & x in LowerSELECT()
24148 // Special handling for __builtin_ffs(X) - 1 pattern which looks like in LowerSELECT()
24149 // (select (seteq X, 0), -1, (cttz_zero_undef X)). Disable the special in LowerSELECT()
24166 // 'X - 1' sets the carry flag if X == 0. in LowerSELECT()
24167 // '0 - X' sets the carry flag if X != 0. in LowerSELECT()
24168 // Convert the carry flag to a -1/0 mask with sbb: in LowerSELECT()
24169 // select (X != 0), -1, Y --> 0 - X; or (sbb), Y in LowerSELECT()
24170 // select (X == 0), Y, -1 --> 0 - X; or (sbb), Y in LowerSELECT()
24171 // select (X != 0), Y, -1 --> X - 1; or (sbb), Y in LowerSELECT()
24172 // select (X == 0), -1, Y --> X - 1; or (sbb), Y in LowerSELECT()
24216 SDValue Mask = DAG.getNegative(Neg, DL, VT); // -(and (x, 0x1)) in LowerSELECT()
24221 Cmp.getNode()->hasOneUse() && (CmpOp0 == Op1) && in LowerSELECT()
24224 // (select (x < 0), x, 0) -> ((x >> (size_in_bits(x)-1))) & x in LowerSELECT()
24229 // (select (x > 0), x, 0) -> (~(x >> (size_in_bits(x)-1))) & x in LowerSELECT()
24230 unsigned ShCt = VT.getSizeInBits() - 1; in LowerSELECT()
24256 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue()); in LowerSELECT()
24296 // a < b ? -1 : 0 -> RES = ~setcc_carry in LowerSELECT()
24297 // a < b ? 0 : -1 -> RES = setcc_carry in LowerSELECT()
24298 // a >= b ? -1 : 0 -> RES = setcc_carry in LowerSELECT()
24299 // a >= b ? 0 : -1 -> RES = ~setcc_carry in LowerSELECT()
24301 unsigned CondCode = CC->getAsZExtVal(); in LowerSELECT()
24349 return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops, Op->getFlags()); in LowerSELECT()
24355 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND_Mask()
24356 SDValue In = Op->getOperand(0); in LowerSIGN_EXTEND_Mask()
24372 // Widen to 512-bits if VLX is not supported. in LowerSIGN_EXTEND_Mask()
24388 SDValue NegOne = DAG.getConstant(-1, dl, WideVT); in LowerSIGN_EXTEND_Mask()
24399 // Extract back to 128/256-bit if we widened. in LowerSIGN_EXTEND_Mask()
24409 SDValue In = Op->getOperand(0); in LowerANY_EXTEND()
24422 // non-SSE4.1 targets. For zero extend this should only handle inputs of
24427 SDValue In = Op->getOperand(0); in LowerEXTEND_VECTOR_INREG()
24428 MVT VT = Op->getSimpleValueType(0); in LowerEXTEND_VECTOR_INREG()
24448 // For 256-bit vectors, we only need the lower (128-bit) half of the input. in LowerEXTEND_VECTOR_INREG()
24449 // For 512-bit vectors, we need 128-bits or 256-bits. in LowerEXTEND_VECTOR_INREG()
24452 // at least 128-bits. in LowerEXTEND_VECTOR_INREG()
24458 // SSE41 targets can use the pmov[sz]x* instructions directly for 128-bit results, in LowerEXTEND_VECTOR_INREG()
24460 // need to be handled here for 256/512-bit results. in LowerEXTEND_VECTOR_INREG()
24462 assert(VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension"); in LowerEXTEND_VECTOR_INREG()
24475 // pre-AVX2 256-bit extensions need to be split into 128-bit instructions. in LowerEXTEND_VECTOR_INREG()
24477 assert(VT.is256BitVector() && "256-bit vector expected"); in LowerEXTEND_VECTOR_INREG()
24497 // If the source elements are already all-signbits, we don't need to extend, in LowerEXTEND_VECTOR_INREG()
24509 // pre-SSE41 targets unpack lower lanes and then sign-extend using SRAI. in LowerEXTEND_VECTOR_INREG()
24526 Mask[i * Scale + (Scale - 1)] = i; in LowerEXTEND_VECTOR_INREG()
24531 unsigned SignExtShift = DestWidth - InSVT.getSizeInBits(); in LowerEXTEND_VECTOR_INREG()
24549 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND()
24550 SDValue In = Op->getOperand(0); in LowerSIGN_EXTEND()
24582 // for v4i32 the high shuffle mask will be {2, 3, -1, -1} in LowerSIGN_EXTEND()
24583 // use vpmovsx instruction to extend v4i32 -> v2i64; v8i16 -> v4i32 in LowerSIGN_EXTEND()
24589 SmallVector<int,8> ShufMask(NumElems, -1); in LowerSIGN_EXTEND()
24599 /// Change a vector store into a pair of half-size vector stores.
24601 SDValue StoredVal = Store->getValue(); in splitVectorStore()
24604 "Expecting 256/512-bit op"); in splitVectorStore()
24611 if (!Store->isSimple()) in splitVectorStore()
24618 SDValue Ptr0 = Store->getBasePtr(); in splitVectorStore()
24622 DAG.getStore(Store->getChain(), DL, Value0, Ptr0, Store->getPointerInfo(), in splitVectorStore()
24623 Store->getOriginalAlign(), in splitVectorStore()
24624 Store->getMemOperand()->getFlags()); in splitVectorStore()
24625 SDValue Ch1 = DAG.getStore(Store->getChain(), DL, Value1, Ptr1, in splitVectorStore()
24626 Store->getPointerInfo().getWithOffset(HalfOffset), in splitVectorStore()
24627 Store->getOriginalAlign(), in splitVectorStore()
24628 Store->getMemOperand()->getFlags()); in splitVectorStore()
24636 SDValue StoredVal = Store->getValue(); in scalarizeVectorStore()
24638 StoredVal.getValueType().is128BitVector() && "Expecting 128-bit op"); in scalarizeVectorStore()
24644 if (!Store->isSimple()) in scalarizeVectorStore()
24655 SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(), in scalarizeVectorStore()
24659 SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr, in scalarizeVectorStore()
24660 Store->getPointerInfo().getWithOffset(Offset), in scalarizeVectorStore()
24661 Store->getOriginalAlign(), in scalarizeVectorStore()
24662 Store->getMemOperand()->getFlags()); in scalarizeVectorStore()
24672 SDValue StoredVal = St->getValue(); in LowerStore()
24679 assert(!St->isTruncatingStore() && "Expected non-truncating store"); in LowerStore()
24694 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), in LowerStore()
24695 St->getPointerInfo(), St->getOriginalAlign(), in LowerStore()
24696 St->getMemOperand()->getFlags()); in LowerStore()
24699 if (St->isTruncatingStore()) in LowerStore()
24702 // If this is a 256-bit store of concatenated ops, we are better off splitting in LowerStore()
24703 // that store into two 128-bit stores. This avoids spurious use of 256-bit ops in LowerStore()
24729 // Widen the vector, cast to a v2x64 type, extract the single 64-bit element in LowerStore()
24737 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), in LowerStore()
24738 St->getPointerInfo(), St->getOriginalAlign(), in LowerStore()
24739 St->getMemOperand()->getFlags()); in LowerStore()
24743 SDValue Ops[] = {St->getChain(), StoredVal, St->getBasePtr()}; in LowerStore()
24745 St->getMemOperand()); in LowerStore()
24767 assert(EVT(RegVT) == Ld->getMemoryVT() && "Expected non-extending load"); in LowerLoad()
24772 SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(), in LowerLoad()
24773 Ld->getPointerInfo(), Ld->getOriginalAlign(), in LowerLoad()
24774 Ld->getMemOperand()->getFlags()); in LowerLoad()
24777 assert(NewLd->getNumValues() == 2 && "Loads must carry a chain!"); in LowerLoad()
24813 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in LowerBRCOND()
24844 // have a fall-through edge, because this requires an explicit in LowerBRCOND()
24846 if (Op.getNode()->hasOneUse()) { in LowerBRCOND()
24847 SDNode *User = *Op.getNode()->use_begin(); in LowerBRCOND()
24851 if (User->getOpcode() == ISD::BR) { in LowerBRCOND()
24852 SDValue FalseBB = User->getOperand(1); in LowerBRCOND()
24854 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest); in LowerBRCOND()
24940 EVT VT = Node->getValueType(0); in LowerDYNAMIC_STACKALLOC()
24974 DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT)); in LowerDYNAMIC_STACKALLOC()
24998 MF.getInfo<X86MachineFunctionInfo>()->setHasDynAlloca(true); in LowerDYNAMIC_STACKALLOC()
25001 Register SPReg = RegInfo->getStackRegister(); in LowerDYNAMIC_STACKALLOC()
25007 DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT)); in LowerDYNAMIC_STACKALLOC()
25025 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); in LowerVASTART()
25032 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); in LowerVASTART()
25038 // gp_offset (0 - 6 * 8) in LowerVASTART()
25039 // fp_offset (48 - 48 + 8 * 16) in LowerVASTART()
25047 DAG.getConstant(FuncInfo->getVarArgsGPOffset(), DL, MVT::i32), FIN, in LowerVASTART()
25055 DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32), FIN, in LowerVASTART()
25061 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); in LowerVASTART()
25069 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT); in LowerVASTART()
25079 "LowerVAARG only handles 64-bit va_arg!"); in LowerVAARG()
25089 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); in LowerVAARG()
25093 EVT ArgVT = Op.getNode()->getValueType(0); in LowerVAARG()
25137 // X86-64 va_list is a struct { i32, i32, i8*, i8* }, except on Windows, in LowerVACOPY()
25139 assert(Subtarget.is64Bit() && "This code only handles 64-bit va_copy!"); in LowerVACOPY()
25148 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); in LowerVACOPY()
25149 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); in LowerVACOPY()
25198 ShiftAmt = ElementType.getSizeInBits() - 1; in getTargetVShiftByConstNode()
25204 && "Unknown target vector shift-by-constant node"); in getTargetVShiftByConstNode()
25244 SmallVector<int> Mask(AmtVT.getVectorNumElements(), -1); in getTargetVShiftNode()
25249 // Peek through any zext node if we can get back to a 128-bit source. in getTargetVShiftNode()
25260 // The shift uses the entire lower 64-bits of the amount vector, so no need to in getTargetVShiftNode()
25266 // If the shift amount has come from a scalar, then zero-extend the scalar in getTargetVShiftNode()
25275 // then we can zero-extend it by setting all the other mask elements to in getTargetVShiftNode()
25290 // Extract if the shift amount vector is larger than 128-bits. in getTargetVShiftNode()
25296 // Zero-extend bottom element to v2i64 vector type, either by extension or in getTargetVShiftNode()
25307 (128 - AmtVT.getScalarSizeInBits()) / 8, SDLoc(ShAmt), MVT::i8); in getTargetVShiftNode()
25316 // Change opcode to non-immediate version. in getTargetVShiftNode()
25319 // The return type has to be a 128-bit type with the same element in getTargetVShiftNode()
25396 if (MaskConst->getZExtValue() & 0x1) in getScalarMaskingNode()
25417 if (!Fn->hasPersonalityFn()) in getSEHRegistrationNodeSize()
25420 // The RegNodeSize is 6 32-bit words for SEH and 4 for C++ EH. See in getSEHRegistrationNodeSize()
25422 switch (classifyEHPersonality(Fn->getPersonalityFn())) { in getSEHRegistrationNodeSize()
25428 "can only recover FP for 32-bit MSVC EH personality functions"); in getSEHRegistrationNodeSize()
25435 /// RegNodeBase = EntryEBP - RegNodeSize
25436 /// ParentFP = RegNodeBase - ParentFrameOffset
25450 if (!Fn->hasPersonalityFn()) in recoverFramePointer()
25456 GlobalValue::dropLLVMManglingEscape(Fn->getName())); in recoverFramePointer()
25468 // RegNodeBase = EntryEBP - RegNodeSize in recoverFramePointer()
25469 // ParentFP = RegNodeBase - ParentFrameOffset in recoverFramePointer()
25480 return C->getAPIntValue() == X86::STATIC_ROUNDING::CUR_DIRECTION; in LowerINTRINSIC_WO_CHAIN()
25486 unsigned RC = C->getZExtValue(); in LowerINTRINSIC_WO_CHAIN()
25500 RC = C->getZExtValue(); in LowerINTRINSIC_WO_CHAIN()
25520 SelectionDAG::FlagInserter FlagsInserter(DAG, Op->getFlags()); in LowerINTRINSIC_WO_CHAIN()
25523 switch(IntrData->Type) { in LowerINTRINSIC_WO_CHAIN()
25526 // First, we check if the intrinsic may have non-default rounding mode, in LowerINTRINSIC_WO_CHAIN()
25527 // (IntrData->Opc1 != 0), then we check the rounding mode operand. in LowerINTRINSIC_WO_CHAIN()
25528 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25539 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25547 Opc = IntrData->Opc0; in LowerINTRINSIC_WO_CHAIN()
25549 Opc = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25559 // First, we check if the intrinsic may have non-default rounding mode, in LowerINTRINSIC_WO_CHAIN()
25560 // (IntrData->Opc1 != 0), then we check the rounding mode operand. in LowerINTRINSIC_WO_CHAIN()
25561 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25573 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25581 Opc = IntrData->Opc0; in LowerINTRINSIC_WO_CHAIN()
25583 Opc = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25596 if (IntrData->Type == INTR_TYPE_3OP_IMM8 && in LowerINTRINSIC_WO_CHAIN()
25598 Src3 = DAG.getTargetConstant(Src3->getAsZExtVal() & 0xff, dl, MVT::i8); in LowerINTRINSIC_WO_CHAIN()
25602 // First, we check if the intrinsic may have non-default rounding mode, in LowerINTRINSIC_WO_CHAIN()
25603 // (IntrData->Opc1 != 0), then we check the rounding mode operand. in LowerINTRINSIC_WO_CHAIN()
25604 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25616 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25620 assert(Op.getOperand(4)->getOpcode() == ISD::TargetConstant); in LowerINTRINSIC_WO_CHAIN()
25623 Src4 = DAG.getTargetConstant(Src4->getAsZExtVal() & 0xff, dl, MVT::i8); in LowerINTRINSIC_WO_CHAIN()
25626 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25635 // - RC Opcode is specified and in LowerINTRINSIC_WO_CHAIN()
25636 // - RC is not "current direction". in LowerINTRINSIC_WO_CHAIN()
25637 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25650 DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru, in LowerINTRINSIC_WO_CHAIN()
25661 Opc = IntrData->Opc0; in LowerINTRINSIC_WO_CHAIN()
25663 Opc = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25675 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25677 // (1) With suppress-all-exceptions (sae) or rounding mode- 6 operands in LowerINTRINSIC_WO_CHAIN()
25678 // (2) With rounding mode and sae - 7 operands. in LowerINTRINSIC_WO_CHAIN()
25692 return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1, in LowerINTRINSIC_WO_CHAIN()
25700 unsigned Opc = IntrData->Opc0; in LowerINTRINSIC_WO_CHAIN()
25722 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2); in LowerINTRINSIC_WO_CHAIN()
25724 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
25739 Opc = IntrData->Opc0; in LowerINTRINSIC_WO_CHAIN()
25741 Opc = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25754 if (IntrData->Opc1 != 0) { in LowerINTRINSIC_WO_CHAIN()
25758 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
25764 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2); in LowerINTRINSIC_WO_CHAIN()
25773 unsigned Opc = IntrData->Opc0; in LowerINTRINSIC_WO_CHAIN()
25774 if (IntrData->Opc1 != 0) { in LowerINTRINSIC_WO_CHAIN()
25777 Opc = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25794 Opc = IntrData->Opc0; in LowerINTRINSIC_WO_CHAIN()
25796 Opc = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25810 unsigned Opc = IntrData->Opc0; in LowerINTRINSIC_WO_CHAIN()
25811 if (IntrData->Opc1 != 0) { in LowerINTRINSIC_WO_CHAIN()
25814 Opc = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25830 return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1); in LowerINTRINSIC_WO_CHAIN()
25837 return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1); in LowerINTRINSIC_WO_CHAIN()
25848 if (IntrData->Type == CFMA_OP_MASKZ) in LowerINTRINSIC_WO_CHAIN()
25852 // - RC Opcode is specified and in LowerINTRINSIC_WO_CHAIN()
25853 // - RC is not "current direction". in LowerINTRINSIC_WO_CHAIN()
25855 if (IntrData->Opc1 != 0) { in LowerINTRINSIC_WO_CHAIN()
25859 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, Src3, in LowerINTRINSIC_WO_CHAIN()
25865 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2, Src3); in LowerINTRINSIC_WO_CHAIN()
25871 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
25877 SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm); in LowerINTRINSIC_WO_CHAIN()
25893 // First, we check if the intrinsic may have non-default rounding mode, in LowerINTRINSIC_WO_CHAIN()
25894 // (IntrData->Opc1 != 0), then we check the rounding mode operand. in LowerINTRINSIC_WO_CHAIN()
25895 if (IntrData->Opc1 != 0) { in LowerINTRINSIC_WO_CHAIN()
25898 return DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
25904 return DAG.getNode(IntrData->Opc0, dl, MaskVT, in LowerINTRINSIC_WO_CHAIN()
25914 if (IntrData->Opc1 != 0) { in LowerINTRINSIC_WO_CHAIN()
25917 Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::v1i1, Src1, Src2, CC, Sae); in LowerINTRINSIC_WO_CHAIN()
25923 Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC); in LowerINTRINSIC_WO_CHAIN()
25935 ISD::CondCode CC = (ISD::CondCode)IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
25942 SDValue Comi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
26000 // Catch shift-by-constant. in LowerINTRINSIC_WO_CHAIN()
26002 return getTargetVShiftByConstNode(IntrData->Opc0, dl, in LowerINTRINSIC_WO_CHAIN()
26004 CShAmt->getZExtValue(), DAG); in LowerINTRINSIC_WO_CHAIN()
26007 return getTargetVShiftNode(IntrData->Opc0, dl, Op.getSimpleValueType(), in LowerINTRINSIC_WO_CHAIN()
26021 return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru, in LowerINTRINSIC_WO_CHAIN()
26031 SDValue Passthru = (IntrData->Type == FIXUPIMM) in LowerINTRINSIC_WO_CHAIN()
26035 unsigned Opc = IntrData->Opc0; in LowerINTRINSIC_WO_CHAIN()
26036 if (IntrData->Opc1 != 0) { in LowerINTRINSIC_WO_CHAIN()
26039 Opc = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
26052 assert(IntrData->Opc0 == X86ISD::VRNDSCALE && "Unexpected opcode"); in LowerINTRINSIC_WO_CHAIN()
26057 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26061 assert(IntrData->Opc0 == X86ISD::VRNDSCALES && "Unexpected opcode"); in LowerINTRINSIC_WO_CHAIN()
26066 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26070 assert(IntrData->Opc0 == X86ISD::BEXTRI && "Unexpected opcode"); in LowerINTRINSIC_WO_CHAIN()
26075 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26080 SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
26087 Res = DAG.getNode(IntrData->Opc1, dl, VTs, Op.getOperand(2), in LowerINTRINSIC_WO_CHAIN()
26091 DAG.getConstant(-1, dl, MVT::i8)); in LowerINTRINSIC_WO_CHAIN()
26092 Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(2), in LowerINTRINSIC_WO_CHAIN()
26108 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src); in LowerINTRINSIC_WO_CHAIN()
26113 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
26123 unsigned Opc = IntrData->Opc0; in LowerINTRINSIC_WO_CHAIN()
26137 Opc = IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
26149 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src); in LowerINTRINSIC_WO_CHAIN()
26155 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, PassThru, in LowerINTRINSIC_WO_CHAIN()
26311 SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops())); in LowerINTRINSIC_WO_CHAIN()
26326 SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops())); in LowerINTRINSIC_WO_CHAIN()
26339 SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops())); in LowerINTRINSIC_WO_CHAIN()
26359 auto *Fn = cast<Function>(cast<GlobalAddressSDNode>(Op1)->getGlobal()); in LowerINTRINSIC_WO_CHAIN()
26361 GlobalValue::dropLLVMManglingEscape(Fn->getName())); in LowerINTRINSIC_WO_CHAIN()
26364 // supported on 32-bit Windows, which isn't PIC. in LowerINTRINSIC_WO_CHAIN()
26373 auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr); in LowerINTRINSIC_WO_CHAIN()
26386 if (RegInfo->hasBasePointer(MF)) in LowerINTRINSIC_WO_CHAIN()
26387 Reg = RegInfo->getBaseRegister(); in LowerINTRINSIC_WO_CHAIN()
26389 bool CantUseFP = RegInfo->hasStackRealignment(MF); in LowerINTRINSIC_WO_CHAIN()
26391 Reg = RegInfo->getPtrSizedStackRegister(MF); in LowerINTRINSIC_WO_CHAIN()
26393 Reg = RegInfo->getPtrSizedFrameRegister(MF); in LowerINTRINSIC_WO_CHAIN()
26410 Op->getOperand(1), Op->getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
26431 // to 8-bits which may make it no longer out of bounds. in LowerINTRINSIC_WO_CHAIN()
26432 unsigned ShiftAmount = C->getAPIntValue().getLimitedValue(255); in LowerINTRINSIC_WO_CHAIN()
26483 // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit). in LowerINTRINSIC_WO_CHAIN()
26505 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, in getAVX2GatherNode()
26523 MemIntr->getMemoryVT(), MemIntr->getMemOperand()); in getAVX2GatherNode()
26538 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, in getGatherNode()
26561 MemIntr->getMemoryVT(), MemIntr->getMemOperand()); in getGatherNode()
26575 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, in getScatterNode()
26592 MemIntr->getMemoryVT(), MemIntr->getMemOperand()); in getScatterNode()
26606 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, in getPrefetchNode()
26623 /// Returns a Glue value which can be used to add extra copy-from-reg if the
26632 SDValue Chain = N->getOperand(0); in expandIntrinsicWChainHelper()
26636 assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); in expandIntrinsicWChainHelper()
26637 Chain = DAG.getCopyToReg(Chain, DL, SrcReg, N->getOperand(2), Glue); in expandIntrinsicWChainHelper()
26662 // Merge the two 32-bit values into a 64-bit one. in expandIntrinsicWChainHelper()
26670 // Use a buildpair to merge the two 32-bit values into a 64-bit one. in expandIntrinsicWChainHelper()
26685 // The processor's time-stamp counter (a 64-bit MSR) is stored into the in getReadTimeStampCounter()
26686 // EDX:EAX registers. EDX is loaded with the high-order 32 bits of the MSR in getReadTimeStampCounter()
26687 // and the EAX register is loaded with the low-order 32 bits. in getReadTimeStampCounter()
26723 EHInfo->EHRegNodeFrameIndex = FINode->getIndex(); in MarkEHRegistrationNode()
26741 EHInfo->EHGuardFrameIndex = FINode->getIndex(); in MarkEHGuard()
26774 // 64-bit targets support extended Swift async frame setup, in isExtendedSwiftAsyncFrameSupported()
26776 return !MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); in isExtendedSwiftAsyncFrameSupported()
26792 X86FI->setHasSwiftAsyncContext(true); in LowerINTRINSIC_W_CHAIN()
26793 SDValue Chain = Op->getOperand(0); in LowerINTRINSIC_W_CHAIN()
26800 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, in LowerINTRINSIC_W_CHAIN()
26805 if (!X86FI->getSwiftAsyncContextFrameIdx()) in LowerINTRINSIC_W_CHAIN()
26806 X86FI->setSwiftAsyncContextFrameIdx( in LowerINTRINSIC_W_CHAIN()
26810 DAG.getFrameIndex(*X86FI->getSwiftAsyncContextFrameIdx(), in LowerINTRINSIC_W_CHAIN()
26813 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, in LowerINTRINSIC_W_CHAIN()
26814 Op->getOperand(0)); in LowerINTRINSIC_W_CHAIN()
26861 SDValue Chain = Op->getOperand(0); in LowerINTRINSIC_W_CHAIN()
26880 DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2), in LowerINTRINSIC_W_CHAIN()
26881 Op->getOperand(3), Op->getOperand(4)); in LowerINTRINSIC_W_CHAIN()
26883 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, in LowerINTRINSIC_W_CHAIN()
26904 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, in LowerINTRINSIC_W_CHAIN()
26933 MachineMemOperand *MMO = MemIntr->getMemOperand(); in LowerINTRINSIC_W_CHAIN()
26934 EVT MemVT = MemIntr->getMemoryVT(); in LowerINTRINSIC_W_CHAIN()
26940 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
26971 MachineMemOperand *MMO = MemIntr->getMemOperand(); in LowerINTRINSIC_W_CHAIN()
26972 EVT MemVT = MemIntr->getMemoryVT(); in LowerINTRINSIC_W_CHAIN()
26981 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
26994 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, in LowerINTRINSIC_W_CHAIN()
27008 MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand(); in LowerINTRINSIC_W_CHAIN()
27014 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain); in LowerINTRINSIC_W_CHAIN()
27028 MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand(); in LowerINTRINSIC_W_CHAIN()
27034 unsigned Imm = Op2->getAsZExtVal(); in LowerINTRINSIC_W_CHAIN()
27038 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain); in LowerINTRINSIC_W_CHAIN()
27048 MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand(); in LowerINTRINSIC_W_CHAIN()
27050 X86ISD::CMPCCXADD, DL, Op->getVTList(), {Chain, Addr, Src1, Src2, CC}, in LowerINTRINSIC_W_CHAIN()
27088 MachineMemOperand *MMO = cast<MemSDNode>(Op)->getMemOperand(); in LowerINTRINSIC_W_CHAIN()
27089 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
27123 MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand(); in LowerINTRINSIC_W_CHAIN()
27135 switch(IntrData->Type) { in LowerINTRINSIC_W_CHAIN()
27140 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32, MVT::Other); in LowerINTRINSIC_W_CHAIN()
27141 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0)); in LowerINTRINSIC_W_CHAIN()
27145 SDValue Ops[] = {DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)), in LowerINTRINSIC_W_CHAIN()
27146 DAG.getConstant(1, dl, Op->getValueType(1)), in LowerINTRINSIC_W_CHAIN()
27149 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops); in LowerINTRINSIC_W_CHAIN()
27152 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid, in LowerINTRINSIC_W_CHAIN()
27162 return getAVX2GatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, in LowerINTRINSIC_W_CHAIN()
27184 return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, in LowerINTRINSIC_W_CHAIN()
27191 unsigned Opcode = (HintVal == 2 ? IntrData->Opc1 : IntrData->Opc0); in LowerINTRINSIC_W_CHAIN()
27203 getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget, in LowerINTRINSIC_W_CHAIN()
27219 expandIntrinsicWChainHelper(Op.getNode(), dl, DAG, IntrData->Opc0, X86::ECX, in LowerINTRINSIC_W_CHAIN()
27225 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other); in LowerINTRINSIC_W_CHAIN()
27226 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0)); in LowerINTRINSIC_W_CHAIN()
27229 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC); in LowerINTRINSIC_W_CHAIN()
27230 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
27244 EVT MemVT = MemIntr->getMemoryVT(); in LowerINTRINSIC_W_CHAIN()
27246 uint16_t TruncationOp = IntrData->Opc0; in LowerINTRINSIC_W_CHAIN()
27251 MemIntr->getMemOperand()); in LowerINTRINSIC_W_CHAIN()
27258 MemVT, MemIntr->getMemOperand(), ISD::UNINDEXED, in LowerINTRINSIC_W_CHAIN()
27266 MemIntr->getMemOperand(), DAG); in LowerINTRINSIC_W_CHAIN()
27272 VMask, MemVT, MemIntr->getMemOperand(), DAG); in LowerINTRINSIC_W_CHAIN()
27296 SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), dl, PtrVT); in LowerRETURNADDR()
27323 if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) { in LowerFRAMEADDR()
27327 int FrameAddrIndex = FuncInfo->getFAIndex(); in LowerFRAMEADDR()
27330 unsigned SlotSize = RegInfo->getSlotSize(); in LowerFRAMEADDR()
27333 FuncInfo->setFAIndex(FrameAddrIndex); in LowerFRAMEADDR()
27339 RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction()); in LowerFRAMEADDR()
27346 while (Depth--) in LowerFRAMEADDR()
27374 Register FrameReg = RegInfo->getPtrSizedFrameRegister(MF); in getRegisterByName()
27390 return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize(), SDLoc(Op)); in LowerFRAME_TO_ARGS_OFFSET()
27421 Register FrameReg = RegInfo->getFrameRegister(DAG.getMachineFunction()); in LowerEH_RETURN()
27429 DAG.getIntPtrConstant(RegInfo->getSlotSize(), in LowerEH_RETURN()
27450 (void)TII->getGlobalBaseReg(&DAG.getMachineFunction()); in lowerEH_SJLJ_SETJMP()
27483 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); in LowerINIT_TRAMPOLINE()
27489 // Large code-model. in LowerINIT_TRAMPOLINE()
27490 const unsigned char JMP64r = 0xFF; // 64-bit jmp through register opcode. in LowerINIT_TRAMPOLINE()
27493 const unsigned char N86R10 = TRI->getEncodingValue(X86::R10) & 0x7; in LowerINIT_TRAMPOLINE()
27494 const unsigned char N86R11 = TRI->getEncodingValue(X86::R11) & 0x7; in LowerINIT_TRAMPOLINE()
27538 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue()); in LowerINIT_TRAMPOLINE()
27539 CallingConv::ID CC = Func->getCallingConv(); in LowerINIT_TRAMPOLINE()
27552 FunctionType *FTy = Func->getFunctionType(); in LowerINIT_TRAMPOLINE()
27553 const AttributeList &Attrs = Func->getAttributes(); in LowerINIT_TRAMPOLINE()
27555 if (!Attrs.isEmpty() && !Func->isVarArg()) { in LowerINIT_TRAMPOLINE()
27559 for (FunctionType::param_iterator I = FTy->param_begin(), in LowerINIT_TRAMPOLINE()
27560 E = FTy->param_end(); I != E; ++I, ++Idx) in LowerINIT_TRAMPOLINE()
27568 report_fatal_error("Nest register in use - reduce number of inreg" in LowerINIT_TRAMPOLINE()
27594 const unsigned char N86Reg = TRI->getEncodingValue(NestReg) & 0x7; in LowerINIT_TRAMPOLINE()
27626 01 Round to -inf in LowerGET_ROUNDING()
27631 -1 Undefined in LowerGET_ROUNDING()
27635 3 Round to -inf in LowerGET_ROUNDING()
27637 To perform the conversion, we use a packed lookup table of the four 2-bit in LowerGET_ROUNDING()
27639 0x2d --> (0b00,10,11,01) --> (0,2,3,1) >> FPSR[11:10] in LowerGET_ROUNDING()
27688 SDValue Chain = Op.getNode()->getOperand(0); in LowerSET_ROUNDING()
27711 SDValue NewRM = Op.getNode()->getOperand(1); in LowerSET_ROUNDING()
27714 uint64_t RM = CVal->getZExtValue(); in LowerSET_ROUNDING()
27717 // clang-format off in LowerSET_ROUNDING()
27724 // clang-format on in LowerSET_ROUNDING()
27729 // 0 Round to 0 -> 11 in LowerSET_ROUNDING()
27730 // 1 Round to nearest -> 00 in LowerSET_ROUNDING()
27731 // 2 Round to +inf -> 10 in LowerSET_ROUNDING()
27732 // 3 Round to -inf -> 01 in LowerSET_ROUNDING()
27733 // The 2-bit value needs then to be shifted so that it occupies bits 11:10. in LowerSET_ROUNDING()
27806 SDValue Chain = Op->getOperand(0); in LowerGET_FPENV_MEM()
27807 SDValue Ptr = Op->getOperand(1); in LowerGET_FPENV_MEM()
27809 EVT MemVT = Node->getMemoryVT(); in LowerGET_FPENV_MEM()
27811 MachineMemOperand *MMO = cast<FPStateAccessSDNode>(Op)->getMemOperand(); in LowerGET_FPENV_MEM()
27822 (MMO->getFlags() & ~MachineMemOperand::MOStore); in LowerGET_FPENV_MEM()
27872 SDValue Chain = Op->getOperand(0); in LowerSET_FPENV_MEM()
27873 SDValue Ptr = Op->getOperand(1); in LowerSET_FPENV_MEM()
27875 EVT MemVT = Node->getMemoryVT(); in LowerSET_FPENV_MEM()
27877 MachineMemOperand *MMO = cast<FPStateAccessSDNode>(Op)->getMemOperand(); in LowerSET_FPENV_MEM()
27885 SDValue Chain = Op.getNode()->getOperand(0); in LowerRESET_FPENV()
27891 // x87 FPU Control Word: mask all floating-point exceptions, sets rounding to in LowerRESET_FPENV()
27900 // MXCSR: mask all floating-point exceptions, sets rounding to nearest, clear in LowerRESET_FPENV()
27944 SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT); in LowerVectorCTLZ_AVX512CDI()
27958 // Per-nibble leading zero PSHUFB lookup table. in LowerVectorCTLZInRegLUT()
28038 // vXi8 vectors need to be promoted to 512-bits for vXi32. in LowerVectorCTLZ()
28042 // Decompose 256-bit ops into smaller 128-bit ops. in LowerVectorCTLZ()
28046 // Decompose 512-bit ops into smaller 256-bit ops. in LowerVectorCTLZ()
28078 SDValue Ops[] = {Op, DAG.getConstant(NumBits + NumBits - 1, dl, OpVT), in LowerCTLZ()
28084 // Finally xor with NumBits-1. in LowerCTLZ()
28086 DAG.getConstant(NumBits - 1, dl, OpVT)); in LowerCTLZ()
28131 "Only handle AVX 256-bit vector integer operation"); in lowerAddSub()
28157 // Handle a special-case with a bit-hack instead of cmp+select: in LowerADDSAT_SUBSAT()
28158 // usubsat X, SMIN --> (X ^ SMIN) & (X s>> BW-1) in LowerADDSAT_SUBSAT()
28163 if (C && C->getAPIntValue().isSignMask()) { in LowerADDSAT_SUBSAT()
28164 SDValue SignMask = DAG.getConstant(C->getAPIntValue(), DL, VT); in LowerADDSAT_SUBSAT()
28165 SDValue ShiftAmt = DAG.getConstant(BitWidth - 1, DL, VT); in LowerADDSAT_SUBSAT()
28172 // usubsat X, Y --> (X >u Y) ? X - Y : 0 in LowerADDSAT_SUBSAT()
28211 // Since X86 does not have CMOV for 8-bit integer, we don't convert in LowerABS()
28212 // 8-bit integer abs to NEG and CMOV. in LowerABS()
28221 // ABS(vXi64 X) --> VPBLENDVPD(X, 0-X, X). in LowerABS()
28230 "Only handle AVX 256-bit vector integer operation"); in LowerABS()
28301 // Num xNaN +0 -0 in LowerFMINIMUM_FMAXIMUM()
28302 // --------------- --------------- in LowerFMINIMUM_FMAXIMUM()
28304 // X --------------- X --------------- in LowerFMINIMUM_FMAXIMUM()
28305 // xNaN | X | X/Y | -0 | +0 | -0 | in LowerFMINIMUM_FMAXIMUM()
28306 // --------------- --------------- in LowerFMINIMUM_FMAXIMUM()
28317 return CstOp->getValueAPF().bitcastToAPInt() == Zero; in LowerFMINIMUM_FMAXIMUM()
28319 return CstOp->getAPIntValue() == Zero; in LowerFMINIMUM_FMAXIMUM()
28320 if (Op->getOpcode() == ISD::BUILD_VECTOR || in LowerFMINIMUM_FMAXIMUM()
28321 Op->getOpcode() == ISD::SPLAT_VECTOR) { in LowerFMINIMUM_FMAXIMUM()
28322 for (const SDValue &OpVal : Op->op_values()) { in LowerFMINIMUM_FMAXIMUM()
28328 if (!CstOp->getValueAPF().isZero()) in LowerFMINIMUM_FMAXIMUM()
28330 if (CstOp->getValueAPF().bitcastToAPInt() != Zero) in LowerFMINIMUM_FMAXIMUM()
28341 Op->getFlags().hasNoSignedZeros() || in LowerFMINIMUM_FMAXIMUM()
28354 (Op->getFlags().hasNoNaNs() || IsXNeverNaN || IsYNeverNaN)) { in LowerFMINIMUM_FMAXIMUM()
28373 return DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags()); in LowerFMINIMUM_FMAXIMUM()
28404 Op->getFlags().hasNoNaNs() || (IsXNeverNaN && IsYNeverNaN); in LowerFMINIMUM_FMAXIMUM()
28412 SDValue MinMax = DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags()); in LowerFMINIMUM_FMAXIMUM()
28441 // abds(lhs, rhs) -> trunc(abs(sub(sext(lhs), sext(rhs)))) in LowerABD()
28442 // abdu(lhs, rhs) -> trunc(abs(sub(zext(lhs), zext(rhs)))) in LowerABD()
28461 // Decompose 256-bit ops into 128-bit ops. in LowerMUL()
28471 // Lower v16i8/v32i8/v64i8 mul as sign-extension to v8i16/v16i16/v32i16 in LowerMUL()
28497 for (auto [Idx, Val] : enumerate(B->ops())) { in LowerMUL()
28519 // pmullw, so it doesn't matter what's in the high byte of each 16-bit in LowerMUL()
28557 static const int UnpackMask[] = { 1, -1, 3, -1 }; in LowerMUL()
28641 // and use pmullw to calculate the full 16-bit product. in LowervXi8MulWithUNPCK()
28644 // pmulhw to calculate the full 16-bit product. This trick means we don't in LowervXi8MulWithUNPCK()
28711 bool IsSigned = Op->getOpcode() == ISD::MULHS; in LowerMULH()
28716 // Decompose 256-bit ops into 128-bit ops. in LowerMULH()
28740 const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1, in LowerMULH()
28741 9, -1, 11, -1, 13, -1, 15, -1}; in LowerMULH()
28796 // With SSE41 we can use sign/zero extend, but for pre-SSE41 we unpack in LowerMULH()
28823 bool IsSigned = Op->getOpcode() == ISD::SMULO; in LowerMULO()
28826 EVT OvfVT = Op->getValueType(1); in LowerMULO()
28935 // UMULO overflows if the high bits are non-zero. in LowerMULO()
28951 if (isa<ConstantSDNode>(Op->getOperand(1))) { in LowerWin64_i128OP()
28959 switch (Op->getOpcode()) { in LowerWin64_i128OP()
28960 // clang-format off in LowerWin64_i128OP()
28966 // clang-format on in LowerWin64_i128OP()
28974 for (unsigned i = 0, e = Op->getNumOperands(); i != e; ++i) { in LowerWin64_i128OP()
28975 EVT ArgVT = Op->getOperand(i).getValueType(); in LowerWin64_i128OP()
28979 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in LowerWin64_i128OP()
28984 DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MPI, Align(16)); in LowerWin64_i128OP()
29015 bool IsStrict = Op->isStrictFPOpcode(); in LowerWin64_FP_TO_INT128()
29024 if (Op->getOpcode() == ISD::FP_TO_SINT || in LowerWin64_FP_TO_INT128()
29025 Op->getOpcode() == ISD::STRICT_FP_TO_SINT) in LowerWin64_FP_TO_INT128()
29048 bool IsStrict = Op->isStrictFPOpcode(); in LowerWin64_INT128_TO_FP()
29057 if (Op->getOpcode() == ISD::SINT_TO_FP || in LowerWin64_INT128_TO_FP()
29058 Op->getOpcode() == ISD::STRICT_SINT_TO_FP) in LowerWin64_INT128_TO_FP()
29070 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in LowerWin64_INT128_TO_FP()
29095 (0x8080808080808080ULL >> (64 - (8 * Amt)))); in getGFNICtrlImm()
29097 return getGFNICtrlImm(ISD::SRL, 8 - Amt) | getGFNICtrlImm(ISD::SHL, Amt); in getGFNICtrlImm()
29099 return getGFNICtrlImm(ISD::SHL, 8 - Amt) | getGFNICtrlImm(ISD::SRL, Amt); in getGFNICtrlImm()
29118 // Return true if the required (according to Opcode) shift-imm form is natively
29147 // These instructions are defined together with shift-immediate.
29154 // Return true if the required (according to Opcode) variable-shift form is
29170 // vXi16 supported only on AVX-512, BWI in supportedVectorVarShift()
29209 ShiftAmt - 32, DAG); in LowerShiftByScalarImmediate()
29245 // shl: (shl V, 1) -> (add (freeze V), (freeze V)) in LowerShiftByScalarImmediate()
29247 // R may be undef at run-time, but (shl R, 1) must be an even number (LSB in LowerShiftByScalarImmediate()
29265 // If we're logical shifting an all-signbits value then we can just perform as in LowerShiftByScalarImmediate()
29281 // R may be undef at run-time, but (shl R, 1) must be an even number (LSB in LowerShiftByScalarImmediate()
29317 APInt Mask = APInt::getHighBitsSet(8, 8 - ShiftAmt); in LowerShiftByScalarImmediate()
29326 APInt Mask = APInt::getLowBitsSet(8, 8 - ShiftAmt); in LowerShiftByScalarImmediate()
29353 int BaseShAmtIdx = -1; in LowerShiftByScalarVariable()
29359 // vXi8 shifts - shift as v8i16 + mask result. in LowerShiftByScalarVariable()
29370 // Create the mask using vXi16 shifts. For shift-rights we need to move in LowerShiftByScalarVariable()
29372 SDValue BitMask = DAG.getConstant(-1, dl, ExtVT); in LowerShiftByScalarVariable()
29390 // SignMask = lshr(SignBit, Amt) - safe to do this with PSRLW. in LowerShiftByScalarVariable()
29502 // XOP has 128-bit variable logical/arithmetic shifts. in LowerShift()
29503 // +ve/-ve Amt = shift left/right. in LowerShift()
29514 // 2i64 vector logical shifts can efficiently avoid scalarization - do the in LowerShift()
29515 // shifts per-lane and then shuffle the partial results back together. in LowerShift()
29541 SDValue A = Amt->getOperand(i); in LowerShift()
29567 if (Cst1 && Cst2 && Cst1->getAPIntValue().ult(EltSizeInBits) && in LowerShift()
29568 Cst2->getAPIntValue().ult(EltSizeInBits)) { in LowerShift()
29570 Cst1->getZExtValue(), DAG); in LowerShift()
29572 Cst2->getZExtValue(), DAG); in LowerShift()
29587 // can replace with ISD::MULHU, creating scale factor from (NumEltBits - Amt). in LowerShift()
29601 // can replace with ISD::MULHS, creating scale factor from (NumEltBits - Amt). in LowerShift()
29603 // of these cases in pre-SSE41/XOP/AVX512 but not both. in LowerShift()
29626 // immediate shifts, else we need to zero-extend each lane to the lower i64 in LowerShift()
29639 // just zero-extending, but for SSE just duplicating the top 16-bits is in LowerShift()
29643 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1}); in LowerShift()
29644 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1}); in LowerShift()
29645 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1}); in LowerShift()
29646 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1}); in LowerShift()
29650 {4, 5, 6, 7, -1, -1, -1, -1}); in LowerShift()
29667 // TODO - ideally shuffle combining would handle this. in LowerShift()
29669 SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1}); in LowerShift()
29670 SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7}); in LowerShift()
29673 SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5}); in LowerShift()
29674 SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7}); in LowerShift()
29681 // NOTE: We honor prefered vector width before promoting to 512-bits. in LowerShift()
29773 // On pre-SSE41 targets we test for the sign bit by comparing to in LowerShift()
29774 // zero - a negative value will set all bits of the lanes to true in LowerShift()
29875 // If we have a constant shift amount, the non-SSE41 path is best as in LowerShift()
29891 // On pre-SSE41 targets we splat the sign bit - a negative value will in LowerShift()
29938 // Decompose 256-bit shifts into 128-bit shifts. in LowerShift()
29984 // fshl(x,y,z) -> unpack(y,x) << (z & (bw-1))) >> bw. in LowerFunnelShift()
29985 // fshr(x,y,z) -> unpack(y,x) >> (z & (bw-1))). in LowerFunnelShift()
29991 uint64_t ShXAmt = IsFSHR ? (EltSizeInBits - ShiftAmt) : ShiftAmt; in LowerFunnelShift()
29992 uint64_t ShYAmt = IsFSHR ? ShiftAmt : (EltSizeInBits - ShiftAmt); in LowerFunnelShift()
30001 // bit-select - lower using vXi16 shifts and then perform the bitmask at in LowerFunnelShift()
30003 APInt MaskX = APInt::getHighBitsSet(8, 8 - ShXAmt); in LowerFunnelShift()
30004 APInt MaskY = APInt::getLowBitsSet(8, 8 - ShYAmt); in LowerFunnelShift()
30025 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT); in LowerFunnelShift()
30037 // Split 256-bit integers on XOP/pre-AVX2 targets. in LowerFunnelShift()
30038 // Split 512-bit integers on non 512-bit BWI targets. in LowerFunnelShift()
30043 // Pre-mask the amount modulo using the wider vector. in LowerFunnelShift()
30050 int ScalarAmtIdx = -1; in LowerFunnelShift()
30070 // If per-element shifts are legal, fallback to generic expansion. in LowerFunnelShift()
30075 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z & (bw-1))) >> bw. in LowerFunnelShift()
30076 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z & (bw-1))). in LowerFunnelShift()
30092 // Attempt to fold per-element (ExtVT) shift as unpack(y,x) << zext(z) in LowerFunnelShift()
30116 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z & (bw-1))) >> bw. in LowerFunnelShift()
30117 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z & (bw-1))). in LowerFunnelShift()
30120 SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, Amt.getValueType()); in LowerFunnelShift()
30183 // Else, fall-back on VPROLV/VPRORV. in LowerRotate()
30187 // AVX512 VBMI2 vXi16 - lower to funnel shifts. in LowerRotate()
30216 // Split 256-bit integers on XOP/pre-AVX2 targets. in LowerRotate()
30220 // XOP has 128-bit vector variable + immediate rotates. in LowerRotate()
30221 // +ve/-ve Amt = rotate left/right - just need to handle ISD::ROTL. in LowerRotate()
30225 assert(VT.is128BitVector() && "Only rotate 128-bit vectors!"); in LowerRotate()
30234 // Use general rotate by variable (per-element). in LowerRotate()
30238 // Rotate by an uniform constant - expand back to shifts. in LowerRotate()
30243 uint64_t ShlAmt = IsROTL ? RotAmt : (EltSizeInBits - RotAmt); in LowerRotate()
30244 uint64_t SrlAmt = IsROTL ? (EltSizeInBits - RotAmt) : RotAmt; in LowerRotate()
30252 // Split 512-bit integers on non 512-bit BWI targets. in LowerRotate()
30266 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT); in LowerRotate()
30270 // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw. in LowerRotate()
30271 // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))). in LowerRotate()
30273 int BaseRotAmtIdx = -1; in LowerRotate()
30294 // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw. in LowerRotate()
30295 // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))). in LowerRotate()
30296 // Const vXi16/vXi32 are excluded in favor of MUL-based lowering. in LowerRotate()
30317 // rotl(x,y) -> (((aext(x) << bw) | zext(x)) << (y & (bw-1))) >> bw. in LowerRotate()
30318 // rotr(x,y) -> (((aext(x) << bw) | zext(x)) >> (y & (bw-1))). in LowerRotate()
30347 // On pre-SSE41 targets we test for the sign bit by comparing to in LowerRotate()
30348 // zero - a negative value will set all bits of the lanes to true in LowerRotate()
30405 // Fallback for non-constants AVX2 vXi16 as well. in LowerRotate()
30440 // to v2i64 results at a time. The upper 32-bits contain the wrapped bits in LowerRotate()
30441 // that can then be OR'd with the lower 32-bits. in LowerRotate()
30443 static const int OddMask[] = {1, -1, 3, -1}; in LowerRotate()
30466 unsigned OpWidth = MemType->getPrimitiveSizeInBits(); in needsCmpXchgNb()
30478 Type *MemType = SI->getValueOperand()->getType(); in shouldExpandAtomicStoreInIR()
30480 if (!SI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat) && in shouldExpandAtomicStoreInIR()
30482 if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() && in shouldExpandAtomicStoreInIR()
30486 if (MemType->getPrimitiveSizeInBits() == 128 && Subtarget.is64Bit() && in shouldExpandAtomicStoreInIR()
30498 Type *MemType = LI->getType(); in shouldExpandAtomicLoadInIR()
30500 if (!LI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat) && in shouldExpandAtomicLoadInIR()
30502 // If this a 64 bit atomic load on a 32-bit target and SSE2 is enabled, we in shouldExpandAtomicLoadInIR()
30503 // can use movq to do the load. If we have X87 we can load into an 80-bit in shouldExpandAtomicLoadInIR()
30505 if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() && in shouldExpandAtomicLoadInIR()
30509 // If this is a 128-bit load with AVX, 128-bit SSE loads/stores are atomic. in shouldExpandAtomicLoadInIR()
30510 if (MemType->getPrimitiveSizeInBits() == 128 && Subtarget.is64Bit() && in shouldExpandAtomicLoadInIR()
30533 if (isPowerOf2_64(C->getZExtValue())) in FindSingleBitChange()
30535 else if (isPowerOf2_64((~C->getValue()).getZExtValue())) in FindSingleBitChange()
30540 // Check if V is some power of 2 pattern known to be non-zero in FindSingleBitChange()
30559 if (I->getOpcode() == Instruction::Shl) { in FindSingleBitChange()
30561 // -X` and some other provable power of 2 patterns that we can use CTZ on in FindSingleBitChange()
30564 // non-zero even where C != 1. Likewise LShr(C, X) and AShr(C, X) may also in FindSingleBitChange()
30565 // be provably a non-zero power of 2. in FindSingleBitChange()
30568 auto *ShiftVal = dyn_cast<ConstantInt>(I->getOperand(0)); in FindSingleBitChange()
30571 if (ShiftVal->equalsInt(1)) in FindSingleBitChange()
30577 Value *BitV = I->getOperand(1); in FindSingleBitChange()
30583 if (*AndC == (I->getType()->getPrimitiveSizeInBits() - 1)) in FindSingleBitChange()
30597 if (AI->use_empty()) in shouldExpandLogicAtomicRMWInIR()
30600 if (AI->getOperation() == AtomicRMWInst::Xor) { in shouldExpandLogicAtomicRMWInIR()
30601 // A ^ SignBit -> A + SignBit. This allows us to use `xadd` which is in shouldExpandLogicAtomicRMWInIR()
30603 if (match(AI->getOperand(1), m_SignMask())) in shouldExpandLogicAtomicRMWInIR()
30609 // Note: InstCombinePass can cause a de-optimization here. It replaces the in shouldExpandLogicAtomicRMWInIR()
30613 Instruction *I = AI->user_back(); in shouldExpandLogicAtomicRMWInIR()
30614 auto BitChange = FindSingleBitChange(AI->getValOperand()); in shouldExpandLogicAtomicRMWInIR()
30615 if (BitChange.second == UndefBit || !AI->hasOneUse() || in shouldExpandLogicAtomicRMWInIR()
30616 I->getOpcode() != Instruction::And || in shouldExpandLogicAtomicRMWInIR()
30617 AI->getType()->getPrimitiveSizeInBits() == 8 || in shouldExpandLogicAtomicRMWInIR()
30618 AI->getParent() != I->getParent()) in shouldExpandLogicAtomicRMWInIR()
30621 unsigned OtherIdx = I->getOperand(0) == AI ? 1 : 0; in shouldExpandLogicAtomicRMWInIR()
30624 if (AI == I->getOperand(OtherIdx)) in shouldExpandLogicAtomicRMWInIR()
30629 auto *C1 = cast<ConstantInt>(AI->getValOperand()); in shouldExpandLogicAtomicRMWInIR()
30630 auto *C2 = dyn_cast<ConstantInt>(I->getOperand(OtherIdx)); in shouldExpandLogicAtomicRMWInIR()
30631 if (!C2 || !isPowerOf2_64(C2->getZExtValue())) { in shouldExpandLogicAtomicRMWInIR()
30634 if (AI->getOperation() == AtomicRMWInst::And) { in shouldExpandLogicAtomicRMWInIR()
30635 return ~C1->getValue() == C2->getValue() in shouldExpandLogicAtomicRMWInIR()
30645 auto BitTested = FindSingleBitChange(I->getOperand(OtherIdx)); in shouldExpandLogicAtomicRMWInIR()
30657 if (AI->getOperation() == AtomicRMWInst::And) in shouldExpandLogicAtomicRMWInIR()
30673 switch (AI->getOperation()) { in emitBitTestAtomicRMWIntrinsic()
30689 Instruction *I = AI->user_back(); in emitBitTestAtomicRMWIntrinsic()
30690 LLVMContext &Ctx = AI->getContext(); in emitBitTestAtomicRMWIntrinsic()
30691 Value *Addr = Builder.CreatePointerCast(AI->getPointerOperand(), in emitBitTestAtomicRMWIntrinsic()
30695 auto BitTested = FindSingleBitChange(AI->getValOperand()); in emitBitTestAtomicRMWIntrinsic()
30699 auto *C = cast<ConstantInt>(I->getOperand(I->getOperand(0) == AI ? 1 : 0)); in emitBitTestAtomicRMWIntrinsic()
30701 BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_C, AI->getType()); in emitBitTestAtomicRMWIntrinsic()
30703 unsigned Imm = llvm::countr_zero(C->getZExtValue()); in emitBitTestAtomicRMWIntrinsic()
30706 BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_I, AI->getType()); in emitBitTestAtomicRMWIntrinsic()
30715 unsigned ShiftBits = SI->getType()->getPrimitiveSizeInBits(); in emitBitTestAtomicRMWIntrinsic()
30717 Builder.CreateAnd(SI, Builder.getIntN(ShiftBits, ShiftBits - 1)); in emitBitTestAtomicRMWIntrinsic()
30725 Result = Builder.CreateZExtOrTrunc(Result, AI->getType()); in emitBitTestAtomicRMWIntrinsic()
30727 // If the result is only used for zero/non-zero status then we don't need to in emitBitTestAtomicRMWIntrinsic()
30729 for (auto It = I->user_begin(); It != I->user_end(); ++It) { in emitBitTestAtomicRMWIntrinsic()
30731 if (ICmp->isEquality()) { in emitBitTestAtomicRMWIntrinsic()
30732 auto *C0 = dyn_cast<ConstantInt>(ICmp->getOperand(0)); in emitBitTestAtomicRMWIntrinsic()
30733 auto *C1 = dyn_cast<ConstantInt>(ICmp->getOperand(1)); in emitBitTestAtomicRMWIntrinsic()
30736 if ((C0 ? C0 : C1)->isZero()) in emitBitTestAtomicRMWIntrinsic()
30746 I->replaceAllUsesWith(Result); in emitBitTestAtomicRMWIntrinsic()
30747 I->eraseFromParent(); in emitBitTestAtomicRMWIntrinsic()
30748 AI->eraseFromParent(); in emitBitTestAtomicRMWIntrinsic()
30753 if (!AI->hasOneUse()) in shouldExpandCmpArithRMWInIR()
30756 Value *Op = AI->getOperand(1); in shouldExpandCmpArithRMWInIR()
30758 Instruction *I = AI->user_back(); in shouldExpandCmpArithRMWInIR()
30759 AtomicRMWInst::BinOp Opc = AI->getOperation(); in shouldExpandCmpArithRMWInIR()
30764 if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) in shouldExpandCmpArithRMWInIR()
30766 if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes()))) in shouldExpandCmpArithRMWInIR()
30775 if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) in shouldExpandCmpArithRMWInIR()
30777 if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes()))) in shouldExpandCmpArithRMWInIR()
30786 if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) in shouldExpandCmpArithRMWInIR()
30789 if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes()))) in shouldExpandCmpArithRMWInIR()
30797 if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt()))) in shouldExpandCmpArithRMWInIR()
30799 if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes()))) in shouldExpandCmpArithRMWInIR()
30813 LLVMContext &Ctx = AI->getContext(); in emitCmpArithAtomicRMWIntrinsic()
30814 ICmpInst *ICI = dyn_cast<ICmpInst>(AI->user_back()); in emitCmpArithAtomicRMWIntrinsic()
30816 TempI = AI->user_back(); in emitCmpArithAtomicRMWIntrinsic()
30817 assert(TempI->hasOneUse() && "Must have one use"); in emitCmpArithAtomicRMWIntrinsic()
30818 ICI = cast<ICmpInst>(TempI->user_back()); in emitCmpArithAtomicRMWIntrinsic()
30821 ICmpInst::Predicate Pred = ICI->getPredicate(); in emitCmpArithAtomicRMWIntrinsic()
30839 switch (AI->getOperation()) { in emitCmpArithAtomicRMWIntrinsic()
30859 Intrinsic::getDeclaration(AI->getModule(), IID, AI->getType()); in emitCmpArithAtomicRMWIntrinsic()
30860 Value *Addr = Builder.CreatePointerCast(AI->getPointerOperand(), in emitCmpArithAtomicRMWIntrinsic()
30863 CmpArith, {Addr, AI->getValOperand(), Builder.getInt32((unsigned)CC)}); in emitCmpArithAtomicRMWIntrinsic()
30865 ICI->replaceAllUsesWith(Result); in emitCmpArithAtomicRMWIntrinsic()
30866 ICI->eraseFromParent(); in emitCmpArithAtomicRMWIntrinsic()
30868 TempI->eraseFromParent(); in emitCmpArithAtomicRMWIntrinsic()
30869 AI->eraseFromParent(); in emitCmpArithAtomicRMWIntrinsic()
30875 Type *MemType = AI->getType(); in shouldExpandAtomicRMWInIR()
30879 if (MemType->getPrimitiveSizeInBits() > NativeWidth) { in shouldExpandAtomicRMWInIR()
30884 AtomicRMWInst::BinOp Op = AI->getOperation(); in shouldExpandAtomicRMWInIR()
30912 // These always require a non-trivial set of data operations on x86. We must in shouldExpandAtomicRMWInIR()
30921 Type *MemType = AI->getType(); in lowerIdempotentRMWIntoFencedLoad()
30925 if (MemType->getPrimitiveSizeInBits() > NativeWidth) in lowerIdempotentRMWIntoFencedLoad()
30931 if (auto *C = dyn_cast<ConstantInt>(AI->getValOperand())) in lowerIdempotentRMWIntoFencedLoad()
30932 if (AI->getOperation() == AtomicRMWInst::Or && C->isZero() && in lowerIdempotentRMWIntoFencedLoad()
30933 AI->use_empty()) in lowerIdempotentRMWIntoFencedLoad()
30938 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); in lowerIdempotentRMWIntoFencedLoad()
30939 auto SSID = AI->getSyncScopeID(); in lowerIdempotentRMWIntoFencedLoad()
30942 auto Order = AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering()); in lowerIdempotentRMWIntoFencedLoad()
30945 // http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf showing why a fence in lowerIdempotentRMWIntoFencedLoad()
30967 // different cache-line to prevent cache-line bouncing. In practice it in lowerIdempotentRMWIntoFencedLoad()
30978 AI->getType(), AI->getPointerOperand(), AI->getAlign()); in lowerIdempotentRMWIntoFencedLoad()
30979 Loaded->setAtomic(Order, SSID); in lowerIdempotentRMWIntoFencedLoad()
30980 AI->replaceAllUsesWith(Loaded); in lowerIdempotentRMWIntoFencedLoad()
30981 AI->eraseFromParent(); in lowerIdempotentRMWIntoFencedLoad()
30996 // See: Intel® 64 and IA-32 ArchitecturesSoftware Developer’s Manual, in emitLockedStackOp()
31007 // c) To minimize concerns about cross thread stack usage - in particular, in emitLockedStackOp()
31009 // captures state in the TOS frame and accesses it from many threads - in emitLockedStackOp()
31014 // https://shipilev.net/blog/2014/on-the-fence-with-dependencies/ in emitLockedStackOp()
31018 const unsigned SPOffset = TFL.has128ByteRedZone(MF) ? -64 : 0; in emitLockedStackOp()
31058 // The only fence that needs an instruction is a sequentially-consistent in LowerATOMIC_FENCE()
31059 // cross-thread fence. in LowerATOMIC_FENCE()
31069 // MEMBARRIER is a compiler barrier; it codegens to a no-op. in LowerATOMIC_FENCE()
31097 MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand(); in LowerCMP_SWAP()
31107 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), in LowerCMP_SWAP()
31149 assert(!Subtarget.is64Bit() && "Expected 32-bit mode"); in LowerBITCAST()
31161 assert(!Subtarget.hasAVX512() && "Should use K-registers with AVX512"); in LowerBITCAST()
31202 /// Compute the horizontal sum of bytes in V for the elements of VT.
31217 "Horizontal byte sum only makes sense for wider elements!"); in LowerHorizontalByteSum()
31241 // Do the horizontal sums into two v2i64s. in LowerHorizontalByteSum()
31282 // http://wm.ite.pl/articles/sse-popcount.html in LowerVectorCTPOPInRegLUT()
31285 // index into a in-register pre-computed pop count table. We then split up the in LowerVectorCTPOPInRegLUT()
31286 // input vector in two new ones: (1) a vector with only the shifted-right in LowerVectorCTPOPInRegLUT()
31289 // to index the in-register table. Next, both are added and the result is a in LowerVectorCTPOPInRegLUT()
31340 // Decompose 256-bit ops into smaller 128-bit ops. in LowerVectorCTPOP()
31344 // Decompose 512-bit ops into smaller 256-bit ops. in LowerVectorCTPOP()
31377 unsigned ActiveBits = Known.getBitWidth() - LZ; in LowerCTPOP()
31378 unsigned ShiftedActiveBits = Known.getBitWidth() - (LZ + TZ); in LowerCTPOP()
31380 // i2 CTPOP - "ctpop(x) --> sub(x, (x >> 1))". in LowerCTPOP()
31392 // i3 CTPOP - perform LUT into i32 integer. in LowerCTPOP()
31407 // i4 CTPOP - perform LUT into i64 integer. in LowerCTPOP()
31424 // i8 CTPOP - with efficient i32 MUL, then attempt multiply-mask-multiply. in LowerCTPOP()
31468 // Decompose 256-bit ops into smaller 128-bit ops. in LowerBITREVERSE_XOP()
31473 "Only 128-bit vector bitreverse lowering supported."); in LowerBITREVERSE_XOP()
31481 for (int j = ScalarSizeInBytes - 1; j >= 0; --j) { in LowerBITREVERSE_XOP()
31507 // Split 512-bit ops without BWI so that we can still use the PSHUFB lowering. in LowerBITREVERSE()
31511 // Decompose 256-bit ops into smaller 128-bit ops on pre-AVX2. in LowerBITREVERSE()
31553 // 0-15 value (moved to the other nibble). in LowerBITREVERSE()
31588 // Special case. If the input fits in 8-bits we can use a single 8-bit TEST. in LowerPARITY()
31605 // Xor the high and low 16-bits together using a 32-bit operation. in LowerPARITY()
31614 // Xor the high and low 16-bits together using a 32-bit operation. in LowerPARITY()
31619 // If the input is 16-bits, we need to extend to use an i32 shift below. in LowerPARITY()
31623 // Finally xor the low 2 bytes together and use a 8-bit flag setting xor. in LowerPARITY()
31624 // This should allow an h-reg to be used to save a shift. in LowerPARITY()
31641 switch (N->getOpcode()) { in lowerAtomicArithWithLOCK()
31661 MachineMemOperand *MMO = cast<MemSDNode>(N)->getMemOperand(); in lowerAtomicArithWithLOCK()
31665 {N->getOperand(0), N->getOperand(1), N->getOperand(2)}, in lowerAtomicArithWithLOCK()
31666 /*MemVT=*/N->getSimpleValueType(0), MMO); in lowerAtomicArithWithLOCK()
31669 /// Lower atomic_load_ops into LOCK-prefixed operations.
31673 SDValue Chain = N->getOperand(0); in lowerAtomicArith()
31674 SDValue LHS = N->getOperand(1); in lowerAtomicArith()
31675 SDValue RHS = N->getOperand(2); in lowerAtomicArith()
31676 unsigned Opc = N->getOpcode(); in lowerAtomicArith()
31677 MVT VT = N->getSimpleValueType(0); in lowerAtomicArith()
31683 if (N->hasAnyUseOfValue(0)) { in lowerAtomicArith()
31684 // Handle (atomic_load_sub p, v) as (atomic_load_add p, -v), to be able to in lowerAtomicArith()
31691 DAG.getNegative(RHS, DL, VT), AN->getMemOperand()); in lowerAtomicArith()
31705 if (Opc == ISD::ATOMIC_LOAD_OR && isNullConstant(RHS) && !AN->isVolatile()) { in lowerAtomicArith()
31711 if (AN->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent && in lowerAtomicArith()
31712 AN->getSyncScopeID() == SyncScope::System) { in lowerAtomicArith()
31717 assert(!N->hasAnyUseOfValue(0)); in lowerAtomicArith()
31719 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), in lowerAtomicArith()
31722 // MEMBARRIER is a compiler barrier; it codegens to a no-op. in lowerAtomicArith()
31724 assert(!N->hasAnyUseOfValue(0)); in lowerAtomicArith()
31726 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), in lowerAtomicArith()
31732 assert(!N->hasAnyUseOfValue(0)); in lowerAtomicArith()
31734 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), in lowerAtomicArith()
31742 EVT VT = Node->getMemoryVT(); in LowerATOMIC_STORE()
31745 Node->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent; in LowerATOMIC_STORE()
31760 SDValue VecVal = DAG.getBitcast(MVT::v2i64, Node->getVal()); in LowerATOMIC_STORE()
31761 Chain = DAG.getStore(Node->getChain(), dl, VecVal, Node->getBasePtr(), in LowerATOMIC_STORE()
31762 Node->getMemOperand()); in LowerATOMIC_STORE()
31770 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Node->getVal()); in LowerATOMIC_STORE()
31774 SDValue Ops[] = {Node->getChain(), SclToVec, Node->getBasePtr()}; in LowerATOMIC_STORE()
31776 MVT::i64, Node->getMemOperand()); in LowerATOMIC_STORE()
31778 // First load this into an 80-bit X87 register using a stack temporary. in LowerATOMIC_STORE()
31781 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in LowerATOMIC_STORE()
31784 Chain = DAG.getStore(Node->getChain(), dl, Node->getVal(), StackPtr, in LowerATOMIC_STORE()
31794 SDValue StoreOps[] = {Chain, Value, Node->getBasePtr()}; in LowerATOMIC_STORE()
31797 StoreOps, MVT::i64, Node->getMemOperand()); in LowerATOMIC_STORE()
31811 // Convert seq_cst store -> xchg in LowerATOMIC_STORE()
31812 // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b) in LowerATOMIC_STORE()
31813 // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment. in LowerATOMIC_STORE()
31814 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, Node->getMemoryVT(), in LowerATOMIC_STORE()
31815 Node->getOperand(0), Node->getOperand(2), in LowerATOMIC_STORE()
31816 Node->getOperand(1), Node->getMemOperand()); in LowerATOMIC_STORE()
31822 MVT VT = N->getSimpleValueType(0); in LowerADDSUBO_CARRY()
31846 if (N->getValueType(1) == MVT::i1) in LowerADDSUBO_CARRY()
31849 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC); in LowerADDSUBO_CARRY()
31947 for (unsigned i = 0; i < WidenNumElts - InNumElts; ++i) in ExtendToType()
31960 "MGATHER/MSCATTER are supported on AVX-512 arch only"); in LowerMSCATTER()
31963 SDValue Src = N->getValue(); in LowerMSCATTER()
31968 SDValue Scale = N->getScale(); in LowerMSCATTER()
31969 SDValue Index = N->getIndex(); in LowerMSCATTER()
31970 SDValue Mask = N->getMask(); in LowerMSCATTER()
31971 SDValue Chain = N->getChain(); in LowerMSCATTER()
31972 SDValue BasePtr = N->getBasePtr(); in LowerMSCATTER()
31984 N->getMemoryVT(), N->getMemOperand()); in LowerMSCATTER()
31996 // If we don't have VLX and neither the passthru or index is 512-bits, we in LowerMSCATTER()
32000 // Determine how much we need to widen by to get a 512-bit type. in LowerMSCATTER()
32017 N->getMemoryVT(), N->getMemOperand()); in LowerMSCATTER()
32026 SDValue Mask = N->getMask(); in LowerMLOAD()
32028 SDValue PassThru = N->getPassThru(); in LowerMLOAD()
32038 VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, in LowerMLOAD()
32039 getZeroVector(VT, Subtarget, DAG, dl), N->getMemoryVT(), in LowerMLOAD()
32040 N->getMemOperand(), N->getAddressingMode(), N->getExtensionType(), in LowerMLOAD()
32041 N->isExpandingLoad()); in LowerMLOAD()
32047 assert((!N->isExpandingLoad() || Subtarget.hasAVX512()) && in LowerMLOAD()
32048 "Expanding masked load is supported on AVX-512 target only!"); in LowerMLOAD()
32050 assert((!N->isExpandingLoad() || ScalarVT.getSizeInBits() >= 32) && in LowerMLOAD()
32051 "Expanding masked load is supported for 32 and 64-bit types only!"); in LowerMLOAD()
32075 WideDataVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, in LowerMLOAD()
32076 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(), in LowerMLOAD()
32077 N->getExtensionType(), N->isExpandingLoad()); in LowerMLOAD()
32089 SDValue DataToStore = N->getValue(); in LowerMSTORE()
32092 SDValue Mask = N->getMask(); in LowerMSTORE()
32095 assert((!N->isCompressingStore() || Subtarget.hasAVX512()) && in LowerMSTORE()
32096 "Expanding masked load is supported on AVX-512 target only!"); in LowerMSTORE()
32098 assert((!N->isCompressingStore() || ScalarVT.getSizeInBits() >= 32) && in LowerMSTORE()
32099 "Expanding masked load is supported for 32 and 64-bit types only!"); in LowerMSTORE()
32122 return DAG.getMaskedStore(N->getChain(), dl, DataToStore, N->getBasePtr(), in LowerMSTORE()
32123 N->getOffset(), Mask, N->getMemoryVT(), in LowerMSTORE()
32124 N->getMemOperand(), N->getAddressingMode(), in LowerMSTORE()
32125 N->isTruncatingStore(), N->isCompressingStore()); in LowerMSTORE()
32131 "MGATHER/MSCATTER are supported on AVX-512/AVX-2 arch only"); in LowerMGATHER()
32136 SDValue Index = N->getIndex(); in LowerMGATHER()
32137 SDValue Mask = N->getMask(); in LowerMGATHER()
32138 SDValue PassThru = N->getPassThru(); in LowerMGATHER()
32147 // If we don't have VLX and neither the passthru or index is 512-bits, we in LowerMGATHER()
32152 // Determine how much we need to widen by to get a 512-bit type. in LowerMGATHER()
32171 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index, in LowerMGATHER()
32172 N->getScale() }; in LowerMGATHER()
32174 X86ISD::MGATHER, dl, DAG.getVTList(VT, MVT::Other), Ops, N->getMemoryVT(), in LowerMGATHER()
32175 N->getMemOperand()); in LowerMGATHER()
32187 unsigned SrcAS = N->getSrcAddressSpace(); in LowerADDRSPACECAST()
32189 assert(SrcAS != N->getDestAddressSpace() && in LowerADDRSPACECAST()
32209 // no-ops in the case of a null GC strategy (or a GC strategy which does not in LowerGC_TRANSITION()
32214 if (Op->getGluedNode()) in LowerGC_TRANSITION()
32215 Ops.push_back(Op->getOperand(Op->getNumOperands() - 1)); in LowerGC_TRANSITION()
32239 // We don't support non-data prefetch without PREFETCHI. in LowerPREFETCH()
32259 // sub-string, e.g. "$12" contain "$1" in getInstrStrFromOpNo()
32261 I = AsmStr.size() - OpNoStr1.size(); in getInstrStrFromOpNo()
32316 // -> in visitMaskedLoad()
32337 // -> in visitMaskedStore()
32352 // clang-format off in LowerOperation()
32504 // clang-format on in LowerOperation()
32514 switch (N->getOpcode()) { in ReplaceNodeResults()
32518 N->dump(&DAG); in ReplaceNodeResults()
32522 EVT VT = N->getValueType(0); in ReplaceNodeResults()
32534 EVT VT = N->getValueType(0); in ReplaceNodeResults()
32540 {N->getOperand(0), Lo}); in ReplaceNodeResults()
32542 {N->getOperand(0), Hi}); in ReplaceNodeResults()
32554 assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!"); in ReplaceNodeResults()
32557 KnownBits Known = DAG.computeKnownBits(N->getOperand(0)); in ReplaceNodeResults()
32561 SDValue Op = DAG.getNode(ISD::SRL, dl, MVT::i64, N->getOperand(0), in ReplaceNodeResults()
32575 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, N->getOperand(0)); in ReplaceNodeResults()
32577 // Bit count should fit in 32-bits, extract it as that and then zero in ReplaceNodeResults()
32588 EVT VT = N->getValueType(0); in ReplaceNodeResults()
32591 // Pre-promote these to vXi16 to avoid op legalization thinking all 16 in ReplaceNodeResults()
32594 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0)); in ReplaceNodeResults()
32595 SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1)); in ReplaceNodeResults()
32607 EVT VT = N->getValueType(0); in ReplaceNodeResults()
32610 bool IsSigned = N->getOpcode() == ISD::SMULO; in ReplaceNodeResults()
32612 SDValue Op0 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(0)); in ReplaceNodeResults()
32613 SDValue Op1 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(1)); in ReplaceNodeResults()
32618 Hi = DAG.getVectorShuffle(MVT::v4i32, dl, Hi, Hi, {1, 3, -1, -1}); in ReplaceNodeResults()
32630 // UMULO overflows if the high bits are non-zero. in ReplaceNodeResults()
32633 SDValue Ovf = DAG.getSetCC(dl, N->getValueType(1), Hi, HiCmp, ISD::SETNE); in ReplaceNodeResults()
32646 EVT VT = N->getValueType(0); in ReplaceNodeResults()
32647 EVT InVT = N->getOperand(0).getValueType(); in ReplaceNodeResults()
32662 Ops[0] = N->getOperand(0); in ReplaceNodeResults()
32664 Ops[0] = N->getOperand(1); in ReplaceNodeResults()
32667 SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1); in ReplaceNodeResults()
32676 EVT VT = N->getValueType(0); in ReplaceNodeResults()
32680 N->getOperand(0), UNDEF); in ReplaceNodeResults()
32682 N->getOperand(1), UNDEF); in ReplaceNodeResults()
32683 Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS)); in ReplaceNodeResults()
32690 EVT VT = N->getValueType(0); in ReplaceNodeResults()
32696 // TODO: Can we do something for non-splat? in ReplaceNodeResults()
32698 if (ISD::isConstantSplatVector(N->getOperand(1).getNode(), SplatVal)) { in ReplaceNodeResults()
32701 Ops0[0] = N->getOperand(0); in ReplaceNodeResults()
32705 SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1); in ReplaceNodeResults()
32716 MVT VT = N->getSimpleValueType(0); in ReplaceNodeResults()
32724 SDValue In = N->getOperand(0); in ReplaceNodeResults()
32749 SmallVector<int, 16> TruncMask(WidenNumElts, -1); in ReplaceNodeResults()
32791 -1, -1, -1, -1, -1, -1, -1, -1 }); in ReplaceNodeResults()
32814 assert(N->getValueType(0) == MVT::v8i8 && in ReplaceNodeResults()
32819 EVT VT = N->getValueType(0); in ReplaceNodeResults()
32820 SDValue In = N->getOperand(0); in ReplaceNodeResults()
32826 assert(N->getOpcode() == ISD::SIGN_EXTEND && "Unexpected opcode"); in ReplaceNodeResults()
32827 // Custom split this so we can extend i8/i16->i32 invec. This is better in ReplaceNodeResults()
32828 // since sign_extend_inreg i8/i16->i64 requires an extend to i32 using in ReplaceNodeResults()
32863 In = DAG.getNode(N->getOpcode(), dl, InVT, In); in ReplaceNodeResults()
32869 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); in ReplaceNodeResults()
32872 SDValue Lo = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, LoVT, In, DAG); in ReplaceNodeResults()
32882 Hi = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, HiVT, Hi, DAG); in ReplaceNodeResults()
32893 bool IsStrict = N->isStrictFPOpcode(); in ReplaceNodeResults()
32894 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || in ReplaceNodeResults()
32895 N->getOpcode() == ISD::STRICT_FP_TO_SINT; in ReplaceNodeResults()
32896 EVT VT = N->getValueType(0); in ReplaceNodeResults()
32897 SDValue Src = N->getOperand(IsStrict ? 1 : 0); in ReplaceNodeResults()
32898 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); in ReplaceNodeResults()
32906 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other}, in ReplaceNodeResults()
32911 Res = DAG.getNode(N->getOpcode(), dl, VT, in ReplaceNodeResults()
32938 DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src}); in ReplaceNodeResults()
32977 {N->getOperand(0), Src}); in ReplaceNodeResults()
33037 // legalization to v8i32<-v8f64. in ReplaceNodeResults()
33044 Opc = N->getOpcode(); in ReplaceNodeResults()
33050 {N->getOperand(0), Src}); in ReplaceNodeResults()
33061 // Custom widen strict v2f32->v2i32 by padding with zeros. in ReplaceNodeResults()
33066 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other}, in ReplaceNodeResults()
33067 {N->getOperand(0), Src}); in ReplaceNodeResults()
33085 // If we use a 128-bit result we might need to use a target specific node. in ReplaceNodeResults()
33090 unsigned Opc = N->getOpcode(); in ReplaceNodeResults()
33105 Res = DAG.getNode(Opc, SDLoc(N), Tys, N->getOperand(0), Res); in ReplaceNodeResults()
33143 bool IsStrict = N->isStrictFPOpcode(); in ReplaceNodeResults()
33144 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || in ReplaceNodeResults()
33145 N->getOpcode() == ISD::STRICT_SINT_TO_FP; in ReplaceNodeResults()
33146 EVT VT = N->getValueType(0); in ReplaceNodeResults()
33147 SDValue Src = N->getOperand(IsStrict ? 1 : 0); in ReplaceNodeResults()
33161 {N->getOperand(0), Src}); in ReplaceNodeResults()
33178 {N->getOperand(0), Src}); in ReplaceNodeResults()
33203 {N->getOperand(0), Elt}); in ReplaceNodeResults()
33220 DAG.getVectorShuffle(MVT::v4i32, dl, IsNeg, IsNeg, {1, 3, -1, -1}); in ReplaceNodeResults()
33235 // Custom widen strict v2i32->v2f32 to avoid scalarization. in ReplaceNodeResults()
33239 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other}, in ReplaceNodeResults()
33240 {N->getOperand(0), Src}); in ReplaceNodeResults()
33255 {N->getOperand(0), Or, VBias}); in ReplaceNodeResults()
33262 // TODO: Are there any fast-math-flags to propagate here? in ReplaceNodeResults()
33270 bool IsStrict = N->isStrictFPOpcode(); in ReplaceNodeResults()
33271 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); in ReplaceNodeResults()
33272 SDValue Src = N->getOperand(IsStrict ? 1 : 0); in ReplaceNodeResults()
33273 SDValue Rnd = N->getOperand(IsStrict ? 2 : 1); in ReplaceNodeResults()
33275 EVT VT = N->getValueType(0); in ReplaceNodeResults()
33315 assert(N->getValueType(0) == MVT::v2f32 && in ReplaceNodeResults()
33319 bool IsStrict = N->isStrictFPOpcode(); in ReplaceNodeResults()
33320 SDValue Src = N->getOperand(IsStrict ? 1 : 0); in ReplaceNodeResults()
33328 {N->getOperand(0), V}); in ReplaceNodeResults()
33337 unsigned IntNo = N->getConstantOperandVal(1); in ReplaceNodeResults()
33365 EVT T = N->getValueType(0); in ReplaceNodeResults()
33369 "64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS requires CMPXCHG16B"); in ReplaceNodeResults()
33373 DAG.SplitScalar(N->getOperand(2), dl, HalfT, HalfT); in ReplaceNodeResults()
33374 cpInL = DAG.getCopyToReg(N->getOperand(0), dl, in ReplaceNodeResults()
33381 DAG.SplitScalar(N->getOperand(3), dl, HalfT, HalfT); in ReplaceNodeResults()
33386 // In 64-bit mode we might need the base pointer in RBX, but we can't know in ReplaceNodeResults()
33391 // live-range. in ReplaceNodeResults()
33394 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand(); in ReplaceNodeResults()
33396 SDValue Ops[] = {swapInH.getValue(0), N->getOperand(1), swapInL, in ReplaceNodeResults()
33403 SDValue Ops[] = {swapInL.getValue(0), N->getOperand(1), in ReplaceNodeResults()
33420 Success = DAG.getZExtOrTrunc(Success, dl, N->getValueType(1)); in ReplaceNodeResults()
33429 (N->getValueType(0) == MVT::i64 || N->getValueType(0) == MVT::i128) && in ReplaceNodeResults()
33437 if (N->getValueType(0) == MVT::i128) { in ReplaceNodeResults()
33439 SDValue Ld = DAG.getLoad(MVT::v2i64, dl, Node->getChain(), in ReplaceNodeResults()
33440 Node->getBasePtr(), Node->getMemOperand()); in ReplaceNodeResults()
33445 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, N->getValueType(0), in ReplaceNodeResults()
33454 // Then extract the lower 64-bits. in ReplaceNodeResults()
33457 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() }; in ReplaceNodeResults()
33459 MVT::i64, Node->getMemOperand()); in ReplaceNodeResults()
33468 // then casts to i64. This avoids a 128-bit stack temporary being in ReplaceNodeResults()
33469 // created by type legalization if we were to cast v4f32->v2i64. in ReplaceNodeResults()
33478 // First load this into an 80-bit X87 register. This will put the whole in ReplaceNodeResults()
33481 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() }; in ReplaceNodeResults()
33484 Node->getMemOperand()); in ReplaceNodeResults()
33492 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); in ReplaceNodeResults()
33531 EVT DstVT = N->getValueType(0); in ReplaceNodeResults()
33532 EVT SrcVT = N->getOperand(0).getValueType(); in ReplaceNodeResults()
33534 // If this is a bitcast from a v64i1 k-register to a i64 on a 32-bit target in ReplaceNodeResults()
33535 // we can split using the k-register rather than memory. in ReplaceNodeResults()
33537 assert(!Subtarget.is64Bit() && "Expected 32-bit mode"); in ReplaceNodeResults()
33554 N->getOperand(0)); in ReplaceNodeResults()
33563 EVT VT = N->getValueType(0); in ReplaceNodeResults()
33567 SDValue Index = Gather->getIndex(); in ReplaceNodeResults()
33573 SDValue Mask = Gather->getMask(); in ReplaceNodeResults()
33576 Gather->getPassThru(), in ReplaceNodeResults()
33585 SDValue Ops[] = { Gather->getChain(), PassThru, Mask, in ReplaceNodeResults()
33586 Gather->getBasePtr(), Index, Gather->getScale() }; in ReplaceNodeResults()
33589 Gather->getMemoryVT(), Gather->getMemOperand()); in ReplaceNodeResults()
33598 // avoids scalarizing in 32-bit mode. In 64-bit mode this avoids a int->fp in ReplaceNodeResults()
33600 MVT VT = N->getSimpleValueType(0); in ReplaceNodeResults()
33609 SDValue Res = DAG.getLoad(LdVT, dl, Ld->getChain(), Ld->getBasePtr(), in ReplaceNodeResults()
33610 Ld->getPointerInfo(), Ld->getOriginalAlign(), in ReplaceNodeResults()
33611 Ld->getMemOperand()->getFlags()); in ReplaceNodeResults()
33623 SDValue Ops[] = {Ld->getChain(), Ld->getBasePtr()}; in ReplaceNodeResults()
33625 MVT::i64, Ld->getMemOperand()); in ReplaceNodeResults()
33636 assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!"); in ReplaceNodeResults()
33645 assert(N->getSimpleValueType(0) == MVT::f16 && in ReplaceNodeResults()
33648 SDValue VecOp = N->getOperand(0); in ReplaceNodeResults()
33650 SDValue Split = DAG.getBitcast(ExtVT, N->getOperand(0)); in ReplaceNodeResults()
33652 N->getOperand(1)); in ReplaceNodeResults()
34081 // X86 allows a sign-extended 32-bit immediate field as a displacement. in isLegalAddressingMode()
34097 // If lower 4G is not available, then we must use rip-relative addressing. in isLegalAddressingMode()
34127 unsigned Bits = Ty->getScalarSizeInBits(); in isVectorShiftByScalarCheap()
34151 // These are non-commutative binops. in isBinOp()
34187 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) in isTruncateFree()
34189 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); in isTruncateFree()
34190 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); in isTruncateFree()
34195 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) in allowTruncateForTailCall()
34201 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop"); in allowTruncateForTailCall()
34230 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers. in isZExtFree()
34231 return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget.is64Bit(); in isZExtFree()
34235 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers. in isZExtFree()
34256 // X86 has 8, 16, and 32-bit zero-extending loads. in isZExtFree()
34267 FixedVectorType *VTy = dyn_cast<FixedVectorType>(I->getType()); in shouldSinkOperands()
34271 if (I->getOpcode() == Instruction::Mul && in shouldSinkOperands()
34272 VTy->getElementType()->isIntegerTy(64)) { in shouldSinkOperands()
34273 for (auto &Op : I->operands()) { in shouldSinkOperands()
34275 if (any_of(Ops, [&](Use *U) { return U->get() == Op; })) in shouldSinkOperands()
34283 Ops.push_back(&cast<Instruction>(Op)->getOperandUse(0)); in shouldSinkOperands()
34298 int ShiftAmountOpNum = -1; in shouldSinkOperands()
34299 if (I->isShift()) in shouldSinkOperands()
34302 if (II->getIntrinsicID() == Intrinsic::fshl || in shouldSinkOperands()
34303 II->getIntrinsicID() == Intrinsic::fshr) in shouldSinkOperands()
34307 if (ShiftAmountOpNum == -1) in shouldSinkOperands()
34310 auto *Shuf = dyn_cast<ShuffleVectorInst>(I->getOperand(ShiftAmountOpNum)); in shouldSinkOperands()
34311 if (Shuf && getSplatIndex(Shuf->getShuffleMask()) >= 0 && in shouldSinkOperands()
34312 isVectorShiftByScalarCheap(I->getType())) { in shouldSinkOperands()
34313 Ops.push_back(&I->getOperandUse(ShiftAmountOpNum)); in shouldSinkOperands()
34369 // TODO: This is too general. There are cases where pre-AVX512 codegen would in shouldFoldSelectWithIdentityConstant()
34393 // Very little shuffling can be done for 64-bit vectors right now. in isShuffleMaskLegal()
34405 // vpblendw and vpshufb for 256-bit vectors are not available on AVX1. in isVectorClearMaskLegal()
34426 // zero-extensions. in getPreferredSwitchConditionType()
34433 //===----------------------------------------------------------------------===//
34435 //===----------------------------------------------------------------------===//
34442 for (const MachineInstr &mi : llvm::make_range(std::next(Itr), BB->end())) { in isEFLAGSLiveAfter()
34452 for (MachineBasicBlock *Succ : BB->successors()) in isEFLAGSLiveAfter()
34453 if (Succ->isLiveIn(X86::EFLAGS)) in isEFLAGSLiveAfter()
34464 const BasicBlock *BB = MBB->getBasicBlock(); in emitXBegin()
34465 MachineFunction::iterator I = ++MBB->getIterator(); in emitXBegin()
34473 // s0 = -1 in emitXBegin()
34483 MachineFunction *MF = MBB->getParent(); in emitXBegin()
34484 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB); in emitXBegin()
34485 MachineBasicBlock *fallMBB = MF->CreateMachineBasicBlock(BB); in emitXBegin()
34486 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB); in emitXBegin()
34487 MF->insert(I, mainMBB); in emitXBegin()
34488 MF->insert(I, fallMBB); in emitXBegin()
34489 MF->insert(I, sinkMBB); in emitXBegin()
34492 mainMBB->addLiveIn(X86::EFLAGS); in emitXBegin()
34493 fallMBB->addLiveIn(X86::EFLAGS); in emitXBegin()
34494 sinkMBB->addLiveIn(X86::EFLAGS); in emitXBegin()
34498 sinkMBB->splice(sinkMBB->begin(), MBB, in emitXBegin()
34499 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); in emitXBegin()
34500 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); in emitXBegin()
34502 MachineRegisterInfo &MRI = MF->getRegInfo(); in emitXBegin()
34512 BuildMI(thisMBB, MIMD, TII->get(X86::XBEGIN_4)).addMBB(fallMBB); in emitXBegin()
34513 thisMBB->addSuccessor(mainMBB); in emitXBegin()
34514 thisMBB->addSuccessor(fallMBB); in emitXBegin()
34517 // mainDstReg := -1 in emitXBegin()
34518 BuildMI(mainMBB, MIMD, TII->get(X86::MOV32ri), mainDstReg).addImm(-1); in emitXBegin()
34519 BuildMI(mainMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB); in emitXBegin()
34520 mainMBB->addSuccessor(sinkMBB); in emitXBegin()
34526 BuildMI(fallMBB, MIMD, TII->get(X86::XABORT_DEF)); in emitXBegin()
34527 BuildMI(fallMBB, MIMD, TII->get(TargetOpcode::COPY), fallDstReg) in emitXBegin()
34529 fallMBB->addSuccessor(sinkMBB); in emitXBegin()
34533 BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg) in emitXBegin()
34544 // Emit va_arg instruction on X86-64. in EmitVAARGWithCustomInserter()
34546 // Operands to this pseudo-instruction: in EmitVAARGWithCustomInserter()
34548 // 1-5) Input : va_list address (addr, i64mem) in EmitVAARGWithCustomInserter()
34552 // 9 ) EFLAGS (implicit-def) in EmitVAARGWithCustomInserter()
34567 MachineFunction *MF = MBB->getParent(); in EmitVAARGWithCustomInserter()
34575 MachineMemOperand *LoadOnlyMMO = MF->getMachineMemOperand( in EmitVAARGWithCustomInserter()
34576 OldMMO, OldMMO->getFlags() & ~MachineMemOperand::MOStore); in EmitVAARGWithCustomInserter()
34577 MachineMemOperand *StoreOnlyMMO = MF->getMachineMemOperand( in EmitVAARGWithCustomInserter()
34578 OldMMO, OldMMO->getFlags() & ~MachineMemOperand::MOLoad); in EmitVAARGWithCustomInserter()
34582 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); in EmitVAARGWithCustomInserter()
34584 getRegClassFor(getPointerTy(MBB->getParent()->getDataLayout())); in EmitVAARGWithCustomInserter()
34643 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); in EmitVAARGWithCustomInserter()
34644 overflowMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitVAARGWithCustomInserter()
34645 offsetMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitVAARGWithCustomInserter()
34646 endMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitVAARGWithCustomInserter()
34648 MachineFunction::iterator MBBIter = ++MBB->getIterator(); in EmitVAARGWithCustomInserter()
34651 MF->insert(MBBIter, offsetMBB); in EmitVAARGWithCustomInserter()
34652 MF->insert(MBBIter, overflowMBB); in EmitVAARGWithCustomInserter()
34653 MF->insert(MBBIter, endMBB); in EmitVAARGWithCustomInserter()
34656 endMBB->splice(endMBB->begin(), thisMBB, in EmitVAARGWithCustomInserter()
34657 std::next(MachineBasicBlock::iterator(MI)), thisMBB->end()); in EmitVAARGWithCustomInserter()
34658 endMBB->transferSuccessorsAndUpdatePHIs(thisMBB); in EmitVAARGWithCustomInserter()
34661 thisMBB->addSuccessor(offsetMBB); in EmitVAARGWithCustomInserter()
34662 thisMBB->addSuccessor(overflowMBB); in EmitVAARGWithCustomInserter()
34665 offsetMBB->addSuccessor(endMBB); in EmitVAARGWithCustomInserter()
34666 overflowMBB->addSuccessor(endMBB); in EmitVAARGWithCustomInserter()
34670 BuildMI(thisMBB, MIMD, TII->get(X86::MOV32rm), OffsetReg) in EmitVAARGWithCustomInserter()
34679 BuildMI(thisMBB, MIMD, TII->get(X86::CMP32ri)) in EmitVAARGWithCustomInserter()
34681 .addImm(MaxOffset + 8 - ArgSizeA8); in EmitVAARGWithCustomInserter()
34685 BuildMI(thisMBB, MIMD, TII->get(X86::JCC_1)) in EmitVAARGWithCustomInserter()
34697 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm), in EmitVAARGWithCustomInserter()
34707 // Zero-extend the offset in EmitVAARGWithCustomInserter()
34709 BuildMI(offsetMBB, MIMD, TII->get(X86::SUBREG_TO_REG), OffsetReg64) in EmitVAARGWithCustomInserter()
34715 BuildMI(offsetMBB, MIMD, TII->get(X86::ADD64rr), OffsetDestReg) in EmitVAARGWithCustomInserter()
34720 BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32rr), OffsetDestReg) in EmitVAARGWithCustomInserter()
34727 BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32ri), NextOffsetReg) in EmitVAARGWithCustomInserter()
34732 BuildMI(offsetMBB, MIMD, TII->get(X86::MOV32mr)) in EmitVAARGWithCustomInserter()
34742 BuildMI(offsetMBB, MIMD, TII->get(X86::JMP_1)) in EmitVAARGWithCustomInserter()
34753 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm), in EmitVAARGWithCustomInserter()
34768 // aligned_addr = (addr + (align-1)) & ~(align-1) in EmitVAARGWithCustomInserter()
34771 TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri), in EmitVAARGWithCustomInserter()
34774 .addImm(Alignment.value() - 1); in EmitVAARGWithCustomInserter()
34778 TII->get(Subtarget.isTarget64BitLP64() ? X86::AND64ri32 : X86::AND32ri), in EmitVAARGWithCustomInserter()
34781 .addImm(~(uint64_t)(Alignment.value() - 1)); in EmitVAARGWithCustomInserter()
34783 BuildMI(overflowMBB, MIMD, TII->get(TargetOpcode::COPY), OverflowDestReg) in EmitVAARGWithCustomInserter()
34788 // (the overflow address should be kept 8-byte aligned) in EmitVAARGWithCustomInserter()
34792 TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri), in EmitVAARGWithCustomInserter()
34799 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64mr : X86::MOV32mr)) in EmitVAARGWithCustomInserter()
34810 BuildMI(*endMBB, endMBB->begin(), MIMD, in EmitVAARGWithCustomInserter()
34811 TII->get(X86::PHI), DestReg) in EmitVAARGWithCustomInserter()
34835 SelectItr->addRegisterKilled(X86::EFLAGS, TRI); in checkAndUpdateEFLAGSKill()
34839 // Return true if it is OK for this CMOV pseudo-opcode to be cascaded
34840 // together with other CMOV pseudo-opcodes into a single basic-block with
34885 MachineFunction *MF = TrueMBB->getParent(); in createPHIsForCMOVsInSinkBB()
34886 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); in createPHIsForCMOVsInSinkBB()
34889 X86::CondCode CC = X86::CondCode(MIItBegin->getOperand(3).getImm()); in createPHIsForCMOVsInSinkBB()
34892 MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin(); in createPHIsForCMOVsInSinkBB()
34904 Register DestReg = MIIt->getOperand(0).getReg(); in createPHIsForCMOVsInSinkBB()
34905 Register Op1Reg = MIIt->getOperand(1).getReg(); in createPHIsForCMOVsInSinkBB()
34906 Register Op2Reg = MIIt->getOperand(2).getReg(); in createPHIsForCMOVsInSinkBB()
34911 if (MIIt->getOperand(3).getImm() == OppCC) in createPHIsForCMOVsInSinkBB()
34921 BuildMI(*SinkMBB, SinkInsertionPoint, MIMD, TII->get(X86::PHI), DestReg) in createPHIsForCMOVsInSinkBB()
34966 // because this custom-inserter would have generated: in EmitLoweredCascadedSelect()
35013 const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock(); in EmitLoweredCascadedSelect()
35014 MachineFunction *F = ThisMBB->getParent(); in EmitLoweredCascadedSelect()
35015 MachineBasicBlock *FirstInsertedMBB = F->CreateMachineBasicBlock(LLVM_BB); in EmitLoweredCascadedSelect()
35016 MachineBasicBlock *SecondInsertedMBB = F->CreateMachineBasicBlock(LLVM_BB); in EmitLoweredCascadedSelect()
35017 MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(LLVM_BB); in EmitLoweredCascadedSelect()
35019 MachineFunction::iterator It = ++ThisMBB->getIterator(); in EmitLoweredCascadedSelect()
35020 F->insert(It, FirstInsertedMBB); in EmitLoweredCascadedSelect()
35021 F->insert(It, SecondInsertedMBB); in EmitLoweredCascadedSelect()
35022 F->insert(It, SinkMBB); in EmitLoweredCascadedSelect()
35027 FirstInsertedMBB->addLiveIn(X86::EFLAGS); in EmitLoweredCascadedSelect()
35034 SecondInsertedMBB->addLiveIn(X86::EFLAGS); in EmitLoweredCascadedSelect()
35035 SinkMBB->addLiveIn(X86::EFLAGS); in EmitLoweredCascadedSelect()
35039 SinkMBB->splice(SinkMBB->begin(), ThisMBB, in EmitLoweredCascadedSelect()
35041 ThisMBB->end()); in EmitLoweredCascadedSelect()
35042 SinkMBB->transferSuccessorsAndUpdatePHIs(ThisMBB); in EmitLoweredCascadedSelect()
35045 ThisMBB->addSuccessor(FirstInsertedMBB); in EmitLoweredCascadedSelect()
35047 ThisMBB->addSuccessor(SinkMBB); in EmitLoweredCascadedSelect()
35049 FirstInsertedMBB->addSuccessor(SecondInsertedMBB); in EmitLoweredCascadedSelect()
35051 FirstInsertedMBB->addSuccessor(SinkMBB); in EmitLoweredCascadedSelect()
35053 SecondInsertedMBB->addSuccessor(SinkMBB); in EmitLoweredCascadedSelect()
35057 BuildMI(ThisMBB, MIMD, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(FirstCC); in EmitLoweredCascadedSelect()
35061 BuildMI(FirstInsertedMBB, MIMD, TII->get(X86::JCC_1)) in EmitLoweredCascadedSelect()
35071 BuildMI(*SinkMBB, SinkMBB->begin(), MIMD, TII->get(X86::PHI), DestReg) in EmitLoweredCascadedSelect()
35095 // diamond control-flow pattern. The incoming instruction knows the in EmitLoweredSelect()
35104 // fallthrough --> FalseMBB in EmitLoweredSelect()
35106 // This code lowers all pseudo-CMOV instructions. Generally it lowers these in EmitLoweredSelect()
35139 // function - EmitLoweredCascadedSelect. in EmitLoweredSelect()
35153 while (NextMIIt != ThisMBB->end() && isCMOVPseudo(*NextMIIt) && in EmitLoweredSelect()
35154 (NextMIIt->getOperand(3).getImm() == CC || in EmitLoweredSelect()
35155 NextMIIt->getOperand(3).getImm() == OppCC)) { in EmitLoweredSelect()
35157 NextMIIt = next_nodbg(NextMIIt, ThisMBB->end()); in EmitLoweredSelect()
35163 if (LastCMOV == &MI && NextMIIt != ThisMBB->end() && in EmitLoweredSelect()
35164 NextMIIt->getOpcode() == MI.getOpcode() && in EmitLoweredSelect()
35165 NextMIIt->getOperand(2).getReg() == MI.getOperand(2).getReg() && in EmitLoweredSelect()
35166 NextMIIt->getOperand(1).getReg() == MI.getOperand(0).getReg() && in EmitLoweredSelect()
35167 NextMIIt->getOperand(1).isKill()) { in EmitLoweredSelect()
35171 const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock(); in EmitLoweredSelect()
35172 MachineFunction *F = ThisMBB->getParent(); in EmitLoweredSelect()
35173 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVM_BB); in EmitLoweredSelect()
35174 MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(LLVM_BB); in EmitLoweredSelect()
35176 MachineFunction::iterator It = ++ThisMBB->getIterator(); in EmitLoweredSelect()
35177 F->insert(It, FalseMBB); in EmitLoweredSelect()
35178 F->insert(It, SinkMBB); in EmitLoweredSelect()
35181 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI); in EmitLoweredSelect()
35182 FalseMBB->setCallFrameSize(CallFrameSize); in EmitLoweredSelect()
35183 SinkMBB->setCallFrameSize(CallFrameSize); in EmitLoweredSelect()
35188 if (!LastCMOV->killsRegister(X86::EFLAGS, /*TRI=*/nullptr) && in EmitLoweredSelect()
35190 FalseMBB->addLiveIn(X86::EFLAGS); in EmitLoweredSelect()
35191 SinkMBB->addLiveIn(X86::EFLAGS); in EmitLoweredSelect()
35199 SinkMBB->push_back(MI.removeFromParent()); in EmitLoweredSelect()
35202 SinkMBB->splice(SinkMBB->end(), ThisMBB, in EmitLoweredSelect()
35204 ThisMBB->end()); in EmitLoweredSelect()
35205 SinkMBB->transferSuccessorsAndUpdatePHIs(ThisMBB); in EmitLoweredSelect()
35208 ThisMBB->addSuccessor(FalseMBB); in EmitLoweredSelect()
35210 ThisMBB->addSuccessor(SinkMBB); in EmitLoweredSelect()
35212 FalseMBB->addSuccessor(SinkMBB); in EmitLoweredSelect()
35215 BuildMI(ThisMBB, MIMD, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(CC); in EmitLoweredSelect()
35226 ThisMBB->erase(MIItBegin, MIItEnd); in EmitLoweredSelect()
35241 MachineFunction *MF = MBB->getParent(); in EmitLoweredProbedAlloca()
35245 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); in EmitLoweredProbedAlloca()
35249 MachineRegisterInfo &MRI = MF->getRegInfo(); in EmitLoweredProbedAlloca()
35250 MachineBasicBlock *testMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitLoweredProbedAlloca()
35251 MachineBasicBlock *tailMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitLoweredProbedAlloca()
35252 MachineBasicBlock *blockMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitLoweredProbedAlloca()
35254 MachineFunction::iterator MBBIter = ++MBB->getIterator(); in EmitLoweredProbedAlloca()
35255 MF->insert(MBBIter, testMBB); in EmitLoweredProbedAlloca()
35256 MF->insert(MBBIter, blockMBB); in EmitLoweredProbedAlloca()
35257 MF->insert(MBBIter, tailMBB); in EmitLoweredProbedAlloca()
35268 BuildMI(*MBB, {MI}, MIMD, TII->get(TargetOpcode::COPY), TmpStackPtr) in EmitLoweredProbedAlloca()
35272 BuildMI(*MBB, {MI}, MIMD, TII->get(Opc), FinalStackPtr) in EmitLoweredProbedAlloca()
35280 TII->get(TFI.Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr)) in EmitLoweredProbedAlloca()
35284 BuildMI(testMBB, MIMD, TII->get(X86::JCC_1)) in EmitLoweredProbedAlloca()
35287 testMBB->addSuccessor(blockMBB); in EmitLoweredProbedAlloca()
35288 testMBB->addSuccessor(tailMBB); in EmitLoweredProbedAlloca()
35294 // + ---- <- ------------ <- ------------- <- ------------ + in EmitLoweredProbedAlloca()
35296 …// [free probe] -> [page alloc] -> [alloc probe] -> [tail alloc] + -> [dyn probe] -> [page alloc] … in EmitLoweredProbedAlloca()
35298 … + <- ----------- <- ------------ <- ----------- <- ----… in EmitLoweredProbedAlloca()
35304 addRegOffset(BuildMI(blockMBB, MIMD, TII->get(XORMIOpc)), physSPReg, false, 0) in EmitLoweredProbedAlloca()
35307 BuildMI(blockMBB, MIMD, TII->get(getSUBriOpcode(TFI.Uses64BitFramePtr)), in EmitLoweredProbedAlloca()
35312 BuildMI(blockMBB, MIMD, TII->get(X86::JMP_1)).addMBB(testMBB); in EmitLoweredProbedAlloca()
35313 blockMBB->addSuccessor(testMBB); in EmitLoweredProbedAlloca()
35316 BuildMI(tailMBB, MIMD, TII->get(TargetOpcode::COPY), in EmitLoweredProbedAlloca()
35320 tailMBB->splice(tailMBB->end(), MBB, in EmitLoweredProbedAlloca()
35321 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); in EmitLoweredProbedAlloca()
35322 tailMBB->transferSuccessorsAndUpdatePHIs(MBB); in EmitLoweredProbedAlloca()
35323 MBB->addSuccessor(testMBB); in EmitLoweredProbedAlloca()
35335 MachineFunction *MF = BB->getParent(); in EmitLoweredSegAlloca()
35338 const BasicBlock *LLVM_BB = BB->getBasicBlock(); in EmitLoweredSegAlloca()
35340 assert(MF->shouldSplitStack()); in EmitLoweredSegAlloca()
35364 MachineBasicBlock *mallocMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitLoweredSegAlloca()
35365 MachineBasicBlock *bumpMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitLoweredSegAlloca()
35366 MachineBasicBlock *continueMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitLoweredSegAlloca()
35368 MachineRegisterInfo &MRI = MF->getRegInfo(); in EmitLoweredSegAlloca()
35370 getRegClassFor(getPointerTy(MF->getDataLayout())); in EmitLoweredSegAlloca()
35380 MachineFunction::iterator MBBIter = ++BB->getIterator(); in EmitLoweredSegAlloca()
35382 MF->insert(MBBIter, bumpMBB); in EmitLoweredSegAlloca()
35383 MF->insert(MBBIter, mallocMBB); in EmitLoweredSegAlloca()
35384 MF->insert(MBBIter, continueMBB); in EmitLoweredSegAlloca()
35386 continueMBB->splice(continueMBB->begin(), BB, in EmitLoweredSegAlloca()
35387 std::next(MachineBasicBlock::iterator(MI)), BB->end()); in EmitLoweredSegAlloca()
35388 continueMBB->transferSuccessorsAndUpdatePHIs(BB); in EmitLoweredSegAlloca()
35392 BuildMI(BB, MIMD, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg); in EmitLoweredSegAlloca()
35393 BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg) in EmitLoweredSegAlloca()
35395 BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::CMP64mr:X86::CMP32mr)) in EmitLoweredSegAlloca()
35398 BuildMI(BB, MIMD, TII->get(X86::JCC_1)).addMBB(mallocMBB).addImm(X86::COND_G); in EmitLoweredSegAlloca()
35402 BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), physSPReg) in EmitLoweredSegAlloca()
35404 BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), bumpSPPtrVReg) in EmitLoweredSegAlloca()
35406 BuildMI(bumpMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB); in EmitLoweredSegAlloca()
35410 Subtarget.getRegisterInfo()->getCallPreservedMask(*MF, CallingConv::C); in EmitLoweredSegAlloca()
35412 BuildMI(mallocMBB, MIMD, TII->get(X86::MOV64rr), X86::RDI) in EmitLoweredSegAlloca()
35414 BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32)) in EmitLoweredSegAlloca()
35420 BuildMI(mallocMBB, MIMD, TII->get(X86::MOV32rr), X86::EDI) in EmitLoweredSegAlloca()
35422 BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32)) in EmitLoweredSegAlloca()
35428 BuildMI(mallocMBB, MIMD, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg) in EmitLoweredSegAlloca()
35430 BuildMI(mallocMBB, MIMD, TII->get(X86::PUSH32r)).addReg(sizeVReg); in EmitLoweredSegAlloca()
35431 BuildMI(mallocMBB, MIMD, TII->get(X86::CALLpcrel32)) in EmitLoweredSegAlloca()
35438 BuildMI(mallocMBB, MIMD, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg) in EmitLoweredSegAlloca()
35441 BuildMI(mallocMBB, MIMD, TII->get(TargetOpcode::COPY), mallocPtrVReg) in EmitLoweredSegAlloca()
35443 BuildMI(mallocMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB); in EmitLoweredSegAlloca()
35446 BB->addSuccessor(bumpMBB); in EmitLoweredSegAlloca()
35447 BB->addSuccessor(mallocMBB); in EmitLoweredSegAlloca()
35448 mallocMBB->addSuccessor(continueMBB); in EmitLoweredSegAlloca()
35449 bumpMBB->addSuccessor(continueMBB); in EmitLoweredSegAlloca()
35452 BuildMI(*continueMBB, continueMBB->begin(), MIMD, TII->get(X86::PHI), in EmitLoweredSegAlloca()
35469 MachineFunction *MF = BB->getParent(); in EmitLoweredCatchRet()
35475 classifyEHPersonality(MF->getFunction().getPersonalityFn())) && in EmitLoweredCatchRet()
35478 // Only 32-bit EH needs to worry about manually restoring stack pointers. in EmitLoweredCatchRet()
35485 MF->CreateMachineBasicBlock(BB->getBasicBlock()); in EmitLoweredCatchRet()
35486 assert(BB->succ_size() == 1); in EmitLoweredCatchRet()
35487 MF->insert(std::next(BB->getIterator()), RestoreMBB); in EmitLoweredCatchRet()
35488 RestoreMBB->transferSuccessorsAndUpdatePHIs(BB); in EmitLoweredCatchRet()
35489 BB->addSuccessor(RestoreMBB); in EmitLoweredCatchRet()
35494 RestoreMBB->setIsEHPad(true); in EmitLoweredCatchRet()
35496 auto RestoreMBBI = RestoreMBB->begin(); in EmitLoweredCatchRet()
35505 // adjust_stackdown -> TLSADDR -> adjust_stackup. in EmitLoweredTLSAddr()
35507 // inside MC, therefore without the two markers shrink-wrapping in EmitLoweredTLSAddr()
35511 MachineFunction &MF = *BB->getParent(); in EmitLoweredTLSAddr()
35518 BB->insert(MachineBasicBlock::iterator(MI), CallseqStart); in EmitLoweredTLSAddr()
35526 BB->insertAfter(MachineBasicBlock::iterator(MI), CallseqEnd); in EmitLoweredTLSAddr()
35535 // our load from the relocation, sticking it in either RDI (x86-64) in EmitLoweredTLSCall()
35538 MachineFunction *F = BB->getParent(); in EmitLoweredTLSCall()
35546 // FIXME: The 32-bit calls have non-standard calling conventions. Use a in EmitLoweredTLSCall()
35550 Subtarget.getRegisterInfo()->getDarwinTLSCallPreservedMask() : in EmitLoweredTLSCall()
35551 Subtarget.getRegisterInfo()->getCallPreservedMask(*F, CallingConv::C); in EmitLoweredTLSCall()
35554 BuildMI(*BB, MI, MIMD, TII->get(X86::MOV64rm), X86::RDI) in EmitLoweredTLSCall()
35561 MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL64m)); in EmitLoweredTLSCall()
35566 BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), X86::EAX) in EmitLoweredTLSCall()
35573 MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL32m)); in EmitLoweredTLSCall()
35578 BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), X86::EAX) in EmitLoweredTLSCall()
35579 .addReg(TII->getGlobalBaseReg(F)) in EmitLoweredTLSCall()
35585 MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL32m)); in EmitLoweredTLSCall()
35614 // aliases and are doing non-trivial configuration of the thunk's body. For in getIndirectThunkSymbol()
35615 // example, the Linux kernel will do boot-time hot patching of the thunk in getIndirectThunkSymbol()
35621 // LLVM will generate calls to specific thunks, we merely make a best-effort in getIndirectThunkSymbol()
35626 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35629 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35632 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35635 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35638 assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!"); in getIndirectThunkSymbol()
35646 // When targeting an internal COMDAT thunk use an LLVM-specific name. in getIndirectThunkSymbol()
35649 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35652 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35655 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35658 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"); in getIndirectThunkSymbol()
35661 assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!"); in getIndirectThunkSymbol()
35668 assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!"); in getIndirectThunkSymbol()
35684 // Find an available scratch register to hold the callee. On 64-bit, we can in EmitLoweredIndirectThunk()
35686 // incorrect code. On 32-bit, we use one of EAX, ECX, or EDX that isn't in EmitLoweredIndirectThunk()
35704 // Choose the first remaining non-zero available register. in EmitLoweredIndirectThunk()
35718 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), AvailableReg) in EmitLoweredIndirectThunk()
35721 MI.setDesc(TII->get(Opc)); in EmitLoweredIndirectThunk()
35722 MachineInstrBuilder(*BB->getParent(), &MI) in EmitLoweredIndirectThunk()
35742 MachineFunction *MF = MBB->getParent(); in emitSetJmpShadowStackFix()
35744 MachineRegisterInfo &MRI = MF->getRegInfo(); in emitSetJmpShadowStackFix()
35752 MVT PVT = getPointerTy(MF->getDataLayout()); in emitSetJmpShadowStackFix()
35756 BuildMI(*MBB, MI, MIMD, TII->get(XorRROpc)) in emitSetJmpShadowStackFix()
35764 BuildMI(*MBB, MI, MIMD, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg); in emitSetJmpShadowStackFix()
35768 MIB = BuildMI(*MBB, MI, MIMD, TII->get(PtrStoreOpc)); in emitSetJmpShadowStackFix()
35785 MachineFunction *MF = MBB->getParent(); in emitEHSjLjSetJmp()
35788 MachineRegisterInfo &MRI = MF->getRegInfo(); in emitEHSjLjSetJmp()
35790 const BasicBlock *BB = MBB->getBasicBlock(); in emitEHSjLjSetJmp()
35791 MachineFunction::iterator I = ++MBB->getIterator(); in emitEHSjLjSetJmp()
35804 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!"); in emitEHSjLjSetJmp()
35811 MVT PVT = getPointerTy(MF->getDataLayout()); in emitEHSjLjSetJmp()
35818 // buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB in emitEHSjLjSetJmp()
35832 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB); in emitEHSjLjSetJmp()
35833 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB); in emitEHSjLjSetJmp()
35834 MachineBasicBlock *restoreMBB = MF->CreateMachineBasicBlock(BB); in emitEHSjLjSetJmp()
35835 MF->insert(I, mainMBB); in emitEHSjLjSetJmp()
35836 MF->insert(I, sinkMBB); in emitEHSjLjSetJmp()
35837 MF->push_back(restoreMBB); in emitEHSjLjSetJmp()
35838 restoreMBB->setMachineBlockAddressTaken(); in emitEHSjLjSetJmp()
35843 sinkMBB->splice(sinkMBB->begin(), MBB, in emitEHSjLjSetJmp()
35844 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); in emitEHSjLjSetJmp()
35845 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); in emitEHSjLjSetJmp()
35851 bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) && in emitEHSjLjSetJmp()
35860 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA64r), LabelReg) in emitEHSjLjSetJmp()
35868 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA32r), LabelReg) in emitEHSjLjSetJmp()
35869 .addReg(XII->getGlobalBaseReg(MF)) in emitEHSjLjSetJmp()
35878 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrStoreOpc)); in emitEHSjLjSetJmp()
35891 if (MF->getFunction().getParent()->getModuleFlag("cf-protection-return")) { in emitEHSjLjSetJmp()
35896 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::EH_SjLj_Setup)) in emitEHSjLjSetJmp()
35900 MIB.addRegMask(RegInfo->getNoPreservedMask()); in emitEHSjLjSetJmp()
35901 thisMBB->addSuccessor(mainMBB); in emitEHSjLjSetJmp()
35902 thisMBB->addSuccessor(restoreMBB); in emitEHSjLjSetJmp()
35906 BuildMI(mainMBB, MIMD, TII->get(X86::MOV32r0), mainDstReg); in emitEHSjLjSetJmp()
35907 mainMBB->addSuccessor(sinkMBB); in emitEHSjLjSetJmp()
35910 BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg) in emitEHSjLjSetJmp()
35917 if (RegInfo->hasBasePointer(*MF)) { in emitEHSjLjSetJmp()
35920 X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); in emitEHSjLjSetJmp()
35921 X86FI->setRestoreBasePointer(MF); in emitEHSjLjSetJmp()
35922 Register FramePtr = RegInfo->getFrameRegister(*MF); in emitEHSjLjSetJmp()
35923 Register BasePtr = RegInfo->getBaseRegister(); in emitEHSjLjSetJmp()
35925 addRegOffset(BuildMI(restoreMBB, MIMD, TII->get(Opm), BasePtr), in emitEHSjLjSetJmp()
35926 FramePtr, true, X86FI->getRestoreBasePointerOffset()) in emitEHSjLjSetJmp()
35929 BuildMI(restoreMBB, MIMD, TII->get(X86::MOV32ri), restoreDstReg).addImm(1); in emitEHSjLjSetJmp()
35930 BuildMI(restoreMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB); in emitEHSjLjSetJmp()
35931 restoreMBB->addSuccessor(sinkMBB); in emitEHSjLjSetJmp()
35946 MachineFunction *MF = MBB->getParent(); in emitLongJmpShadowStackFix()
35948 MachineRegisterInfo &MRI = MF->getRegInfo(); in emitLongJmpShadowStackFix()
35954 MVT PVT = getPointerTy(MF->getDataLayout()); in emitLongJmpShadowStackFix()
35981 MachineFunction::iterator I = ++MBB->getIterator(); in emitLongJmpShadowStackFix()
35982 const BasicBlock *BB = MBB->getBasicBlock(); in emitLongJmpShadowStackFix()
35984 MachineBasicBlock *checkSspMBB = MF->CreateMachineBasicBlock(BB); in emitLongJmpShadowStackFix()
35985 MachineBasicBlock *fallMBB = MF->CreateMachineBasicBlock(BB); in emitLongJmpShadowStackFix()
35986 MachineBasicBlock *fixShadowMBB = MF->CreateMachineBasicBlock(BB); in emitLongJmpShadowStackFix()
35987 MachineBasicBlock *fixShadowLoopPrepareMBB = MF->CreateMachineBasicBlock(BB); in emitLongJmpShadowStackFix()
35988 MachineBasicBlock *fixShadowLoopMBB = MF->CreateMachineBasicBlock(BB); in emitLongJmpShadowStackFix()
35989 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB); in emitLongJmpShadowStackFix()
35990 MF->insert(I, checkSspMBB); in emitLongJmpShadowStackFix()
35991 MF->insert(I, fallMBB); in emitLongJmpShadowStackFix()
35992 MF->insert(I, fixShadowMBB); in emitLongJmpShadowStackFix()
35993 MF->insert(I, fixShadowLoopPrepareMBB); in emitLongJmpShadowStackFix()
35994 MF->insert(I, fixShadowLoopMBB); in emitLongJmpShadowStackFix()
35995 MF->insert(I, sinkMBB); in emitLongJmpShadowStackFix()
35998 sinkMBB->splice(sinkMBB->begin(), MBB, MachineBasicBlock::iterator(MI), in emitLongJmpShadowStackFix()
35999 MBB->end()); in emitLongJmpShadowStackFix()
36000 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); in emitLongJmpShadowStackFix()
36002 MBB->addSuccessor(checkSspMBB); in emitLongJmpShadowStackFix()
36006 BuildMI(checkSspMBB, MIMD, TII->get(X86::MOV32r0), ZReg); in emitLongJmpShadowStackFix()
36010 BuildMI(checkSspMBB, MIMD, TII->get(X86::SUBREG_TO_REG), TmpZReg) in emitLongJmpShadowStackFix()
36020 BuildMI(checkSspMBB, MIMD, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg); in emitLongJmpShadowStackFix()
36025 BuildMI(checkSspMBB, MIMD, TII->get(TestRROpc)) in emitLongJmpShadowStackFix()
36028 BuildMI(checkSspMBB, MIMD, TII->get(X86::JCC_1)) in emitLongJmpShadowStackFix()
36031 checkSspMBB->addSuccessor(sinkMBB); in emitLongJmpShadowStackFix()
36032 checkSspMBB->addSuccessor(fallMBB); in emitLongJmpShadowStackFix()
36039 BuildMI(fallMBB, MIMD, TII->get(PtrLoadOpc), PrevSSPReg); in emitLongJmpShadowStackFix()
36055 BuildMI(fallMBB, MIMD, TII->get(SubRROpc), SspSubReg) in emitLongJmpShadowStackFix()
36060 BuildMI(fallMBB, MIMD, TII->get(X86::JCC_1)) in emitLongJmpShadowStackFix()
36063 fallMBB->addSuccessor(sinkMBB); in emitLongJmpShadowStackFix()
36064 fallMBB->addSuccessor(fixShadowMBB); in emitLongJmpShadowStackFix()
36070 BuildMI(fixShadowMBB, MIMD, TII->get(ShrRIOpc), SspFirstShrReg) in emitLongJmpShadowStackFix()
36076 BuildMI(fixShadowMBB, MIMD, TII->get(IncsspOpc)).addReg(SspFirstShrReg); in emitLongJmpShadowStackFix()
36080 BuildMI(fixShadowMBB, MIMD, TII->get(ShrRIOpc), SspSecondShrReg) in emitLongJmpShadowStackFix()
36085 BuildMI(fixShadowMBB, MIMD, TII->get(X86::JCC_1)) in emitLongJmpShadowStackFix()
36088 fixShadowMBB->addSuccessor(sinkMBB); in emitLongJmpShadowStackFix()
36089 fixShadowMBB->addSuccessor(fixShadowLoopPrepareMBB); in emitLongJmpShadowStackFix()
36094 BuildMI(fixShadowLoopPrepareMBB, MIMD, TII->get(ShlR1Opc), SspAfterShlReg) in emitLongJmpShadowStackFix()
36101 BuildMI(fixShadowLoopPrepareMBB, MIMD, TII->get(MovRIOpc), Value128InReg) in emitLongJmpShadowStackFix()
36103 fixShadowLoopPrepareMBB->addSuccessor(fixShadowLoopMBB); in emitLongJmpShadowStackFix()
36109 BuildMI(fixShadowLoopMBB, MIMD, TII->get(X86::PHI), CounterReg) in emitLongJmpShadowStackFix()
36116 BuildMI(fixShadowLoopMBB, MIMD, TII->get(IncsspOpc)).addReg(Value128InReg); in emitLongJmpShadowStackFix()
36120 BuildMI(fixShadowLoopMBB, MIMD, TII->get(DecROpc), DecReg).addReg(CounterReg); in emitLongJmpShadowStackFix()
36123 BuildMI(fixShadowLoopMBB, MIMD, TII->get(X86::JCC_1)) in emitLongJmpShadowStackFix()
36126 fixShadowLoopMBB->addSuccessor(sinkMBB); in emitLongJmpShadowStackFix()
36127 fixShadowLoopMBB->addSuccessor(fixShadowLoopMBB); in emitLongJmpShadowStackFix()
36136 MachineFunction *MF = MBB->getParent(); in emitEHSjLjLongJmp()
36138 MachineRegisterInfo &MRI = MF->getRegInfo(); in emitEHSjLjLongJmp()
36144 MVT PVT = getPointerTy(MF->getDataLayout()); in emitEHSjLjLongJmp()
36154 Register SP = RegInfo->getStackRegister(); in emitEHSjLjLongJmp()
36167 if (MF->getFunction().getParent()->getModuleFlag("cf-protection-return")) { in emitEHSjLjLongJmp()
36172 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), FP); in emitEHSjLjLongJmp()
36184 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), Tmp); in emitEHSjLjLongJmp()
36198 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), SP); in emitEHSjLjLongJmp()
36209 BuildMI(*thisMBB, MI, MIMD, TII->get(IJmpOpc)).addReg(Tmp); in emitEHSjLjLongJmp()
36220 MachineFunction *MF = MBB->getParent(); in SetupEntryBlockForSjLj()
36221 MachineRegisterInfo *MRI = &MF->getRegInfo(); in SetupEntryBlockForSjLj()
36224 MVT PVT = getPointerTy(MF->getDataLayout()); in SetupEntryBlockForSjLj()
36230 bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) && in SetupEntryBlockForSjLj()
36238 VR = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
36242 BuildMI(*MBB, MI, MIMD, TII->get(X86::LEA64r), VR) in SetupEntryBlockForSjLj()
36249 BuildMI(*MBB, MI, MIMD, TII->get(X86::LEA32r), VR) in SetupEntryBlockForSjLj()
36250 .addReg(0) /* TII->getGlobalBaseReg(MF) */ in SetupEntryBlockForSjLj()
36257 MachineInstrBuilder MIB = BuildMI(*MBB, MI, MIMD, TII->get(Op)); in SetupEntryBlockForSjLj()
36269 MachineFunction *MF = BB->getParent(); in EmitSjLjDispatchBlock()
36270 MachineRegisterInfo *MRI = &MF->getRegInfo(); in EmitSjLjDispatchBlock()
36272 int FI = MF->getFrameInfo().getFunctionContextIndex(); in EmitSjLjDispatchBlock()
36292 if (!MF->hasCallSiteLandingPad(Sym)) in EmitSjLjDispatchBlock()
36295 for (unsigned CSI : MF->getCallSiteLandingPad(Sym)) { in EmitSjLjDispatchBlock()
36309 InvokeBBs.insert(LP->pred_begin(), LP->pred_end()); in EmitSjLjDispatchBlock()
36319 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock(); in EmitSjLjDispatchBlock()
36320 DispatchBB->setIsEHPad(true); in EmitSjLjDispatchBlock()
36322 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock(); in EmitSjLjDispatchBlock()
36323 BuildMI(TrapBB, MIMD, TII->get(X86::TRAP)); in EmitSjLjDispatchBlock()
36324 DispatchBB->addSuccessor(TrapBB); in EmitSjLjDispatchBlock()
36326 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock(); in EmitSjLjDispatchBlock()
36327 DispatchBB->addSuccessor(DispContBB); in EmitSjLjDispatchBlock()
36330 MF->push_back(DispatchBB); in EmitSjLjDispatchBlock()
36331 MF->push_back(DispContBB); in EmitSjLjDispatchBlock()
36332 MF->push_back(TrapBB); in EmitSjLjDispatchBlock()
36340 MachineJumpTableInfo *JTI = MF->getOrCreateJumpTableInfo(JTE); in EmitSjLjDispatchBlock()
36341 unsigned MJTI = JTI->createJumpTableIndex(LPadList); in EmitSjLjDispatchBlock()
36343 const X86RegisterInfo &RI = TII->getRegisterInfo(); in EmitSjLjDispatchBlock()
36349 X86MachineFunctionInfo *MFI = MF->getInfo<X86MachineFunctionInfo>(); in EmitSjLjDispatchBlock()
36350 MFI->setRestoreBasePointer(MF); in EmitSjLjDispatchBlock()
36355 addRegOffset(BuildMI(DispatchBB, MIMD, TII->get(Op), BP), FP, true, in EmitSjLjDispatchBlock()
36356 MFI->getRestoreBasePointerOffset()) in EmitSjLjDispatchBlock()
36359 BuildMI(DispatchBB, MIMD, TII->get(X86::NOOP)) in EmitSjLjDispatchBlock()
36364 Register IReg = MRI->createVirtualRegister(&X86::GR32_NOSPRegClass); in EmitSjLjDispatchBlock()
36365 addFrameReference(BuildMI(DispatchBB, MIMD, TII->get(X86::MOV32rm), IReg), FI, in EmitSjLjDispatchBlock()
36367 BuildMI(DispatchBB, MIMD, TII->get(X86::CMP32ri)) in EmitSjLjDispatchBlock()
36370 BuildMI(DispatchBB, MIMD, TII->get(X86::JCC_1)) in EmitSjLjDispatchBlock()
36375 Register BReg = MRI->createVirtualRegister(&X86::GR64RegClass); in EmitSjLjDispatchBlock()
36376 Register IReg64 = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass); in EmitSjLjDispatchBlock()
36379 BuildMI(DispContBB, MIMD, TII->get(X86::LEA64r), BReg) in EmitSjLjDispatchBlock()
36386 BuildMI(DispContBB, MIMD, TII->get(TargetOpcode::SUBREG_TO_REG), IReg64) in EmitSjLjDispatchBlock()
36394 BuildMI(DispContBB, MIMD, TII->get(X86::JMP64m)) in EmitSjLjDispatchBlock()
36402 Register OReg = MRI->createVirtualRegister(&X86::GR32RegClass); in EmitSjLjDispatchBlock()
36403 Register OReg64 = MRI->createVirtualRegister(&X86::GR64RegClass); in EmitSjLjDispatchBlock()
36404 Register TReg = MRI->createVirtualRegister(&X86::GR64RegClass); in EmitSjLjDispatchBlock()
36407 BuildMI(DispContBB, MIMD, TII->get(X86::MOV32rm), OReg) in EmitSjLjDispatchBlock()
36414 BuildMI(DispContBB, MIMD, TII->get(X86::MOVSX64rr32), OReg64) in EmitSjLjDispatchBlock()
36417 BuildMI(DispContBB, MIMD, TII->get(X86::ADD64rr), TReg) in EmitSjLjDispatchBlock()
36421 BuildMI(DispContBB, MIMD, TII->get(X86::JMP64r)).addReg(TReg); in EmitSjLjDispatchBlock()
36429 BuildMI(DispContBB, MIMD, TII->get(X86::JMP32m)) in EmitSjLjDispatchBlock()
36441 DispContBB->addSuccessor(LP); in EmitSjLjDispatchBlock()
36445 const MCPhysReg *SavedRegs = MF->getRegInfo().getCalleeSavedRegs(); in EmitSjLjDispatchBlock()
36450 SmallVector<MachineBasicBlock *, 8> Successors(MBB->succ_rbegin(), in EmitSjLjDispatchBlock()
36451 MBB->succ_rend()); in EmitSjLjDispatchBlock()
36454 if (MBBS->isEHPad()) { in EmitSjLjDispatchBlock()
36455 MBB->removeSuccessor(MBBS); in EmitSjLjDispatchBlock()
36460 MBB->addSuccessor(DispatchBB); in EmitSjLjDispatchBlock()
36462 // Find the invoke call and mark all of the callee-saved registers as in EmitSjLjDispatchBlock()
36486 // Mark all former landing pads as non-landing pads. The dispatch is the only in EmitSjLjDispatchBlock()
36489 LP->setIsEHPad(false); in EmitSjLjDispatchBlock()
36503 MachineFunction &MF = *BB->getParent(); in emitPatchableEventCall()
36510 BB->insert(MachineBasicBlock::iterator(MI), CallseqStart); in emitPatchableEventCall()
36516 BB->insertAfter(MachineBasicBlock::iterator(MI), CallseqEnd); in emitPatchableEventCall()
36524 MachineFunction *MF = BB->getParent(); in EmitInstrWithCustomInserter()
36591 MF->getFrameInfo().CreateStackObject(2, Align(2), false); in EmitInstrWithCustomInserter()
36592 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FNSTCW16m)), in EmitInstrWithCustomInserter()
36596 Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass); in EmitInstrWithCustomInserter()
36597 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOVZX32rm16), OldCW), in EmitInstrWithCustomInserter()
36602 Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass); in EmitInstrWithCustomInserter()
36603 BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW) in EmitInstrWithCustomInserter()
36609 MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass); in EmitInstrWithCustomInserter()
36610 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), NewCW16) in EmitInstrWithCustomInserter()
36615 MF->getFrameInfo().CreateStackObject(2, Align(2), false); in EmitInstrWithCustomInserter()
36616 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)), in EmitInstrWithCustomInserter()
36621 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)), in EmitInstrWithCustomInserter()
36626 BuildMI(*BB, MI, MIMD, TII->get(X86::ADD_Fp80)) in EmitInstrWithCustomInserter()
36631 BuildMI(*BB, MI, MIMD, TII->get(X86::ADD_Fp80m32)) in EmitInstrWithCustomInserter()
36642 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)), in EmitInstrWithCustomInserter()
36661 MF->getFrameInfo().CreateStackObject(2, Align(2), false); in EmitInstrWithCustomInserter()
36662 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FNSTCW16m)), in EmitInstrWithCustomInserter()
36666 Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass); in EmitInstrWithCustomInserter()
36667 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOVZX32rm16), OldCW), in EmitInstrWithCustomInserter()
36671 Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass); in EmitInstrWithCustomInserter()
36672 BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW) in EmitInstrWithCustomInserter()
36677 MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass); in EmitInstrWithCustomInserter()
36678 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), NewCW16) in EmitInstrWithCustomInserter()
36683 MF->getFrameInfo().CreateStackObject(2, Align(2), false); in EmitInstrWithCustomInserter()
36684 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)), in EmitInstrWithCustomInserter()
36690 TII->get(X86::FLDCW16m)), NewCWFrameIdx); in EmitInstrWithCustomInserter()
36695 // clang-format off in EmitInstrWithCustomInserter()
36706 // clang-format on in EmitInstrWithCustomInserter()
36710 addFullAddress(BuildMI(*BB, MI, MIMD, TII->get(Opc)), AM) in EmitInstrWithCustomInserter()
36714 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)), in EmitInstrWithCustomInserter()
36758 // - which is ESI for i686 - register allocator would not be able to in EmitInstrWithCustomInserter()
36760 // - there never would be enough unreserved registers during regalloc in EmitInstrWithCustomInserter()
36765 // If it is not i686 or there is no base pointer - nothing to do here. in EmitInstrWithCustomInserter()
36766 if (!Subtarget.is32Bit() || !TRI->hasBasePointer(*MF)) in EmitInstrWithCustomInserter()
36773 assert(TRI->getBaseRegister() == X86::ESI && in EmitInstrWithCustomInserter()
36777 MachineRegisterInfo &MRI = MF->getRegInfo(); in EmitInstrWithCustomInserter()
36778 MVT SPTy = getPointerTy(MF->getDataLayout()); in EmitInstrWithCustomInserter()
36792 while (RMBBI != BB->rend() && in EmitInstrWithCustomInserter()
36793 (RMBBI->definesRegister(X86::EAX, /*TRI=*/nullptr) || in EmitInstrWithCustomInserter()
36794 RMBBI->definesRegister(X86::EBX, /*TRI=*/nullptr) || in EmitInstrWithCustomInserter()
36795 RMBBI->definesRegister(X86::ECX, /*TRI=*/nullptr) || in EmitInstrWithCustomInserter()
36796 RMBBI->definesRegister(X86::EDX, /*TRI=*/nullptr))) { in EmitInstrWithCustomInserter()
36801 BuildMI(*BB, *MBBI, MIMD, TII->get(X86::LEA32r), computedAddrVReg), AM); in EmitInstrWithCustomInserter()
36809 Register BasePtr = TRI->getBaseRegister(); in EmitInstrWithCustomInserter()
36810 if (TRI->hasBasePointer(*MF) && in EmitInstrWithCustomInserter()
36812 if (!BB->isLiveIn(BasePtr)) in EmitInstrWithCustomInserter()
36813 BB->addLiveIn(BasePtr); in EmitInstrWithCustomInserter()
36816 MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass); in EmitInstrWithCustomInserter()
36817 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), SaveRBX) in EmitInstrWithCustomInserter()
36819 Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass); in EmitInstrWithCustomInserter()
36821 BuildMI(*BB, MI, MIMD, TII->get(X86::LCMPXCHG16B_SAVE_RBX), Dst); in EmitInstrWithCustomInserter()
36828 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::RBX) in EmitInstrWithCustomInserter()
36831 BuildMI(*BB, MI, MIMD, TII->get(X86::LCMPXCHG16B)); in EmitInstrWithCustomInserter()
36840 Register BasePtr = TRI->getBaseRegister(); in EmitInstrWithCustomInserter()
36844 if (!IsRBX || !TRI->hasBasePointer(*MF)) { in EmitInstrWithCustomInserter()
36845 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::ECX) in EmitInstrWithCustomInserter()
36847 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EAX) in EmitInstrWithCustomInserter()
36849 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EBX) in EmitInstrWithCustomInserter()
36851 BuildMI(*BB, MI, MIMD, TII->get(X86::MWAITXrrr)); in EmitInstrWithCustomInserter()
36854 if (!BB->isLiveIn(BasePtr)) { in EmitInstrWithCustomInserter()
36855 BB->addLiveIn(BasePtr); in EmitInstrWithCustomInserter()
36858 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::ECX) in EmitInstrWithCustomInserter()
36860 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EAX) in EmitInstrWithCustomInserter()
36862 assert(Subtarget.is64Bit() && "Expected 64-bit mode!"); in EmitInstrWithCustomInserter()
36865 MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass); in EmitInstrWithCustomInserter()
36866 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), SaveRBX) in EmitInstrWithCustomInserter()
36869 Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass); in EmitInstrWithCustomInserter()
36870 BuildMI(*BB, MI, MIMD, TII->get(X86::MWAITX_SAVE_RBX)) in EmitInstrWithCustomInserter()
36879 assert(Subtarget.is32Bit() && "preallocated only used in 32-bit"); in EmitInstrWithCustomInserter()
36880 auto *MFI = MF->getInfo<X86MachineFunctionInfo>(); in EmitInstrWithCustomInserter()
36881 MFI->setHasPreallocatedCall(true); in EmitInstrWithCustomInserter()
36883 size_t StackAdjustment = MFI->getPreallocatedStackSize(PreallocatedId); in EmitInstrWithCustomInserter()
36887 BuildMI(*BB, MI, MIMD, TII->get(X86::SUB32ri), X86::ESP) in EmitInstrWithCustomInserter()
36894 assert(Subtarget.is32Bit() && "preallocated calls only used in 32-bit"); in EmitInstrWithCustomInserter()
36897 auto *MFI = MF->getInfo<X86MachineFunctionInfo>(); in EmitInstrWithCustomInserter()
36898 size_t ArgOffset = MFI->getPreallocatedArgOffsets(PreallocatedId)[ArgIdx]; in EmitInstrWithCustomInserter()
36902 addRegOffset(BuildMI(*BB, MI, MIMD, TII->get(X86::LEA32r), in EmitInstrWithCustomInserter()
36916 // clang-format off in EmitInstrWithCustomInserter()
36924 // clang-format on in EmitInstrWithCustomInserter()
36927 MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc)); in EmitInstrWithCustomInserter()
36938 BuildMI(*BB, MI, MIMD, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm)); in EmitInstrWithCustomInserter()
36940 auto *MFI = MF->getInfo<X86MachineFunctionInfo>(); in EmitInstrWithCustomInserter()
36941 MFI->setAMXProgModel(AMXProgModelEnum::DirectReg); in EmitInstrWithCustomInserter()
36945 auto *MFI = MF->getInfo<X86MachineFunctionInfo>(); in EmitInstrWithCustomInserter()
36946 MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA); in EmitInstrWithCustomInserter()
36968 MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc)); in EmitInstrWithCustomInserter()
36976 MIB.add(MI.getOperand(CurOp++)); // index -- stride in EmitInstrWithCustomInserter()
36992 // clang-format off in EmitInstrWithCustomInserter()
36996 // clang-format on in EmitInstrWithCustomInserter()
36998 MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc)); in EmitInstrWithCustomInserter()
37009 //===----------------------------------------------------------------------===//
37011 //===----------------------------------------------------------------------===//
37039 // For vectors - if we have a constant, then try to sign extend. in targetShrinkDemandedConstant()
37068 const APInt &Mask = C->getAPIntValue(); in targetShrinkDemandedConstant()
37070 // Clear all non-demanded bits initially. in targetShrinkDemandedConstant()
37094 // and non-demanded bits. in targetShrinkDemandedConstant()
37254 ShAmt = VT.getScalarSizeInBits() - 1; in computeKnownBitsForTargetNode()
37303 // PSHUFB is being used as a LUT (ctpop etc.) - the target shuffle handling in computeKnownBitsForTargetNode()
37419 unsigned Shift = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 0); in computeKnownBitsForTargetNode()
37420 unsigned Length = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 8); in computeKnownBitsForTargetNode()
37442 // The result will have at least as many trailing zeros as the non-mask in computeKnownBitsForTargetNode()
37474 // Truncations/Conversions - upper elements are known zero. in computeKnownBitsForTargetNode()
37489 // Strict Conversions - upper elements are known zero. in computeKnownBitsForTargetNode()
37538 switch (Op->getConstantOperandVal(0)) { in computeKnownBitsForTargetNode()
37581 // TODO - use resolveTargetShuffleInputs once we can limit recursive depth. in computeKnownBitsForTargetNode()
37611 // TODO - handle target shuffle ops with different value types. in computeKnownBitsForTargetNode()
37648 if (Tmp > (NumSrcBits - VTBits)) in ComputeNumSignBitsForTargetNode()
37649 return Tmp - (NumSrcBits - VTBits); in ComputeNumSignBitsForTargetNode()
37661 auto NumSignBitsPACKSS = [&](SDValue V, const APInt &Elts) -> unsigned { in ComputeNumSignBitsForTargetNode()
37684 if (Tmp > (SrcBits - VTBits)) in ComputeNumSignBitsForTargetNode()
37685 return Tmp - (SrcBits - VTBits); in ComputeNumSignBitsForTargetNode()
37700 return VTBits; // Shifted all bits out --> zero. in ComputeNumSignBitsForTargetNode()
37703 return 1; // Shifted all sign bits out --> unknown. in ComputeNumSignBitsForTargetNode()
37704 return Tmp - ShiftVal.getZExtValue(); in ComputeNumSignBitsForTargetNode()
37710 if (ShiftVal.uge(VTBits - 1)) in ComputeNumSignBitsForTargetNode()
37718 // cmpss/cmpsd return zero/all-bits result values in the bottom element. in ComputeNumSignBitsForTargetNode()
37729 // Vector compares return zero/all-bits result values. in ComputeNumSignBitsForTargetNode()
37750 // TODO - use resolveTargetShuffleInputs once we can limit recursive depth. in ComputeNumSignBitsForTargetNode()
37777 // TODO - handle target shuffle ops with different value types. in ComputeNumSignBitsForTargetNode()
37800 if (N->getOpcode() == X86ISD::Wrapper || N->getOpcode() == X86ISD::WrapperRIP) in unwrapAddress()
37801 return N->getOperand(0); in unwrapAddress()
37810 if (!LN->isSimple()) in narrowLoadToVZLoad()
37814 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()}; in narrowLoadToVZLoad()
37816 LN->getPointerInfo(), LN->getOriginalAlign(), in narrowLoadToVZLoad()
37817 LN->getMemOperand()->getFlags()); in narrowLoadToVZLoad()
37831 // Match against a VZEXT_MOVL vXi32 and vXi16 zero-extending instruction. in matchUnaryShuffle()
37834 if ((isUndefOrZero(Mask[1]) && isUndefInRange(Mask, 2, NumMaskElts - 2)) || in matchUnaryShuffle()
37836 isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1))) { in matchUnaryShuffle()
37847 // TODO: Add 512-bit vector support (split AVX512F and AVX512BW). in matchUnaryShuffle()
37865 unsigned Len = Scale - 1; in matchUnaryShuffle()
37891 // Match against a VZEXT_MOVL instruction, SSE1 only supports 32-bits (MOVSS). in matchUnaryShuffle()
37895 isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1)) { in matchUnaryShuffle()
37926 assert(Subtarget.hasAVX() && "AVX required for 256-bit vector shuffles"); in matchUnaryShuffle()
37948 "AVX512 required for 512-bit vector shuffles"); in matchUnaryShuffle()
37994 // PERMPD/PERMQ permutes within a 256-bit vector (AVX2+). in matchUnaryPermuteShuffle()
38011 // VPERMILPD can permute with a non-repeating shuffle. in matchUnaryPermuteShuffle()
38032 // had to use 2-input SHUFPD/SHUFPS shuffles (not handled here). in matchUnaryPermuteShuffle()
38037 // Narrow the repeated mask to create 32-bit element permutes. in matchUnaryPermuteShuffle()
38075 OffsetHiMask[i] = (HiMask[i] < 0 ? HiMask[i] : HiMask[i] - 4); in matchUnaryPermuteShuffle()
38189 // Use (SSE41) PACKUSWD if the leading zerobits goto the lowest 16-bits. in matchBinaryShuffle()
38198 // Use PACKUSBW if the leading zerobits goto the lowest 8-bits. in matchBinaryShuffle()
38205 // Use PACKSSWD if the signbits extend to the lowest 16-bits. in matchBinaryShuffle()
38231 // non-blended source element is zero in each case. in matchBinaryShuffle()
38441 S0 = (SM_SentinelUndef == M0 ? -1 : 0); in matchBinaryPermuteShuffle()
38442 S1 = (SM_SentinelUndef == M1 ? -1 : 1); in matchBinaryPermuteShuffle()
38445 S0 = (SM_SentinelUndef == M0 ? -1 : M0 & 3); in matchBinaryPermuteShuffle()
38446 S1 = (SM_SentinelUndef == M1 ? -1 : M1 & 3); in matchBinaryPermuteShuffle()
38449 S0 = (SM_SentinelUndef == M0 ? -1 : M0 & 3); in matchBinaryPermuteShuffle()
38450 S1 = (SM_SentinelUndef == M1 ? -1 : M1 & 3); in matchBinaryPermuteShuffle()
38457 int ShufMask[4] = {-1, -1, -1, -1}; in matchBinaryPermuteShuffle()
38494 /// chain of single-use x86 shuffle instructions and accumulated the combined
38551 // is different from the root element size - this would prevent writemasks in combineX86ShuffleChain()
38555 if (Root.hasOneUse() && Root->use_begin()->getOpcode() == ISD::VSELECT && in combineX86ShuffleChain()
38556 Root->use_begin()->getOperand(0).getScalarValueSizeInBits() == 1) { in combineX86ShuffleChain()
38573 // See if the shuffle is a hidden identity shuffle - repeated args in HOPs in combineX86ShuffleChain()
38588 // Handle 128/256-bit lane shuffles of 512-bit vectors. in combineX86ShuffleChain()
38594 if (isUndefOrZeroInRange(Mask, 1, NumBaseMaskElts - 1)) { in combineX86ShuffleChain()
38615 int PermMask[4] = {-1, -1, -1, -1}; in combineX86ShuffleChain()
38619 assert(ScaledMask[i] >= -1 && "Illegal shuffle sentinel value"); in combineX86ShuffleChain()
38660 // Handle 128-bit lane shuffles of 256-bit vectors. in combineX86ShuffleChain()
38675 // If we're inserting the low subvector, an insert-subvector 'concat' in combineX86ShuffleChain()
38708 // TODO - handle AVX512VL cases with X86ISD::SHUF128. in combineX86ShuffleChain()
38727 // For masks that have been widened to 128-bit elements or more, in combineX86ShuffleChain()
38728 // narrow back down to 64-bit elements. in combineX86ShuffleChain()
38739 // TODO - variable shuffles might need this to be widened again. in combineX86ShuffleChain()
38779 // Attempt to match against broadcast-from-vector. in combineX86ShuffleChain()
38965 // Don't try to re-form single instruction chains under any circumstances now in combineX86ShuffleChain()
38987 // If we have a single input lane-crossing shuffle then lower to VPERMV. in combineX86ShuffleChain()
38996 // AVX512 variants (non-VLX will pad to 512-bit shuffles). in combineX86ShuffleChain()
39011 // Lower a unary+zero lane-crossing shuffle as VPERMV3 with a zero in combineX86ShuffleChain()
39012 // vector as the second source (non-VLX will pad to 512-bit shuffles). in combineX86ShuffleChain()
39023 // Adjust shuffle mask - replace SM_SentinelZero with second source index. in combineX86ShuffleChain()
39041 // If we have a dual input lane-crossing shuffle then lower to VPERMV3, in combineX86ShuffleChain()
39042 // (non-VLX will pad to 512-bit shuffles). in combineX86ShuffleChain()
39061 // See if we can combine a single input shuffle with zeros to a bit-mask, in combineX86ShuffleChain()
39089 // the 128-bit lanes use the variable mask to VPERMILPS. in combineX86ShuffleChain()
39106 // With XOP, binary shuffles of 128/256-bit floating point vectors can combine in combineX86ShuffleChain()
39112 // Bits[3] - Match Bit. in combineX86ShuffleChain()
39113 // Bits[2:1] - (Per Lane) PD Shuffle Mask. in combineX86ShuffleChain()
39114 // Bits[2:0] - (Per Lane) PS Shuffle Mask. in combineX86ShuffleChain()
39121 VPerm2Idx.push_back(-1); in combineX86ShuffleChain()
39174 // With XOP, if we have a 128-bit binary input shuffle we can always combine in combineX86ShuffleChain()
39175 // to VPPERM. We match the depth requirement of PSHUFB - VPPERM is never in combineX86ShuffleChain()
39180 // Bits[4:0] - Byte Index (0 - 31) in combineX86ShuffleChain()
39181 // Bits[7:5] - Permute Operation (0 - Source byte, 4 - ZERO) in combineX86ShuffleChain()
39214 // (non-VLX will pad to 512-bit shuffles) in combineX86ShuffleChain()
39244 // -->
39286 WideMask.append((Scale - 1) * NumMaskElts, SM_SentinelUndef); in combineX86ShuffleChainWithExtract()
39325 // elements, and shrink them to the half-width mask. It does this in a loop in combineX86ShuffleChainWithExtract()
39366 // Canonicalize the combined shuffle mask chain with horizontal ops.
39413 // the HOP args are pre-shuffled. in canonicalizeShuffleMaskWithHorizOp()
39425 if (Src1.getOpcode() == Opcode0 && Src0->isOnlyUserOf(Src1.getNode())) in canonicalizeShuffleMaskWithHorizOp()
39439 // shuffle(hop(x,y),hop(z,w)) -> permute(hop(x,z)) etc. in canonicalizeShuffleMaskWithHorizOp()
39459 int PostMask[4] = {-1, -1, -1, -1}; in canonicalizeShuffleMaskWithHorizOp()
39481 SDValue BC1 = BC[BC.size() - 1]; in canonicalizeShuffleMaskWithHorizOp()
39483 // Canonicalize binary shuffles of horizontal ops that use the in canonicalizeShuffleMaskWithHorizOp()
39505 M -= NumElts + (SubLane * NumHalfEltsPerLane); in canonicalizeShuffleMaskWithHorizOp()
39512 // Canonicalize unary horizontal ops to only refer to lower halves. in canonicalizeShuffleMaskWithHorizOp()
39519 M -= NumHalfEltsPerLane; in canonicalizeShuffleMaskWithHorizOp()
39522 M -= NumHalfEltsPerLane; in canonicalizeShuffleMaskWithHorizOp()
39526 // Combine binary shuffle of 2 similar 'Horizontal' instructions into a in canonicalizeShuffleMaskWithHorizOp()
39552 // If we are post-shuffling a 256-bit hop and not requiring the upper in canonicalizeShuffleMaskWithHorizOp()
39553 // elements, then try to narrow to a 128-bit hop directly. in canonicalizeShuffleMaskWithHorizOp()
39602 llvm::none_of(Ops, [](SDValue SrcOp) { return SrcOp->hasOneUse(); })) in combineX86ShufflesConstants()
39674 /// of single-use shuffle instructions, build a generic model of the cumulative
39681 /// special-purpose shuffle.
39697 /// combine-ordering. To fix this, we should do the redundant instruction
39724 return SDValue(); // Bail if we hit a non-simple non-vector. in combineX86ShufflesRecursively()
39738 OpDemandedElts.setBit(M - BaseIdx); in combineX86ShufflesRecursively()
39741 // Op is smaller than Root - extract the demanded elts for the subvector. in combineX86ShufflesRecursively()
39746 .extractBits(RootMask.size() - NumOpMaskElts, NumOpMaskElts) in combineX86ShufflesRecursively()
39798 OpMask.append((NumSubVecs - 1) * OpMaskSize, SM_SentinelUndef); in combineX86ShufflesRecursively()
39834 auto AddOp = [&Ops](SDValue Input, int InsertionPoint) -> int { in combineX86ShufflesRecursively()
39840 // Match failed - should we replace an existing Op? in combineX86ShufflesRecursively()
39847 return Ops.size() - 1; in combineX86ShufflesRecursively()
39853 AddOp(OpInput, OpInputIdx.empty() ? SrcOpIndex : -1)); in combineX86ShufflesRecursively()
39862 // This function can be performance-critical, so we rely on the power-of-2 in combineX86ShufflesRecursively()
39864 // bit-masks and shifts. in combineX86ShufflesRecursively()
39866 "Non-power-of-2 shuffle mask sizes"); in combineX86ShufflesRecursively()
39868 "Non-power-of-2 shuffle mask sizes"); in combineX86ShufflesRecursively()
39879 assert(isPowerOf2_32(MaskWidth) && "Non-power-of-2 shuffle mask sizes"); in combineX86ShufflesRecursively()
39880 assert(isPowerOf2_32(RootRatio) && "Non-power-of-2 shuffle mask sizes"); in combineX86ShufflesRecursively()
39881 assert(isPowerOf2_32(OpRatio) && "Non-power-of-2 shuffle mask sizes"); in combineX86ShufflesRecursively()
39902 : (RootMask[RootIdx] << RootRatioLog2) + (i & (RootRatio - 1)); in combineX86ShufflesRecursively()
39912 RootMaskedIdx = RootMaskedIdx & (MaskWidth - 1); in combineX86ShufflesRecursively()
39921 // Ok, we have non-zero lanes, map them through to one of the Op's inputs. in combineX86ShufflesRecursively()
39924 (RootMaskedIdx & (OpRatio - 1)); in combineX86ShufflesRecursively()
39926 OpMaskedIdx = OpMaskedIdx & (MaskWidth - 1); in combineX86ShufflesRecursively()
39987 if (Ops.size() < (MaxDepth - Depth)) { in combineX86ShufflesRecursively()
39996 if (Ops[i].getNode()->hasOneUse() || in combineX86ShufflesRecursively()
40014 // If constant fold failed and we only have constants - then we have in combineX86ShufflesRecursively()
40015 // multiple uses by a single non-variable shuffle - just bail. in combineX86ShufflesRecursively()
40028 // Canonicalize the combined shuffle mask chain with horizontal ops. in combineX86ShufflesRecursively()
40047 int OpEltIdx = MaskElt - Lo; in combineX86ShufflesRecursively()
40059 NumExpectedVectorElts - NumOpVectorElts, NumOpVectorElts) && in combineX86ShufflesRecursively()
40086 // Reresolve - we might have repeated subvector sources. in combineX86ShufflesRecursively()
40094 // elements, and shrink them to the half-width mask. It does this in a loop in combineX86ShufflesRecursively()
40146 /// Get the PSHUF-style mask from PSHUF node.
40149 /// PSHUF-style masks that can be reused with such instructions.
40158 // If we have more than 128-bits, only the low 128-bits of shuffle mask in getPSHUFShuffleMask()
40165 assert(Mask[j] == Mask[i * LaneElts + j] - (LaneElts * i) && in getPSHUFShuffleMask()
40166 "Mask doesn't repeat in high 128-bit lanes!"); in getPSHUFShuffleMask()
40180 M -= 4; in getPSHUFShuffleMask()
40197 "Called with something other than an x86 128-bit half shuffle!"); in combineRedundantDWordShuffle()
40199 // Walk up a single-use chain looking for a combinable shuffle. Keep a stack in combineRedundantDWordShuffle()
40220 // dword shuffle, and the high words are self-contained. in combineRedundantDWordShuffle()
40230 // dword shuffle, and the low words are self-contained. in combineRedundantDWordShuffle()
40240 // For either i8 -> i16 or i16 -> i32 unpacks, we can combine a dword in combineRedundantDWordShuffle()
40246 // Search for a half-shuffle which we can combine with. in combineRedundantDWordShuffle()
40250 !V->isOnlyUserOf(V.getOperand(0).getNode())) in combineRedundantDWordShuffle()
40321 // permilps(shufps(load(),x)) --> permilps(shufps(x,load()))
40328 // SHUFP(LHS, RHS) -> SHUFP(RHS, LHS) iff LHS is foldable + RHS is not. in combineCommutableSHUFP()
40330 if (V.getOpcode() != X86ISD::SHUFP || !Parent->isOnlyUserOf(V.getNode())) in combineCommutableSHUFP()
40374 // Attempt to fold BLEND(PERMUTE(X),PERMUTE(Y)) -> PERMUTE(BLEND(X,Y))
40444 // the blend mask is the same in the 128-bit subvectors (or can widen to in combineBlendOfPermutes()
40454 // Don't introduce lane-crossing permutes without AVX2, unless it can be in combineBlendOfPermutes()
40469 // TODO - move this to TLI like isBinOp?
40480 // Canonicalize SHUFFLE(UNARYOP(X)) -> UNARYOP(SHUFFLE(X)).
40481 // Canonicalize SHUFFLE(BINOP(X,Y)) -> BINOP(SHUFFLE(X),SHUFFLE(Y)).
40499 (Op.getOpcode() == Opc && Op->hasOneUse()) || in canonicalizeShuffleWithOp()
40500 (Op.getOpcode() == ISD::INSERT_SUBVECTOR && Op->hasOneUse()) || in canonicalizeShuffleWithOp()
40501 (FoldShuf && isTargetShuffle(Op.getOpcode()) && Op->hasOneUse()) || in canonicalizeShuffleWithOp()
40530 N->isOnlyUserOf(N.getOperand(0).getNode())) { in canonicalizeShuffleWithOp()
40575 if (N->isOnlyUserOf(N.getOperand(0).getNode()) && in canonicalizeShuffleWithOp()
40576 N->isOnlyUserOf(N.getOperand(1).getNode())) { in canonicalizeShuffleWithOp()
40639 /// Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()).
40708 // Turn a 128-bit MOVDDUP of a full vector load into movddup+vzload. in combineTargetShuffle()
40730 // TODO - we really need a general SimplifyDemandedVectorElts mechanism. in combineTargetShuffle()
40747 // broadcast(bitcast(src)) -> bitcast(broadcast(src)) in combineTargetShuffle()
40748 // 32-bit targets have to bitcast i64 to f64, so better to bitcast upward. in combineTargetShuffle()
40759 // vbroadcast(bitcast(vbroadcast(src))) -> bitcast(vbroadcast(src)) in combineTargetShuffle()
40760 // If we're re-broadcasting a smaller type then broadcast with that type and in combineTargetShuffle()
40774 // Reduce broadcast source vector to lowest 128-bits. in combineTargetShuffle()
40779 // broadcast(scalar_to_vector(x)) -> broadcast(x). in combineTargetShuffle()
40784 // broadcast(extract_vector_elt(x, 0)) -> broadcast(x). in combineTargetShuffle()
40794 for (SDNode *User : Src->uses()) in combineTargetShuffle()
40795 if (User != N.getNode() && User->getOpcode() == X86ISD::VBROADCAST && in combineTargetShuffle()
40796 Src == User->getOperand(0) && in combineTargetShuffle()
40797 User->getValueSizeInBits(0).getFixedValue() > in combineTargetShuffle()
40803 // vbroadcast(scalarload X) -> vbroadcast_load X in combineTargetShuffle()
40809 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() }; in combineTargetShuffle()
40812 LN->getMemoryVT(), LN->getMemOperand()); in combineTargetShuffle()
40839 if (LN->isSimple()) { in combineTargetShuffle()
40841 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() }; in combineTargetShuffle()
40844 LN->getPointerInfo(), LN->getOriginalAlign(), in combineTargetShuffle()
40845 LN->getMemOperand()->getFlags()); in combineTargetShuffle()
40857 if (LN->getMemoryVT().getSizeInBits() == 16) { in combineTargetShuffle()
40859 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() }; in combineTargetShuffle()
40862 LN->getMemoryVT(), LN->getMemOperand()); in combineTargetShuffle()
40881 LN->isSimple()) { in combineTargetShuffle()
40885 LN->getBasePtr(), TypeSize::getFixed(Offset), DL); in combineTargetShuffle()
40886 SDValue Ops[] = { LN->getChain(), Ptr }; in combineTargetShuffle()
40889 LN->getPointerInfo().getWithOffset(Offset), in combineTargetShuffle()
40890 LN->getOriginalAlign(), in combineTargetShuffle()
40891 LN->getMemOperand()->getFlags()); in combineTargetShuffle()
40900 // vbroadcast(vzload X) -> vbroadcast_load X in combineTargetShuffle()
40903 if (LN->getMemoryVT().getSizeInBits() == VT.getScalarSizeInBits()) { in combineTargetShuffle()
40905 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() }; in combineTargetShuffle()
40908 LN->getMemoryVT(), LN->getMemOperand()); in combineTargetShuffle()
40916 // vbroadcast(vector load X) -> vbroadcast_load in combineTargetShuffle()
40922 if (LN->isSimple()) { in combineTargetShuffle()
40924 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()}; in combineTargetShuffle()
40927 LN->getPointerInfo(), LN->getOriginalAlign(), in combineTargetShuffle()
40928 LN->getMemOperand()->getFlags()); in combineTargetShuffle()
40959 if (VT.getScalarSizeInBits() == LN->getMemoryVT().getSizeInBits()) { in combineTargetShuffle()
40961 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()}; in combineTargetShuffle()
40964 LN->getMemoryVT(), LN->getMemOperand()); in combineTargetShuffle()
40991 // vzext_movl (scalar_to_vector C) --> load [C,0...] in combineTargetShuffle()
40994 // Create a vector constant - scalar constant followed by zeros. in combineTargetShuffle()
41000 ConstantVec[0] = const_cast<ConstantInt *>(C->getConstantIntValue()); in combineTargetShuffle()
41007 Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign(); in combineTargetShuffle()
41016 // 128-bit scalar_to_vector. This reduces the number of isel patterns. in combineTargetShuffle()
41042 // blend(bitcast(x),bitcast(y)) -> bitcast(blend(x,y)) to narrower types. in combineTargetShuffle()
41061 // --> m3 = blend(m1,m2) in combineTargetShuffle()
41097 // Fold shufps(shuffle(x),shuffle(y)) -> shufps(x,y). in combineTargetShuffle()
41131 // vpermi(bitcast(x)) -> bitcast(vpermi(x)) for same number of elements. in combineTargetShuffle()
41146 // If we're permuting the upper 256-bits subvectors of a concatenation, then in combineTargetShuffle()
41149 // 512-bit mask uses 4 x i2 indices - if the msb is always set then only the in combineTargetShuffle()
41151 SDValue LHS = N->getOperand(0); in combineTargetShuffle()
41152 SDValue RHS = N->getOperand(1); in combineTargetShuffle()
41153 uint64_t Mask = N->getConstantOperandVal(2); in combineTargetShuffle()
41174 // Fold vperm2x128(bitcast(x),bitcast(y),c) -> bitcast(vperm2x128(x,y,c)). in combineTargetShuffle()
41175 SDValue LHS = N->getOperand(0); in combineTargetShuffle()
41176 SDValue RHS = N->getOperand(1); in combineTargetShuffle()
41184 N->getOperand(2))); in combineTargetShuffle()
41188 // Fold vperm2x128(op(),op()) -> op(vperm2x128(),vperm2x128()). in combineTargetShuffle()
41193 // vperm2x128(concat(X,Y),concat(Z,W)) --> concat X,Y etc. in combineTargetShuffle()
41225 if (N0->hasOneUse()) { in combineTargetShuffle()
41260 // MOVS*(N0, OP(N0, N1)) --> MOVS*(N0, SCALAR_TO_VECTOR(OP(N0[0], N1[0]))) in combineTargetShuffle()
41309 // Zero/UNDEF insertion - zero out element and remove dependency. in combineTargetShuffle()
41375 if (MemIntr->getMemoryVT().getScalarSizeInBits() == 32) { in combineTargetShuffle()
41376 SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(), in combineTargetShuffle()
41377 MemIntr->getBasePtr(), in combineTargetShuffle()
41378 MemIntr->getMemOperand()); in combineTargetShuffle()
41420 // Nuke no-op shuffles that show up after combining. in combineTargetShuffle()
41434 // combine with more operations. Note that it has to at least flip the in combineTargetShuffle()
41435 // dwords as otherwise it would have been removed as a no-op. in combineTargetShuffle()
41450 // only works when we have a PSHUFD followed by two half-shuffles. in combineTargetShuffle()
41499 int ParitySrc[2] = {-1, -1}; in isAddSubOrSubAddMask()
41538 EVT VT = N->getValueType(0); in isAddSubOrSubAdd()
41544 // We only handle target-independent shuffles. in isAddSubOrSubAdd()
41547 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) in isAddSubOrSubAdd()
41550 SDValue V1 = N->getOperand(0); in isAddSubOrSubAdd()
41551 SDValue V2 = N->getOperand(1); in isAddSubOrSubAdd()
41560 if (!V1->hasOneUse() || !V2->hasOneUse()) in isAddSubOrSubAdd()
41567 LHS = V1->getOperand(0); RHS = V1->getOperand(1); in isAddSubOrSubAdd()
41568 if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) && in isAddSubOrSubAdd()
41569 (V2->getOperand(0) != RHS || V2->getOperand(1) != LHS)) in isAddSubOrSubAdd()
41573 LHS = V2->getOperand(0); RHS = V2->getOperand(1); in isAddSubOrSubAdd()
41574 if ((V1->getOperand(0) != LHS || V1->getOperand(1) != RHS) && in isAddSubOrSubAdd()
41575 (V1->getOperand(0) != RHS || V1->getOperand(1) != LHS)) in isAddSubOrSubAdd()
41579 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask(); in isAddSubOrSubAdd()
41585 IsSubAdd = Op0Even ? V1->getOpcode() == ISD::FADD in isAddSubOrSubAdd()
41586 : V2->getOpcode() == ISD::FADD; in isAddSubOrSubAdd()
41597 // We only handle target-independent shuffles. in combineShuffleToFMAddSub()
41600 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) in combineShuffleToFMAddSub()
41603 MVT VT = N->getSimpleValueType(0); in combineShuffleToFMAddSub()
41609 SDValue Op0 = N->getOperand(0); in combineShuffleToFMAddSub()
41610 SDValue Op1 = N->getOperand(1); in combineShuffleToFMAddSub()
41622 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask(); in combineShuffleToFMAddSub()
41634 /// Try to combine a shuffle into a target-specific add-sub or
41635 /// mul-add-sub node.
41647 MVT VT = N->getSimpleValueType(0); in combineShuffleToAddSubOrFMAddSub()
41659 // Do not generate X86ISD::ADDSUB node for 512-bit types even though in combineShuffleToAddSubOrFMAddSub()
41661 // X86 targets with 512-bit ADDSUB instructions! in combineShuffleToAddSubOrFMAddSub()
41676 // if we can express this as a single-source shuffle, that's preferable.
41683 EVT VT = N->getValueType(0); in combineShuffleOfConcatUndef()
41685 // We only care about shuffles of 128/256-bit vectors of 32/64-bit values. in combineShuffleOfConcatUndef()
41695 SDValue N0 = N->getOperand(0); in combineShuffleOfConcatUndef()
41696 SDValue N1 = N->getOperand(1); in combineShuffleOfConcatUndef()
41711 for (int Elt : SVOp->getMask()) in combineShuffleOfConcatUndef()
41712 Mask.push_back(Elt < NumElts ? Elt : (Elt - NumElts / 2)); in combineShuffleOfConcatUndef()
41723 EVT VT = Shuf->getValueType(0); in narrowShuffle()
41724 if (!DAG.getTargetLoweringInfo().isTypeLegal(Shuf->getValueType(0))) in narrowShuffle()
41730 ArrayRef<int> Mask = Shuf->getMask(); in narrowShuffle()
41735 // (half-index output is 0 or 2). in narrowShuffle()
41742 // Create a half-width shuffle to replace the unnecessarily wide shuffle. in narrowShuffle()
41744 // subregister (zmm<->ymm or ymm<->xmm) ops. That leaves us with a shuffle in narrowShuffle()
41747 return getShuffleHalfVectors(SDLoc(Shuf), Shuf->getOperand(0), in narrowShuffle()
41748 Shuf->getOperand(1), HalfMask, HalfIdx1, in narrowShuffle()
41762 EVT VT = N->getValueType(0); in combineShuffle()
41783 if (isTargetShuffle(N->getOpcode())) { in combineShuffle()
41789 // instructions into higher-order shuffles. We do this after combining in combineShuffle()
41797 // TODO - merge this into combineX86ShufflesRecursively. in combineShuffle()
41802 // Canonicalize SHUFFLE(UNARYOP(X)) -> UNARYOP(SHUFFLE(X)). in combineShuffle()
41803 // Canonicalize SHUFFLE(BINOP(X,Y)) -> BINOP(SHUFFLE(X),SHUFFLE(Y)). in combineShuffle()
41838 if (!Load || !Load->getBasePtr().hasOneUse()) in SimplifyDemandedVectorEltsForTargetShuffle()
41845 Type *CTy = C->getType(); in SimplifyDemandedVectorEltsForTargetShuffle()
41846 if (!CTy->isVectorTy() || in SimplifyDemandedVectorEltsForTargetShuffle()
41847 CTy->getPrimitiveSizeInBits() != Mask.getValueSizeInBits()) in SimplifyDemandedVectorEltsForTargetShuffle()
41850 // Handle scaling for i64 elements on 32-bit targets. in SimplifyDemandedVectorEltsForTargetShuffle()
41851 unsigned NumCstElts = cast<FixedVectorType>(CTy)->getNumElements(); in SimplifyDemandedVectorEltsForTargetShuffle()
41860 Constant *Elt = C->getAggregateElement(i); in SimplifyDemandedVectorEltsForTargetShuffle()
41862 ConstVecOps.push_back(UndefValue::get(Elt->getType())); in SimplifyDemandedVectorEltsForTargetShuffle()
41878 Load->getAlign()); in SimplifyDemandedVectorEltsForTargetShuffle()
41963 // We only need the bottom 64-bits of the (128-bit) shift amount. in SimplifyDemandedVectorEltsForTargetNode()
41969 // only the bottom 64-bits are only ever used. in SimplifyDemandedVectorEltsForTargetNode()
41970 bool AssumeSingleUse = llvm::all_of(Amt->uses(), [&Amt](SDNode *Use) { in SimplifyDemandedVectorEltsForTargetNode()
41971 unsigned UseOpc = Use->getOpcode(); in SimplifyDemandedVectorEltsForTargetNode()
41974 Use->getOperand(0) != Amt; in SimplifyDemandedVectorEltsForTargetNode()
41994 // Fold shift(0,x) -> 0 in SimplifyDemandedVectorEltsForTargetNode()
42020 // Fold shift(0,x) -> 0 in SimplifyDemandedVectorEltsForTargetNode()
42049 assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount"); in SimplifyDemandedVectorEltsForTargetNode()
42050 unsigned ShiftAmt = Amt->getZExtValue(); in SimplifyDemandedVectorEltsForTargetNode()
42062 int Diff = ShiftAmt - C1; in SimplifyDemandedVectorEltsForTargetNode()
42064 Diff = -Diff; in SimplifyDemandedVectorEltsForTargetNode()
42088 assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount"); in SimplifyDemandedVectorEltsForTargetNode()
42089 unsigned ShiftAmt = Amt->getZExtValue(); in SimplifyDemandedVectorEltsForTargetNode()
42101 int Diff = ShiftAmt - C1; in SimplifyDemandedVectorEltsForTargetNode()
42103 Diff = -Diff; in SimplifyDemandedVectorEltsForTargetNode()
42144 // We can't assume an undef src element gives an undef dst - the in SimplifyDemandedVectorEltsForTargetNode()
42215 // TODO - pass on known zero/undef. in SimplifyDemandedVectorEltsForTargetNode()
42218 // TODO - we should do this for all target/faux shuffles ops. in SimplifyDemandedVectorEltsForTargetNode()
42252 // TODO - pass on known zero/undef. in SimplifyDemandedVectorEltsForTargetNode()
42330 SDValue Elt = TLO.DAG.getLoad(SVT, DL, Mem->getChain(), Mem->getBasePtr(), in SimplifyDemandedVectorEltsForTargetNode()
42331 Mem->getMemOperand()); in SimplifyDemandedVectorEltsForTargetNode()
42355 // TODO - we should do this for all target/faux shuffles ops. in SimplifyDemandedVectorEltsForTargetNode()
42381 // For 256/512-bit ops that are 128/256-bit ops glued together, if we do not in SimplifyDemandedVectorEltsForTargetNode()
42382 // demand any of the high elements, then narrow the op to 128/256-bits: e.g. in SimplifyDemandedVectorEltsForTargetNode()
42383 // (op ymm0, ymm1) --> insert undef, (op xmm0, xmm1), 0 in SimplifyDemandedVectorEltsForTargetNode()
42389 // See if 512-bit ops only use the bottom 128-bits. in SimplifyDemandedVectorEltsForTargetNode()
42412 SDValue Ops[] = {MemIntr->getOperand(0), MemIntr->getOperand(1)}; in SimplifyDemandedVectorEltsForTargetNode()
42414 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MemIntr->getMemoryVT(), in SimplifyDemandedVectorEltsForTargetNode()
42415 MemIntr->getMemOperand()); in SimplifyDemandedVectorEltsForTargetNode()
42424 EVT MemVT = MemIntr->getMemoryVT(); in SimplifyDemandedVectorEltsForTargetNode()
42428 TLO.DAG.getLoad(MemVT, DL, MemIntr->getChain(), in SimplifyDemandedVectorEltsForTargetNode()
42429 MemIntr->getBasePtr(), MemIntr->getMemOperand()); in SimplifyDemandedVectorEltsForTargetNode()
42506 // (Non-Lane Crossing) Target Shuffles. in SimplifyDemandedVectorEltsForTargetNode()
42528 // Horizontal Ops. in SimplifyDemandedVectorEltsForTargetNode()
42610 int M = OpMask[i] - Lo; in SimplifyDemandedVectorEltsForTargetNode()
42615 // TODO - Propagate input undef/zero elts. in SimplifyDemandedVectorEltsForTargetNode()
42625 // can handle - so pretend its Depth == 0 again, and reduce the max depth in SimplifyDemandedVectorEltsForTargetNode()
42638 {Op}, 0, Op, DemandedMask, {}, 0, X86::MaxShuffleCombineDepth - Depth, in SimplifyDemandedVectorEltsForTargetNode()
42676 // Don't mask bits on 32-bit AVX512 targets which might lose a broadcast. in SimplifyDemandedBitsForTargetNode()
42695 // PMULUDQ(X,1) -> AND(X,(1<<32)-1) 'getZeroExtendInReg'. in SimplifyDemandedBitsForTargetNode()
42756 int Diff = ShAmt - Shift2Amt; in SimplifyDemandedBitsForTargetNode()
42771 unsigned UpperDemandedBits = BitWidth - OriginalDemandedBits.countr_zero(); in SimplifyDemandedBitsForTargetNode()
42772 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= UpperDemandedBits) in SimplifyDemandedBitsForTargetNode()
42808 unsigned ShAmt = Op1->getAsZExtVal(); in SimplifyDemandedBitsForTargetNode()
42818 // fold (VSRAI (VSHLI X, C1), C1) --> X iff NumSignBits(X) > C1 in SimplifyDemandedBitsForTargetNode()
42842 if (Known.Zero[BitWidth - ShAmt - 1] || in SimplifyDemandedBitsForTargetNode()
42848 if (Known.One[BitWidth - ShAmt - 1]) in SimplifyDemandedBitsForTargetNode()
42881 if (CIdx && CIdx->getAPIntValue().ult(NumVecElts)) { in SimplifyDemandedBitsForTargetNode()
42882 unsigned Idx = CIdx->getZExtValue(); in SimplifyDemandedBitsForTargetNode()
42886 // bits from the implict zext - simplify to zero. in SimplifyDemandedBitsForTargetNode()
42919 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) { in SimplifyDemandedBitsForTargetNode()
42920 unsigned Idx = CIdx->getZExtValue(); in SimplifyDemandedBitsForTargetNode()
42946 // TODO - add known bits handling. in SimplifyDemandedBitsForTargetNode()
42960 // Attempt to avoid multi-use ops if we don't need anything from them. in SimplifyDemandedBitsForTargetNode()
42971 // TODO - add general PACKSS/PACKUS SimplifyDemandedBits support. in SimplifyDemandedBitsForTargetNode()
42986 Src->hasOneUse()) { in SimplifyDemandedBitsForTargetNode()
42998 // icmp sgt(0, R) == ashr(R, BitWidth-1). in SimplifyDemandedBitsForTargetNode()
43014 // See if we only demand bits from the lower 128-bit vector. in SimplifyDemandedBitsForTargetNode()
43029 Known.Zero.setHighBits(BitWidth - NumElts); in SimplifyDemandedBitsForTargetNode()
43038 if (KnownSrc.One[SrcBits - 1]) in SimplifyDemandedBitsForTargetNode()
43040 else if (KnownSrc.Zero[SrcBits - 1]) in SimplifyDemandedBitsForTargetNode()
43043 // Attempt to avoid multi-use os if we don't need anything from it. in SimplifyDemandedBitsForTargetNode()
43060 bool AssumeSingleUse = (Op0 == Op1) && Op->isOnlyUserOf(Op0.getNode()); in SimplifyDemandedBitsForTargetNode()
43084 // Only bottom 16-bits of the control bits are required. in SimplifyDemandedBitsForTargetNode()
43087 uint64_t Val1 = Cst1->getZExtValue(); in SimplifyDemandedBitsForTargetNode()
43096 unsigned Shift = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 0); in SimplifyDemandedBitsForTargetNode()
43097 unsigned Length = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 8); in SimplifyDemandedBitsForTargetNode()
43134 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ); in SimplifyDemandedBitsForTargetNode()
43152 // The result will have at least as many trailing zeros as the non-mask in SimplifyDemandedBitsForTargetNode()
43177 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) && in SimplifyMultipleUseDemandedBitsForTargetNode()
43178 !DemandedElts[CIdx->getZExtValue()]) in SimplifyMultipleUseDemandedBitsForTargetNode()
43189 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero(); in SimplifyMultipleUseDemandedBitsForTargetNode()
43190 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= UpperDemandedBits) in SimplifyMultipleUseDemandedBitsForTargetNode()
43201 // icmp sgt(0, R) == ashr(R, BitWidth-1). in SimplifyMultipleUseDemandedBitsForTargetNode()
43391 // Helper to flip between AND/OR/XOR opcodes and their X86ISD FP equivalents.
43394 // clang-format off in getAltBitOpcode()
43399 // clang-format on in getAltBitOpcode()
43404 // Helper to adjust v4i32 MOVMSK expansion to work with SSE1-only targets.
43415 cast<CondCodeSDNode>(Src.getOperand(2))->get() == ISD::SETLT) { in adjustBitcastSrcVectorSSE1()
43466 // ->
43499 cast<CondCodeSDNode>(Src.getOperand(2))->get() == ISD::SETLT && in combineBitcastvxi1()
43508 // With AVX512 vxi1 types are legal and we prefer using k-regs. in combineBitcastvxi1()
43532 // v8f64. So all legal 128-bit and 256-bit vectors are covered except for in combineBitcastvxi1()
43538 // avoid sign-extending to this type entirely. in combineBitcastvxi1()
43552 // sign-extend to a 256-bit operation to avoid truncation. in combineBitcastvxi1()
43562 // sign-extend to a 256-bit operation to match the compare. in combineBitcastvxi1()
43563 // If the setcc operand is 128-bit, prefer sign-extending to 128-bit over in combineBitcastvxi1()
43564 // 256-bit because the shuffle is cheaper than sign extending the result of in combineBitcastvxi1()
43575 // it is not profitable to sign-extend to 256-bit because this will in combineBitcastvxi1()
43576 // require an extra cross-lane shuffle which is more expensive than in combineBitcastvxi1()
43577 // truncating the result of the compare to 128-bits. in combineBitcastvxi1()
43629 if (!In.isUndef() && (In->getAsZExtVal() & 0x1)) in combinevXi1ConstantToInteger()
43639 assert(N->getOpcode() == ISD::BITCAST && "Expected a bitcast"); in combineCastedMaskArithmetic()
43644 // Only do this if we have k-registers. in combineCastedMaskArithmetic()
43648 EVT DstVT = N->getValueType(0); in combineCastedMaskArithmetic()
43649 SDValue Op = N->getOperand(0); in combineCastedMaskArithmetic()
43681 // If the RHS is a vXi1 build vector, this is a good reason to flip too. in combineCastedMaskArithmetic()
43695 unsigned NumElts = BV->getNumOperands(); in createMMXBuildVector()
43696 SDValue Splat = BV->getSplatValue(); in createMMXBuildVector()
43720 // Broadcast - use (PUNPCKL+)PSHUFW to broadcast single element. in createMMXBuildVector()
43728 // Unpack v8i8 to splat i8 elements to lowest 16-bits. in createMMXBuildVector()
43736 // Use PSHUFW to repeat 16-bit elements. in createMMXBuildVector()
43747 Ops.push_back(CreateMMXElement(BV->getOperand(i))); in createMMXBuildVector()
43792 if (C->isZero()) in combineBitcastToBoolVector()
43794 if (C->isAllOnes()) in combineBitcastToBoolVector()
43848 DAG.getTargetConstant(Amt->getZExtValue(), DL, MVT::i8)); in combineBitcastToBoolVector()
43864 SDValue N0 = N->getOperand(0); in combineBitcast()
43865 EVT VT = N->getValueType(0); in combineBitcast()
43871 // -> in combineBitcast()
43902 SDValue LastOp = N0.getOperand(N0.getNumOperands() - 1); in combineBitcast()
43906 SmallVector<SDValue, 4> Ops(N0->op_begin(), N0->op_end()); in combineBitcast()
43951 unsigned MemSize = BCast->getMemoryVT().getScalarSizeInBits(); in combineBitcast()
43961 SDValue Ops[] = { BCast->getChain(), BCast->getBasePtr() }; in combineBitcast()
43964 MemVT, BCast->getMemOperand()); in combineBitcast()
43972 // avoiding store-load conversions. in combineBitcast()
43981 // Handle zero-extension of i32 with MOVD. in combineBitcast()
43986 // TODO - investigate supporting sext 32-bit immediates on x86_64. in combineBitcast()
44010 // Detect bitcasts of 64-bit build vectors and convert to a in combineBitcast()
44049 if (C->isAllOnes()) in combineBitcast()
44051 if (C->isZero()) in combineBitcast()
44057 // Turn it into a sign bit compare that produces a k-register. This avoids in combineBitcast()
44098 // remove GPR<->K-register crossings. in combineBitcast()
44103 // floating-point operand into a floating-point logic operation. This may in combineBitcast()
44109 // clang-format off in combineBitcast()
44114 // clang-format on in combineBitcast()
44129 // bitcast(logic(bitcast(X), Y)) --> logic'(X, bitcast(Y)) in combineBitcast()
44138 // bitcast(logic(X, bitcast(Y))) --> logic'(bitcast(X), Y) in combineBitcast()
44161 auto IsFreeTruncation = [](SDValue &Op) -> bool { in detectExtMul()
44168 return (BV && BV->isConstant()); in detectExtMul()
44186 SDValue AbsOp1 = Abs->getOperand(0); in detectZextAbsDiff()
44193 // Check if the operands of the sub are zero-extended from vectors of i8. in detectZextAbsDiff()
44223 // "Zero-extend" the i8 vectors. This is not a per-element zext, rather we in createVPDPBUSD()
44256 // "Zero-extend" the i8 vectors. This is not a per-element zext, rather we in createPSADBW()
44277 // Attempt to replace an min/max v8i16/v16i8 horizontal reduction with
44285 EVT ExtractVT = Extract->getValueType(0); in combineMinMaxReduction()
44289 // Check for SMAX/SMIN/UMAX/UMIN horizontal reduction patterns. in combineMinMaxReduction()
44304 // First, reduce the source down to 128-bit, applying BinOp to lo/hi. in combineMinMaxReduction()
44316 // to flip the value accordingly. in combineMinMaxReduction()
44331 // v16i8 UMIN will leave the upper element as zero, performing zero-extension in combineMinMaxReduction()
44352 // Attempt to replace an all_of/any_of/parity style horizontal reduction with a MOVMSK.
44359 EVT ExtractVT = Extract->getValueType(0); in combinePredicateReduction()
44365 // Check for OR(any_of)/AND(all_of)/XOR(parity) horizontal reduction patterns. in combinePredicateReduction()
44387 // Special case for (pre-legalization) vXi1 reductions. in combinePredicateReduction()
44391 ISD::CondCode CC = cast<CondCodeSDNode>(Match.getOperand(2))->get(); in combinePredicateReduction()
44394 // For all_of(setcc(x,y,eq)) - use (iX)x == (iX)y. in combinePredicateReduction()
44395 // For any_of(setcc(x,y,ne)) - use (iX)x != (iX)y. in combinePredicateReduction()
44425 // FIXME: Better handling of k-registers or 512-bit vectors? in combinePredicateReduction()
44466 // parity -> (PARITY(MOVMSK X)) in combinePredicateReduction()
44474 // any_of -> MOVMSK != 0 in combinePredicateReduction()
44478 // all_of -> MOVMSK == ((1 << NumElts) - 1) in combinePredicateReduction()
44485 // negate to get the final 0/-1 mask value. in combinePredicateReduction()
44497 EVT ExtractVT = Extract->getValueType(0); in combineVPDPBUSDPattern()
44503 EVT VT = Extract->getOperand(0).getValueType(); in combineVPDPBUSDPattern()
44512 // done by vpdpbusd compute a signed 16-bit product that will be sign extended in combineVPDPBUSDPattern()
44543 for (unsigned i = Stages - StageBias; i > 0; --i) { in combineVPDPBUSDPattern()
44544 SmallVector<int, 16> Mask(DpElems, -1); in combineVPDPBUSDPattern()
44545 for (unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j) in combineVPDPBUSDPattern()
44560 Extract->getOperand(1)); in combineVPDPBUSDPattern()
44569 EVT ExtractVT = Extract->getValueType(0); in combineBasicSADPattern()
44575 EVT VT = Extract->getOperand(0).getValueType(); in combineBasicSADPattern()
44597 // abs-diff pattern. in combineBasicSADPattern()
44601 // Check whether we have an abs-diff pattern feeding into the select. in combineBasicSADPattern()
44617 for(unsigned i = Stages - 3; i > 0; --i) { in combineBasicSADPattern()
44618 SmallVector<int, 16> Mask(SadElems, -1); in combineBasicSADPattern()
44619 for(unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j) in combineBasicSADPattern()
44634 Extract->getOperand(1)); in combineBasicSADPattern()
44638 // integer, that requires a potentially expensive XMM -> GPR transfer.
44643 // to a single-use of the loaded vector. For the reasons above, we
44649 assert(N->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in combineExtractFromVectorLoad()
44653 EVT VT = N->getValueType(0); in combineExtractFromVectorLoad()
44655 bool LikelyUsedAsVector = any_of(N->uses(), [](SDNode *Use) { in combineExtractFromVectorLoad()
44656 return Use->getOpcode() == ISD::STORE || in combineExtractFromVectorLoad()
44657 Use->getOpcode() == ISD::INSERT_VECTOR_ELT || in combineExtractFromVectorLoad()
44658 Use->getOpcode() == ISD::SCALAR_TO_VECTOR; in combineExtractFromVectorLoad()
44665 DCI.isAfterLegalizeDAG() && !LikelyUsedAsVector && LoadVec->isSimple()) { in combineExtractFromVectorLoad()
44667 DAG, LoadVec->getBasePtr(), VecVT, DAG.getVectorIdxConstant(Idx, dl)); in combineExtractFromVectorLoad()
44669 MachinePointerInfo MPI = LoadVec->getPointerInfo().getWithOffset(PtrOff); in combineExtractFromVectorLoad()
44670 Align Alignment = commonAlignment(LoadVec->getAlign(), PtrOff); in combineExtractFromVectorLoad()
44672 DAG.getLoad(VT, dl, LoadVec->getChain(), NewPtr, MPI, Alignment, in combineExtractFromVectorLoad()
44673 LoadVec->getMemOperand()->getFlags(), LoadVec->getAAInfo()); in combineExtractFromVectorLoad()
44690 SDValue Src = N->getOperand(0); in combineExtractWithShuffle()
44691 SDValue Idx = N->getOperand(1); in combineExtractWithShuffle()
44693 EVT VT = N->getValueType(0); in combineExtractWithShuffle()
44703 const APInt &IdxC = N->getConstantOperandAPInt(1); in combineExtractWithShuffle()
44717 // TODO support non-zero offsets. in combineExtractWithShuffle()
44731 if (MemIntr->getMemoryVT().getSizeInBits() == SrcBCWidth && in combineExtractWithShuffle()
44733 SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(), in combineExtractWithShuffle()
44734 MemIntr->getBasePtr(), in combineExtractWithShuffle()
44735 MemIntr->getPointerInfo(), in combineExtractWithShuffle()
44736 MemIntr->getOriginalAlign(), in combineExtractWithShuffle()
44737 MemIntr->getMemOperand()->getFlags()); in combineExtractWithShuffle()
44772 return DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(ExtractVT, Src), in combineExtractWithShuffle()
44776 // We can only legally extract other elements from 128-bit vectors and in in combineExtractWithShuffle()
44777 // certain circumstances, depending on SSE-level. in combineExtractWithShuffle()
44787 unsigned LaneOffset = (Idx & ~(NumEltsPerLane - 1)) * EltSizeInBits; in combineExtractWithShuffle()
44791 Idx &= (NumEltsPerLane - 1); in combineExtractWithShuffle()
44844 // If narrowing/widening failed, see if we can extract+zero-extend. in combineExtractWithShuffle()
44855 if (!isUndefOrZeroInRange(Mask, ScaledIdx + 1, Scale - 1)) in combineExtractWithShuffle()
44877 if (N->getOpcode() == ISD::EXTRACT_VECTOR_ELT && ExtractVT == SrcVT) in combineExtractWithShuffle()
44889 assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Expected extract"); in scalarizeExtEltFP()
44890 SDValue Vec = ExtElt->getOperand(0); in scalarizeExtEltFP()
44891 SDValue Index = ExtElt->getOperand(1); in scalarizeExtEltFP()
44892 EVT VT = ExtElt->getValueType(0); in scalarizeExtEltFP()
44896 // non-zero element because the shuffle+scalar op will be cheaper? in scalarizeExtEltFP()
44901 // extract, the condition code), so deal with those as a special-case. in scalarizeExtEltFP()
44907 // extract (setcc X, Y, CC), 0 --> setcc (extract X, 0), (extract Y, 0), CC in scalarizeExtEltFP()
44930 // ext (sel Cond, X, Y), 0 --> sel (ext Cond, 0), (ext X, 0), (ext Y, 0) in scalarizeExtEltFP()
44942 // TODO: This switch could include FNEG and the x86-specific FP logic ops in scalarizeExtEltFP()
44973 // extract (fp X, Y, ...), 0 --> fp (extract X, 0), (extract Y, 0), ... in scalarizeExtEltFP()
44976 for (SDValue Op : Vec->ops()) in scalarizeExtEltFP()
44987 /// into horizontal ops.
44990 assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unexpected caller"); in combineArithReduction()
45002 SDValue Index = ExtElt->getOperand(1); in combineArithReduction()
45006 EVT VT = ExtElt->getValueType(0); in combineArithReduction()
45015 // Extend v4i8/v8i8 vector to v16i8, with undef upper 64-bits. in combineArithReduction()
45033 // vXi8 mul reduction - promote to vXi16 mul reduction. in combineArithReduction()
45056 {4, 5, 6, 7, -1, -1, -1, -1})); in combineArithReduction()
45059 {2, 3, -1, -1, -1, -1, -1, -1})); in combineArithReduction()
45062 {1, -1, -1, -1, -1, -1, -1, -1})); in combineArithReduction()
45067 // vXi8 add reduction - sub 128-bit vector. in combineArithReduction()
45076 // Must be a >=128-bit vector with pow2 elements. in combineArithReduction()
45080 // vXi8 add reduction - sum lo/hi halves then use PSADBW. in combineArithReduction()
45092 {8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1}); in combineArithReduction()
45101 // If the source vector values are 0-255, then we can use PSADBW to in combineArithReduction()
45138 SDValue RdxHi = DAG.getVectorShuffle(MVT::v2i64, DL, Rdx, Rdx, {1, -1}); in combineArithReduction()
45153 // 256-bit horizontal instructions operate on 128-bit chunks rather than in combineArithReduction()
45156 // TODO: We could extend this to handle 512-bit or even longer vectors. in combineArithReduction()
45169 // extract (add (shuf X), X), 0 --> extract (hadd X, X), 0 in combineArithReduction()
45180 /// scalars back, while for x64 we should use 64-bit extracts and shifts.
45187 SDValue InputVector = N->getOperand(0); in combineExtractVectorElt()
45188 SDValue EltIdx = N->getOperand(1); in combineExtractVectorElt()
45192 EVT VT = N->getValueType(0); in combineExtractVectorElt()
45194 bool IsPextr = N->getOpcode() != ISD::EXTRACT_VECTOR_ELT; in combineExtractVectorElt()
45199 if (CIdx && CIdx->getAPIntValue().uge(NumSrcElts)) in combineExtractVectorElt()
45210 uint64_t Idx = CIdx->getZExtValue(); in combineExtractVectorElt()
45216 // Convert extract_element(bitcast(<X x i1>) -> bitcast(extract_subvector()). in combineExtractVectorElt()
45224 DAG.getIntPtrConstant(CIdx->getZExtValue() * NumEltBits, dl)); in combineExtractVectorElt()
45235 // PEXTR*(PINSR*(v, s, c), c) -> s (with implicit zext handling). in combineExtractVectorElt()
45246 // TODO - Remove this once we can handle the implicit zero-extension of in combineExtractVectorElt()
45269 // pre-legalization, in combineExtractVectorElt()
45276 // Attempt to replace an all_of/any_of horizontal reduction with a MOVMSK. in combineExtractVectorElt()
45293 N, InputVector.getValueType(), InputVector, CIdx->getZExtValue(), in combineExtractVectorElt()
45312 if (Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in combineExtractVectorElt()
45313 Use->getOperand(0).getResNo() == ResNo && in combineExtractVectorElt()
45314 Use->getValueType(0) == MVT::i1) { in combineExtractVectorElt()
45316 IsVar |= !isa<ConstantSDNode>(Use->getOperand(1)); in combineExtractVectorElt()
45322 if (all_of(InputVector->uses(), IsBoolExtract) && in combineExtractVectorElt()
45328 // extractelement vXi1 X, MaskIdx --> ((movmsk X) & Mask) == Mask in combineExtractVectorElt()
45330 SDValue MaskIdx = DAG.getZExtOrTrunc(Use->getOperand(1), dl, MVT::i8); in combineExtractVectorElt()
45342 // Attempt to fold extract(trunc(x),c) -> trunc(extract(x,c)). in combineExtractVectorElt()
45373 // Input type must be extending a bool vector (bit-casted from a scalar in combineToExtendBoolVectorInReg()
45395 // must split it down into sub-sections for broadcasting. For example: in combineToExtendBoolVectorInReg()
45396 // i16 -> v16i8 (i16 -> v8i16 -> v16i8) with 2 sub-sections. in combineToExtendBoolVectorInReg()
45397 // i32 -> v32i8 (i32 -> v8i32 -> v32i8) with 4 sub-sections. in combineToExtendBoolVectorInReg()
45422 // For smaller scalar integers, we can simply any-extend it to the vector in combineToExtendBoolVectorInReg()
45447 // zero-extension. in combineToExtendBoolVectorInReg()
45451 DAG.getConstant(EltSizeInBits - 1, DL, VT)); in combineToExtendBoolVectorInReg()
45454 /// If a vector select has an operand that is -1 or 0, try to simplify the
45461 SDValue Cond = N->getOperand(0); in combineVSelectWithAllOnesOrZeros()
45462 SDValue LHS = N->getOperand(1); in combineVSelectWithAllOnesOrZeros()
45463 SDValue RHS = N->getOperand(2); in combineVSelectWithAllOnesOrZeros()
45468 if (N->getOpcode() != ISD::VSELECT) in combineVSelectWithAllOnesOrZeros()
45491 // vector floating-point selects. in combineVSelectWithAllOnesOrZeros()
45509 cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType()); in combineVSelectWithAllOnesOrZeros()
45522 // vselect Cond, 111..., 000... -> Cond in combineVSelectWithAllOnesOrZeros()
45529 // vselect Cond, 111..., X -> or Cond, X in combineVSelectWithAllOnesOrZeros()
45536 // vselect Cond, X, 000... -> and Cond, X in combineVSelectWithAllOnesOrZeros()
45543 // vselect Cond, 000..., X -> andn Cond, X in combineVSelectWithAllOnesOrZeros()
45547 // The canonical form differs for i1 vectors - x86andnp is not used in combineVSelectWithAllOnesOrZeros()
45560 /// and concatenate the result to eliminate a wide (256-bit) vector instruction:
45561 /// vselect Cond, (concat T0, T1), (concat F0, F1) -->
45565 unsigned Opcode = N->getOpcode(); in narrowVectorSelect()
45569 // TODO: Split 512-bit vectors too? in narrowVectorSelect()
45570 EVT VT = N->getValueType(0); in narrowVectorSelect()
45575 SDValue Cond = N->getOperand(0); in narrowVectorSelect()
45576 SDValue TVal = N->getOperand(1); in narrowVectorSelect()
45577 SDValue FVal = N->getOperand(2); in narrowVectorSelect()
45593 SDValue Cond = N->getOperand(0); in combineSelectOfTwoConstants()
45594 SDValue LHS = N->getOperand(1); in combineSelectOfTwoConstants()
45595 SDValue RHS = N->getOperand(2); in combineSelectOfTwoConstants()
45603 EVT VT = N->getValueType(0); in combineSelectOfTwoConstants()
45608 // this with a wider condition value (post-legalization it becomes an i8), in combineSelectOfTwoConstants()
45613 // A power-of-2 multiply is just a shift. LEA also cheaply handles multiply by in combineSelectOfTwoConstants()
45615 // TODO: For constants that overflow or do not differ by power-of-2 or small in combineSelectOfTwoConstants()
45617 const APInt &TrueVal = TrueC->getAPIntValue(); in combineSelectOfTwoConstants()
45618 const APInt &FalseVal = FalseC->getAPIntValue(); in combineSelectOfTwoConstants()
45620 // We have a more efficient lowering for "(X == 0) ? Y : -1" using SBB. in combineSelectOfTwoConstants()
45623 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in combineSelectOfTwoConstants()
45646 // select Cond, TC, FC --> (zext(Cond) * (TC - FC)) + FC in combineSelectOfTwoConstants()
45649 // Multiply condition by the difference if non-one. in combineSelectOfTwoConstants()
45653 // Add the base if non-zero. in combineSelectOfTwoConstants()
45654 if (!FalseC->isZero()) in combineSelectOfTwoConstants()
45663 /// If this is a *dynamic* select (non-constant condition) and we can match
45672 SDValue Cond = N->getOperand(0); in combineVSelectToBLENDV()
45673 if ((N->getOpcode() != ISD::VSELECT && in combineVSelectToBLENDV()
45674 N->getOpcode() != X86ISD::BLENDV) || in combineVSelectToBLENDV()
45680 EVT VT = N->getValueType(0); in combineVSelectToBLENDV()
45686 // cases where a *dynamic* blend will fail even though a constant-condition in combineVSelectToBLENDV()
45689 // Potentially, we should combine constant-condition vselect nodes in combineVSelectToBLENDV()
45690 // pre-legalization into shuffles and not mark as many types as custom in combineVSelectToBLENDV()
45694 // FIXME: We don't support i16-element blends currently. We could and in combineVSelectToBLENDV()
45696 // rather than just the high bit and using an i8-element blend. in combineVSelectToBLENDV()
45705 // There are no 512-bit blend instructions that use sign bits. in combineVSelectToBLENDV()
45710 // and don't ever optimize vector selects that map to AVX512 mask-registers. in combineVSelectToBLENDV()
45715 for (SDNode::use_iterator UI = Cond->use_begin(), UE = Cond->use_end(); in combineVSelectToBLENDV()
45717 if ((UI->getOpcode() != ISD::VSELECT && in combineVSelectToBLENDV()
45718 UI->getOpcode() != X86ISD::BLENDV) || in combineVSelectToBLENDV()
45739 for (SDNode *U : Cond->uses()) { in combineVSelectToBLENDV()
45740 if (U->getOpcode() == X86ISD::BLENDV) in combineVSelectToBLENDV()
45743 SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0), in combineVSelectToBLENDV()
45744 Cond, U->getOperand(1), U->getOperand(2)); in combineVSelectToBLENDV()
45754 return DAG.getNode(X86ISD::BLENDV, DL, N->getValueType(0), V, in combineVSelectToBLENDV()
45755 N->getOperand(1), N->getOperand(2)); in combineVSelectToBLENDV()
45767 // (fNegate ? -v : v) == ((v ^ -fNegate) + fNegate)
45770 // ((M & 1) ? -X : X) == ((X ^ -(M & 1)) + (M & 1))
45771 // ( M ? -X : X) == ((X ^ M ) + (M & 1))
45782 "Mask must be zero/all-bits"); in combineLogicBlendIntoConditionalNegate()
45790 return N->getOpcode() == ISD::SUB && N->getOperand(1) == V && in combineLogicBlendIntoConditionalNegate()
45791 ISD::isBuildVectorAllZeros(N->getOperand(0).getNode()); in combineLogicBlendIntoConditionalNegate()
45808 // (vselect M, (sub (0, X), X) -> (sub (xor X, M), M) in combineLogicBlendIntoConditionalNegate()
45811 // above, -(vselect M, (sub 0, X), X), and therefore the replacement in combineLogicBlendIntoConditionalNegate()
45813 // And -(sub X, Y) is just sub (Y, X), so swapping the operands of the in combineLogicBlendIntoConditionalNegate()
45826 if (N->getOpcode() != ISD::VSELECT) in commuteSelect()
45829 SDValue Cond = N->getOperand(0); in commuteSelect()
45830 SDValue LHS = N->getOperand(1); in commuteSelect()
45831 SDValue RHS = N->getOperand(2); in commuteSelect()
45843 // (vselect M, L, R) -> (vselect ~M, R, L) in commuteSelect()
45845 ISD::getSetCCInverse(cast<CondCodeSDNode>(Cond.getOperand(2))->get(), in commuteSelect()
45852 /// Do target-specific dag combines on SELECT and VSELECT nodes.
45857 SDValue Cond = N->getOperand(0); in combineSelect()
45858 SDValue LHS = N->getOperand(1); in combineSelect()
45859 SDValue RHS = N->getOperand(2); in combineSelect()
45878 // Attempt to combine (select M, (sub 0, X), X) -> (sub (xor X, M), M). in combineSelect()
45879 // Limit this to cases of non-constant masks that createShuffleMaskFromVSELECT in combineSelect()
45891 (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::BLENDV)) { in combineSelect()
45894 N->getOpcode() == X86ISD::BLENDV)) in combineSelect()
45898 // fold vselect(cond, pshufb(x), pshufb(y)) -> or (pshufb(x), pshufb(y)) in combineSelect()
45901 if (N->getOpcode() == ISD::VSELECT && CondVT.isVector() && in combineSelect()
45933 // ignored in unsafe-math mode). in combineSelect()
45940 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in combineSelect()
46012 // Check for x CC y ? y : x -- a min/max with reversed arms. in combineSelect()
46080 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS); in combineSelect()
46087 // fold (select (setcc (and (X, 1), 0, seteq), Y, Z)) -> select(and(X, 1),Z,Y) in combineSelect()
46088 if (Subtarget.hasAVX512() && N->getOpcode() == ISD::SELECT && in combineSelect()
46090 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in combineSelect()
46114 return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS); in combineSelect()
46117 // AVX512 - Extend select with zero to merge with target shuffle. in combineSelect()
46118 // select(mask, extract_subvector(shuffle(x)), zero) --> in combineSelect()
46120 // TODO - support non target shuffles as well. in combineSelect()
46155 if (N->getOpcode() == ISD::SELECT && Cond.getOpcode() == ISD::SETCC && in combineSelect()
46160 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in combineSelect()
46163 // (x > 0) ? x : 0 -> (x >= 0) ? x : 0 in combineSelect()
46164 // (x < -1) ? x : -1 -> (x <= -1) ? x : -1 in combineSelect()
46167 // (a - b) > 0 : (a - b) ? 0 -> (a - b) >= 0 : (a - b) ? 0 in combineSelect()
46178 // (x s> 1) ? x : 1 -> (x s>= 1) ? x : 1 -> (x s> 0) ? x : 1 in combineSelect()
46179 // (x u> 1) ? x : 1 -> (x u>= 1) ? x : 1 -> (x != 0) ? x : 1 in combineSelect()
46198 // --> (select (cmpuge Cond0, Cond1), LHS, Y) in combineSelect()
46200 // --> (select (cmpsle Cond0, Cond1), LHS, Y) in combineSelect()
46206 cast<CondCodeSDNode>(InnerSetCC.getOperand(2))->get(); in combineSelect()
46212 // clang-format off in combineSelect()
46218 // clang-format on in combineSelect()
46231 if (N->getOpcode() == ISD::VSELECT && Cond.hasOneUse() && in combineSelect()
46243 if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::BITCAST && in combineSelect()
46255 // with out-of-bounds clamping. in combineSelect()
46259 // to bitwidth-1 for unsigned shifts, effectively performing a maximum left in combineSelect()
46260 // shift of bitwidth-1 positions. and returns zero for unsigned right shifts in combineSelect()
46261 // exceeding bitwidth-1. in combineSelect()
46262 if (N->getOpcode() == ISD::VSELECT) { in combineSelect()
46264 // fold select(icmp_ult(amt,BW),shl(x,amt),0) -> avx2 psllv(x,amt) in combineSelect()
46265 // fold select(icmp_ult(amt,BW),srl(x,amt),0) -> avx2 psrlv(x,amt) in combineSelect()
46276 // fold select(icmp_uge(amt,BW),0,shl(x,amt)) -> avx2 psllv(x,amt) in combineSelect()
46277 // fold select(icmp_uge(amt,BW),0,srl(x,amt)) -> avx2 psrlv(x,amt) in combineSelect()
46303 // select(~Cond, X, Y) -> select(Cond, Y, X) in combineSelect()
46306 return DAG.getNode(N->getOpcode(), DL, VT, in combineSelect()
46309 // pcmpgt(X, -1) -> pcmpgt(0, X) to help select/blendv just use the in combineSelect()
46316 return DAG.getNode(N->getOpcode(), DL, VT, Cond, RHS, LHS); in combineSelect()
46325 if (N->getOpcode() == ISD::SELECT && VT.isVector() && in combineSelect()
46355 // This can lower using a vector shift bit-hack rather than mask and compare. in combineSelect()
46357 N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC && in combineSelect()
46361 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETEQ && in combineSelect()
46363 // The 'and' mask must be composed of power-of-2 constants. in combineSelect()
46366 if (C && C->getAPIntValue().isPowerOf2()) { in combineSelect()
46367 // vselect (X & C == 0), LHS, RHS --> vselect (X & C != 0), RHS, LHS in combineSelect()
46373 // If we have a non-splat but still powers-of-2 mask, AVX1 can use pmulld in combineSelect()
46374 // and AVX2 can use vpsllv{dq}. 8-bit lacks a proper shift or multiply. in combineSelect()
46375 // 16-bit lacks a proper blendv. in combineSelect()
46383 return C->getAPIntValue().isPowerOf2(); in combineSelect()
46385 // Create a left-shift constant to get the mask bits over to the sign-bit. in combineSelect()
46390 ShlVals.push_back(EltBitWidth - 1 - in combineSelect()
46391 MaskVal->getAPIntValue().exactLogBase2()); in combineSelect()
46393 // vsel ((X & C) == 0), LHS, RHS --> vsel ((shl X, C') < 0), RHS, LHS in combineSelect()
46414 // This combine only operates on CMP-like nodes. in combineSetCCAtomicArith()
46416 (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0)))) in combineSetCCAtomicArith()
46426 // (icmp slt x, 0) -> (icmp sle (add x, 1), 0) in combineSetCCAtomicArith()
46427 // (icmp sge x, 0) -> (icmp sgt (add x, 1), 0) in combineSetCCAtomicArith()
46428 // (icmp sle x, 0) -> (icmp slt (sub x, 1), 0) in combineSetCCAtomicArith()
46429 // (icmp sgt x, 0) -> (icmp sge (sub x, 1), 0) in combineSetCCAtomicArith()
46433 // - XOR/OR/AND (if they were made to survive AtomicExpand) in combineSetCCAtomicArith()
46434 // - LHS != 1 in combineSetCCAtomicArith()
46453 APInt Addend = OpRHSC->getAPIntValue(); in combineSetCCAtomicArith()
46455 Addend = -Addend; in combineSetCCAtomicArith()
46461 APInt Comparison = CmpRHSC->getAPIntValue(); in combineSetCCAtomicArith()
46462 APInt NegAddend = -Addend; in combineSetCCAtomicArith()
46477 APInt DecComparison = Comparison - 1; in combineSetCCAtomicArith()
46499 AN->getMemOperand()); in combineSetCCAtomicArith()
46515 else if (CC == X86::COND_G && Addend == -1) in combineSetCCAtomicArith()
46517 else if (CC == X86::COND_LE && Addend == -1) in combineSetCCAtomicArith()
46540 // CMP(X,0) -> signbit test in checkSignTestSetCCCombine()
46545 // TODO: Remove one use limit once sdiv-fix regressions are fixed. in checkSignTestSetCCCombine()
46551 // OR(X,Y) -> see if only one operand contributes to the signbit. in checkSignTestSetCCCombine()
46552 // TODO: XOR(X,Y) -> see if only one operand contributes to the signbit. in checkSignTestSetCCCombine()
46600 // This combine only operates on CMP-like nodes. in checkBoolTestSetCCCombine()
46602 (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0)))) in checkBoolTestSetCCCombine()
46626 if (C->getZExtValue() == 1) { in checkBoolTestSetCCCombine()
46629 } else if (C->getZExtValue() != 0) in checkBoolTestSetCCCombine()
46639 int OpIdx = -1; in checkBoolTestSetCCCombine()
46691 if (FVal && FVal->getZExtValue() != 0) { in checkBoolTestSetCCCombine()
46692 if (FVal->getZExtValue() != 1) in checkBoolTestSetCCCombine()
46699 if (FValIsFalse && TVal->getZExtValue() != 1) in checkBoolTestSetCCCombine()
46701 if (!FValIsFalse && TVal->getZExtValue() != 0) in checkBoolTestSetCCCombine()
46720 if (Cond->getOpcode() == X86ISD::CMP) { in checkBoolTestAndOrSetCCCombine()
46721 if (!isNullConstant(Cond->getOperand(1))) in checkBoolTestAndOrSetCCCombine()
46724 Cond = Cond->getOperand(0); in checkBoolTestAndOrSetCCCombine()
46730 switch (Cond->getOpcode()) { in checkBoolTestAndOrSetCCCombine()
46738 SetCC0 = Cond->getOperand(0); in checkBoolTestAndOrSetCCCombine()
46739 SetCC1 = Cond->getOperand(1); in checkBoolTestAndOrSetCCCombine()
46746 SetCC0->getOperand(1) != SetCC1->getOperand(1)) in checkBoolTestAndOrSetCCCombine()
46749 CC0 = (X86::CondCode)SetCC0->getConstantOperandVal(0); in checkBoolTestAndOrSetCCCombine()
46750 CC1 = (X86::CondCode)SetCC1->getConstantOperandVal(0); in checkBoolTestAndOrSetCCCombine()
46751 Flags = SetCC0->getOperand(1); in checkBoolTestAndOrSetCCCombine()
46755 // When legalizing carry, we create carries via add X, -1
46781 // Do not flip "e > c", where "c" is a constant, because Cmp in combineCarryThroughADD()
46785 CarryOp1.getNode()->hasOneUse() && in combineCarryThroughADD()
46789 DAG.getNode(X86ISD::SUB, SDLoc(CarryOp1), CarryOp1->getVTList(), in combineCarryThroughADD()
46840 // testc -> testz. in combinePTESTCC()
46844 // !testc -> !testz. in combinePTESTCC()
46848 // testz -> testc. in combinePTESTCC()
46852 // !testz -> !testc. in combinePTESTCC()
46857 // testnzc -> testnzc (no change). in combinePTESTCC()
46873 // TESTC(X,~X) == TESTC(X,-1) in combinePTESTCC()
46912 // If every element is an all-sign value, see if we can use TESTP/MOVMSK in combinePTESTCC()
46949 // TESTZ(-1,X) == TESTZ(X,X) in combinePTESTCC()
46953 // TESTZ(X,-1) == TESTZ(X,X) in combinePTESTCC()
46957 // TESTZ(OR(LO(X),HI(X)),OR(LO(Y),HI(Y))) -> TESTZ(X,Y) in combinePTESTCC()
46985 // Handle eq/ne against -1 (all_of). in combineSetCCMOVMSK()
46996 const APInt &CmpVal = CmpConstant->getAPIntValue(); in combineSetCCMOVMSK()
47027 bool IsOneUse = CmpOp.getNode()->hasOneUse(); in combineSetCCMOVMSK()
47030 // signbits extend down to all the sub-elements as well. in combineSetCCMOVMSK()
47044 DAG.ComputeNumSignBits(BC) > (BCNumEltBits - NumEltBits)) { in combineSetCCMOVMSK()
47053 // MOVMSK(CONCAT(X,Y)) == 0 -> MOVMSK(OR(X,Y)). in combineSetCCMOVMSK()
47054 // MOVMSK(CONCAT(X,Y)) != 0 -> MOVMSK(OR(X,Y)). in combineSetCCMOVMSK()
47055 // MOVMSK(CONCAT(X,Y)) == -1 -> MOVMSK(AND(X,Y)). in combineSetCCMOVMSK()
47056 // MOVMSK(CONCAT(X,Y)) != -1 -> MOVMSK(AND(X,Y)). in combineSetCCMOVMSK()
47074 // MOVMSK(PCMPEQ(X,0)) == -1 -> PTESTZ(X,X). in combineSetCCMOVMSK()
47075 // MOVMSK(PCMPEQ(X,0)) != -1 -> !PTESTZ(X,X). in combineSetCCMOVMSK()
47076 // MOVMSK(PCMPEQ(X,Y)) == -1 -> PTESTZ(XOR(X,Y),XOR(X,Y)). in combineSetCCMOVMSK()
47077 // MOVMSK(PCMPEQ(X,Y)) != -1 -> !PTESTZ(XOR(X,Y),XOR(X,Y)). in combineSetCCMOVMSK()
47089 // Check for 256-bit split vector cases. in combineSetCCMOVMSK()
47117 // PMOVMSKB(PACKSSBW(X, undef)) -> PMOVMSKB(BITCAST_v16i8(X)) & 0xAAAA. in combineSetCCMOVMSK()
47131 // -> PMOVMSKB(BITCAST_v32i8(X)) & 0xAAAAAAAA. in combineSetCCMOVMSK()
47159 // MOVMSK(SHUFFLE(X,u)) -> MOVMSK(X) iff every element is referenced. in combineSetCCMOVMSK()
47190 // MOVMSKPS(V) !=/== 0 -> TESTPS(V,V) in combineSetCCMOVMSK()
47191 // MOVMSKPD(V) !=/== 0 -> TESTPD(V,V) in combineSetCCMOVMSK()
47192 // MOVMSKPS(V) !=/== -1 -> TESTPS(V,V) in combineSetCCMOVMSK()
47193 // MOVMSKPD(V) !=/== -1 -> TESTPD(V,V) in combineSetCCMOVMSK()
47244 SDValue FalseOp = N->getOperand(0); in combineCMov()
47245 SDValue TrueOp = N->getOperand(1); in combineCMov()
47246 X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2); in combineCMov()
47247 SDValue Cond = N->getOperand(3); in combineCMov()
47249 // cmov X, X, ?, ? --> X in combineCMov()
47262 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops); in combineCMov()
47273 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) { in combineCMov()
47279 // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3. Likewise for any pow2/0. in combineCMov()
47282 if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) { in combineCMov()
47286 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond); in combineCMov()
47288 unsigned ShAmt = TrueC->getAPIntValue().logBase2(); in combineCMov()
47294 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. This is efficient in combineCMov()
47296 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) { in combineCMov()
47301 FalseC->getValueType(0), Cond); in combineCMov()
47309 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { in combineCMov()
47310 APInt Diff = TrueC->getAPIntValue() - FalseC->getAPIntValue(); in combineCMov()
47311 assert(Diff.getBitWidth() == N->getValueType(0).getSizeInBits() && in combineCMov()
47333 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), in combineCMov()
47340 // Add the base if non-zero. in combineCMov()
47341 if (FalseC->getAPIntValue() != 0) in combineCMov()
47351 // (select (x != c), e, c) -> select (x != c), e, x), in combineCMov()
47352 // (select (x == c), c, e) -> select (x == c), x, e) in combineCMov()
47356 // The rationale for this change is that the conditional-move from a constant in combineCMov()
47357 // needs two instructions, however, conditional-move from a register needs in combineCMov()
47361 // some instruction-combining opportunities. This opt needs to be in combineCMov()
47383 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops); in combineCMov()
47396 Cond.getOpcode() == X86ISD::SUB && Cond->hasOneUse()) { in combineCMov()
47401 if (Cond0 == TrueOp && Sub1C && Sub1C->getZExtValue() == 2) { in combineCMov()
47402 EVT CondVT = Cond->getValueType(0); in combineCMov()
47403 EVT OuterVT = N->getValueType(0); in combineCMov()
47406 DAG.getNode(X86ISD::SUB, DL, Cond->getVTList(), Cond.getOperand(0), in combineCMov()
47415 // (CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2) in combineCMov()
47416 // (CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2) in combineCMov()
47444 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps); in combineCMov()
47447 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops); in combineCMov()
47452 // Fold (CMOV C1, (ADD (CTTZ X), C2), (X != 0)) -> in combineCMov()
47453 // (ADD (CMOV C1-C2, (CTTZ X), (X != 0)), C2) in combineCMov()
47454 // Or (CMOV (ADD (CTTZ X), C2), C1, (X == 0)) -> in combineCMov()
47455 // (ADD (CMOV (CTTZ X), C1-C2, (X == 0)), C2) in combineCMov()
47475 EVT VT = N->getValueType(0); in combineCMov()
47492 EVT VT = N->getOperand(0).getValueType(); in canReduceVMulWidth()
47496 assert(N->getNumOperands() == 2 && "NumOperands of Mul are 2"); in canReduceVMulWidth()
47500 SDValue Opd = N->getOperand(i); in canReduceVMulWidth()
47508 // When ranges are from -128 ~ 127, use MULS8 mode. in canReduceVMulWidth()
47514 // When ranges are from -32768 ~ 32767, use MULS16 mode. in canReduceVMulWidth()
47542 /// -128 to 128, and the scalar value range of %4 is also -128 to 128,
47548 /// -32768 to 32767, and the scalar value range of %4 is also -32768 to 32767,
47572 SDValue N0 = N->getOperand(0); in reduceVMULWidth()
47573 SDValue N1 = N->getOperand(1); in reduceVMULWidth()
47574 EVT VT = N->getOperand(0).getValueType(); in reduceVMULWidth()
47626 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMulSpecial()
47631 N->getOperand(0)); in combineMulSpecial()
47636 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMulSpecial()
47641 N->getOperand(0)); in combineMulSpecial()
47659 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), in combineMulSpecial()
47684 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), in combineMulSpecial()
47693 if (isPowerOf2_64(MulAmt & (MulAmt - 1))) { in combineMulSpecial()
47696 unsigned ShiftAmt = Log2_64((MulAmt & (MulAmt - 1))); in combineMulSpecial()
47697 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMulSpecial()
47699 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMulSpecial()
47720 EVT VT = N->getValueType(0); in combineMulToPMADDWD()
47736 SDValue N0 = N->getOperand(0); in combineMulToPMADDWD()
47737 SDValue N1 = N->getOperand(1); in combineMulToPMADDWD()
47772 // Mask off upper 16-bits of sign-extended constants. in combineMulToPMADDWD()
47775 if (Op.getOpcode() == ISD::SIGN_EXTEND && N->isOnlyUserOf(Op.getNode())) { in combineMulToPMADDWD()
47780 // Convert sext(vXi8) to zext(vXi16 sext(vXi8)) on pre-SSE41 targets in combineMulToPMADDWD()
47790 N->isOnlyUserOf(Op.getNode())) { in combineMulToPMADDWD()
47797 N->isOnlyUserOf(Op.getNode())) { in combineMulToPMADDWD()
47827 EVT VT = N->getValueType(0); in combineMulToPMULDQ()
47835 SDValue N0 = N->getOperand(0); in combineMulToPMULDQ()
47836 SDValue N1 = N->getOperand(1); in combineMulToPMULDQ()
47838 // MULDQ returns the 64-bit result of the signed multiplication of the lower in combineMulToPMULDQ()
47839 // 32-bits. We can lower with this if the sign bits stretch that far. in combineMulToPMULDQ()
47867 EVT VT = N->getValueType(0); in combineMul()
47896 N->getOperand(1), /*AllowUndefs*/ true, /*AllowTrunc*/ false); in combineMul()
47900 if (auto *RawC = getTargetConstantFromNode(N->getOperand(1))) in combineMul()
47901 if (auto *SplatC = RawC->getSplatValue()) in combineMul()
47903 C = &(SplatCI->getValue()); in combineMul()
47905 if (!C || C->getBitWidth() != VT.getScalarSizeInBits()) in combineMul()
47908 C = &(CNode->getAPIntValue()); in combineMul()
47911 if (isPowerOf2_64(C->getZExtValue())) in combineMul()
47914 int64_t SignMulAmt = C->getSExtValue(); in combineMul()
47916 uint64_t AbsMulAmt = SignMulAmt < 0 ? -SignMulAmt : SignMulAmt; in combineMul()
47921 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMul()
47947 if (isPowerOf2_64(MulAmt2) && !(SignMulAmt >= 0 && N->hasOneUse() && in combineMul()
47948 N->use_begin()->getOpcode() == ISD::ADD)) in combineMul()
47956 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
47959 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMul()
47973 NewMul = combineMulSpecial(C->getZExtValue(), N, DAG, VT, DL); in combineMul()
47977 assert(C->getZExtValue() != 0 && in combineMul()
47978 C->getZExtValue() != maxUIntN(VT.getScalarSizeInBits()) && in combineMul()
47981 if (isPowerOf2_64(AbsMulAmt - 1)) { in combineMul()
47984 ISD::ADD, DL, VT, N->getOperand(0), in combineMul()
47985 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
47986 DAG.getConstant(Log2_64(AbsMulAmt - 1), DL, ShiftVT))); in combineMul()
47990 // (mul x, 2^N - 1) => (sub (shl x, N), x) in combineMul()
47992 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
47996 NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul); in combineMul()
47998 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0)); in combineMul()
47999 } else if (SignMulAmt >= 0 && isPowerOf2_64(AbsMulAmt - 2) && in combineMul()
48003 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48004 DAG.getConstant(Log2_64(AbsMulAmt - 2), DL, ShiftVT)); in combineMul()
48007 DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0))); in combineMul()
48010 // (mul x, 2^N - 2) => (sub (shl x, N), (add x, x)) in combineMul()
48012 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48016 DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0))); in combineMul()
48019 uint64_t AbsMulAmtLowBit = AbsMulAmt & (-AbsMulAmt); in combineMul()
48022 if (isPowerOf2_64(AbsMulAmt - AbsMulAmtLowBit)) { in combineMul()
48023 ShiftAmt1 = AbsMulAmt - AbsMulAmtLowBit; in combineMul()
48032 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48035 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48055 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) && in combineShiftToPMULH()
48062 SDValue ShiftOperand = N->getOperand(0); in combineShiftToPMULH()
48067 EVT VT = N->getValueType(0); in combineShiftToPMULH()
48073 if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), ShiftAmt) || in combineShiftToPMULH()
48097 ExtOpc = N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in combineShiftToPMULH()
48104 SDValue N0 = N->getOperand(0); in combineShiftLeft()
48105 SDValue N1 = N->getOperand(1); in combineShiftLeft()
48112 // with out-of-bounds clamping. in combineShiftLeft()
48118 // fold shl(select(icmp_ult(amt,BW),x,0),amt) -> avx2 psllv(x,amt) in combineShiftLeft()
48124 // fold shl(select(icmp_uge(amt,BW),0,x),amt) -> avx2 psllv(x,amt) in combineShiftLeft()
48132 // fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2)) in combineShiftLeft()
48139 Mask <<= N1C->getAPIntValue(); in combineShiftLeft()
48141 // We can handle cases concerning bit-widening nodes containing setcc_c if in combineShiftLeft()
48147 // zext(setcc_c) -> i32 0x0000FFFF in combineShiftLeft()
48148 // c1 -> i32 0x0000FFFF in combineShiftLeft()
48149 // c2 -> i32 0x00000001 in combineShiftLeft()
48150 // (shl (and (setcc_c), c1), c2) -> i32 0x0001FFFE in combineShiftLeft()
48151 // (and setcc_c, (c1 << c2)) -> i32 0x0000FFFE in combineShiftLeft()
48172 SDValue N0 = N->getOperand(0); in combineShiftRightArithmetic()
48173 SDValue N1 = N->getOperand(1); in combineShiftRightArithmetic()
48181 // fold sra(x,umin(amt,bw-1)) -> avx2 psrav(x,amt) in combineShiftRightArithmetic()
48185 m_SpecificInt(VT.getScalarSizeInBits() - 1)))) in combineShiftRightArithmetic()
48190 // into (SHL (sext_in_reg X), ShlConst - SraConst) in combineShiftRightArithmetic()
48192 // or (SRA (sext_in_reg X), SraConst - ShlConst) in combineShiftRightArithmetic()
48194 // We only do this if (Size - ShlConst) is equal to 8, 16 or 32. That allows in combineShiftRightArithmetic()
48210 APInt ShlConst = N01->getAsAPIntVal(); in combineShiftRightArithmetic()
48211 APInt SraConst = N1->getAsAPIntVal(); in combineShiftRightArithmetic()
48221 // Only deal with (Size - ShlConst) being equal to 8, 16 or 32. in combineShiftRightArithmetic()
48222 if (ShiftSize >= Size || ShlConst != Size - ShiftSize) in combineShiftRightArithmetic()
48230 DAG.getConstant(ShlConst - SraConst, DL, CVT)); in combineShiftRightArithmetic()
48232 DAG.getConstant(SraConst - ShlConst, DL, CVT)); in combineShiftRightArithmetic()
48241 SDValue N0 = N->getOperand(0); in combineShiftRightLogical()
48242 SDValue N1 = N->getOperand(1); in combineShiftRightLogical()
48251 // with out-of-bounds clamping. in combineShiftRightLogical()
48257 // fold srl(select(icmp_ult(amt,BW),x,0),amt) -> avx2 psrlv(x,amt) in combineShiftRightLogical()
48263 // fold srl(select(icmp_uge(amt,BW),0,x),amt) -> avx2 psrlv(x,amt) in combineShiftRightLogical()
48277 // TODO: This is a generic DAG combine that became an x86-only combine to in combineShiftRightLogical()
48278 // avoid shortcomings in other folds such as bswap, bit-test ('bt'), and in combineShiftRightLogical()
48279 // and-not ('andn'). in combineShiftRightLogical()
48288 // If we can shrink the constant mask below 8-bits or 32-bits, then this in combineShiftRightLogical()
48290 // from improved known-bits analysis or instruction selection. in combineShiftRightLogical()
48291 APInt MaskVal = AndC->getAPIntValue(); in combineShiftRightLogical()
48300 APInt NewMaskVal = MaskVal.lshr(ShiftC->getAPIntValue()); in combineShiftRightLogical()
48305 // srl (and X, AndC), ShiftC --> and (srl X, ShiftC), (AndC >> ShiftC) in combineShiftRightLogical()
48315 unsigned Opcode = N->getOpcode(); in combineHorizOpWithShuffle()
48319 EVT VT = N->getValueType(0); in combineHorizOpWithShuffle()
48320 SDValue N0 = N->getOperand(0); in combineHorizOpWithShuffle()
48321 SDValue N1 = N->getOperand(1); in combineHorizOpWithShuffle()
48325 N->isOnlyUserOf(N0.getNode()) ? peekThroughOneUseBitcasts(N0) : N0; in combineHorizOpWithShuffle()
48327 N->isOnlyUserOf(N1.getNode()) ? peekThroughOneUseBitcasts(N1) : N1; in combineHorizOpWithShuffle()
48342 // shuffle to a v4X64 width - we can probably relax this in the future. in combineHorizOpWithShuffle()
48360 // Attempt to fold HOP(SHUFFLE(X,Y),SHUFFLE(Z,W)) -> SHUFFLE(HOP()). in combineHorizOpWithShuffle()
48385 int PostShuffle[4] = {-1, -1, -1, -1}; in combineHorizOpWithShuffle()
48417 // Attempt to fold HOP(SHUFFLE(X,Y),SHUFFLE(X,Y)) -> SHUFFLE(HOP(X,Y)). in combineHorizOpWithShuffle()
48460 unsigned Opcode = N->getOpcode(); in combineVectorPack()
48464 EVT VT = N->getValueType(0); in combineVectorPack()
48465 SDValue N0 = N->getOperand(0); in combineVectorPack()
48466 SDValue N1 = N->getOperand(1); in combineVectorPack()
48479 if ((N0.isUndef() || N->isOnlyUserOf(N0.getNode())) && in combineVectorPack()
48480 (N1.isUndef() || N->isOnlyUserOf(N1.getNode())) && in combineVectorPack()
48530 // Try to fold PACK(SHUFFLE(),SHUFFLE()) -> SHUFFLE(PACK()). in combineVectorPack()
48534 // Try to fold PACKSS(NOT(X),NOT(Y)) -> NOT(PACKSS(X,Y)). in combineVectorPack()
48570 // Try to fold PACK(EXTEND(X),EXTEND(Y)) -> CONCAT(X,Y) subvectors. in combineVectorPack()
48611 assert((X86ISD::HADD == N->getOpcode() || X86ISD::FHADD == N->getOpcode() || in combineVectorHADDSUB()
48612 X86ISD::HSUB == N->getOpcode() || X86ISD::FHSUB == N->getOpcode()) && in combineVectorHADDSUB()
48613 "Unexpected horizontal add/sub opcode"); in combineVectorHADDSUB()
48616 MVT VT = N->getSimpleValueType(0); in combineVectorHADDSUB()
48617 SDValue LHS = N->getOperand(0); in combineVectorHADDSUB()
48618 SDValue RHS = N->getOperand(1); in combineVectorHADDSUB()
48620 // HOP(HOP'(X,X),HOP'(Y,Y)) -> HOP(PERMUTE(HOP'(X,Y)),PERMUTE(HOP'(X,Y)). in combineVectorHADDSUB()
48621 if (LHS != RHS && LHS.getOpcode() == N->getOpcode() && in combineVectorHADDSUB()
48624 N->isOnlyUserOf(LHS.getNode()) && N->isOnlyUserOf(RHS.getNode())) { in combineVectorHADDSUB()
48643 return DAG.getNode(N->getOpcode(), DL, VT, DAG.getBitcast(VT, NewLHS), in combineVectorHADDSUB()
48649 // Try to fold HOP(SHUFFLE(),SHUFFLE()) -> SHUFFLE(HOP()). in combineVectorHADDSUB()
48659 assert((X86ISD::VSHL == N->getOpcode() || X86ISD::VSRA == N->getOpcode() || in combineVectorShiftVar()
48660 X86ISD::VSRL == N->getOpcode()) && in combineVectorShiftVar()
48662 EVT VT = N->getValueType(0); in combineVectorShiftVar()
48663 SDValue N0 = N->getOperand(0); in combineVectorShiftVar()
48664 SDValue N1 = N->getOperand(1); in combineVectorShiftVar()
48666 // Shift zero -> zero. in combineVectorShiftVar()
48676 unsigned X86Opc = getTargetVShiftUniformOpcode(N->getOpcode(), false); in combineVectorShiftVar()
48692 unsigned Opcode = N->getOpcode(); in combineVectorShiftImm()
48697 EVT VT = N->getValueType(0); in combineVectorShiftImm()
48698 SDValue N0 = N->getOperand(0); in combineVectorShiftImm()
48699 SDValue N1 = N->getOperand(1); in combineVectorShiftImm()
48705 // (shift undef, X) -> 0 in combineVectorShiftImm()
48711 unsigned ShiftVal = N->getConstantOperandVal(1); in combineVectorShiftImm()
48715 ShiftVal = NumBitsPerElt - 1; in combineVectorShiftImm()
48718 // (shift X, 0) -> X in combineVectorShiftImm()
48722 // (shift 0, C) -> 0 in combineVectorShiftImm()
48728 // (VSRAI -1, C) -> -1 in combineVectorShiftImm()
48732 return DAG.getConstant(-1, SDLoc(N), VT); in combineVectorShiftImm()
48741 NewShiftVal = NumBitsPerElt - 1; in combineVectorShiftImm()
48747 // (shift (shift X, C2), C1) -> (shift X, (C1 + C2)) in combineVectorShiftImm()
48751 // (shl (add X, X), C) -> (shl X, (C + 1)) in combineVectorShiftImm()
48765 // psrad(pshufd(psllq(X,63),1,1,3,3),31) -> in combineVectorShiftImm()
48770 N0->hasOneUse()) { in combineVectorShiftImm()
48815 if (N->isOnlyUserOf(N0.getNode())) { in combineVectorShiftImm()
48819 // Fold (shift (logic X, C2), C1) -> (logic (shift X, C1), (shift C2, C1)) in combineVectorShiftImm()
48823 BC->isOnlyUserOf(BC.getOperand(1).getNode()) && in combineVectorShiftImm()
48845 EVT VT = N->getValueType(0); in combineVectorInsert()
48846 unsigned Opcode = N->getOpcode(); in combineVectorInsert()
48852 SDValue Vec = N->getOperand(0); in combineVectorInsert()
48853 SDValue Scl = N->getOperand(1); in combineVectorInsert()
48854 SDValue Idx = N->getOperand(2); in combineVectorInsert()
48856 // Fold insert_vector_elt(undef, elt, 0) --> scalar_to_vector(elt). in combineVectorInsert()
48880 /// OR -> CMPNEQSS.
48889 SDValue N0 = N->getOperand(0); in combineCompareEqual()
48890 SDValue N1 = N->getOperand(1); in combineCompareEqual()
48899 SDValue CMP00 = CMP0->getOperand(0); in combineCompareEqual()
48900 SDValue CMP01 = CMP0->getOperand(1); in combineCompareEqual()
48907 for (const SDNode *U : N->uses()) { in combineCompareEqual()
48911 switch (U->getOpcode()) { in combineCompareEqual()
48951 N->getSimpleValueType(0)); in combineCompareEqual()
48961 // On a 32-bit target, we cannot bitcast the 64-bit float to a in combineCompareEqual()
48962 // 64-bit integer, since that's not a legal type. Since in combineCompareEqual()
48987 /// Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
48989 assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDNP"); in combineAndNotIntoANDNP()
48991 MVT VT = N->getSimpleValueType(0); in combineAndNotIntoANDNP()
48996 SDValue N0 = N->getOperand(0); in combineAndNotIntoANDNP()
48997 SDValue N1 = N->getOperand(1); in combineAndNotIntoANDNP()
49015 /// (insert_vector_elt undef, (xor X, -1), Z), undef), Y
49016 /// ->
49021 assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDNP"); in combineAndShuffleNot()
49023 EVT VT = N->getValueType(0); in combineAndShuffleNot()
49032 // TODO: SVN->hasOneUse() is a strong condition. It can be relaxed if all in combineAndShuffleNot()
49033 // end-users are ISD::AND including cases in combineAndShuffleNot()
49035 if (!SVN || !SVN->hasOneUse() || !SVN->isSplat() || in combineAndShuffleNot()
49036 !SVN->getOperand(1).isUndef()) { in combineAndShuffleNot()
49039 SDValue IVEN = SVN->getOperand(0); in combineAndShuffleNot()
49044 IVEN.getConstantOperandAPInt(2) != SVN->getSplatIndex()) in combineAndShuffleNot()
49052 return DAG.getVectorShuffle(SVN->getValueType(0), SDLoc(SVN), NotIVEN, in combineAndShuffleNot()
49053 SVN->getOperand(1), SVN->getMask()); in combineAndShuffleNot()
49059 SDValue N0 = N->getOperand(0); in combineAndShuffleNot()
49060 SDValue N1 = N->getOperand(1); in combineAndShuffleNot()
49154 // register. In most cases we actually compare or select YMM-sized registers
49157 // Even with AVX-512 this is still useful for removing casts around logical
49190 // clang-format off in convertIntLogicToFPLogicOpcode()
49195 // clang-format on in convertIntLogicToFPLogicOpcode()
49200 /// If both input operands of a logic op are being cast from floating-point
49201 /// types or FP compares, try to convert this into a floating-point logic node
49206 EVT VT = N->getValueType(0); in convertIntLogicToFPLogic()
49207 SDValue N0 = N->getOperand(0); in convertIntLogicToFPLogic()
49208 SDValue N1 = N->getOperand(1); in convertIntLogicToFPLogic()
49227 unsigned FPOpcode = convertIntLogicToFPLogicOpcode(N->getOpcode()); in convertIntLogicToFPLogic()
49236 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in convertIntLogicToFPLogic()
49237 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1.getOperand(2))->get(); in convertIntLogicToFPLogic()
49247 // logic (setcc N00, N01), (setcc N10, N11) --> in convertIntLogicToFPLogic()
49261 SDValue Logic = DAG.getNode(N->getOpcode(), DL, BoolVecVT, Setcc0, Setcc1); in convertIntLogicToFPLogic()
49265 // Attempt to fold BITOP(MOVMSK(X),MOVMSK(Y)) -> MOVMSK(BITOP(X,Y))
49266 // to reduce XMM->GPR traffic.
49268 unsigned Opc = N->getOpcode(); in combineBitOpWithMOVMSK()
49272 SDValue N0 = N->getOperand(0); in combineBitOpWithMOVMSK()
49273 SDValue N1 = N->getOperand(1); in combineBitOpWithMOVMSK()
49299 // Attempt to fold BITOP(SHIFT(X,Z),SHIFT(Y,Z)) -> SHIFT(BITOP(X,Y),Z).
49303 unsigned Opc = N->getOpcode(); in combineBitOpWithShift()
49307 SDValue N0 = N->getOperand(0); in combineBitOpWithShift()
49308 SDValue N1 = N->getOperand(1); in combineBitOpWithShift()
49309 EVT VT = N->getValueType(0); in combineBitOpWithShift()
49321 if (BCOpc != BC1->getOpcode() || BCVT != BC1.getValueType()) in combineBitOpWithShift()
49343 // BITOP(PACKSS(X,Z),PACKSS(Y,W)) --> PACKSS(BITOP(X,Y),BITOP(Z,W)).
49346 unsigned Opc = N->getOpcode(); in combineBitOpWithPACK()
49350 SDValue N0 = N->getOperand(0); in combineBitOpWithPACK()
49351 SDValue N1 = N->getOperand(1); in combineBitOpWithPACK()
49352 EVT VT = N->getValueType(0); in combineBitOpWithPACK()
49385 /// If this is a zero/all-bits result that is bitwise-anded with a low bits
49387 /// with a shift-right to eliminate loading the vector constant mask value.
49390 SDValue Op0 = peekThroughBitcasts(N->getOperand(0)); in combineAndMaskToShift()
49391 SDValue Op1 = peekThroughBitcasts(N->getOperand(1)); in combineAndMaskToShift()
49397 // shift and "andn". This saves a materialization of a -1 vector constant. in combineAndMaskToShift()
49400 // and (pcmpgt X, -1), Y --> pandn (vsrai X, BitWidth - 1), Y in combineAndMaskToShift()
49405 if (N->getValueType(0) == VT && in combineAndMaskToShift()
49421 VT.getScalarSizeInBits() - 1, DAG); in combineAndMaskToShift()
49443 SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8); in combineAndMaskToShift()
49445 return DAG.getBitcast(N->getValueType(0), Shift); in combineAndMaskToShift()
49451 if (Ld->isIndexed()) in getIndexFromUnindexedLoad()
49454 SDValue Base = Ld->getBasePtr(); in getIndexFromUnindexedLoad()
49474 // 'and-load' sequence.
49478 // int array[SIZE] = {0x0, 0x1, 0x3, 0x7, 0xF ..., 2^(SIZE-1) - 1}
49485 MVT VT = Node->getSimpleValueType(0); in combineAndLoadToBZHI()
49494 SDValue N = Node->getOperand(i); in combineAndLoadToBZHI()
49501 const Value *MemOp = Ld->getMemOperand()->getValue(); in combineAndLoadToBZHI()
49507 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) { in combineAndLoadToBZHI()
49508 if (GV->isConstant() && GV->hasDefinitiveInitializer()) { in combineAndLoadToBZHI()
49510 Constant *Init = GV->getInitializer(); in combineAndLoadToBZHI()
49511 Type *Ty = Init->getType(); in combineAndLoadToBZHI()
49513 !Ty->getArrayElementType()->isIntegerTy() || in combineAndLoadToBZHI()
49514 Ty->getArrayElementType()->getScalarSizeInBits() != in combineAndLoadToBZHI()
49516 Ty->getArrayNumElements() > in combineAndLoadToBZHI()
49517 Ty->getArrayElementType()->getScalarSizeInBits()) in combineAndLoadToBZHI()
49521 uint64_t ArrayElementCount = Init->getType()->getArrayNumElements(); in combineAndLoadToBZHI()
49524 auto *Elem = cast<ConstantInt>(Init->getAggregateElement(j)); in combineAndLoadToBZHI()
49525 if (Elem->getZExtValue() != (((uint64_t)1 << j) - 1)) { in combineAndLoadToBZHI()
49533 // Do the transformation (For 32-bit type): in combineAndLoadToBZHI()
49534 // -> (and (load arr[idx]), inp) in combineAndLoadToBZHI()
49535 // <- (and (srl 0xFFFFFFFF, (sub 32, idx))) in combineAndLoadToBZHI()
49537 SDValue Inp = (i == 0) ? Node->getOperand(1) : Node->getOperand(0); in combineAndLoadToBZHI()
49562 // where the setcc will freely 0 upper bits of k-register. We can replace the
49567 assert(N->getOpcode() == ISD::AND && "Unexpected opcode!"); in combineScalarAndWithMaskSetcc()
49569 EVT VT = N->getValueType(0); in combineScalarAndWithMaskSetcc()
49573 auto *C1 = dyn_cast<ConstantSDNode>(N->getOperand(1)); in combineScalarAndWithMaskSetcc()
49580 SDValue Src = N->getOperand(0); in combineScalarAndWithMaskSetcc()
49612 !C1->getAPIntValue().isMask(SubVecVT.getVectorNumElements())) in combineScalarAndWithMaskSetcc()
49651 // Only do this re-ordering if op has one use. in getBMIMatchingOp()
49666 Op.getOperand(1 - OpIdx)); in getBMIMatchingOp()
49682 // BLSR: (and x, (add x, -1)) in getBMIMatchingOp()
49683 // BLSMSK: (xor x, (add x, -1)) in getBMIMatchingOp()
49692 EVT VT = N->getValueType(0); in combineBMILogicOp()
49698 assert(N->getOpcode() == ISD::AND || N->getOpcode() == ISD::XOR); in combineBMILogicOp()
49703 getBMIMatchingOp(N->getOpcode(), DAG, N->getOperand(OpIdx), in combineBMILogicOp()
49704 N->getOperand(1 - OpIdx), 0)) in combineBMILogicOp()
49715 // -> in combineX86SubCmpForFlags()
49721 // -> in combineX86SubCmpForFlags()
49727 SDValue SetCC = N->getOperand(0); in combineX86SubCmpForFlags()
49733 SDNode *BrCond = *Flag->uses().begin(); in combineX86SubCmpForFlags()
49734 if (BrCond->getOpcode() != X86ISD::BRCOND) in combineX86SubCmpForFlags()
49737 if (static_cast<X86::CondCode>(BrCond->getConstantOperandVal(CondNo)) != in combineX86SubCmpForFlags()
49744 if (N->getOpcode() == X86ISD::SUB) in combineX86SubCmpForFlags()
49745 X = DAG.getMergeValues({N->getOperand(0), X}, SDLoc(N)); in combineX86SubCmpForFlags()
49749 static_cast<X86::CondCode>(CCN->getAsAPIntVal().getSExtValue()); in combineX86SubCmpForFlags()
49755 SmallVector<SDValue> Ops(BrCond->op_values()); in combineX86SubCmpForFlags()
49756 if (isNullConstant(N->getOperand(1))) in combineX86SubCmpForFlags()
49758 else if (isOneConstant(N->getOperand(1))) in combineX86SubCmpForFlags()
49764 DAG.getNode(X86ISD::BRCOND, SDLoc(BrCond), BrCond->getValueType(0), Ops); in combineX86SubCmpForFlags()
49765 // Avoid self-assign error b/c CC1 can be `e/ne`. in combineX86SubCmpForFlags()
49775 // -> in combineAndOrForCcmpCtest()
49779 // -> in combineAndOrForCcmpCtest()
49787 SDValue SetCC0 = N->getOperand(0); in combineAndOrForCcmpCtest()
49788 SDValue SetCC1 = N->getOperand(1); in combineAndOrForCcmpCtest()
49793 auto GetCombineToOpc = [&](SDValue V) -> unsigned { in combineAndOrForCcmpCtest()
49819 bool IsOR = N->getOpcode() == ISD::OR; in combineAndOrForCcmpCtest()
49830 static_cast<X86::CondCode>(CC1N->getAsAPIntVal().getSExtValue()); in combineAndOrForCcmpCtest()
49854 SDValue N0 = N->getOperand(0); in combineAnd()
49855 SDValue N1 = N->getOperand(1); in combineAnd()
49856 EVT VT = N->getValueType(0); in combineAnd()
49868 // Use a 32-bit and+zext if upper bits known zero. in combineAnd()
49880 // Match all-of bool scalar reductions into a bitcast/movmsk + cmp. in combineAnd()
49903 // `(-x << C0) & C1` in combineAnd()
49905 // `(x * (Pow2_Ceil(C1) - (1 << C0))) & C1` in combineAnd()
49918 const APInt &MulC = N01C->getAPIntValue(); in combineAnd()
49919 const APInt &AndC = N1C->getAPIntValue(); in combineAnd()
49920 APInt MulCLowBit = MulC & (-MulC); in combineAnd()
49925 assert(MulCLowBitLog != -1 && in combineAnd()
49970 // fold (and (mul x, c1), c2) -> (mul x, (and c1, c2)) in combineAnd()
49971 // iff c2 is all/no bits mask - i.e. a select-with-zero mask. in combineAnd()
49978 N0->hasOneUse() && N0.getOperand(1)->hasOneUse()) { in combineAnd()
49984 // Fold AND(SRL(X,Y),1) -> SETCC(BT(X,Y), COND_B) iff Y is not a constant in combineAnd()
49986 if (isOneConstant(N1) && N0->hasOneUse()) { in combineAnd()
49990 Src.getOperand(0)->hasOneUse()) in combineAnd()
50038 // We can't assume an undef src element gives an undef dst - the in combineAnd()
50059 if (N->getOpcode() != ISD::DELETED_NODE) in combineAnd()
50074 isa<ConstantSDNode>(N0.getOperand(1)) && N0->hasOneUse()) { in combineAnd()
50082 if (VT == SrcVecVT.getScalarType() && N0->isOnlyUserOf(SrcVec.getNode()) && in combineAnd()
50116 // Canonicalize OR(AND(X,C),AND(Y,~C)) -> OR(AND(X,C),ANDNP(C,Y))
50119 assert(N->getOpcode() == ISD::OR && "Unexpected Opcode"); in canonicalizeBitSelect()
50121 MVT VT = N->getSimpleValueType(0); in canonicalizeBitSelect()
50126 SDValue N0 = peekThroughBitcasts(N->getOperand(0)); in canonicalizeBitSelect()
50127 SDValue N1 = peekThroughBitcasts(N->getOperand(1)); in canonicalizeBitSelect()
50150 // TODO - add UNDEF elts support. in canonicalizeBitSelect()
50160 // Emit a VPTERNLOG node directly - 0xCA is the imm code for A?B:C. in canonicalizeBitSelect()
50161 // VPTERNLOG is only available as vXi32/64-bit types. in canonicalizeBitSelect()
50174 SDValue X = N->getOperand(0); in canonicalizeBitSelect()
50183 if (N->getOpcode() != ISD::OR) in matchLogicBlend()
50186 SDValue N0 = N->getOperand(0); in matchLogicBlend()
50187 SDValue N1 = N->getOperand(1); in matchLogicBlend()
50208 // TODO: Attempt to match against AND(XOR(-1,M),Y) as well, waiting for in matchLogicBlend()
50223 assert(N->getOpcode() == ISD::OR && "Unexpected Opcode"); in combineLogicBlendIntoPBLENDV()
50225 EVT VT = N->getValueType(0); in combineLogicBlendIntoPBLENDV()
50281 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0)); in lowerX86CmpEqZeroToCtlzSrl()
50282 // The result of the shift is true or false, and on X86, the 32-bit in lowerX86CmpEqZeroToCtlzSrl()
50300 if (DCI.isBeforeLegalize() || !Subtarget.getTargetLowering()->isCtlzFast()) in combineOrCmpEqZeroToCtlzSrl()
50304 return (N->getOpcode() == ISD::OR && N->hasOneUse()); in combineOrCmpEqZeroToCtlzSrl()
50307 // Check the zero extend is extending to 32-bit or more. The code generated by in combineOrCmpEqZeroToCtlzSrl()
50308 // srl(ctlz) for 16-bit or less variants of the pattern would require extra in combineOrCmpEqZeroToCtlzSrl()
50310 if (!N->hasOneUse() || !N->getSimpleValueType(0).bitsGE(MVT::i32) || in combineOrCmpEqZeroToCtlzSrl()
50311 !isORCandidate(N->getOperand(0))) in combineOrCmpEqZeroToCtlzSrl()
50316 return N->getOpcode() == X86ISD::SETCC && N->hasOneUse() && in combineOrCmpEqZeroToCtlzSrl()
50317 X86::CondCode(N->getConstantOperandVal(0)) == X86::COND_E && in combineOrCmpEqZeroToCtlzSrl()
50318 N->getOperand(1).getOpcode() == X86ISD::CMP && in combineOrCmpEqZeroToCtlzSrl()
50319 isNullConstant(N->getOperand(1).getOperand(1)) && in combineOrCmpEqZeroToCtlzSrl()
50320 N->getOperand(1).getValueType().bitsGE(MVT::i32); in combineOrCmpEqZeroToCtlzSrl()
50323 SDNode *OR = N->getOperand(0).getNode(); in combineOrCmpEqZeroToCtlzSrl()
50324 SDValue LHS = OR->getOperand(0); in combineOrCmpEqZeroToCtlzSrl()
50325 SDValue RHS = OR->getOperand(1); in combineOrCmpEqZeroToCtlzSrl()
50332 OR = (LHS->getOpcode() == ISD::OR) ? LHS.getNode() : RHS.getNode(); in combineOrCmpEqZeroToCtlzSrl()
50333 LHS = OR->getOperand(0); in combineOrCmpEqZeroToCtlzSrl()
50334 RHS = OR->getOperand(1); in combineOrCmpEqZeroToCtlzSrl()
50358 LHS = OR->getOperand(0); in combineOrCmpEqZeroToCtlzSrl()
50359 RHS = OR->getOperand(1); in combineOrCmpEqZeroToCtlzSrl()
50361 if (RHS->getOpcode() == ISD::OR) in combineOrCmpEqZeroToCtlzSrl()
50369 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), Ret); in combineOrCmpEqZeroToCtlzSrl()
50375 if (!isBitwiseNot(And0_L, true) || !And0_L->hasOneUse()) in foldMaskedMergeImpl()
50377 SDValue NotOp = And0_L->getOperand(0); in foldMaskedMergeImpl()
50384 // --> ((And0_R ^ And1_R) & NotOp) ^ And1_R in foldMaskedMergeImpl()
50385 EVT VT = And1_L->getValueType(0); in foldMaskedMergeImpl()
50396 /// "and-not" operation. This function is intended to be called from a
50399 // Note that masked-merge variants using XOR or ADD expressions are in foldMaskedMerge()
50401 assert(Node->getOpcode() == ISD::OR && "Must be called with ISD::OR node"); in foldMaskedMerge()
50402 SDValue N0 = Node->getOperand(0); in foldMaskedMerge()
50403 if (N0->getOpcode() != ISD::AND || !N0->hasOneUse()) in foldMaskedMerge()
50405 SDValue N1 = Node->getOperand(1); in foldMaskedMerge()
50406 if (N1->getOpcode() != ISD::AND || !N1->hasOneUse()) in foldMaskedMerge()
50410 SDValue N00 = N0->getOperand(0); in foldMaskedMerge()
50411 SDValue N01 = N0->getOperand(1); in foldMaskedMerge()
50412 SDValue N10 = N1->getOperand(0); in foldMaskedMerge()
50413 SDValue N11 = N1->getOperand(1); in foldMaskedMerge()
50436 // Look through a one-use zext. in combineAddOrSubToADCOrSBB()
50453 // If X is -1 or 0, then we have an opportunity to avoid constants required in in combineAddOrSubToADCOrSBB()
50457 if ((!IsSub && CC == X86::COND_AE && ConstantX->isAllOnes()) || in combineAddOrSubToADCOrSBB()
50458 (IsSub && CC == X86::COND_B && ConstantX->isZero())) { in combineAddOrSubToADCOrSBB()
50459 // This is a complicated way to get -1 or 0 from the carry flag: in combineAddOrSubToADCOrSBB()
50460 // -1 + SETAE --> -1 + (!CF) --> CF ? -1 : 0 --> SBB %eax, %eax in combineAddOrSubToADCOrSBB()
50461 // 0 - SETB --> 0 - (CF) --> CF ? -1 : 0 --> SBB %eax, %eax in combineAddOrSubToADCOrSBB()
50467 if ((!IsSub && CC == X86::COND_BE && ConstantX->isAllOnes()) || in combineAddOrSubToADCOrSBB()
50468 (IsSub && CC == X86::COND_A && ConstantX->isZero())) { in combineAddOrSubToADCOrSBB()
50473 // -1 + SETBE (SUB A, B) --> -1 + SETAE (SUB B, A) --> SUB + SBB in combineAddOrSubToADCOrSBB()
50474 // 0 - SETA (SUB A, B) --> 0 - SETB (SUB B, A) --> SUB + SBB in combineAddOrSubToADCOrSBB()
50476 X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(), in combineAddOrSubToADCOrSBB()
50487 // X + SETB Z --> adc X, 0 in combineAddOrSubToADCOrSBB()
50488 // X - SETB Z --> sbb X, 0 in combineAddOrSubToADCOrSBB()
50501 // Do not flip "e > c", where "c" is a constant, because Cmp instruction in combineAddOrSubToADCOrSBB()
50504 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() && in combineAddOrSubToADCOrSBB()
50508 DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(), in combineAddOrSubToADCOrSBB()
50518 // X + SETAE --> sbb X, -1 in combineAddOrSubToADCOrSBB()
50519 // X - SETAE --> adc X, -1 in combineAddOrSubToADCOrSBB()
50522 DAG.getConstant(-1, DL, VT), EFLAGS); in combineAddOrSubToADCOrSBB()
50526 // X + SETBE --> sbb X, -1 in combineAddOrSubToADCOrSBB()
50527 // X - SETBE --> adc X, -1 in combineAddOrSubToADCOrSBB()
50531 // Do not flip "e <= c", where "c" is a constant, because Cmp instruction in combineAddOrSubToADCOrSBB()
50534 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() && in combineAddOrSubToADCOrSBB()
50538 DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(), in combineAddOrSubToADCOrSBB()
50543 DAG.getConstant(-1, DL, VT), NewEFLAGS); in combineAddOrSubToADCOrSBB()
50558 // If X is -1 or 0, then we have an opportunity to avoid constants required in in combineAddOrSubToADCOrSBB()
50561 // 'neg' sets the carry flag when Z != 0, so create 0 or -1 using 'sbb' with in combineAddOrSubToADCOrSBB()
50563 // 0 - (Z != 0) --> sbb %eax, %eax, (neg Z) in combineAddOrSubToADCOrSBB()
50564 // -1 + (Z == 0) --> sbb %eax, %eax, (neg Z) in combineAddOrSubToADCOrSBB()
50565 if ((IsSub && CC == X86::COND_NE && ConstantX->isZero()) || in combineAddOrSubToADCOrSBB()
50566 (!IsSub && CC == X86::COND_E && ConstantX->isAllOnes())) { in combineAddOrSubToADCOrSBB()
50575 // cmp with 1 sets the carry flag when Z == 0, so create 0 or -1 using 'sbb' in combineAddOrSubToADCOrSBB()
50577 // 0 - (Z == 0) --> sbb %eax, %eax, (cmp Z, 1) in combineAddOrSubToADCOrSBB()
50578 // -1 + (Z != 0) --> sbb %eax, %eax, (cmp Z, 1) in combineAddOrSubToADCOrSBB()
50579 if ((IsSub && CC == X86::COND_E && ConstantX->isZero()) || in combineAddOrSubToADCOrSBB()
50580 (!IsSub && CC == X86::COND_NE && ConstantX->isAllOnes())) { in combineAddOrSubToADCOrSBB()
50598 // X - (Z != 0) --> sub X, (zext(setne Z, 0)) --> adc X, -1, (cmp Z, 1) in combineAddOrSubToADCOrSBB()
50599 // X + (Z != 0) --> add X, (zext(setne Z, 0)) --> sbb X, -1, (cmp Z, 1) in combineAddOrSubToADCOrSBB()
50602 DAG.getConstant(-1ULL, DL, VT), Cmp1.getValue(1)); in combineAddOrSubToADCOrSBB()
50604 // X - (Z == 0) --> sub X, (zext(sete Z, 0)) --> sbb X, 0, (cmp Z, 1) in combineAddOrSubToADCOrSBB()
50605 // X + (Z == 0) --> add X, (zext(sete Z, 0)) --> adc X, 0, (cmp Z, 1) in combineAddOrSubToADCOrSBB()
50615 bool IsSub = N->getOpcode() == ISD::SUB; in combineAddOrSubToADCOrSBB()
50616 SDValue X = N->getOperand(0); in combineAddOrSubToADCOrSBB()
50617 SDValue Y = N->getOperand(1); in combineAddOrSubToADCOrSBB()
50618 EVT VT = N->getValueType(0); in combineAddOrSubToADCOrSBB()
50635 assert((N->getOpcode() == ISD::XOR || N->getOpcode() == ISD::OR) && in combineOrXorWithSETCC()
50647 bool IsSub = N->getOpcode() == ISD::XOR; in combineOrXorWithSETCC()
50648 bool N1COdd = N1C->getZExtValue() & 1; in combineOrXorWithSETCC()
50651 EVT VT = N->getValueType(0); in combineOrXorWithSETCC()
50658 // not(pcmpeq(and(X,CstPow2),0)) -> pcmpeq(and(X,CstPow2),CstPow2) in combineOrXorWithSETCC()
50659 if (N->getOpcode() == ISD::XOR && N0.getOpcode() == X86ISD::PCMPEQ && in combineOrXorWithSETCC()
50663 MVT VT = N->getSimpleValueType(0); in combineOrXorWithSETCC()
50685 SDValue N0 = N->getOperand(0); in combineOr()
50686 SDValue N1 = N->getOperand(1); in combineOr()
50687 EVT VT = N->getValueType(0); in combineOr()
50699 // Match any-of bool scalar reductions into a bitcast/movmsk + cmp. in combineOr()
50749 // (0 - SetCC) | C -> (zext (not SetCC)) * (C + 1) - 1 if we can get a LEA out of it. in combineOr()
50759 uint64_t Val = CN->getZExtValue(); in combineOr()
50774 // Combine OR(X,KSHIFTL(Y,Elts/2)) -> CONCAT_VECTORS(X,Y) == KUNPCK(X,Y). in combineOr()
50775 // Combine OR(KSHIFTL(X,Elts/2),Y) -> CONCAT_VECTORS(Y,X) == KUNPCK(Y,X). in combineOr()
50776 // iff the upper elements of the non-shifted arg are zero. in combineOr()
50824 if (N->getOpcode() != ISD::DELETED_NODE) in combineOr()
50842 /// XOR(TRUNCATE(SRL(X, size(X)-1)), 1)
50844 /// SETGT(X, -1)
50847 EVT ResultType = N->getValueType(0); in foldXorTruncShiftIntoCmp()
50851 SDValue N0 = N->getOperand(0); in foldXorTruncShiftIntoCmp()
50852 SDValue N1 = N->getOperand(1); in foldXorTruncShiftIntoCmp()
50874 Shift.getConstantOperandAPInt(1) != (ShiftTy.getSizeInBits() - 1)) in foldXorTruncShiftIntoCmp()
50877 // Create a greater-than comparison against -1. in foldXorTruncShiftIntoCmp()
50887 DAG.getConstant(-1, DL, ShiftOpTy), ISD::SETGT); in foldXorTruncShiftIntoCmp()
50894 /// xor (sra X, elt_size(X)-1), -1
50896 /// pcmpgt X, -1
50902 EVT VT = N->getValueType(0); in foldVectorXorShiftIntoCmp()
50907 // clang-format off in foldVectorXorShiftIntoCmp()
50917 // clang-format on in foldVectorXorShiftIntoCmp()
50922 SDValue Shift = N->getOperand(0); in foldVectorXorShiftIntoCmp()
50923 SDValue Ones = N->getOperand(1); in foldVectorXorShiftIntoCmp()
50932 ShiftAmt->getAPIntValue() != (Shift.getScalarValueSizeInBits() - 1)) in foldVectorXorShiftIntoCmp()
50935 // Create a greater-than comparison against -1. We don't use the more obvious in foldVectorXorShiftIntoCmp()
50936 // greater-than-or-equal-to-zero because SSE/AVX don't have that instruction. in foldVectorXorShiftIntoCmp()
50965 auto MatchMinMax = [](SDValue V, unsigned Opcode, APInt &Limit) -> SDValue { in detectUSatPattern()
51009 const APInt &Limit) -> SDValue { in detectSSatPattern()
51047 // If we're clamping a signed 32-bit vector to 0-255 and the 32-bit vector is in combineTruncateWithSat()
51048 // split across two registers. We can use a packusdw+perm to clamp to 0-65535 in combineTruncateWithSat()
51050 // clip to 0-255. in combineTruncateWithSat()
51064 // For 256-bit or smaller vectors, we require VLX. in combineTruncateWithSat()
51066 // If the result type is 256-bits or larger and we have disable 512-bit in combineTruncateWithSat()
51079 // vXi32 -> vXi8 must be performed as PACKUSWB(PACKSSDW,PACKSSDW). in combineTruncateWithSat()
51142 EVT RegVT = Ld->getValueType(0); in combineConstantPoolLoads()
51143 SDValue Ptr = Ld->getBasePtr(); in combineConstantPoolLoads()
51144 SDValue Chain = Ld->getChain(); in combineConstantPoolLoads()
51145 ISD::LoadExtType Ext = Ld->getExtensionType(); in combineConstantPoolLoads()
51147 if (Ext != ISD::NON_EXTLOAD || !Subtarget.hasAVX() || !Ld->isSimple()) in combineConstantPoolLoads()
51170 for (SDNode *User : Chain->uses()) { in combineConstantPoolLoads()
51173 (User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD || in combineConstantPoolLoads()
51174 User->getOpcode() == X86ISD::VBROADCAST_LOAD || in combineConstantPoolLoads()
51176 UserLd->getChain() == Chain && !User->hasAnyUseOfValue(1) && in combineConstantPoolLoads()
51177 User->getValueSizeInBits(0).getFixedValue() > in combineConstantPoolLoads()
51179 EVT UserVT = User->getValueType(0); in combineConstantPoolLoads()
51180 SDValue UserPtr = UserLd->getBasePtr(); in combineConstantPoolLoads()
51186 unsigned LdSize = LdC->getType()->getPrimitiveSizeInBits(); in combineConstantPoolLoads()
51187 unsigned UserSize = UserC->getType()->getPrimitiveSizeInBits(); in combineConstantPoolLoads()
51216 EVT RegVT = Ld->getValueType(0); in combineLoad()
51217 EVT MemVT = Ld->getMemoryVT(); in combineLoad()
51221 // For chips with slow 32-byte unaligned loads, break the 32-byte operation in combineLoad()
51222 // into two 16-byte operations. Also split non-temporal aligned loads on in combineLoad()
51223 // pre-AVX2 targets as 32-byte loads will lower to regular temporal loads. in combineLoad()
51224 ISD::LoadExtType Ext = Ld->getExtensionType(); in combineLoad()
51228 ((Ld->isNonTemporal() && !Subtarget.hasInt256() && in combineLoad()
51229 Ld->getAlign() >= Align(16)) || in combineLoad()
51231 *Ld->getMemOperand(), &Fast) && in combineLoad()
51238 SDValue Ptr1 = Ld->getBasePtr(); in combineLoad()
51244 DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr1, Ld->getPointerInfo(), in combineLoad()
51245 Ld->getOriginalAlign(), in combineLoad()
51246 Ld->getMemOperand()->getFlags()); in combineLoad()
51247 SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr2, in combineLoad()
51248 Ld->getPointerInfo().getWithOffset(HalfOffset), in combineLoad()
51249 Ld->getOriginalAlign(), in combineLoad()
51250 Ld->getMemOperand()->getFlags()); in combineLoad()
51258 // Bool vector load - attempt to cast to an integer, as we have good in combineLoad()
51265 SDValue IntLoad = DAG.getLoad(IntVT, dl, Ld->getChain(), Ld->getBasePtr(), in combineLoad()
51266 Ld->getPointerInfo(), in combineLoad()
51267 Ld->getOriginalAlign(), in combineLoad()
51268 Ld->getMemOperand()->getFlags()); in combineLoad()
51276 if (Ext == ISD::NON_EXTLOAD && Subtarget.hasAVX() && Ld->isSimple() && in combineLoad()
51278 SDValue Ptr = Ld->getBasePtr(); in combineLoad()
51279 SDValue Chain = Ld->getChain(); in combineLoad()
51280 for (SDNode *User : Chain->uses()) { in combineLoad()
51283 User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD && in combineLoad()
51284 UserLd->getChain() == Chain && UserLd->getBasePtr() == Ptr && in combineLoad()
51285 UserLd->getMemoryVT().getSizeInBits() == MemVT.getSizeInBits() && in combineLoad()
51286 !User->hasAnyUseOfValue(1) && in combineLoad()
51287 User->getValueSizeInBits(0).getFixedValue() > in combineLoad()
51301 unsigned AddrSpace = Ld->getAddressSpace(); in combineLoad()
51305 if (PtrVT != Ld->getBasePtr().getSimpleValueType()) { in combineLoad()
51307 DAG.getAddrSpaceCast(dl, PtrVT, Ld->getBasePtr(), AddrSpace, 0); in combineLoad()
51308 return DAG.getExtLoad(Ext, dl, RegVT, Ld->getChain(), Cast, in combineLoad()
51309 Ld->getPointerInfo(), MemVT, Ld->getOriginalAlign(), in combineLoad()
51310 Ld->getMemOperand()->getFlags()); in combineLoad()
51319 /// Otherwise, return -1.
51329 if (!BV || BV->getValueType(0).getVectorElementType() != MVT::i1) in getOneTrueElt()
51330 return -1; in getOneTrueElt()
51332 int TrueIndex = -1; in getOneTrueElt()
51333 unsigned NumElts = BV->getValueType(0).getVectorNumElements(); in getOneTrueElt()
51335 const SDValue &Op = BV->getOperand(i); in getOneTrueElt()
51340 return -1; in getOneTrueElt()
51341 if (ConstNode->getAPIntValue().countr_one() >= 1) { in getOneTrueElt()
51344 return -1; in getOneTrueElt()
51359 int TrueMaskElt = getOneTrueElt(MaskedOp->getMask()); in getParamsForOneTrueMaskedElt()
51365 EVT EltVT = MaskedOp->getMemoryVT().getVectorElementType(); in getParamsForOneTrueMaskedElt()
51367 Addr = MaskedOp->getBasePtr(); in getParamsForOneTrueMaskedElt()
51375 Alignment = commonAlignment(MaskedOp->getOriginalAlign(), in getParamsForOneTrueMaskedElt()
51380 /// If exactly one element of the mask is set for a non-extending masked load,
51382 /// Note: It is expected that the degenerate cases of an all-zeros or all-ones
51388 assert(ML->isUnindexed() && "Unexpected indexed masked load!"); in reduceMaskedLoadToScalarLoad()
51389 // TODO: This is not x86-specific, so it could be lifted to DAGCombiner. in reduceMaskedLoadToScalarLoad()
51402 EVT VT = ML->getValueType(0); in reduceMaskedLoadToScalarLoad()
51412 DAG.getLoad(EltVT, DL, ML->getChain(), Addr, in reduceMaskedLoadToScalarLoad()
51413 ML->getPointerInfo().getWithOffset(Offset), in reduceMaskedLoadToScalarLoad()
51414 Alignment, ML->getMemOperand()->getFlags()); in reduceMaskedLoadToScalarLoad()
51416 SDValue PassThru = DAG.getBitcast(CastVT, ML->getPassThru()); in reduceMaskedLoadToScalarLoad()
51428 assert(ML->isUnindexed() && "Unexpected indexed masked load!"); in combineMaskedLoadConstantMask()
51429 if (!ISD::isBuildVectorOfConstantSDNodes(ML->getMask().getNode())) in combineMaskedLoadConstantMask()
51433 EVT VT = ML->getValueType(0); in combineMaskedLoadConstantMask()
51439 BuildVectorSDNode *MaskBV = cast<BuildVectorSDNode>(ML->getMask()); in combineMaskedLoadConstantMask()
51440 bool LoadFirstElt = !isNullConstant(MaskBV->getOperand(0)); in combineMaskedLoadConstantMask()
51441 bool LoadLastElt = !isNullConstant(MaskBV->getOperand(NumElts - 1)); in combineMaskedLoadConstantMask()
51443 SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(), in combineMaskedLoadConstantMask()
51444 ML->getMemOperand()); in combineMaskedLoadConstantMask()
51445 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd, in combineMaskedLoadConstantMask()
51446 ML->getPassThru()); in combineMaskedLoadConstantMask()
51452 // (for example, vblendvps -> vblendps). in combineMaskedLoadConstantMask()
51454 // Don't try this if the pass-through operand is already undefined. That would in combineMaskedLoadConstantMask()
51456 if (ML->getPassThru().isUndef()) in combineMaskedLoadConstantMask()
51459 if (ISD::isBuildVectorAllZeros(ML->getPassThru().getNode())) in combineMaskedLoadConstantMask()
51462 // The new masked load has an undef pass-through operand. The select uses the in combineMaskedLoadConstantMask()
51463 // original pass-through operand. in combineMaskedLoadConstantMask()
51465 VT, DL, ML->getChain(), ML->getBasePtr(), ML->getOffset(), ML->getMask(), in combineMaskedLoadConstantMask()
51466 DAG.getUNDEF(VT), ML->getMemoryVT(), ML->getMemOperand(), in combineMaskedLoadConstantMask()
51467 ML->getAddressingMode(), ML->getExtensionType()); in combineMaskedLoadConstantMask()
51468 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML, in combineMaskedLoadConstantMask()
51469 ML->getPassThru()); in combineMaskedLoadConstantMask()
51480 if (Mld->isExpandingLoad()) in combineMaskedLoad()
51483 if (Mld->getExtensionType() == ISD::NON_EXTLOAD) { in combineMaskedLoad()
51494 // If the mask value has been legalized to a non-boolean vector, try to in combineMaskedLoad()
51496 SDValue Mask = Mld->getMask(); in combineMaskedLoad()
51498 EVT VT = Mld->getValueType(0); in combineMaskedLoad()
51502 if (N->getOpcode() != ISD::DELETED_NODE) in combineMaskedLoad()
51509 VT, SDLoc(N), Mld->getChain(), Mld->getBasePtr(), Mld->getOffset(), in combineMaskedLoad()
51510 NewMask, Mld->getPassThru(), Mld->getMemoryVT(), Mld->getMemOperand(), in combineMaskedLoad()
51511 Mld->getAddressingMode(), Mld->getExtensionType()); in combineMaskedLoad()
51517 /// If exactly one element of the mask is set for a non-truncating masked store,
51519 /// Note: It is expected that the degenerate cases of an all-zeros or all-ones
51524 // TODO: This is not x86-specific, so it could be lifted to DAGCombiner. in reduceMaskedStoreToScalarStore()
51536 SDValue Value = MS->getValue(); in reduceMaskedStoreToScalarStore()
51548 return DAG.getStore(MS->getChain(), DL, Extract, Addr, in reduceMaskedStoreToScalarStore()
51549 MS->getPointerInfo().getWithOffset(Offset), in reduceMaskedStoreToScalarStore()
51550 Alignment, MS->getMemOperand()->getFlags()); in reduceMaskedStoreToScalarStore()
51557 if (Mst->isCompressingStore()) in combineMaskedStore()
51560 EVT VT = Mst->getValue().getValueType(); in combineMaskedStore()
51564 if (Mst->isTruncatingStore()) in combineMaskedStore()
51570 // If the mask value has been legalized to a non-boolean vector, try to in combineMaskedStore()
51572 SDValue Mask = Mst->getMask(); in combineMaskedStore()
51576 if (N->getOpcode() != ISD::DELETED_NODE) in combineMaskedStore()
51582 return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Mst->getValue(), in combineMaskedStore()
51583 Mst->getBasePtr(), Mst->getOffset(), NewMask, in combineMaskedStore()
51584 Mst->getMemoryVT(), Mst->getMemOperand(), in combineMaskedStore()
51585 Mst->getAddressingMode()); in combineMaskedStore()
51588 SDValue Value = Mst->getValue(); in combineMaskedStore()
51589 if (Value.getOpcode() == ISD::TRUNCATE && Value.getNode()->hasOneUse() && in combineMaskedStore()
51591 Mst->getMemoryVT())) { in combineMaskedStore()
51592 return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Value.getOperand(0), in combineMaskedStore()
51593 Mst->getBasePtr(), Mst->getOffset(), Mask, in combineMaskedStore()
51594 Mst->getMemoryVT(), Mst->getMemOperand(), in combineMaskedStore()
51595 Mst->getAddressingMode(), true); in combineMaskedStore()
51605 EVT StVT = St->getMemoryVT(); in combineStore()
51607 SDValue StoredVal = St->getValue(); in combineStore()
51618 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), in combineStore()
51619 St->getPointerInfo(), St->getOriginalAlign(), in combineStore()
51620 St->getMemOperand()->getFlags()); in combineStore()
51624 // This will avoid a copy to k-register. in combineStore()
51631 return DAG.getStore(St->getChain(), dl, Val, in combineStore()
51632 St->getBasePtr(), St->getPointerInfo(), in combineStore()
51633 St->getOriginalAlign(), in combineStore()
51634 St->getMemOperand()->getFlags()); in combineStore()
51645 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), in combineStore()
51646 St->getPointerInfo(), St->getOriginalAlign(), in combineStore()
51647 St->getMemOperand()->getFlags()); in combineStore()
51654 // If its a v64i1 store without 64-bit support, we need two stores. in combineStore()
51657 StoredVal->ops().slice(0, 32)); in combineStore()
51660 StoredVal->ops().slice(32, 32)); in combineStore()
51663 SDValue Ptr0 = St->getBasePtr(); in combineStore()
51667 DAG.getStore(St->getChain(), dl, Lo, Ptr0, St->getPointerInfo(), in combineStore()
51668 St->getOriginalAlign(), in combineStore()
51669 St->getMemOperand()->getFlags()); in combineStore()
51671 DAG.getStore(St->getChain(), dl, Hi, Ptr1, in combineStore()
51672 St->getPointerInfo().getWithOffset(4), in combineStore()
51673 St->getOriginalAlign(), in combineStore()
51674 St->getMemOperand()->getFlags()); in combineStore()
51679 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), in combineStore()
51680 St->getPointerInfo(), St->getOriginalAlign(), in combineStore()
51681 St->getMemOperand()->getFlags()); in combineStore()
51684 // If we are saving a 32-byte vector and 32-byte stores are slow, such as on in combineStore()
51685 // Sandy Bridge, perform two 16-byte stores. in combineStore()
51689 *St->getMemOperand(), &Fast) && in combineStore()
51698 // Split under-aligned vector non-temporal stores. in combineStore()
51699 if (St->isNonTemporal() && StVT == VT && in combineStore()
51700 St->getAlign().value() < VT.getStoreSize()) { in combineStore()
51701 // ZMM/YMM nt-stores - either it can be stored as a series of shorter in combineStore()
51710 // XMM nt-stores - scalarize this to f64 nt-stores on SSE4A, else i32/i64 in combineStore()
51720 // Try to optimize v16i16->v16i8 truncating stores when BWI is not in combineStore()
51722 if (!St->isTruncatingStore() && VT == MVT::v16i8 && !Subtarget.hasBWI() && in combineStore()
51723 St->getValue().getOpcode() == ISD::TRUNCATE && in combineStore()
51724 St->getValue().getOperand(0).getValueType() == MVT::v16i16 && in combineStore()
51726 St->getValue().hasOneUse() && !DCI.isBeforeLegalizeOps()) { in combineStore()
51728 St->getValue().getOperand(0)); in combineStore()
51729 return DAG.getTruncStore(St->getChain(), dl, Ext, St->getBasePtr(), in combineStore()
51730 MVT::v16i8, St->getMemOperand()); in combineStore()
51734 if (!St->isTruncatingStore() && in combineStore()
51740 return EmitTruncSStore(IsSigned, St->getChain(), in combineStore()
51741 dl, StoredVal.getOperand(0), St->getBasePtr(), in combineStore()
51742 VT, St->getMemOperand(), DAG); in combineStore()
51746 if (!St->isTruncatingStore()) { in combineStore()
51768 return DAG.getTruncStore(St->getChain(), dl, Src, St->getBasePtr(), in combineStore()
51769 TruncVT, St->getMemOperand()); in combineStore()
51778 if (St->isTruncatingStore() && VT.isVector()) { in combineStore()
51780 if (SDValue Val = detectSSatPattern(St->getValue(), St->getMemoryVT())) in combineStore()
51781 return EmitTruncSStore(true /* Signed saturation */, St->getChain(), in combineStore()
51782 dl, Val, St->getBasePtr(), in combineStore()
51783 St->getMemoryVT(), St->getMemOperand(), DAG); in combineStore()
51784 if (SDValue Val = detectUSatPattern(St->getValue(), St->getMemoryVT(), in combineStore()
51786 return EmitTruncSStore(false /* Unsigned saturation */, St->getChain(), in combineStore()
51787 dl, Val, St->getBasePtr(), in combineStore()
51788 St->getMemoryVT(), St->getMemOperand(), DAG); in combineStore()
51795 unsigned AddrSpace = St->getAddressSpace(); in combineStore()
51799 if (PtrVT != St->getBasePtr().getSimpleValueType()) { in combineStore()
51801 DAG.getAddrSpaceCast(dl, PtrVT, St->getBasePtr(), AddrSpace, 0); in combineStore()
51803 St->getChain(), dl, StoredVal, Cast, St->getPointerInfo(), StVT, in combineStore()
51804 St->getOriginalAlign(), St->getMemOperand()->getFlags(), in combineStore()
51805 St->getAAInfo()); in combineStore()
51809 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering in combineStore()
51814 // Similarly, turn load->store of i64 into double load/stores in 32-bit mode. in combineStore()
51826 if (VT == MVT::i64 && isa<LoadSDNode>(St->getValue()) && in combineStore()
51827 cast<LoadSDNode>(St->getValue())->isSimple() && in combineStore()
51828 St->getChain().hasOneUse() && St->isSimple()) { in combineStore()
51829 auto *Ld = cast<LoadSDNode>(St->getValue()); in combineStore()
51835 if (!Ld->hasNUsesOfValue(1, 0)) in combineStore()
51841 SDValue NewLd = DAG.getLoad(MVT::f64, LdDL, Ld->getChain(), in combineStore()
51842 Ld->getBasePtr(), Ld->getMemOperand()); in combineStore()
51846 return DAG.getStore(St->getChain(), StDL, NewLd, St->getBasePtr(), in combineStore()
51847 St->getMemOperand()); in combineStore()
51850 // This is similar to the above case, but here we handle a scalar 64-bit in combineStore()
51851 // integer store that is extracted from a vector on a 32-bit target. in combineStore()
51852 // If we have SSE2, then we can treat it like a floating-point double in combineStore()
51857 St->getOperand(1).getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in combineStore()
51858 SDValue OldExtract = St->getOperand(1); in combineStore()
51865 return DAG.getStore(St->getChain(), dl, NewExtract, St->getBasePtr(), in combineStore()
51866 St->getPointerInfo(), St->getOriginalAlign(), in combineStore()
51867 St->getMemOperand()->getFlags()); in combineStore()
51878 SDValue StoredVal = N->getOperand(1); in combineVEXTRACT_STORE()
51880 EVT MemVT = St->getMemoryVT(); in combineVEXTRACT_STORE()
51888 if (N->getOpcode() != ISD::DELETED_NODE) in combineVEXTRACT_STORE()
51896 /// Return 'true' if this vector operation is "horizontal"
51897 /// and return the operands for the horizontal operation in LHS and RHS. A
51898 /// horizontal operation performs the binary operation on successive elements
51904 /// then the result of doing a horizontal operation on A and B is
51905 /// A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >.
51907 /// A horizontal-op B, for some already available A and B, and if so then LHS is
51925 // which is A horizontal-op B. in isHorizontalBinOp()
51929 "Unsupported vector type for horizontal add/sub"); in isHorizontalBinOp()
51967 // LHS = VECTOR_SHUFFLE LHS, undef, <0, 1, ..., N-1> in isHorizontalBinOp()
52023 // Check that the masks correspond to performing a horizontal operation. in isHorizontalBinOp()
52024 // AVX defines horizontal add/sub to operate independently on 128-bit lanes, in isHorizontalBinOp()
52025 // so we just repeat the inner loop if this is a 256-bit op. in isHorizontalBinOp()
52041 // this is not a horizontal operation. in isHorizontalBinOp()
52046 // Compute the post-shuffle mask index based on where the element in isHorizontalBinOp()
52050 ((Base % NumElts) & ~(NumEltsPer128BitChunk - 1)); in isHorizontalBinOp()
52052 // The low half of the 128-bit result must choose from A. in isHorizontalBinOp()
52053 // The high half of the 128-bit result must choose from B, in isHorizontalBinOp()
52069 // Avoid 128-bit multi lane shuffles if pre-AVX2 and FP (integer will split). in isHorizontalBinOp()
52077 return User->getOpcode() == HOpcode && User->getValueType(0) == VT; in isHorizontalBinOp()
52080 ForceHorizOp || (llvm::any_of(NewLHS->uses(), FoundHorizUser) && in isHorizontalBinOp()
52081 llvm::any_of(NewRHS->uses(), FoundHorizUser)); in isHorizontalBinOp()
52096 // Try to synthesize horizontal (f)hadd/hsub from (f)adds/subs of shuffles.
52099 EVT VT = N->getValueType(0); in combineToHorizontalAddSub()
52100 unsigned Opcode = N->getOpcode(); in combineToHorizontalAddSub()
52105 return N->hasOneUse() && in combineToHorizontalAddSub()
52106 N->use_begin()->getOpcode() == ISD::VECTOR_SHUFFLE && in combineToHorizontalAddSub()
52107 (N->use_begin()->getOperand(0).getOpcode() == HorizOpcode || in combineToHorizontalAddSub()
52108 N->use_begin()->getOperand(1).getOpcode() == HorizOpcode); in combineToHorizontalAddSub()
52116 SDValue LHS = N->getOperand(0); in combineToHorizontalAddSub()
52117 SDValue RHS = N->getOperand(1); in combineToHorizontalAddSub()
52133 SDValue LHS = N->getOperand(0); in combineToHorizontalAddSub()
52134 SDValue RHS = N->getOperand(1); in combineToHorizontalAddSub()
52158 // <i32 -2147483648[float -0.000000e+00]> 0
52160 // <(load 4 from constant-pool)> t0, t29
52171 EVT VT = N->getValueType(0); in combineFMulcFCMulc()
52172 SDValue LHS = N->getOperand(0); in combineFMulcFCMulc()
52173 SDValue RHS = N->getOperand(1); in combineFMulcFCMulc()
52175 N->getOpcode() == X86ISD::VFCMULC ? X86ISD::VFMULC : X86ISD::VFCMULC; in combineFMulcFCMulc()
52177 if (LHS->getOpcode() == ISD::BITCAST && RHS.hasOneUse()) { in combineFMulcFCMulc()
52179 if (XOR->getOpcode() == ISD::XOR && XOR.hasOneUse()) { in combineFMulcFCMulc()
52228 if (N->getOpcode() != ISD::FADD || !Subtarget.hasFP16() || in combineFaddCFmul()
52229 !AllowContract(N->getFlags())) in combineFaddCFmul()
52232 EVT VT = N->getValueType(0); in combineFaddCFmul()
52236 SDValue LHS = N->getOperand(0); in combineFaddCFmul()
52237 SDValue RHS = N->getOperand(1); in combineFaddCFmul()
52242 &HasNoSignedZero](SDValue N) -> bool { in combineFaddCFmul()
52247 if (Op0.hasOneUse() && AllowContract(Op0->getFlags())) { in combineFaddCFmul()
52255 ((ISD::isBuildVectorAllZeros(Op0->getOperand(2).getNode()) && in combineFaddCFmul()
52256 HasNoSignedZero(Op0->getFlags())) || in combineFaddCFmul()
52257 IsVectorAllNegativeZero(Op0->getOperand(2)))) { in combineFaddCFmul()
52280 DAG.getNode(NewOp, SDLoc(N), CVT, MulOp0, MulOp1, FAddOp1, N->getFlags()); in combineFaddCFmul()
52284 /// Do target-specific dag combines on floating-point adds/subs.
52298 EVT VT = N->getValueType(0); in combineLRINT_LLRINT()
52299 SDValue Src = N->getOperand(0); in combineLRINT_LLRINT()
52312 /// Attempt to pre-truncate inputs to arithmetic ops if it will simplify
52314 /// e.g. TRUNC( BINOP( X, Y ) ) --> BINOP( TRUNC( X ), TRUNC( Y ) )
52320 assert(N->getOpcode() == ISD::TRUNCATE && "Wrong opcode"); in combineTruncatedArithmetic()
52321 SDValue Src = N->getOperand(0); in combineTruncatedArithmetic()
52325 EVT VT = N->getValueType(0); in combineTruncatedArithmetic()
52362 // In most cases its only worth pre-truncating if we're only facing the cost in combineTruncatedArithmetic()
52367 // X86 is rubbish at scalar and vector i64 multiplies (until AVX512DQ) - its in combineTruncatedArithmetic()
52408 // Only handle vXi16 types that are at least 128-bits unless they will be in combinePMULH()
52427 // Count leading sign/zero bits on both inputs - if there are enough then in combinePMULH()
52428 // truncation back to vXi16 will be cheap - either as a pack/shuffle in combinePMULH()
52476 // adjacent pairs of 16-bit products, and saturates the result before
52477 // truncating to 16-bits.
52570 unsigned IdxN00 = ConstN00Elt->getZExtValue(); in detectPMADDUBSW()
52571 unsigned IdxN01 = ConstN01Elt->getZExtValue(); in detectPMADDUBSW()
52572 unsigned IdxN10 = ConstN10Elt->getZExtValue(); in detectPMADDUBSW()
52573 unsigned IdxN11 = ConstN11Elt->getZExtValue(); in detectPMADDUBSW()
52626 EVT VT = N->getValueType(0); in combineTruncate()
52627 SDValue Src = N->getOperand(0); in combineTruncate()
52630 // Attempt to pre-truncate inputs to arithmetic ops instead. in combineTruncate()
52664 EVT VT = N->getValueType(0); in combineVTRUNC()
52665 SDValue In = N->getOperand(0); in combineVTRUNC()
52683 /// FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000)
52691 if (N->getOpcode() == ISD::FNEG) in isFNEG()
52692 return N->getOperand(0); in isFNEG()
52698 unsigned ScalarSize = N->getValueType(0).getScalarSizeInBits(); in isFNEG()
52701 EVT VT = Op->getValueType(0); in isFNEG()
52711 // of this is VECTOR_SHUFFLE(-VEC1, UNDEF). The mask can be anything here. in isFNEG()
52717 cast<ShuffleVectorSDNode>(Op)->getMask()); in isFNEG()
52722 // -V, INDEX). in isFNEG()
52757 // Only allow bitcast from correctly-sized constant. in isFNEG()
52773 // clang-format off in negateFMAOpcode()
52787 // clang-format on in negateFMAOpcode()
52793 // clang-format off in negateFMAOpcode()
52811 // clang-format on in negateFMAOpcode()
52818 // clang-format off in negateFMAOpcode()
52828 // clang-format on in negateFMAOpcode()
52835 /// Do target-specific dag combines on floating point negations.
52839 EVT OrigVT = N->getValueType(0); in combineFneg()
52854 // use of a constant by performing (-0 - A*B) instead. in combineFneg()
52857 Arg->getFlags().hasNoSignedZeros() && Subtarget.hasAnyFMA()) { in combineFneg()
52887 SDNodeFlags Flags = Op.getNode()->getFlags(); in getNegatedExpression()
52922 // Fill in the non-negated ops with the original values. in getNegatedExpression()
52942 MVT VT = N->getSimpleValueType(0); in lowerX86FPLogicOp()
52953 SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0)); in lowerX86FPLogicOp()
52954 SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1)); in lowerX86FPLogicOp()
52956 switch (N->getOpcode()) { in lowerX86FPLogicOp()
52957 // clang-format off in lowerX86FPLogicOp()
52963 // clang-format on in lowerX86FPLogicOp()
52970 /// Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
52972 if (N->getOpcode() != ISD::XOR) in foldXor1SetCC()
52975 SDValue LHS = N->getOperand(0); in foldXor1SetCC()
52976 if (!isOneConstant(N->getOperand(1)) || LHS->getOpcode() != X86ISD::SETCC) in foldXor1SetCC()
52980 X86::CondCode(LHS->getConstantOperandVal(0))); in foldXor1SetCC()
52982 return getSETCC(NewCC, LHS->getOperand(1), DL, DAG); in foldXor1SetCC()
52987 assert((N->getOpcode() == ISD::XOR || N->getOpcode() == ISD::SUB) && in combineXorSubCTLZ()
52992 EVT VT = N->getValueType(0); in combineXorSubCTLZ()
52997 SDValue N0 = N->getOperand(0); in combineXorSubCTLZ()
52998 SDValue N1 = N->getOperand(1); in combineXorSubCTLZ()
53010 } else if (N->getOpcode() == ISD::SUB) { in combineXorSubCTLZ()
53023 if (C->getZExtValue() != uint64_t(OpCTLZ.getValueSizeInBits() - 1)) in combineXorSubCTLZ()
53044 SDValue N0 = N->getOperand(0); in combineXor()
53045 SDValue N1 = N->getOperand(1); in combineXor()
53046 EVT VT = N->getValueType(0); in combineXor()
53087 // Fold not(iX bitcast(vXi1)) -> (iX bitcast(not(vec))) for legal boolvecs. in combineXor()
53098 // Fold not(insert_subvector(undef,sub)) -> insert_subvector(undef,not(sub)) in combineXor()
53109 // Fold xor(zext(xor(x,c1)),c2) -> xor(zext(x),xor(zext(c1),c2)) in combineXor()
53110 // Fold xor(truncate(xor(x,c1)),c2) -> xor(truncate(x),xor(truncate(c1),c2)) in combineXor()
53113 N0.getOperand(0).getOpcode() == N->getOpcode()) { in combineXor()
53117 if (N1C && !N1C->isOpaque() && N001C && !N001C->isOpaque()) { in combineXor()
53134 SDValue N0 = N->getOperand(0); in combineBITREVERSE()
53135 EVT VT = N->getValueType(0); in combineBITREVERSE()
53137 // Convert a (iX bitreverse(bitcast(vXi1 X))) -> (iX bitcast(shuffle(X))) in combineBITREVERSE()
53148 ReverseMask[I] = (NumElts - 1) - I; in combineBITREVERSE()
53162 unsigned Opcode = N->getOpcode(); in combineAVG()
53163 SDValue N0 = N->getOperand(0); in combineAVG()
53164 SDValue N1 = N->getOperand(1); in combineAVG()
53165 EVT VT = N->getValueType(0); in combineAVG()
53169 // avgceils(x,y) -> flipsign(avgceilu(flipsign(x),flipsign(y))) in combineAVG()
53186 EVT VT = N->getValueType(0); in combineBEXTR()
53189 // TODO - Constant Folding. in combineBEXTR()
53208 /// to be used as a replacement operand with operations (eg, bitwise-and) where
53223 SDValue N0 = N->getOperand(0); in combineFAndFNotToFAndn()
53224 SDValue N1 = N->getOperand(1); in combineFAndFNotToFAndn()
53225 EVT VT = N->getValueType(0); in combineFAndFNotToFAndn()
53238 return C && C->getConstantFPValue()->isAllOnesValue(); in combineFAndFNotToFAndn()
53241 // fand (fxor X, -1), Y --> fandn X, Y in combineFAndFNotToFAndn()
53245 // fand X, (fxor Y, -1) --> fandn Y, X in combineFAndFNotToFAndn()
53252 /// Do target-specific dag combines on X86ISD::FAND nodes.
53255 // FAND(0.0, x) -> 0.0 in combineFAnd()
53256 if (SDValue V = getNullFPConstForNullVal(N->getOperand(0), DAG, Subtarget)) in combineFAnd()
53259 // FAND(x, 0.0) -> 0.0 in combineFAnd()
53260 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget)) in combineFAnd()
53269 /// Do target-specific dag combines on X86ISD::FANDN nodes.
53272 // FANDN(0.0, x) -> x in combineFAndn()
53273 if (isNullFPScalarOrVectorConst(N->getOperand(0))) in combineFAndn()
53274 return N->getOperand(1); in combineFAndn()
53276 // FANDN(x, 0.0) -> 0.0 in combineFAndn()
53277 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget)) in combineFAndn()
53283 /// Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
53287 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR); in combineFOr()
53289 // F[X]OR(0.0, x) -> x in combineFOr()
53290 if (isNullFPScalarOrVectorConst(N->getOperand(0))) in combineFOr()
53291 return N->getOperand(1); in combineFOr()
53293 // F[X]OR(x, 0.0) -> x in combineFOr()
53294 if (isNullFPScalarOrVectorConst(N->getOperand(1))) in combineFOr()
53295 return N->getOperand(0); in combineFOr()
53303 /// Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
53305 assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX); in combineFMinFMax()
53312 // If we run in unsafe-math mode, then convert the FMAX and FMIN nodes in combineFMinFMax()
53315 switch (N->getOpcode()) { in combineFMinFMax()
53321 return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0), in combineFMinFMax()
53322 N->getOperand(0), N->getOperand(1)); in combineFMinFMax()
53327 EVT VT = N->getValueType(0); in combineFMinNumFMaxNum()
53339 SDValue Op0 = N->getOperand(0); in combineFMinNumFMaxNum()
53340 SDValue Op1 = N->getOperand(1); in combineFMinNumFMaxNum()
53342 auto MinMaxOp = N->getOpcode() == ISD::FMAXNUM ? X86ISD::FMAX : X86ISD::FMIN; in combineFMinNumFMaxNum()
53346 if (DAG.getTarget().Options.NoNaNsFPMath || N->getFlags().hasNoNaNs()) in combineFMinNumFMaxNum()
53347 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags()); in combineFMinNumFMaxNum()
53349 // If one of the operands is known non-NaN use the native min/max instructions in combineFMinNumFMaxNum()
53350 // with the non-NaN input as second operand. in combineFMinNumFMaxNum()
53352 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags()); in combineFMinNumFMaxNum()
53354 return DAG.getNode(MinMaxOp, DL, VT, Op1, Op0, N->getFlags()); in combineFMinNumFMaxNum()
53368 // ---------------- in combineFMinNumFMaxNum()
53370 // Op0 ---------------- in combineFMinNumFMaxNum()
53372 // ---------------- in combineFMinNumFMaxNum()
53393 EVT VT = N->getValueType(0); in combineX86INT_TO_FP()
53401 SDValue In = N->getOperand(0); in combineX86INT_TO_FP()
53405 assert(InVT.is128BitVector() && "Expected 128-bit input vector"); in combineX86INT_TO_FP()
53406 LoadSDNode *LN = cast<LoadSDNode>(N->getOperand(0)); in combineX86INT_TO_FP()
53412 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT, in combineX86INT_TO_FP()
53426 bool IsStrict = N->isTargetStrictFPOpcode(); in combineCVTP2I_CVTTP2I()
53427 EVT VT = N->getValueType(0); in combineCVTP2I_CVTTP2I()
53430 SDValue In = N->getOperand(IsStrict ? 1 : 0); in combineCVTP2I_CVTTP2I()
53434 assert(InVT.is128BitVector() && "Expected 128-bit input vector"); in combineCVTP2I_CVTTP2I()
53443 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other}, in combineCVTP2I_CVTTP2I()
53444 {N->getOperand(0), DAG.getBitcast(InVT, VZLoad)}); in combineCVTP2I_CVTTP2I()
53448 DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(InVT, VZLoad)); in combineCVTP2I_CVTTP2I()
53460 /// Do target-specific dag combines on X86ISD::ANDNP nodes.
53464 SDValue N0 = N->getOperand(0); in combineAndnp()
53465 SDValue N1 = N->getOperand(1); in combineAndnp()
53466 MVT VT = N->getSimpleValueType(0); in combineAndnp()
53471 // ANDNP(undef, x) -> 0 in combineAndnp()
53472 // ANDNP(x, undef) -> 0 in combineAndnp()
53476 // ANDNP(0, x) -> x in combineAndnp()
53480 // ANDNP(x, 0) -> 0 in combineAndnp()
53484 // ANDNP(x, -1) -> NOT(x) -> XOR(x, -1) in combineAndnp()
53493 // ANDNP(x,NOT(y)) -> AND(NOT(x),NOT(y)) -> NOT(OR(X,Y)). in combineAndnp()
53494 if (N1->hasOneUse()) in combineAndnp()
53517 if (N0->hasOneUse()) { in combineAndnp()
53547 // We can't assume an undef src element gives an undef dst - the in combineAndnp()
53570 if (N->getOpcode() != ISD::DELETED_NODE) in combineAndnp()
53581 SDValue N1 = N->getOperand(1); in combineBT()
53587 if (N->getOpcode() != ISD::DELETED_NODE) in combineBT()
53597 bool IsStrict = N->getOpcode() == X86ISD::STRICT_CVTPH2PS; in combineCVTPH2PS()
53598 SDValue Src = N->getOperand(IsStrict ? 1 : 0); in combineCVTPH2PS()
53600 if (N->getValueType(0) == MVT::v4f32 && Src.getValueType() == MVT::v8i16) { in combineCVTPH2PS()
53604 if (N->getOpcode() != ISD::DELETED_NODE) in combineCVTPH2PS()
53611 LoadSDNode *LN = cast<LoadSDNode>(N->getOperand(IsStrict ? 1 : 0)); in combineCVTPH2PS()
53616 N->getOpcode(), dl, {MVT::v4f32, MVT::Other}, in combineCVTPH2PS()
53617 {N->getOperand(0), DAG.getBitcast(MVT::v8i16, VZLoad)}); in combineCVTPH2PS()
53620 SDValue Convert = DAG.getNode(N->getOpcode(), dl, MVT::v4f32, in combineCVTPH2PS()
53637 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG); in combineSextInRegCmov()
53639 EVT DstVT = N->getValueType(0); in combineSextInRegCmov()
53641 SDValue N0 = N->getOperand(0); in combineSextInRegCmov()
53642 SDValue N1 = N->getOperand(1); in combineSextInRegCmov()
53643 EVT ExtraVT = cast<VTSDNode>(N1)->getVT(); in combineSextInRegCmov()
53699 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG); in combineSignExtendInReg()
53704 EVT VT = N->getValueType(0); in combineSignExtendInReg()
53705 SDValue N0 = N->getOperand(0); in combineSignExtendInReg()
53706 SDValue N1 = N->getOperand(1); in combineSignExtendInReg()
53707 EVT ExtraVT = cast<VTSDNode>(N1)->getVT(); in combineSignExtendInReg()
53711 // both SSE and AVX2 since there is no sign-extended shift right in combineSignExtendInReg()
53712 // operation on a vector with 64-bit elements. in combineSignExtendInReg()
53713 //(sext_in_reg (v4i64 anyext (v4i32 x )), ExtraVT) -> in combineSignExtendInReg()
53739 /// sext(add_nsw(x, C)) --> add(sext(x), C_sext)
53740 /// zext(add_nuw(x, C)) --> add(zext(x), C_zext)
53746 if (Ext->getOpcode() != ISD::SIGN_EXTEND && in promoteExtBeforeAdd()
53747 Ext->getOpcode() != ISD::ZERO_EXTEND) in promoteExtBeforeAdd()
53751 EVT VT = Ext->getValueType(0); in promoteExtBeforeAdd()
53755 SDValue Add = Ext->getOperand(0); in promoteExtBeforeAdd()
53761 bool Sext = Ext->getOpcode() == ISD::SIGN_EXTEND; in promoteExtBeforeAdd()
53762 bool NSW = Add->getFlags().hasNoSignedWrap(); in promoteExtBeforeAdd()
53763 bool NUW = Add->getFlags().hasNoUnsignedWrap(); in promoteExtBeforeAdd()
53785 for (auto *User : Ext->uses()) { in promoteExtBeforeAdd()
53786 if (User->getOpcode() == ISD::ADD || User->getOpcode() == ISD::SHL) { in promoteExtBeforeAdd()
53795 int64_t AddC = Sext ? AddOp1C->getSExtValue() : AddOp1C->getZExtValue(); in promoteExtBeforeAdd()
53796 SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0); in promoteExtBeforeAdd()
53800 // sign-extended. in promoteExtBeforeAdd()
53808 // operands and the result of CMOV is not used anywhere else - promote CMOV
53811 // (or more) pseudo-CMOVs only when they go one-after-another and
53815 // 3) 16-bit CMOV encoding is 4 bytes, 32-bit CMOV is 3-byte, so this
53816 // promotion is also good in terms of code-size.
53817 // (64-bit CMOV is 4-bytes, that's why we don't do 32-bit => 64-bit
53820 SDValue CMovN = Extend->getOperand(0); in combineToExtendCMOV()
53824 EVT TargetVT = Extend->getValueType(0); in combineToExtendCMOV()
53825 unsigned ExtendOpcode = Extend->getOpcode(); in combineToExtendCMOV()
53868 SDValue N0 = N->getOperand(0); in combineExtSetcc()
53869 EVT VT = N->getValueType(0); in combineExtSetcc()
53892 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in combineExtSetcc()
53904 if (N->getOpcode() == ISD::ZERO_EXTEND) in combineExtSetcc()
53913 SDValue N0 = N->getOperand(0); in combineSext()
53914 EVT VT = N->getValueType(0); in combineSext()
53917 // (i32 (sext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry)) in combineSext()
53920 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0), in combineSext()
53921 N0->getOperand(1)); in combineSext()
53943 if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), DL, VT, N0, in combineSext()
53974 return Use->getOpcode() != ISD::FMA && Use->getOpcode() != ISD::STRICT_FMA; in getInvertedVectorForFMA()
53976 if (llvm::any_of(V->uses(), IsNotFMA)) in getInvertedVectorForFMA()
53982 for (const SDValue &Op : V->op_values()) { in getInvertedVectorForFMA()
53984 Ops.push_back(DAG.getConstantFP(-Cst->getValueAPF(), SDLoc(Op), EltVT)); in getInvertedVectorForFMA()
53997 if (llvm::any_of(NV->uses(), IsNotFMA)) in getInvertedVectorForFMA()
54004 for (const SDValue &Op : V->op_values()) { in getInvertedVectorForFMA()
54006 if (Cst->isNegative()) in getInvertedVectorForFMA()
54018 EVT VT = N->getValueType(0); in combineFMA()
54019 bool IsStrict = N->isStrictFPOpcode() || N->isTargetStrictFPOpcode(); in combineFMA()
54026 SDValue A = N->getOperand(IsStrict ? 1 : 0); in combineFMA()
54027 SDValue B = N->getOperand(IsStrict ? 2 : 1); in combineFMA()
54028 SDValue C = N->getOperand(IsStrict ? 3 : 2); in combineFMA()
54030 // If the operation allows fast-math and the target does not support FMA, in combineFMA()
54032 SDNodeFlags Flags = N->getFlags(); in combineFMA()
54085 negateFMAOpcode(N->getOpcode(), NegA != NegB, NegC, false); in combineFMA()
54087 // Propagate fast-math-flags to new FMA node. in combineFMA()
54090 assert(N->getNumOperands() == 4 && "Shouldn't be greater than 4"); in combineFMA()
54092 {N->getOperand(0), A, B, C}); in combineFMA()
54094 if (N->getNumOperands() == 4) in combineFMA()
54095 return DAG.getNode(NewOpcode, dl, VT, A, B, C, N->getOperand(3)); in combineFMA()
54100 // Combine FMADDSUB(A, B, FNEG(C)) -> FMSUBADD(A, B, C)
54101 // Combine FMSUBADD(A, B, FNEG(C)) -> FMADDSUB(A, B, C)
54105 EVT VT = N->getValueType(0); in combineFMADDSUB()
54110 SDValue N2 = N->getOperand(2); in combineFMADDSUB()
54116 unsigned NewOpcode = negateFMAOpcode(N->getOpcode(), false, true, false); in combineFMADDSUB()
54118 if (N->getNumOperands() == 4) in combineFMADDSUB()
54119 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1), in combineFMADDSUB()
54120 NegN2, N->getOperand(3)); in combineFMADDSUB()
54121 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1), in combineFMADDSUB()
54129 SDValue N0 = N->getOperand(0); in combineZext()
54130 EVT VT = N->getValueType(0); in combineZext()
54132 // (i32 (aext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry)) in combineZext()
54134 if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ANY_EXTEND && in combineZext()
54136 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0), in combineZext()
54137 N0->getOperand(1)); in combineZext()
54157 if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), dl, VT, N0, in combineZext()
54188 /// pre-promote its result type since vXi1 vectors don't get promoted
54207 const ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); in combineSetCC()
54208 const SDValue LHS = N->getOperand(0); in combineSetCC()
54209 const SDValue RHS = N->getOperand(1); in combineSetCC()
54210 EVT VT = N->getValueType(0); in combineSetCC()
54227 // cmpeq(or(X,Y),X) --> cmpeq(and(~X,Y),0) in combineSetCC()
54228 // cmpne(or(X,Y),X) --> cmpne(and(~X,Y),0) in combineSetCC()
54230 if (N0.getOpcode() == ISD::OR && N0->hasOneUse()) { in combineSetCC()
54245 // cmpeq(and(X,Y),Y) --> cmpeq(and(~X,Y),0) in combineSetCC()
54246 // cmpne(and(X,Y),Y) --> cmpne(and(~X,Y),0) in combineSetCC()
54248 if (N0.getOpcode() == ISD::AND && N0->hasOneUse()) { in combineSetCC()
54263 // cmpeq(trunc(x),C) --> cmpeq(x,C) in combineSetCC()
54264 // cmpne(trunc(x),C) --> cmpne(x,C) in combineSetCC()
54280 // icmp eq Abs(X) C -> in combineSetCC()
54281 // (icmp eq A, C) | (icmp eq A, -C) in combineSetCC()
54282 // icmp ne Abs(X) C -> in combineSetCC()
54283 // (icmp ne A, C) & (icmp ne A, -C) in combineSetCC()
54289 const APInt &CInt = C->getAPIntValue(); in combineSetCC()
54295 DL, VT, BaseOp, DAG.getConstant(-CInt, DL, OpVT), CC); in combineSetCC()
54413 // -> `(icmp ult (add x, -C), 2)` in combineSetCC()
54417 // in worse codegen. So, undo the middle-end transform and go back to `(or in combineSetCC()
54440 // If we had `(add x, -1)` and can lower with `umin`, don't transform as in combineSetCC()
54457 else if ((CC == ISD::SETUGT && (-CmpC) == 3) || in combineSetCC()
54458 (CC == ISD::SETUGE && (-CmpC) == 2)) { in combineSetCC()
54474 // For an SSE1-only target, lower a comparison of v4f32 to X86ISD::CMPP early in combineSetCC()
54480 // X pred 0.0 --> X pred -X in combineSetCC()
54496 SDValue Src = N->getOperand(0); in combineMOVMSK()
54498 MVT VT = N->getSimpleValueType(0); in combineMOVMSK()
54518 // Look through int->fp bitcasts that don't change the element width. in combineMOVMSK()
54524 // Fold movmsk(not(x)) -> not(movmsk(x)) to improve folding of movmsk results in combineMOVMSK()
54535 // Fold movmsk(icmp_sgt(x,-1)) -> not(movmsk(x)) to improve folding of movmsk in combineMOVMSK()
54546 // Fold movmsk(icmp_eq(and(x,c1),c1)) -> movmsk(shl(x,c2)) in combineMOVMSK()
54547 // Fold movmsk(icmp_eq(and(x,c1),0)) -> movmsk(not(shl(x,c2))) in combineMOVMSK()
54549 // Use KnownBits to determine if only a single bit is non-zero in combineMOVMSK()
54563 // vXi8 shifts - we only care about the signbit so can use PSLLW. in combineMOVMSK()
54579 // Fold movmsk(logic(X,C)) -> logic(movmsk(X),C) in combineMOVMSK()
54580 if (N->isOnlyUserOf(Src.getNode())) { in combineMOVMSK()
54613 MVT VT = N->getSimpleValueType(0); in combineTESTP()
54628 SDValue Mask = MemOp->getMask(); in combineX86GatherScatter()
54635 if (N->getOpcode() != ISD::DELETED_NODE) in combineX86GatherScatter()
54650 SDValue Ops[] = { Gather->getChain(), Gather->getPassThru(), in rebuildGatherScatter()
54651 Gather->getMask(), Base, Index, Scale } ; in rebuildGatherScatter()
54652 return DAG.getMaskedGather(Gather->getVTList(), in rebuildGatherScatter()
54653 Gather->getMemoryVT(), DL, Ops, in rebuildGatherScatter()
54654 Gather->getMemOperand(), in rebuildGatherScatter()
54655 Gather->getIndexType(), in rebuildGatherScatter()
54656 Gather->getExtensionType()); in rebuildGatherScatter()
54659 SDValue Ops[] = { Scatter->getChain(), Scatter->getValue(), in rebuildGatherScatter()
54660 Scatter->getMask(), Base, Index, Scale }; in rebuildGatherScatter()
54661 return DAG.getMaskedScatter(Scatter->getVTList(), in rebuildGatherScatter()
54662 Scatter->getMemoryVT(), DL, in rebuildGatherScatter()
54663 Ops, Scatter->getMemOperand(), in rebuildGatherScatter()
54664 Scatter->getIndexType(), in rebuildGatherScatter()
54665 Scatter->isTruncatingStore()); in rebuildGatherScatter()
54672 SDValue Index = GorS->getIndex(); in combineGatherScatter()
54673 SDValue Base = GorS->getBasePtr(); in combineGatherScatter()
54674 SDValue Scale = GorS->getScale(); in combineGatherScatter()
54680 // Shrink constant indices if they are larger than 32-bits. in combineGatherScatter()
54688 if (BV->isConstant() && IndexWidth > 32 && in combineGatherScatter()
54689 DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) { in combineGatherScatter()
54703 DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) { in combineGatherScatter()
54718 uint64_t ScaleAmt = Scale->getAsZExtVal(); in combineGatherScatter()
54721 if (ConstantSDNode *C = BV->getConstantSplatNode(&UndefElts)) { in combineGatherScatter()
54722 // FIXME: Allow non-constant? in combineGatherScatter()
54725 APInt Adder = C->getAPIntValue() * ScaleAmt; in combineGatherScatter()
54736 if (BV->isConstant() && isa<ConstantSDNode>(Base) && in combineGatherScatter()
54764 SDValue Mask = GorS->getMask(); in combineGatherScatter()
54768 if (N->getOpcode() != ISD::DELETED_NODE) in combineGatherScatter()
54781 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(0)); in combineX86SetCC()
54782 SDValue EFLAGS = N->getOperand(1); in combineX86SetCC()
54795 SDValue EFLAGS = N->getOperand(3); in combineBrCond()
54796 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(2)); in combineBrCond()
54803 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0), in combineBrCond()
54804 N->getOperand(1), Cond, Flags); in combineBrCond()
54813 // Take advantage of vector comparisons (etc.) producing 0 or -1 in each lane in combineVectorCompareAndMaskUnaryOp()
54817 // UNARYOP(AND(VECTOR_CMP(x,y), constant)) --> in combineVectorCompareAndMaskUnaryOp()
54824 EVT VT = N->getValueType(0); in combineVectorCompareAndMaskUnaryOp()
54825 bool IsStrict = N->isStrictFPOpcode(); in combineVectorCompareAndMaskUnaryOp()
54827 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0); in combineVectorCompareAndMaskUnaryOp()
54834 // make the transformation for non-constant splats as well, but it's unclear in combineVectorCompareAndMaskUnaryOp()
54839 if (!BV->isConstant()) in combineVectorCompareAndMaskUnaryOp()
54844 EVT IntVT = BV->getValueType(0); in combineVectorCompareAndMaskUnaryOp()
54849 SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other}, in combineVectorCompareAndMaskUnaryOp()
54850 {N->getOperand(0), SDValue(BV, 0)}); in combineVectorCompareAndMaskUnaryOp()
54852 SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0)); in combineVectorCompareAndMaskUnaryOp()
54855 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0), in combineVectorCompareAndMaskUnaryOp()
54866 /// If we are converting a value to floating-point, try to replace scalar
54873 SDValue Trunc = N->getOperand(0); in combineToFPTruncExtElt()
54889 // inttofp (trunc (extelt X, 0)) --> inttofp (extelt (bitcast X), 0) in combineToFPTruncExtElt()
54898 return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), NewExtElt); in combineToFPTruncExtElt()
54903 bool IsStrict = N->isStrictFPOpcode(); in combineUIntToFP()
54904 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0); in combineUIntToFP()
54905 EVT VT = N->getValueType(0); in combineUIntToFP()
54911 // UINT_TO_FP(vXi1~15) -> SINT_TO_FP(ZEXT(vXi1~15 to vXi16)) in combineUIntToFP()
54912 // UINT_TO_FP(vXi17~31) -> SINT_TO_FP(ZEXT(vXi17~31 to vXi32)) in combineUIntToFP()
54914 // UINT_TO_FP(vXi1~31) -> SINT_TO_FP(ZEXT(vXi1~31 to vXi32)) in combineUIntToFP()
54915 // UINT_TO_FP(vXi33~63) -> SINT_TO_FP(ZEXT(vXi33~63 to vXi64)) in combineUIntToFP()
54931 {N->getOperand(0), P}); in combineUIntToFP()
54935 // UINT_TO_FP(vXi1) -> SINT_TO_FP(ZEXT(vXi1 to vXi32)) in combineUIntToFP()
54936 // UINT_TO_FP(vXi8) -> SINT_TO_FP(ZEXT(vXi8 to vXi32)) in combineUIntToFP()
54937 // UINT_TO_FP(vXi16) -> SINT_TO_FP(ZEXT(vXi16 to vXi32)) in combineUIntToFP()
54947 {N->getOperand(0), P}); in combineUIntToFP()
54954 SDNodeFlags Flags = N->getFlags(); in combineUIntToFP()
54958 {N->getOperand(0), Op0}); in combineUIntToFP()
54970 bool IsStrict = N->isStrictFPOpcode(); in combineSIntToFP()
54975 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0); in combineSIntToFP()
54976 EVT VT = N->getValueType(0); in combineSIntToFP()
54982 // SINT_TO_FP(vXi1~15) -> SINT_TO_FP(SEXT(vXi1~15 to vXi16)) in combineSIntToFP()
54983 // SINT_TO_FP(vXi17~31) -> SINT_TO_FP(SEXT(vXi17~31 to vXi32)) in combineSIntToFP()
54985 // SINT_TO_FP(vXi1~31) -> SINT_TO_FP(ZEXT(vXi1~31 to vXi32)) in combineSIntToFP()
54986 // SINT_TO_FP(vXi33~63) -> SINT_TO_FP(SEXT(vXi33~63 to vXi64)) in combineSIntToFP()
55002 {N->getOperand(0), P}); in combineSIntToFP()
55006 // SINT_TO_FP(vXi1) -> SINT_TO_FP(SEXT(vXi1 to vXi32)) in combineSIntToFP()
55007 // SINT_TO_FP(vXi8) -> SINT_TO_FP(SEXT(vXi8 to vXi32)) in combineSIntToFP()
55008 // SINT_TO_FP(vXi16) -> SINT_TO_FP(SEXT(vXi16 to vXi32)) in combineSIntToFP()
55016 {N->getOperand(0), P}); in combineSIntToFP()
55026 if (NumSignBits >= (BitWidth - 31)) { in combineSIntToFP()
55035 {N->getOperand(0), Trunc}); in combineSIntToFP()
55043 { 0, 2, -1, -1 }); in combineSIntToFP()
55046 {N->getOperand(0), Shuf}); in combineSIntToFP()
55052 // a 32-bit target where SSE doesn't support i64->FP operations. in combineSIntToFP()
55066 if (Ld->isSimple() && !VT.isVector() && ISD::isNormalLoad(Op0.getNode()) && in combineSIntToFP()
55069 Subtarget.getTargetLowering()->BuildFILD( in combineSIntToFP()
55070 VT, InVT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(), in combineSIntToFP()
55071 Ld->getPointerInfo(), Ld->getOriginalAlign(), DAG); in combineSIntToFP()
55089 for (const SDNode *User : Flags->uses()) { in needCarryOrOverflowFlag()
55091 switch (User->getOpcode()) { in needCarryOrOverflowFlag()
55097 CC = (X86::CondCode)User->getConstantOperandVal(0); in needCarryOrOverflowFlag()
55101 CC = (X86::CondCode)User->getConstantOperandVal(2); in needCarryOrOverflowFlag()
55106 // clang-format off in needCarryOrOverflowFlag()
55114 // clang-format on in needCarryOrOverflowFlag()
55124 for (const SDNode *User : Flags->uses()) { in onlyZeroFlagUsed()
55126 switch (User->getOpcode()) { in onlyZeroFlagUsed()
55140 X86::CondCode CC = (X86::CondCode)User->getConstantOperandVal(CCOpNo); in onlyZeroFlagUsed()
55152 if (!isNullConstant(N->getOperand(1))) in combineCMP()
55160 SDValue Op = N->getOperand(0); in combineCMP()
55177 unsigned MaskBits = BitWidth - ShAmt.getZExtValue(); in combineCMP()
55192 // (and (extract_elt (kshiftr vXi1, C), 0), 1) -> (and (bc vXi1), 1<<C) in combineCMP()
55220 // Peek through any zero-extend if we're only testing for a zero result. in combineCMP()
55238 // i32 truncated op to prevent partial-reg compares of promoted ops. in combineCMP()
55298 assert((X86ISD::ADD == N->getOpcode() || X86ISD::SUB == N->getOpcode()) && in combineX86AddSub()
55302 SDValue LHS = N->getOperand(0); in combineX86AddSub()
55303 SDValue RHS = N->getOperand(1); in combineX86AddSub()
55305 bool IsSub = X86ISD::SUB == N->getOpcode(); in combineX86AddSub()
55308 if (IsSub && isOneConstant(N->getOperand(1)) && !N->hasAnyUseOfValue(0)) in combineX86AddSub()
55313 if (!N->hasAnyUseOfValue(1)) { in combineX86AddSub()
55321 SDVTList VTs = DAG.getVTList(N->getValueType(0)); in combineX86AddSub()
55330 MatchGeneric(RHS, LHS, X86ISD::SUB == N->getOpcode()); in combineX86AddSub()
55339 SDValue LHS = N->getOperand(0); in combineSBB()
55340 SDValue RHS = N->getOperand(1); in combineSBB()
55341 SDValue BorrowIn = N->getOperand(2); in combineSBB()
55344 MVT VT = N->getSimpleValueType(0); in combineSBB()
55349 // Fold SBB(SUB(X,Y),0,Carry) -> SBB(X,Y,Carry) in combineSBB()
55352 !N->hasAnyUseOfValue(1)) in combineSBB()
55353 return DAG.getNode(X86ISD::SBB, SDLoc(N), N->getVTList(), LHS.getOperand(0), in combineSBB()
55362 SDValue LHS = N->getOperand(0); in combineADC()
55363 SDValue RHS = N->getOperand(1); in combineADC()
55364 SDValue CarryIn = N->getOperand(2); in combineADC()
55370 return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), RHS, LHS, in combineADC()
55376 if (LHSC && RHSC && LHSC->isZero() && RHSC->isZero() && in combineADC()
55381 EVT VT = N->getValueType(0); in combineADC()
55382 SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1)); in combineADC()
55391 // Fold ADC(C1,C2,Carry) -> ADC(0,C1+C2,Carry) in combineADC()
55394 if (LHSC && RHSC && !LHSC->isZero() && !N->hasAnyUseOfValue(1)) { in combineADC()
55396 APInt Sum = LHSC->getAPIntValue() + RHSC->getAPIntValue(); in combineADC()
55397 return DAG.getNode(X86ISD::ADC, DL, N->getVTList(), in combineADC()
55403 MVT VT = N->getSimpleValueType(0); in combineADC()
55408 // Fold ADC(ADD(X,Y),0,Carry) -> ADC(X,Y,Carry) in combineADC()
55410 if (LHS.getOpcode() == ISD::ADD && RHSC && RHSC->isZero() && in combineADC()
55411 !N->hasAnyUseOfValue(1)) in combineADC()
55412 return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), LHS.getOperand(0), in combineADC()
55457 SDValue Op0L = Op0->getOperand(i), Op1L = Op1->getOperand(i), in matchPMADDWD()
55458 Op0H = Op0->getOperand(i + 1), Op1H = Op1->getOperand(i + 1); in matchPMADDWD()
55465 auto *Const0L = dyn_cast<ConstantSDNode>(Op0L->getOperand(1)); in matchPMADDWD()
55466 auto *Const1L = dyn_cast<ConstantSDNode>(Op1L->getOperand(1)); in matchPMADDWD()
55467 auto *Const0H = dyn_cast<ConstantSDNode>(Op0H->getOperand(1)); in matchPMADDWD()
55468 auto *Const1H = dyn_cast<ConstantSDNode>(Op1H->getOperand(1)); in matchPMADDWD()
55471 unsigned Idx0L = Const0L->getZExtValue(), Idx1L = Const1L->getZExtValue(), in matchPMADDWD()
55472 Idx0H = Const0H->getZExtValue(), Idx1H = Const1H->getZExtValue(); in matchPMADDWD()
55490 Mul = Op0L->getOperand(0); in matchPMADDWD()
55491 if (Mul->getOpcode() != ISD::MUL || in matchPMADDWD()
55496 if (Mul != Op0L->getOperand(0) || Mul != Op1L->getOperand(0) || in matchPMADDWD()
55497 Mul != Op0H->getOperand(0) || Mul != Op1H->getOperand(0)) in matchPMADDWD()
55596 unsigned IdxN00 = ConstN00Elt->getZExtValue(); in matchPMADDWD_2()
55597 unsigned IdxN01 = ConstN01Elt->getZExtValue(); in matchPMADDWD_2()
55598 unsigned IdxN10 = ConstN10Elt->getZExtValue(); in matchPMADDWD_2()
55599 unsigned IdxN11 = ConstN11Elt->getZExtValue(); in matchPMADDWD_2()
55662 // ADD(VPMADDWD(X,Y),VPMADDWD(Z,W)) -> VPMADDWD(SHUFFLE(X,Z), SHUFFLE(Y,W))
55671 // TODO: Add 256/512-bit support once VPMADDWD combines with shuffles. in combineAddOfPMADDWD()
55709 /// earlier folds that may be used to turn select-of-constants into logic hacks.
55713 // If an operand is zero, add-of-0 gets simplified away, so that's clearly in pushAddIntoCmovOfConsts()
55714 // better because we eliminate 1-2 instructions. This transform is still in pushAddIntoCmovOfConsts()
55717 // immediate asm operands (fit in 32-bits). in pushAddIntoCmovOfConsts()
55730 SDValue Cmov = N->getOperand(0); in pushAddIntoCmovOfConsts()
55731 SDValue OtherOp = N->getOperand(1); in pushAddIntoCmovOfConsts()
55742 EVT VT = N->getValueType(0); in pushAddIntoCmovOfConsts()
55749 // a 3-operand LEA which is likely slower than a 2-operand LEA. in pushAddIntoCmovOfConsts()
55753 all_of(N->uses(), [&](SDNode *Use) { in pushAddIntoCmovOfConsts()
55755 return MemNode && MemNode->getBasePtr().getNode() == N; in pushAddIntoCmovOfConsts()
55757 // add (cmov C1, C2), add (X, Y) --> add (cmov (add X, C1), (add X, C2)), Y in pushAddIntoCmovOfConsts()
55768 // add (cmov C1, C2), OtherOp --> cmov (add OtherOp, C1), (add OtherOp, C2) in pushAddIntoCmovOfConsts()
55778 EVT VT = N->getValueType(0); in combineAdd()
55779 SDValue Op0 = N->getOperand(0); in combineAdd()
55780 SDValue Op1 = N->getOperand(1); in combineAdd()
55793 // Try to synthesize horizontal adds from adds of shuffles. in combineAdd()
55797 // add(psadbw(X,0),psadbw(Y,0)) -> psadbw(add(X,Y),0) in combineAdd()
55813 // FIXME: We have the (sub Y, (zext (vXi1 X))) -> (add (sext (vXi1 X)), Y) in in combineAdd()
55833 // Fold ADD(ADC(Y,0,W),X) -> ADC(X,Y,W) in combineAdd()
55834 if (Op0.getOpcode() == X86ISD::ADC && Op0->hasOneUse() && in combineAdd()
55836 assert(!Op0->hasAnyUseOfValue(1) && "Overflow bit in use"); in combineAdd()
55837 return DAG.getNode(X86ISD::ADC, SDLoc(Op0), Op0->getVTList(), Op1, in combineAdd()
55844 // Try to fold (sub Y, cmovns X, -X) -> (add Y, cmovns -X, X) if the cmov
55845 // condition comes from the subtract node that produced -X. This matches the
55849 SDValue N0 = N->getOperand(0); in combineSubABS()
55850 SDValue N1 = N->getOperand(1); in combineSubABS()
55865 // Get the X and -X from the negate. in combineSubABS()
55878 MVT VT = N->getSimpleValueType(0); in combineSubABS()
55886 SDValue Op0 = N->getOperand(0); in combineSubSetcc()
55887 SDValue Op1 = N->getOperand(1); in combineSubSetcc()
55891 // (add (zero_extend (setcc inverted) C-1)) if C is a nonzero immediate in combineSubSetcc()
55893 EVT VT = N->getValueType(0); in combineSubSetcc()
55896 !Op0C->isZero() && Op1.getOperand(0).getOpcode() == X86ISD::SETCC && in combineSubSetcc()
55901 APInt NewImm = Op0C->getAPIntValue() - 1; in combineSubSetcc()
55915 // -> in combineX86CloadCstore()
55917 if (N->getConstantOperandVal(3) != X86::COND_NE) in combineX86CloadCstore()
55920 SDValue Sub = N->getOperand(4); in combineX86CloadCstore()
55929 SmallVector<SDValue, 5> Ops(N->op_values()); in combineX86CloadCstore()
55933 return DAG.getMemIntrinsicNode(N->getOpcode(), SDLoc(N), N->getVTList(), Ops, in combineX86CloadCstore()
55934 cast<MemSDNode>(N)->getMemoryVT(), in combineX86CloadCstore()
55935 cast<MemSDNode>(N)->getMemOperand()); in combineX86CloadCstore()
55941 SDValue Op0 = N->getOperand(0); in combineSub()
55942 SDValue Op1 = N->getOperand(1); in combineSub()
55949 return !Cst->isOpaque(); in combineSub()
55959 // sub(C1, xor(X, C2)) -> add(xor(X, ~C2), C1+1) in combineSub()
55962 Op1->hasOneUse()) { in combineSub()
55974 // Try to synthesize horizontal subs from subs of shuffles. in combineSub()
55978 // Fold SUB(X,ADC(Y,0,W)) -> SBB(X,Y,W) in combineSub()
55979 if (Op1.getOpcode() == X86ISD::ADC && Op1->hasOneUse() && in combineSub()
55981 assert(!Op1->hasAnyUseOfValue(1) && "Overflow bit in use"); in combineSub()
55982 return DAG.getNode(X86ISD::SBB, SDLoc(Op1), Op1->getVTList(), Op0, in combineSub()
55986 // Fold SUB(X,SBB(Y,Z,W)) -> SUB(ADC(X,Z,W),Y) in combineSub()
55988 if (Op1.getOpcode() == X86ISD::SBB && Op1->hasOneUse() && in combineSub()
55990 assert(!Op1->hasAnyUseOfValue(1) && "Overflow bit in use"); in combineSub()
55991 SDValue ADC = DAG.getNode(X86ISD::ADC, SDLoc(Op1), Op1->getVTList(), Op0, in combineSub()
56008 unsigned Opcode = N->getOpcode(); in combineVectorCompare()
56012 SDValue LHS = N->getOperand(0); in combineVectorCompare()
56013 SDValue RHS = N->getOperand(1); in combineVectorCompare()
56014 MVT VT = N->getSimpleValueType(0); in combineVectorCompare()
56024 // PCMPEQ(X,UNDEF) -> UNDEF in combineVectorCompare()
56025 // PCMPGT(X,UNDEF) -> 0 in combineVectorCompare()
56026 // PCMPGT(UNDEF,X) -> 0 in combineVectorCompare()
56103 // concat_vectors(movddup(x),movddup(x)) -> broadcast(x) in combineConcatVectorOps()
56113 // concat_vectors(scalar_to_vector(x),scalar_to_vector(x)) -> broadcast(x) in combineConcatVectorOps()
56122 // extract_subvector(broadcast(x))) -> broadcast(x) in combineConcatVectorOps()
56124 // extract_subvector(subv_broadcast(x))) -> subv_broadcast(x) in combineConcatVectorOps()
56132 Op0.getValueType() == cast<MemSDNode>(SrcVec)->getMemoryVT()) in combineConcatVectorOps()
56136 // concat_vectors(permq(x),permq(x)) -> permq(concat_vectors(x,x)) in combineConcatVectorOps()
56145 // concat(extract_subvector(v0,c0), extract_subvector(v1,c1)) -> vperm2x128. in combineConcatVectorOps()
56169 // TODO - combineX86ShufflesRecursively should handle shuffle concatenation in combineConcatVectorOps()
56419 // Special case: SHL/SRL AVX1 V4i64 by 32-bits can lower as a shuffle. in combineConcatVectorOps()
56668 if (TLI->allowsMemoryAccess(Ctx, DAG.getDataLayout(), VT, in combineConcatVectorOps()
56669 *FirstLd->getMemOperand(), &Fast) && in combineConcatVectorOps()
56717 getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) { in combineConcatVectorOps()
56726 // If we're splatting a 128-bit subvector to 512-bits, use SHUF128 directly. in combineConcatVectorOps()
56743 EVT VT = N->getValueType(0); in combineCONCAT_VECTORS()
56744 EVT SrcVT = N->getOperand(0).getValueType(); in combineCONCAT_VECTORS()
56746 SmallVector<SDValue, 4> Ops(N->op_begin(), N->op_end()); in combineCONCAT_VECTORS()
56755 Constant.insertBits(C->getAPIntValue(), I * SubSizeInBits); in combineCONCAT_VECTORS()
56756 if (I == (E - 1)) { in combineCONCAT_VECTORS()
56782 MVT OpVT = N->getSimpleValueType(0); in combineINSERT_SUBVECTOR()
56787 SDValue Vec = N->getOperand(0); in combineINSERT_SUBVECTOR()
56788 SDValue SubVec = N->getOperand(1); in combineINSERT_SUBVECTOR()
56790 uint64_t IdxVal = N->getConstantOperandVal(2); in combineINSERT_SUBVECTOR()
56827 Ins.getOperand(1), N->getOperand(2)); in combineINSERT_SUBVECTOR()
56836 // insert_subvector X, (insert_subvector undef, Y, 0), Idx --> in combineINSERT_SUBVECTOR()
56843 SubVec.getOperand(1), N->getOperand(2)); in combineINSERT_SUBVECTOR()
56905 SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() }; in combineINSERT_SUBVECTOR()
56908 MemIntr->getMemoryVT(), in combineINSERT_SUBVECTOR()
56909 MemIntr->getMemOperand()); in combineINSERT_SUBVECTOR()
56932 /// is a common pattern for AVX1 integer code because 256-bit selects may be
56933 /// legal, but there is almost no integer math/logic available for 256-bit.
56938 SDValue Sel = Ext->getOperand(0); in narrowExtractedVectorSelect()
56945 // TODO: This can be extended to handle extraction to 256-bits. in narrowExtractedVectorSelect()
56946 MVT VT = Ext->getSimpleValueType(0); in narrowExtractedVectorSelect()
56954 MVT WideVT = Ext->getOperand(0).getSimpleValueType(); in narrowExtractedVectorSelect()
56961 unsigned ExtIdx = Ext->getConstantOperandVal(1); in narrowExtractedVectorSelect()
56990 // For AVX1 only, if we are extracting from a 256-bit and+not (which will in combineEXTRACT_SUBVECTOR()
56992 // split the 'and' into 128-bit ops to avoid the concatenate and extract. in combineEXTRACT_SUBVECTOR()
57000 if (!N->getValueType(0).isSimple()) in combineEXTRACT_SUBVECTOR()
57003 MVT VT = N->getSimpleValueType(0); in combineEXTRACT_SUBVECTOR()
57004 SDValue InVec = N->getOperand(0); in combineEXTRACT_SUBVECTOR()
57005 unsigned IdxVal = N->getConstantOperandVal(1); in combineEXTRACT_SUBVECTOR()
57021 SDValue NotOp = V->getOperand(0); in combineEXTRACT_SUBVECTOR()
57026 // extract (and v4i64 X, (not (concat Y1, Y2))), n -> andnp v2i64 X(n), Y1 in combineEXTRACT_SUBVECTOR()
57029 DAG.getBitcast(InVecVT, Concat), N->getOperand(1)); in combineEXTRACT_SUBVECTOR()
57049 return DAG.getBuildVector(VT, DL, InVec->ops().slice(IdxVal, NumSubElts)); in combineEXTRACT_SUBVECTOR()
57060 InVec.getOperand(0), N->getOperand(1)); in combineEXTRACT_SUBVECTOR()
57061 unsigned NewIdxVal = InVec.getConstantOperandVal(2) - IdxVal; in combineEXTRACT_SUBVECTOR()
57077 cast<MemIntrinsicSDNode>(InVec)->getMemoryVT() == VT) in combineEXTRACT_SUBVECTOR()
57188 // Always split vXi64 logical shifts where we're extracting the upper 32-bits in combineEXTRACT_SUBVECTOR()
57202 EVT VT = N->getValueType(0); in combineScalarToVector()
57203 SDValue Src = N->getOperand(0); in combineScalarToVector()
57234 if (Ld->getExtensionType() == Ext && in combineScalarToVector()
57235 Ld->getMemoryVT().getScalarSizeInBits() <= 32) in combineScalarToVector()
57266 for (SDNode *User : Src->uses()) in combineScalarToVector()
57267 if (User->getOpcode() == X86ISD::VBROADCAST && in combineScalarToVector()
57268 Src == User->getOperand(0)) { in combineScalarToVector()
57271 User->getValueSizeInBits(0).getFixedValue(); in combineScalarToVector()
57287 SDValue LHS = N->getOperand(0); in combinePMULDQ()
57288 SDValue RHS = N->getOperand(1); in combinePMULDQ()
57293 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), RHS, LHS); in combinePMULDQ()
57298 return DAG.getConstant(0, SDLoc(N), N->getValueType(0)); in combinePMULDQ()
57312 if (N->getValueType(0) == MVT::v2i64 && LHS.hasOneUse() && in combinePMULDQ()
57318 LHS.getOperand(0), { 0, -1, 1, -1 }); in combinePMULDQ()
57320 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS); in combinePMULDQ()
57322 if (N->getValueType(0) == MVT::v2i64 && RHS.hasOneUse() && in combinePMULDQ()
57328 RHS.getOperand(0), { 0, -1, 1, -1 }); in combinePMULDQ()
57330 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS); in combinePMULDQ()
57339 MVT VT = N->getSimpleValueType(0); in combineVPMADD()
57340 SDValue LHS = N->getOperand(0); in combineVPMADD()
57341 SDValue RHS = N->getOperand(1); in combineVPMADD()
57342 unsigned Opc = N->getOpcode(); in combineVPMADD()
57385 EVT VT = N->getValueType(0); in combineEXTEND_VECTOR_INREG()
57386 SDValue In = N->getOperand(0); in combineEXTEND_VECTOR_INREG()
57387 unsigned Opcode = N->getOpcode(); in combineEXTEND_VECTOR_INREG()
57396 if (Ld->isSimple()) { in combineEXTEND_VECTOR_INREG()
57404 Ext, DL, VT, Ld->getChain(), Ld->getBasePtr(), Ld->getPointerInfo(), in combineEXTEND_VECTOR_INREG()
57405 MemVT, Ld->getOriginalAlign(), Ld->getMemOperand()->getFlags()); in combineEXTEND_VECTOR_INREG()
57412 // Fold EXTEND_VECTOR_INREG(EXTEND_VECTOR_INREG(X)) -> EXTEND_VECTOR_INREG(X). in combineEXTEND_VECTOR_INREG()
57417 // -> EXTEND_VECTOR_INREG(X). in combineEXTEND_VECTOR_INREG()
57418 // TODO: Handle non-zero subvector indices. in combineEXTEND_VECTOR_INREG()
57425 // Fold EXTEND_VECTOR_INREG(BUILD_VECTOR(X,Y,?,?)) -> BUILD_VECTOR(X,0,Y,0). in combineEXTEND_VECTOR_INREG()
57452 EVT VT = N->getValueType(0); in combineKSHIFT()
57454 if (ISD::isBuildVectorAllZeros(N->getOperand(0).getNode())) in combineKSHIFT()
57473 if (N->getOperand(0).getOpcode() != ISD::FP_TO_FP16) in combineFP16_TO_FP()
57476 if (N->getValueType(0) != MVT::f32 || in combineFP16_TO_FP()
57477 N->getOperand(0).getOperand(0).getValueType() != MVT::f32) in combineFP16_TO_FP()
57482 N->getOperand(0).getOperand(0)); in combineFP16_TO_FP()
57493 EVT VT = N->getValueType(0); in combineFP_EXTEND()
57494 bool IsStrict = N->isStrictFPOpcode(); in combineFP_EXTEND()
57495 SDValue Src = N->getOperand(IsStrict ? 1 : 0); in combineFP_EXTEND()
57558 {N->getOperand(0), Src}); in combineFP_EXTEND()
57589 assert((N->getOpcode() == X86ISD::VBROADCAST_LOAD || in combineBROADCAST_LOAD()
57590 N->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) && in combineBROADCAST_LOAD()
57594 if (N->hasAnyUseOfValue(1)) in combineBROADCAST_LOAD()
57599 SDValue Ptr = MemIntrin->getBasePtr(); in combineBROADCAST_LOAD()
57600 SDValue Chain = MemIntrin->getChain(); in combineBROADCAST_LOAD()
57601 EVT VT = N->getSimpleValueType(0); in combineBROADCAST_LOAD()
57602 EVT MemVT = MemIntrin->getMemoryVT(); in combineBROADCAST_LOAD()
57606 for (SDNode *User : Ptr->uses()) in combineBROADCAST_LOAD()
57607 if (User != N && User->getOpcode() == N->getOpcode() && in combineBROADCAST_LOAD()
57608 cast<MemIntrinsicSDNode>(User)->getBasePtr() == Ptr && in combineBROADCAST_LOAD()
57609 cast<MemIntrinsicSDNode>(User)->getChain() == Chain && in combineBROADCAST_LOAD()
57610 cast<MemIntrinsicSDNode>(User)->getMemoryVT().getSizeInBits() == in combineBROADCAST_LOAD()
57612 !User->hasAnyUseOfValue(1) && in combineBROADCAST_LOAD()
57613 User->getValueSizeInBits(0).getFixedValue() > VT.getFixedSizeInBits()) { in combineBROADCAST_LOAD()
57628 bool IsStrict = N->isStrictFPOpcode(); in combineFP_ROUND()
57629 EVT VT = N->getValueType(0); in combineFP_ROUND()
57630 SDValue Src = N->getOperand(IsStrict ? 1 : 0); in combineFP_ROUND()
57651 bool IsOp0Strict = Op0->isStrictFPOpcode(); in combineFP_ROUND()
57693 {N->getOperand(0), Src, Rnd}); in combineFP_ROUND()
57715 SDValue Src = N->getOperand(0); in combineMOVDQ2Q()
57721 if (LN->isSimple()) { in combineMOVDQ2Q()
57722 SDValue NewLd = DAG.getLoad(MVT::x86mmx, SDLoc(N), LN->getChain(), in combineMOVDQ2Q()
57723 LN->getBasePtr(), in combineMOVDQ2Q()
57724 LN->getPointerInfo(), in combineMOVDQ2Q()
57725 LN->getOriginalAlign(), in combineMOVDQ2Q()
57726 LN->getMemOperand()->getFlags()); in combineMOVDQ2Q()
57737 unsigned NumBits = N->getSimpleValueType(0).getSizeInBits(); in combinePDEP()
57748 switch (N->getOpcode()) { in PerformDAGCombine()
57749 // clang-format off in PerformDAGCombine()
57935 // clang-format on in PerformDAGCombine()
57945 // Prefer (non-AVX512) vector TRUNCATE(SIGN_EXTEND_INREG(X)) to use of PACKSS.
57959 // TODO: Almost no 8-bit ops are desirable because they have no actual in isTypeDesirableForOp()
57960 // size/speed advantages vs. 32-bit ops, but they do have a major in isTypeDesirableForOp()
57963 // 8-bit multiply/shl is probably not cheaper than 32-bit multiply/shl, and in isTypeDesirableForOp()
57964 // we have specializations to turn 32-bit multiply/shl into LEA or other ops. in isTypeDesirableForOp()
57965 // Also, see the comment in "IsDesirableToPromoteOp" - where we additionally in isTypeDesirableForOp()
58006 Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch"); in expandIndirectJTBranch()
58008 // In case control-flow branch protection is enabled, we need to add in expandIndirectJTBranch()
58023 EVT VT = LogicOp->getValueType(0); in isDesirableToCombineLogicOpOfSETCC()
58024 EVT OpVT = SETCC0->getOperand(0).getValueType(); in isDesirableToCombineLogicOpOfSETCC()
58037 // TODO: Currently we lower (icmp eq/ne (and ~X, Y), 0) -> `test (not X), Y`, in isDesirableToCombineLogicOpOfSETCC()
58049 // 8-bit multiply-by-constant can usually be expanded to something cheaper in IsDesirableToPromoteOp()
58057 SDNode *User = *Op->use_begin(); in IsDesirableToPromoteOp()
58062 return Ld->getBasePtr() == St->getBasePtr(); in IsDesirableToPromoteOp()
58070 SDNode *User = *Op->use_begin(); in IsDesirableToPromoteOp()
58071 if (User->getOpcode() != ISD::ATOMIC_STORE) in IsDesirableToPromoteOp()
58075 return Ld->getBasePtr() == St->getBasePtr(); in IsDesirableToPromoteOp()
58123 //===----------------------------------------------------------------------===//
58125 //===----------------------------------------------------------------------===//
58163 InlineAsm *IA = cast<InlineAsm>(CI->getCalledOperand()); in ExpandInlineAsm()
58165 const std::string &AsmStr = IA->getAsmString(); in ExpandInlineAsm()
58167 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType()); in ExpandInlineAsm()
58168 if (!Ty || Ty->getBitWidth() % 16 != 0) in ExpandInlineAsm()
58171 // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a" in ExpandInlineAsm()
58194 // rorw $$8, ${0:w} --> llvm.bswap.i16 in ExpandInlineAsm()
58195 if (CI->getType()->isIntegerTy(16) && in ExpandInlineAsm()
58196 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 && in ExpandInlineAsm()
58200 StringRef ConstraintsStr = IA->getConstraintString(); in ExpandInlineAsm()
58208 if (CI->getType()->isIntegerTy(32) && in ExpandInlineAsm()
58209 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 && in ExpandInlineAsm()
58214 StringRef ConstraintsStr = IA->getConstraintString(); in ExpandInlineAsm()
58221 if (CI->getType()->isIntegerTy(64)) { in ExpandInlineAsm()
58222 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints(); in ExpandInlineAsm()
58226 // bswap %eax / bswap %edx / xchgl %eax, %edx -> llvm.bswap.i64 in ExpandInlineAsm()
58361 Type *Ty = CallOperandVal->getType(); in getSingleConstraintMatchWeight()
58377 if (CallOperandVal->getType()->isIntegerTy()) in getSingleConstraintMatchWeight()
58383 if (Ty->isFloatingPointTy()) in getSingleConstraintMatchWeight()
58387 if (Ty->isX86_MMXTy() && Subtarget.hasMMX()) in getSingleConstraintMatchWeight()
58398 if (((Ty->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1()) || in getSingleConstraintMatchWeight()
58399 ((Ty->getPrimitiveSizeInBits() == 256) && Subtarget.hasAVX()) || in getSingleConstraintMatchWeight()
58400 ((Ty->getPrimitiveSizeInBits() == 512) && Subtarget.hasAVX512())) in getSingleConstraintMatchWeight()
58405 if ((Ty->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512()) in getSingleConstraintMatchWeight()
58410 if (Ty->isX86_MMXTy() && Subtarget.hasMMX()) in getSingleConstraintMatchWeight()
58430 if (CallOperandVal->getType()->isIntegerTy()) in getSingleConstraintMatchWeight()
58436 if ((Ty->getPrimitiveSizeInBits() == 512) && Subtarget.hasAVX512()) in getSingleConstraintMatchWeight()
58440 if (((Ty->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1()) || in getSingleConstraintMatchWeight()
58441 ((Ty->getPrimitiveSizeInBits() == 256) && Subtarget.hasAVX())) in getSingleConstraintMatchWeight()
58446 if ((Ty->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512()) in getSingleConstraintMatchWeight()
58451 if (C->getZExtValue() <= 31) in getSingleConstraintMatchWeight()
58456 if (C->getZExtValue() <= 63) in getSingleConstraintMatchWeight()
58461 if ((C->getSExtValue() >= -0x80) && (C->getSExtValue() <= 0x7f)) in getSingleConstraintMatchWeight()
58466 if ((C->getZExtValue() == 0xff) || (C->getZExtValue() == 0xffff)) in getSingleConstraintMatchWeight()
58471 if (C->getZExtValue() <= 3) in getSingleConstraintMatchWeight()
58476 if (C->getZExtValue() <= 0xff) in getSingleConstraintMatchWeight()
58486 if ((C->getSExtValue() >= -0x80000000LL) && in getSingleConstraintMatchWeight()
58487 (C->getSExtValue() <= 0x7fffffffLL)) in getSingleConstraintMatchWeight()
58492 if (C->getZExtValue() <= 0xffffffff) in getSingleConstraintMatchWeight()
58534 // Extend to 32-bits in LowerAsmOutputForConstraint()
58552 if (C->getZExtValue() <= 31) { in LowerAsmOperandForConstraint()
58553 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58561 if (C->getZExtValue() <= 63) { in LowerAsmOperandForConstraint()
58562 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58570 if (isInt<8>(C->getSExtValue())) { in LowerAsmOperandForConstraint()
58571 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58579 if (C->getZExtValue() == 0xff || C->getZExtValue() == 0xffff || in LowerAsmOperandForConstraint()
58580 (Subtarget.is64Bit() && C->getZExtValue() == 0xffffffff)) { in LowerAsmOperandForConstraint()
58581 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58589 if (C->getZExtValue() <= 3) { in LowerAsmOperandForConstraint()
58590 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58598 if (C->getZExtValue() <= 255) { in LowerAsmOperandForConstraint()
58599 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58607 if (C->getZExtValue() <= 127) { in LowerAsmOperandForConstraint()
58608 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58615 // 32-bit signed value in LowerAsmOperandForConstraint()
58618 C->getSExtValue())) { in LowerAsmOperandForConstraint()
58620 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64); in LowerAsmOperandForConstraint()
58633 Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(), in LowerAsmOperandForConstraint()
58634 BA->getValueType(0))); in LowerAsmOperandForConstraint()
58637 if (Op->getOpcode() == ISD::ADD && in LowerAsmOperandForConstraint()
58638 isa<ConstantSDNode>(Op->getOperand(1))) { in LowerAsmOperandForConstraint()
58639 Offset = cast<ConstantSDNode>(Op->getOperand(1))->getSExtValue(); in LowerAsmOperandForConstraint()
58640 Op = Op->getOperand(0); in LowerAsmOperandForConstraint()
58643 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op), in LowerAsmOperandForConstraint()
58644 GA->getValueType(0), Offset)); in LowerAsmOperandForConstraint()
58649 // 32-bit unsigned value in LowerAsmOperandForConstraint()
58652 C->getZExtValue())) { in LowerAsmOperandForConstraint()
58653 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), in LowerAsmOperandForConstraint()
58665 bool IsBool = CST->getConstantIntValue()->getBitWidth() == 1; in LowerAsmOperandForConstraint()
58669 int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? CST->getZExtValue() in LowerAsmOperandForConstraint()
58670 : CST->getSExtValue(); in LowerAsmOperandForConstraint()
58682 // If we are in non-pic codegen mode, we allow the address of a global (with in LowerAsmOperandForConstraint()
58688 Subtarget.classifyGlobalReference(GA->getGlobal()))) in LowerAsmOperandForConstraint()
58775 case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode. in getRegForInlineAsmConstraint()
58796 // 32-bit fallthrough in getRegForInlineAsmConstraint()
59090 // Map st(0) -> st(7) -> ST0 in getRegForInlineAsmConstraint()
59100 return std::make_pair(X86::FP0 + Constraint[4] - '0', in getRegForInlineAsmConstraint()
59109 // flags -> EFLAGS in getRegForInlineAsmConstraint()
59113 // dirflag -> DF in getRegForInlineAsmConstraint()
59119 // fpsr -> FPSW in getRegForInlineAsmConstraint()
59127 // Make sure it isn't a register that requires 64-bit mode. in getRegForInlineAsmConstraint()
59130 TRI->getEncodingValue(Res.first) >= 8) { in getRegForInlineAsmConstraint()
59131 // Register requires REX prefix, but we're in 32-bit mode. in getRegForInlineAsmConstraint()
59137 TRI->getEncodingValue(Res.first) & 0x10) { in getRegForInlineAsmConstraint()
59143 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to in getRegForInlineAsmConstraint()
59146 if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other) in getRegForInlineAsmConstraint()
59171 // Model GCC's behavior here and select a fixed pair of 32-bit in getRegForInlineAsmConstraint()
59192 if (RC && RC->contains(DestReg)) in getRegForInlineAsmConstraint()
59211 else if (TRI->isTypeLegalForClass(X86::VR128XRegClass, VT)) in getRegForInlineAsmConstraint()
59213 else if (TRI->isTypeLegalForClass(X86::VR256XRegClass, VT)) in getRegForInlineAsmConstraint()
59215 else if (TRI->isTypeLegalForClass(X86::VR512RegClass, VT)) in getRegForInlineAsmConstraint()
59248 // integer division, leaving the division as-is is a loss even in terms of in isIntDivCheap()
59261 Entry->getParent()->getInfo<X86MachineFunctionInfo>(); in initializeSplitCSR()
59262 AFI->setIsSplitCSR(true); in initializeSplitCSR()
59269 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent()); in insertCopiesSplitCSR()
59274 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo(); in insertCopiesSplitCSR()
59275 MachineBasicBlock::iterator MBBI = Entry->begin(); in insertCopiesSplitCSR()
59283 Register NewVR = MRI->createVirtualRegister(RC); in insertCopiesSplitCSR()
59285 // FIXME: this currently does not emit CFI pseudo-instructions, it works in insertCopiesSplitCSR()
59286 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be in insertCopiesSplitCSR()
59288 // CFI pseudo-instructions. in insertCopiesSplitCSR()
59290 Entry->getParent()->getFunction().hasFnAttribute(Attribute::NoUnwind) && in insertCopiesSplitCSR()
59292 Entry->addLiveIn(*I); in insertCopiesSplitCSR()
59293 BuildMI(*Entry, MBBI, MIMetadata(), TII->get(TargetOpcode::COPY), NewVR) in insertCopiesSplitCSR()
59296 // Insert the copy-back instructions right before the terminator. in insertCopiesSplitCSR()
59298 BuildMI(*Exit, Exit->getFirstTerminator(), MIMetadata(), in insertCopiesSplitCSR()
59299 TII->get(TargetOpcode::COPY), *I) in insertCopiesSplitCSR()
59312 assert(MBBI->isCall() && MBBI->getCFIType() && in EmitKCFICheck()
59318 switch (MBBI->getOpcode()) { in EmitKCFICheck()
59325 if (!TII->unfoldMemoryOperand(MF, *OrigCall, X86::R11, /*UnfoldLoad=*/true, in EmitKCFICheck()
59330 assert(MBBI->isCall() && in EmitKCFICheck()
59332 if (OrigCall->shouldUpdateCallSiteInfo()) in EmitKCFICheck()
59334 MBBI->setCFIType(MF, OrigCall->getCFIType()); in EmitKCFICheck()
59335 OrigCall->eraseFromParent(); in EmitKCFICheck()
59342 MachineOperand &Target = MBBI->getOperand(0); in EmitKCFICheck()
59344 switch (MBBI->getOpcode()) { in EmitKCFICheck()
59357 // 64-bit indirect thunk calls. in EmitKCFICheck()
59367 return BuildMI(MBB, MBBI, MIMetadata(*MBBI), TII->get(X86::KCFI_CHECK)) in EmitKCFICheck()
59369 .addImm(MBBI->getCFIType()) in EmitKCFICheck()
59383 MF.getFunction().hasFnAttribute("no-stack-arg-probe")) in hasInlineStackProbe()
59387 if (MF.getFunction().hasFnAttribute("probe-stack")) in hasInlineStackProbe()
59388 return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() == in hasInlineStackProbe()
59389 "inline-asm"; in hasInlineStackProbe()
59403 if (MF.getFunction().hasFnAttribute("probe-stack")) in getStackProbeSymbolName()
59404 return MF.getFunction().getFnAttribute("probe-stack").getValueAsString(); in getStackProbeSymbolName()
59409 MF.getFunction().hasFnAttribute("no-stack-arg-probe")) in getStackProbeSymbolName()
59423 return MF.getFunction().getFnAttributeAsParsedInteger("stack-probe-size", in getStackProbeSize()
59428 if (ML && ML->isInnermost() && in getPrefLoopAlignment()